Implemented Plugin Cache + its unit tests

wlauto/core/configuration/configuration.py

@@ -494,7 +494,7 @@ def merge_using_priority_specificity(generic_name, specific_name, plugin_cache):
     """
     generic_config = plugin_cache.get_plugin_config(generic_name)
     specific_config = plugin_cache.get_plugin_config(specific_name)
-    cfg_points = plugin_cache.get_plugin_config_points(specific_name)
+    cfg_points = plugin_cache.get_plugin_parameters(specific_name)
     sources = plugin_cache.sources
     final_config = obj_dict(not_in_dict=['name'])
     seen_specific_config = defaultdict(list)
@@ -776,7 +776,7 @@ class RunConfiguration(Configuration):
         instance = super(RunConfiguration, cls).from_pod(pod, plugin_cache)
 
         device_config.name = "device_config"
-        cfg_points = plugin_cache.get_plugin_config_points(instance.device)
+        cfg_points = plugin_cache.get_plugin_parameters(instance.device)
         for entry_name in device_config.iterkeys():
             if entry_name not in cfg_points.iterkeys():
                 msg = 'Invalid entry "{}" for device "{}".'
@@ -866,7 +866,7 @@ class JobSpec(Configuration):
 
         # Merge entry "workload_parameters"
         # TODO: Wrap in - "error in [agenda path]"
-        cfg_points = plugin_cache.get_plugin_config_points(self.workload_name)
+        cfg_points = plugin_cache.get_plugin_parameters(self.workload_name)
         for source in self._sources:
             if source in self._to_merge["workload_params"]:
                 config = self._to_merge["workload_params"][source]

wlauto/core/configuration/plugin_cache.py

@@ -12,10 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+from copy import copy
+from collections import defaultdict
 
-from collections import OrderedDict
+from wlauto.core import pluginloader
-
+from wlauto.exceptions import ConfigError
 from wlauto.utils.types import obj_dict
+from devlib.utils.misc import memoized
+
+GENERIC_CONFIGS = ["device_config", "workload_parameters",
+                   "boot_parameters", "runtime_parameters"]
 
 
 class PluginCache(object):
@@ -27,41 +33,165 @@ class PluginCache(object):
     from, and the priority order of said sources.
     """
 
-    def __init__(self):
+    def __init__(self, loader=pluginloader):
-        self.plugin_configs = {}
+        self.loader = loader
-        self.global_alias = {}
         self.sources = []
-        self.finalised = False
+        self.plugin_configs = defaultdict(lambda: defaultdict(dict))
-        # TODO: Build dicts of global_alias: [list of destinations]
+        self.global_alias_values = defaultdict(dict)
+
+        # Generate a mapping of what global aliases belong to
+        self._global_alias_map = defaultdict(dict)
+        self._list_of_global_aliases = set()
+        for plugin in self.loader.list_plugins():
+            for param in plugin.parameters:
+                if param.aliases:
+                    self._global_alias_map[plugin.name][param.aliases] = param
+                    self._list_of_global_aliases.add(param.aliases)
 
     def add_source(self, source):
         if source in self.sources:
             raise Exception("Source has already been added.")
         self.sources.append(source)
 
-    def _add_config(self, destination, name, value, source):
+    def add_global_alias(self, alias, value, source):
         if source not in self.sources:
             msg = "Source '{}' has not been added to the plugin cache."
-            raise Exception(msg.format(source))
+            raise RuntimeError(msg.format(source))
 
-        if name not in destination:
+        if not self.is_global_alias(alias):
-            destination[name] = OrderedDict()
+            msg = "'{} is not a valid global alias'"
-        destination[name][source] = value
+            raise RuntimeError(msg.format(alias))
 
-    def add_plugin_config(self, name, config, source):
+        self.global_alias_values[alias][source] = value
-        self._add_config(self.plugin_configs, name, config, source)
 
-    def add_global_alias(self, name, config, source):
+    def add_configs(self, plugin_name, values, source):
-        self._add_config(self.global_alias, name, config, source)
+        print plugin_name, values
+        if self.is_global_alias(plugin_name):
+            self.add_global_alias(plugin_name, values, source)
+            return
+        for name, value in values.iteritems():
+            self.add_config(plugin_name, name, value, source)
 
-    def finalise_config(self):
+    def add_config(self, plugin_name, name, value, source):
-        pass
+        if source not in self.sources:
+            msg = "Source '{}' has not been added to the plugin cache."
+            raise RuntimeError(msg.format(source))
+
+        if not self.loader.has_plugin(plugin_name) and plugin_name not in GENERIC_CONFIGS:
+            msg = 'configuration provided for unknown plugin "{}"'
+            raise ConfigError(msg.format(plugin_name))
+
+        if (plugin_name not in GENERIC_CONFIGS and
+                name not in self.get_plugin_parameters(plugin_name)):
+            msg = "'{}' is not a valid parameter for '{}'"
+            raise ConfigError(msg.format(name, plugin_name))
+
+        self.plugin_configs[plugin_name][source][name] = value
 
     def is_global_alias(self, name):
-        pass
+        return name in self._list_of_global_aliases
 
-    def get_plugin_config(self, name):
+    def get_plugin_config(self, plugin_name, generic_name=None):
-        return self.plugin_configs[name]
+        config = obj_dict(not_in_dict=['name'])
+        config.name = plugin_name
 
-    def get_plugin_config_points(self, name):
+        # Load plugin defaults
-        pass
+        cfg_points = self.get_plugin_parameters(plugin_name)
+        for cfg_point in cfg_points.itervalues():
+            cfg_point.set_value(config, check_mandatory=False)
+
+        # Merge global aliases
+        for alias, param in self._global_alias_map[plugin_name].iteritems():
+            if alias in self.global_alias_values:
+                for source in self.sources:
+                    if source not in self.global_alias_values[alias]:
+                        continue
+                    param.set_value(config, value=self.global_alias_values[alias][source])
+
+        # Merge user config
+        # Perform a simple merge with the order of sources representing priority
+        if generic_name is None:
+            plugin_config = self.plugin_configs[plugin_name]
+            for source in self.sources:
+                if source not in plugin_config:
+                    continue
+                for name, value in plugin_config[source].iteritems():
+                    cfg_points[name].set_value(config, value=value)
+        # A more complicated merge that involves priority of sources and specificity
+        else:
+            self._merge_using_priority_specificity(plugin_name, generic_name, config)
+
+        return config
+
+    @memoized
+    def get_plugin_parameters(self, name):
+        params = self.loader.get_plugin_class(name).parameters
+        return {param.name: param for param in params}
+
+    # pylint: disable=too-many-nested-blocks, too-many-branches
+    def _merge_using_priority_specificity(self, specific_name, generic_name, final_config):
+        """
+        WA configuration can come from various sources of increasing priority, as well
+        as being specified in a generic and specific manner (e.g. ``device_config``
+        and ``nexus10`` respectivly). WA has two rules for the priority of configuration:
+
+            - Configuration from higher priority sources overrides configuration from
+              lower priority sources.
+            - More specific configuration overrides less specific configuration.
+
+        There is a situation where these two rules come into conflict. When a generic
+        configuration is given in config source of high priority and a specific
+        configuration is given in a config source of lower priority. In this situation
+        it is not possible to know the end users intention and WA will error.
+
+        :param generic_name: The name of the generic configuration e.g ``device_config``
+        :param specific_name: The name of the specific configuration used, e.g ``nexus10``
+        :param cfg_point: A dict of ``ConfigurationPoint``s to be used when merging configuration.
+                          keys=config point name, values=config point
+
+        :rtype: A fully merged and validated configuration in the form of a obj_dict.
+        """
+        generic_config = copy(self.plugin_configs[generic_name])
+        specific_config = copy(self.plugin_configs[specific_name])
+        cfg_points = self.get_plugin_parameters(specific_name)
+        sources = self.sources
+        seen_specific_config = defaultdict(list)
+
+        # set_value uses the 'name' attribute of the passed object in it error
+        # messages, to ensure these messages make sense the name will have to be
+        # changed several times during this function.
+        final_config.name = specific_name
+
+        # pylint: disable=too-many-nested-blocks
+        for source in sources:
+            try:
+                if source in generic_config:
+                    final_config.name = generic_name
+                    for name, cfg_point in cfg_points.iteritems():
+                        if name in generic_config[source]:
+                            if name in seen_specific_config:
+                                msg = ('"{generic_name}" configuration "{config_name}" has already been '
+                                       'specified more specifically for {specific_name} in:\n\t\t{sources}')
+                                msg = msg.format(generic_name=generic_name,
+                                                 config_name=name,
+                                                 specific_name=specific_name,
+                                                 sources=", ".join(seen_specific_config[name]))
+                                raise ConfigError(msg)
+                            value = generic_config[source][name]
+                            cfg_point.set_value(final_config, value, check_mandatory=False)
+
+                if source in specific_config:
+                    final_config.name = specific_name
+                    for name, cfg_point in cfg_points.iteritems():
+                        if name in specific_config[source]:
+                            seen_specific_config[name].append(str(source))
+                            value = specific_config[source][name]
+                            cfg_point.set_value(final_config, value, check_mandatory=False)
+
+            except ConfigError as e:
+                raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e)))
+
+        # Validate final configuration
+        final_config.name = specific_name
+        for cfg_point in cfg_points.itervalues():
+            cfg_point.validate(final_config)

wlauto/tests/test_configuration.py

@@ -13,7 +13,7 @@ from wlauto.core.configuration.configuration import (ConfigurationPoint,
                                                      RunConfiguration,
                                                      merge_using_priority_specificity,
                                                      get_type_name)
-from wlauto.core.configuration.plugin_cache import PluginCache
+from wlauto.core.configuration.plugin_cache import PluginCache, GENERIC_CONFIGS
 from wlauto.utils.types import obj_dict
 #       A1
 #     /    \
@@ -65,9 +65,9 @@ def _construct_mock_plugin_cache(values=None):
         return values[plugin_name]
     plugin_cache.get_plugin_config.side_effect = get_plugin_config
 
-    def get_plugin_config_points(_):
+    def get_plugin_parameters(_):
         return TestConfiguration.configuration
-    plugin_cache.get_plugin_config_points.side_effect = get_plugin_config_points
+    plugin_cache.get_plugin_parameters.side_effect = get_plugin_parameters
 
     return plugin_cache
 
@@ -454,3 +454,168 @@ class ConfigurationTest(TestCase):
 
     def test_generate_job_spec(self):
         pass
+
+
+class PluginCacheTest(TestCase):
+
+    param1 = ConfigurationPoint("param1", aliases="test_global_alias")
+    param2 = ConfigurationPoint("param2", aliases="some_other_alias")
+    param3 = ConfigurationPoint("param3")
+
+    plugin1 = obj_dict(values={
+        "name": "plugin 1",
+        "parameters": [
+            param1,
+            param2,
+        ]
+    })
+    plugin2 = obj_dict(values={
+        "name": "plugin 2",
+        "parameters": [
+            param1,
+            param3,
+        ]
+    })
+
+    def get_plugin(self, name):
+        if name == "plugin 1":
+            return self.plugin1
+        if name == "plugin 2":
+            return self.plugin2
+
+    def has_plugin(self, name):
+        return name in ["plugin 1", "plugin 2"]
+
+    def make_mock_cache(self):
+        mock_loader = Mock()
+        mock_loader.get_plugin_class.side_effect = self.get_plugin
+        mock_loader.list_plugins = Mock(return_value=[self.plugin1, self.plugin2])
+        mock_loader.has_plugin.side_effect = self.has_plugin
+        return PluginCache(loader=mock_loader)
+
+    def test_get_params(self):
+        plugin_cache = self.make_mock_cache()
+
+        expected_params = {
+            self.param1.name: self.param1,
+            self.param2.name: self.param2,
+        }
+
+        assert_equal(expected_params, plugin_cache.get_plugin_parameters("plugin 1"))
+
+    def test_global_aliases(self):
+        plugin_cache = self.make_mock_cache()
+
+        # Check the alias map
+        expected_map = {
+            "plugin 1": {
+                self.param1.aliases: self.param1,
+                self.param2.aliases: self.param2,
+            },
+            "plugin 2": {
+                self.param1.aliases: self.param1,
+            }
+        }
+        expected_set = set(["test_global_alias", "some_other_alias"])
+
+        assert_equal(expected_map, plugin_cache._global_alias_map)
+        assert_equal(expected_set, plugin_cache._list_of_global_aliases)
+        assert_equal(True, plugin_cache.is_global_alias("test_global_alias"))
+        assert_equal(False, plugin_cache.is_global_alias("not_a_global_alias"))
+
+        # Error when adding to unknown source
+        with self.assertRaises(RuntimeError):
+            plugin_cache.add_global_alias("adding", "too", "early")
+
+        # Test adding sources
+        for x in xrange(5):
+            plugin_cache.add_source(x)
+        assert_equal([0, 1, 2, 3, 4], plugin_cache.sources)
+
+        # Error when adding non plugin/global alias/generic
+        with self.assertRaises(RuntimeError):
+            plugin_cache.add_global_alias("unknow_alias", "some_value", 0)
+
+        # Test adding global alias values
+        plugin_cache.add_global_alias("test_global_alias", "some_value", 0)
+        expected_aliases = {"test_global_alias": {0: "some_value"}}
+        assert_equal(expected_aliases, plugin_cache.global_alias_values)
+
+    def test_add_config(self):
+        plugin_cache = self.make_mock_cache()
+
+        # Test adding sources
+        for x in xrange(5):
+            plugin_cache.add_source(x)
+        assert_equal([0, 1, 2, 3, 4], plugin_cache.sources)
+
+        # Test adding plugin config
+        plugin_cache.add_config("plugin 1", "param1", "some_other_value", 0)
+        expected_plugin_config = {"plugin 1": {0: {"param1": "some_other_value"}}}
+        assert_equal(expected_plugin_config, plugin_cache.plugin_configs)
+
+        # Test adding generic config
+        for name in GENERIC_CONFIGS:
+            plugin_cache.add_config(name, "param1", "some_value", 0)
+            expected_plugin_config[name] = {}
+            expected_plugin_config[name][0] = {"param1": "some_value"}
+        assert_equal(expected_plugin_config, plugin_cache.plugin_configs)
+
+    def test_get_plugin_config(self):
+        plugin_cache = self.make_mock_cache()
+        for x in xrange(5):
+            plugin_cache.add_source(x)
+
+        # Add some global aliases
+        plugin_cache.add_global_alias("test_global_alias", "1", 0)
+        plugin_cache.add_global_alias("test_global_alias", "2", 4)
+        plugin_cache.add_global_alias("test_global_alias", "3", 3)
+
+        # Test if they are being merged in source order
+        expected_config = {
+            "param1": "2",
+            "param2": None,
+        }
+        assert_equal(expected_config, plugin_cache.get_plugin_config("plugin 1"))
+
+        # Add some plugin specific config
+        plugin_cache.add_config("plugin 1", "param1", "3", 0)
+        plugin_cache.add_config("plugin 1", "param1", "4", 2)
+        plugin_cache.add_config("plugin 1", "param1", "5", 1)
+
+        # Test if they are being merged in source order on top of the global aliases
+        expected_config = {
+            "param1": "4",
+            "param2": None,
+        }
+        assert_equal(expected_config, plugin_cache.get_plugin_config("plugin 1"))
+
+    def test_merge_using_priority_specificity(self):
+        plugin_cache = self.make_mock_cache()
+        for x in xrange(5):
+            plugin_cache.add_source(x)
+
+        # Add generic configs
+        plugin_cache.add_config("device_config", "param1", '1', 1)
+        plugin_cache.add_config("device_config", "param1", '2', 2)
+        assert_equal(plugin_cache.get_plugin_config("plugin 1", generic_name="device_config"),
+                     {"param1": '2', "param2": None})
+
+        # Add specific configs at same level as generic config
+        plugin_cache.add_config("plugin 1", "param1", '3', 2)
+        assert_equal(plugin_cache.get_plugin_config("plugin 1", generic_name="device_config"),
+                     {"param1": '3', "param2": None})
+
+        # Add specific config at higher level
+        plugin_cache.add_config("plugin 1", "param1", '4', 3)
+        assert_equal(plugin_cache.get_plugin_config("plugin 1", generic_name="device_config"),
+                     {"param1": '4', "param2": None})
+
+        # Add generic config at higher level - should be an error
+        plugin_cache.add_config("device_config", "param1", '5', 4)
+        msg = 'Error in "4":\n' \
+              '\t"device_config" configuration "param1" has already been specified' \
+              ' more specifically for plugin 1 in:\n' \
+              '\t\t2, 3'
+        with self.assertRaisesRegexp(ConfigError, msg):
+            plugin_cache.get_plugin_config("plugin 1", generic_name="device_config")