From 06e95abc788e2660bc6eba195171611d6f0c969d Mon Sep 17 00:00:00 2001 From: Sebastian Goscik Date: Wed, 14 Sep 2016 12:57:40 +0100 Subject: [PATCH] Implemented Plugin Cache + its unit tests --- wlauto/core/configuration/configuration.py | 6 +- wlauto/core/configuration/plugin_cache.py | 176 ++++++++++++++++++--- wlauto/tests/test_configuration.py | 171 +++++++++++++++++++- 3 files changed, 324 insertions(+), 29 deletions(-) diff --git a/wlauto/core/configuration/configuration.py b/wlauto/core/configuration/configuration.py index 1dd3f8b4..c477ec99 100644 --- a/wlauto/core/configuration/configuration.py +++ b/wlauto/core/configuration/configuration.py @@ -494,7 +494,7 @@ def merge_using_priority_specificity(generic_name, specific_name, plugin_cache): """ generic_config = plugin_cache.get_plugin_config(generic_name) specific_config = plugin_cache.get_plugin_config(specific_name) - cfg_points = plugin_cache.get_plugin_config_points(specific_name) + cfg_points = plugin_cache.get_plugin_parameters(specific_name) sources = plugin_cache.sources final_config = obj_dict(not_in_dict=['name']) seen_specific_config = defaultdict(list) @@ -776,7 +776,7 @@ class RunConfiguration(Configuration): instance = super(RunConfiguration, cls).from_pod(pod, plugin_cache) device_config.name = "device_config" - cfg_points = plugin_cache.get_plugin_config_points(instance.device) + cfg_points = plugin_cache.get_plugin_parameters(instance.device) for entry_name in device_config.iterkeys(): if entry_name not in cfg_points.iterkeys(): msg = 'Invalid entry "{}" for device "{}".' @@ -866,7 +866,7 @@ class JobSpec(Configuration): # Merge entry "workload_parameters" # TODO: Wrap in - "error in [agenda path]" - cfg_points = plugin_cache.get_plugin_config_points(self.workload_name) + cfg_points = plugin_cache.get_plugin_parameters(self.workload_name) for source in self._sources: if source in self._to_merge["workload_params"]: config = self._to_merge["workload_params"][source] diff --git a/wlauto/core/configuration/plugin_cache.py b/wlauto/core/configuration/plugin_cache.py index 44643950..fbff1746 100644 --- a/wlauto/core/configuration/plugin_cache.py +++ b/wlauto/core/configuration/plugin_cache.py @@ -12,10 +12,16 @@ # See the License for the specific language governing permissions and # limitations under the License. +from copy import copy +from collections import defaultdict -from collections import OrderedDict - +from wlauto.core import pluginloader +from wlauto.exceptions import ConfigError from wlauto.utils.types import obj_dict +from devlib.utils.misc import memoized + +GENERIC_CONFIGS = ["device_config", "workload_parameters", + "boot_parameters", "runtime_parameters"] class PluginCache(object): @@ -27,41 +33,165 @@ class PluginCache(object): from, and the priority order of said sources. """ - def __init__(self): - self.plugin_configs = {} - self.global_alias = {} + def __init__(self, loader=pluginloader): + self.loader = loader self.sources = [] - self.finalised = False - # TODO: Build dicts of global_alias: [list of destinations] + self.plugin_configs = defaultdict(lambda: defaultdict(dict)) + self.global_alias_values = defaultdict(dict) + + # Generate a mapping of what global aliases belong to + self._global_alias_map = defaultdict(dict) + self._list_of_global_aliases = set() + for plugin in self.loader.list_plugins(): + for param in plugin.parameters: + if param.aliases: + self._global_alias_map[plugin.name][param.aliases] = param + self._list_of_global_aliases.add(param.aliases) def add_source(self, source): if source in self.sources: raise Exception("Source has already been added.") self.sources.append(source) - def _add_config(self, destination, name, value, source): + def add_global_alias(self, alias, value, source): if source not in self.sources: msg = "Source '{}' has not been added to the plugin cache." - raise Exception(msg.format(source)) + raise RuntimeError(msg.format(source)) - if name not in destination: - destination[name] = OrderedDict() - destination[name][source] = value + if not self.is_global_alias(alias): + msg = "'{} is not a valid global alias'" + raise RuntimeError(msg.format(alias)) - def add_plugin_config(self, name, config, source): - self._add_config(self.plugin_configs, name, config, source) + self.global_alias_values[alias][source] = value - def add_global_alias(self, name, config, source): - self._add_config(self.global_alias, name, config, source) + def add_configs(self, plugin_name, values, source): + print plugin_name, values + if self.is_global_alias(plugin_name): + self.add_global_alias(plugin_name, values, source) + return + for name, value in values.iteritems(): + self.add_config(plugin_name, name, value, source) - def finalise_config(self): - pass + def add_config(self, plugin_name, name, value, source): + if source not in self.sources: + msg = "Source '{}' has not been added to the plugin cache." + raise RuntimeError(msg.format(source)) + + if not self.loader.has_plugin(plugin_name) and plugin_name not in GENERIC_CONFIGS: + msg = 'configuration provided for unknown plugin "{}"' + raise ConfigError(msg.format(plugin_name)) + + if (plugin_name not in GENERIC_CONFIGS and + name not in self.get_plugin_parameters(plugin_name)): + msg = "'{}' is not a valid parameter for '{}'" + raise ConfigError(msg.format(name, plugin_name)) + + self.plugin_configs[plugin_name][source][name] = value def is_global_alias(self, name): - pass + return name in self._list_of_global_aliases - def get_plugin_config(self, name): - return self.plugin_configs[name] + def get_plugin_config(self, plugin_name, generic_name=None): + config = obj_dict(not_in_dict=['name']) + config.name = plugin_name - def get_plugin_config_points(self, name): - pass + # Load plugin defaults + cfg_points = self.get_plugin_parameters(plugin_name) + for cfg_point in cfg_points.itervalues(): + cfg_point.set_value(config, check_mandatory=False) + + # Merge global aliases + for alias, param in self._global_alias_map[plugin_name].iteritems(): + if alias in self.global_alias_values: + for source in self.sources: + if source not in self.global_alias_values[alias]: + continue + param.set_value(config, value=self.global_alias_values[alias][source]) + + # Merge user config + # Perform a simple merge with the order of sources representing priority + if generic_name is None: + plugin_config = self.plugin_configs[plugin_name] + for source in self.sources: + if source not in plugin_config: + continue + for name, value in plugin_config[source].iteritems(): + cfg_points[name].set_value(config, value=value) + # A more complicated merge that involves priority of sources and specificity + else: + self._merge_using_priority_specificity(plugin_name, generic_name, config) + + return config + + @memoized + def get_plugin_parameters(self, name): + params = self.loader.get_plugin_class(name).parameters + return {param.name: param for param in params} + + # pylint: disable=too-many-nested-blocks, too-many-branches + def _merge_using_priority_specificity(self, specific_name, generic_name, final_config): + """ + WA configuration can come from various sources of increasing priority, as well + as being specified in a generic and specific manner (e.g. ``device_config`` + and ``nexus10`` respectivly). WA has two rules for the priority of configuration: + + - Configuration from higher priority sources overrides configuration from + lower priority sources. + - More specific configuration overrides less specific configuration. + + There is a situation where these two rules come into conflict. When a generic + configuration is given in config source of high priority and a specific + configuration is given in a config source of lower priority. In this situation + it is not possible to know the end users intention and WA will error. + + :param generic_name: The name of the generic configuration e.g ``device_config`` + :param specific_name: The name of the specific configuration used, e.g ``nexus10`` + :param cfg_point: A dict of ``ConfigurationPoint``s to be used when merging configuration. + keys=config point name, values=config point + + :rtype: A fully merged and validated configuration in the form of a obj_dict. + """ + generic_config = copy(self.plugin_configs[generic_name]) + specific_config = copy(self.plugin_configs[specific_name]) + cfg_points = self.get_plugin_parameters(specific_name) + sources = self.sources + seen_specific_config = defaultdict(list) + + # set_value uses the 'name' attribute of the passed object in it error + # messages, to ensure these messages make sense the name will have to be + # changed several times during this function. + final_config.name = specific_name + + # pylint: disable=too-many-nested-blocks + for source in sources: + try: + if source in generic_config: + final_config.name = generic_name + for name, cfg_point in cfg_points.iteritems(): + if name in generic_config[source]: + if name in seen_specific_config: + msg = ('"{generic_name}" configuration "{config_name}" has already been ' + 'specified more specifically for {specific_name} in:\n\t\t{sources}') + msg = msg.format(generic_name=generic_name, + config_name=name, + specific_name=specific_name, + sources=", ".join(seen_specific_config[name])) + raise ConfigError(msg) + value = generic_config[source][name] + cfg_point.set_value(final_config, value, check_mandatory=False) + + if source in specific_config: + final_config.name = specific_name + for name, cfg_point in cfg_points.iteritems(): + if name in specific_config[source]: + seen_specific_config[name].append(str(source)) + value = specific_config[source][name] + cfg_point.set_value(final_config, value, check_mandatory=False) + + except ConfigError as e: + raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e))) + + # Validate final configuration + final_config.name = specific_name + for cfg_point in cfg_points.itervalues(): + cfg_point.validate(final_config) diff --git a/wlauto/tests/test_configuration.py b/wlauto/tests/test_configuration.py index dfb76dfd..3884763c 100644 --- a/wlauto/tests/test_configuration.py +++ b/wlauto/tests/test_configuration.py @@ -13,7 +13,7 @@ from wlauto.core.configuration.configuration import (ConfigurationPoint, RunConfiguration, merge_using_priority_specificity, get_type_name) -from wlauto.core.configuration.plugin_cache import PluginCache +from wlauto.core.configuration.plugin_cache import PluginCache, GENERIC_CONFIGS from wlauto.utils.types import obj_dict # A1 # / \ @@ -65,9 +65,9 @@ def _construct_mock_plugin_cache(values=None): return values[plugin_name] plugin_cache.get_plugin_config.side_effect = get_plugin_config - def get_plugin_config_points(_): + def get_plugin_parameters(_): return TestConfiguration.configuration - plugin_cache.get_plugin_config_points.side_effect = get_plugin_config_points + plugin_cache.get_plugin_parameters.side_effect = get_plugin_parameters return plugin_cache @@ -454,3 +454,168 @@ class ConfigurationTest(TestCase): def test_generate_job_spec(self): pass + + +class PluginCacheTest(TestCase): + + param1 = ConfigurationPoint("param1", aliases="test_global_alias") + param2 = ConfigurationPoint("param2", aliases="some_other_alias") + param3 = ConfigurationPoint("param3") + + plugin1 = obj_dict(values={ + "name": "plugin 1", + "parameters": [ + param1, + param2, + ] + }) + plugin2 = obj_dict(values={ + "name": "plugin 2", + "parameters": [ + param1, + param3, + ] + }) + + def get_plugin(self, name): + if name == "plugin 1": + return self.plugin1 + if name == "plugin 2": + return self.plugin2 + + def has_plugin(self, name): + return name in ["plugin 1", "plugin 2"] + + def make_mock_cache(self): + mock_loader = Mock() + mock_loader.get_plugin_class.side_effect = self.get_plugin + mock_loader.list_plugins = Mock(return_value=[self.plugin1, self.plugin2]) + mock_loader.has_plugin.side_effect = self.has_plugin + return PluginCache(loader=mock_loader) + + def test_get_params(self): + plugin_cache = self.make_mock_cache() + + expected_params = { + self.param1.name: self.param1, + self.param2.name: self.param2, + } + + assert_equal(expected_params, plugin_cache.get_plugin_parameters("plugin 1")) + + def test_global_aliases(self): + plugin_cache = self.make_mock_cache() + + # Check the alias map + expected_map = { + "plugin 1": { + self.param1.aliases: self.param1, + self.param2.aliases: self.param2, + }, + "plugin 2": { + self.param1.aliases: self.param1, + } + } + expected_set = set(["test_global_alias", "some_other_alias"]) + + assert_equal(expected_map, plugin_cache._global_alias_map) + assert_equal(expected_set, plugin_cache._list_of_global_aliases) + assert_equal(True, plugin_cache.is_global_alias("test_global_alias")) + assert_equal(False, plugin_cache.is_global_alias("not_a_global_alias")) + + # Error when adding to unknown source + with self.assertRaises(RuntimeError): + plugin_cache.add_global_alias("adding", "too", "early") + + # Test adding sources + for x in xrange(5): + plugin_cache.add_source(x) + assert_equal([0, 1, 2, 3, 4], plugin_cache.sources) + + # Error when adding non plugin/global alias/generic + with self.assertRaises(RuntimeError): + plugin_cache.add_global_alias("unknow_alias", "some_value", 0) + + # Test adding global alias values + plugin_cache.add_global_alias("test_global_alias", "some_value", 0) + expected_aliases = {"test_global_alias": {0: "some_value"}} + assert_equal(expected_aliases, plugin_cache.global_alias_values) + + def test_add_config(self): + plugin_cache = self.make_mock_cache() + + # Test adding sources + for x in xrange(5): + plugin_cache.add_source(x) + assert_equal([0, 1, 2, 3, 4], plugin_cache.sources) + + # Test adding plugin config + plugin_cache.add_config("plugin 1", "param1", "some_other_value", 0) + expected_plugin_config = {"plugin 1": {0: {"param1": "some_other_value"}}} + assert_equal(expected_plugin_config, plugin_cache.plugin_configs) + + # Test adding generic config + for name in GENERIC_CONFIGS: + plugin_cache.add_config(name, "param1", "some_value", 0) + expected_plugin_config[name] = {} + expected_plugin_config[name][0] = {"param1": "some_value"} + assert_equal(expected_plugin_config, plugin_cache.plugin_configs) + + def test_get_plugin_config(self): + plugin_cache = self.make_mock_cache() + for x in xrange(5): + plugin_cache.add_source(x) + + # Add some global aliases + plugin_cache.add_global_alias("test_global_alias", "1", 0) + plugin_cache.add_global_alias("test_global_alias", "2", 4) + plugin_cache.add_global_alias("test_global_alias", "3", 3) + + # Test if they are being merged in source order + expected_config = { + "param1": "2", + "param2": None, + } + assert_equal(expected_config, plugin_cache.get_plugin_config("plugin 1")) + + # Add some plugin specific config + plugin_cache.add_config("plugin 1", "param1", "3", 0) + plugin_cache.add_config("plugin 1", "param1", "4", 2) + plugin_cache.add_config("plugin 1", "param1", "5", 1) + + # Test if they are being merged in source order on top of the global aliases + expected_config = { + "param1": "4", + "param2": None, + } + assert_equal(expected_config, plugin_cache.get_plugin_config("plugin 1")) + + def test_merge_using_priority_specificity(self): + plugin_cache = self.make_mock_cache() + for x in xrange(5): + plugin_cache.add_source(x) + + # Add generic configs + plugin_cache.add_config("device_config", "param1", '1', 1) + plugin_cache.add_config("device_config", "param1", '2', 2) + assert_equal(plugin_cache.get_plugin_config("plugin 1", generic_name="device_config"), + {"param1": '2', "param2": None}) + + # Add specific configs at same level as generic config + plugin_cache.add_config("plugin 1", "param1", '3', 2) + assert_equal(plugin_cache.get_plugin_config("plugin 1", generic_name="device_config"), + {"param1": '3', "param2": None}) + + # Add specific config at higher level + plugin_cache.add_config("plugin 1", "param1", '4', 3) + assert_equal(plugin_cache.get_plugin_config("plugin 1", generic_name="device_config"), + {"param1": '4', "param2": None}) + + # Add generic config at higher level - should be an error + plugin_cache.add_config("device_config", "param1", '5', 4) + msg = 'Error in "4":\n' \ + '\t"device_config" configuration "param1" has already been specified' \ + ' more specifically for plugin 1 in:\n' \ + '\t\t2, 3' + with self.assertRaisesRegexp(ConfigError, msg): + plugin_cache.get_plugin_config("plugin 1", generic_name="device_config")