mirror of
https://github.com/ARM-software/workload-automation.git
synced 2025-01-18 20:11:20 +00:00
Implemented Plugin Cache + its unit tests
This commit is contained in:
Sebastian Goscik
parent
c02c6cbceb
commit
06e95abc78
@ -494,7 +494,7 @@ def merge_using_priority_specificity(generic_name, specific_name, plugin_cache):
|
||||
"""
|
||||
generic_config = plugin_cache.get_plugin_config(generic_name)
|
||||
specific_config = plugin_cache.get_plugin_config(specific_name)
|
||||
cfg_points = plugin_cache.get_plugin_config_points(specific_name)
|
||||
cfg_points = plugin_cache.get_plugin_parameters(specific_name)
|
||||
sources = plugin_cache.sources
|
||||
final_config = obj_dict(not_in_dict=['name'])
|
||||
seen_specific_config = defaultdict(list)
|
||||
@ -776,7 +776,7 @@ class RunConfiguration(Configuration):
|
||||
instance = super(RunConfiguration, cls).from_pod(pod, plugin_cache)
|
||||
|
||||
device_config.name = "device_config"
|
||||
cfg_points = plugin_cache.get_plugin_config_points(instance.device)
|
||||
cfg_points = plugin_cache.get_plugin_parameters(instance.device)
|
||||
for entry_name in device_config.iterkeys():
|
||||
if entry_name not in cfg_points.iterkeys():
|
||||
msg = 'Invalid entry "{}" for device "{}".'
|
||||
@ -866,7 +866,7 @@ class JobSpec(Configuration):
|
||||
|
||||
# Merge entry "workload_parameters"
|
||||
# TODO: Wrap in - "error in [agenda path]"
|
||||
cfg_points = plugin_cache.get_plugin_config_points(self.workload_name)
|
||||
cfg_points = plugin_cache.get_plugin_parameters(self.workload_name)
|
||||
for source in self._sources:
|
||||
if source in self._to_merge["workload_params"]:
|
||||
config = self._to_merge["workload_params"][source]
|
||||
|
||||
@ -12,10 +12,16 @@
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
from copy import copy
|
||||
from collections import defaultdict
|
||||
|
||||
from collections import OrderedDict
|
||||
|
||||
from wlauto.core import pluginloader
|
||||
from wlauto.exceptions import ConfigError
|
||||
from wlauto.utils.types import obj_dict
|
||||
from devlib.utils.misc import memoized
|
||||
|
||||
GENERIC_CONFIGS = ["device_config", "workload_parameters",
|
||||
"boot_parameters", "runtime_parameters"]
|
||||
|
||||
|
||||
class PluginCache(object):
|
||||
@ -27,41 +33,165 @@ class PluginCache(object):
|
||||
from, and the priority order of said sources.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.plugin_configs = {}
|
||||
self.global_alias = {}
|
||||
def __init__(self, loader=pluginloader):
|
||||
self.loader = loader
|
||||
self.sources = []
|
||||
self.finalised = False
|
||||
# TODO: Build dicts of global_alias: [list of destinations]
|
||||
self.plugin_configs = defaultdict(lambda: defaultdict(dict))
|
||||
self.global_alias_values = defaultdict(dict)
|
||||
|
||||
# Generate a mapping of what global aliases belong to
|
||||
self._global_alias_map = defaultdict(dict)
|
||||
self._list_of_global_aliases = set()
|
||||
for plugin in self.loader.list_plugins():
|
||||
for param in plugin.parameters:
|
||||
if param.aliases:
|
||||
self._global_alias_map[plugin.name][param.aliases] = param
|
||||
self._list_of_global_aliases.add(param.aliases)
|
||||
|
||||
def add_source(self, source):
|
||||
if source in self.sources:
|
||||
raise Exception("Source has already been added.")
|
||||
self.sources.append(source)
|
||||
|
||||
def _add_config(self, destination, name, value, source):
|
||||
def add_global_alias(self, alias, value, source):
|
||||
if source not in self.sources:
|
||||
msg = "Source '{}' has not been added to the plugin cache."
|
||||
raise Exception(msg.format(source))
|
||||
raise RuntimeError(msg.format(source))
|
||||
|
||||
if name not in destination:
|
||||
destination[name] = OrderedDict()
|
||||
destination[name][source] = value
|
||||
if not self.is_global_alias(alias):
|
||||
msg = "'{} is not a valid global alias'"
|
||||
raise RuntimeError(msg.format(alias))
|
||||
|
||||
def add_plugin_config(self, name, config, source):
|
||||
self._add_config(self.plugin_configs, name, config, source)
|
||||
self.global_alias_values[alias][source] = value
|
||||
|
||||
def add_global_alias(self, name, config, source):
|
||||
self._add_config(self.global_alias, name, config, source)
|
||||
def add_configs(self, plugin_name, values, source):
|
||||
print plugin_name, values
|
||||
if self.is_global_alias(plugin_name):
|
||||
self.add_global_alias(plugin_name, values, source)
|
||||
return
|
||||
for name, value in values.iteritems():
|
||||
self.add_config(plugin_name, name, value, source)
|
||||
|
||||
def finalise_config(self):
|
||||
pass
|
||||
def add_config(self, plugin_name, name, value, source):
|
||||
if source not in self.sources:
|
||||
msg = "Source '{}' has not been added to the plugin cache."
|
||||
raise RuntimeError(msg.format(source))
|
||||
|
||||
if not self.loader.has_plugin(plugin_name) and plugin_name not in GENERIC_CONFIGS:
|
||||
msg = 'configuration provided for unknown plugin "{}"'
|
||||
raise ConfigError(msg.format(plugin_name))
|
||||
|
||||
if (plugin_name not in GENERIC_CONFIGS and
|
||||
name not in self.get_plugin_parameters(plugin_name)):
|
||||
msg = "'{}' is not a valid parameter for '{}'"
|
||||
raise ConfigError(msg.format(name, plugin_name))
|
||||
|
||||
self.plugin_configs[plugin_name][source][name] = value
|
||||
|
||||
def is_global_alias(self, name):
|
||||
pass
|
||||
return name in self._list_of_global_aliases
|
||||
|
||||
def get_plugin_config(self, name):
|
||||
return self.plugin_configs[name]
|
||||
def get_plugin_config(self, plugin_name, generic_name=None):
|
||||
config = obj_dict(not_in_dict=['name'])
|
||||
config.name = plugin_name
|
||||
|
||||
def get_plugin_config_points(self, name):
|
||||
pass
|
||||
# Load plugin defaults
|
||||
cfg_points = self.get_plugin_parameters(plugin_name)
|
||||
for cfg_point in cfg_points.itervalues():
|
||||
cfg_point.set_value(config, check_mandatory=False)
|
||||
|
||||
# Merge global aliases
|
||||
for alias, param in self._global_alias_map[plugin_name].iteritems():
|
||||
if alias in self.global_alias_values:
|
||||
for source in self.sources:
|
||||
if source not in self.global_alias_values[alias]:
|
||||
continue
|
||||
param.set_value(config, value=self.global_alias_values[alias][source])
|
||||
|
||||
# Merge user config
|
||||
# Perform a simple merge with the order of sources representing priority
|
||||
if generic_name is None:
|
||||
plugin_config = self.plugin_configs[plugin_name]
|
||||
for source in self.sources:
|
||||
if source not in plugin_config:
|
||||
continue
|
||||
for name, value in plugin_config[source].iteritems():
|
||||
cfg_points[name].set_value(config, value=value)
|
||||
# A more complicated merge that involves priority of sources and specificity
|
||||
else:
|
||||
self._merge_using_priority_specificity(plugin_name, generic_name, config)
|
||||
|
||||
return config
|
||||
|
||||
@memoized
|
||||
def get_plugin_parameters(self, name):
|
||||
params = self.loader.get_plugin_class(name).parameters
|
||||
return {param.name: param for param in params}
|
||||
|
||||
# pylint: disable=too-many-nested-blocks, too-many-branches
|
||||
def _merge_using_priority_specificity(self, specific_name, generic_name, final_config):
|
||||
"""
|
||||
WA configuration can come from various sources of increasing priority, as well
|
||||
as being specified in a generic and specific manner (e.g. ``device_config``
|
||||
and ``nexus10`` respectivly). WA has two rules for the priority of configuration:
|
||||
|
||||
- Configuration from higher priority sources overrides configuration from
|
||||
lower priority sources.
|
||||
- More specific configuration overrides less specific configuration.
|
||||
|
||||
There is a situation where these two rules come into conflict. When a generic
|
||||
configuration is given in config source of high priority and a specific
|
||||
configuration is given in a config source of lower priority. In this situation
|
||||
it is not possible to know the end users intention and WA will error.
|
||||
|
||||
:param generic_name: The name of the generic configuration e.g ``device_config``
|
||||
:param specific_name: The name of the specific configuration used, e.g ``nexus10``
|
||||
:param cfg_point: A dict of ``ConfigurationPoint``s to be used when merging configuration.
|
||||
keys=config point name, values=config point
|
||||
|
||||
:rtype: A fully merged and validated configuration in the form of a obj_dict.
|
||||
"""
|
||||
generic_config = copy(self.plugin_configs[generic_name])
|
||||
specific_config = copy(self.plugin_configs[specific_name])
|
||||
cfg_points = self.get_plugin_parameters(specific_name)
|
||||
sources = self.sources
|
||||
seen_specific_config = defaultdict(list)
|
||||
|
||||
# set_value uses the 'name' attribute of the passed object in it error
|
||||
# messages, to ensure these messages make sense the name will have to be
|
||||
# changed several times during this function.
|
||||
final_config.name = specific_name
|
||||
|
||||
# pylint: disable=too-many-nested-blocks
|
||||
for source in sources:
|
||||
try:
|
||||
if source in generic_config:
|
||||
final_config.name = generic_name
|
||||
for name, cfg_point in cfg_points.iteritems():
|
||||
if name in generic_config[source]:
|
||||
if name in seen_specific_config:
|
||||
msg = ('"{generic_name}" configuration "{config_name}" has already been '
|
||||
'specified more specifically for {specific_name} in:\n\t\t{sources}')
|
||||
msg = msg.format(generic_name=generic_name,
|
||||
config_name=name,
|
||||
specific_name=specific_name,
|
||||
sources=", ".join(seen_specific_config[name]))
|
||||
raise ConfigError(msg)
|
||||
value = generic_config[source][name]
|
||||
cfg_point.set_value(final_config, value, check_mandatory=False)
|
||||
|
||||
if source in specific_config:
|
||||
final_config.name = specific_name
|
||||
for name, cfg_point in cfg_points.iteritems():
|
||||
if name in specific_config[source]:
|
||||
seen_specific_config[name].append(str(source))
|
||||
value = specific_config[source][name]
|
||||
cfg_point.set_value(final_config, value, check_mandatory=False)
|
||||
|
||||
except ConfigError as e:
|
||||
raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e)))
|
||||
|
||||
# Validate final configuration
|
||||
final_config.name = specific_name
|
||||
for cfg_point in cfg_points.itervalues():
|
||||
cfg_point.validate(final_config)
|
||||
|
||||
@ -13,7 +13,7 @@ from wlauto.core.configuration.configuration import (ConfigurationPoint,
|
||||
RunConfiguration,
|
||||
merge_using_priority_specificity,
|
||||
get_type_name)
|
||||
from wlauto.core.configuration.plugin_cache import PluginCache
|
||||
from wlauto.core.configuration.plugin_cache import PluginCache, GENERIC_CONFIGS
|
||||
from wlauto.utils.types import obj_dict
|
||||
# A1
|
||||
# / \
|
||||
@ -65,9 +65,9 @@ def _construct_mock_plugin_cache(values=None):
|
||||
return values[plugin_name]
|
||||
plugin_cache.get_plugin_config.side_effect = get_plugin_config
|
||||
|
||||
def get_plugin_config_points(_):
|
||||
def get_plugin_parameters(_):
|
||||
return TestConfiguration.configuration
|
||||
plugin_cache.get_plugin_config_points.side_effect = get_plugin_config_points
|
||||
plugin_cache.get_plugin_parameters.side_effect = get_plugin_parameters
|
||||
|
||||
return plugin_cache
|
||||
|
||||
@ -454,3 +454,168 @@ class ConfigurationTest(TestCase):
|
||||
|
||||
def test_generate_job_spec(self):
|
||||
pass
|
||||
|
||||
|
||||
class PluginCacheTest(TestCase):
|
||||
|
||||
param1 = ConfigurationPoint("param1", aliases="test_global_alias")
|
||||
param2 = ConfigurationPoint("param2", aliases="some_other_alias")
|
||||
param3 = ConfigurationPoint("param3")
|
||||
|
||||
plugin1 = obj_dict(values={
|
||||
"name": "plugin 1",
|
||||
"parameters": [
|
||||
param1,
|
||||
param2,
|
||||
]
|
||||
})
|
||||
plugin2 = obj_dict(values={
|
||||
"name": "plugin 2",
|
||||
"parameters": [
|
||||
param1,
|
||||
param3,
|
||||
]
|
||||
})
|
||||
|
||||
def get_plugin(self, name):
|
||||
if name == "plugin 1":
|
||||
return self.plugin1
|
||||
if name == "plugin 2":
|
||||
return self.plugin2
|
||||
|
||||
def has_plugin(self, name):
|
||||
return name in ["plugin 1", "plugin 2"]
|
||||
|
||||
def make_mock_cache(self):
|
||||
mock_loader = Mock()
|
||||
mock_loader.get_plugin_class.side_effect = self.get_plugin
|
||||
mock_loader.list_plugins = Mock(return_value=[self.plugin1, self.plugin2])
|
||||
mock_loader.has_plugin.side_effect = self.has_plugin
|
||||
return PluginCache(loader=mock_loader)
|
||||
|
||||
def test_get_params(self):
|
||||
plugin_cache = self.make_mock_cache()
|
||||
|
||||
expected_params = {
|
||||
self.param1.name: self.param1,
|
||||
self.param2.name: self.param2,
|
||||
}
|
||||
|
||||
assert_equal(expected_params, plugin_cache.get_plugin_parameters("plugin 1"))
|
||||
|
||||
def test_global_aliases(self):
|
||||
plugin_cache = self.make_mock_cache()
|
||||
|
||||
# Check the alias map
|
||||
expected_map = {
|
||||
"plugin 1": {
|
||||
self.param1.aliases: self.param1,
|
||||
self.param2.aliases: self.param2,
|
||||
},
|
||||
"plugin 2": {
|
||||
self.param1.aliases: self.param1,
|
||||
}
|
||||
}
|
||||
expected_set = set(["test_global_alias", "some_other_alias"])
|
||||
|
||||
assert_equal(expected_map, plugin_cache._global_alias_map)
|
||||
assert_equal(expected_set, plugin_cache._list_of_global_aliases)
|
||||
assert_equal(True, plugin_cache.is_global_alias("test_global_alias"))
|
||||
assert_equal(False, plugin_cache.is_global_alias("not_a_global_alias"))
|
||||
|
||||
# Error when adding to unknown source
|
||||
with self.assertRaises(RuntimeError):
|
||||
plugin_cache.add_global_alias("adding", "too", "early")
|
||||
|
||||
# Test adding sources
|
||||
for x in xrange(5):
|
||||
plugin_cache.add_source(x)
|
||||
assert_equal([0, 1, 2, 3, 4], plugin_cache.sources)
|
||||
|
||||
# Error when adding non plugin/global alias/generic
|
||||
with self.assertRaises(RuntimeError):
|
||||
plugin_cache.add_global_alias("unknow_alias", "some_value", 0)
|
||||
|
||||
# Test adding global alias values
|
||||
plugin_cache.add_global_alias("test_global_alias", "some_value", 0)
|
||||
expected_aliases = {"test_global_alias": {0: "some_value"}}
|
||||
assert_equal(expected_aliases, plugin_cache.global_alias_values)
|
||||
|
||||
def test_add_config(self):
|
||||
plugin_cache = self.make_mock_cache()
|
||||
|
||||
# Test adding sources
|
||||
for x in xrange(5):
|
||||
plugin_cache.add_source(x)
|
||||
assert_equal([0, 1, 2, 3, 4], plugin_cache.sources)
|
||||
|
||||
# Test adding plugin config
|
||||
plugin_cache.add_config("plugin 1", "param1", "some_other_value", 0)
|
||||
expected_plugin_config = {"plugin 1": {0: {"param1": "some_other_value"}}}
|
||||
assert_equal(expected_plugin_config, plugin_cache.plugin_configs)
|
||||
|
||||
# Test adding generic config
|
||||
for name in GENERIC_CONFIGS:
|
||||
plugin_cache.add_config(name, "param1", "some_value", 0)
|
||||
expected_plugin_config[name] = {}
|
||||
expected_plugin_config[name][0] = {"param1": "some_value"}
|
||||
assert_equal(expected_plugin_config, plugin_cache.plugin_configs)
|
||||
|
||||
def test_get_plugin_config(self):
|
||||
plugin_cache = self.make_mock_cache()
|
||||
for x in xrange(5):
|
||||
plugin_cache.add_source(x)
|
||||
|
||||
# Add some global aliases
|
||||
plugin_cache.add_global_alias("test_global_alias", "1", 0)
|
||||
plugin_cache.add_global_alias("test_global_alias", "2", 4)
|
||||
plugin_cache.add_global_alias("test_global_alias", "3", 3)
|
||||
|
||||
# Test if they are being merged in source order
|
||||
expected_config = {
|
||||
"param1": "2",
|
||||
"param2": None,
|
||||
}
|
||||
assert_equal(expected_config, plugin_cache.get_plugin_config("plugin 1"))
|
||||
|
||||
# Add some plugin specific config
|
||||
plugin_cache.add_config("plugin 1", "param1", "3", 0)
|
||||
plugin_cache.add_config("plugin 1", "param1", "4", 2)
|
||||
plugin_cache.add_config("plugin 1", "param1", "5", 1)
|
||||
|
||||
# Test if they are being merged in source order on top of the global aliases
|
||||
expected_config = {
|
||||
"param1": "4",
|
||||
"param2": None,
|
||||
}
|
||||
assert_equal(expected_config, plugin_cache.get_plugin_config("plugin 1"))
|
||||
|
||||
def test_merge_using_priority_specificity(self):
|
||||
plugin_cache = self.make_mock_cache()
|
||||
for x in xrange(5):
|
||||
plugin_cache.add_source(x)
|
||||
|
||||
# Add generic configs
|
||||
plugin_cache.add_config("device_config", "param1", '1', 1)
|
||||
plugin_cache.add_config("device_config", "param1", '2', 2)
|
||||
assert_equal(plugin_cache.get_plugin_config("plugin 1", generic_name="device_config"),
|
||||
{"param1": '2', "param2": None})
|
||||
|
||||
# Add specific configs at same level as generic config
|
||||
plugin_cache.add_config("plugin 1", "param1", '3', 2)
|
||||
assert_equal(plugin_cache.get_plugin_config("plugin 1", generic_name="device_config"),
|
||||
{"param1": '3', "param2": None})
|
||||
|
||||
# Add specific config at higher level
|
||||
plugin_cache.add_config("plugin 1", "param1", '4', 3)
|
||||
assert_equal(plugin_cache.get_plugin_config("plugin 1", generic_name="device_config"),
|
||||
{"param1": '4', "param2": None})
|
||||
|
||||
# Add generic config at higher level - should be an error
|
||||
plugin_cache.add_config("device_config", "param1", '5', 4)
|
||||
msg = 'Error in "4":\n' \
|
||||
'\t"device_config" configuration "param1" has already been specified' \
|
||||
' more specifically for plugin 1 in:\n' \
|
||||
'\t\t2, 3'
|
||||
with self.assertRaisesRegexp(ConfigError, msg):
|
||||
plugin_cache.get_plugin_config("plugin 1", generic_name="device_config")
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user