mirror of
https://github.com/ARM-software/workload-automation.git
synced 2025-01-31 10:11:17 +00:00
Moved wlauto.core.config.core into wlauto.core.configuration
This commit is contained in:
Sebastian Goscik
parent
c51593a729
commit
4485333454
@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from wlauto.core.config.core import settings # NOQA
|
||||
from wlauto.core.configuration import settings # NOQA
|
||||
from wlauto.core.device_manager import DeviceManager, RuntimeParameter, CoreParameter # NOQA
|
||||
from wlauto.core.command import Command # NOQA
|
||||
from wlauto.core.workload import Workload # NOQA
|
||||
|
||||
@ -16,7 +16,6 @@
|
||||
import os
|
||||
import sys
|
||||
|
||||
from wlauto import PluginLoader, Command, settings
|
||||
from wlauto.common.resources import Executable
|
||||
from wlauto.core.resource import NO_ONE
|
||||
from wlauto.core.resolver import ResourceResolver
|
||||
|
||||
@ -19,7 +19,7 @@ import subprocess
|
||||
from cStringIO import StringIO
|
||||
|
||||
from wlauto import Command
|
||||
from wlauto.core.config.core import settings
|
||||
from wlauto.core.configuration import settings
|
||||
from wlauto.core import pluginloader
|
||||
from wlauto.utils.doc import (get_summary, get_description, get_type_name, format_column, format_body,
|
||||
format_paragraph, indent, strip_inlined_text)
|
||||
|
||||
@ -1,2 +0,0 @@
|
||||
from wlauto.core.config.core import settings, ConfigurationPoint, PluginConfiguration
|
||||
from wlauto.core.config.core import merge_config_values, WA_CONFIGURATION
|
||||
@ -1,650 +0,0 @@
|
||||
import os
|
||||
import logging
|
||||
import shutil
|
||||
from glob import glob
|
||||
from copy import copy
|
||||
from itertools import chain
|
||||
|
||||
from wlauto.core import pluginloader
|
||||
from wlauto.exceptions import ConfigError
|
||||
from wlauto.utils.types import integer, boolean, identifier, list_of_strings
|
||||
from wlauto.utils.misc import isiterable, get_article
|
||||
from wlauto.utils.serializer import read_pod, yaml
|
||||
|
||||
|
||||
class ConfigurationPoint(object):
|
||||
"""
|
||||
This defines a gneric configuration point for workload automation. This is
|
||||
used to handle global settings, plugin parameters, etc.
|
||||
|
||||
"""
|
||||
|
||||
# Mapping for kind conversion; see docs for convert_types below
|
||||
kind_map = {
|
||||
int: integer,
|
||||
bool: boolean,
|
||||
}
|
||||
|
||||
def __init__(self, name,
|
||||
kind=None,
|
||||
mandatory=None,
|
||||
default=None,
|
||||
override=False,
|
||||
allowed_values=None,
|
||||
description=None,
|
||||
constraint=None,
|
||||
merge=False,
|
||||
aliases=None,
|
||||
convert_types=True):
|
||||
"""
|
||||
Create a new Parameter object.
|
||||
|
||||
:param name: The name of the parameter. This will become an instance
|
||||
member of the plugin object to which the parameter is
|
||||
applied, so it must be a valid python identifier. This
|
||||
is the only mandatory parameter.
|
||||
:param kind: The type of parameter this is. This must be a callable
|
||||
that takes an arbitrary object and converts it to the
|
||||
expected type, or raised ``ValueError`` if such conversion
|
||||
is not possible. Most Python standard types -- ``str``,
|
||||
``int``, ``bool``, etc. -- can be used here. This
|
||||
defaults to ``str`` if not specified.
|
||||
:param mandatory: If set to ``True``, then a non-``None`` value for
|
||||
this parameter *must* be provided on plugin
|
||||
object construction, otherwise ``ConfigError``
|
||||
will be raised.
|
||||
:param default: The default value for this parameter. If no value
|
||||
is specified on plugin construction, this value
|
||||
will be used instead. (Note: if this is specified
|
||||
and is not ``None``, then ``mandatory`` parameter
|
||||
will be ignored).
|
||||
:param override: A ``bool`` that specifies whether a parameter of
|
||||
the same name further up the hierarchy should
|
||||
be overridden. If this is ``False`` (the
|
||||
default), an exception will be raised by the
|
||||
``AttributeCollection`` instead.
|
||||
:param allowed_values: This should be the complete list of allowed
|
||||
values for this parameter. Note: ``None``
|
||||
value will always be allowed, even if it is
|
||||
not in this list. If you want to disallow
|
||||
``None``, set ``mandatory`` to ``True``.
|
||||
:param constraint: If specified, this must be a callable that takes
|
||||
the parameter value as an argument and return a
|
||||
boolean indicating whether the constraint has been
|
||||
satisfied. Alternatively, can be a two-tuple with
|
||||
said callable as the first element and a string
|
||||
describing the constraint as the second.
|
||||
:param merge: The default behaviour when setting a value on an object
|
||||
that already has that attribute is to overrided with
|
||||
the new value. If this is set to ``True`` then the two
|
||||
values will be merged instead. The rules by which the
|
||||
values are merged will be determined by the types of
|
||||
the existing and new values -- see
|
||||
``merge_config_values`` documentation for details.
|
||||
:param aliases: Alternative names for the same configuration point.
|
||||
These are largely for backwards compatibility.
|
||||
:param convert_types: If ``True`` (the default), will automatically
|
||||
convert ``kind`` values from native Python
|
||||
types to WA equivalents. This allows more
|
||||
ituitive interprestation of parameter values,
|
||||
e.g. the string ``"false"`` being interpreted
|
||||
as ``False`` when specifed as the value for
|
||||
a boolean Parameter.
|
||||
|
||||
"""
|
||||
self.name = identifier(name)
|
||||
if kind is not None and not callable(kind):
|
||||
raise ValueError('Kind must be callable.')
|
||||
if convert_types and kind in self.kind_map:
|
||||
kind = self.kind_map[kind]
|
||||
self.kind = kind
|
||||
self.mandatory = mandatory
|
||||
self.default = default
|
||||
self.override = override
|
||||
self.allowed_values = allowed_values
|
||||
self.description = description
|
||||
if self.kind is None and not self.override:
|
||||
self.kind = str
|
||||
if constraint is not None and not callable(constraint) and not isinstance(constraint, tuple):
|
||||
raise ValueError('Constraint must be callable or a (callable, str) tuple.')
|
||||
self.constraint = constraint
|
||||
self.merge = merge
|
||||
self.aliases = aliases or []
|
||||
|
||||
def match(self, name):
|
||||
if name == self.name:
|
||||
return True
|
||||
elif name in self.aliases:
|
||||
return True
|
||||
return False
|
||||
|
||||
def set_value(self, obj, value=None):
|
||||
if value is None:
|
||||
if self.default is not None:
|
||||
value = self.default
|
||||
elif self.mandatory:
|
||||
msg = 'No values specified for mandatory parameter {} in {}'
|
||||
raise ConfigError(msg.format(self.name, obj.name))
|
||||
else:
|
||||
try:
|
||||
value = self.kind(value)
|
||||
except (ValueError, TypeError):
|
||||
typename = self.get_type_name()
|
||||
msg = 'Bad value "{}" for {}; must be {} {}'
|
||||
article = get_article(typename)
|
||||
raise ConfigError(msg.format(value, self.name, article, typename))
|
||||
if self.merge and hasattr(obj, self.name):
|
||||
value = merge_config_values(getattr(obj, self.name), value)
|
||||
setattr(obj, self.name, value)
|
||||
|
||||
def validate(self, obj):
|
||||
value = getattr(obj, self.name, None)
|
||||
|
||||
if value is not None:
|
||||
if self.allowed_values:
|
||||
self._validate_allowed_values(obj, value)
|
||||
if self.constraint:
|
||||
self._validate_constraint(obj, value)
|
||||
else:
|
||||
if self.mandatory:
|
||||
msg = 'No value specified for mandatory parameter {} in {}.'
|
||||
raise ConfigError(msg.format(self.name, obj.name))
|
||||
|
||||
def get_type_name(self):
|
||||
typename = str(self.kind)
|
||||
if '\'' in typename:
|
||||
typename = typename.split('\'')[1]
|
||||
elif typename.startswith('<function'):
|
||||
typename = typename.split()[1]
|
||||
return typename
|
||||
|
||||
def _validate_allowed_values(self, obj, value):
|
||||
if 'list' in str(self.kind):
|
||||
for v in value:
|
||||
if v not in self.allowed_values:
|
||||
msg = 'Invalid value {} for {} in {}; must be in {}'
|
||||
raise ConfigError(msg.format(v, self.name, obj.name, self.allowed_values))
|
||||
else:
|
||||
if value not in self.allowed_values:
|
||||
msg = 'Invalid value {} for {} in {}; must be in {}'
|
||||
raise ConfigError(msg.format(value, self.name, obj.name, self.allowed_values))
|
||||
|
||||
def _validate_constraint(self, obj, value):
|
||||
msg_vals = {'value': value, 'param': self.name, 'plugin': obj.name}
|
||||
if isinstance(self.constraint, tuple) and len(self.constraint) == 2:
|
||||
constraint, msg = self.constraint # pylint: disable=unpacking-non-sequence
|
||||
elif callable(self.constraint):
|
||||
constraint = self.constraint
|
||||
msg = '"{value}" failed constraint validation for {param} in {plugin}.'
|
||||
else:
|
||||
raise ValueError('Invalid constraint for {}: must be callable or a 2-tuple'.format(self.name))
|
||||
if not constraint(value):
|
||||
raise ConfigError(value, msg.format(**msg_vals))
|
||||
|
||||
def __repr__(self):
|
||||
d = copy(self.__dict__)
|
||||
del d['description']
|
||||
return 'ConfPoint({})'.format(d)
|
||||
|
||||
__str__ = __repr__
|
||||
|
||||
|
||||
class ConfigurationPointCollection(object):
|
||||
|
||||
def __init__(self):
|
||||
self._configs = []
|
||||
self._config_map = {}
|
||||
|
||||
def get(self, name, default=None):
|
||||
return self._config_map.get(name, default)
|
||||
|
||||
def add(self, point):
|
||||
if not isinstance(point, ConfigurationPoint):
|
||||
raise ValueError('Mustbe a ConfigurationPoint, got {}'.format(point.__class__))
|
||||
existing = self.get(point.name)
|
||||
if existing:
|
||||
if point.override:
|
||||
new_point = copy(existing)
|
||||
for a, v in point.__dict__.iteritems():
|
||||
if v is not None:
|
||||
setattr(new_point, a, v)
|
||||
self.remove(existing)
|
||||
point = new_point
|
||||
else:
|
||||
raise ValueError('Duplicate ConfigurationPoint "{}"'.format(point.name))
|
||||
self._add(point)
|
||||
|
||||
def remove(self, point):
|
||||
self._configs.remove(point)
|
||||
del self._config_map[point.name]
|
||||
for alias in point.aliases:
|
||||
del self._config_map[alias]
|
||||
|
||||
append = add
|
||||
|
||||
def _add(self, point):
|
||||
self._configs.append(point)
|
||||
self._config_map[point.name] = point
|
||||
for alias in point.aliases:
|
||||
if alias in self._config_map:
|
||||
message = 'Clashing alias "{}" between "{}" and "{}"'
|
||||
raise ValueError(message.format(alias, point.name,
|
||||
self._config_map[alias].name))
|
||||
|
||||
def __str__(self):
|
||||
str(self._configs)
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
def __iadd__(self, other):
|
||||
for p in other:
|
||||
self.add(p)
|
||||
return self
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._configs)
|
||||
|
||||
def __contains__(self, p):
|
||||
if isinstance(p, basestring):
|
||||
return p in self._config_map
|
||||
return p.name in self._config_map
|
||||
|
||||
def __getitem__(self, i):
|
||||
if isinstance(i, int):
|
||||
return self._configs[i]
|
||||
return self._config_map[i]
|
||||
|
||||
def __len__(self):
|
||||
return len(self._configs)
|
||||
|
||||
|
||||
class LoggingConfig(dict):
|
||||
|
||||
defaults = {
|
||||
'file_format': '%(asctime)s %(levelname)-8s %(name)s: %(message)s',
|
||||
'verbose_format': '%(asctime)s %(levelname)-8s %(name)s: %(message)s',
|
||||
'regular_format': '%(levelname)-8s %(message)s',
|
||||
'color': True,
|
||||
}
|
||||
|
||||
def __init__(self, config=None):
|
||||
dict.__init__(self)
|
||||
if isinstance(config, dict):
|
||||
config = {identifier(k.lower()): v for k, v in config.iteritems()}
|
||||
self['regular_format'] = config.pop('regular_format', self.defaults['regular_format'])
|
||||
self['verbose_format'] = config.pop('verbose_format', self.defaults['verbose_format'])
|
||||
self['file_format'] = config.pop('file_format', self.defaults['file_format'])
|
||||
self['color'] = config.pop('colour_enabled', self.defaults['color']) # legacy
|
||||
self['color'] = config.pop('color', self.defaults['color'])
|
||||
if config:
|
||||
message = 'Unexpected logging configuation parameters: {}'
|
||||
raise ValueError(message.format(bad_vals=', '.join(config.keys())))
|
||||
elif config is None:
|
||||
for k, v in self.defaults.iteritems():
|
||||
self[k] = v
|
||||
else:
|
||||
raise ValueError(config)
|
||||
|
||||
|
||||
__WA_CONFIGURATION = [
|
||||
ConfigurationPoint(
|
||||
'user_directory',
|
||||
description="""
|
||||
Path to the user directory. This is the location WA will look for
|
||||
user configuration, additional plugins and plugin dependencies.
|
||||
""",
|
||||
kind=str,
|
||||
default=os.path.join(os.path.expanduser('~'), '.workload_automation'),
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'plugin_packages',
|
||||
kind=list_of_strings,
|
||||
default=[
|
||||
'wlauto.commands',
|
||||
'wlauto.workloads',
|
||||
'wlauto.instrumentation',
|
||||
'wlauto.result_processors',
|
||||
'wlauto.managers',
|
||||
'wlauto.resource_getters',
|
||||
],
|
||||
description="""
|
||||
List of packages that will be scanned for WA plugins.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'plugin_paths',
|
||||
kind=list_of_strings,
|
||||
default=[
|
||||
'workloads',
|
||||
'instruments',
|
||||
'targets',
|
||||
'processors',
|
||||
|
||||
# Legacy
|
||||
'managers',
|
||||
'result_processors',
|
||||
],
|
||||
description="""
|
||||
List of paths that will be scanned for WA plugins.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'plugin_ignore_paths',
|
||||
kind=list_of_strings,
|
||||
default=[],
|
||||
description="""
|
||||
List of (sub)paths that will be ignored when scanning
|
||||
``plugin_paths`` for WA plugins.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'filer_mount_point',
|
||||
description="""
|
||||
The local mount point for the filer hosting WA assets.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'logging',
|
||||
kind=LoggingConfig,
|
||||
default=LoggingConfig.defaults,
|
||||
description="""
|
||||
WA logging configuration. This should be a dict with a subset
|
||||
of the following keys::
|
||||
|
||||
:normal_format: Logging format used for console output
|
||||
:verbose_format: Logging format used for verbose console output
|
||||
:file_format: Logging format used for run.log
|
||||
:color: If ``True`` (the default), console logging output will
|
||||
contain bash color escape codes. Set this to ``False`` if
|
||||
console output will be piped somewhere that does not know
|
||||
how to handle those.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'verbosity',
|
||||
kind=int,
|
||||
default=0,
|
||||
description="""
|
||||
Verbosity of console output.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'default_output_directory',
|
||||
default="wa_output",
|
||||
description="""
|
||||
The default output directory that will be created if not
|
||||
specified when invoking a run.
|
||||
""",
|
||||
),
|
||||
]
|
||||
|
||||
WA_CONFIGURATION = {cp.name: cp for cp in __WA_CONFIGURATION}
|
||||
|
||||
ENVIRONMENT_VARIABLES = {
|
||||
'WA_USER_DIRECTORY': WA_CONFIGURATION['user_directory'],
|
||||
'WA_PLUGIN_PATHS': WA_CONFIGURATION['plugin_paths'],
|
||||
'WA_EXTENSION_PATHS': WA_CONFIGURATION['plugin_paths'], # plugin_paths (legacy)
|
||||
}
|
||||
|
||||
|
||||
class WAConfiguration(object):
|
||||
"""
|
||||
This is configuration for Workload Automation framework as a whole. This
|
||||
does not track configuration for WA runs. Rather, this tracks "meta"
|
||||
configuration, such as various locations WA looks for things, logging
|
||||
configuration etc.
|
||||
|
||||
"""
|
||||
|
||||
basename = 'config'
|
||||
|
||||
@property
|
||||
def dependencies_directory(self):
|
||||
return os.path.join(self.user_directory, 'dependencies')
|
||||
|
||||
def __init__(self):
|
||||
self.user_directory = ''
|
||||
self.plugin_packages = []
|
||||
self.plugin_paths = []
|
||||
self.plugin_ignore_paths = []
|
||||
self.config_paths = []
|
||||
self.logging = {}
|
||||
self._logger = logging.getLogger('settings')
|
||||
for confpoint in WA_CONFIGURATION.itervalues():
|
||||
confpoint.set_value(self)
|
||||
|
||||
def load_environment(self):
|
||||
for name, confpoint in ENVIRONMENT_VARIABLES.iteritems():
|
||||
value = os.getenv(name)
|
||||
if value:
|
||||
confpoint.set_value(self, value)
|
||||
self._expand_paths()
|
||||
|
||||
def load_config_file(self, path):
|
||||
self.load(read_pod(path))
|
||||
if path not in self.config_paths:
|
||||
self.config_paths.append(config_paths)
|
||||
|
||||
def load_user_config(self):
|
||||
globpath = os.path.join(self.user_directory, '{}.*'.format(self.basename))
|
||||
for path in glob(globpath):
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
if ext in ['.pyc', '.pyo']:
|
||||
continue
|
||||
self.load_config_file(path)
|
||||
|
||||
def load(self, config):
|
||||
for name, value in config.iteritems():
|
||||
if name in WA_CONFIGURATION:
|
||||
confpoint = WA_CONFIGURATION[name]
|
||||
confpoint.set_value(self, value)
|
||||
self._expand_paths()
|
||||
|
||||
def set(self, name, value):
|
||||
if name not in WA_CONFIGURATION:
|
||||
raise ConfigError('Unknown WA configuration "{}"'.format(name))
|
||||
WA_CONFIGURATION[name].set_value(self, value)
|
||||
|
||||
def initialize_user_directory(self, overwrite=False):
|
||||
"""
|
||||
Initialize a fresh user environment creating the workload automation.
|
||||
|
||||
"""
|
||||
if os.path.exists(self.user_directory):
|
||||
if not overwrite:
|
||||
raise ConfigError('Environment {} already exists.'.format(self.user_directory))
|
||||
shutil.rmtree(self.user_directory)
|
||||
|
||||
self._expand_paths()
|
||||
os.makedirs(self.dependencies_directory)
|
||||
for path in self.plugin_paths:
|
||||
os.makedirs(path)
|
||||
|
||||
with open(os.path.join(self.user_directory, 'config.yaml'), 'w') as _:
|
||||
yaml.dump(self.to_pod())
|
||||
|
||||
if os.getenv('USER') == 'root':
|
||||
# If running with sudo on POSIX, change the ownership to the real user.
|
||||
real_user = os.getenv('SUDO_USER')
|
||||
if real_user:
|
||||
import pwd # done here as module won't import on win32
|
||||
user_entry = pwd.getpwnam(real_user)
|
||||
uid, gid = user_entry.pw_uid, user_entry.pw_gid
|
||||
os.chown(self.user_directory, uid, gid)
|
||||
# why, oh why isn't there a recusive=True option for os.chown?
|
||||
for root, dirs, files in os.walk(self.user_directory):
|
||||
for d in dirs:
|
||||
os.chown(os.path.join(root, d), uid, gid)
|
||||
for f in files:
|
||||
os.chown(os.path.join(root, f), uid, gid)
|
||||
|
||||
@staticmethod
|
||||
def from_pod(pod):
|
||||
instance = WAConfiguration()
|
||||
instance.load(pod)
|
||||
return instance
|
||||
|
||||
def to_pod(self):
|
||||
return dict(
|
||||
user_directory=self.user_directory,
|
||||
plugin_packages=self.plugin_packages,
|
||||
plugin_paths=self.plugin_paths,
|
||||
plugin_ignore_paths=self.plugin_ignore_paths,
|
||||
logging=self.logging,
|
||||
)
|
||||
|
||||
def _expand_paths(self):
|
||||
self.dependencies_directory = os.path.join(self.user_directory,
|
||||
self.dependencies_directory)
|
||||
expanded = []
|
||||
for path in self.plugin_paths:
|
||||
path = os.path.expanduser(path)
|
||||
path = os.path.expandvars(path)
|
||||
expanded.append(os.path.join(self.user_directory, path))
|
||||
self.plugin_paths = expanded
|
||||
expanded = []
|
||||
for path in self.plugin_ignore_paths:
|
||||
path = os.path.expanduser(path)
|
||||
path = os.path.expandvars(path)
|
||||
expanded.append(os.path.join(self.user_directory, path))
|
||||
self.plugin_ignore_paths = expanded
|
||||
|
||||
|
||||
class PluginConfiguration(object):
|
||||
""" Maintains a mapping of plugin_name --> plugin_config. """
|
||||
|
||||
def __init__(self, loader=pluginloader):
|
||||
self.loader = loader
|
||||
self.config = {}
|
||||
|
||||
def update(self, name, config):
|
||||
if not hasattr(config, 'get'):
|
||||
raise ValueError('config must be a dict-like object got: {}'.format(config))
|
||||
name, alias_config = self.loader.resolve_alias(name)
|
||||
existing_config = self.config.get(name)
|
||||
if existing_config is None:
|
||||
existing_config = alias_config
|
||||
|
||||
new_config = config or {}
|
||||
self.config[name] = merge_config_values(existing_config, new_config)
|
||||
|
||||
|
||||
def merge_config_values(base, other):
|
||||
"""
|
||||
This is used to merge two objects, typically when setting the value of a
|
||||
``ConfigurationPoint``. First, both objects are categorized into
|
||||
|
||||
c: A scalar value. Basically, most objects. These values
|
||||
are treated as atomic, and not mergeable.
|
||||
s: A sequence. Anything iterable that is not a dict or
|
||||
a string (strings are considered scalars).
|
||||
m: A key-value mapping. ``dict`` and it's derivatives.
|
||||
n: ``None``.
|
||||
o: A mergeable object; this is an object that implements both
|
||||
``merge_with`` and ``merge_into`` methods.
|
||||
|
||||
The merge rules based on the two categories are then as follows:
|
||||
|
||||
(c1, c2) --> c2
|
||||
(s1, s2) --> s1 . s2
|
||||
(m1, m2) --> m1 . m2
|
||||
(c, s) --> [c] . s
|
||||
(s, c) --> s . [c]
|
||||
(s, m) --> s . [m]
|
||||
(m, s) --> [m] . s
|
||||
(m, c) --> ERROR
|
||||
(c, m) --> ERROR
|
||||
(o, X) --> o.merge_with(X)
|
||||
(X, o) --> o.merge_into(X)
|
||||
(X, n) --> X
|
||||
(n, X) --> X
|
||||
|
||||
where:
|
||||
|
||||
'.' means concatenation (for maps, contcationation of (k, v) streams
|
||||
then converted back into a map). If the types of the two objects
|
||||
differ, the type of ``other`` is used for the result.
|
||||
'X' means "any category"
|
||||
'[]' used to indicate a literal sequence (not necessarily a ``list``).
|
||||
when this is concatenated with an actual sequence, that sequencies
|
||||
type is used.
|
||||
|
||||
notes:
|
||||
|
||||
- When a mapping is combined with a sequence, that mapping is
|
||||
treated as a scalar value.
|
||||
- When combining two mergeable objects, they're combined using
|
||||
``o1.merge_with(o2)`` (_not_ using o2.merge_into(o1)).
|
||||
- Combining anything with ``None`` yields that value, irrespective
|
||||
of the order. So a ``None`` value is eqivalent to the corresponding
|
||||
item being omitted.
|
||||
- When both values are scalars, merging is equivalent to overwriting.
|
||||
- There is no recursion (e.g. if map values are lists, they will not
|
||||
be merged; ``other`` will overwrite ``base`` values). If complicated
|
||||
merging semantics (such as recursion) are required, they should be
|
||||
implemented within custom mergeable types (i.e. those that implement
|
||||
``merge_with`` and ``merge_into``).
|
||||
|
||||
While this can be used as a generic "combine any two arbitry objects"
|
||||
function, the semantics have been selected specifically for merging
|
||||
configuration point values.
|
||||
|
||||
"""
|
||||
cat_base = categorize(base)
|
||||
cat_other = categorize(other)
|
||||
|
||||
if cat_base == 'n':
|
||||
return other
|
||||
elif cat_other == 'n':
|
||||
return base
|
||||
|
||||
if cat_base == 'o':
|
||||
return base.merge_with(other)
|
||||
elif cat_other == 'o':
|
||||
return other.merge_into(base)
|
||||
|
||||
if cat_base == 'm':
|
||||
if cat_other == 's':
|
||||
return merge_sequencies([base], other)
|
||||
elif cat_other == 'm':
|
||||
return merge_maps(base, other)
|
||||
else:
|
||||
message = 'merge error ({}, {}): "{}" and "{}"'
|
||||
raise ValueError(message.format(cat_base, cat_other, base, other))
|
||||
elif cat_base == 's':
|
||||
if cat_other == 's':
|
||||
return merge_sequencies(base, other)
|
||||
else:
|
||||
return merge_sequencies(base, [other])
|
||||
else: # cat_base == 'c'
|
||||
if cat_other == 's':
|
||||
return merge_sequencies([base], other)
|
||||
elif cat_other == 'm':
|
||||
message = 'merge error ({}, {}): "{}" and "{}"'
|
||||
raise ValueError(message.format(cat_base, cat_other, base, other))
|
||||
else:
|
||||
return other
|
||||
|
||||
|
||||
def merge_sequencies(s1, s2):
|
||||
return type(s2)(chain(s1, s2))
|
||||
|
||||
|
||||
def merge_maps(m1, m2):
|
||||
return type(m2)(chain(m1.iteritems(), m2.iteritems()))
|
||||
|
||||
|
||||
def categorize(v):
|
||||
if hasattr(v, 'merge_with') and hasattr(v, 'merge_into'):
|
||||
return 'o'
|
||||
elif hasattr(v, 'iteritems'):
|
||||
return 'm'
|
||||
elif isiterable(v):
|
||||
return 's'
|
||||
elif v is None:
|
||||
return 'n'
|
||||
else:
|
||||
return 'c'
|
||||
|
||||
|
||||
settings = WAConfiguration()
|
||||
@ -15,17 +15,657 @@
|
||||
|
||||
|
||||
import os
|
||||
import json
|
||||
from copy import copy
|
||||
from collections import OrderedDict
|
||||
import logging
|
||||
import shutil
|
||||
from glob import glob
|
||||
from itertools import chain
|
||||
|
||||
from wlauto.exceptions import ConfigError
|
||||
from wlauto.utils.misc import merge_dicts, merge_lists, load_struct_from_file
|
||||
from wlauto.utils.types import regex_type, identifier
|
||||
from wlauto.core.config.core import settings
|
||||
from wlauto.utils.types import regex_type, identifier, integer, boolean, list_of_strings
|
||||
from wlauto.core import pluginloader
|
||||
from wlauto.utils.serializer import json, WAJSONEncoder
|
||||
from wlauto.exceptions import ConfigError
|
||||
from wlauto.utils.misc import isiterable, get_article
|
||||
from wlauto.utils.serializer import read_pod, yaml
|
||||
|
||||
|
||||
class ConfigurationPoint(object):
|
||||
"""
|
||||
This defines a gneric configuration point for workload automation. This is
|
||||
used to handle global settings, plugin parameters, etc.
|
||||
|
||||
"""
|
||||
|
||||
# Mapping for kind conversion; see docs for convert_types below
|
||||
kind_map = {
|
||||
int: integer,
|
||||
bool: boolean,
|
||||
}
|
||||
|
||||
def __init__(self, name,
|
||||
kind=None,
|
||||
mandatory=None,
|
||||
default=None,
|
||||
override=False,
|
||||
allowed_values=None,
|
||||
description=None,
|
||||
constraint=None,
|
||||
merge=False,
|
||||
aliases=None,
|
||||
convert_types=True):
|
||||
"""
|
||||
Create a new Parameter object.
|
||||
|
||||
:param name: The name of the parameter. This will become an instance
|
||||
member of the plugin object to which the parameter is
|
||||
applied, so it must be a valid python identifier. This
|
||||
is the only mandatory parameter.
|
||||
:param kind: The type of parameter this is. This must be a callable
|
||||
that takes an arbitrary object and converts it to the
|
||||
expected type, or raised ``ValueError`` if such conversion
|
||||
is not possible. Most Python standard types -- ``str``,
|
||||
``int``, ``bool``, etc. -- can be used here. This
|
||||
defaults to ``str`` if not specified.
|
||||
:param mandatory: If set to ``True``, then a non-``None`` value for
|
||||
this parameter *must* be provided on plugin
|
||||
object construction, otherwise ``ConfigError``
|
||||
will be raised.
|
||||
:param default: The default value for this parameter. If no value
|
||||
is specified on plugin construction, this value
|
||||
will be used instead. (Note: if this is specified
|
||||
and is not ``None``, then ``mandatory`` parameter
|
||||
will be ignored).
|
||||
:param override: A ``bool`` that specifies whether a parameter of
|
||||
the same name further up the hierarchy should
|
||||
be overridden. If this is ``False`` (the
|
||||
default), an exception will be raised by the
|
||||
``AttributeCollection`` instead.
|
||||
:param allowed_values: This should be the complete list of allowed
|
||||
values for this parameter. Note: ``None``
|
||||
value will always be allowed, even if it is
|
||||
not in this list. If you want to disallow
|
||||
``None``, set ``mandatory`` to ``True``.
|
||||
:param constraint: If specified, this must be a callable that takes
|
||||
the parameter value as an argument and return a
|
||||
boolean indicating whether the constraint has been
|
||||
satisfied. Alternatively, can be a two-tuple with
|
||||
said callable as the first element and a string
|
||||
describing the constraint as the second.
|
||||
:param merge: The default behaviour when setting a value on an object
|
||||
that already has that attribute is to overrided with
|
||||
the new value. If this is set to ``True`` then the two
|
||||
values will be merged instead. The rules by which the
|
||||
values are merged will be determined by the types of
|
||||
the existing and new values -- see
|
||||
``merge_config_values`` documentation for details.
|
||||
:param aliases: Alternative names for the same configuration point.
|
||||
These are largely for backwards compatibility.
|
||||
:param convert_types: If ``True`` (the default), will automatically
|
||||
convert ``kind`` values from native Python
|
||||
types to WA equivalents. This allows more
|
||||
ituitive interprestation of parameter values,
|
||||
e.g. the string ``"false"`` being interpreted
|
||||
as ``False`` when specifed as the value for
|
||||
a boolean Parameter.
|
||||
|
||||
"""
|
||||
self.name = identifier(name)
|
||||
if kind is not None and not callable(kind):
|
||||
raise ValueError('Kind must be callable.')
|
||||
if convert_types and kind in self.kind_map:
|
||||
kind = self.kind_map[kind]
|
||||
self.kind = kind
|
||||
self.mandatory = mandatory
|
||||
self.default = default
|
||||
self.override = override
|
||||
self.allowed_values = allowed_values
|
||||
self.description = description
|
||||
if self.kind is None and not self.override:
|
||||
self.kind = str
|
||||
if constraint is not None and not callable(constraint) and not isinstance(constraint, tuple):
|
||||
raise ValueError('Constraint must be callable or a (callable, str) tuple.')
|
||||
self.constraint = constraint
|
||||
self.merge = merge
|
||||
self.aliases = aliases or []
|
||||
|
||||
def match(self, name):
|
||||
if name == self.name:
|
||||
return True
|
||||
elif name in self.aliases:
|
||||
return True
|
||||
return False
|
||||
|
||||
def set_value(self, obj, value=None):
|
||||
if value is None:
|
||||
if self.default is not None:
|
||||
value = self.default
|
||||
elif self.mandatory:
|
||||
msg = 'No values specified for mandatory parameter {} in {}'
|
||||
raise ConfigError(msg.format(self.name, obj.name))
|
||||
else:
|
||||
try:
|
||||
value = self.kind(value)
|
||||
except (ValueError, TypeError):
|
||||
typename = self.get_type_name()
|
||||
msg = 'Bad value "{}" for {}; must be {} {}'
|
||||
article = get_article(typename)
|
||||
raise ConfigError(msg.format(value, self.name, article, typename))
|
||||
if self.merge and hasattr(obj, self.name):
|
||||
value = merge_config_values(getattr(obj, self.name), value)
|
||||
setattr(obj, self.name, value)
|
||||
|
||||
def validate(self, obj):
|
||||
value = getattr(obj, self.name, None)
|
||||
|
||||
if value is not None:
|
||||
if self.allowed_values:
|
||||
self._validate_allowed_values(obj, value)
|
||||
if self.constraint:
|
||||
self._validate_constraint(obj, value)
|
||||
else:
|
||||
if self.mandatory:
|
||||
msg = 'No value specified for mandatory parameter {} in {}.'
|
||||
raise ConfigError(msg.format(self.name, obj.name))
|
||||
|
||||
def get_type_name(self):
|
||||
typename = str(self.kind)
|
||||
if '\'' in typename:
|
||||
typename = typename.split('\'')[1]
|
||||
elif typename.startswith('<function'):
|
||||
typename = typename.split()[1]
|
||||
return typename
|
||||
|
||||
def _validate_allowed_values(self, obj, value):
|
||||
if 'list' in str(self.kind):
|
||||
for v in value:
|
||||
if v not in self.allowed_values:
|
||||
msg = 'Invalid value {} for {} in {}; must be in {}'
|
||||
raise ConfigError(msg.format(v, self.name, obj.name, self.allowed_values))
|
||||
else:
|
||||
if value not in self.allowed_values:
|
||||
msg = 'Invalid value {} for {} in {}; must be in {}'
|
||||
raise ConfigError(msg.format(value, self.name, obj.name, self.allowed_values))
|
||||
|
||||
def _validate_constraint(self, obj, value):
|
||||
msg_vals = {'value': value, 'param': self.name, 'plugin': obj.name}
|
||||
if isinstance(self.constraint, tuple) and len(self.constraint) == 2:
|
||||
constraint, msg = self.constraint # pylint: disable=unpacking-non-sequence
|
||||
elif callable(self.constraint):
|
||||
constraint = self.constraint
|
||||
msg = '"{value}" failed constraint validation for {param} in {plugin}.'
|
||||
else:
|
||||
raise ValueError('Invalid constraint for {}: must be callable or a 2-tuple'.format(self.name))
|
||||
if not constraint(value):
|
||||
raise ConfigError(value, msg.format(**msg_vals))
|
||||
|
||||
def __repr__(self):
|
||||
d = copy(self.__dict__)
|
||||
del d['description']
|
||||
return 'ConfPoint({})'.format(d)
|
||||
|
||||
__str__ = __repr__
|
||||
|
||||
|
||||
class ConfigurationPointCollection(object):
|
||||
|
||||
def __init__(self):
|
||||
self._configs = []
|
||||
self._config_map = {}
|
||||
|
||||
def get(self, name, default=None):
|
||||
return self._config_map.get(name, default)
|
||||
|
||||
def add(self, point):
|
||||
if not isinstance(point, ConfigurationPoint):
|
||||
raise ValueError('Mustbe a ConfigurationPoint, got {}'.format(point.__class__))
|
||||
existing = self.get(point.name)
|
||||
if existing:
|
||||
if point.override:
|
||||
new_point = copy(existing)
|
||||
for a, v in point.__dict__.iteritems():
|
||||
if v is not None:
|
||||
setattr(new_point, a, v)
|
||||
self.remove(existing)
|
||||
point = new_point
|
||||
else:
|
||||
raise ValueError('Duplicate ConfigurationPoint "{}"'.format(point.name))
|
||||
self._add(point)
|
||||
|
||||
def remove(self, point):
|
||||
self._configs.remove(point)
|
||||
del self._config_map[point.name]
|
||||
for alias in point.aliases:
|
||||
del self._config_map[alias]
|
||||
|
||||
append = add
|
||||
|
||||
def _add(self, point):
|
||||
self._configs.append(point)
|
||||
self._config_map[point.name] = point
|
||||
for alias in point.aliases:
|
||||
if alias in self._config_map:
|
||||
message = 'Clashing alias "{}" between "{}" and "{}"'
|
||||
raise ValueError(message.format(alias, point.name,
|
||||
self._config_map[alias].name))
|
||||
|
||||
def __str__(self):
|
||||
str(self._configs)
|
||||
|
||||
__repr__ = __str__
|
||||
|
||||
def __iadd__(self, other):
|
||||
for p in other:
|
||||
self.add(p)
|
||||
return self
|
||||
|
||||
def __iter__(self):
|
||||
return iter(self._configs)
|
||||
|
||||
def __contains__(self, p):
|
||||
if isinstance(p, basestring):
|
||||
return p in self._config_map
|
||||
return p.name in self._config_map
|
||||
|
||||
def __getitem__(self, i):
|
||||
if isinstance(i, int):
|
||||
return self._configs[i]
|
||||
return self._config_map[i]
|
||||
|
||||
def __len__(self):
|
||||
return len(self._configs)
|
||||
|
||||
|
||||
class LoggingConfig(dict):
|
||||
|
||||
defaults = {
|
||||
'file_format': '%(asctime)s %(levelname)-8s %(name)s: %(message)s',
|
||||
'verbose_format': '%(asctime)s %(levelname)-8s %(name)s: %(message)s',
|
||||
'regular_format': '%(levelname)-8s %(message)s',
|
||||
'color': True,
|
||||
}
|
||||
|
||||
def __init__(self, config=None):
|
||||
dict.__init__(self)
|
||||
if isinstance(config, dict):
|
||||
config = {identifier(k.lower()): v for k, v in config.iteritems()}
|
||||
self['regular_format'] = config.pop('regular_format', self.defaults['regular_format'])
|
||||
self['verbose_format'] = config.pop('verbose_format', self.defaults['verbose_format'])
|
||||
self['file_format'] = config.pop('file_format', self.defaults['file_format'])
|
||||
self['color'] = config.pop('colour_enabled', self.defaults['color']) # legacy
|
||||
self['color'] = config.pop('color', self.defaults['color'])
|
||||
if config:
|
||||
message = 'Unexpected logging configuation parameters: {}'
|
||||
raise ValueError(message.format(bad_vals=', '.join(config.keys())))
|
||||
elif config is None:
|
||||
for k, v in self.defaults.iteritems():
|
||||
self[k] = v
|
||||
else:
|
||||
raise ValueError(config)
|
||||
|
||||
|
||||
__WA_CONFIGURATION = [
|
||||
ConfigurationPoint(
|
||||
'user_directory',
|
||||
description="""
|
||||
Path to the user directory. This is the location WA will look for
|
||||
user configuration, additional plugins and plugin dependencies.
|
||||
""",
|
||||
kind=str,
|
||||
default=os.path.join(os.path.expanduser('~'), '.workload_automation'),
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'plugin_packages',
|
||||
kind=list_of_strings,
|
||||
default=[
|
||||
'wlauto.commands',
|
||||
'wlauto.workloads',
|
||||
'wlauto.instrumentation',
|
||||
'wlauto.result_processors',
|
||||
'wlauto.managers',
|
||||
'wlauto.resource_getters',
|
||||
],
|
||||
description="""
|
||||
List of packages that will be scanned for WA plugins.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'plugin_paths',
|
||||
kind=list_of_strings,
|
||||
default=[
|
||||
'workloads',
|
||||
'instruments',
|
||||
'targets',
|
||||
'processors',
|
||||
|
||||
# Legacy
|
||||
'managers',
|
||||
'result_processors',
|
||||
],
|
||||
description="""
|
||||
List of paths that will be scanned for WA plugins.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'plugin_ignore_paths',
|
||||
kind=list_of_strings,
|
||||
default=[],
|
||||
description="""
|
||||
List of (sub)paths that will be ignored when scanning
|
||||
``plugin_paths`` for WA plugins.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'assets_repository',
|
||||
description="""
|
||||
The local mount point for the filer hosting WA assets.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'logging',
|
||||
kind=LoggingConfig,
|
||||
default=LoggingConfig.defaults,
|
||||
description="""
|
||||
WA logging configuration. This should be a dict with a subset
|
||||
of the following keys::
|
||||
|
||||
:normal_format: Logging format used for console output
|
||||
:verbose_format: Logging format used for verbose console output
|
||||
:file_format: Logging format used for run.log
|
||||
:color: If ``True`` (the default), console logging output will
|
||||
contain bash color escape codes. Set this to ``False`` if
|
||||
console output will be piped somewhere that does not know
|
||||
how to handle those.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'verbosity',
|
||||
kind=int,
|
||||
default=0,
|
||||
description="""
|
||||
Verbosity of console output.
|
||||
""",
|
||||
),
|
||||
ConfigurationPoint(
|
||||
'default_output_directory',
|
||||
default="wa_output",
|
||||
description="""
|
||||
The default output directory that will be created if not
|
||||
specified when invoking a run.
|
||||
""",
|
||||
),
|
||||
]
|
||||
|
||||
WA_CONFIGURATION = {cp.name: cp for cp in __WA_CONFIGURATION}
|
||||
|
||||
ENVIRONMENT_VARIABLES = {
|
||||
'WA_USER_DIRECTORY': WA_CONFIGURATION['user_directory'],
|
||||
'WA_PLUGIN_PATHS': WA_CONFIGURATION['plugin_paths'],
|
||||
'WA_EXTENSION_PATHS': WA_CONFIGURATION['plugin_paths'], # plugin_paths (legacy)
|
||||
}
|
||||
|
||||
|
||||
class WAConfiguration(object):
|
||||
"""
|
||||
This is configuration for Workload Automation framework as a whole. This
|
||||
does not track configuration for WA runs. Rather, this tracks "meta"
|
||||
configuration, such as various locations WA looks for things, logging
|
||||
configuration etc.
|
||||
|
||||
"""
|
||||
|
||||
basename = 'config'
|
||||
|
||||
@property
|
||||
def dependencies_directory(self):
|
||||
return os.path.join(self.user_directory, 'dependencies')
|
||||
|
||||
def __init__(self):
|
||||
self.user_directory = ''
|
||||
self.plugin_packages = []
|
||||
self.plugin_paths = []
|
||||
self.plugin_ignore_paths = []
|
||||
self.config_paths = []
|
||||
self.logging = {}
|
||||
self._logger = logging.getLogger('settings')
|
||||
for confpoint in WA_CONFIGURATION.itervalues():
|
||||
confpoint.set_value(self)
|
||||
|
||||
def load_environment(self):
|
||||
for name, confpoint in ENVIRONMENT_VARIABLES.iteritems():
|
||||
value = os.getenv(name)
|
||||
if value:
|
||||
confpoint.set_value(self, value)
|
||||
self._expand_paths()
|
||||
|
||||
def load_config_file(self, path):
|
||||
self.load(read_pod(path))
|
||||
if path not in self.config_paths:
|
||||
self.config_paths.append(path)
|
||||
|
||||
def load_user_config(self):
|
||||
globpath = os.path.join(self.user_directory, '{}.*'.format(self.basename))
|
||||
for path in glob(globpath):
|
||||
ext = os.path.splitext(path)[1].lower()
|
||||
if ext in ['.pyc', '.pyo', '.py~']:
|
||||
continue
|
||||
self.load_config_file(path)
|
||||
|
||||
def load(self, config):
|
||||
for name, value in config.iteritems():
|
||||
if name in WA_CONFIGURATION:
|
||||
confpoint = WA_CONFIGURATION[name]
|
||||
confpoint.set_value(self, value)
|
||||
self._expand_paths()
|
||||
|
||||
def set(self, name, value):
|
||||
if name not in WA_CONFIGURATION:
|
||||
raise ConfigError('Unknown WA configuration "{}"'.format(name))
|
||||
WA_CONFIGURATION[name].set_value(self, value)
|
||||
|
||||
def initialize_user_directory(self, overwrite=False):
|
||||
"""
|
||||
Initialize a fresh user environment creating the workload automation.
|
||||
|
||||
"""
|
||||
if os.path.exists(self.user_directory):
|
||||
if not overwrite:
|
||||
raise ConfigError('Environment {} already exists.'.format(self.user_directory))
|
||||
shutil.rmtree(self.user_directory)
|
||||
|
||||
self._expand_paths()
|
||||
os.makedirs(self.dependencies_directory)
|
||||
for path in self.plugin_paths:
|
||||
os.makedirs(path)
|
||||
|
||||
with open(os.path.join(self.user_directory, 'config.yaml'), 'w') as _:
|
||||
yaml.dump(self.to_pod())
|
||||
|
||||
if os.getenv('USER') == 'root':
|
||||
# If running with sudo on POSIX, change the ownership to the real user.
|
||||
real_user = os.getenv('SUDO_USER')
|
||||
if real_user:
|
||||
import pwd # done here as module won't import on win32
|
||||
user_entry = pwd.getpwnam(real_user)
|
||||
uid, gid = user_entry.pw_uid, user_entry.pw_gid
|
||||
os.chown(self.user_directory, uid, gid)
|
||||
# why, oh why isn't there a recusive=True option for os.chown?
|
||||
for root, dirs, files in os.walk(self.user_directory):
|
||||
for d in dirs:
|
||||
os.chown(os.path.join(root, d), uid, gid)
|
||||
for f in files:
|
||||
os.chown(os.path.join(root, f), uid, gid)
|
||||
|
||||
@staticmethod
|
||||
def from_pod(pod):
|
||||
instance = WAConfiguration()
|
||||
instance.load(pod)
|
||||
return instance
|
||||
|
||||
def to_pod(self):
|
||||
return dict(
|
||||
user_directory=self.user_directory,
|
||||
plugin_packages=self.plugin_packages,
|
||||
plugin_paths=self.plugin_paths,
|
||||
plugin_ignore_paths=self.plugin_ignore_paths,
|
||||
logging=self.logging,
|
||||
)
|
||||
|
||||
def _expand_paths(self):
|
||||
expanded = []
|
||||
for path in self.plugin_paths:
|
||||
path = os.path.expanduser(path)
|
||||
path = os.path.expandvars(path)
|
||||
expanded.append(os.path.join(self.user_directory, path))
|
||||
self.plugin_paths = expanded
|
||||
expanded = []
|
||||
for path in self.plugin_ignore_paths:
|
||||
path = os.path.expanduser(path)
|
||||
path = os.path.expandvars(path)
|
||||
expanded.append(os.path.join(self.user_directory, path))
|
||||
self.plugin_ignore_paths = expanded
|
||||
|
||||
|
||||
class PluginConfiguration(object):
|
||||
""" Maintains a mapping of plugin_name --> plugin_config. """
|
||||
|
||||
def __init__(self, loader=pluginloader):
|
||||
self.loader = loader
|
||||
self.config = {}
|
||||
|
||||
def update(self, name, config):
|
||||
if not hasattr(config, 'get'):
|
||||
raise ValueError('config must be a dict-like object got: {}'.format(config))
|
||||
name, alias_config = self.loader.resolve_alias(name)
|
||||
existing_config = self.config.get(name)
|
||||
if existing_config is None:
|
||||
existing_config = alias_config
|
||||
|
||||
new_config = config or {}
|
||||
self.config[name] = merge_config_values(existing_config, new_config)
|
||||
|
||||
|
||||
def merge_config_values(base, other):
|
||||
"""
|
||||
This is used to merge two objects, typically when setting the value of a
|
||||
``ConfigurationPoint``. First, both objects are categorized into
|
||||
|
||||
c: A scalar value. Basically, most objects. These values
|
||||
are treated as atomic, and not mergeable.
|
||||
s: A sequence. Anything iterable that is not a dict or
|
||||
a string (strings are considered scalars).
|
||||
m: A key-value mapping. ``dict`` and it's derivatives.
|
||||
n: ``None``.
|
||||
o: A mergeable object; this is an object that implements both
|
||||
``merge_with`` and ``merge_into`` methods.
|
||||
|
||||
The merge rules based on the two categories are then as follows:
|
||||
|
||||
(c1, c2) --> c2
|
||||
(s1, s2) --> s1 . s2
|
||||
(m1, m2) --> m1 . m2
|
||||
(c, s) --> [c] . s
|
||||
(s, c) --> s . [c]
|
||||
(s, m) --> s . [m]
|
||||
(m, s) --> [m] . s
|
||||
(m, c) --> ERROR
|
||||
(c, m) --> ERROR
|
||||
(o, X) --> o.merge_with(X)
|
||||
(X, o) --> o.merge_into(X)
|
||||
(X, n) --> X
|
||||
(n, X) --> X
|
||||
|
||||
where:
|
||||
|
||||
'.' means concatenation (for maps, contcationation of (k, v) streams
|
||||
then converted back into a map). If the types of the two objects
|
||||
differ, the type of ``other`` is used for the result.
|
||||
'X' means "any category"
|
||||
'[]' used to indicate a literal sequence (not necessarily a ``list``).
|
||||
when this is concatenated with an actual sequence, that sequencies
|
||||
type is used.
|
||||
|
||||
notes:
|
||||
|
||||
- When a mapping is combined with a sequence, that mapping is
|
||||
treated as a scalar value.
|
||||
- When combining two mergeable objects, they're combined using
|
||||
``o1.merge_with(o2)`` (_not_ using o2.merge_into(o1)).
|
||||
- Combining anything with ``None`` yields that value, irrespective
|
||||
of the order. So a ``None`` value is eqivalent to the corresponding
|
||||
item being omitted.
|
||||
- When both values are scalars, merging is equivalent to overwriting.
|
||||
- There is no recursion (e.g. if map values are lists, they will not
|
||||
be merged; ``other`` will overwrite ``base`` values). If complicated
|
||||
merging semantics (such as recursion) are required, they should be
|
||||
implemented within custom mergeable types (i.e. those that implement
|
||||
``merge_with`` and ``merge_into``).
|
||||
|
||||
While this can be used as a generic "combine any two arbitry objects"
|
||||
function, the semantics have been selected specifically for merging
|
||||
configuration point values.
|
||||
|
||||
"""
|
||||
cat_base = categorize(base)
|
||||
cat_other = categorize(other)
|
||||
|
||||
if cat_base == 'n':
|
||||
return other
|
||||
elif cat_other == 'n':
|
||||
return base
|
||||
|
||||
if cat_base == 'o':
|
||||
return base.merge_with(other)
|
||||
elif cat_other == 'o':
|
||||
return other.merge_into(base)
|
||||
|
||||
if cat_base == 'm':
|
||||
if cat_other == 's':
|
||||
return merge_sequencies([base], other)
|
||||
elif cat_other == 'm':
|
||||
return merge_maps(base, other)
|
||||
else:
|
||||
message = 'merge error ({}, {}): "{}" and "{}"'
|
||||
raise ValueError(message.format(cat_base, cat_other, base, other))
|
||||
elif cat_base == 's':
|
||||
if cat_other == 's':
|
||||
return merge_sequencies(base, other)
|
||||
else:
|
||||
return merge_sequencies(base, [other])
|
||||
else: # cat_base == 'c'
|
||||
if cat_other == 's':
|
||||
return merge_sequencies([base], other)
|
||||
elif cat_other == 'm':
|
||||
message = 'merge error ({}, {}): "{}" and "{}"'
|
||||
raise ValueError(message.format(cat_base, cat_other, base, other))
|
||||
else:
|
||||
return other
|
||||
|
||||
|
||||
def merge_sequencies(s1, s2):
|
||||
return type(s2)(chain(s1, s2))
|
||||
|
||||
|
||||
def merge_maps(m1, m2):
|
||||
return type(m2)(chain(m1.iteritems(), m2.iteritems()))
|
||||
|
||||
|
||||
def categorize(v):
|
||||
if hasattr(v, 'merge_with') and hasattr(v, 'merge_into'):
|
||||
return 'o'
|
||||
elif hasattr(v, 'iteritems'):
|
||||
return 'm'
|
||||
elif isiterable(v):
|
||||
return 's'
|
||||
elif v is None:
|
||||
return 'n'
|
||||
else:
|
||||
return 'c'
|
||||
|
||||
settings = WAConfiguration()
|
||||
|
||||
class SharedConfiguration(object):
|
||||
|
||||
def __init__(self):
|
||||
|
||||
@ -21,7 +21,7 @@ import os
|
||||
import subprocess
|
||||
import warnings
|
||||
|
||||
from wlauto.core.config.core import settings
|
||||
from wlauto.core.configuration import settings
|
||||
from wlauto.core import pluginloader
|
||||
from wlauto.exceptions import WAError
|
||||
from wlauto.utils.misc import get_traceback
|
||||
|
||||
@ -49,9 +49,8 @@ from itertools import izip_longest
|
||||
|
||||
import wlauto.core.signal as signal
|
||||
from wlauto.core import instrumentation
|
||||
from wlauto.core.config.core import settings
|
||||
from wlauto.core.configuration import settings
|
||||
from wlauto.core.plugin import Artifact
|
||||
from wlauto.core.configuration import RunConfiguration
|
||||
from wlauto.core import pluginloader
|
||||
from wlauto.core.resolver import ResourceResolver
|
||||
from wlauto.core.result import ResultManager, IterationResult, RunResult
|
||||
|
||||
@ -15,7 +15,7 @@
|
||||
|
||||
|
||||
# Separate module to avoid circular dependencies
|
||||
from wlauto.core.config.core import settings
|
||||
from wlauto.core.configuration import settings
|
||||
from wlauto.core.plugin import Plugin
|
||||
from wlauto.utils.misc import load_class
|
||||
from wlauto.core import pluginloader
|
||||
|
||||
@ -27,9 +27,9 @@ from copy import copy
|
||||
|
||||
from wlauto.exceptions import NotFoundError, LoaderError, ValidationError, ConfigError
|
||||
from wlauto.utils.misc import isiterable, ensure_directory_exists as _d, walk_modules, load_class, merge_dicts, get_article
|
||||
from wlauto.core.config.core import settings
|
||||
from wlauto.core.configuration import settings
|
||||
from wlauto.utils.types import identifier, integer, boolean
|
||||
from wlauto.core.config.core import ConfigurationPoint, ConfigurationPointCollection
|
||||
from wlauto.core.configuration import ConfigurationPoint, ConfigurationPointCollection
|
||||
|
||||
MODNAME_TRANS = string.maketrans(':/\\.', '____')
|
||||
|
||||
|
||||
@ -36,7 +36,7 @@ class __LoaderWrapper(object):
|
||||
# These imports cannot be done at top level, because of
|
||||
# sys.modules manipulation below
|
||||
from wlauto.core.plugin import PluginLoader
|
||||
from wlauto.core.config.core import settings
|
||||
from wlauto.core.configuration import settings
|
||||
self._loader = PluginLoader(settings.plugin_packages,
|
||||
settings.plugin_paths,
|
||||
settings.plugin_ignore_paths)
|
||||
|
||||
@ -13,7 +13,7 @@
|
||||
# limitations under the License.
|
||||
#
|
||||
|
||||
from wlauto.core.config.core import settings
|
||||
from wlauto.core.configuration import settings
|
||||
from wlauto.core.plugin import Plugin
|
||||
|
||||
|
||||
|
||||
@ -21,7 +21,7 @@ import threading
|
||||
|
||||
import colorama
|
||||
|
||||
from wlauto.core.config.core import settings
|
||||
from wlauto.core.configuration import settings
|
||||
import wlauto.core.signal as signal
|
||||
|
||||
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user