From 44853334542ebe44e3e419cf3891d8961f1ba4ab Mon Sep 17 00:00:00 2001 From: Sebastian Goscik Date: Thu, 17 Mar 2016 13:43:45 +0000 Subject: [PATCH] Moved wlauto.core.config.core into wlauto.core.configuration --- wlauto/__init__.py | 2 +- wlauto/commands/record.py | 1 - wlauto/commands/show.py | 2 +- wlauto/core/config/__init__.py | 2 - wlauto/core/config/core.py | 650 --------------------------------- wlauto/core/configuration.py | 646 +++++++++++++++++++++++++++++++- wlauto/core/entry_point.py | 2 +- wlauto/core/execution.py | 3 +- wlauto/core/exttype.py | 2 +- wlauto/core/plugin.py | 4 +- wlauto/core/pluginloader.py | 2 +- wlauto/core/resource.py | 2 +- wlauto/utils/log.py | 2 +- 13 files changed, 653 insertions(+), 667 deletions(-) delete mode 100644 wlauto/core/config/__init__.py delete mode 100644 wlauto/core/config/core.py diff --git a/wlauto/__init__.py b/wlauto/__init__.py index 4149850a..b73f57f4 100644 --- a/wlauto/__init__.py +++ b/wlauto/__init__.py @@ -13,7 +13,7 @@ # limitations under the License. # -from wlauto.core.config.core import settings # NOQA +from wlauto.core.configuration import settings # NOQA from wlauto.core.device_manager import DeviceManager, RuntimeParameter, CoreParameter # NOQA from wlauto.core.command import Command # NOQA from wlauto.core.workload import Workload # NOQA diff --git a/wlauto/commands/record.py b/wlauto/commands/record.py index 6a77e5a7..373099f9 100644 --- a/wlauto/commands/record.py +++ b/wlauto/commands/record.py @@ -16,7 +16,6 @@ import os import sys -from wlauto import PluginLoader, Command, settings from wlauto.common.resources import Executable from wlauto.core.resource import NO_ONE from wlauto.core.resolver import ResourceResolver diff --git a/wlauto/commands/show.py b/wlauto/commands/show.py index a1c9a482..e89085e3 100644 --- a/wlauto/commands/show.py +++ b/wlauto/commands/show.py @@ -19,7 +19,7 @@ import subprocess from cStringIO import StringIO from wlauto import Command -from wlauto.core.config.core import settings +from wlauto.core.configuration import settings from wlauto.core import pluginloader from wlauto.utils.doc import (get_summary, get_description, get_type_name, format_column, format_body, format_paragraph, indent, strip_inlined_text) diff --git a/wlauto/core/config/__init__.py b/wlauto/core/config/__init__.py deleted file mode 100644 index d4154638..00000000 --- a/wlauto/core/config/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from wlauto.core.config.core import settings, ConfigurationPoint, PluginConfiguration -from wlauto.core.config.core import merge_config_values, WA_CONFIGURATION diff --git a/wlauto/core/config/core.py b/wlauto/core/config/core.py deleted file mode 100644 index f6822dca..00000000 --- a/wlauto/core/config/core.py +++ /dev/null @@ -1,650 +0,0 @@ -import os -import logging -import shutil -from glob import glob -from copy import copy -from itertools import chain - -from wlauto.core import pluginloader -from wlauto.exceptions import ConfigError -from wlauto.utils.types import integer, boolean, identifier, list_of_strings -from wlauto.utils.misc import isiterable, get_article -from wlauto.utils.serializer import read_pod, yaml - - -class ConfigurationPoint(object): - """ - This defines a gneric configuration point for workload automation. This is - used to handle global settings, plugin parameters, etc. - - """ - - # Mapping for kind conversion; see docs for convert_types below - kind_map = { - int: integer, - bool: boolean, - } - - def __init__(self, name, - kind=None, - mandatory=None, - default=None, - override=False, - allowed_values=None, - description=None, - constraint=None, - merge=False, - aliases=None, - convert_types=True): - """ - Create a new Parameter object. - - :param name: The name of the parameter. This will become an instance - member of the plugin object to which the parameter is - applied, so it must be a valid python identifier. This - is the only mandatory parameter. - :param kind: The type of parameter this is. This must be a callable - that takes an arbitrary object and converts it to the - expected type, or raised ``ValueError`` if such conversion - is not possible. Most Python standard types -- ``str``, - ``int``, ``bool``, etc. -- can be used here. This - defaults to ``str`` if not specified. - :param mandatory: If set to ``True``, then a non-``None`` value for - this parameter *must* be provided on plugin - object construction, otherwise ``ConfigError`` - will be raised. - :param default: The default value for this parameter. If no value - is specified on plugin construction, this value - will be used instead. (Note: if this is specified - and is not ``None``, then ``mandatory`` parameter - will be ignored). - :param override: A ``bool`` that specifies whether a parameter of - the same name further up the hierarchy should - be overridden. If this is ``False`` (the - default), an exception will be raised by the - ``AttributeCollection`` instead. - :param allowed_values: This should be the complete list of allowed - values for this parameter. Note: ``None`` - value will always be allowed, even if it is - not in this list. If you want to disallow - ``None``, set ``mandatory`` to ``True``. - :param constraint: If specified, this must be a callable that takes - the parameter value as an argument and return a - boolean indicating whether the constraint has been - satisfied. Alternatively, can be a two-tuple with - said callable as the first element and a string - describing the constraint as the second. - :param merge: The default behaviour when setting a value on an object - that already has that attribute is to overrided with - the new value. If this is set to ``True`` then the two - values will be merged instead. The rules by which the - values are merged will be determined by the types of - the existing and new values -- see - ``merge_config_values`` documentation for details. - :param aliases: Alternative names for the same configuration point. - These are largely for backwards compatibility. - :param convert_types: If ``True`` (the default), will automatically - convert ``kind`` values from native Python - types to WA equivalents. This allows more - ituitive interprestation of parameter values, - e.g. the string ``"false"`` being interpreted - as ``False`` when specifed as the value for - a boolean Parameter. - - """ - self.name = identifier(name) - if kind is not None and not callable(kind): - raise ValueError('Kind must be callable.') - if convert_types and kind in self.kind_map: - kind = self.kind_map[kind] - self.kind = kind - self.mandatory = mandatory - self.default = default - self.override = override - self.allowed_values = allowed_values - self.description = description - if self.kind is None and not self.override: - self.kind = str - if constraint is not None and not callable(constraint) and not isinstance(constraint, tuple): - raise ValueError('Constraint must be callable or a (callable, str) tuple.') - self.constraint = constraint - self.merge = merge - self.aliases = aliases or [] - - def match(self, name): - if name == self.name: - return True - elif name in self.aliases: - return True - return False - - def set_value(self, obj, value=None): - if value is None: - if self.default is not None: - value = self.default - elif self.mandatory: - msg = 'No values specified for mandatory parameter {} in {}' - raise ConfigError(msg.format(self.name, obj.name)) - else: - try: - value = self.kind(value) - except (ValueError, TypeError): - typename = self.get_type_name() - msg = 'Bad value "{}" for {}; must be {} {}' - article = get_article(typename) - raise ConfigError(msg.format(value, self.name, article, typename)) - if self.merge and hasattr(obj, self.name): - value = merge_config_values(getattr(obj, self.name), value) - setattr(obj, self.name, value) - - def validate(self, obj): - value = getattr(obj, self.name, None) - - if value is not None: - if self.allowed_values: - self._validate_allowed_values(obj, value) - if self.constraint: - self._validate_constraint(obj, value) - else: - if self.mandatory: - msg = 'No value specified for mandatory parameter {} in {}.' - raise ConfigError(msg.format(self.name, obj.name)) - - def get_type_name(self): - typename = str(self.kind) - if '\'' in typename: - typename = typename.split('\'')[1] - elif typename.startswith(' plugin_config. """ - - def __init__(self, loader=pluginloader): - self.loader = loader - self.config = {} - - def update(self, name, config): - if not hasattr(config, 'get'): - raise ValueError('config must be a dict-like object got: {}'.format(config)) - name, alias_config = self.loader.resolve_alias(name) - existing_config = self.config.get(name) - if existing_config is None: - existing_config = alias_config - - new_config = config or {} - self.config[name] = merge_config_values(existing_config, new_config) - - -def merge_config_values(base, other): - """ - This is used to merge two objects, typically when setting the value of a - ``ConfigurationPoint``. First, both objects are categorized into - - c: A scalar value. Basically, most objects. These values - are treated as atomic, and not mergeable. - s: A sequence. Anything iterable that is not a dict or - a string (strings are considered scalars). - m: A key-value mapping. ``dict`` and it's derivatives. - n: ``None``. - o: A mergeable object; this is an object that implements both - ``merge_with`` and ``merge_into`` methods. - - The merge rules based on the two categories are then as follows: - - (c1, c2) --> c2 - (s1, s2) --> s1 . s2 - (m1, m2) --> m1 . m2 - (c, s) --> [c] . s - (s, c) --> s . [c] - (s, m) --> s . [m] - (m, s) --> [m] . s - (m, c) --> ERROR - (c, m) --> ERROR - (o, X) --> o.merge_with(X) - (X, o) --> o.merge_into(X) - (X, n) --> X - (n, X) --> X - - where: - - '.' means concatenation (for maps, contcationation of (k, v) streams - then converted back into a map). If the types of the two objects - differ, the type of ``other`` is used for the result. - 'X' means "any category" - '[]' used to indicate a literal sequence (not necessarily a ``list``). - when this is concatenated with an actual sequence, that sequencies - type is used. - - notes: - - - When a mapping is combined with a sequence, that mapping is - treated as a scalar value. - - When combining two mergeable objects, they're combined using - ``o1.merge_with(o2)`` (_not_ using o2.merge_into(o1)). - - Combining anything with ``None`` yields that value, irrespective - of the order. So a ``None`` value is eqivalent to the corresponding - item being omitted. - - When both values are scalars, merging is equivalent to overwriting. - - There is no recursion (e.g. if map values are lists, they will not - be merged; ``other`` will overwrite ``base`` values). If complicated - merging semantics (such as recursion) are required, they should be - implemented within custom mergeable types (i.e. those that implement - ``merge_with`` and ``merge_into``). - - While this can be used as a generic "combine any two arbitry objects" - function, the semantics have been selected specifically for merging - configuration point values. - - """ - cat_base = categorize(base) - cat_other = categorize(other) - - if cat_base == 'n': - return other - elif cat_other == 'n': - return base - - if cat_base == 'o': - return base.merge_with(other) - elif cat_other == 'o': - return other.merge_into(base) - - if cat_base == 'm': - if cat_other == 's': - return merge_sequencies([base], other) - elif cat_other == 'm': - return merge_maps(base, other) - else: - message = 'merge error ({}, {}): "{}" and "{}"' - raise ValueError(message.format(cat_base, cat_other, base, other)) - elif cat_base == 's': - if cat_other == 's': - return merge_sequencies(base, other) - else: - return merge_sequencies(base, [other]) - else: # cat_base == 'c' - if cat_other == 's': - return merge_sequencies([base], other) - elif cat_other == 'm': - message = 'merge error ({}, {}): "{}" and "{}"' - raise ValueError(message.format(cat_base, cat_other, base, other)) - else: - return other - - -def merge_sequencies(s1, s2): - return type(s2)(chain(s1, s2)) - - -def merge_maps(m1, m2): - return type(m2)(chain(m1.iteritems(), m2.iteritems())) - - -def categorize(v): - if hasattr(v, 'merge_with') and hasattr(v, 'merge_into'): - return 'o' - elif hasattr(v, 'iteritems'): - return 'm' - elif isiterable(v): - return 's' - elif v is None: - return 'n' - else: - return 'c' - - -settings = WAConfiguration() diff --git a/wlauto/core/configuration.py b/wlauto/core/configuration.py index 082fe110..e450088b 100644 --- a/wlauto/core/configuration.py +++ b/wlauto/core/configuration.py @@ -15,17 +15,657 @@ import os -import json from copy import copy from collections import OrderedDict +import logging +import shutil +from glob import glob +from itertools import chain from wlauto.exceptions import ConfigError from wlauto.utils.misc import merge_dicts, merge_lists, load_struct_from_file -from wlauto.utils.types import regex_type, identifier -from wlauto.core.config.core import settings +from wlauto.utils.types import regex_type, identifier, integer, boolean, list_of_strings from wlauto.core import pluginloader +from wlauto.utils.serializer import json, WAJSONEncoder +from wlauto.exceptions import ConfigError +from wlauto.utils.misc import isiterable, get_article +from wlauto.utils.serializer import read_pod, yaml +class ConfigurationPoint(object): + """ + This defines a gneric configuration point for workload automation. This is + used to handle global settings, plugin parameters, etc. + + """ + + # Mapping for kind conversion; see docs for convert_types below + kind_map = { + int: integer, + bool: boolean, + } + + def __init__(self, name, + kind=None, + mandatory=None, + default=None, + override=False, + allowed_values=None, + description=None, + constraint=None, + merge=False, + aliases=None, + convert_types=True): + """ + Create a new Parameter object. + + :param name: The name of the parameter. This will become an instance + member of the plugin object to which the parameter is + applied, so it must be a valid python identifier. This + is the only mandatory parameter. + :param kind: The type of parameter this is. This must be a callable + that takes an arbitrary object and converts it to the + expected type, or raised ``ValueError`` if such conversion + is not possible. Most Python standard types -- ``str``, + ``int``, ``bool``, etc. -- can be used here. This + defaults to ``str`` if not specified. + :param mandatory: If set to ``True``, then a non-``None`` value for + this parameter *must* be provided on plugin + object construction, otherwise ``ConfigError`` + will be raised. + :param default: The default value for this parameter. If no value + is specified on plugin construction, this value + will be used instead. (Note: if this is specified + and is not ``None``, then ``mandatory`` parameter + will be ignored). + :param override: A ``bool`` that specifies whether a parameter of + the same name further up the hierarchy should + be overridden. If this is ``False`` (the + default), an exception will be raised by the + ``AttributeCollection`` instead. + :param allowed_values: This should be the complete list of allowed + values for this parameter. Note: ``None`` + value will always be allowed, even if it is + not in this list. If you want to disallow + ``None``, set ``mandatory`` to ``True``. + :param constraint: If specified, this must be a callable that takes + the parameter value as an argument and return a + boolean indicating whether the constraint has been + satisfied. Alternatively, can be a two-tuple with + said callable as the first element and a string + describing the constraint as the second. + :param merge: The default behaviour when setting a value on an object + that already has that attribute is to overrided with + the new value. If this is set to ``True`` then the two + values will be merged instead. The rules by which the + values are merged will be determined by the types of + the existing and new values -- see + ``merge_config_values`` documentation for details. + :param aliases: Alternative names for the same configuration point. + These are largely for backwards compatibility. + :param convert_types: If ``True`` (the default), will automatically + convert ``kind`` values from native Python + types to WA equivalents. This allows more + ituitive interprestation of parameter values, + e.g. the string ``"false"`` being interpreted + as ``False`` when specifed as the value for + a boolean Parameter. + + """ + self.name = identifier(name) + if kind is not None and not callable(kind): + raise ValueError('Kind must be callable.') + if convert_types and kind in self.kind_map: + kind = self.kind_map[kind] + self.kind = kind + self.mandatory = mandatory + self.default = default + self.override = override + self.allowed_values = allowed_values + self.description = description + if self.kind is None and not self.override: + self.kind = str + if constraint is not None and not callable(constraint) and not isinstance(constraint, tuple): + raise ValueError('Constraint must be callable or a (callable, str) tuple.') + self.constraint = constraint + self.merge = merge + self.aliases = aliases or [] + + def match(self, name): + if name == self.name: + return True + elif name in self.aliases: + return True + return False + + def set_value(self, obj, value=None): + if value is None: + if self.default is not None: + value = self.default + elif self.mandatory: + msg = 'No values specified for mandatory parameter {} in {}' + raise ConfigError(msg.format(self.name, obj.name)) + else: + try: + value = self.kind(value) + except (ValueError, TypeError): + typename = self.get_type_name() + msg = 'Bad value "{}" for {}; must be {} {}' + article = get_article(typename) + raise ConfigError(msg.format(value, self.name, article, typename)) + if self.merge and hasattr(obj, self.name): + value = merge_config_values(getattr(obj, self.name), value) + setattr(obj, self.name, value) + + def validate(self, obj): + value = getattr(obj, self.name, None) + + if value is not None: + if self.allowed_values: + self._validate_allowed_values(obj, value) + if self.constraint: + self._validate_constraint(obj, value) + else: + if self.mandatory: + msg = 'No value specified for mandatory parameter {} in {}.' + raise ConfigError(msg.format(self.name, obj.name)) + + def get_type_name(self): + typename = str(self.kind) + if '\'' in typename: + typename = typename.split('\'')[1] + elif typename.startswith(' plugin_config. """ + + def __init__(self, loader=pluginloader): + self.loader = loader + self.config = {} + + def update(self, name, config): + if not hasattr(config, 'get'): + raise ValueError('config must be a dict-like object got: {}'.format(config)) + name, alias_config = self.loader.resolve_alias(name) + existing_config = self.config.get(name) + if existing_config is None: + existing_config = alias_config + + new_config = config or {} + self.config[name] = merge_config_values(existing_config, new_config) + + +def merge_config_values(base, other): + """ + This is used to merge two objects, typically when setting the value of a + ``ConfigurationPoint``. First, both objects are categorized into + + c: A scalar value. Basically, most objects. These values + are treated as atomic, and not mergeable. + s: A sequence. Anything iterable that is not a dict or + a string (strings are considered scalars). + m: A key-value mapping. ``dict`` and it's derivatives. + n: ``None``. + o: A mergeable object; this is an object that implements both + ``merge_with`` and ``merge_into`` methods. + + The merge rules based on the two categories are then as follows: + + (c1, c2) --> c2 + (s1, s2) --> s1 . s2 + (m1, m2) --> m1 . m2 + (c, s) --> [c] . s + (s, c) --> s . [c] + (s, m) --> s . [m] + (m, s) --> [m] . s + (m, c) --> ERROR + (c, m) --> ERROR + (o, X) --> o.merge_with(X) + (X, o) --> o.merge_into(X) + (X, n) --> X + (n, X) --> X + + where: + + '.' means concatenation (for maps, contcationation of (k, v) streams + then converted back into a map). If the types of the two objects + differ, the type of ``other`` is used for the result. + 'X' means "any category" + '[]' used to indicate a literal sequence (not necessarily a ``list``). + when this is concatenated with an actual sequence, that sequencies + type is used. + + notes: + + - When a mapping is combined with a sequence, that mapping is + treated as a scalar value. + - When combining two mergeable objects, they're combined using + ``o1.merge_with(o2)`` (_not_ using o2.merge_into(o1)). + - Combining anything with ``None`` yields that value, irrespective + of the order. So a ``None`` value is eqivalent to the corresponding + item being omitted. + - When both values are scalars, merging is equivalent to overwriting. + - There is no recursion (e.g. if map values are lists, they will not + be merged; ``other`` will overwrite ``base`` values). If complicated + merging semantics (such as recursion) are required, they should be + implemented within custom mergeable types (i.e. those that implement + ``merge_with`` and ``merge_into``). + + While this can be used as a generic "combine any two arbitry objects" + function, the semantics have been selected specifically for merging + configuration point values. + + """ + cat_base = categorize(base) + cat_other = categorize(other) + + if cat_base == 'n': + return other + elif cat_other == 'n': + return base + + if cat_base == 'o': + return base.merge_with(other) + elif cat_other == 'o': + return other.merge_into(base) + + if cat_base == 'm': + if cat_other == 's': + return merge_sequencies([base], other) + elif cat_other == 'm': + return merge_maps(base, other) + else: + message = 'merge error ({}, {}): "{}" and "{}"' + raise ValueError(message.format(cat_base, cat_other, base, other)) + elif cat_base == 's': + if cat_other == 's': + return merge_sequencies(base, other) + else: + return merge_sequencies(base, [other]) + else: # cat_base == 'c' + if cat_other == 's': + return merge_sequencies([base], other) + elif cat_other == 'm': + message = 'merge error ({}, {}): "{}" and "{}"' + raise ValueError(message.format(cat_base, cat_other, base, other)) + else: + return other + + +def merge_sequencies(s1, s2): + return type(s2)(chain(s1, s2)) + + +def merge_maps(m1, m2): + return type(m2)(chain(m1.iteritems(), m2.iteritems())) + + +def categorize(v): + if hasattr(v, 'merge_with') and hasattr(v, 'merge_into'): + return 'o' + elif hasattr(v, 'iteritems'): + return 'm' + elif isiterable(v): + return 's' + elif v is None: + return 'n' + else: + return 'c' + +settings = WAConfiguration() + class SharedConfiguration(object): def __init__(self): diff --git a/wlauto/core/entry_point.py b/wlauto/core/entry_point.py index 24e48fd9..d7867050 100644 --- a/wlauto/core/entry_point.py +++ b/wlauto/core/entry_point.py @@ -21,7 +21,7 @@ import os import subprocess import warnings -from wlauto.core.config.core import settings +from wlauto.core.configuration import settings from wlauto.core import pluginloader from wlauto.exceptions import WAError from wlauto.utils.misc import get_traceback diff --git a/wlauto/core/execution.py b/wlauto/core/execution.py index c0a7b5a4..0bc99e14 100644 --- a/wlauto/core/execution.py +++ b/wlauto/core/execution.py @@ -49,9 +49,8 @@ from itertools import izip_longest import wlauto.core.signal as signal from wlauto.core import instrumentation -from wlauto.core.config.core import settings +from wlauto.core.configuration import settings from wlauto.core.plugin import Artifact -from wlauto.core.configuration import RunConfiguration from wlauto.core import pluginloader from wlauto.core.resolver import ResourceResolver from wlauto.core.result import ResultManager, IterationResult, RunResult diff --git a/wlauto/core/exttype.py b/wlauto/core/exttype.py index 19014111..5d7a7617 100644 --- a/wlauto/core/exttype.py +++ b/wlauto/core/exttype.py @@ -15,7 +15,7 @@ # Separate module to avoid circular dependencies -from wlauto.core.config.core import settings +from wlauto.core.configuration import settings from wlauto.core.plugin import Plugin from wlauto.utils.misc import load_class from wlauto.core import pluginloader diff --git a/wlauto/core/plugin.py b/wlauto/core/plugin.py index 38483fe4..d37169c8 100644 --- a/wlauto/core/plugin.py +++ b/wlauto/core/plugin.py @@ -27,9 +27,9 @@ from copy import copy from wlauto.exceptions import NotFoundError, LoaderError, ValidationError, ConfigError from wlauto.utils.misc import isiterable, ensure_directory_exists as _d, walk_modules, load_class, merge_dicts, get_article -from wlauto.core.config.core import settings +from wlauto.core.configuration import settings from wlauto.utils.types import identifier, integer, boolean -from wlauto.core.config.core import ConfigurationPoint, ConfigurationPointCollection +from wlauto.core.configuration import ConfigurationPoint, ConfigurationPointCollection MODNAME_TRANS = string.maketrans(':/\\.', '____') diff --git a/wlauto/core/pluginloader.py b/wlauto/core/pluginloader.py index ead0a5eb..0aa8dd3f 100644 --- a/wlauto/core/pluginloader.py +++ b/wlauto/core/pluginloader.py @@ -36,7 +36,7 @@ class __LoaderWrapper(object): # These imports cannot be done at top level, because of # sys.modules manipulation below from wlauto.core.plugin import PluginLoader - from wlauto.core.config.core import settings + from wlauto.core.configuration import settings self._loader = PluginLoader(settings.plugin_packages, settings.plugin_paths, settings.plugin_ignore_paths) diff --git a/wlauto/core/resource.py b/wlauto/core/resource.py index 8e613b99..24e0ae19 100644 --- a/wlauto/core/resource.py +++ b/wlauto/core/resource.py @@ -13,7 +13,7 @@ # limitations under the License. # -from wlauto.core.config.core import settings +from wlauto.core.configuration import settings from wlauto.core.plugin import Plugin diff --git a/wlauto/utils/log.py b/wlauto/utils/log.py index 31ea91f5..a4b5d51d 100644 --- a/wlauto/utils/log.py +++ b/wlauto/utils/log.py @@ -21,7 +21,7 @@ import threading import colorama -from wlauto.core.config.core import settings +from wlauto.core.configuration import settings import wlauto.core.signal as signal