From 7d833ec112dd1fa71d21e44fe48e76be5ae49e3b Mon Sep 17 00:00:00 2001 From: Sergei Trofimov Date: Thu, 19 Jul 2018 08:17:02 +0100 Subject: [PATCH] fw/config: add includes Add the ability to include other YAML files inside agendas and config files using "include#:" entries. --- .../user_reference/configuration.rst | 87 ++++++++++++++++++- wa/framework/configuration/parsers.py | 43 ++++++++- 2 files changed, 126 insertions(+), 4 deletions(-) diff --git a/doc/source/user_information/user_reference/configuration.rst b/doc/source/user_information/user_reference/configuration.rst index 88260e3a..44890346 100644 --- a/doc/source/user_information/user_reference/configuration.rst +++ b/doc/source/user_information/user_reference/configuration.rst @@ -102,6 +102,91 @@ remove the high level configuration. Dependent on specificity, configuration parameters from different sources will have different inherent priorities. Within an agenda, the configuration in -"workload" entries wil be more specific than "sections" entries, which in turn +"workload" entries will be more specific than "sections" entries, which in turn are more specific than parameters in the "config" entry. +.. _config-include: + +Configuration Includes +---------------------- + +It is possible to include other files in your config files and agendas. This is +done by specifying ``include#`` (note the trailing hash) as a key in one of the +mappings, with the value being the path to the file to be included. The path +must be either absolute, or relative to the location of the file it is being +included from (*not* to the current working directory). The path may also +include ``~`` to indicate current user's home directory. + +The include is performed by removing the ``include#`` loading the contents of +the specified into the mapping that contained it. In cases where the mapping +already contains the key to be loaded, values will be merged using the usual +merge method (for overwrites, values in the mapping take precedence over those +from the included files). + +Below is an example of an agenda that includes other files. The assumption is +that all of those files are in one directory + +.. code-block:: yaml + + # agenda.yaml + config: + augmentations: [trace-cmd] + include#: ./my-config.yaml + sections: + - include#: ./section1.yaml + - include#: ./section2.yaml + include#: ./workloads.yaml + +.. code-block:: yaml + + # my-config.yaml + augmentations: [cpufreq] + + +.. code-block:: yaml + + # section1.yaml + runtime_parameters: + frequency: max + +.. code-block:: yaml + + # section2.yaml + runtime_parameters: + frequency: min + +.. code-block:: yaml + + # workloads.yaml + workloads: + - dhrystone + - memcpy + +The above is equivalent to having a single file like this: + +.. code-block:: yaml + + # agenda.yaml + config: + augmentations: [cpufreq, trace-cmd] + sections: + - runtime_parameters: + frequency: max + - runtime_parameters: + frequency: min + workloads: + - dhrystone + - memcpy + +Some additional details about the implementation and its limitations: + +- The ``include#`` *must* be a key in a mapping, and the contents of the + included file *must* be a mapping as well; it is not possible to include a + list (e.g. in the examples above ``workload:`` part *must* be in the included + file. +- Being a key in a mapping, there can only be one ``include#`` entry per block. +- The included file *must* have a ``.yaml`` extension. +- Nested inclusions *are* allowed. I.e. included files may themselves include + files; in such cases the included paths must be relative to *that* file, and + not the "main" file. + diff --git a/wa/framework/configuration/parsers.py b/wa/framework/configuration/parsers.py index a6c70562..54a9ff85 100644 --- a/wa/framework/configuration/parsers.py +++ b/wa/framework/configuration/parsers.py @@ -25,6 +25,7 @@ from wa.framework.exception import ConfigError from wa.utils import log from wa.utils.serializer import json, read_pod, SerializerSyntaxError from wa.utils.types import toggle_set, counter +from wa.utils.misc import merge_config_values, isiterable logger = logging.getLogger('config') @@ -33,7 +34,9 @@ logger = logging.getLogger('config') class ConfigParser(object): def load_from_path(self, state, filepath): - self.load(state, _load_file(filepath, "Config"), filepath) + raw, includes = _load_file(filepath, "Config") + self.load(state, raw, filepath) + return includes def load(self, state, raw, source, wrap_exceptions=True): # pylint: disable=too-many-branches logger.debug('Parsing config from "{}"'.format(source)) @@ -89,8 +92,9 @@ class ConfigParser(object): class AgendaParser(object): def load_from_path(self, state, filepath): - raw = _load_file(filepath, 'Agenda') + raw, includes = _load_file(filepath, 'Agenda') self.load(state, raw, filepath) + return includes def load(self, state, raw, source): logger.debug('Parsing agenda from "{}"'.format(source)) @@ -224,12 +228,45 @@ def _load_file(filepath, error_name): raise ValueError("{} does not exist".format(filepath)) try: raw = read_pod(filepath) + includes = _process_includes(raw, filepath, error_name) except SerializerSyntaxError as e: raise ConfigError('Error parsing {} {}: {}'.format(error_name, filepath, e)) if not isinstance(raw, dict): message = '{} does not contain a valid {} structure; top level must be a dict.' raise ConfigError(message.format(filepath, error_name)) - return raw + return raw, includes + + +def _process_includes(raw, filepath, error_name): + if not raw: + return [] + + source_dir = os.path.dirname(filepath) + included_files = [] + replace_value = None + + if hasattr(raw, 'items'): + for key, value in raw.items(): + if key == 'include#': + include_path = os.path.expanduser(os.path.join(source_dir, value)) + included_files.append(include_path) + replace_value, includes = _load_file(include_path, error_name) + included_files.extend(includes) + elif hasattr(value, 'items') or isiterable(value): + includes = _process_includes(value, filepath, error_name) + included_files.extend(includes) + elif isiterable(raw): + for element in raw: + if hasattr(element, 'items') or isiterable(element): + includes = _process_includes(element, filepath, error_name) + included_files.extend(includes) + + if replace_value is not None: + del raw['include#'] + for key, value in replace_value.items(): + raw[key] = merge_config_values(value, raw.get(key, None)) + + return included_files def merge_augmentations(raw):