fw/config: add includes

Add the ability to include other YAML files inside agendas and config
files using "include#:" entries.

doc/source/user_information/user_reference/configuration.rst

@@ -102,6 +102,91 @@ remove the high level configuration.
 
 Dependent on specificity, configuration parameters from different sources will
 have different inherent priorities. Within an agenda, the configuration in
-"workload" entries wil be more specific than "sections" entries, which in turn
+"workload" entries will be more specific than "sections" entries, which in turn
 are more specific than parameters in the "config" entry.
 
+.. _config-include:
+
+Configuration Includes
+----------------------
+
+It is possible to include other files in your config files and agendas. This is
+done by specifying ``include#`` (note the trailing hash) as a key in one of the
+mappings, with the value being the path to the file to be included. The path
+must be either absolute, or relative to the location of the file it is being
+included from (*not* to the current working directory). The path may also
+include ``~`` to indicate current user's home directory.
+
+The include is performed by removing the ``include#`` loading the contents of
+the specified into the mapping that contained it. In cases where the mapping
+already contains the key to be loaded, values will be merged using the usual
+merge method (for overwrites, values in the mapping take precedence over those
+from the included files).
+
+Below is an example of an agenda that includes other files. The assumption is
+that all of those files are in one directory
+
+.. code-block:: yaml
+
+    # agenda.yaml
+    config:
+       augmentations: [trace-cmd]
+       include#: ./my-config.yaml
+    sections:
+       - include#: ./section1.yaml
+       - include#: ./section2.yaml
+    include#: ./workloads.yaml
+
+.. code-block:: yaml
+
+   # my-config.yaml
+   augmentations: [cpufreq]
+
+
+.. code-block:: yaml
+
+   # section1.yaml
+   runtime_parameters:
+      frequency: max
+
+.. code-block:: yaml
+
+   # section2.yaml
+   runtime_parameters:
+      frequency: min
+
+.. code-block:: yaml
+
+   # workloads.yaml
+   workloads:
+      - dhrystone
+      - memcpy
+
+The above is equivalent to having a single file like this:
+
+.. code-block:: yaml
+
+    # agenda.yaml
+    config:
+       augmentations: [cpufreq, trace-cmd]
+    sections:
+       - runtime_parameters:
+            frequency: max
+       - runtime_parameters:
+            frequency: min
+    workloads:
+       - dhrystone
+       - memcpy
+
+Some additional details about the implementation and its limitations:
+
+- The ``include#`` *must* be a key in a mapping, and the contents of the
+  included file *must* be a mapping as well; it is not possible to include a
+  list (e.g. in the examples above ``workload:`` part *must* be in the included
+  file.
+- Being a key in a mapping, there can only be one ``include#`` entry per block.
+- The included file *must* have a ``.yaml`` extension.
+- Nested inclusions *are* allowed. I.e. included files may themselves include
+  files; in such cases the included paths must be relative to *that* file, and
+  not the "main" file.
+

wa/framework/configuration/parsers.py

@@ -25,6 +25,7 @@ from wa.framework.exception import ConfigError
 from wa.utils import log
 from wa.utils.serializer import json, read_pod, SerializerSyntaxError
 from wa.utils.types import toggle_set, counter
+from wa.utils.misc import merge_config_values, isiterable
 
 
 logger = logging.getLogger('config')
@@ -33,7 +34,9 @@ logger = logging.getLogger('config')
 class ConfigParser(object):
 
     def load_from_path(self, state, filepath):
-        self.load(state, _load_file(filepath, "Config"), filepath)
+        raw, includes = _load_file(filepath, "Config")
+        self.load(state, raw, filepath)
+        return includes
 
     def load(self, state, raw, source, wrap_exceptions=True):  # pylint: disable=too-many-branches
         logger.debug('Parsing config from "{}"'.format(source))
@@ -89,8 +92,9 @@ class ConfigParser(object):
 class AgendaParser(object):
 
     def load_from_path(self, state, filepath):
-        raw = _load_file(filepath, 'Agenda')
+        raw, includes = _load_file(filepath, 'Agenda')
         self.load(state, raw, filepath)
+        return includes
 
     def load(self, state, raw, source):
         logger.debug('Parsing agenda from "{}"'.format(source))
@@ -224,12 +228,45 @@ def _load_file(filepath, error_name):
         raise ValueError("{} does not exist".format(filepath))
     try:
         raw = read_pod(filepath)
+        includes = _process_includes(raw, filepath, error_name)
     except SerializerSyntaxError as e:
         raise ConfigError('Error parsing {} {}: {}'.format(error_name, filepath, e))
     if not isinstance(raw, dict):
         message = '{} does not contain a valid {} structure; top level must be a dict.'
         raise ConfigError(message.format(filepath, error_name))
-    return raw
+    return raw, includes
+
+
+def _process_includes(raw, filepath, error_name):
+    if not raw:
+        return []
+
+    source_dir = os.path.dirname(filepath)
+    included_files = []
+    replace_value = None
+
+    if hasattr(raw, 'items'):
+        for key, value in raw.items():
+            if key == 'include#':
+                include_path = os.path.expanduser(os.path.join(source_dir, value))
+                included_files.append(include_path)
+                replace_value, includes = _load_file(include_path, error_name)
+                included_files.extend(includes)
+            elif hasattr(value, 'items') or isiterable(value):
+                includes = _process_includes(value, filepath, error_name)
+                included_files.extend(includes)
+    elif isiterable(raw):
+        for element in raw:
+            if hasattr(element, 'items') or isiterable(element):
+                includes = _process_includes(element, filepath, error_name)
+                included_files.extend(includes)
+
+    if replace_value is not None:
+        del raw['include#']
+        for key, value in replace_value.items():
+            raw[key] = merge_config_values(value, raw.get(key, None))
+
+    return included_files
 
 
 def merge_augmentations(raw):