From f19ca4c00cef1a3ffa0a0e0867f3bdc8a79e27cd Mon Sep 17 00:00:00 2001 From: sergei Trofimov Date: Wed, 25 Apr 2018 16:03:50 +0100 Subject: [PATCH] fw/config: add logging to parsers Add logging to config/agenda parsing. --- wa/framework/configuration/core.py | 10 +++++++--- wa/framework/configuration/parsers.py | 24 ++++++++++++++++++++---- wa/framework/configuration/tree.py | 17 +++++++++++++++++ 3 files changed, 44 insertions(+), 7 deletions(-) diff --git a/wa/framework/configuration/core.py b/wa/framework/configuration/core.py index 6becdbd0..233ef8de 100644 --- a/wa/framework/configuration/core.py +++ b/wa/framework/configuration/core.py @@ -19,6 +19,7 @@ from collections import OrderedDict, defaultdict from wa.framework.exception import ConfigError, NotFoundError from wa.framework.configuration.tree import SectionNode +from wa.utils import log from wa.utils.misc import (get_article, merge_config_values) from wa.utils.types import (identifier, integer, boolean, list_of_strings, list_of, toggle_set, obj_dict, enum) @@ -37,7 +38,6 @@ Status = enum(['UNKNOWN', 'NEW', 'PENDING', 'OK', 'PARTIAL', 'FAILED', 'ABORTED', 'SKIPPED']) - ########################## ### CONFIG POINT TYPES ### ########################## @@ -933,8 +933,12 @@ class JobGenerator(object): def add_section(self, section, workloads): new_node = self.root_node.add_section(section) - for workload in workloads: - new_node.add_workload(workload) + log.indent() + try: + for workload in workloads: + new_node.add_workload(workload) + finally: + log.dedent() def add_workload(self, workload): self.root_node.add_workload(workload) diff --git a/wa/framework/configuration/parsers.py b/wa/framework/configuration/parsers.py index 22f8765c..3360c8fa 100644 --- a/wa/framework/configuration/parsers.py +++ b/wa/framework/configuration/parsers.py @@ -14,16 +14,17 @@ # import os +import logging from wa.framework.configuration.core import JobSpec from wa.framework.exception import ConfigError +from wa.utils import log from wa.utils.serializer import json, read_pod, SerializerSyntaxError from wa.utils.types import toggle_set, counter -############### -### Parsers ### -############### +logger = logging.getLogger('config') + class ConfigParser(object): @@ -31,6 +32,8 @@ class ConfigParser(object): self.load(state, _load_file(filepath, "Config"), filepath) def load(self, state, raw, source, wrap_exceptions=True): # pylint: disable=too-many-branches + logger.debug('Parsing config from "{}"'.format(source)) + log.indent() try: state.plugin_cache.add_source(source) if 'run_name' in raw: @@ -47,23 +50,27 @@ class ConfigParser(object): for cfg_point in state.settings.configuration.itervalues(): value = pop_aliased_param(cfg_point, raw) if value is not None: + logger.debug('Setting meta "{}" to "{}"'.format(cfg_point.name, value)) state.settings.set(cfg_point.name, value) # Get run specific configuration for cfg_point in state.run_config.configuration.itervalues(): value = pop_aliased_param(cfg_point, raw) if value is not None: + logger.debug('Setting run "{}" to "{}"'.format(cfg_point.name, value)) state.run_config.set(cfg_point.name, value) # Get global job spec configuration for cfg_point in JobSpec.configuration.itervalues(): value = pop_aliased_param(cfg_point, raw) if value is not None: + logger.debug('Setting global "{}" to "{}"'.format(cfg_point.name, value)) state.jobs_config.set_global_value(cfg_point.name, value) for name, values in raw.iteritems(): # Assume that all leftover config is for a plug-in or a global # alias it is up to PluginCache to assert this assumption + logger.debug('Caching "{}" with "{}"'.format(name, values)) state.plugin_cache.add_configs(name, values, source) except ConfigError as e: @@ -71,6 +78,9 @@ class ConfigParser(object): raise ConfigError('Error in "{}":\n{}'.format(source, str(e))) else: raise e + finally: + log.dedent() + class AgendaParser(object): @@ -80,6 +90,8 @@ class AgendaParser(object): self.load(state, raw, filepath) def load(self, state, raw, source): + logger.debug('Parsing agenda from "{}"'.format(source)) + log.indent() try: if not isinstance(raw, dict): raise ConfigError('Invalid agenda, top level entry must be a dict') @@ -104,6 +116,8 @@ class AgendaParser(object): except (ConfigError, SerializerSyntaxError) as e: raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e))) + finally: + log.dedent() def _populate_and_validate_config(self, state, raw, source): for name in ['config', 'global']: @@ -116,7 +130,9 @@ class AgendaParser(object): raise ConfigError(msg.format(name)) if 'run_name' in entry: - state.run_config.set('run_name', entry.pop('run_name')) + value = entry.pop('run_name') + logger.debug('Setting run name to "{}"'.format(value)) + state.run_config.set('run_name', value) state.load_config(entry, '{}/{}'.format(source, name), wrap_exceptions=False) diff --git a/wa/framework/configuration/tree.py b/wa/framework/configuration/tree.py index 1cec5d38..a1880b1e 100644 --- a/wa/framework/configuration/tree.py +++ b/wa/framework/configuration/tree.py @@ -12,6 +12,13 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging + +from wa.utils import log + + +logger = logging.getLogger('config') + class JobSpecSource(object): @@ -20,6 +27,7 @@ class JobSpecSource(object): def __init__(self, config, parent=None): self.config = config self.parent = parent + self._log_self() @property def id(self): @@ -28,6 +36,15 @@ class JobSpecSource(object): def name(self): raise NotImplementedError() + def _log_self(self): + logger.debug('Creating {} node'.format(self.kind)) + log.indent() + try: + for key, value in self.config.iteritems(): + logger.debug('"{}" to "{}"'.format(key, value)) + finally: + log.dedent() + class WorkloadEntry(JobSpecSource): kind = "workload"