diff --git a/wlauto/commands/run.py b/wlauto/commands/run.py index 44bcf3ee..7fbf1c4f 100644 --- a/wlauto/commands/run.py +++ b/wlauto/commands/run.py @@ -88,9 +88,11 @@ class RunCommand(Command): self.logger.debug('Updating agenda to disable {}'.format(itd)) agenda.config['instrumentation'].append('~{}'.format(itd)) - file_name = 'config_{}.py' + basename = 'config_' for file_number, path in enumerate(settings.get_config_paths(), 1): - shutil.copy(path, os.path.join(settings.meta_directory, file_name.format(file_number))) + file_ext = os.path.splitext(path)[1] + shutil.copy(path, os.path.join(settings.meta_directory, + basename + str(file_number) + file_ext)) executor = Executor() executor.execute(agenda, selectors={'ids': args.only_run_ids}) diff --git a/wlauto/core/bootstrap.py b/wlauto/core/bootstrap.py index c299ebae..dcc8dee0 100644 --- a/wlauto/core/bootstrap.py +++ b/wlauto/core/bootstrap.py @@ -16,13 +16,13 @@ import os import shutil -import imp import sys import re from collections import namedtuple, OrderedDict from wlauto.exceptions import ConfigError from wlauto.utils.misc import merge_dicts, normalize, unique +from wlauto.utils.misc import load_struct_from_yaml, load_struct_from_python, LoadSyntaxError from wlauto.utils.types import identifier @@ -76,7 +76,7 @@ class ConfigLoader(object): self._loaded = False self._config = {} self.config_count = 0 - self._loaded_files = [] + self.loaded_files = [] self.environment_root = None self.output_directory = 'wa_output' self.reboot_after_each_iteration = True @@ -106,14 +106,22 @@ class ConfigLoader(object): self.update_from_file(source) def update_from_file(self, source): + ext = os.path.splitext(source)[1].lower() # pylint: disable=redefined-outer-name try: - new_config = imp.load_source('config_{}'.format(self.config_count), source) - except SyntaxError, e: - message = 'Sytax error in config: {}'.format(str(e)) - raise ConfigError(message) - self._config = merge_dicts(self._config, vars(new_config), - list_duplicates='first', match_types=False, dict_type=OrderedDict) - self._loaded_files.append(source) + if ext in ['.py', '.pyo', '.pyc']: + new_config = load_struct_from_python(source) + elif ext == '.yaml': + new_config = load_struct_from_yaml(source) + else: + raise ConfigError('Unknown config format: {}'.format(source)) + except LoadSyntaxError as e: + raise ConfigError(e) + + self._config = merge_dicts(self._config, new_config, + list_duplicates='first', + match_types=False, + dict_type=OrderedDict) + self.loaded_files.append(source) self._loaded = True def update_from_dict(self, source): @@ -123,7 +131,7 @@ class ConfigLoader(object): self._loaded = True def get_config_paths(self): - return [lf.rstrip('c') for lf in self._loaded_files] + return [lf.rstrip('c') for lf in self.loaded_files] def _check_loaded(self): if not self._loaded: @@ -174,13 +182,21 @@ _env_var_paths = os.getenv('WA_EXTENSION_PATHS', '') if _env_var_paths: _extension_paths.extend(_env_var_paths.split(os.pathsep)) +_env_configs = [] +for filename in ['config.py', 'config.yaml']: + filepath = os.path.join(_env_root, filename) + if os.path.isfile(filepath): + _env_configs.append(filepath) + if not os.path.isdir(_env_root): init_environment(_env_root, _dep_dir, _extension_paths) -elif not os.path.isfile(os.path.join(_env_root, 'config.py')): +elif not _env_configs: + filepath = os.path.join(_env_root, 'config.py') with open(os.path.join(_this_dir, '..', 'config_example.py')) as f: f_text = re.sub(r'""".*?"""', '', f.read(), 1, re.DOTALL) - with open(os.path.join(_env_root, 'config.py'), 'w') as f: + with open(filepath, 'w') as f: f.write(f_text) + _env_configs.append(filepath) settings = ConfigLoader() settings.environment_root = _env_root @@ -193,6 +209,6 @@ if os.path.isfile(_packages_file): with open(_packages_file) as fh: settings.extension_packages = unique(fh.read().split()) -_env_config = os.path.join(settings.environment_root, 'config.py') -settings.update(_env_config) +for config in _env_configs: + settings.update(config) diff --git a/wlauto/core/execution.py b/wlauto/core/execution.py index ff6900a5..a3e0a56f 100644 --- a/wlauto/core/execution.py +++ b/wlauto/core/execution.py @@ -165,10 +165,12 @@ class ExecutionContext(object): 'meta', mandatory=True, description='Agenda for this run.')) - for i in xrange(1, settings.config_count + 1): - self.run_artifacts.append(Artifact('config_{}'.format(i), - os.path.join(self.host_working_directory, - 'config_{}.py'.format(i)), + for i, filepath in enumerate(settings.loaded_files, 1): + name = 'config_{}'.format(i) + path = os.path.join(self.host_working_directory, + name + os.path.splitext(filepath)[1]) + self.run_artifacts.append(Artifact(name, + path, kind='meta', mandatory=True, description='Config file used for the run.'))