mirror of
https://github.com/ARM-software/workload-automation.git
synced 2025-02-20 20:09:11 +00:00
Generating jobs.
This commit is contained in:
Sergei Trofimov
parent
9cfa4e7f51
commit
390e9ca78a
@ -25,6 +25,7 @@ from wlauto.core.configuration import RunConfiguration
|
||||
from wlauto.core.configuration.parsers import AgendaParser, ConfigParser
|
||||
from wlauto.core.execution import Executor
|
||||
from wlauto.core.output import init_wa_output
|
||||
from wlauto.core.version import get_wa_version
|
||||
from wlauto.exceptions import NotFoundError, ConfigError
|
||||
from wlauto.utils.log import add_log_file
|
||||
from wlauto.utils.types import toggle_set
|
||||
@ -74,23 +75,26 @@ class RunCommand(Command):
|
||||
This option may be specified multiple times.
|
||||
""")
|
||||
|
||||
def execute(self, state, args):
|
||||
output = self.set_up_output_directory(state, args)
|
||||
def execute(self, config, args):
|
||||
output = self.set_up_output_directory(config, args)
|
||||
add_log_file(output.logfile)
|
||||
|
||||
self.logger.debug('Version: {}'.format(get_wa_version()))
|
||||
self.logger.debug('Command Line: {}'.format(' '.join(sys.argv)))
|
||||
|
||||
disabled_instruments = toggle_set(["~{}".format(i)
|
||||
for i in args.instruments_to_disable])
|
||||
state.jobs_config.disable_instruments(disabled_instruments)
|
||||
state.jobs_config.only_run_ids(args.only_run_ids)
|
||||
config.jobs_config.disable_instruments(disabled_instruments)
|
||||
config.jobs_config.only_run_ids(args.only_run_ids)
|
||||
|
||||
parser = AgendaParser()
|
||||
if os.path.isfile(args.agenda):
|
||||
parser.load_from_path(state, args.agenda)
|
||||
parser.load_from_path(config, args.agenda)
|
||||
else:
|
||||
try:
|
||||
pluginloader.get_plugin_class(args.agenda, kind='workload')
|
||||
agenda = {'workloads': [{'name': args.agenda}]}
|
||||
parser.load(state, agenda, 'CMDLINE_ARGS')
|
||||
parser.load(config, agenda, 'CMDLINE_ARGS')
|
||||
except NotFoundError:
|
||||
msg = 'Agenda file "{}" does not exist, and there no workload '\
|
||||
'with that name.\nYou can get a list of available '\
|
||||
@ -98,16 +102,16 @@ class RunCommand(Command):
|
||||
raise ConfigError(msg.format(args.agenda))
|
||||
|
||||
executor = Executor()
|
||||
executor.execute(state, output)
|
||||
executor.execute(config, output)
|
||||
|
||||
def set_up_output_directory(self, state, args):
|
||||
def set_up_output_directory(self, config, args):
|
||||
if args.output_directory:
|
||||
output_directory = args.output_directory
|
||||
else:
|
||||
output_directory = settings.default_output_directory
|
||||
self.logger.debug('Using output directory: {}'.format(output_directory))
|
||||
try:
|
||||
return init_wa_output(output_directory, state, args.force)
|
||||
return init_wa_output(output_directory, config, args.force)
|
||||
except RuntimeError as e:
|
||||
if 'path exists' in str(e):
|
||||
msg = 'Output directory "{}" exists.\nPlease specify another '\
|
||||
|
||||
@ -70,7 +70,7 @@ class Command(Plugin):
|
||||
"""
|
||||
Execute this command.
|
||||
|
||||
:state: An initialized ``WAState`` that contains the current state of
|
||||
:state: An initialized ``ConfigManager`` that contains the current state of
|
||||
WA exeuction up to that point (processed configuraition, loaded
|
||||
plugins, etc).
|
||||
:args: An ``argparse.Namespace`` containing command line arguments (as returned by
|
||||
|
||||
@ -17,7 +17,7 @@ import re
|
||||
from copy import copy
|
||||
from collections import OrderedDict, defaultdict
|
||||
|
||||
from wlauto.exceptions import ConfigError
|
||||
from wlauto.exceptions import ConfigError, NotFoundError
|
||||
from wlauto.utils.misc import (get_article, merge_config_values)
|
||||
from wlauto.utils.types import (identifier, integer, boolean,
|
||||
list_of_strings, toggle_set,
|
||||
@ -489,6 +489,15 @@ class CpuFreqParameters(object):
|
||||
#####################
|
||||
|
||||
|
||||
def _to_pod(cfg_point, value):
|
||||
if is_pod(value):
|
||||
return value
|
||||
if hasattr(cfg_point.kind, 'to_pod'):
|
||||
return value.to_pod()
|
||||
msg = '{} value "{}" is not serializable'
|
||||
raise ValueError(msg.format(cfg_point.name, value))
|
||||
|
||||
|
||||
class Configuration(object):
|
||||
|
||||
config_points = []
|
||||
@ -498,16 +507,17 @@ class Configuration(object):
|
||||
configuration = {cp.name: cp for cp in config_points}
|
||||
|
||||
@classmethod
|
||||
# pylint: disable=unused-argument
|
||||
def from_pod(cls, pod, plugin_cache):
|
||||
def from_pod(cls, pod):
|
||||
instance = cls()
|
||||
for name, cfg_point in cls.configuration.iteritems():
|
||||
for cfg_point in cls.config_points:
|
||||
if name in pod:
|
||||
cfg_point.set_value(instance, pod.pop(name))
|
||||
value = pod.pop(name)
|
||||
if hasattr(cfg_point.kind, 'from_pod'):
|
||||
value = cfg_point.kind.from_pod(value)
|
||||
cfg_point.set_value(instance, value)
|
||||
if pod:
|
||||
msg = 'Invalid entry(ies) for "{}": "{}"'
|
||||
raise ConfigError(msg.format(cls.name, '", "'.join(pod.keys())))
|
||||
instance.validate()
|
||||
raise ValueError(msg.format(cls.name, '", "'.join(pod.keys())))
|
||||
return instance
|
||||
|
||||
def __init__(self):
|
||||
@ -531,17 +541,17 @@ class Configuration(object):
|
||||
|
||||
def to_pod(self):
|
||||
pod = {}
|
||||
for cfg_point_name in self.configuration.iterkeys():
|
||||
value = getattr(self, cfg_point_name, None)
|
||||
for cfg_point in self.configuration.itervalues():
|
||||
value = getattr(self, cfg_point.name, None)
|
||||
if value is not None:
|
||||
pod[cfg_point_name] = value
|
||||
pod[cfg_point.name] = _to_pod(cfg_point, value)
|
||||
return pod
|
||||
|
||||
|
||||
# This configuration for the core WA framework
|
||||
class WAConfiguration(Configuration):
|
||||
class MetaConfiguration(Configuration):
|
||||
|
||||
name = "WA Configuration"
|
||||
name = "Meta Configuration"
|
||||
|
||||
plugin_packages = [
|
||||
'wlauto.commands',
|
||||
@ -617,7 +627,7 @@ class WAConfiguration(Configuration):
|
||||
return os.path.join(self.user_directory, 'config.yaml')
|
||||
|
||||
def __init__(self, environ):
|
||||
super(WAConfiguration, self).__init__()
|
||||
super(MetaConfiguration, self).__init__()
|
||||
user_directory = environ.pop('WA_USER_DIRECTORY', '')
|
||||
if user_directory:
|
||||
self.set('user_directory', user_directory)
|
||||
@ -748,39 +758,30 @@ class RunConfiguration(Configuration):
|
||||
selected device.
|
||||
"""
|
||||
# pylint: disable=no-member
|
||||
self.device_config = plugin_cache.get_plugin_config(self.device_config,
|
||||
if self.device is None:
|
||||
msg = 'Attemting to merge device config with unspecified device'
|
||||
raise RuntimeError(msg)
|
||||
self.device_config = plugin_cache.get_plugin_config(self.device,
|
||||
generic_name="device_config")
|
||||
|
||||
def to_pod(self):
|
||||
pod = super(RunConfiguration, self).to_pod()
|
||||
pod['device_config'] = self.device_config
|
||||
pod['device_config'] = dict(self.device_config or {})
|
||||
return pod
|
||||
|
||||
# pylint: disable=no-member
|
||||
@classmethod
|
||||
def from_pod(cls, pod, plugin_cache):
|
||||
try:
|
||||
device_config = obj_dict(values=pod.pop("device_config"), not_in_dict=['name'])
|
||||
except KeyError as e:
|
||||
msg = 'No value specified for mandatory parameter "{}".'
|
||||
raise ConfigError(msg.format(e.args[0]))
|
||||
def from_pod(cls, pod):
|
||||
meta_pod = {}
|
||||
for cfg_point in cls.meta_data:
|
||||
meta_pod[cfg_point.name] = pod.pop(cfg_point.name, None)
|
||||
|
||||
instance = super(RunConfiguration, cls).from_pod(pod, plugin_cache)
|
||||
instance = super(RunConfiguration, cls).from_pod(pod)
|
||||
for cfg_point in cls.meta_data:
|
||||
cfg_point.set_value(instance, meta_pod[cfg_point.name])
|
||||
|
||||
device_config.name = "device_config"
|
||||
cfg_points = plugin_cache.get_plugin_parameters(instance.device)
|
||||
for entry_name in device_config.iterkeys():
|
||||
if entry_name not in cfg_points.iterkeys():
|
||||
msg = 'Invalid entry "{}" for device "{}".'
|
||||
raise ConfigError(msg.format(entry_name, instance.device, cls.name))
|
||||
else:
|
||||
cfg_points[entry_name].validate(device_config)
|
||||
|
||||
instance.device_config = device_config
|
||||
return instance
|
||||
|
||||
|
||||
# This is the configuration for WA jobs
|
||||
class JobSpec(Configuration):
|
||||
|
||||
name = "Job Spec"
|
||||
@ -795,6 +796,23 @@ class JobSpec(Configuration):
|
||||
description='''
|
||||
The name of the workload to run.
|
||||
'''),
|
||||
ConfigurationPoint('workload_parameters', kind=obj_dict,
|
||||
aliases=["params", "workload_params"],
|
||||
description='''
|
||||
Parameter to be passed to the workload
|
||||
'''),
|
||||
ConfigurationPoint('runtime_parameters', kind=obj_dict,
|
||||
aliases=["runtime_params"],
|
||||
description='''
|
||||
Runtime parameters to be set prior to running
|
||||
the workload.
|
||||
'''),
|
||||
ConfigurationPoint('boot_parameters', kind=obj_dict,
|
||||
aliases=["boot_params"],
|
||||
description='''
|
||||
Parameters to be used when rebooting the target
|
||||
prior to running the workload.
|
||||
'''),
|
||||
ConfigurationPoint('label', kind=str,
|
||||
description='''
|
||||
Similar to IDs but do not have the uniqueness restriction.
|
||||
@ -823,14 +841,23 @@ class JobSpec(Configuration):
|
||||
]
|
||||
configuration = {cp.name: cp for cp in config_points}
|
||||
|
||||
@classmethod
|
||||
def from_pod(cls, pod):
|
||||
job_id = pod.pop('id')
|
||||
instance = super(JobSpec, cls).from_pod(pod)
|
||||
instance['id'] = job_id
|
||||
return instance
|
||||
|
||||
def __init__(self):
|
||||
super(JobSpec, self).__init__()
|
||||
self.to_merge = defaultdict(OrderedDict)
|
||||
self._sources = []
|
||||
self.id = None
|
||||
self.workload_parameters = None
|
||||
self.runtime_parameters = None
|
||||
self.boot_parameters = None
|
||||
|
||||
def to_pod(self):
|
||||
pod = super(JobSpec, self).to_pod()
|
||||
pod['id'] = self.id
|
||||
return pod
|
||||
|
||||
def update_config(self, source, check_mandatory=True):
|
||||
self._sources.append(source)
|
||||
@ -848,7 +875,6 @@ class JobSpec(Configuration):
|
||||
msg = 'Error in {}:\n\t{}'
|
||||
raise ConfigError(msg.format(source.name, e.message))
|
||||
|
||||
|
||||
def merge_workload_parameters(self, plugin_cache):
|
||||
# merge global generic and specific config
|
||||
workload_params = plugin_cache.get_plugin_config(self.workload_name,
|
||||
@ -876,7 +902,10 @@ class JobSpec(Configuration):
|
||||
|
||||
# Order global runtime parameters
|
||||
runtime_parameters = OrderedDict()
|
||||
global_runtime_params = plugin_cache.get_plugin_config("runtime_parameters")
|
||||
try:
|
||||
global_runtime_params = plugin_cache.get_plugin_config("runtime_parameters")
|
||||
except NotFoundError:
|
||||
global_runtime_params = {}
|
||||
for source in plugin_cache.sources:
|
||||
runtime_parameters[source] = global_runtime_params[source]
|
||||
|
||||
@ -890,27 +919,6 @@ class JobSpec(Configuration):
|
||||
def finalize(self):
|
||||
self.id = "-".join([source.config['id'] for source in self._sources[1:]]) # ignore first id, "global"
|
||||
|
||||
def to_pod(self):
|
||||
pod = super(JobSpec, self).to_pod()
|
||||
pod['workload_parameters'] = self.workload_parameters
|
||||
pod['runtime_parameters'] = self.runtime_parameters
|
||||
pod['boot_parameters'] = self.boot_parameters
|
||||
return pod
|
||||
|
||||
@classmethod
|
||||
def from_pod(cls, pod, plugin_cache):
|
||||
try:
|
||||
workload_parameters = pod['workload_parameters']
|
||||
runtime_parameters = pod['runtime_parameters']
|
||||
boot_parameters = pod['boot_parameters']
|
||||
except KeyError as e:
|
||||
msg = 'No value specified for mandatory parameter "{}}".'
|
||||
raise ConfigError(msg.format(e.args[0]))
|
||||
|
||||
instance = super(JobSpec, cls).from_pod(pod, plugin_cache)
|
||||
|
||||
# TODO: validate parameters and construct the rest of the instance
|
||||
|
||||
|
||||
# This is used to construct the list of Jobs WA will run
|
||||
class JobGenerator(object):
|
||||
@ -970,6 +978,7 @@ class JobGenerator(object):
|
||||
self.ids_to_run = ids
|
||||
|
||||
def generate_job_specs(self, target_manager):
|
||||
specs = []
|
||||
for leaf in self.root_node.leaves():
|
||||
workload_entries = leaf.workload_entries
|
||||
sections = [leaf]
|
||||
@ -978,18 +987,23 @@ class JobGenerator(object):
|
||||
sections.insert(0, ancestor)
|
||||
|
||||
for workload_entry in workload_entries:
|
||||
job_spec = create_job_spec(workload_entry, sections, target_manager)
|
||||
for job_id in self.ids_to_run:
|
||||
if job_id in job_spec.id:
|
||||
break
|
||||
else:
|
||||
continue
|
||||
job_spec = create_job_spec(workload_entry, sections,
|
||||
target_manager, self.plugin_cache,
|
||||
self.disabled_instruments)
|
||||
if self.ids_to_run:
|
||||
for job_id in self.ids_to_run:
|
||||
if job_id in job_spec.id:
|
||||
break
|
||||
else:
|
||||
continue
|
||||
self.update_enabled_instruments(job_spec.instrumentation.values())
|
||||
yield job_spec
|
||||
specs.append(job_spec)
|
||||
return specs
|
||||
|
||||
|
||||
|
||||
def create_job_spec(workload_entry, sections, target_manager):
|
||||
def create_job_spec(workload_entry, sections, target_manager, plugin_cache,
|
||||
disabled_instruments):
|
||||
job_spec = JobSpec()
|
||||
|
||||
# PHASE 2.1: Merge general job spec configuration
|
||||
@ -998,18 +1012,17 @@ def create_job_spec(workload_entry, sections, target_manager):
|
||||
job_spec.update_config(workload_entry, check_mandatory=False)
|
||||
|
||||
# PHASE 2.2: Merge global, section and workload entry "workload_parameters"
|
||||
job_spec.merge_workload_parameters(self.plugin_cache)
|
||||
target_manager.static_runtime_parameter_validation(job_spec.runtime_parameters)
|
||||
job_spec.merge_workload_parameters(plugin_cache)
|
||||
|
||||
# TODO: PHASE 2.3: Validate device runtime/boot paramerers
|
||||
job_spec.merge_runtime_parameters(self.plugin_cache, target_manager)
|
||||
job_spec.merge_runtime_parameters(plugin_cache, target_manager)
|
||||
target_manager.validate_runtime_parameters(job_spec.runtime_parameters)
|
||||
|
||||
# PHASE 2.4: Disable globally disabled instrumentation
|
||||
job_spec.set("instrumentation", self.disabled_instruments)
|
||||
job_spec.set("instrumentation", disabled_instruments)
|
||||
job_spec.finalize()
|
||||
|
||||
return job_spec
|
||||
|
||||
|
||||
settings = WAConfiguration(os.environ)
|
||||
settings = MetaConfiguration(os.environ)
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
from wlauto.core.configuration.configuration import WAConfiguration, RunConfiguration
|
||||
from wlauto.core.configuration.configuration import MetaConfiguration, RunConfiguration
|
||||
from wlauto.core.configuration.plugin_cache import PluginCache
|
||||
from wlauto.utils.serializer import yaml
|
||||
from wlauto.utils.doc import strip_inlined_text
|
||||
@ -33,7 +33,7 @@ def _format_instruments(output):
|
||||
|
||||
def generate_default_config(path):
|
||||
with open(path, 'w') as output:
|
||||
for param in WAConfiguration.config_points + RunConfiguration.config_points:
|
||||
for param in MetaConfiguration.config_points + RunConfiguration.config_points:
|
||||
entry = {param.name: param.default}
|
||||
comment = _format_yaml_comment(param)
|
||||
output.writelines(comment)
|
||||
|
||||
@ -1,10 +1,29 @@
|
||||
from wlauto.core.configuration.configuration import (RunConfiguration,
|
||||
from wlauto.core.configuration.configuration import (MetaConfiguration,
|
||||
RunConfiguration,
|
||||
JobGenerator, settings)
|
||||
from wlauto.core.configuration.parsers import ConfigParser
|
||||
from wlauto.core.configuration.plugin_cache import PluginCache
|
||||
|
||||
|
||||
class WAState(object):
|
||||
class CombinedConfig(object):
|
||||
|
||||
@staticmethod
|
||||
def from_pod(pod):
|
||||
instance = CombinedConfig()
|
||||
instance.settings = MetaConfiguration.from_pod(pod.get('setttings', {}))
|
||||
instance.run_config = RunConfiguration.from_pod(pod.get('run_config', {}))
|
||||
return instance
|
||||
|
||||
def __init__(self, settings=None, run_config=None):
|
||||
self.settings = settings
|
||||
self.run_config = run_config
|
||||
|
||||
def to_pod(self):
|
||||
return {'settings': self.settings.to_pod(),
|
||||
'run_config': self.run_config.to_pod()}
|
||||
|
||||
|
||||
class ConfigManager(object):
|
||||
"""
|
||||
Represents run-time state of WA. Mostly used as a container for loaded
|
||||
configuration and discovered plugins.
|
||||
@ -20,6 +39,8 @@ class WAState(object):
|
||||
self.jobs_config = JobGenerator(self.plugin_cache)
|
||||
self.loaded_config_sources = []
|
||||
self._config_parser = ConfigParser()
|
||||
self._job_specs = []
|
||||
self.jobs = []
|
||||
|
||||
def load_config_file(self, filepath):
|
||||
self._config_parser.load_from_path(self, filepath)
|
||||
@ -29,4 +50,7 @@ class WAState(object):
|
||||
self._config_parser.load(self, values, source)
|
||||
self.loaded_config_sources.append(source)
|
||||
|
||||
def finalize(self):
|
||||
self.run_config.merge_device_config(self.plugin_cache)
|
||||
return CombinedConfig(self.settings, self.run_config)
|
||||
|
||||
@ -26,7 +26,8 @@ class TargetInfo(object):
|
||||
instance.os_version = pod['os_version']
|
||||
instance.abi = pod['abi']
|
||||
instance.is_rooted = pod['is_rooted']
|
||||
instance.kernel_version = KernelVersion(pod['kernel_version'])
|
||||
instance.kernel_version = KernelVersion(pod['kernel_release'],
|
||||
pod['kernel_version'])
|
||||
instance.kernel_config = KernelConfig(pod['kernel_config'])
|
||||
|
||||
if pod["target"] == "AndroidTarget":
|
||||
@ -69,15 +70,16 @@ class TargetInfo(object):
|
||||
|
||||
def to_pod(self):
|
||||
pod = {}
|
||||
pod['target'] = self.target.__class__.__name__
|
||||
pod['target'] = self.target
|
||||
pod['abi'] = self.abi
|
||||
pod['cpuinfo'] = self.cpuinfo.text
|
||||
pod['cpuinfo'] = self.cpuinfo.sections
|
||||
pod['os'] = self.os
|
||||
pod['os_version'] = self.os_version
|
||||
pod['abi'] = self.abi
|
||||
pod['is_rooted'] = self.is_rooted
|
||||
pod['kernel_release'] = self.kernel_version.release
|
||||
pod['kernel_version'] = self.kernel_version.version
|
||||
pod['kernel_config'] = self.kernel_config.text
|
||||
pod['kernel_config'] = dict(self.kernel_config.iteritems())
|
||||
|
||||
if self.target == "AndroidTarget":
|
||||
pod['screen_resolution'] = self.screen_resolution
|
||||
|
||||
@ -24,8 +24,8 @@ import warnings
|
||||
from wlauto.core import pluginloader
|
||||
from wlauto.core.command import init_argument_parser
|
||||
from wlauto.core.configuration import settings
|
||||
from wlauto.core.configuration.manager import ConfigManager
|
||||
from wlauto.core.host import init_user_directory
|
||||
from wlauto.core.state import WAState
|
||||
from wlauto.exceptions import WAError, DevlibError, ConfigError
|
||||
from wlauto.utils.doc import format_body
|
||||
from wlauto.utils.log import init_logging
|
||||
@ -46,7 +46,7 @@ def load_commands(subparsers):
|
||||
|
||||
|
||||
def main():
|
||||
state = WAState()
|
||||
config = ConfigManager()
|
||||
|
||||
if not os.path.exists(settings.user_directory):
|
||||
init_user_directory()
|
||||
@ -68,16 +68,16 @@ def main():
|
||||
|
||||
settings.set("verbosity", args.verbose)
|
||||
|
||||
state.load_config_file(settings.user_config_file)
|
||||
config.load_config_file(settings.user_config_file)
|
||||
for config_file in args.config:
|
||||
if not os.path.exists(config_file):
|
||||
raise ConfigError("Config file {} not found".format(config_file))
|
||||
state.load_config_file(config_file)
|
||||
config.load_config_file(config_file)
|
||||
|
||||
init_logging(settings.verbosity)
|
||||
|
||||
command = commands[args.command]
|
||||
sys.exit(command.execute(state, args))
|
||||
sys.exit(command.execute(config, args))
|
||||
|
||||
except KeyboardInterrupt:
|
||||
logging.info('Got CTRL-C. Aborting.')
|
||||
|
||||
@ -51,6 +51,7 @@ import wlauto.core.signal as signal
|
||||
from wlauto.core import instrumentation
|
||||
from wlauto.core import pluginloader
|
||||
from wlauto.core.configuration import settings
|
||||
from wlauto.core.device_manager import TargetInfo
|
||||
from wlauto.core.plugin import Artifact
|
||||
from wlauto.core.resolver import ResourceResolver
|
||||
from wlauto.core.result import ResultManager, IterationResult, RunResult
|
||||
@ -213,6 +214,30 @@ def _check_artifact_path(path, rootpath):
|
||||
return full_path
|
||||
|
||||
|
||||
|
||||
class FakeTargetManager(object):
|
||||
|
||||
def __init__(self, name, config):
|
||||
self.device_name = name
|
||||
self.device_config = config
|
||||
|
||||
from devlib import LocalLinuxTarget
|
||||
self.target = LocalLinuxTarget({'unrooted': True})
|
||||
|
||||
def get_target_info(self):
|
||||
return TargetInfo(self.target)
|
||||
|
||||
def validate_runtime_parameters(self, params):
|
||||
pass
|
||||
|
||||
def merge_runtime_parameters(self, params):
|
||||
pass
|
||||
|
||||
|
||||
def init_target_manager(config):
|
||||
return FakeTargetManager(config.device, config.device_config)
|
||||
|
||||
|
||||
class Executor(object):
|
||||
"""
|
||||
The ``Executor``'s job is to set up the execution context and pass to a
|
||||
@ -237,14 +262,14 @@ class Executor(object):
|
||||
self.device = None
|
||||
self.context = None
|
||||
|
||||
def execute(self, state, output):
|
||||
def execute(self, config_manager, output):
|
||||
"""
|
||||
Execute the run specified by an agenda. Optionally, selectors may be
|
||||
used to only selecute a subset of the specified agenda.
|
||||
|
||||
Params::
|
||||
|
||||
:state: a ``WAState`` containing processed configuraiton
|
||||
:state: a ``ConfigManager`` containing processed configuraiton
|
||||
:output: an initialized ``RunOutput`` that will be used to
|
||||
store the results.
|
||||
|
||||
@ -253,8 +278,17 @@ class Executor(object):
|
||||
signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED)
|
||||
|
||||
self.logger.info('Initializing run')
|
||||
self.logger.debug('Finalizing run configuration.')
|
||||
config = config_manager.finalize()
|
||||
output.write_config(config)
|
||||
|
||||
self.logger.debug('Loading run configuration.')
|
||||
self.logger.info('Connecting to target')
|
||||
target_manager = init_target_manager(config.run_config)
|
||||
output.write_target_info(target_manager.get_target_info())
|
||||
|
||||
self.logger.info('Generationg jobs')
|
||||
job_specs = config_manager.jobs_config.generate_job_specs(target_manager)
|
||||
output.write_job_specs(job_specs)
|
||||
|
||||
def old_exec(self, agenda, selectors={}):
|
||||
self.config.set_agenda(agenda, selectors)
|
||||
|
||||
@ -6,6 +6,9 @@ import sys
|
||||
import uuid
|
||||
from copy import copy
|
||||
|
||||
from wlauto.core.configuration.configuration import JobSpec
|
||||
from wlauto.core.configuration.manager import ConfigManager
|
||||
from wlauto.core.device_manager import TargetInfo
|
||||
from wlauto.utils.misc import touch
|
||||
from wlauto.utils.serializer import write_pod, read_pod
|
||||
|
||||
@ -78,6 +81,18 @@ class RunOutput(object):
|
||||
def statefile(self):
|
||||
return os.path.join(self.basepath, '.run_state.json')
|
||||
|
||||
@property
|
||||
def configfile(self):
|
||||
return os.path.join(self.metadir, 'config.json')
|
||||
|
||||
@property
|
||||
def targetfile(self):
|
||||
return os.path.join(self.metadir, 'target_info.json')
|
||||
|
||||
@property
|
||||
def jobsfile(self):
|
||||
return os.path.join(self.metadir, 'jobs.json')
|
||||
|
||||
def __init__(self, path):
|
||||
self.basepath = path
|
||||
self.info = None
|
||||
@ -98,6 +113,32 @@ class RunOutput(object):
|
||||
def write_state(self):
|
||||
write_pod(self.state.to_pod(), self.statefile)
|
||||
|
||||
def write_config(self, config):
|
||||
write_pod(config.to_pod(), self.configfile)
|
||||
|
||||
def read_config(self):
|
||||
if not os.path.isfile(self.configfile):
|
||||
return None
|
||||
return ConfigManager.from_pod(read_pod(self.configfile))
|
||||
|
||||
def write_target_info(self, ti):
|
||||
write_pod(ti.to_pod(), self.targetfile)
|
||||
|
||||
def read_config(self):
|
||||
if not os.path.isfile(self.targetfile):
|
||||
return None
|
||||
return TargetInfo.from_pod(read_pod(self.targetfile))
|
||||
|
||||
def write_job_specs(self, job_specs):
|
||||
job_specs[0].to_pod()
|
||||
js_pod = {'jobs': [js.to_pod() for js in job_specs]}
|
||||
write_pod(js_pod, self.jobsfile)
|
||||
|
||||
def read_job_specs(self):
|
||||
if not os.path.isfile(self.jobsfile):
|
||||
return None
|
||||
pod = read_pod(self.jobsfile)
|
||||
return [JobSpec.from_pod(jp) for jp in pod['jobs']]
|
||||
|
||||
|
||||
def init_wa_output(path, wa_state, force=False):
|
||||
|
||||
@ -9,7 +9,7 @@ from wlauto.exceptions import ConfigError
|
||||
from wlauto.core.configuration.parsers import * # pylint: disable=wildcard-import
|
||||
from wlauto.core.configuration.parsers import _load_file, _collect_valid_id, _resolve_params_alias
|
||||
from wlauto.core.configuration import RunConfiguration, JobGenerator, PluginCache, ConfigurationPoint
|
||||
from wlauto.core.configuration.configuration import WAConfiguration
|
||||
from wlauto.core.configuration.configuration import MetaConfiguration
|
||||
from wlauto.utils.types import toggle_set, reset_counter
|
||||
|
||||
|
||||
@ -129,8 +129,8 @@ class TestFunctions(TestCase):
|
||||
class TestConfigParser(TestCase):
|
||||
|
||||
def test_error_cases(self):
|
||||
wa_config = Mock(spec=WAConfiguration)
|
||||
wa_config.configuration = WAConfiguration.configuration
|
||||
wa_config = Mock(spec=MetaConfiguration)
|
||||
wa_config.configuration = MetaConfiguration.configuration
|
||||
run_config = Mock(spec=RunConfiguration)
|
||||
run_config.configuration = RunConfiguration.configuration
|
||||
config_parser = ConfigParser(wa_config,
|
||||
@ -155,8 +155,8 @@ class TestConfigParser(TestCase):
|
||||
"Unit test")
|
||||
|
||||
def test_config_points(self):
|
||||
wa_config = Mock(spec=WAConfiguration)
|
||||
wa_config.configuration = WAConfiguration.configuration
|
||||
wa_config = Mock(spec=MetaConfiguration)
|
||||
wa_config.configuration = MetaConfiguration.configuration
|
||||
|
||||
run_config = Mock(spec=RunConfiguration)
|
||||
run_config.configuration = RunConfiguration.configuration
|
||||
@ -211,8 +211,8 @@ class TestAgendaParser(TestCase):
|
||||
|
||||
# Tests Phase 1 & 2
|
||||
def test_valid_structures(self):
|
||||
wa_config = Mock(spec=WAConfiguration)
|
||||
wa_config.configuration = WAConfiguration.configuration
|
||||
wa_config = Mock(spec=MetaConfiguration)
|
||||
wa_config.configuration = MetaConfiguration.configuration
|
||||
run_config = Mock(spec=RunConfiguration)
|
||||
run_config.configuration = RunConfiguration.configuration
|
||||
jobs_config = Mock(spec=JobGenerator)
|
||||
@ -241,8 +241,8 @@ class TestAgendaParser(TestCase):
|
||||
|
||||
# Test Phase 3
|
||||
def test_id_collection(self):
|
||||
wa_config = Mock(spec=WAConfiguration)
|
||||
wa_config.configuration = WAConfiguration.configuration
|
||||
wa_config = Mock(spec=MetaConfiguration)
|
||||
wa_config.configuration = MetaConfiguration.configuration
|
||||
run_config = Mock(spec=RunConfiguration)
|
||||
run_config.configuration = RunConfiguration.configuration
|
||||
jobs_config = Mock(spec=JobGenerator)
|
||||
@ -267,8 +267,8 @@ class TestAgendaParser(TestCase):
|
||||
|
||||
# Test Phase 4
|
||||
def test_id_assignment(self):
|
||||
wa_config = Mock(spec=WAConfiguration)
|
||||
wa_config.configuration = WAConfiguration.configuration
|
||||
wa_config = Mock(spec=MetaConfiguration)
|
||||
wa_config.configuration = MetaConfiguration.configuration
|
||||
run_config = Mock(spec=RunConfiguration)
|
||||
run_config.configuration = RunConfiguration.configuration
|
||||
jobs_config = Mock(spec=JobGenerator)
|
||||
@ -362,7 +362,7 @@ class TestAgendaParser(TestCase):
|
||||
|
||||
|
||||
class TestCommandLineArgsParser(TestCase):
|
||||
wa_config = Mock(spec=WAConfiguration)
|
||||
wa_config = Mock(spec=MetaConfiguration)
|
||||
run_config = Mock(spec=RunConfiguration)
|
||||
jobs_config = Mock(spec=JobGenerator)
|
||||
|
||||
|
||||
@ -322,7 +322,8 @@ class prioritylist(object):
|
||||
raise ValueError('Invalid index {}'.format(index))
|
||||
current_global_offset = 0
|
||||
priority_counts = {priority: count for (priority, count) in
|
||||
zip(self.priorities, [len(self.elements[p]) for p in self.priorities])}
|
||||
zip(self.priorities, [len(self.elements[p])
|
||||
for p in self.priorities])}
|
||||
for priority in self.priorities:
|
||||
if not index_range:
|
||||
break
|
||||
@ -351,13 +352,9 @@ class toggle_set(set):
|
||||
and ``cherries`` but disables ``oranges``.
|
||||
"""
|
||||
|
||||
def merge_with(self, other):
|
||||
new_self = copy(self)
|
||||
return toggle_set.merge(other, new_self)
|
||||
|
||||
def merge_into(self, other):
|
||||
other = copy(other)
|
||||
return toggle_set.merge(self, other)
|
||||
@staticmethod
|
||||
def from_pod(pod):
|
||||
return toggle_set(pod)
|
||||
|
||||
@staticmethod
|
||||
def merge(source, dest):
|
||||
@ -372,6 +369,14 @@ class toggle_set(set):
|
||||
dest.add(item)
|
||||
return dest
|
||||
|
||||
def merge_with(self, other):
|
||||
new_self = copy(self)
|
||||
return toggle_set.merge(other, new_self)
|
||||
|
||||
def merge_into(self, other):
|
||||
other = copy(other)
|
||||
return toggle_set.merge(self, other)
|
||||
|
||||
def values(self):
|
||||
"""
|
||||
returns a list of enabled items.
|
||||
@ -396,6 +401,9 @@ class toggle_set(set):
|
||||
conflicts.append(item)
|
||||
return conflicts
|
||||
|
||||
def to_pod(self):
|
||||
return list(self.values())
|
||||
|
||||
class ID(str):
|
||||
|
||||
def merge_with(self, other):
|
||||
@ -411,11 +419,19 @@ class obj_dict(MutableMapping):
|
||||
as an attribute.
|
||||
|
||||
:param not_in_dict: A list of keys that can only be accessed as attributes
|
||||
|
||||
"""
|
||||
|
||||
def __init__(self, not_in_dict=None, values={}):
|
||||
@staticmethod
|
||||
def from_pod(pod):
|
||||
return obj_dict(pod)
|
||||
|
||||
def __init__(self, values=None, not_in_dict=None):
|
||||
self.__dict__['dict'] = dict(values or {})
|
||||
self.__dict__['not_in_dict'] = not_in_dict if not_in_dict is not None else []
|
||||
self.__dict__['dict'] = dict(values)
|
||||
|
||||
def to_pod(self):
|
||||
return self.__dict__['dict']
|
||||
|
||||
def __getitem__(self, key):
|
||||
if key in self.not_in_dict:
|
||||
@ -457,13 +473,3 @@ class obj_dict(MutableMapping):
|
||||
return self.__dict__['dict'][name]
|
||||
else:
|
||||
raise AttributeError("No such attribute: " + name)
|
||||
|
||||
def to_pod(self):
|
||||
return self.__dict__.copy()
|
||||
|
||||
@staticmethod
|
||||
def from_pod(pod):
|
||||
instance = ObjDict()
|
||||
for k, v in pod.iteritems():
|
||||
instance[k] = v
|
||||
return instance
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user