Generating jobs.

wlauto/core/command.py

@@ -70,7 +70,7 @@ class Command(Plugin):
         """
         Execute this command.
 
-        :state: An initialized ``WAState`` that contains the current state of
+        :state: An initialized ``ConfigManager`` that contains the current state of
                 WA exeuction up to that point (processed configuraition, loaded
                 plugins, etc).
         :args: An ``argparse.Namespace`` containing command line arguments (as returned by

wlauto/core/configuration/configuration.py

@@ -17,7 +17,7 @@ import re
 from copy import copy
 from collections import OrderedDict, defaultdict
 
-from wlauto.exceptions import ConfigError
+from wlauto.exceptions import ConfigError, NotFoundError
 from wlauto.utils.misc import (get_article, merge_config_values)
 from wlauto.utils.types import (identifier, integer, boolean,
                                 list_of_strings, toggle_set,
@@ -489,6 +489,15 @@ class CpuFreqParameters(object):
 #####################
 
 
+def _to_pod(cfg_point, value):
+    if is_pod(value):
+        return value
+    if hasattr(cfg_point.kind, 'to_pod'):
+        return value.to_pod()
+    msg = '{} value "{}" is not serializable'
+    raise ValueError(msg.format(cfg_point.name, value))
+
+
 class Configuration(object):
 
     config_points = []
@@ -498,16 +507,17 @@ class Configuration(object):
     configuration = {cp.name: cp for cp in config_points}
 
     @classmethod
-    # pylint: disable=unused-argument
-    def from_pod(cls, pod, plugin_cache):
+    def from_pod(cls, pod):
         instance = cls()
-        for name, cfg_point in cls.configuration.iteritems():
+        for cfg_point in cls.config_points:
             if name in pod:
-                cfg_point.set_value(instance, pod.pop(name))
+                value = pod.pop(name)
+                if hasattr(cfg_point.kind, 'from_pod'):
+                    value = cfg_point.kind.from_pod(value)
+                cfg_point.set_value(instance, value)
         if pod:
             msg = 'Invalid entry(ies) for "{}": "{}"'
-            raise ConfigError(msg.format(cls.name, '", "'.join(pod.keys())))
-        instance.validate()
+            raise ValueError(msg.format(cls.name, '", "'.join(pod.keys())))
         return instance
 
     def __init__(self):
@@ -531,17 +541,17 @@ class Configuration(object):
 
     def to_pod(self):
         pod = {}
-        for cfg_point_name in self.configuration.iterkeys():
-            value = getattr(self, cfg_point_name, None)
+        for cfg_point in self.configuration.itervalues():
+            value = getattr(self, cfg_point.name, None)
             if value is not None:
-                pod[cfg_point_name] = value
+                pod[cfg_point.name] = _to_pod(cfg_point, value)
         return pod
 
 
 # This configuration for the core WA framework
-class WAConfiguration(Configuration):
+class MetaConfiguration(Configuration):
 
-    name = "WA Configuration"
+    name = "Meta Configuration"
 
     plugin_packages = [
         'wlauto.commands',
@@ -617,7 +627,7 @@ class WAConfiguration(Configuration):
         return os.path.join(self.user_directory, 'config.yaml')
 
     def __init__(self, environ):
-        super(WAConfiguration, self).__init__()
+        super(MetaConfiguration, self).__init__()
         user_directory = environ.pop('WA_USER_DIRECTORY', '')
         if user_directory:
             self.set('user_directory', user_directory)
@@ -748,39 +758,30 @@ class RunConfiguration(Configuration):
         selected device.
         """
         # pylint: disable=no-member
-        self.device_config = plugin_cache.get_plugin_config(self.device_config,
+        if self.device is None:
+            msg = 'Attemting to merge device config with unspecified device'
+            raise RuntimeError(msg)
+        self.device_config = plugin_cache.get_plugin_config(self.device,
                                                             generic_name="device_config")
 
     def to_pod(self):
         pod = super(RunConfiguration, self).to_pod()
-        pod['device_config'] = self.device_config
+        pod['device_config'] = dict(self.device_config or {})
         return pod
 
-    # pylint: disable=no-member
     @classmethod
-    def from_pod(cls, pod, plugin_cache):
-        try:
-            device_config = obj_dict(values=pod.pop("device_config"), not_in_dict=['name'])
-        except KeyError as e:
-            msg = 'No value specified for mandatory parameter "{}".'
-            raise ConfigError(msg.format(e.args[0]))
+    def from_pod(cls, pod):
+        meta_pod = {}
+        for cfg_point in cls.meta_data:
+            meta_pod[cfg_point.name] = pod.pop(cfg_point.name, None)
 
-        instance = super(RunConfiguration, cls).from_pod(pod, plugin_cache)
+        instance = super(RunConfiguration, cls).from_pod(pod)
+        for cfg_point in cls.meta_data:
+            cfg_point.set_value(instance, meta_pod[cfg_point.name])
 
-        device_config.name = "device_config"
-        cfg_points = plugin_cache.get_plugin_parameters(instance.device)
-        for entry_name in device_config.iterkeys():
-            if entry_name not in cfg_points.iterkeys():
-                msg = 'Invalid entry "{}" for device "{}".'
-                raise ConfigError(msg.format(entry_name, instance.device, cls.name))
-            else:
-                cfg_points[entry_name].validate(device_config)
-
-        instance.device_config = device_config
         return instance
 
 
-# This is the configuration for WA jobs
 class JobSpec(Configuration):
 
     name = "Job Spec"
@@ -795,6 +796,23 @@ class JobSpec(Configuration):
                            description='''
                            The name of the workload to run.
                            '''),
+        ConfigurationPoint('workload_parameters', kind=obj_dict,
+                           aliases=["params", "workload_params"],
+                           description='''
+                           Parameter to be passed to the workload
+                           '''),
+        ConfigurationPoint('runtime_parameters', kind=obj_dict,
+                           aliases=["runtime_params"],
+                           description='''
+                           Runtime parameters to be set prior to running
+                           the workload.
+                           '''),
+        ConfigurationPoint('boot_parameters', kind=obj_dict,
+                           aliases=["boot_params"],
+                           description='''
+                           Parameters to be used when rebooting the target
+                           prior to running the workload.
+                           '''),
         ConfigurationPoint('label', kind=str,
                            description='''
                            Similar to IDs but do not have the uniqueness restriction.
@@ -823,14 +841,23 @@ class JobSpec(Configuration):
     ]
     configuration = {cp.name: cp for cp in config_points}
 
+    @classmethod
+    def from_pod(cls, pod):
+        job_id = pod.pop('id')
+        instance = super(JobSpec, cls).from_pod(pod)
+        instance['id'] = job_id
+        return instance
+
     def __init__(self):
         super(JobSpec, self).__init__()
         self.to_merge = defaultdict(OrderedDict)
         self._sources = []
         self.id = None
-        self.workload_parameters = None
-        self.runtime_parameters = None
-        self.boot_parameters = None
+
+    def to_pod(self):
+        pod = super(JobSpec, self).to_pod()
+        pod['id'] = self.id
+        return pod
 
     def update_config(self, source, check_mandatory=True):
         self._sources.append(source)
@@ -848,7 +875,6 @@ class JobSpec(Configuration):
                     msg = 'Error in {}:\n\t{}'
                     raise ConfigError(msg.format(source.name, e.message))
 
-
     def merge_workload_parameters(self, plugin_cache):
         # merge global generic and specific config
         workload_params = plugin_cache.get_plugin_config(self.workload_name,
@@ -876,7 +902,10 @@ class JobSpec(Configuration):
 
         # Order global runtime parameters
         runtime_parameters = OrderedDict()
-        global_runtime_params = plugin_cache.get_plugin_config("runtime_parameters")
+        try:
+            global_runtime_params = plugin_cache.get_plugin_config("runtime_parameters")
+        except NotFoundError:
+            global_runtime_params = {}
         for source in plugin_cache.sources:
             runtime_parameters[source] = global_runtime_params[source]
 
@@ -890,27 +919,6 @@ class JobSpec(Configuration):
     def finalize(self):
         self.id = "-".join([source.config['id'] for source in self._sources[1:]])  # ignore first id, "global"
 
-    def to_pod(self):
-        pod = super(JobSpec, self).to_pod()
-        pod['workload_parameters'] = self.workload_parameters
-        pod['runtime_parameters'] = self.runtime_parameters
-        pod['boot_parameters'] = self.boot_parameters
-        return pod
-
-    @classmethod
-    def from_pod(cls, pod, plugin_cache):
-        try:
-            workload_parameters = pod['workload_parameters']
-            runtime_parameters = pod['runtime_parameters']
-            boot_parameters = pod['boot_parameters']
-        except KeyError as e:
-            msg = 'No value specified for mandatory parameter "{}}".'
-            raise ConfigError(msg.format(e.args[0]))
-
-        instance = super(JobSpec, cls).from_pod(pod, plugin_cache)
-
-        # TODO: validate parameters and construct the rest of the instance
-
 
 # This is used to construct the list of Jobs WA will run
 class JobGenerator(object):
@@ -970,6 +978,7 @@ class JobGenerator(object):
         self.ids_to_run = ids
 
     def generate_job_specs(self, target_manager):
+        specs = []
         for leaf in self.root_node.leaves():
             workload_entries = leaf.workload_entries
             sections = [leaf]
@@ -978,18 +987,23 @@ class JobGenerator(object):
                 sections.insert(0, ancestor)
 
             for workload_entry in workload_entries:
-                job_spec = create_job_spec(workload_entry, sections, target_manager)
-                for job_id in self.ids_to_run:
-                    if job_id in job_spec.id:
-                        break
-                else:
-                    continue
+                job_spec = create_job_spec(workload_entry, sections, 
+                                           target_manager, self.plugin_cache,
+                                           self.disabled_instruments)
+                if self.ids_to_run:
+                    for job_id in self.ids_to_run:
+                        if job_id in job_spec.id:
+                            break
+                    else:
+                        continue
                 self.update_enabled_instruments(job_spec.instrumentation.values())
-                yield  job_spec
+                specs.append(job_spec)
+        return specs
 
 
 
-def create_job_spec(workload_entry, sections, target_manager):
+def create_job_spec(workload_entry, sections, target_manager, plugin_cache,
+                    disabled_instruments):
     job_spec = JobSpec()
 
     # PHASE 2.1: Merge general job spec configuration
@@ -998,18 +1012,17 @@ def create_job_spec(workload_entry, sections, target_manager):
     job_spec.update_config(workload_entry, check_mandatory=False)
 
     # PHASE 2.2: Merge global, section and workload entry "workload_parameters"
-    job_spec.merge_workload_parameters(self.plugin_cache)
-    target_manager.static_runtime_parameter_validation(job_spec.runtime_parameters)
+    job_spec.merge_workload_parameters(plugin_cache)
 
     # TODO: PHASE 2.3: Validate device runtime/boot paramerers
-    job_spec.merge_runtime_parameters(self.plugin_cache, target_manager)
+    job_spec.merge_runtime_parameters(plugin_cache, target_manager)
     target_manager.validate_runtime_parameters(job_spec.runtime_parameters)
 
     # PHASE 2.4: Disable globally disabled instrumentation
-    job_spec.set("instrumentation", self.disabled_instruments)
+    job_spec.set("instrumentation", disabled_instruments)
     job_spec.finalize()
 
     return job_spec
 
 
-settings = WAConfiguration(os.environ)
+settings = MetaConfiguration(os.environ)

wlauto/core/configuration/default.py

@@ -1,4 +1,4 @@
-from wlauto.core.configuration.configuration import WAConfiguration, RunConfiguration
+from wlauto.core.configuration.configuration import MetaConfiguration, RunConfiguration
 from wlauto.core.configuration.plugin_cache import PluginCache
 from wlauto.utils.serializer import yaml
 from wlauto.utils.doc import strip_inlined_text
@@ -33,7 +33,7 @@ def _format_instruments(output):
 
 def generate_default_config(path):
     with open(path, 'w') as output:
-        for param in WAConfiguration.config_points + RunConfiguration.config_points:
+        for param in MetaConfiguration.config_points + RunConfiguration.config_points:
             entry = {param.name: param.default}
             comment = _format_yaml_comment(param)
             output.writelines(comment)

wlauto/core/configuration/manager.py

@@ -1,10 +1,29 @@
-from wlauto.core.configuration.configuration import (RunConfiguration,
+from wlauto.core.configuration.configuration import (MetaConfiguration,
+                                                     RunConfiguration,
                                                      JobGenerator, settings)
 from wlauto.core.configuration.parsers import ConfigParser
 from wlauto.core.configuration.plugin_cache import PluginCache
 
 
-class WAState(object):
+class CombinedConfig(object):
+
+    @staticmethod
+    def from_pod(pod):
+        instance = CombinedConfig()
+        instance.settings = MetaConfiguration.from_pod(pod.get('setttings', {}))
+        instance.run_config = RunConfiguration.from_pod(pod.get('run_config', {}))
+        return instance
+
+    def __init__(self, settings=None, run_config=None):
+        self.settings = settings
+        self.run_config = run_config
+
+    def to_pod(self):
+        return {'settings': self.settings.to_pod(),
+                'run_config': self.run_config.to_pod()}
+
+
+class ConfigManager(object):
     """
     Represents run-time state of WA. Mostly used as a container for loaded 
     configuration and discovered plugins.
@@ -20,6 +39,8 @@ class WAState(object):
         self.jobs_config = JobGenerator(self.plugin_cache)
         self.loaded_config_sources = []
         self._config_parser = ConfigParser()
+        self._job_specs = []
+        self.jobs = []
 
     def load_config_file(self, filepath):
         self._config_parser.load_from_path(self, filepath)
@@ -29,4 +50,7 @@ class WAState(object):
         self._config_parser.load(self, values, source)
         self.loaded_config_sources.append(source)
 
+    def finalize(self):
+        self.run_config.merge_device_config(self.plugin_cache)
+        return CombinedConfig(self.settings, self.run_config)
 

wlauto/core/device_manager.py

@@ -26,7 +26,8 @@ class TargetInfo(object):
         instance.os_version = pod['os_version']
         instance.abi = pod['abi']
         instance.is_rooted = pod['is_rooted']
-        instance.kernel_version = KernelVersion(pod['kernel_version'])
+        instance.kernel_version = KernelVersion(pod['kernel_release'], 
+                                                pod['kernel_version'])
         instance.kernel_config = KernelConfig(pod['kernel_config'])
 
         if pod["target"] == "AndroidTarget":
@@ -69,15 +70,16 @@ class TargetInfo(object):
 
     def to_pod(self):
         pod = {}
-        pod['target'] = self.target.__class__.__name__
+        pod['target'] = self.target
         pod['abi'] = self.abi
-        pod['cpuinfo'] = self.cpuinfo.text
+        pod['cpuinfo'] = self.cpuinfo.sections
         pod['os'] = self.os
         pod['os_version'] = self.os_version
         pod['abi'] = self.abi
         pod['is_rooted'] = self.is_rooted
+        pod['kernel_release'] = self.kernel_version.release
         pod['kernel_version'] = self.kernel_version.version
-        pod['kernel_config'] = self.kernel_config.text
+        pod['kernel_config'] = dict(self.kernel_config.iteritems())
 
         if self.target == "AndroidTarget":
             pod['screen_resolution'] = self.screen_resolution

wlauto/core/entry_point.py

@@ -24,8 +24,8 @@ import warnings
 from wlauto.core import pluginloader
 from wlauto.core.command import init_argument_parser
 from wlauto.core.configuration import settings
+from wlauto.core.configuration.manager import ConfigManager
 from wlauto.core.host import init_user_directory
-from wlauto.core.state import WAState
 from wlauto.exceptions import WAError, DevlibError, ConfigError
 from wlauto.utils.doc import format_body
 from wlauto.utils.log import init_logging
@@ -46,7 +46,7 @@ def load_commands(subparsers):
 
 
 def main():
-    state = WAState()
+    config = ConfigManager()
 
     if not os.path.exists(settings.user_directory):
         init_user_directory()
@@ -68,16 +68,16 @@ def main():
 
         settings.set("verbosity", args.verbose)
 
-        state.load_config_file(settings.user_config_file)
+        config.load_config_file(settings.user_config_file)
         for config_file in args.config:
             if not os.path.exists(config_file):
                 raise ConfigError("Config file {} not found".format(config_file))
-            state.load_config_file(config_file)
+            config.load_config_file(config_file)
 
         init_logging(settings.verbosity)
 
         command = commands[args.command]
-        sys.exit(command.execute(state, args))
+        sys.exit(command.execute(config, args))
 
     except KeyboardInterrupt:
         logging.info('Got CTRL-C. Aborting.')

wlauto/core/execution.py

@@ -51,6 +51,7 @@ import wlauto.core.signal as signal
 from wlauto.core import instrumentation
 from wlauto.core import pluginloader
 from wlauto.core.configuration import settings
+from wlauto.core.device_manager import TargetInfo
 from wlauto.core.plugin import Artifact
 from wlauto.core.resolver import ResourceResolver
 from wlauto.core.result import ResultManager, IterationResult, RunResult
@@ -213,6 +214,30 @@ def _check_artifact_path(path, rootpath):
     return full_path
 
 
+
+class FakeTargetManager(object):
+
+    def __init__(self, name, config):
+        self.device_name = name
+        self.device_config = config
+
+        from devlib import LocalLinuxTarget
+        self.target = LocalLinuxTarget({'unrooted': True})
+        
+    def get_target_info(self):
+        return TargetInfo(self.target)
+
+    def validate_runtime_parameters(self, params):
+        pass
+
+    def merge_runtime_parameters(self, params):
+        pass
+
+
+def init_target_manager(config):
+    return FakeTargetManager(config.device, config.device_config)
+
+
 class Executor(object):
     """
     The ``Executor``'s job is to set up the execution context and pass to a
@@ -237,14 +262,14 @@ class Executor(object):
         self.device = None
         self.context = None
 
-    def execute(self, state, output):
+    def execute(self, config_manager, output):
         """
         Execute the run specified by an agenda. Optionally, selectors may be
         used to only selecute a subset of the specified agenda.
 
         Params::
 
-            :state: a ``WAState`` containing processed configuraiton
+            :state: a ``ConfigManager`` containing processed configuraiton
             :output: an initialized ``RunOutput`` that will be used to
                      store the results.
 
@@ -253,8 +278,17 @@ class Executor(object):
         signal.connect(self._warning_signalled_callback, signal.WARNING_LOGGED)
 
         self.logger.info('Initializing run')
+        self.logger.debug('Finalizing run configuration.')
+        config = config_manager.finalize()
+        output.write_config(config)
 
-        self.logger.debug('Loading run configuration.')
+        self.logger.info('Connecting to target')
+        target_manager = init_target_manager(config.run_config)
+        output.write_target_info(target_manager.get_target_info())
+
+        self.logger.info('Generationg jobs')
+        job_specs = config_manager.jobs_config.generate_job_specs(target_manager)
+        output.write_job_specs(job_specs)
 
     def old_exec(self, agenda, selectors={}):
         self.config.set_agenda(agenda, selectors)

wlauto/core/output.py

@@ -6,6 +6,9 @@ import sys
 import uuid
 from copy import copy
 
+from wlauto.core.configuration.configuration import JobSpec
+from wlauto.core.configuration.manager import ConfigManager
+from wlauto.core.device_manager import TargetInfo
 from wlauto.utils.misc import touch
 from wlauto.utils.serializer import write_pod, read_pod
 
@@ -78,6 +81,18 @@ class RunOutput(object):
     def statefile(self):
         return os.path.join(self.basepath, '.run_state.json')
 
+    @property
+    def configfile(self):
+        return os.path.join(self.metadir, 'config.json')
+
+    @property
+    def targetfile(self):
+        return os.path.join(self.metadir, 'target_info.json')
+
+    @property
+    def jobsfile(self):
+        return os.path.join(self.metadir, 'jobs.json')
+
     def __init__(self, path):
         self.basepath = path
         self.info = None
@@ -98,6 +113,32 @@ class RunOutput(object):
     def write_state(self):
         write_pod(self.state.to_pod(), self.statefile)
 
+    def write_config(self, config):
+        write_pod(config.to_pod(), self.configfile)
+
+    def read_config(self):
+        if not os.path.isfile(self.configfile):
+            return None
+        return ConfigManager.from_pod(read_pod(self.configfile))
+
+    def write_target_info(self, ti):
+        write_pod(ti.to_pod(), self.targetfile)
+
+    def read_config(self):
+        if not os.path.isfile(self.targetfile):
+            return None
+        return TargetInfo.from_pod(read_pod(self.targetfile))
+
+    def write_job_specs(self, job_specs):
+        job_specs[0].to_pod()
+        js_pod = {'jobs': [js.to_pod() for js in job_specs]}
+        write_pod(js_pod, self.jobsfile)
+
+    def read_job_specs(self):
+        if not os.path.isfile(self.jobsfile):
+            return None
+        pod = read_pod(self.jobsfile)
+        return [JobSpec.from_pod(jp) for jp in pod['jobs']]
 
 
 def init_wa_output(path, wa_state, force=False):