Integerated new target stuff into execution so far

wa/framework/configuration/core.py

@@ -638,7 +638,8 @@ class RunConfiguration(Configuration):
 
     name = "Run Configuration"
 
-    # Metadata is separated out because it is not loaded into the auto generated config file
+    # Metadata is separated out because it is not loaded into the auto
+    # generated config file
     meta_data = [
         ConfigurationPoint('run_name', kind=str,
                            description='''
@@ -917,7 +918,8 @@ class JobSpec(Configuration):
         except NotFoundError:
             global_runtime_params = {}
         for source in plugin_cache.sources:
-            runtime_parameters[source] = global_runtime_params[source]
+            if source in global_runtime_params:
+                runtime_parameters[source] = global_runtime_params[source]
 
         # Add runtime parameters from JobSpec
         for source, values in self.to_merge['runtime_parameters'].iteritems():

wa/framework/configuration/parsers.py

@@ -32,6 +32,7 @@ class ConfigParser(object):
 
     def load(self, state, raw, source, wrap_exceptions=True):  # pylint: disable=too-many-branches
         try:
+            state.plugin_cache.add_source(source)
             if 'run_name' in raw:
                 msg = '"run_name" can only be specified in the config '\
                       'section of an agenda'

wa/framework/configuration/plugin_cache.py

@@ -181,47 +181,74 @@ class PluginCache(object):
         :rtype: A fully merged and validated configuration in the form of a
                 obj_dict.
         """
-        generic_config = copy(self.plugin_configs[generic_name])
+        ms = MergeState()
-        specific_config = copy(self.plugin_configs[specific_name])
+        ms.generic_name = generic_name
-        cfg_points = self.get_plugin_parameters(specific_name)
+        ms.specific_name = specific_name
+        ms.generic_config = copy(self.plugin_configs[generic_name])
+        ms.specific_config = copy(self.plugin_configs[specific_name])
+        ms.cfg_points = self.get_plugin_parameters(specific_name)
         sources = self.sources
-        seen_specific_config = defaultdict(list)
 
         # set_value uses the 'name' attribute of the passed object in it error
         # messages, to ensure these messages make sense the name will have to be
         # changed several times during this function.
         final_config.name = specific_name
 
-        # pylint: disable=too-many-nested-blocks
         for source in sources:
             try:
-                if source in generic_config:
+                update_config_from_source(final_config, source, ms)
-                    final_config.name = generic_name
-                    for name, cfg_point in cfg_points.iteritems():
-                        if name in generic_config[source]:
-                            if name in seen_specific_config:
-                                msg = ('"{generic_name}" configuration "{config_name}" has already been '
-                                       'specified more specifically for {specific_name} in:\n\t\t{sources}')
-                                msg = msg.format(generic_name=generic_name,
-                                                 config_name=name,
-                                                 specific_name=specific_name,
-                                                 sources=", ".join(seen_specific_config[name]))
-                                raise ConfigError(msg)
-                            value = generic_config[source][name]
-                            cfg_point.set_value(final_config, value, check_mandatory=False)
-
-                if source in specific_config:
-                    final_config.name = specific_name
-                    for name, cfg_point in cfg_points.iteritems():
-                        if name in specific_config[source]:
-                            seen_specific_config[name].append(str(source))
-                            value = specific_config[source][name]
-                            cfg_point.set_value(final_config, value, check_mandatory=False)
-
             except ConfigError as e:
                 raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e)))
 
         # Validate final configuration
         final_config.name = specific_name
-        for cfg_point in cfg_points.itervalues():
+        for cfg_point in ms.cfg_points.itervalues():
             cfg_point.validate(final_config)
+
+
+class MergeState(object):
+
+    def __init__(self):
+        self.generic_name = None
+        self.specific_name = None
+        self.generic_config = None
+        self.specific_config = None
+        self.cfg_points = None
+        self.seen_specific_config = defaultdict(list)
+
+
+def update_config_from_source(final_config, source, state):
+    if source in state.generic_config:
+        final_config.name = state.generic_name
+        for name, cfg_point in state.cfg_points.iteritems():
+            if name in state.generic_config[source]:
+                if name in state.seen_specific_config:
+                    msg = ('"{generic_name}" configuration "{config_name}" has '
+                            'already been specified more specifically for '
+                            '{specific_name} in:\n\t\t{sources}')
+                    seen_sources = state.seen_specific_config[name]
+                    msg = msg.format(generic_name=generic_name,
+                                        config_name=name,
+                                        specific_name=specific_name,
+                                        sources=", ".join(seen_sources))
+                    raise ConfigError(msg)
+                value = state.generic_config[source].pop(name)
+                cfg_point.set_value(final_config, value, check_mandatory=False)
+
+        if state.generic_config[source]:
+            msg = 'Unexected values for {}: {}'
+            raise ConfigError(msg.format(state.generic_name,
+                                         state.generic_config[source]))
+
+    if source in state.specific_config:
+        final_config.name = state.specific_name
+        for name, cfg_point in state.cfg_points.iteritems():
+            if name in state.specific_config[source]:
+                seen_state.specific_config[name].append(str(source))
+                value = state.specific_config[source].pop(name)
+                cfg_point.set_value(final_config, value, check_mandatory=False)
+
+        if state.specific_config[source]:
+            msg = 'Unexected values for {}: {}'
+            raise ConfigError(msg.format(state.specific_name,
+                                         state.specific_config[source]))

wa/framework/execution.py

@@ -57,6 +57,7 @@ from wa.framework.exception import (WAError, ConfigError, TimeoutError,
 from wa.framework.plugin import Artifact
 from wa.framework.resource import ResourceResolver
 from wa.framework.target.info import TargetInfo
+from wa.framework.target.manager import TargetManager
 from wa.utils.misc import (ensure_directory_exists as _d, 
                            get_traceback, format_duration)
 from wa.utils.serializer import json
@@ -228,30 +229,6 @@ def _check_artifact_path(path, rootpath):
     return full_path
 
 
-class FakeTargetManager(object):
-    # TODO: this is a FAKE
-
-    def __init__(self, name, config):
-        self.device_name = name
-        self.device_config = config
-
-        from devlib import LocalLinuxTarget
-        self.target = LocalLinuxTarget({'unrooted': True})
-        
-    def get_target_info(self):
-        return TargetInfo(self.target)
-
-    def validate_runtime_parameters(self, params):
-        pass
-
-    def merge_runtime_parameters(self, params):
-        pass
-
-
-def init_target_manager(config):
-    return FakeTargetManager(config.device, config.device_config)
-
-
 class Executor(object):
     """
     The ``Executor``'s job is to set up the execution context and pass to a
@@ -297,7 +274,8 @@ class Executor(object):
         output.write_config(config)
 
         self.logger.info('Connecting to target')
-        target_manager = init_target_manager(config.run_config)
+        target_manager = TargetManager(config.run_config.device,
+                                       config.run_config.device_config)
         output.write_target_info(target_manager.get_target_info())
 
         self.logger.info('Initializing execution conetext')

wa/framework/output.py

@@ -6,11 +6,11 @@ import sys
 import uuid
 from copy import copy
 
-from wlauto.core.configuration.configuration import JobSpec
+from wa.framework.configuration.core import JobSpec
-from wlauto.core.configuration.manager import ConfigManager
+from wa.framework.configuration.manager import ConfigManager
-from wlauto.core.device_manager import TargetInfo
+from wa.framework.target.info import TargetInfo
-from wlauto.utils.misc import touch
+from wa.utils.misc import touch
-from wlauto.utils.serializer import write_pod, read_pod
+from wa.utils.serializer import write_pod, read_pod
 
 
 logger = logging.getLogger('output')

wa/framework/target/descriptor.py

@@ -24,6 +24,33 @@ def get_target_descriptions(loader=pluginloader):
     return targets.values()
 
 
+def instantiate_target(tdesc, params):
+    target_params = {p.name: p for p in tdesc.target_params}
+    platform_params = {p.name: p for p in tdesc.platform_params}
+    conn_params = {p.name: p for p in tdesc.conn_params}
+
+    tp, pp, cp = {}, {}, {}
+
+    for name, value in params.iteritems():
+        if name in target_params:
+            tp[name] = value
+        elif name in platform_params:
+            pp[name] = value
+        elif name in conn_params:
+            cp[name] = value
+        else:
+            msg = 'Unexpected parameter for {}: {}'
+            raise ValueError(msg.format(tdesc.name, name))
+
+    tp['platform'] = (tdesc.platform or Platform)(**pp)
+    if cp:
+        tp['connection_settings'] = cp
+    if tdesc.connection:
+        tp['conn_cls'] = tdesc.connection
+
+    return tdesc.target(**tp)
+
+
 class TargetDescription(object):
 
     def __init__(self, name, source, description=None, target=None, platform=None, 
@@ -86,6 +113,18 @@ COMMON_TARGET_PARAMS = [
               Please see ``devlab`` documentation for information on the available
               modules.
               '''),
+    Parameter('load_default_modules', kind=bool, default=True,
+              description='''
+              A number of modules (e.g. for working with the cpufreq subsystem) are
+              loaded by default when a Target is instantiated. Setting this to
+              ``True`` would suppress that, ensuring that only the base Target
+              interface is initialized.
+
+              You may want to set this if there is a problem with one or more default
+              modules on your platform (e.g. your device is unrooted and cpufreq is
+              not accessible to unprivileged users), or if Target initialization is
+              taking too long for your platform.
+              '''),
 ]
 
 COMMON_PLATFORM_PARAMS = [

wa/framework/target/info.py

@@ -1,6 +1,7 @@
 from devlib import AndroidTarget
 from devlib.exception import TargetError
 from devlib.target import KernelConfig, KernelVersion, Cpuinfo
+from devlib.utils.android import AndroidProperties
 
 
 class TargetInfo(object):
@@ -21,8 +22,9 @@ class TargetInfo(object):
 
         if pod["target"] == "AndroidTarget":
             instance.screen_resolution = pod['screen_resolution']
-            instance.prop = pod['prop']
+            instance.prop = AndroidProperties('')
-            instance.prop = pod['android_id']
+            instance.prop._properties = pod['prop']
+            instance.android_id = pod['android_id']
 
         return instance
 
@@ -72,7 +74,7 @@ class TargetInfo(object):
 
         if self.target == "AndroidTarget":
             pod['screen_resolution'] = self.screen_resolution
-            pod['prop'] = self.prop
+            pod['prop'] = self.prop._properties
             pod['android_id'] = self.android_id
 
         return pod

wa/framework/target/manager.py

@@ -9,6 +9,8 @@ import sys
 from wa.framework import signal
 from wa.framework.exception import WorkerThreadError, ConfigError
 from wa.framework.plugin import Parameter
+from wa.framework.target.descriptor import (get_target_descriptions,
+                                            instantiate_target)
 from wa.framework.target.info import TargetInfo
 from wa.framework.target.runtime_config import (SysfileValuesRuntimeConfig,
                                                 HotplugRuntimeConfig,
@@ -41,54 +43,29 @@ class TargetManager(object):
                   """),
     ]
 
-    DEVICE_MAPPING = {'test' : {'platform_name':'generic',
-                               'target_name': 'android'},
-                      'other':  {'platform_name':'test',
-                                'target_name': 'linux'},
-                      }
-
     runtime_config_cls = [
-                            # order matters
+        # order matters
-                            SysfileValuesRuntimeConfig,
+        SysfileValuesRuntimeConfig,
-                            HotplugRuntimeConfig,
+        HotplugRuntimeConfig,
-                            CpufreqRuntimeConfig,
+        CpufreqRuntimeConfig,
-                            CpuidleRuntimeConfig,
+        CpuidleRuntimeConfig,
-                          ]
+    ]
 
     def __init__(self, name, parameters):
-        self.name = name
+        self.target_name = name
         self.target = None
         self.assistant = None
-        self.target_name = None
         self.platform_name = None
         self.parameters = parameters
         self.disconnect = parameters.get('disconnect')
         self.info = TargetInfo()
 
-        # Determine platform and target based on passed name
+        self._init_target()
-        self._parse_name()
+        self._init_assistant()
-        # Create target
-        self._get_target()
-        # Create an assistant to perform target specific configuration
-        self._get_assistant()
 
-        ### HERE FOR TESTING, WILL BE CALLED EXTERNALLY ###
-        # Connect to device and retrieve details.
-        # self.initialize()
-        # self.add_parameters()
-        # self.validate_parameters()
-        # self.set_parameters()
-
-    def initialize(self):
         self.runtime_configs = [cls(self.target) for cls in self.runtime_config_cls]
-        # if self.parameters:
-        # self.logger.info('Connecting to the device')
         with signal.wrap('TARGET_CONNECT'):
             self.target.connect()
-            # self.info.load(self.target)
-            # info_file = os.path.join(self.context.info_directory, 'target.json')
-            # with open(info_file, 'w') as wfh:
-            #     json.dump(self.info.to_pod(), wfh)
 
     def finalize(self):
         # self.logger.info('Disconnecting from the device')
@@ -108,10 +85,16 @@ class TargetManager(object):
                 if any(parameter in name for parameter in cfg.supported_parameters):
                     cfg.add(name, self.parameters.pop(name))
 
-    def validate_parameters(self):
+    def get_target_info(self):
+        return TargetInfo(self.target)
+
+    def validate_runtime_parameters(self, params):
         for cfg in self.runtime_configs:
             cfg.validate()
 
+    def merge_runtime_parameters(self, params):
+        pass
+
     def set_parameters(self):
         for cfg in self.runtime_configs:
             cfg.set()
@@ -120,47 +103,23 @@ class TargetManager(object):
         for cfg in self.runtime_configs:
             cfg.clear()
 
-    def _parse_name(self):
+    def _init_target(self):
-        # Try and get platform and target
+        target_map = {td.name: td for td in get_target_descriptions()}
-        self.name = identifier(self.name.replace('-', '_'))
+        if self.target_name not in target_map:
-        if '_' in self.name:
+            raise ValueError('Unknown Target: {}'.format(self.target_name))
-            self.platform_name, self.target_name = self.name.split('_', 1)
+        tdesc = target_map[self.target_name]
-        elif self.name in self.DEVICE_MAPPING:
+        self.target = instantiate_target(tdesc, self.parameters)
-            self.platform_name = self.DEVICE_MAPPING[self.name]['platform_name']
+        self.target.setup()
-            self.target_name = self.DEVICE_MAPPING[self.name]['target_name']
-        else:
-            raise ConfigError('Unknown Device Specified {}'.format(self.name))
 
-    def _get_target(self):
+    def _init_assistant(self):
-        # Create a corresponding target and target-assistant
+        # Create a corresponding target and target-assistant to help with
-        if self.target_name == 'android':
+        # platformy stuff?
-            self.target = AndroidTarget()
+        if self.target.os == 'android':
-        elif self.target_name == 'linux':
-            self.target = LinuxTarget()  # pylint: disable=redefined-variable-type
-        elif self.target_name == 'localLinux':
-            self.target = LocalLinuxTarget()
-        else:
-            raise ConfigError('Unknown Target Specified {}'.format(self.target_name))
-
-    def _get_assistant(self):
-        # Create a corresponding target and target-assistant to help with platformy stuff?
-        if self.target_name == 'android':
             self.assistant = AndroidAssistant(self.target)
-        elif self.target_name in ['linux', 'localLinux']:
+        elif self.target.os == 'linux':
             self.assistant = LinuxAssistant(self.target)  # pylint: disable=redefined-variable-type
         else:
-            raise ConfigError('Unknown Target Specified {}'.format(self.target_name))
+            raise ValueError('Unknown Target OS: {}'.format(self.target.os))
-
-    # def validate_runtime_parameters(self, parameters):
-    #     for  name, value in parameters.iteritems():
-    #         self.add_parameter(name, value)
-    #     self.validate_parameters()
-
-    # def set_runtime_parameters(self, parameters):
-    #     # self.clear()
-    #     for  name, value in parameters.iteritems():
-    #         self.add_parameter(name, value)
-    #     self.set_parameters()
 
 
 class LinuxAssistant(object):

wa/framework/target/runtime_config.py

@@ -15,8 +15,6 @@ class RuntimeConfig(Plugin):
     parameters = [
     ]
 
-# class RuntimeConfig(object):
-
     @property
     def supported_parameters(self):
         raise NotImplementedError()
@@ -25,8 +23,8 @@ class RuntimeConfig(Plugin):
     def core_names(self):
         return unique(self.target.core_names)
 
-    def __init__(self, target):
+    def __init__(self, target, **kwargs):
-        super(RuntimeConfig, self).__init__()
+        super(RuntimeConfig, self).__init__(**kwargs)
         self.target = target
 
     def initialize(self, context):
@@ -47,6 +45,9 @@ class RuntimeConfig(Plugin):
 
 class HotplugRuntimeConfig(RuntimeConfig):
 ##### NOTE: Currently if initialized with cores hotplugged, this will fail when trying to hotplug back in
+
+    name = 'rt-hotplug'
+
     @property
     def supported_parameters(self):
         params = ['cores']
@@ -93,6 +94,8 @@ class HotplugRuntimeConfig(RuntimeConfig):
 
 class SysfileValuesRuntimeConfig(RuntimeConfig):
 
+    name = 'rt-sysfiles'
+
     @property
     def supported_parameters(self):
         return ['sysfile_values']
@@ -132,6 +135,8 @@ class SysfileValuesRuntimeConfig(RuntimeConfig):
 
 class CpufreqRuntimeConfig(RuntimeConfig):
 
+    name = 'rt-cpufreq'
+
     @property
     def supported_parameters(self):
         params = ['frequency']
@@ -151,9 +156,14 @@ class CpufreqRuntimeConfig(RuntimeConfig):
         self.min_supported_freq = {}
         self.max_supported_freq = {}
 
-        for cpu in self.target.list_online_cpus():
+        if self.target.has('cpufreq'):
-            self.supported_freqs[cpu] = self.target.cpufreq.list_frequencies(cpu) or []
+            for cpu in self.target.list_online_cpus():
-            self.supported_govenors[cpu] = self.target.cpufreq.list_governors(cpu) or []
+                freqs = self.target.cpufreq.list_frequencies(cpu) or []
+                self.supported_freqs[cpu] = freqs
+                govs = self.target.cpufreq.list_governors(cpu) or []
+                self.supported_govenors[cpu] = govs
+        else:
+            self.logger.debug('Target does not support cpufreq')
 
     def add(self, name, value):
         if not self.target.has('cpufreq'):
@@ -319,6 +329,8 @@ class CpufreqRuntimeConfig(RuntimeConfig):
 
 class CpuidleRuntimeConfig(RuntimeConfig):
 
+    name = 'rt-cpuidle'
+
     @property
     def supported_parameters(self):
         params = ['idle_states']
@@ -330,12 +342,15 @@ class CpuidleRuntimeConfig(RuntimeConfig):
         self.aliases = ['ENABLE_ALL', 'DISABLE_ALL']
         self.available_states = {}
 
-        for cpu in self.target.list_online_cpus():
+        if self.target.has('cpuidle'):
-            self.available_states[cpu] = self.target.cpuidle.get_states(cpu) or []
+            for cpu in self.target.list_online_cpus():
+                self.available_states[cpu] = self.target.cpuidle.get_states(cpu) or []
+        else:
+            self.logger.debug('Target does not support cpuidle.')
 
     def add(self, name, values):
-        if not self.target.has('cpufreq'):
+        if not self.target.has('cpuidle'):
-            raise TargetError('Target does not support cpufreq.')
+            raise TargetError('Target does not support cpuidle.')
 
         prefix, _ = split_parameter_name(name, self.supported_parameters)
         cpus = uniqueDomainCpusFromPrefix(prefix, self.target)