New target description + moving target stuff under "framework"

Changing the way target descriptions work from a static mapping to
something that is dynamically generated and is extensible via plugins.
Also moving core target implementation stuff under "framework".

wa/framework/configuration/core.py

@@ -638,8 +638,7 @@ class RunConfiguration(Configuration):
 
     name = "Run Configuration"
 
-    # Metadata is separated out because it is not loaded into the auto
+    # Metadata is separated out because it is not loaded into the auto generated config file
-    # generated config file
     meta_data = [
         ConfigurationPoint('run_name', kind=str,
                            description='''
@@ -918,8 +917,7 @@ class JobSpec(Configuration):
         except NotFoundError:
             global_runtime_params = {}
         for source in plugin_cache.sources:
-            if source in global_runtime_params:
+            runtime_parameters[source] = global_runtime_params[source]
-                runtime_parameters[source] = global_runtime_params[source]
 
         # Add runtime parameters from JobSpec
         for source, values in self.to_merge['runtime_parameters'].iteritems():

wa/framework/configuration/parsers.py

@@ -32,7 +32,6 @@ class ConfigParser(object):
 
     def load(self, state, raw, source, wrap_exceptions=True):  # pylint: disable=too-many-branches
         try:
-            state.plugin_cache.add_source(source)
             if 'run_name' in raw:
                 msg = '"run_name" can only be specified in the config '\
                       'section of an agenda'

wa/framework/configuration/plugin_cache.py

@@ -181,74 +181,47 @@ class PluginCache(object):
         :rtype: A fully merged and validated configuration in the form of a
                 obj_dict.
         """
-        ms = MergeState()
+        generic_config = copy(self.plugin_configs[generic_name])
-        ms.generic_name = generic_name
+        specific_config = copy(self.plugin_configs[specific_name])
-        ms.specific_name = specific_name
+        cfg_points = self.get_plugin_parameters(specific_name)
-        ms.generic_config = copy(self.plugin_configs[generic_name])
-        ms.specific_config = copy(self.plugin_configs[specific_name])
-        ms.cfg_points = self.get_plugin_parameters(specific_name)
         sources = self.sources
+        seen_specific_config = defaultdict(list)
 
         # set_value uses the 'name' attribute of the passed object in it error
         # messages, to ensure these messages make sense the name will have to be
         # changed several times during this function.
         final_config.name = specific_name
 
+        # pylint: disable=too-many-nested-blocks
         for source in sources:
             try:
-                update_config_from_source(final_config, source, ms)
+                if source in generic_config:
+                    final_config.name = generic_name
+                    for name, cfg_point in cfg_points.iteritems():
+                        if name in generic_config[source]:
+                            if name in seen_specific_config:
+                                msg = ('"{generic_name}" configuration "{config_name}" has already been '
+                                       'specified more specifically for {specific_name} in:\n\t\t{sources}')
+                                msg = msg.format(generic_name=generic_name,
+                                                 config_name=name,
+                                                 specific_name=specific_name,
+                                                 sources=", ".join(seen_specific_config[name]))
+                                raise ConfigError(msg)
+                            value = generic_config[source][name]
+                            cfg_point.set_value(final_config, value, check_mandatory=False)
+
+                if source in specific_config:
+                    final_config.name = specific_name
+                    for name, cfg_point in cfg_points.iteritems():
+                        if name in specific_config[source]:
+                            seen_specific_config[name].append(str(source))
+                            value = specific_config[source][name]
+                            cfg_point.set_value(final_config, value, check_mandatory=False)
+
             except ConfigError as e:
                 raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e)))
 
         # Validate final configuration
         final_config.name = specific_name
-        for cfg_point in ms.cfg_points.itervalues():
+        for cfg_point in cfg_points.itervalues():
             cfg_point.validate(final_config)
-
-
-class MergeState(object):
-
-    def __init__(self):
-        self.generic_name = None
-        self.specific_name = None
-        self.generic_config = None
-        self.specific_config = None
-        self.cfg_points = None
-        self.seen_specific_config = defaultdict(list)
-
-
-def update_config_from_source(final_config, source, state):
-    if source in state.generic_config:
-        final_config.name = state.generic_name
-        for name, cfg_point in state.cfg_points.iteritems():
-            if name in state.generic_config[source]:
-                if name in state.seen_specific_config:
-                    msg = ('"{generic_name}" configuration "{config_name}" has '
-                            'already been specified more specifically for '
-                            '{specific_name} in:\n\t\t{sources}')
-                    seen_sources = state.seen_specific_config[name]
-                    msg = msg.format(generic_name=generic_name,
-                                        config_name=name,
-                                        specific_name=specific_name,
-                                        sources=", ".join(seen_sources))
-                    raise ConfigError(msg)
-                value = state.generic_config[source].pop(name)
-                cfg_point.set_value(final_config, value, check_mandatory=False)
-
-        if state.generic_config[source]:
-            msg = 'Unexected values for {}: {}'
-            raise ConfigError(msg.format(state.generic_name,
-                                         state.generic_config[source]))
-
-    if source in state.specific_config:
-        final_config.name = state.specific_name
-        for name, cfg_point in state.cfg_points.iteritems():
-            if name in state.specific_config[source]:
-                seen_state.specific_config[name].append(str(source))
-                value = state.specific_config[source].pop(name)
-                cfg_point.set_value(final_config, value, check_mandatory=False)
-
-        if state.specific_config[source]:
-            msg = 'Unexected values for {}: {}'
-            raise ConfigError(msg.format(state.specific_name,
-                                         state.specific_config[source]))

wa/framework/execution.py

@@ -57,7 +57,6 @@ from wa.framework.exception import (WAError, ConfigError, TimeoutError,
 from wa.framework.plugin import Artifact
 from wa.framework.resource import ResourceResolver
 from wa.framework.target.info import TargetInfo
-from wa.framework.target.manager import TargetManager
 from wa.utils.misc import (ensure_directory_exists as _d, 
                            get_traceback, format_duration)
 from wa.utils.serializer import json
@@ -229,6 +228,30 @@ def _check_artifact_path(path, rootpath):
     return full_path
 
 
+class FakeTargetManager(object):
+    # TODO: this is a FAKE
+
+    def __init__(self, name, config):
+        self.device_name = name
+        self.device_config = config
+
+        from devlib import LocalLinuxTarget
+        self.target = LocalLinuxTarget({'unrooted': True})
+        
+    def get_target_info(self):
+        return TargetInfo(self.target)
+
+    def validate_runtime_parameters(self, params):
+        pass
+
+    def merge_runtime_parameters(self, params):
+        pass
+
+
+def init_target_manager(config):
+    return FakeTargetManager(config.device, config.device_config)
+
+
 class Executor(object):
     """
     The ``Executor``'s job is to set up the execution context and pass to a
@@ -274,8 +297,7 @@ class Executor(object):
         output.write_config(config)
 
         self.logger.info('Connecting to target')
-        target_manager = TargetManager(config.run_config.device,
+        target_manager = init_target_manager(config.run_config)
-                                       config.run_config.device_config)
         output.write_target_info(target_manager.get_target_info())
 
         self.logger.info('Initializing execution conetext')

wa/framework/output.py

@@ -6,11 +6,11 @@ import sys
 import uuid
 from copy import copy
 
-from wa.framework.configuration.core import JobSpec
+from wlauto.core.configuration.configuration import JobSpec
-from wa.framework.configuration.manager import ConfigManager
+from wlauto.core.configuration.manager import ConfigManager
-from wa.framework.target.info import TargetInfo
+from wlauto.core.device_manager import TargetInfo
-from wa.utils.misc import touch
+from wlauto.utils.misc import touch
-from wa.utils.serializer import write_pod, read_pod
+from wlauto.utils.serializer import write_pod, read_pod
 
 
 logger = logging.getLogger('output')

wa/framework/target/descriptor.py

@@ -24,35 +24,6 @@ def get_target_descriptions(loader=pluginloader):
     return targets.values()
 
 
-def instantiate_target(tdesc, params, connect=None):
-    target_params = {p.name: p for p in tdesc.target_params}
-    platform_params = {p.name: p for p in tdesc.platform_params}
-    conn_params = {p.name: p for p in tdesc.conn_params}
-
-    tp, pp, cp = {}, {}, {}
-
-    for name, value in params.iteritems():
-        if name in target_params:
-            tp[name] = value
-        elif name in platform_params:
-            pp[name] = value
-        elif name in conn_params:
-            cp[name] = value
-        else:
-            msg = 'Unexpected parameter for {}: {}'
-            raise ValueError(msg.format(tdesc.name, name))
-
-    tp['platform'] = (tdesc.platform or Platform)(**pp)
-    if cp:
-        tp['connection_settings'] = cp
-    if tdesc.connection:
-        tp['conn_cls'] = tdesc.connection
-    if connect is not None:
-        tp['connect'] = connect
-
-    return tdesc.target(**tp)
-
-
 class TargetDescription(object):
 
     def __init__(self, name, source, description=None, target=None, platform=None, 
@@ -115,18 +86,6 @@ COMMON_TARGET_PARAMS = [
               Please see ``devlab`` documentation for information on the available
               modules.
               '''),
-    Parameter('load_default_modules', kind=bool, default=True,
-              description='''
-              A number of modules (e.g. for working with the cpufreq subsystem) are
-              loaded by default when a Target is instantiated. Setting this to
-              ``True`` would suppress that, ensuring that only the base Target
-              interface is initialized.
-
-              You may want to set this if there is a problem with one or more default
-              modules on your platform (e.g. your device is unrooted and cpufreq is
-              not accessible to unprivileged users), or if Target initialization is
-              taking too long for your platform.
-              '''),
 ]
 
 COMMON_PLATFORM_PARAMS = [

wa/framework/target/info.py

@@ -1,7 +1,6 @@
 from devlib import AndroidTarget
 from devlib.exception import TargetError
 from devlib.target import KernelConfig, KernelVersion, Cpuinfo
-from devlib.utils.android import AndroidProperties
 
 
 class TargetInfo(object):
@@ -22,9 +21,8 @@ class TargetInfo(object):
 
         if pod["target"] == "AndroidTarget":
             instance.screen_resolution = pod['screen_resolution']
-            instance.prop = AndroidProperties('')
+            instance.prop = pod['prop']
-            instance.prop._properties = pod['prop']
+            instance.prop = pod['android_id']
-            instance.android_id = pod['android_id']
 
         return instance
 
@@ -74,7 +72,7 @@ class TargetInfo(object):
 
         if self.target == "AndroidTarget":
             pod['screen_resolution'] = self.screen_resolution
-            pod['prop'] = self.prop._properties
+            pod['prop'] = self.prop
             pod['android_id'] = self.android_id
 
         return pod

wa/framework/target/manager.py

@@ -9,8 +9,6 @@ import sys
 from wa.framework import signal
 from wa.framework.exception import WorkerThreadError, ConfigError
 from wa.framework.plugin import Parameter
-from wa.framework.target.descriptor import (get_target_descriptions,
-                                            instantiate_target)
 from wa.framework.target.info import TargetInfo
 from wa.framework.target.runtime_config import (SysfileValuesRuntimeConfig,
                                                 HotplugRuntimeConfig,
@@ -43,26 +41,54 @@ class TargetManager(object):
                   """),
     ]
 
+    DEVICE_MAPPING = {'test' : {'platform_name':'generic',
+                               'target_name': 'android'},
+                      'other':  {'platform_name':'test',
+                                'target_name': 'linux'},
+                      }
+
     runtime_config_cls = [
-        # order matters
+                            # order matters
-        SysfileValuesRuntimeConfig,
+                            SysfileValuesRuntimeConfig,
-        HotplugRuntimeConfig,
+                            HotplugRuntimeConfig,
-        CpufreqRuntimeConfig,
+                            CpufreqRuntimeConfig,
-        CpuidleRuntimeConfig,
+                            CpuidleRuntimeConfig,
-    ]
+                          ]
 
     def __init__(self, name, parameters):
-        self.target_name = name
+        self.name = name
         self.target = None
         self.assistant = None
+        self.target_name = None
         self.platform_name = None
         self.parameters = parameters
         self.disconnect = parameters.get('disconnect')
         self.info = TargetInfo()
 
-        self._init_target()
+        # Determine platform and target based on passed name
-        self._init_assistant()
+        self._parse_name()
+        # Create target
+        self._get_target()
+        # Create an assistant to perform target specific configuration
+        self._get_assistant()
+
+        ### HERE FOR TESTING, WILL BE CALLED EXTERNALLY ###
+        # Connect to device and retrieve details.
+        # self.initialize()
+        # self.add_parameters()
+        # self.validate_parameters()
+        # self.set_parameters()
+
+    def initialize(self):
         self.runtime_configs = [cls(self.target) for cls in self.runtime_config_cls]
+        # if self.parameters:
+        # self.logger.info('Connecting to the device')
+        with signal.wrap('TARGET_CONNECT'):
+            self.target.connect()
+            # self.info.load(self.target)
+            # info_file = os.path.join(self.context.info_directory, 'target.json')
+            # with open(info_file, 'w') as wfh:
+            #     json.dump(self.info.to_pod(), wfh)
 
     def finalize(self):
         # self.logger.info('Disconnecting from the device')
@@ -82,16 +108,10 @@ class TargetManager(object):
                 if any(parameter in name for parameter in cfg.supported_parameters):
                     cfg.add(name, self.parameters.pop(name))
 
-    def get_target_info(self):
+    def validate_parameters(self):
-        return TargetInfo(self.target)
-
-    def validate_runtime_parameters(self, params):
         for cfg in self.runtime_configs:
             cfg.validate()
 
-    def merge_runtime_parameters(self, params):
-        pass
-
     def set_parameters(self):
         for cfg in self.runtime_configs:
             cfg.set()
@@ -100,25 +120,47 @@ class TargetManager(object):
         for cfg in self.runtime_configs:
             cfg.clear()
 
-    def _init_target(self):
+    def _parse_name(self):
-        target_map = {td.name: td for td in get_target_descriptions()}
+        # Try and get platform and target
-        if self.target_name not in target_map:
+        self.name = identifier(self.name.replace('-', '_'))
-            raise ValueError('Unknown Target: {}'.format(self.target_name))
+        if '_' in self.name:
-        tdesc = target_map[self.target_name]
+            self.platform_name, self.target_name = self.name.split('_', 1)
-        self.target = instantiate_target(tdesc, self.parameters, connect=False)
+        elif self.name in self.DEVICE_MAPPING:
-        with signal.wrap('TARGET_CONNECT'):
+            self.platform_name = self.DEVICE_MAPPING[self.name]['platform_name']
-            self.target.connect()
+            self.target_name = self.DEVICE_MAPPING[self.name]['target_name']
-        self.target.setup()
+        else:
+            raise ConfigError('Unknown Device Specified {}'.format(self.name))
 
-    def _init_assistant(self):
+    def _get_target(self):
-        # Create a corresponding target and target-assistant to help with
+        # Create a corresponding target and target-assistant
-        # platformy stuff?
+        if self.target_name == 'android':
-        if self.target.os == 'android':
+            self.target = AndroidTarget()
+        elif self.target_name == 'linux':
+            self.target = LinuxTarget()  # pylint: disable=redefined-variable-type
+        elif self.target_name == 'localLinux':
+            self.target = LocalLinuxTarget()
+        else:
+            raise ConfigError('Unknown Target Specified {}'.format(self.target_name))
+
+    def _get_assistant(self):
+        # Create a corresponding target and target-assistant to help with platformy stuff?
+        if self.target_name == 'android':
             self.assistant = AndroidAssistant(self.target)
-        elif self.target.os == 'linux':
+        elif self.target_name in ['linux', 'localLinux']:
             self.assistant = LinuxAssistant(self.target)  # pylint: disable=redefined-variable-type
         else:
-            raise ValueError('Unknown Target OS: {}'.format(self.target.os))
+            raise ConfigError('Unknown Target Specified {}'.format(self.target_name))
+
+    # def validate_runtime_parameters(self, parameters):
+    #     for  name, value in parameters.iteritems():
+    #         self.add_parameter(name, value)
+    #     self.validate_parameters()
+
+    # def set_runtime_parameters(self, parameters):
+    #     # self.clear()
+    #     for  name, value in parameters.iteritems():
+    #         self.add_parameter(name, value)
+    #     self.set_parameters()
 
 
 class LinuxAssistant(object):

wa/utils/log.py

@@ -78,7 +78,7 @@ def set_level(level):
 
 
 def add_file(filepath, level=logging.DEBUG,
-             fmt='%(asctime)s %(levelname)-8s %(name)s: %(message)-10.10s'):
+             fmt='%(asctime)s %(levelname)-8s %(name)10.10s: %(message)s'):
     root_logger = logging.getLogger()
     file_handler = logging.FileHandler(filepath)
     file_handler.setLevel(level)