From 6eb5c3681d5fbeb3f45e63f1603ae78e5c2ad4a1 Mon Sep 17 00:00:00 2001 From: Sergei Trofimov Date: Mon, 6 Mar 2017 11:10:25 +0000 Subject: [PATCH] New target description + moving target stuff under "framework" Changing the way target descriptions work from a static mapping to something that is dynamically generated and is extensible via plugins. Also moving core target implementation stuff under "framework". --- wa/framework/configuration/core.py | 6 +- wa/framework/configuration/parsers.py | 1 - wa/framework/configuration/plugin_cache.py | 85 ++++++---------- wa/framework/execution.py | 28 +++++- wa/framework/output.py | 10 +- wa/framework/target/descriptor.py | 41 -------- wa/framework/target/info.py | 8 +- wa/framework/target/manager.py | 108 ++++++++++++++------- wa/utils/log.py | 2 +- 9 files changed, 140 insertions(+), 149 deletions(-) diff --git a/wa/framework/configuration/core.py b/wa/framework/configuration/core.py index 8d11ceb5..c79df8b8 100644 --- a/wa/framework/configuration/core.py +++ b/wa/framework/configuration/core.py @@ -638,8 +638,7 @@ class RunConfiguration(Configuration): name = "Run Configuration" - # Metadata is separated out because it is not loaded into the auto - # generated config file + # Metadata is separated out because it is not loaded into the auto generated config file meta_data = [ ConfigurationPoint('run_name', kind=str, description=''' @@ -918,8 +917,7 @@ class JobSpec(Configuration): except NotFoundError: global_runtime_params = {} for source in plugin_cache.sources: - if source in global_runtime_params: - runtime_parameters[source] = global_runtime_params[source] + runtime_parameters[source] = global_runtime_params[source] # Add runtime parameters from JobSpec for source, values in self.to_merge['runtime_parameters'].iteritems(): diff --git a/wa/framework/configuration/parsers.py b/wa/framework/configuration/parsers.py index 70f50857..df6d019e 100644 --- a/wa/framework/configuration/parsers.py +++ b/wa/framework/configuration/parsers.py @@ -32,7 +32,6 @@ class ConfigParser(object): def load(self, state, raw, source, wrap_exceptions=True): # pylint: disable=too-many-branches try: - state.plugin_cache.add_source(source) if 'run_name' in raw: msg = '"run_name" can only be specified in the config '\ 'section of an agenda' diff --git a/wa/framework/configuration/plugin_cache.py b/wa/framework/configuration/plugin_cache.py index d8a3f8e8..bfabb97c 100644 --- a/wa/framework/configuration/plugin_cache.py +++ b/wa/framework/configuration/plugin_cache.py @@ -181,74 +181,47 @@ class PluginCache(object): :rtype: A fully merged and validated configuration in the form of a obj_dict. """ - ms = MergeState() - ms.generic_name = generic_name - ms.specific_name = specific_name - ms.generic_config = copy(self.plugin_configs[generic_name]) - ms.specific_config = copy(self.plugin_configs[specific_name]) - ms.cfg_points = self.get_plugin_parameters(specific_name) + generic_config = copy(self.plugin_configs[generic_name]) + specific_config = copy(self.plugin_configs[specific_name]) + cfg_points = self.get_plugin_parameters(specific_name) sources = self.sources + seen_specific_config = defaultdict(list) # set_value uses the 'name' attribute of the passed object in it error # messages, to ensure these messages make sense the name will have to be # changed several times during this function. final_config.name = specific_name + # pylint: disable=too-many-nested-blocks for source in sources: try: - update_config_from_source(final_config, source, ms) + if source in generic_config: + final_config.name = generic_name + for name, cfg_point in cfg_points.iteritems(): + if name in generic_config[source]: + if name in seen_specific_config: + msg = ('"{generic_name}" configuration "{config_name}" has already been ' + 'specified more specifically for {specific_name} in:\n\t\t{sources}') + msg = msg.format(generic_name=generic_name, + config_name=name, + specific_name=specific_name, + sources=", ".join(seen_specific_config[name])) + raise ConfigError(msg) + value = generic_config[source][name] + cfg_point.set_value(final_config, value, check_mandatory=False) + + if source in specific_config: + final_config.name = specific_name + for name, cfg_point in cfg_points.iteritems(): + if name in specific_config[source]: + seen_specific_config[name].append(str(source)) + value = specific_config[source][name] + cfg_point.set_value(final_config, value, check_mandatory=False) + except ConfigError as e: raise ConfigError('Error in "{}":\n\t{}'.format(source, str(e))) # Validate final configuration final_config.name = specific_name - for cfg_point in ms.cfg_points.itervalues(): + for cfg_point in cfg_points.itervalues(): cfg_point.validate(final_config) - - -class MergeState(object): - - def __init__(self): - self.generic_name = None - self.specific_name = None - self.generic_config = None - self.specific_config = None - self.cfg_points = None - self.seen_specific_config = defaultdict(list) - - -def update_config_from_source(final_config, source, state): - if source in state.generic_config: - final_config.name = state.generic_name - for name, cfg_point in state.cfg_points.iteritems(): - if name in state.generic_config[source]: - if name in state.seen_specific_config: - msg = ('"{generic_name}" configuration "{config_name}" has ' - 'already been specified more specifically for ' - '{specific_name} in:\n\t\t{sources}') - seen_sources = state.seen_specific_config[name] - msg = msg.format(generic_name=generic_name, - config_name=name, - specific_name=specific_name, - sources=", ".join(seen_sources)) - raise ConfigError(msg) - value = state.generic_config[source].pop(name) - cfg_point.set_value(final_config, value, check_mandatory=False) - - if state.generic_config[source]: - msg = 'Unexected values for {}: {}' - raise ConfigError(msg.format(state.generic_name, - state.generic_config[source])) - - if source in state.specific_config: - final_config.name = state.specific_name - for name, cfg_point in state.cfg_points.iteritems(): - if name in state.specific_config[source]: - seen_state.specific_config[name].append(str(source)) - value = state.specific_config[source].pop(name) - cfg_point.set_value(final_config, value, check_mandatory=False) - - if state.specific_config[source]: - msg = 'Unexected values for {}: {}' - raise ConfigError(msg.format(state.specific_name, - state.specific_config[source])) diff --git a/wa/framework/execution.py b/wa/framework/execution.py index 699f6494..a5c79714 100644 --- a/wa/framework/execution.py +++ b/wa/framework/execution.py @@ -57,7 +57,6 @@ from wa.framework.exception import (WAError, ConfigError, TimeoutError, from wa.framework.plugin import Artifact from wa.framework.resource import ResourceResolver from wa.framework.target.info import TargetInfo -from wa.framework.target.manager import TargetManager from wa.utils.misc import (ensure_directory_exists as _d, get_traceback, format_duration) from wa.utils.serializer import json @@ -229,6 +228,30 @@ def _check_artifact_path(path, rootpath): return full_path +class FakeTargetManager(object): + # TODO: this is a FAKE + + def __init__(self, name, config): + self.device_name = name + self.device_config = config + + from devlib import LocalLinuxTarget + self.target = LocalLinuxTarget({'unrooted': True}) + + def get_target_info(self): + return TargetInfo(self.target) + + def validate_runtime_parameters(self, params): + pass + + def merge_runtime_parameters(self, params): + pass + + +def init_target_manager(config): + return FakeTargetManager(config.device, config.device_config) + + class Executor(object): """ The ``Executor``'s job is to set up the execution context and pass to a @@ -274,8 +297,7 @@ class Executor(object): output.write_config(config) self.logger.info('Connecting to target') - target_manager = TargetManager(config.run_config.device, - config.run_config.device_config) + target_manager = init_target_manager(config.run_config) output.write_target_info(target_manager.get_target_info()) self.logger.info('Initializing execution conetext') diff --git a/wa/framework/output.py b/wa/framework/output.py index 07912bb4..77d5853e 100644 --- a/wa/framework/output.py +++ b/wa/framework/output.py @@ -6,11 +6,11 @@ import sys import uuid from copy import copy -from wa.framework.configuration.core import JobSpec -from wa.framework.configuration.manager import ConfigManager -from wa.framework.target.info import TargetInfo -from wa.utils.misc import touch -from wa.utils.serializer import write_pod, read_pod +from wlauto.core.configuration.configuration import JobSpec +from wlauto.core.configuration.manager import ConfigManager +from wlauto.core.device_manager import TargetInfo +from wlauto.utils.misc import touch +from wlauto.utils.serializer import write_pod, read_pod logger = logging.getLogger('output') diff --git a/wa/framework/target/descriptor.py b/wa/framework/target/descriptor.py index c1252289..34966367 100644 --- a/wa/framework/target/descriptor.py +++ b/wa/framework/target/descriptor.py @@ -24,35 +24,6 @@ def get_target_descriptions(loader=pluginloader): return targets.values() -def instantiate_target(tdesc, params, connect=None): - target_params = {p.name: p for p in tdesc.target_params} - platform_params = {p.name: p for p in tdesc.platform_params} - conn_params = {p.name: p for p in tdesc.conn_params} - - tp, pp, cp = {}, {}, {} - - for name, value in params.iteritems(): - if name in target_params: - tp[name] = value - elif name in platform_params: - pp[name] = value - elif name in conn_params: - cp[name] = value - else: - msg = 'Unexpected parameter for {}: {}' - raise ValueError(msg.format(tdesc.name, name)) - - tp['platform'] = (tdesc.platform or Platform)(**pp) - if cp: - tp['connection_settings'] = cp - if tdesc.connection: - tp['conn_cls'] = tdesc.connection - if connect is not None: - tp['connect'] = connect - - return tdesc.target(**tp) - - class TargetDescription(object): def __init__(self, name, source, description=None, target=None, platform=None, @@ -115,18 +86,6 @@ COMMON_TARGET_PARAMS = [ Please see ``devlab`` documentation for information on the available modules. '''), - Parameter('load_default_modules', kind=bool, default=True, - description=''' - A number of modules (e.g. for working with the cpufreq subsystem) are - loaded by default when a Target is instantiated. Setting this to - ``True`` would suppress that, ensuring that only the base Target - interface is initialized. - - You may want to set this if there is a problem with one or more default - modules on your platform (e.g. your device is unrooted and cpufreq is - not accessible to unprivileged users), or if Target initialization is - taking too long for your platform. - '''), ] COMMON_PLATFORM_PARAMS = [ diff --git a/wa/framework/target/info.py b/wa/framework/target/info.py index f3e40119..4341e155 100644 --- a/wa/framework/target/info.py +++ b/wa/framework/target/info.py @@ -1,7 +1,6 @@ from devlib import AndroidTarget from devlib.exception import TargetError from devlib.target import KernelConfig, KernelVersion, Cpuinfo -from devlib.utils.android import AndroidProperties class TargetInfo(object): @@ -22,9 +21,8 @@ class TargetInfo(object): if pod["target"] == "AndroidTarget": instance.screen_resolution = pod['screen_resolution'] - instance.prop = AndroidProperties('') - instance.prop._properties = pod['prop'] - instance.android_id = pod['android_id'] + instance.prop = pod['prop'] + instance.prop = pod['android_id'] return instance @@ -74,7 +72,7 @@ class TargetInfo(object): if self.target == "AndroidTarget": pod['screen_resolution'] = self.screen_resolution - pod['prop'] = self.prop._properties + pod['prop'] = self.prop pod['android_id'] = self.android_id return pod diff --git a/wa/framework/target/manager.py b/wa/framework/target/manager.py index 545178e6..659516d6 100644 --- a/wa/framework/target/manager.py +++ b/wa/framework/target/manager.py @@ -9,8 +9,6 @@ import sys from wa.framework import signal from wa.framework.exception import WorkerThreadError, ConfigError from wa.framework.plugin import Parameter -from wa.framework.target.descriptor import (get_target_descriptions, - instantiate_target) from wa.framework.target.info import TargetInfo from wa.framework.target.runtime_config import (SysfileValuesRuntimeConfig, HotplugRuntimeConfig, @@ -43,26 +41,54 @@ class TargetManager(object): """), ] + DEVICE_MAPPING = {'test' : {'platform_name':'generic', + 'target_name': 'android'}, + 'other': {'platform_name':'test', + 'target_name': 'linux'}, + } + runtime_config_cls = [ - # order matters - SysfileValuesRuntimeConfig, - HotplugRuntimeConfig, - CpufreqRuntimeConfig, - CpuidleRuntimeConfig, - ] + # order matters + SysfileValuesRuntimeConfig, + HotplugRuntimeConfig, + CpufreqRuntimeConfig, + CpuidleRuntimeConfig, + ] def __init__(self, name, parameters): - self.target_name = name + self.name = name self.target = None self.assistant = None + self.target_name = None self.platform_name = None self.parameters = parameters self.disconnect = parameters.get('disconnect') self.info = TargetInfo() - self._init_target() - self._init_assistant() + # Determine platform and target based on passed name + self._parse_name() + # Create target + self._get_target() + # Create an assistant to perform target specific configuration + self._get_assistant() + + ### HERE FOR TESTING, WILL BE CALLED EXTERNALLY ### + # Connect to device and retrieve details. + # self.initialize() + # self.add_parameters() + # self.validate_parameters() + # self.set_parameters() + + def initialize(self): self.runtime_configs = [cls(self.target) for cls in self.runtime_config_cls] + # if self.parameters: + # self.logger.info('Connecting to the device') + with signal.wrap('TARGET_CONNECT'): + self.target.connect() + # self.info.load(self.target) + # info_file = os.path.join(self.context.info_directory, 'target.json') + # with open(info_file, 'w') as wfh: + # json.dump(self.info.to_pod(), wfh) def finalize(self): # self.logger.info('Disconnecting from the device') @@ -82,16 +108,10 @@ class TargetManager(object): if any(parameter in name for parameter in cfg.supported_parameters): cfg.add(name, self.parameters.pop(name)) - def get_target_info(self): - return TargetInfo(self.target) - - def validate_runtime_parameters(self, params): + def validate_parameters(self): for cfg in self.runtime_configs: cfg.validate() - def merge_runtime_parameters(self, params): - pass - def set_parameters(self): for cfg in self.runtime_configs: cfg.set() @@ -100,25 +120,47 @@ class TargetManager(object): for cfg in self.runtime_configs: cfg.clear() - def _init_target(self): - target_map = {td.name: td for td in get_target_descriptions()} - if self.target_name not in target_map: - raise ValueError('Unknown Target: {}'.format(self.target_name)) - tdesc = target_map[self.target_name] - self.target = instantiate_target(tdesc, self.parameters, connect=False) - with signal.wrap('TARGET_CONNECT'): - self.target.connect() - self.target.setup() + def _parse_name(self): + # Try and get platform and target + self.name = identifier(self.name.replace('-', '_')) + if '_' in self.name: + self.platform_name, self.target_name = self.name.split('_', 1) + elif self.name in self.DEVICE_MAPPING: + self.platform_name = self.DEVICE_MAPPING[self.name]['platform_name'] + self.target_name = self.DEVICE_MAPPING[self.name]['target_name'] + else: + raise ConfigError('Unknown Device Specified {}'.format(self.name)) - def _init_assistant(self): - # Create a corresponding target and target-assistant to help with - # platformy stuff? - if self.target.os == 'android': + def _get_target(self): + # Create a corresponding target and target-assistant + if self.target_name == 'android': + self.target = AndroidTarget() + elif self.target_name == 'linux': + self.target = LinuxTarget() # pylint: disable=redefined-variable-type + elif self.target_name == 'localLinux': + self.target = LocalLinuxTarget() + else: + raise ConfigError('Unknown Target Specified {}'.format(self.target_name)) + + def _get_assistant(self): + # Create a corresponding target and target-assistant to help with platformy stuff? + if self.target_name == 'android': self.assistant = AndroidAssistant(self.target) - elif self.target.os == 'linux': + elif self.target_name in ['linux', 'localLinux']: self.assistant = LinuxAssistant(self.target) # pylint: disable=redefined-variable-type else: - raise ValueError('Unknown Target OS: {}'.format(self.target.os)) + raise ConfigError('Unknown Target Specified {}'.format(self.target_name)) + + # def validate_runtime_parameters(self, parameters): + # for name, value in parameters.iteritems(): + # self.add_parameter(name, value) + # self.validate_parameters() + + # def set_runtime_parameters(self, parameters): + # # self.clear() + # for name, value in parameters.iteritems(): + # self.add_parameter(name, value) + # self.set_parameters() class LinuxAssistant(object): diff --git a/wa/utils/log.py b/wa/utils/log.py index 8dbe5f20..567943e5 100644 --- a/wa/utils/log.py +++ b/wa/utils/log.py @@ -78,7 +78,7 @@ def set_level(level): def add_file(filepath, level=logging.DEBUG, - fmt='%(asctime)s %(levelname)-8s %(name)s: %(message)-10.10s'): + fmt='%(asctime)s %(levelname)-8s %(name)10.10s: %(message)s'): root_logger = logging.getLogger() file_handler = logging.FileHandler(filepath) file_handler.setLevel(level)