DeviceManager: Introduced DeviceManager extension

DeviceManagers will replace devices and will wrap devlib targets for use in WA

setup.py

@@ -76,7 +76,8 @@ params = dict(
         'pyserial',  # Serial port interface
         'colorama',  # Printing with colors
         'pyYAML',  # YAML-formatted agenda parsing
-        'requests', # Fetch assets over HTTP
+        'requests',  # Fetch assets over HTTP
+        'devlib',  # Interacting with devices
     ],
     extras_require={
         'other': ['jinja2', 'pandas>=0.13.1'],

wlauto/__init__.py

@@ -14,7 +14,8 @@
 #
 
 from wlauto.core.bootstrap import settings  # NOQA
-from wlauto.core.device import Device, RuntimeParameter, CoreParameter  # NOQA
+from wlauto.core.device import Device  # NOQA
+from wlauto.core.device_manager import DeviceManager, RuntimeParameter, CoreParameter  # NOQA
 from wlauto.core.command import Command  # NOQA
 from wlauto.core.workload import Workload  # NOQA
 from wlauto.core.extension import Module, Parameter, Artifact, Alias  # NOQA

wlauto/core/bootstrap.py

@@ -55,7 +55,8 @@ sys.path.insert(0, os.path.join(_this_dir, '..', 'external'))
 _EXTENSION_TYPE_TABLE = [
     # name,               class,                                    default package,            default path
     ('command',           'wlauto.core.command.Command',            'wlauto.commands',          'commands'),
-    ('device',            'wlauto.core.device.Device',              'wlauto.devices',           'devices'),
+#    ('device',            'wlauto.core.device.Device',              'wlauto.devices',           'devices'),
+    ('device_manager',    'wlauto.core.device.DeviceManager',       'wlauto.managers',          'devices'),
     ('instrument',        'wlauto.core.instrumentation.Instrument', 'wlauto.instrumentation',   'instruments'),
     ('module',            'wlauto.core.extension.Module',           'wlauto.modules',           'modules'),
     ('resource_getter',   'wlauto.core.resource.ResourceGetter',    'wlauto.resource_getters',  'resource_getters'),
@@ -211,4 +212,3 @@ if os.path.isfile(_packages_file):
 
 for config in _env_configs:
     settings.update(config)
-

wlauto/core/device_manager.py

@@ -0,0 +1,318 @@
+import string
+from collections import OrderedDict
+
+from wlauto.core.extension import Extension, Parameter
+from wlauto.exceptions import ConfigError
+from wlauto.utils.types import list_of_integers, list_of, caseless_string
+
+from devlib.platform import Platform
+from devlib.target import AndroidTarget, Cpuinfo, KernelVersion, KernelConfig
+
+__all__ = ['RuntimeParameter', 'CoreParameter', 'DeviceManager', 'TargetInfo']
+
+
+class RuntimeParameter(object):
+    """
+    A runtime parameter which has its getter and setter methods associated it
+    with it.
+
+    """
+
+    def __init__(self, name, getter, setter,
+                 getter_args=None, setter_args=None,
+                 value_name='value', override=False):
+        """
+        :param name: the name of the parameter.
+        :param getter: the getter method which returns the value of this parameter.
+        :param setter: the setter method which sets the value of this parameter. The setter
+                       always expects to be passed one argument when it is called.
+        :param getter_args: keyword arguments to be used when invoking the getter.
+        :param setter_args: keyword arguments to be used when invoking the setter.
+        :param override: A ``bool`` that specifies whether a parameter of the same name further up the
+                            hierarchy should be overridden. If this is ``False`` (the default), an exception
+                            will be raised by the ``AttributeCollection`` instead.
+
+        """
+        self.name = name
+        self.getter = getter
+        self.setter = setter
+        self.getter_args = getter_args or {}
+        self.setter_args = setter_args or {}
+        self.value_name = value_name
+        self.override = override
+
+    def __str__(self):
+        return self.name
+
+    __repr__ = __str__
+
+
+class CoreParameter(RuntimeParameter):
+    """A runtime parameter that will get expanded into a RuntimeParameter for each core type."""
+
+    def get_runtime_parameters(self, core_names):
+        params = []
+        for core in set(core_names):
+            name = string.Template(self.name).substitute(core=core)
+            getter = string.Template(self.getter).substitute(core=core)
+            setter = string.Template(self.setter).substitute(core=core)
+            getargs = dict(self.getter_args.items() + [('core', core)])
+            setargs = dict(self.setter_args.items() + [('core', core)])
+            params.append(RuntimeParameter(name, getter, setter, getargs, setargs, self.value_name, self.override))
+        return params
+
+
+class TargetInfo(object):
+
+    @staticmethod
+    def from_pod(pod):
+        instance = TargetInfo()
+        instance.target = pod['target']
+        instance.abi = pod['abi']
+        instance.cpuinfo = Cpuinfo(pod['cpuinfo'])
+        instance.os = pod['os']
+        instance.os_version = pod['os_version']
+        instance.abi = pod['abi']
+        instance.is_rooted = pod['is_rooted']
+        instance.kernel_version = KernelVersion(pod['kernel_version'])
+        instance.kernel_config = KernelConfig(pod['kernel_config'])
+
+        if pod["target"] == "AndroidTarget":
+            instance.screen_resolution = pod['screen_resolution']
+            instance.prop = pod['prop']
+            instance.prop = pod['android_id']
+
+        return instance
+
+    def __init__(self, target=None):
+        if target:
+            self.target = target.__class__.__name__
+            self.cpuinfo = target.cpuinfo
+            self.os = target.os
+            self.os_version = target.os_version
+            self.abi = target.abi
+            self.is_rooted = target.is_rooted
+            self.kernel_version = target.kernel_version
+            self.kernel_config = target.config
+
+            if isinstance(target, AndroidTarget):
+                self.screen_resolution = target.screen_resolution
+                self.prop = target.getprop()
+                self.android_id = target.android_id
+
+        else:
+            self.target = None
+            self.cpuinfo = None
+            self.os = None
+            self.os_version = None
+            self.abi = None
+            self.is_rooted = None
+            self.kernel_version = None
+            self.kernel_config = None
+
+            if isinstance(target, AndroidTarget):
+                self.screen_resolution = None
+                self.prop = None
+                self.android_id = None
+
+    def to_pod(self):
+        pod = {}
+        pod['target'] = self.target.__class__.__name__
+        pod['abi'] = self.abi
+        pod['cpuinfo'] = self.cpuinfo.text
+        pod['os'] = self.os
+        pod['os_version'] = self.os_version
+        pod['abi'] = self.abi
+        pod['is_rooted'] = self.is_rooted
+        pod['kernel_version'] = self.kernel_version.version
+        pod['kernel_config'] = self.kernel_config.text
+
+        if self.target == "AndroidTarget":
+            pod['screen_resolution'] = self.screen_resolution
+            pod['prop'] = self.prop
+            pod['android_id'] = self.android_id
+
+        return pod
+
+
+class DeviceManager(Extension):
+
+    name = None
+    target_type = None
+    platform_type = Platform
+    has_gpu = None
+    path_module = None
+    info = None
+
+    parameters = [
+        Parameter('core_names', kind=list_of(caseless_string),
+                  description="""
+                  This is a list of all cpu cores on the device with each
+                  element being the core type, e.g. ``['a7', 'a7', 'a15']``. The
+                  order of the cores must match the order they are listed in
+                  ``'/sys/devices/system/cpu'``. So in this case, ``'cpu0'`` must
+                  be an A7 core, and ``'cpu2'`` an A15.'
+                  """),
+        Parameter('core_clusters', kind=list_of_integers,
+                  description="""
+                  This is a list indicating the cluster affinity of the CPU cores,
+                  each element correponding to the cluster ID of the core coresponding
+                  to its index. E.g. ``[0, 0, 1]`` indicates that cpu0 and cpu1 are on
+                  cluster 0, while cpu2 is on cluster 1. If this is not specified, this
+                  will be inferred from ``core_names`` if possible (assuming all cores with
+                  the same name are on the same cluster).
+                  """),
+        Parameter('working_directory',
+                  description='''
+                  Working directory to be used by WA. This must be in a location where the specified user
+                  has write permissions. This will default to /home/<username>/wa (or to /root/wa, if
+                  username is 'root').
+                  '''),
+        Parameter('binaries_directory',
+                  description='Location of executable binaries on this device (must be in PATH).'),
+    ]
+    modules = []
+
+    runtime_parameters = [
+        RuntimeParameter('sysfile_values', 'get_sysfile_values', 'set_sysfile_values', value_name='params'),
+        CoreParameter('${core}_cores', 'get_number_of_online_cpus', 'set_number_of_online_cpus',
+                      value_name='number'),
+        CoreParameter('${core}_min_frequency', 'get_core_min_frequency', 'set_core_min_frequency',
+                      value_name='freq'),
+        CoreParameter('${core}_max_frequency', 'get_core_max_frequency', 'set_core_max_frequency',
+                      value_name='freq'),
+        CoreParameter('${core}_frequency', 'get_core_cur_frequency', 'set_core_cur_frequency',
+                      value_name='freq'),
+        CoreParameter('${core}_governor', 'get_core_governor', 'set_core_governor',
+                      value_name='governor'),
+        CoreParameter('${core}_governor_tunables', 'get_core_governor_tunables', 'set_core_governor_tunables',
+                      value_name='tunables'),
+    ]
+
+    # Framework
+
+    def connect(self):
+        raise NotImplementedError("connect method must be implemented for device managers")
+
+    def initialize(self, context):
+        super(DeviceManager, self).initialize(context)
+        self.info = TargetInfo(self.target)
+        self.target.setup()
+
+    def start(self):
+        pass
+
+    def stop(self):
+        pass
+
+    def validate(self):
+        pass
+
+    # Runtime Parameters
+
+    def get_runtime_parameter_names(self):
+        return [p.name for p in self._expand_runtime_parameters()]
+
+    def get_runtime_parameters(self):
+        """ returns the runtime parameters that have been set. """
+        # pylint: disable=cell-var-from-loop
+        runtime_parameters = OrderedDict()
+        for rtp in self._expand_runtime_parameters():
+            if not rtp.getter:
+                continue
+            getter = getattr(self, rtp.getter)
+            rtp_value = getter(**rtp.getter_args)
+            runtime_parameters[rtp.name] = rtp_value
+        return runtime_parameters
+
+    def set_runtime_parameters(self, params):
+        """
+        The parameters are taken from the keyword arguments and are specific to
+        a particular device. See the device documentation.
+
+        """
+        runtime_parameters = self._expand_runtime_parameters()
+        rtp_map = {rtp.name.lower(): rtp for rtp in runtime_parameters}
+
+        params = OrderedDict((k.lower(), v) for k, v in params.iteritems() if v is not None)
+
+        expected_keys = rtp_map.keys()
+        if not set(params.keys()).issubset(set(expected_keys)):
+            unknown_params = list(set(params.keys()).difference(set(expected_keys)))
+            raise ConfigError('Unknown runtime parameter(s): {}'.format(unknown_params))
+
+        for param in params:
+            self.logger.debug('Setting runtime parameter "{}"'.format(param))
+            rtp = rtp_map[param]
+            setter = getattr(self, rtp.setter)
+            args = dict(rtp.setter_args.items() + [(rtp.value_name, params[rtp.name.lower()])])
+            setter(**args)
+
+    def _expand_runtime_parameters(self):
+        expanded_params = []
+        for param in self.runtime_parameters:
+            if isinstance(param, CoreParameter):
+                expanded_params.extend(param.get_runtime_parameters(self.target.core_names))  # pylint: disable=no-member
+            else:
+                expanded_params.append(param)
+        return expanded_params
+
+    #Runtime parameter getters/setters
+
+    _written_sysfiles = []
+
+    def get_sysfile_values(self):
+        return self._written_sysfiles
+
+    def set_sysfile_values(self, params):
+        for sysfile, value in params.iteritems():
+            verify = not sysfile.endswith('!')
+            sysfile = sysfile.rstrip('!')
+            self._written_sysfiles.append((sysfile, value))
+            self.target.write_value(sysfile, value, verify=verify)
+
+    # pylint: disable=E1101
+
+    def _get_core_online_cpu(self, core):
+        try:
+            return self.target.list_online_core_cpus(core)[0]
+        except IndexError:
+            raise ValueError("No {} cores are online".format(core))
+
+    def get_number_of_online_cpus(self, core):
+        return len(self._get_core_online_cpu(core))
+
+    def set_number_of_online_cpus(self, core, number):
+        for cpu in self.target.core_cpus(core)[:number]:
+            self.target.hotplug.online(cpu)
+
+    def get_core_min_frequency(self, core):
+        return self.target.cpufreq.get_min_frequency(self._get_core_online_cpu(core))
+
+    def set_core_min_frequency(self, core, frequency):
+        self.target.cpufreq.set_min_frequency(self._get_core_online_cpu(core), frequency)
+
+    def get_core_max_frequency(self, core):
+        return self.target.cpufreq.get_max_frequency(self._get_core_online_cpu(core))
+
+    def set_core_max_frequency(self, core, frequency):
+        self.target.cpufreq.set_max_frequency(self._get_core_online_cpu(core), frequency)
+
+    def get_core_frequency(self, core):
+        return self.target.cpufreq.get_frequency(self._get_core_online_cpu(core))
+
+    def set_core_frequency(self, core, frequency):
+        self.target.cpufreq.set_frequency(self._get_core_online_cpu(core), frequency)
+
+    def get_core_governor(self, core):
+        return self.target.cpufreq.get_cpu_governor(self._get_core_online_cpu(core))
+
+    def set_core_governor(self, core, governor):
+        self.target.cpufreq.set_cpu_governor(self._get_core_online_cpu(core), governor)
+
+    def get_core_governor_tunables(self, core):
+        return self.target.cpufreq.get_governor_tunables(self._get_core_online_cpu(core))
+
+    def set_core_governor_tunables(self, core, tunables):
+        self.target.cpufreq.set_governor_tunables(self._get_core_online_cpu(core),
+                                                  *tunables)

wlauto/core/execution.py

@@ -142,8 +142,9 @@ class ExecutionContext(object):
     def result(self):
         return getattr(self.current_job, 'result', self.run_result)
 
-    def __init__(self, device, config):
+    def __init__(self, device_manager, config):
-        self.device = device
+        self.device_manager = device_manager
+        self.device = self.device_manager.target
         self.config = config
         self.reboot_policy = config.reboot_policy
         self.output_directory = None
@@ -258,6 +259,7 @@ class Executor(object):
         self.warning_logged = False
         self.config = None
         self.ext_loader = None
+        self.device_manager = None
         self.device = None
         self.context = None
 
@@ -301,10 +303,11 @@ class Executor(object):
         self.logger.debug('Initialising device configuration.')
         if not self.config.device:
             raise ConfigError('Make sure a device is specified in the config.')
-        self.device = self.ext_loader.get_device(self.config.device, **self.config.device_config)
+        self.device_manager = self.ext_loader.get_device_manager(self.config.device, **self.config.device_config)
-        self.device.validate()
+        self.device_manager.validate()
+        self.device = self.device_manager.target
 
-        self.context = ExecutionContext(self.device, self.config)
+        self.context = ExecutionContext(self.device_manager, self.config)
 
         self.logger.debug('Loading resource discoverers.')
         self.context.initialize()
@@ -384,7 +387,7 @@ class Executor(object):
             runnercls = RandomRunner
         else:
             raise ConfigError('Unexpected execution order: {}'.format(self.config.execution_order))
-        return runnercls(self.device, self.context, result_manager)
+        return runnercls(self.device_manager, self.context, result_manager)
 
     def _error_signalled_callback(self):
         self.error_logged = True
@@ -464,8 +467,9 @@ class Runner(object):
             return True
         return self.current_job.spec.id != self.next_job.spec.id
 
-    def __init__(self, device, context, result_manager):
+    def __init__(self, device_manager, context, result_manager):
-        self.device = device
+        self.device_manager = device_manager
+        self.device = device_manager.target
         self.context = context
         self.result_manager = result_manager
         self.logger = logging.getLogger('Runner')
@@ -533,14 +537,13 @@ class Runner(object):
         self.context.run_info.start_time = datetime.utcnow()
         self._connect_to_device()
         self.logger.info('Initializing device')
-        self.device.initialize(self.context)
+        self.device_manager.initialize(self.context)
 
         self.logger.info('Initializing workloads')
         for workload_spec in self.context.config.workload_specs:
             workload_spec.workload.initialize(self.context)
 
-        props = self.device.get_properties(self.context)
+        self.context.run_info.device_properties = self.device_manager.info
-        self.context.run_info.device_properties = props
         self.result_manager.initialize(self.context)
         self._send(signal.RUN_INIT)
 
@@ -550,7 +553,7 @@ class Runner(object):
     def _connect_to_device(self):
         if self.context.reboot_policy.perform_initial_boot:
             try:
-                self.device.connect()
+                self.device_manager.connect()
             except DeviceError:  # device may be offline
                 if self.device.can('reset_power'):
                     with self._signal_wrap('INITIAL_BOOT'):
@@ -564,7 +567,7 @@ class Runner(object):
                     self._reboot_device()
         else:
             self.logger.info('Connecting to device')
-            self.device.connect()
+            self.device_manager.connect()
 
     def _init_job(self):
         self.current_job.result.status = IterationResult.RUNNING
@@ -597,7 +600,7 @@ class Runner(object):
 
         instrumentation.disable_all()
         instrumentation.enable(spec.instrumentation)
-        self.device.start()
+        self.device_manager.start()
 
         if self.spec_changed:
             self._send(signal.WORKLOAD_SPEC_START)
@@ -606,7 +609,7 @@ class Runner(object):
         try:
             setup_ok = False
             with self._handle_errors('Setting up device parameters'):
-                self.device.set_runtime_parameters(spec.runtime_parameters)
+                self.device_manager.set_runtime_parameters(spec.runtime_parameters)
                 setup_ok = True
 
             if setup_ok:
@@ -625,7 +628,7 @@ class Runner(object):
             if self.spec_will_change or not spec.enabled:
                 self._send(signal.WORKLOAD_SPEC_END)
         finally:
-            self.device.stop()
+            self.device_manager.stop()
 
     def _finalize_job(self):
         self.context.run_result.iteration_results.append(self.current_job.result)
@@ -737,7 +740,7 @@ class Runner(object):
         except (KeyboardInterrupt, DeviceNotRespondingError):
             raise
         except (WAError, TimeoutError), we:
-            self.device.ping()
+            self.device.check_responsive()
             if self.current_job:
                 self.current_job.result.status = on_error_status
                 self.current_job.result.add_event(str(we))