Merge pull request #373 from marcbonnici/TM

TM: Update and Refactor

wa/framework/job.py

@@ -55,6 +55,7 @@ class Job(object):
 
     def configure_target(self, context):
         self.logger.info('Configuring target for job {}'.format(self.id))
+        context.tm.commit_runtime_parameters(self.spec.runtime_parameters)
 
     def setup(self, context):
         self.logger.info('Setting up job {}'.format(self.id))
@@ -83,4 +84,3 @@ class Job(object):
         self.logger.info('Finalizing job {}'.format(self.id))
         with signal.wrap('WORKLOAD_FINALIZED', self, context):
             self.workload.finalize(context)
-

wa/framework/target/assistant.py

@@ -8,6 +8,7 @@ import time
 
 from wa import Parameter
 from wa.framework.exception import WorkerThreadError
+from wa.utils.misc import touch
 
 
 class LinuxAssistant(object):
@@ -121,8 +122,7 @@ class LogcatPoller(threading.Thread):
         self.logger.debug('clearing logcat buffer')
         with self.lock:
             self.target.clear_logcat()
-            with open(self.buffer_file, 'w') as _:  # NOQA
+            touch(self.buffer_file)
-                pass
 
     def write_log(self, outfile):
         with self.lock:
@@ -130,8 +130,7 @@ class LogcatPoller(threading.Thread):
             if os.path.isfile(self.buffer_file):
                 shutil.copy(self.buffer_file, outfile)
             else:  # there was no logcat trace at this time
-                with open(outfile, 'w') as _:  # NOQA
+                touch(outfile)
-                    pass
 
     def close(self):
         self.logger.debug('closing poller')

wa/framework/target/manager.py

@@ -1,39 +1,19 @@
 import logging
-import tempfile
-import threading
-import os
-import time
-import shutil
-import sys
 
 from wa.framework import signal
-from wa.framework.exception import WorkerThreadError, ConfigError
 from wa.framework.plugin import Parameter
 from wa.framework.target.descriptor import (get_target_descriptions,
                                             instantiate_target,
                                             instantiate_assistant)
 from wa.framework.target.info import TargetInfo
-from wa.framework.target.runtime_config import (SysfileValuesRuntimeConfig,
+from wa.framework.target.runtime_parameter_manager import RuntimeParameterManager
-                                                HotplugRuntimeConfig,
-                                                CpufreqRuntimeConfig,
-                                                CpuidleRuntimeConfig)
-from wa.utils.misc import isiterable
-from wa.utils.serializer import json
 
-
-from devlib import LocalLinuxTarget, LinuxTarget, AndroidTarget
-from devlib.utils.types import identifier
 from devlib.utils.misc import memoized
 
 
 class TargetManager(object):
-
+    """
-    name = 'target-manager'
+    Instantiate the required target and perform configuration and validation of the device.
-
-    description = """
-    Instanciated the required target and performs configuration and validation
-    of the device.
-
     """
 
     parameters = [
@@ -44,14 +24,6 @@ class TargetManager(object):
                   """),
     ]
 
-    runtime_config_cls = [
-        # order matters
-        SysfileValuesRuntimeConfig,
-        HotplugRuntimeConfig,
-        CpufreqRuntimeConfig,
-        CpuidleRuntimeConfig,
-    ]
-
     def __init__(self, name, parameters):
         self.logger = logging.getLogger('tm')
         self.target_name = name
@@ -60,29 +32,26 @@ class TargetManager(object):
         self.platform_name = None
         self.parameters = parameters
         self.disconnect = parameters.get('disconnect')
-        self.info = TargetInfo()
 
         self._init_target()
-        self.runtime_configs = [cls(self.target) for cls in self.runtime_config_cls]
+
+        # If target supports hotplugging, online all cpus before perform discovery
+        # and restore orignal configuration after completed.
+        if self.target.has('hotplug'):
+            online_cpus = self.target.list_online_cpus()
+            self.target.hotplug.online_all()
+            self.rpm = RuntimeParameterManager(self.target)
+            all_cpus = set(range(self.target.number_of_cpus))
+            self.target.hotplug.offline(*all_cpus.difference(online_cpus))
+        else:
+            self.rpm = RuntimeParameterManager(self.target)
 
     def finalize(self):
-        # self.logger.info('Disconnecting from the device')
+        self.logger.info('Disconnecting from the device')
         if self.disconnect:
             with signal.wrap('TARGET_DISCONNECT'):
                 self.target.disconnect()
 
-    def add_parameters(self, parameters=None):
-        if parameters:
-            self.parameters = parameters
-        if not self.parameters:
-            raise ConfigError('No Configuration Provided')
-
-        for name in self.parameters.keys():
-            for cfg in self.runtime_configs:
-                # if name in cfg.supported_parameters:
-                if any(parameter in name for parameter in cfg.supported_parameters):
-                    cfg.add(name, self.parameters.pop(name))
-
     def start(self):
         self.assistant.start()
 
@@ -96,27 +65,21 @@ class TargetManager(object):
     def get_target_info(self):
         return TargetInfo(self.target)
 
-    def validate_runtime_parameters(self, params):
+    def merge_runtime_parameters(self, parameters):
-        for cfg in self.runtime_configs:
+        return self.rpm.merge_runtime_parameters(parameters)
-            cfg.validate()
 
-    def merge_runtime_parameters(self, params):
+    def validate_runtime_parameters(self, parameters):
-        pass
+        self.rpm.validate_runtime_parameters(parameters)
 
-    def set_parameters(self):
+    def commit_runtime_parameters(self, parameters):
-        for cfg in self.runtime_configs:
+        self.rpm.commit_runtime_parameters(parameters)
-            cfg.set()
-
-    def clear_parameters(self):
-        for cfg in self.runtime_configs:
-            cfg.clear()
 
     def _init_target(self):
         target_map = {td.name: td for td in get_target_descriptions()}
         if self.target_name not in target_map:
             raise ValueError('Unknown Target: {}'.format(self.target_name))
         tdesc = target_map[self.target_name]
-
+        self.logger.debug('Creating {} target'.format(self.target_name))
         self.target = instantiate_target(tdesc, self.parameters, connect=False)
 
         with signal.wrap('TARGET_CONNECT'):

wa/framework/target/runtime_parameter_manager.py

@@ -0,0 +1,83 @@
+import re
+from collections import namedtuple
+
+from wa.framework.exception import ConfigError
+from wa.framework.target.runtime_config import (SysfileValuesRuntimeConfig,
+                                                HotplugRuntimeConfig,
+                                                CpufreqRuntimeConfig,
+                                                CpuidleRuntimeConfig)
+from wa.utils.types import obj_dict
+
+
+class RuntimeParameterManager(object):
+
+    runtime_config_cls = [
+        # order matters
+        SysfileValuesRuntimeConfig,
+        HotplugRuntimeConfig,
+        CpufreqRuntimeConfig,
+        CpuidleRuntimeConfig,
+    ]
+
+    def __init__(self, target):
+        self.target = target
+        self.runtime_configs = [cls(self.target) for cls in self.runtime_config_cls]
+        self.runtime_params = {}
+
+        runtime_parameter = namedtuple('RuntimeParameter', 'cfg_point, rt_config')
+        for cfg in self.runtime_configs:
+            for param in cfg.supported_parameters:
+                if param.name in self.runtime_params:
+                    msg = 'Duplicate runtime parameter name "{}": in both {} and {}'
+                    raise RuntimeError(msg.format(param.name,
+                                                  self.runtime_params[param.name].rt_config.name,
+                                                  cfg.name))
+                self.runtime_params[param.name] = runtime_parameter(param, cfg)
+
+    # Uses corresponding config point to merge parameters
+    def merge_runtime_parameters(self, parameters):
+        merged_params = obj_dict()
+        for source in parameters:
+            for name, value in parameters[source].iteritems():
+                cp = self.get_cfg_point(name)
+                cp.set_value(merged_params, value)
+        return dict(merged_params)
+
+    # Validates runtime_parameters against each other
+    def validate_runtime_parameters(self, parameters):
+        self.clear_runtime_parameters()
+        self.set_runtime_parameters(parameters)
+        for cfg in self.runtime_configs:
+            cfg.validate_parameters()
+
+    # Writes the given parameters to the device.
+    def commit_runtime_parameters(self, parameters):
+        self.clear_runtime_parameters()
+        self.set_runtime_parameters(parameters)
+        for cfg in self.runtime_configs:
+            cfg.commit()
+
+    # Stores a set of parameters performing isolated validation when appropriate
+    def set_runtime_parameters(self, parameters):
+        for name, value in parameters.iteritems():
+            cfg = self.get_config_for_name(name)
+            if cfg is None:
+                msg = 'Unsupported runtime parameter: "{}"'
+                raise ConfigError(msg.format(name))
+            cfg.set_runtime_parameter(name, value)
+
+    def clear_runtime_parameters(self):
+        for cfg in self.runtime_configs:
+            cfg.clear()
+
+    def get_config_for_name(self, name):
+        for rp_name, rp in self.runtime_params.iteritems():
+            if re.search(name, rp_name):
+                return rp.rt_config
+        return None
+
+    def get_cfg_point(self, name):
+        for rp_name, rp in self.runtime_params.iteritems():
+            if re.search(name, rp_name, re.IGNORECASE):
+                return rp.cfg_point
+        raise ConfigError('Unknown Runtime Parameter: {}'.format(name))

wa/tests/test_runtime_param_utils.py

@@ -0,0 +1,50 @@
+import unittest
+from nose.tools import assert_equal
+from mock.mock import Mock
+
+from wa.utils.misc import resolve_cpus, resolve_unique_domain_cpus
+
+class TestRuntimeParameterUtils(unittest.TestCase):
+
+    def test_resolve_cpu(self):
+        # Set up a mock target
+        mock = Mock()
+        mock.big_core = "A72"
+        mock.little_core = "A53"
+        mock.core_names = ['A72', 'A72', 'A53', 'A53']
+        mock.number_of_cpus = 4
+        def mock_core_cpus(core):
+            return [i for i, c in enumerate(mock.core_names) if c == core]
+        def mock_online_cpus():
+            return [0, 1, 2]
+        def mock_offline_cpus():
+            return [3]
+        def mock_domain_cpus(core):
+            if core in [0, 1]:
+                return [0, 1]
+            elif core in [2, 3]:
+                return [2, 3]
+
+        mock.list_online_cpus = mock_online_cpus
+        mock.list_offline_cpus = mock_offline_cpus
+        mock.core_cpus = mock_core_cpus
+        mock.core_cpus = mock_core_cpus
+        mock.cpufreq.get_domain_cpus = mock_domain_cpus
+
+        # Check retrieving cpus from a given prefix
+        assert_equal(resolve_cpus('A72', mock), [0, 1])
+        assert_equal(resolve_cpus('A53', mock), [2, 3])
+        assert_equal(resolve_cpus('big', mock), [0, 1])
+        assert_equal(resolve_cpus('little', mock), [2, 3])
+        assert_equal(resolve_cpus('', mock), [0, 1, 2, 3])
+        assert_equal(resolve_cpus('cpu0', mock), [0])
+        assert_equal(resolve_cpus('cpu3', mock), [3])
+
+        # Check get unique domain cpus
+        assert_equal(resolve_unique_domain_cpus('A72', mock), [0])
+        assert_equal(resolve_unique_domain_cpus('A53', mock), [2])
+        assert_equal(resolve_unique_domain_cpus('big', mock), [0])
+        assert_equal(resolve_unique_domain_cpus('little', mock), [2])
+        assert_equal(resolve_unique_domain_cpus('', mock), [0, 2])
+        assert_equal(resolve_unique_domain_cpus('cpu0', mock), [0])
+        assert_equal(resolve_unique_domain_cpus('cpu3', mock), [2])

wa/utils/misc.py

@@ -43,11 +43,12 @@ from distutils.spawn import find_executable
 import yaml
 from dateutil import tz
 
+from devlib.exception import TargetError
 from devlib.utils.misc import (ABI_MAP, check_output, walk_modules,
                                ensure_directory_exists, ensure_file_directory_exists,
                                normalize, convert_new_lines, get_cpu_mask, unique,
                                escape_quotes, escape_single_quotes, escape_double_quotes,
-                               isiterable, getch, as_relative, ranges_to_list,
+                               isiterable, getch, as_relative, ranges_to_list, memoized,
                                list_to_ranges, list_to_mask, mask_to_list, which)
 
 check_output_logger = logging.getLogger('check_output')
@@ -575,7 +576,7 @@ def merge_maps(m1, m2):
 
 def merge_dicts_simple(base, other):
     result = base.copy()
-    for key, value in (base or {}).iteritems():
+    for key, value in (other or {}).iteritems():
         result[key] = merge_config_values(result.get(key), value)
     return result
 
@@ -598,3 +599,76 @@ def get_object_name(obj):
     elif hasattr(obj, '__class__'):
         return obj.__class__.__name__
     return None
+
+
+def resolve_cpus(name, target):
+    """
+    Returns a list of cpu numbers that corresponds to a passed name.
+    Allowed formats are:
+        - 'big'
+        - 'little'
+        - '<core_name> e.g. 'A15'
+        - 'cpuX'
+        - 'all' - returns all cpus
+        - '' - Empty name will also return all cpus
+    """
+    cpu_list = range(target.number_of_cpus)
+
+    # Support for passing cpu no directly
+    if isinstance(name, int):
+        cpu = name
+        if cpu not in cpu_list:
+            message = 'CPU{} is not available, must be in {}'
+            raise ValueError(message.format(cpu, cpu_list))
+        return [cpu]
+
+    # Apply to all cpus
+    if not name or name.lower() == 'all':
+        return cpu_list
+    # Deal with big.little substitution
+    elif name.lower() == 'big':
+        name = target.big_core
+        if not name:
+            raise ValueError('big core name could not be retrieved')
+    elif name.lower() == 'little':
+        name = target.little_core
+        if not name:
+            raise ValueError('little core name could not be retrieved')
+
+    # Return all cores with specified name
+    if name in target.core_names:
+        return target.core_cpus(name)
+
+    # Check if core number has been supplied.
+    else:
+        core_no = re.match('cpu([0-9]+)', name, re.IGNORECASE)
+        if core_no:
+            cpu = int(core_no.group(1))
+            if cpu not in cpu_list:
+                message = 'CPU{} is not available, must be in {}'
+                raise ValueError(message.format(cpu, cpu_list))
+            return [cpu]
+        else:
+            msg = 'Unexpected core name "{}"'
+            raise ValueError(msg.format(name))
+
+@memoized
+def resolve_unique_domain_cpus(name, target):
+    """
+    Same as `resolve_cpus` above but only returns only the first cpu
+    in each of the different frequency domains. Requires cpufreq.
+    """
+    cpus = resolve_cpus(name, target)
+    if not target.has('cpufreq'):
+        msg = 'Device does not appear to support cpufreq; ' \
+              'Cannot obtain cpu domain information'
+        raise TargetError(msg)
+
+    unique_cpus = []
+    domain_cpus = []
+    for cpu in cpus:
+        if cpu not in domain_cpus:
+            domain_cpus = target.cpufreq.get_domain_cpus(cpu)
+        if domain_cpus[0] not in unique_cpus:
+            unique_cpus.append(domain_cpus[0])
+    return unique_cpus