Version bump for release

Update the interface to make use of the collector interface.
Notable changes are the removal of the `output_path` path provided on
initialisation which will now be provided by the dedicated `set_output`
method.

devlib/__init__.py

@@ -45,18 +45,21 @@ from devlib.derived import DerivedMeasurements, DerivedMetric
 from devlib.derived.energy import DerivedEnergyMeasurements
 from devlib.derived.fps import DerivedGfxInfoStats, DerivedSurfaceFlingerStats
 
-from devlib.trace.ftrace import FtraceCollector
-from devlib.trace.perf import PerfCollector
-from devlib.trace.serial_trace import SerialTraceCollector
+from devlib.collector.ftrace import FtraceCollector
+from devlib.collector.perf import PerfCollector
+from devlib.collector.serial_trace import SerialTraceCollector
+from devlib.collector.dmesg import DmesgCollector
+from devlib.collector.logcat import LogcatCollector
 
 from devlib.host import LocalConnection
 from devlib.utils.android import AdbConnection
 from devlib.utils.ssh import SshConnection, TelnetConnection, Gem5Connection
 
-from devlib.utils.version import get_commit as __get_commit
+from devlib.utils.version import (get_devlib_version as __get_devlib_version,
+                                  get_commit as __get_commit)
 
 
-__version__ = '1.1.0'
+__version__ = __get_devlib_version()
 
 __commit = __get_commit()
 if __commit:

devlib/bin/scripts/shutils.in

@@ -238,6 +238,19 @@ hotplug_online_all() {
     done
 }
 
+
+################################################################################
+# Scheduler
+################################################################################
+
+sched_get_kernel_attributes() {
+	MATCH=${1:-'.*'}
+	[ -d /proc/sys/kernel/ ] || exit 1
+	$GREP '' /proc/sys/kernel/sched_* | \
+		$SED -e 's|/proc/sys/kernel/sched_||' | \
+		$GREP -e "$MATCH"
+}
+
 ################################################################################
 # Misc
 ################################################################################
@@ -264,6 +277,34 @@ read_tree_values() {
     fi
 }
 
+read_tree_tgz_b64() {
+    BASEPATH=$1
+    MAXDEPTH=$2
+    TMPBASE=$3
+
+    if [ ! -e $BASEPATH ]; then
+        echo "ERROR: $BASEPATH does not exist"
+        exit 1
+    fi
+
+    cd $TMPBASE
+    TMP_FOLDER=$($BUSYBOX realpath $($BUSYBOX mktemp -d XXXXXX))
+
+    # 'tar' doesn't work as expected on debugfs, so copy the tree first to
+    # workaround the issue
+    cd $BASEPATH
+    for CUR_FILE in $($BUSYBOX find . -follow -type f -maxdepth $MAXDEPTH); do
+        $BUSYBOX cp --parents $CUR_FILE $TMP_FOLDER/ 2> /dev/null
+    done
+
+    cd $TMP_FOLDER
+    $BUSYBOX tar cz * 2>/dev/null | $BUSYBOX base64
+
+    # Clean-up the tmp folder since we won't need it any more
+    cd $TMPBASE
+    rm -rf $TMP_FOLDER
+}
+
 get_linux_system_id() {
 	kernel=$($BUSYBOX uname -r)
 	hardware=$($BUSYBOX ip a | $BUSYBOX grep 'link/ether' | $BUSYBOX sed 's/://g' | $BUSYBOX awk '{print $2}' | $BUSYBOX tr -d '\n')
@@ -337,12 +378,18 @@ hotplug_online_all)
 read_tree_values)
 	read_tree_values $*
     ;;
+read_tree_tgz_b64)
+	read_tree_tgz_b64 $*
+    ;;
 get_linux_system_id)
 	get_linux_system_id $*
     ;;
 get_android_system_id)
 	get_android_system_id $*
     ;;
+sched_get_kernel_attributes)
+	sched_get_kernel_attributes $*
+	;;
 *)
     echo "Command [$CMD] not supported"
     exit -1

devlib/collector/__init__.py

@@ -15,12 +15,14 @@
 
 import logging
 
+from devlib.utils.types import caseless_string
 
-class TraceCollector(object):
+class CollectorBase(object):
 
     def __init__(self, target):
         self.target = target
         self.logger = logging.getLogger(self.__class__.__name__)
+        self.output_path = None
 
     def reset(self):
         pass
@@ -31,6 +33,12 @@ class TraceCollector(object):
     def stop(self):
         pass
 
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
+        return CollectorOutput()
+
     def __enter__(self):
         self.reset()
         self.start()
@@ -39,5 +47,29 @@ class TraceCollector(object):
     def __exit__(self, exc_type, exc_value, traceback):
         self.stop()
 
-    def get_trace(self, outfile):
-        pass
+class CollectorOutputEntry(object):
+
+    path_kinds = ['file', 'directory']
+
+    def __init__(self, path, path_kind):
+        self.path = path
+
+        path_kind = caseless_string(path_kind)
+        if path_kind not in self.path_kinds:
+            msg = '{} is not a valid path_kind [{}]'
+            raise ValueError(msg.format(path_kind, ' '.join(self.path_kinds)))
+        self.path_kind = path_kind
+
+    def __str__(self):
+        return self.path
+
+    def __repr__(self):
+        return '<{} ({})>'.format(self.path, self.path_kind)
+
+    def __fspath__(self):
+        """Allow using with os.path operations"""
+        return self.path
+
+
+class CollectorOutput(list):
+    pass

devlib/collector/dmesg.py

@@ -0,0 +1,208 @@
+#    Copyright 2019 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import division
+import re
+from itertools import takewhile
+from datetime import timedelta
+
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
+
+
+class KernelLogEntry(object):
+    """
+    Entry of the kernel ring buffer.
+
+    :param facility: facility the entry comes from
+    :type facility: str
+
+    :param level: log level
+    :type level: str
+
+    :param timestamp: Timestamp of the entry
+    :type timestamp: datetime.timedelta
+
+    :param msg: Content of the entry
+    :type msg: str
+    """
+
+    _TIMESTAMP_MSG_REGEX = re.compile(r'\[(.*?)\] (.*)')
+    _RAW_LEVEL_REGEX = re.compile(r'<([0-9]+)>(.*)')
+    _PRETTY_LEVEL_REGEX = re.compile(r'\s*([a-z]+)\s*:([a-z]+)\s*:\s*(.*)')
+
+    def __init__(self, facility, level, timestamp, msg):
+        self.facility = facility
+        self.level = level
+        self.timestamp = timestamp
+        self.msg = msg
+
+    @classmethod
+    def from_str(cls, line):
+        """
+        Parses a "dmesg --decode" output line, formatted as following:
+        kern  :err   : [3618282.310743] nouveau 0000:01:00.0: systemd-logind[988]: nv50cal_space: -16
+
+        Or the more basic output given by "dmesg -r":
+        <3>[3618282.310743] nouveau 0000:01:00.0: systemd-logind[988]: nv50cal_space: -16
+
+        """
+
+        def parse_raw_level(line):
+            match = cls._RAW_LEVEL_REGEX.match(line)
+            if not match:
+                raise ValueError('dmesg entry format not recognized: {}'.format(line))
+            level, remainder = match.groups()
+            levels = DmesgCollector.LOG_LEVELS
+            # BusyBox dmesg can output numbers that need to wrap around
+            level = levels[int(level) % len(levels)]
+            return level, remainder
+
+        def parse_pretty_level(line):
+            match = cls._PRETTY_LEVEL_REGEX.match(line)
+            facility, level, remainder = match.groups()
+            return facility, level, remainder
+
+        def parse_timestamp_msg(line):
+            match = cls._TIMESTAMP_MSG_REGEX.match(line)
+            timestamp, msg = match.groups()
+            timestamp = timedelta(seconds=float(timestamp.strip()))
+            return timestamp, msg
+
+        line = line.strip()
+
+        # If we can parse the raw prio directly, that is a basic line
+        try:
+            level, remainder = parse_raw_level(line)
+            facility = None
+        except ValueError:
+            facility, level, remainder = parse_pretty_level(line)
+
+        timestamp, msg = parse_timestamp_msg(remainder)
+
+        return cls(
+            facility=facility,
+            level=level,
+            timestamp=timestamp,
+            msg=msg.strip(),
+        )
+
+    @classmethod
+    def from_dmesg_output(cls, dmesg_out):
+        """
+        Return a generator of :class:`KernelLogEntry` for each line of the
+        output of dmesg command.
+
+        .. note:: The same restrictions on the dmesg output format as for
+            :meth:`from_str` apply.
+        """
+        for line in dmesg_out.splitlines():
+            if line.strip():
+                yield cls.from_str(line)
+
+    def __str__(self):
+        facility = self.facility + ': ' if self.facility else ''
+        return '{facility}{level}: [{timestamp}] {msg}'.format(
+            facility=facility,
+            level=self.level,
+            timestamp=self.timestamp.total_seconds(),
+            msg=self.msg,
+        )
+
+
+class DmesgCollector(CollectorBase):
+    """
+    Dmesg output collector.
+
+    :param level: Minimum log level to enable. All levels that are more
+        critical will be collected as well.
+    :type level: str
+
+    :param facility: Facility to record, see dmesg --help for the list.
+    :type level: str
+
+    .. warning:: If BusyBox dmesg is used, facility and level will be ignored,
+        and the parsed entries will also lack that information.
+    """
+
+    # taken from "dmesg --help"
+    # This list needs to be ordered by priority
+    LOG_LEVELS = [
+        "emerg",        # system is unusable
+        "alert",        # action must be taken immediately
+        "crit",         # critical conditions
+        "err",          # error conditions
+        "warn",         # warning conditions
+        "notice",       # normal but significant condition
+        "info",         # informational
+        "debug",        # debug-level messages
+    ]
+
+    def __init__(self, target, level=LOG_LEVELS[-1], facility='kern'):
+        super(DmesgCollector, self).__init__(target)
+        self.output_path = None
+
+        if level not in self.LOG_LEVELS:
+            raise ValueError('level needs to be one of: {}'.format(
+                ', '.join(self.LOG_LEVELS)
+            ))
+        self.level = level
+
+        # Check if dmesg is the BusyBox one, or the one from util-linux in a
+        # recent version.
+        # Note: BusyBox dmesg does not support -h, but will still print the
+        # help with an exit code of 1
+        self.basic_dmesg = '--force-prefix' not in \
+                self.target.execute('dmesg -h', check_exit_code=False)
+        self.facility = facility
+        self.reset()
+
+    @property
+    def entries(self):
+        return KernelLogEntry.from_dmesg_output(self.dmesg_out)
+
+    def reset(self):
+        self.dmesg_out = None
+
+    def start(self):
+        self.reset()
+        # Empty the dmesg ring buffer
+        self.target.execute('dmesg -c', as_root=True)
+
+    def stop(self):
+        levels_list = list(takewhile(
+            lambda level: level != self.level,
+            self.LOG_LEVELS
+        ))
+        levels_list.append(self.level)
+        if self.basic_dmesg:
+            cmd = 'dmesg -r'
+        else:
+            cmd = 'dmesg --facility={facility} --force-prefix --decode --level={levels}'.format(
+                levels=','.join(levels_list),
+                facility=self.facility,
+            )
+
+        self.dmesg_out = self.target.execute(cmd)
+
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
+        with open(self.output_path, 'wt') as f:
+            f.write(self.dmesg_out + '\n')
+        return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')])

devlib/collector/ftrace.py

@@ -20,11 +20,14 @@ import time
 import re
 import subprocess
 import sys
+import contextlib
+from pipes import quote
 
-from devlib.trace import TraceCollector
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
 from devlib.host import PACKAGE_BIN_DIRECTORY
 from devlib.exception import TargetStableError, HostError
-from devlib.utils.misc import check_output, which
+from devlib.utils.misc import check_output, which, memoized
 
 
 TRACE_MARKER_START = 'TRACE_MARKER_START'
@@ -48,12 +51,14 @@ TIMEOUT = 180
 CPU_RE = re.compile(r'  Function \(CPU([0-9]+)\)')
 STATS_RE = re.compile(r'([^ ]*) +([0-9]+) +([0-9.]+) us +([0-9.]+) us +([0-9.]+) us')
 
-class FtraceCollector(TraceCollector):
+class FtraceCollector(CollectorBase):
 
     # pylint: disable=too-many-locals,too-many-branches,too-many-statements
     def __init__(self, target,
                  events=None,
                  functions=None,
+                 tracer=None,
+                 trace_children_functions=False,
                  buffer_size=None,
                  buffer_size_step=1000,
                  tracing_path='/sys/kernel/debug/tracing',
@@ -63,26 +68,34 @@ class FtraceCollector(TraceCollector):
                  no_install=False,
                  strict=False,
                  report_on_target=False,
+                 trace_clock='local',
+                 saved_cmdlines_nr=4096,
                  ):
         super(FtraceCollector, self).__init__(target)
         self.events = events if events is not None else DEFAULT_EVENTS
         self.functions = functions
+        self.tracer = tracer
+        self.trace_children_functions = trace_children_functions
         self.buffer_size = buffer_size
         self.buffer_size_step = buffer_size_step
         self.tracing_path = tracing_path
         self.automark = automark
         self.autoreport = autoreport
         self.autoview = autoview
+        self.strict = strict
         self.report_on_target = report_on_target
         self.target_output_file = target.path.join(self.target.working_directory, OUTPUT_TRACE_FILE)
         text_file_name = target.path.splitext(OUTPUT_TRACE_FILE)[0] + '.txt'
         self.target_text_file = target.path.join(self.target.working_directory, text_file_name)
+        self.output_path = None
         self.target_binary = None
         self.host_binary = None
         self.start_time = None
         self.stop_time = None
         self.event_string = None
         self.function_string = None
+        self.trace_clock = trace_clock
+        self.saved_cmdlines_nr = saved_cmdlines_nr
         self._reset_needed = True
 
         # pylint: disable=bad-whitespace
@@ -94,6 +107,9 @@ class FtraceCollector(TraceCollector):
         self.function_profile_file    = self.target.path.join(self.tracing_path, 'function_profile_enabled')
         self.marker_file              = self.target.path.join(self.tracing_path, 'trace_marker')
         self.ftrace_filter_file       = self.target.path.join(self.tracing_path, 'set_ftrace_filter')
+        self.trace_clock_file         = self.target.path.join(self.tracing_path, 'trace_clock')
+        self.save_cmdlines_size_file  = self.target.path.join(self.tracing_path, 'saved_cmdlines_size')
+        self.available_tracers_file  = self.target.path.join(self.tracing_path, 'available_tracers')
 
         self.host_binary = which('trace-cmd')
         self.kernelshark = which('kernelshark')
@@ -113,51 +129,98 @@ class FtraceCollector(TraceCollector):
             self.target_binary = 'trace-cmd'
 
         # Validate required events to be traced
-        available_events = self.target.execute(
-                'cat {}'.format(self.available_events_file),
-                as_root=True).splitlines()
-        selected_events = []
-        for event in self.events:
-            # Convert globs supported by FTrace into valid regexp globs
-            _event = event
-            if event[0] != '*':
-                _event = '*' + event
-            event_re = re.compile(_event.replace('*', '.*'))
-            # Select events matching the required ones
-            if not list(filter(event_re.match, available_events)):
-                message = 'Event [{}] not available for tracing'.format(event)
-                if strict:
-                    raise TargetStableError(message)
-                self.target.logger.warning(message)
+        def event_to_regex(event):
+            if not event.startswith('*'):
+                event = '*' + event
+
+            return re.compile(event.replace('*', '.*'))
+
+        def event_is_in_list(event, events):
+            return any(
+                event_to_regex(event).match(_event)
+                for _event in events
+            )
+
+        unavailable_events = [
+            event
+            for event in self.events
+            if not event_is_in_list(event, self.available_events)
+        ]
+        if unavailable_events:
+            message = 'Events not available for tracing: {}'.format(
+                ', '.join(unavailable_events)
+            )
+            if self.strict:
+                raise TargetStableError(message)
             else:
-                selected_events.append(event)
-        # If function profiling is enabled we always need at least one event.
-        # Thus, if not other events have been specified, try to add at least
-        # a tracepoint which is always available and possibly triggered few
-        # times.
-        if self.functions and not selected_events:
-            selected_events = ['sched_wakeup_new']
-        self.event_string = _build_trace_events(selected_events)
+                self.target.logger.warning(message)
+
+        selected_events = sorted(set(self.events) - set(unavailable_events))
+
+        if self.tracer and self.tracer not in self.available_tracers:
+            raise TargetStableError('Unsupported tracer "{}". Available tracers: {}'.format(
+                self.tracer, ', '.join(self.available_tracers)))
 
         # Check for function tracing support
         if self.functions:
-            if not self.target.file_exists(self.function_profile_file):
-                raise TargetStableError('Function profiling not supported. '\
-                        'A kernel build with CONFIG_FUNCTION_PROFILER enable is required')
             # Validate required functions to be traced
-            available_functions = self.target.execute(
-                    'cat {}'.format(self.available_functions_file),
-                    as_root=True).splitlines()
             selected_functions = []
             for function in self.functions:
-                if function not in available_functions:
-                    message = 'Function [{}] not available for profiling'.format(function)
-                    if strict:
+                if function not in self.available_functions:
+                    message = 'Function [{}] not available for tracing/profiling'.format(function)
+                    if self.strict:
                         raise TargetStableError(message)
                     self.target.logger.warning(message)
                 else:
                     selected_functions.append(function)
-            self.function_string = _build_trace_functions(selected_functions)
+
+            # Function profiling
+            if self.tracer is None:
+                if not self.target.file_exists(self.function_profile_file):
+                    raise TargetStableError('Function profiling not supported. '\
+                                            'A kernel build with CONFIG_FUNCTION_PROFILER enable is required')
+                self.function_string = _build_trace_functions(selected_functions)
+                # If function profiling is enabled we always need at least one event.
+                # Thus, if not other events have been specified, try to add at least
+                # a tracepoint which is always available and possibly triggered few
+                # times.
+                if not selected_events:
+                    selected_events = ['sched_wakeup_new']
+
+            # Function tracing
+            elif self.tracer == 'function':
+                self.function_string = _build_graph_functions(selected_functions, False)
+
+            # Function graphing
+            elif self.tracer == 'function_graph':
+                self.function_string = _build_graph_functions(selected_functions, trace_children_functions)
+
+        self.event_string = _build_trace_events(selected_events)
+
+
+    @property
+    @memoized
+    def available_tracers(self):
+        """
+        List of ftrace tracers supported by the target's kernel.
+        """
+        return self.target.read_value(self.available_tracers_file).split(' ')
+
+    @property
+    @memoized
+    def available_events(self):
+        """
+        List of ftrace events supported by the target's kernel.
+        """
+        return self.target.read_value(self.available_events_file).splitlines()
+
+    @property
+    @memoized
+    def available_functions(self):
+        """
+        List of functions whose tracing/profiling is supported by the target's kernel.
+        """
+        return self.target.read_value(self.available_functions_file).splitlines()
 
     def reset(self):
         if self.buffer_size:
@@ -170,8 +233,40 @@ class FtraceCollector(TraceCollector):
         self.start_time = time.time()
         if self._reset_needed:
             self.reset()
-        self.target.execute('{} start {}'.format(self.target_binary, self.event_string),
-                            as_root=True)
+
+        if self.tracer is not None and 'function' in self.tracer:
+            tracecmd_functions = self.function_string
+        else:
+            tracecmd_functions = ''
+
+        tracer_string = '-p {}'.format(self.tracer) if self.tracer else ''
+
+        # Ensure kallsyms contains addresses if possible, so that function the
+        # collected trace contains enough data for pretty printing
+        with contextlib.suppress(TargetStableError):
+            self.target.write_value('/proc/sys/kernel/kptr_restrict', 0)
+
+        self.target.write_value(self.trace_clock_file, self.trace_clock, verify=False)
+        try:
+            self.target.write_value(self.save_cmdlines_size_file, self.saved_cmdlines_nr)
+        except TargetStableError as e:
+            message = 'Could not set "save_cmdlines_size"'
+            if self.strict:
+                self.logger.error(message)
+                raise e
+            else:
+                self.logger.warning(message)
+                self.logger.debug(e)
+
+        self.target.execute(
+            '{} start {events} {tracer} {functions}'.format(
+                self.target_binary,
+                events=self.event_string,
+                tracer=tracer_string,
+                functions=tracecmd_functions,
+            ),
+            as_root=True,
+        )
         if self.automark:
             self.mark_start()
         if 'cpufreq' in self.target.modules:
@@ -181,7 +276,7 @@ class FtraceCollector(TraceCollector):
             self.logger.debug('Trace CPUIdle states')
             self.target.cpuidle.perturb_cpus()
         # Enable kernel function profiling
-        if self.functions:
+        if self.functions and self.tracer is None:
             self.target.execute('echo nop > {}'.format(self.current_tracer_file),
                                 as_root=True)
             self.target.execute('echo 0 > {}'.format(self.function_profile_file),
@@ -194,7 +289,7 @@ class FtraceCollector(TraceCollector):
 
     def stop(self):
         # Disable kernel function profiling
-        if self.functions:
+        if self.functions and self.tracer is None:
             self.target.execute('echo 1 > {}'.format(self.function_profile_file),
                                 as_root=True)
         if 'cpufreq' in self.target.modules:
@@ -207,9 +302,14 @@ class FtraceCollector(TraceCollector):
                             timeout=TIMEOUT, as_root=True)
         self._reset_needed = True
 
-    def get_trace(self, outfile):
-        if os.path.isdir(outfile):
-            outfile = os.path.join(outfile, os.path.basename(self.target_output_file))
+    def set_output(self, output_path):
+        if os.path.isdir(output_path):
+            output_path = os.path.join(output_path, os.path.basename(self.target_output_file))
+        self.output_path = output_path
+
+    def get_data(self):
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
         self.target.execute('{0} extract -o {1}; chmod 666 {1}'.format(self.target_binary,
                                                                        self.target_output_file),
                             timeout=TIMEOUT, as_root=True)
@@ -218,23 +318,27 @@ class FtraceCollector(TraceCollector):
         # Therefore timout for the pull command must also be adjusted
         # accordingly.
         pull_timeout = 10 * (self.stop_time - self.start_time)
-        self.target.pull(self.target_output_file, outfile, timeout=pull_timeout)
-        if not os.path.isfile(outfile):
+        self.target.pull(self.target_output_file, self.output_path, timeout=pull_timeout)
+        output = CollectorOutput()
+        if not os.path.isfile(self.output_path):
             self.logger.warning('Binary trace not pulled from device.')
         else:
+            output.append(CollectorOutputEntry(self.output_path, 'file'))
             if self.autoreport:
-                textfile = os.path.splitext(outfile)[0] + '.txt'
+                textfile = os.path.splitext(self.output_path)[0] + '.txt'
                 if self.report_on_target:
                     self.generate_report_on_target()
                     self.target.pull(self.target_text_file,
                                      textfile, timeout=pull_timeout)
                 else:
-                    self.report(outfile, textfile)
+                    self.report(self.output_path, textfile)
+                output.append(CollectorOutputEntry(textfile, 'file'))
             if self.autoview:
-                self.view(outfile)
+                self.view(self.output_path)
+        return output
 
     def get_stats(self, outfile):
-        if not self.functions:
+        if not (self.functions and self.tracer is None):
             return
 
         if os.path.isdir(outfile):
@@ -351,3 +455,10 @@ def _build_trace_events(events):
 def _build_trace_functions(functions):
     function_string = " ".join(functions)
     return function_string
+
+def _build_graph_functions(functions, trace_children_functions):
+    opt = 'g' if trace_children_functions else 'l'
+    return ' '.join(
+        '-{} {}'.format(opt, quote(f))
+        for f in functions
+    )

devlib/collector/logcat.py

@@ -16,14 +16,16 @@
 import os
 import shutil
 
-from devlib.trace import TraceCollector
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
 from devlib.utils.android import LogcatMonitor
 
-class LogcatCollector(TraceCollector):
+class LogcatCollector(CollectorBase):
 
     def __init__(self, target, regexps=None):
         super(LogcatCollector, self).__init__(target)
         self.regexps = regexps
+        self.output_path = None
         self._collecting = False
         self._prev_log = None
         self._monitor = None
@@ -45,12 +47,14 @@ class LogcatCollector(TraceCollector):
         """
         Start collecting logcat lines
         """
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
         self._monitor = LogcatMonitor(self.target, self.regexps)
         if self._prev_log:
             # Append new data collection to previous collection
             self._monitor.start(self._prev_log)
         else:
-            self._monitor.start()
+            self._monitor.start(self.output_path)
 
         self._collecting = True
 
@@ -65,9 +69,10 @@ class LogcatCollector(TraceCollector):
         self._collecting = False
         self._prev_log = self._monitor.logfile
 
-    def get_trace(self, outfile):
-        """
-        Output collected logcat lines to designated file
-        """
-        # copy self._monitor.logfile to outfile
-        shutil.copy(self._monitor.logfile, outfile)
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
+        if self.output_path is None:
+            raise RuntimeError("No data collected.")
+        return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')])

devlib/collector/perf.py

@@ -0,0 +1,253 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import time
+from past.builtins import basestring, zip
+
+from devlib.host import PACKAGE_BIN_DIRECTORY
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
+from devlib.utils.misc import ensure_file_directory_exists as _f
+
+
+PERF_COMMAND_TEMPLATE = '{binary} {command} {options} {events} sleep 1000 > {outfile} 2>&1 '
+PERF_REPORT_COMMAND_TEMPLATE= '{binary} report {options} -i {datafile} > {outfile} 2>&1 '
+PERF_RECORD_COMMAND_TEMPLATE= '{binary} record {options} {events} -o {outfile}'
+
+PERF_DEFAULT_EVENTS = [
+    'cpu-migrations',
+    'context-switches',
+]
+
+SIMPLEPERF_DEFAULT_EVENTS = [
+    'raw-cpu-cycles',
+    'raw-l1-dcache',
+    'raw-l1-dcache-refill',
+    'raw-br-mis-pred',
+    'raw-instruction-retired',
+]
+
+DEFAULT_EVENTS = {'perf':PERF_DEFAULT_EVENTS, 'simpleperf':SIMPLEPERF_DEFAULT_EVENTS}
+
+class PerfCollector(CollectorBase):
+    """
+    Perf is a Linux profiling with performance counters.
+    Simpleperf is an Android profiling tool with performance counters.
+
+    It is highly recomended to use perf_type = simpleperf when using this instrument
+    on android devices, since it recognises android symbols in record mode and is much more stable
+    when reporting record .data files. For more information see simpleperf documentation at:
+    https://android.googlesource.com/platform/system/extras/+/master/simpleperf/doc/README.md
+
+    Performance counters are CPU hardware registers that count hardware events
+    such as instructions executed, cache-misses suffered, or branches
+    mispredicted. They form a basis for profiling applications to trace dynamic
+    control flow and identify hotspots.
+
+    pref accepts options and events. If no option is given the default '-a' is
+    used. For events, the default events are migrations and cs for perf and raw-cpu-cycles,
+    raw-l1-dcache, raw-l1-dcache-refill, raw-instructions-retired. They both can
+    be specified in the config file.
+
+    Events must be provided as a list that contains them and they will look like
+    this ::
+
+        perf_events = ['migrations', 'cs']
+
+    Events can be obtained by typing the following in the command line on the
+    device ::
+
+        perf list
+        simpleperf list
+
+    Whereas options, they can be provided as a single string as following ::
+
+        perf_options = '-a -i'
+
+    Options can be obtained by running the following in the command line ::
+
+        man perf-stat
+    """
+
+    def __init__(self,
+                 target,
+                 perf_type='perf',
+                 command='stat',
+                 events=None,
+                 optionstring=None,
+                 report_options=None,
+                 labels=None,
+                 force_install=False):
+        super(PerfCollector, self).__init__(target)
+        self.force_install = force_install
+        self.labels = labels
+        self.report_options = report_options
+        self.output_path = None
+
+        # Validate parameters
+        if isinstance(optionstring, list):
+            self.optionstrings = optionstring
+        else:
+            self.optionstrings = [optionstring]
+        if perf_type in ['perf', 'simpleperf']:
+            self.perf_type = perf_type
+        else:
+            raise ValueError('Invalid perf type: {}, must be perf or simpleperf'.format(perf_type))
+        if not events:
+            self.events = DEFAULT_EVENTS[self.perf_type]
+        else:
+            self.events = events
+        if isinstance(self.events, basestring):
+            self.events = [self.events]
+        if not self.labels:
+            self.labels = ['perf_{}'.format(i) for i in range(len(self.optionstrings))]
+        if len(self.labels) != len(self.optionstrings):
+            raise ValueError('The number of labels must match the number of optstrings provided for perf.')
+        if command in ['stat', 'record']:
+            self.command = command
+        else:
+            raise ValueError('Unsupported perf command, must be stat or record')
+
+        self.binary = self.target.get_installed(self.perf_type)
+        if self.force_install or not self.binary:
+            self.binary = self._deploy_perf()
+
+        self._validate_events(self.events)
+
+        self.commands = self._build_commands()
+
+    def reset(self):
+        self.target.killall(self.perf_type, as_root=self.target.is_rooted)
+        self.target.remove(self.target.get_workpath('TemporaryFile*'))
+        for label in self.labels:
+            filepath = self._get_target_file(label, 'data')
+            self.target.remove(filepath)
+            filepath = self._get_target_file(label, 'rpt')
+            self.target.remove(filepath)
+
+    def start(self):
+        for command in self.commands:
+            self.target.kick_off(command)
+
+    def stop(self):
+        self.target.killall(self.perf_type, signal='SIGINT',
+                            as_root=self.target.is_rooted)
+        # perf doesn't transmit the signal to its sleep call so handled here:
+        self.target.killall('sleep', as_root=self.target.is_rooted)
+        # NB: we hope that no other "important" sleep is on-going
+
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
+
+        output = CollectorOutput()
+
+        for label in self.labels:
+            if self.command == 'record':
+                self._wait_for_data_file_write(label, self.output_path)
+                path = self._pull_target_file_to_host(label, 'rpt', self.output_path)
+                output.append(CollectorOutputEntry(path, 'file'))
+            else:
+                path = self._pull_target_file_to_host(label, 'out', self.output_path)
+                output.append(CollectorOutputEntry(path, 'file'))
+        return output
+
+    def _deploy_perf(self):
+        host_executable = os.path.join(PACKAGE_BIN_DIRECTORY,
+                                       self.target.abi, self.perf_type)
+        return self.target.install(host_executable)
+
+    def _get_target_file(self, label, extension):
+        return self.target.get_workpath('{}.{}'.format(label, extension))
+
+    def _build_commands(self):
+        commands = []
+        for opts, label in zip(self.optionstrings, self.labels):
+            if self.command == 'stat':
+                commands.append(self._build_perf_stat_command(opts, self.events, label))
+            else:
+                commands.append(self._build_perf_record_command(opts, label))
+        return commands
+
+    def _build_perf_stat_command(self, options, events, label):
+        event_string = ' '.join(['-e {}'.format(e) for e in events])
+        command = PERF_COMMAND_TEMPLATE.format(binary = self.binary,
+                                               command = self.command,
+                                               options = options or '',
+                                               events = event_string,
+                                               outfile = self._get_target_file(label, 'out'))
+        return command
+
+    def _build_perf_report_command(self, report_options, label):
+        command = PERF_REPORT_COMMAND_TEMPLATE.format(binary=self.binary,
+                                                      options=report_options or '',
+                                                      datafile=self._get_target_file(label, 'data'),
+                                                      outfile=self._get_target_file(label, 'rpt'))
+        return command
+
+    def _build_perf_record_command(self, options, label):
+        event_string = ' '.join(['-e {}'.format(e) for e in self.events])
+        command = PERF_RECORD_COMMAND_TEMPLATE.format(binary=self.binary,
+                                                      options=options or '',
+                                                      events=event_string,
+                                                      outfile=self._get_target_file(label, 'data'))
+        return command
+
+    def _pull_target_file_to_host(self, label, extension, output_path):
+        target_file = self._get_target_file(label, extension)
+        host_relpath = os.path.basename(target_file)
+        host_file = _f(os.path.join(output_path, host_relpath))
+        self.target.pull(target_file, host_file)
+        return host_file
+
+    def _wait_for_data_file_write(self, label, output_path):
+        data_file_finished_writing = False
+        max_tries = 80
+        current_tries = 0
+        while not data_file_finished_writing:
+            files = self.target.execute('cd {} && ls'.format(self.target.get_workpath('')))
+            # Perf stores data in tempory files whilst writing to data output file. Check if they have been removed.
+            if 'TemporaryFile' in files and current_tries <= max_tries:
+                time.sleep(0.25)
+                current_tries += 1
+            else:
+                if current_tries >= max_tries:
+                    self.logger.warning('''writing {}.data file took longer than expected,
+                                        file may not have written correctly'''.format(label))
+                data_file_finished_writing = True
+        report_command = self._build_perf_report_command(self.report_options, label)
+        self.target.execute(report_command)
+
+    def _validate_events(self, events):
+        available_events_string = self.target.execute('{} list'.format(self.perf_type))
+        available_events = available_events_string.splitlines()
+        for available_event in available_events:
+            if available_event == '':
+                continue
+            if 'OR' in available_event:
+                available_events.append(available_event.split('OR')[1])
+            available_events[available_events.index(available_event)] = available_event.split()[0].strip()
+        # Raw hex event codes can also be passed in that do not appear on perf/simpleperf list, prefixed with 'r'
+        raw_event_code_regex = re.compile(r"^r(0x|0X)?[A-Fa-f0-9]+$")
+        for event in events:
+            if event in available_events or re.match(raw_event_code_regex, event):
+                continue
+            else:
+                raise ValueError('Event: {} is not in available event list for {}'.format(event, self.perf_type))

devlib/collector/screencapture.py

@@ -19,13 +19,14 @@ import sys
 import threading
 import time
 
-from devlib.trace import TraceCollector
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
 from devlib.exception import WorkerThreadError
 
 
 class ScreenCapturePoller(threading.Thread):
 
-    def __init__(self, target, period, output_path=None, timeout=30):
+    def __init__(self, target, period, timeout=30):
         super(ScreenCapturePoller, self).__init__()
         self.target = target
         self.logger = logging.getLogger('screencapture')
@@ -36,11 +37,16 @@ class ScreenCapturePoller(threading.Thread):
         self.last_poll = 0
         self.daemon = True
         self.exc = None
+        self.output_path = None
+
+    def set_output(self, output_path):
         self.output_path = output_path
 
     def run(self):
         self.logger.debug('Starting screen capture polling')
         try:
+            if self.output_path is None:
+                raise RuntimeError("Output path was not set.")
             while True:
                 if self.stop_signal.is_set():
                     break
@@ -66,24 +72,33 @@ class ScreenCapturePoller(threading.Thread):
         self.target.capture_screen(os.path.join(self.output_path, "screencap_{ts}.png"))
 
 
-class ScreenCaptureCollector(TraceCollector):
+class ScreenCaptureCollector(CollectorBase):
 
-    def __init__(self, target, output_path=None, period=None):
+    def __init__(self, target, period=None):
         super(ScreenCaptureCollector, self).__init__(target)
         self._collecting = False
-        self.output_path = output_path
+        self.output_path = None
         self.period = period
         self.target = target
-        self._poller = ScreenCapturePoller(self.target, self.period,
-                                           self.output_path)
+
+    def set_output(self, output_path):
+        self.output_path = output_path
 
     def reset(self):
-        pass
+        self._poller = ScreenCapturePoller(self.target, self.period)
+
+    def get_data(self):
+        if self.output_path is None:
+            raise RuntimeError("No data collected.")
+        return CollectorOutput([CollectorOutputEntry(self.output_path, 'directory')])
 
     def start(self):
         """
         Start collecting the screenshots
         """
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
+        self._poller.set_output(self.output_path)
         self._poller.start()
         self._collecting = True
 

devlib/collector/serial_trace.py

@@ -17,11 +17,12 @@ import shutil
 from tempfile import NamedTemporaryFile
 from pexpect.exceptions import TIMEOUT
 
-from devlib.trace import TraceCollector
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
 from devlib.utils.serial_port import get_connection
 
 
-class SerialTraceCollector(TraceCollector):
+class SerialTraceCollector(CollectorBase):
 
     @property
     def collecting(self):
@@ -32,33 +33,35 @@ class SerialTraceCollector(TraceCollector):
         self.serial_port = serial_port
         self.baudrate = baudrate
         self.timeout = timeout
+        self.output_path - None
 
         self._serial_target = None
         self._conn = None
-        self._tmpfile = None
+        self._outfile_fh = None
         self._collecting = False
 
     def reset(self):
         if self._collecting:
             raise RuntimeError("reset was called whilst collecting")
 
-        if self._tmpfile:
-            self._tmpfile.close()
-            self._tmpfile = None
+        if self._outfile_fh:
+            self._outfile_fh.close()
+            self._outfile_fh = None
 
     def start(self):
         if self._collecting:
             raise RuntimeError("start was called whilst collecting")
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
 
-
-        self._tmpfile = NamedTemporaryFile()
+        self._outfile_fh = open(self.output_path, 'w')
         start_marker = "-------- Starting serial logging --------\n"
-        self._tmpfile.write(start_marker.encode('utf-8'))
+        self._outfile_fh.write(start_marker.encode('utf-8'))
 
         self._serial_target, self._conn = get_connection(port=self.serial_port,
                                                          baudrate=self.baudrate,
                                                          timeout=self.timeout,
-                                                         logfile=self._tmpfile,
+                                                         logfile=self._outfile_fh,
                                                          init_dtr=0)
         self._collecting = True
 
@@ -78,17 +81,19 @@ class SerialTraceCollector(TraceCollector):
         del self._conn
 
         stop_marker = "-------- Stopping serial logging --------\n"
-        self._tmpfile.write(stop_marker.encode('utf-8'))
+        self._outfile_fh.write(stop_marker.encode('utf-8'))
+        self._outfile_fh.flush()
+        self._outfile_fh.close()
+        self._outfile_fh = None
 
         self._collecting = False
 
-    def get_trace(self, outfile):
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
         if self._collecting:
-            raise RuntimeError("get_trace was called whilst collecting")
-
-        self._tmpfile.flush()
-
-        shutil.copy(self._tmpfile.name, outfile)
-
-        self._tmpfile.close()
-        self._tmpfile = None
+            raise RuntimeError("get_data was called whilst collecting")
+        if self.output_path is None:
+            raise RuntimeError("No data collected.")
+        return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')])

devlib/collector/systrace.py

@@ -19,8 +19,9 @@ import subprocess
 from shutil import copyfile
 from tempfile import NamedTemporaryFile
 
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
 from devlib.exception import TargetStableError, HostError
-from devlib.trace import TraceCollector
 import devlib.utils.android
 from devlib.utils.misc import memoized
 
@@ -33,7 +34,7 @@ DEFAULT_CATEGORIES = [
     'idle'
 ]
 
-class SystraceCollector(TraceCollector):
+class SystraceCollector(CollectorBase):
     """
     A trace collector based on Systrace
 
@@ -74,9 +75,10 @@ class SystraceCollector(TraceCollector):
 
         self.categories = categories or DEFAULT_CATEGORIES
         self.buffer_size = buffer_size
+        self.output_path = None
 
         self._systrace_process = None
-        self._tmpfile = None
+        self._outfile_fh = None
 
         # Try to find a systrace binary
         self.systrace_binary = None
@@ -104,12 +106,12 @@ class SystraceCollector(TraceCollector):
         self.reset()
 
     def _build_cmd(self):
-        self._tmpfile = NamedTemporaryFile()
+        self._outfile_fh = open(self.output_path, 'w')
 
         # pylint: disable=attribute-defined-outside-init
-        self.systrace_cmd = '{} -o {} -e {}'.format(
+        self.systrace_cmd = 'python2 -u {} -o {} -e {}'.format(
             self.systrace_binary,
-            self._tmpfile.name,
+            self._outfile_fh.name,
             self.target.adb_name
         )
 
@@ -122,13 +124,11 @@ class SystraceCollector(TraceCollector):
         if self._systrace_process:
             self.stop()
 
-        if self._tmpfile:
-            self._tmpfile.close()
-            self._tmpfile = None
-
     def start(self):
         if self._systrace_process:
             raise RuntimeError("Tracing is already underway, call stop() first")
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
 
         self.reset()
 
@@ -137,9 +137,11 @@ class SystraceCollector(TraceCollector):
         self._systrace_process = subprocess.Popen(
             self.systrace_cmd,
             stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
             shell=True,
             universal_newlines=True
         )
+        self._systrace_process.stdout.read(1)
 
     def stop(self):
         if not self._systrace_process:
@@ -149,11 +151,16 @@ class SystraceCollector(TraceCollector):
         self._systrace_process.communicate('\n')
         self._systrace_process = None
 
-    def get_trace(self, outfile):
+        if self._outfile_fh:
+            self._outfile_fh.close()
+            self._outfile_fh = None
+
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
         if self._systrace_process:
             raise RuntimeError("Tracing is underway, call stop() first")
-
-        if not self._tmpfile:
-            raise RuntimeError("No tracing data available")
-
-        copyfile(self._tmpfile.name, outfile)
+        if self.output_path is None:
+            raise RuntimeError("No data collected.")
+        return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')])

devlib/derived/fps.py

@@ -106,17 +106,17 @@ class DerivedGfxInfoStats(DerivedFpsStats):
             frame_count += 1
 
             if start_vsync is None:
-                start_vsync = frame_data.Vsync_time_us
-            end_vsync = frame_data.Vsync_time_us
+                start_vsync = frame_data.Vsync_time_ns
+            end_vsync = frame_data.Vsync_time_ns
 
-            frame_time = frame_data.FrameCompleted_time_us - frame_data.IntendedVsync_time_us
+            frame_time = frame_data.FrameCompleted_time_ns - frame_data.IntendedVsync_time_ns
             pff = 1e9 / frame_time
             if pff > self.drop_threshold:
                 per_frame_fps.append([pff])
 
         if frame_count:
             duration = end_vsync - start_vsync
-            fps = (1e6 * frame_count) / float(duration)
+            fps = (1e9 * frame_count) / float(duration)
         else:
             duration = 0
             fps = 0
@@ -133,15 +133,15 @@ class DerivedGfxInfoStats(DerivedFpsStats):
     def _process_with_pandas(self, measurements_csv):
         data = pd.read_csv(measurements_csv.path)
         data = data[data.Flags_flags == 0]
-        frame_time = data.FrameCompleted_time_us - data.IntendedVsync_time_us
-        per_frame_fps = (1e6 / frame_time)
+        frame_time = data.FrameCompleted_time_ns - data.IntendedVsync_time_ns
+        per_frame_fps = (1e9 / frame_time)
         keep_filter = per_frame_fps > self.drop_threshold
         per_frame_fps = per_frame_fps[keep_filter]
         per_frame_fps.name = 'fps'
 
         frame_count = data.index.size
         if frame_count > 1:
-            duration = data.Vsync_time_us.iloc[-1] - data.Vsync_time_us.iloc[0]
+            duration = data.Vsync_time_ns.iloc[-1] - data.Vsync_time_ns.iloc[0]
             fps = (1e9 * frame_count) / float(duration)
         else:
             duration = 0

devlib/exception.py

@@ -15,11 +15,17 @@
 
 class DevlibError(Exception):
     """Base class for all Devlib exceptions."""
+
+    def __init__(self, *args):
+        message = args[0] if args else None
+        self._message = message
+
     @property
     def message(self):
-        if self.args:
-            return self.args[0]
-        return str(self)
+        if self._message is not None:
+            return self._message
+        else:
+            return str(self)
 
 
 class DevlibStableError(DevlibError):
@@ -105,6 +111,16 @@ class WorkerThreadError(DevlibError):
         super(WorkerThreadError, self).__init__(message)
 
 
+class KernelConfigKeyError(KeyError, IndexError, DevlibError):
+    """
+    Exception raised when a kernel config option cannot be found.
+
+    It inherits from :exc:`IndexError` for backward compatibility, and
+    :exc:`KeyError` to behave like a regular mapping.
+    """
+    pass
+
+
 def get_traceback(exc=None):
     """
     Returns the string with the traceback for the specifiec exc
@@ -117,7 +133,7 @@ def get_traceback(exc=None):
     if not exc:
         return None
     tb = exc[2]
-    sio = io.BytesIO()
+    sio = io.StringIO()
     traceback.print_tb(tb, file=sio)
     del tb  # needs to be done explicitly see: http://docs.python.org/2/library/sys.html#sys.exc_info
     return sio.getvalue()

devlib/host.py

@@ -38,9 +38,21 @@ class LocalConnection(object):
 
     name = 'local'
 
+    @property
+    def connected_as_root(self):
+        if self._connected_as_root is None:
+            result = self.execute('id', as_root=False)
+            self._connected_as_root = 'uid=0(' in result
+        return self._connected_as_root
+
+    @connected_as_root.setter
+    def connected_as_root(self, state):
+        self._connected_as_root = state
+
     # pylint: disable=unused-argument
     def __init__(self, platform=None, keep_password=True, unrooted=False,
                  password=None, timeout=None):
+        self._connected_as_root = None
         self.logger = logging.getLogger('local_connection')
         self.keep_password = keep_password
         self.unrooted = unrooted
@@ -67,11 +79,11 @@ class LocalConnection(object):
     def execute(self, command, timeout=None, check_exit_code=True,
                 as_root=False, strip_colors=True, will_succeed=False):
         self.logger.debug(command)
-        if as_root:
+        if as_root and not self.connected_as_root:
             if self.unrooted:
                 raise TargetStableError('unrooted')
             password = self._get_password()
-            command = 'echo {} | sudo -S '.format(quote(password)) + command
+            command = 'echo {} | sudo -S -- sh -c '.format(quote(password)) + quote(command)
         ignore = None if check_exit_code else 'all'
         try:
             return check_output(command, shell=True, timeout=timeout, ignore=ignore)[0]
@@ -84,7 +96,7 @@ class LocalConnection(object):
                 raise TargetStableError(message)
 
     def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
-        if as_root:
+        if as_root and not self.connected_as_root:
             if self.unrooted:
                 raise TargetStableError('unrooted')
             password = self._get_password()
@@ -97,6 +109,12 @@ class LocalConnection(object):
     def cancel_running_command(self):
         pass
 
+    def wait_for_device(self, timeout=30):
+        return
+
+    def reboot_bootloader(self, timeout=30):
+        raise NotImplementedError()
+
     def _get_password(self):
         if self.password:
             return self.password

devlib/instrument/__init__.py

@@ -97,20 +97,30 @@ _measurement_types = [
     # covert without being familar with individual instruments.
     MeasurementType('time', 'seconds', 'time',
         conversions={
-            'time_us': lambda x: x * 1000000,
-            'time_ms': lambda x: x * 1000,
+            'time_us': lambda x: x * 1e6,
+            'time_ms': lambda x: x * 1e3,
+            'time_ns': lambda x: x * 1e9,
         }
     ),
     MeasurementType('time_us', 'microseconds', 'time',
         conversions={
-            'time': lambda x: x / 1000000,
-            'time_ms': lambda x: x / 1000,
+            'time': lambda x: x / 1e6,
+            'time_ms': lambda x: x / 1e3,
+            'time_ns': lambda x: x * 1e3,
         }
     ),
     MeasurementType('time_ms', 'milliseconds', 'time',
         conversions={
-            'time': lambda x: x / 1000,
-            'time_us': lambda x: x * 1000,
+            'time': lambda x: x / 1e3,
+            'time_us': lambda x: x * 1e3,
+            'time_ns': lambda x: x * 1e6,
+        }
+    ),
+    MeasurementType('time_ns', 'nanoseconds', 'time',
+    conversions={
+        'time': lambda x: x / 1e9,
+        'time_ms': lambda x: x / 1e6,
+        'time_us': lambda x: x / 1e3,
         }
     ),
 

devlib/instrument/acmecape.py

@@ -58,12 +58,14 @@ class AcmeCapeInstrument(Instrument):
                  iio_capture=which('iio-capture'),
                  host='baylibre-acme.local',
                  iio_device='iio:device0',
-                 buffer_size=256):
+                 buffer_size=256,
+                 keep_raw=False):
         super(AcmeCapeInstrument, self).__init__(target)
         self.iio_capture = iio_capture
         self.host = host
         self.iio_device = iio_device
         self.buffer_size = buffer_size
+        self.keep_raw = keep_raw
         self.sample_rate_hz = 100
         if self.iio_capture is None:
             raise HostError('Missing iio-capture binary')
@@ -87,7 +89,8 @@ class AcmeCapeInstrument(Instrument):
         params = dict(
             iio_capture=self.iio_capture,
             host=self.host,
-            buffer_size=self.buffer_size,
+            # This must be a string for quote()
+            buffer_size=str(self.buffer_size),
             iio_device=self.iio_device,
             outfile=self.raw_data_file
         )
@@ -158,3 +161,8 @@ class AcmeCapeInstrument(Instrument):
 
     def get_raw(self):
         return [self.raw_data_file]
+
+    def teardown(self):
+        if not self.keep_raw:
+            if os.path.isfile(self.raw_data_file):
+                os.remove(self.raw_data_file)

devlib/instrument/arm_energy_probe.py

@@ -71,7 +71,7 @@ class ArmEnergyProbeInstrument(Instrument):
 
     MAX_CHANNELS = 12 # 4 Arm Energy Probes
 
-    def __init__(self, target, config_file='./config-aep', ):
+    def __init__(self, target, config_file='./config-aep', keep_raw=False):
         super(ArmEnergyProbeInstrument, self).__init__(target)
         self.arm_probe = which('arm-probe')
         if self.arm_probe is None:
@@ -80,6 +80,7 @@ class ArmEnergyProbeInstrument(Instrument):
         self.attributes = ['power', 'voltage', 'current']
         self.sample_rate_hz = 10000
         self.config_file = config_file
+        self.keep_raw = keep_raw
 
         self.parser = AepParser()
         #TODO make it generic
@@ -142,3 +143,8 @@ class ArmEnergyProbeInstrument(Instrument):
 
     def get_raw(self):
         return [self.output_file_raw]
+
+    def teardown(self):
+        if not self.keep_raw:
+            if os.path.isfile(self.output_file_raw):
+                os.remove(self.output_file_raw)

devlib/instrument/daq.py

@@ -14,6 +14,7 @@
 #
 
 import os
+import shutil
 import tempfile
 from itertools import chain
 
@@ -23,11 +24,11 @@ from devlib.utils.csvutil import csvwriter, create_reader
 from devlib.utils.misc import unique
 
 try:
-    from daqpower.client import execute_command, Status
-    from daqpower.config import DeviceConfiguration, ServerConfiguration
+    from daqpower.client import DaqClient
+    from daqpower.config import DeviceConfiguration
 except ImportError as e:
-    execute_command, Status = None, None
-    DeviceConfiguration, ServerConfiguration, ConfigurationError = None, None, None
+    DaqClient = None
+    DeviceConfiguration = None
     import_error_mesg = e.args[0] if e.args else str(e)
 
 
@@ -44,26 +45,28 @@ class DaqInstrument(Instrument):
                  dv_range=0.2,
                  sample_rate_hz=10000,
                  channel_map=(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23),
+                 keep_raw=False
                  ):
         # pylint: disable=no-member
         super(DaqInstrument, self).__init__(target)
+        self.keep_raw = keep_raw
         self._need_reset = True
         self._raw_files = []
-        if execute_command is None:
+        self.tempdir = None
+        if DaqClient is None:
             raise HostError('Could not import "daqpower": {}'.format(import_error_mesg))
         if labels is None:
             labels = ['PORT_{}'.format(i) for i in range(len(resistor_values))]
         if len(labels) != len(resistor_values):
             raise ValueError('"labels" and "resistor_values" must be of the same length')
-        self.server_config = ServerConfiguration(host=host,
-                                                 port=port)
-        result = self.execute('list_devices')
-        if result.status == Status.OK:
-            if device_id not in result.data:
+        self.daq_client = DaqClient(host, port)
+        try:
+            devices = self.daq_client.list_devices()
+            if device_id not in devices:
                 msg = 'Device "{}" is not found on the DAQ server. Available devices are: "{}"'
-                raise ValueError(msg.format(device_id, ', '.join(result.data)))
-        elif result.status != Status.OKISH:
-            raise HostError('Problem querying DAQ server: {}'.format(result.message))
+                raise ValueError(msg.format(device_id, ', '.join(devices)))
+        except Exception as e:
+            raise HostError('Problem querying DAQ server: {}'.format(e))
 
         self.device_config = DeviceConfiguration(device_id=device_id,
                                                  v_range=v_range,
@@ -80,29 +83,27 @@ class DaqInstrument(Instrument):
 
     def reset(self, sites=None, kinds=None, channels=None):
         super(DaqInstrument, self).reset(sites, kinds, channels)
-        self.execute('close')
-        result = self.execute('configure', config=self.device_config)
-        if not result.status == Status.OK:  # pylint: disable=no-member
-            raise HostError(result.message)
+        self.daq_client.close()
+        self.daq_client.configure(self.device_config)
         self._need_reset = False
         self._raw_files = []
 
     def start(self):
         if self._need_reset:
             self.reset()
-        self.execute('start')
+        self.daq_client.start()
 
     def stop(self):
-        self.execute('stop')
+        self.daq_client.stop()
         self._need_reset = True
 
     def get_data(self, outfile):  # pylint: disable=R0914
-        tempdir = tempfile.mkdtemp(prefix='daq-raw-')
-        self.execute('get_data', output_directory=tempdir)
+        self.tempdir = tempfile.mkdtemp(prefix='daq-raw-')
+        self.daq_client.get_data(self.tempdir)
         raw_file_map = {}
-        for entry in os.listdir(tempdir):
+        for entry in os.listdir(self.tempdir):
             site = os.path.splitext(entry)[0]
-            path = os.path.join(tempdir, entry)
+            path = os.path.join(self.tempdir, entry)
             raw_file_map[site] = path
             self._raw_files.append(path)
 
@@ -118,7 +119,7 @@ class DaqInstrument(Instrument):
                     file_handles.append(fh)
                 except KeyError:
                     message = 'Could not get DAQ trace for {}; Obtained traces are in {}'
-                    raise HostError(message.format(site, tempdir))
+                    raise HostError(message.format(site, self.tempdir))
 
             # The first row is the headers
             channel_order = []
@@ -153,7 +154,7 @@ class DaqInstrument(Instrument):
         return self._raw_files
 
     def teardown(self):
-        self.execute('close')
-
-    def execute(self, command, **kwargs):
-        return execute_command(self.server_config, command, **kwargs)
+        self.daq_client.close()
+        if not self.keep_raw:
+            if os.path.isdir(self.tempdir):
+                shutil.rmtree(self.tempdir)

devlib/instrument/energy_probe.py

@@ -34,9 +34,11 @@ class EnergyProbeInstrument(Instrument):
     def __init__(self, target, resistor_values,
                  labels=None,
                  device_entry='/dev/ttyACM0',
+                 keep_raw=False
                  ):
         super(EnergyProbeInstrument, self).__init__(target)
         self.resistor_values = resistor_values
+        self.keep_raw = keep_raw
         if labels is not None:
             self.labels = labels
         else:
@@ -126,3 +128,8 @@ class EnergyProbeInstrument(Instrument):
 
     def get_raw(self):
         return [self.raw_data_file]
+
+    def teardown(self):
+        if self.keep_raw:
+            if os.path.isfile(self.raw_data_file):
+                os.remove(self.raw_data_file)

devlib/instrument/frames.py

@@ -14,6 +14,8 @@
 #
 
 from __future__ import division
+import os
+
 from devlib.instrument import (Instrument, CONTINUOUS,
                                MeasurementsCsv, MeasurementType)
 from devlib.utils.rendering import (GfxinfoFrameCollector,
@@ -70,6 +72,11 @@ class FramesInstrument(Instrument):
     def _init_channels(self):
         raise NotImplementedError()
 
+    def teardown(self):
+        if not self.keep_raw:
+            if os.path.isfile(self._raw_file):
+                os.remove(self._raw_file)
+
 
 class GfxInfoFramesInstrument(FramesInstrument):
 
@@ -82,7 +89,7 @@ class GfxInfoFramesInstrument(FramesInstrument):
             if entry == 'Flags':
                 self.add_channel('Flags', MeasurementType('flags', 'flags'))
             else:
-                self.add_channel(entry, 'time_us')
+                self.add_channel(entry, 'time_ns')
         self.header = [chan.label for chan in self.channels.values()]
 
 

devlib/module/__init__.py

@@ -91,7 +91,7 @@ class FlashModule(Module):
 
     kind = 'flash'
 
-    def __call__(self, image_bundle=None, images=None, boot_config=None):
+    def __call__(self, image_bundle=None, images=None, boot_config=None, connect=True):
         raise NotImplementedError()
 
 

devlib/module/android.py

@@ -54,7 +54,7 @@ class FastbootFlashModule(FlashModule):
     def probe(target):
         return target.os == 'android'
 
-    def __call__(self, image_bundle=None, images=None, bootargs=None):
+    def __call__(self, image_bundle=None, images=None, bootargs=None, connect=True):
         if bootargs:
             raise ValueError('{} does not support boot configuration'.format(self.name))
         self.prelude_done = False
@@ -67,7 +67,8 @@ class FastbootFlashModule(FlashModule):
             self.logger.debug('flashing {}'.format(partition))
             self._flash_image(self.target, partition, expand_path(image_path))
         fastboot_command('reboot')
-        self.target.connect(timeout=180)
+        if connect:
+            self.target.connect(timeout=180)
 
     def _validate_image_bundle(self, image_bundle):
         if not tarfile.is_tarfile(image_bundle):

devlib/module/cgroups.py

@@ -124,11 +124,10 @@ class Controller(object):
     def move_tasks(self, source, dest, exclude=None):
         if exclude is None:
             exclude = []
-        try:
-            srcg = self._cgroups[source]
-            dstg = self._cgroups[dest]
-        except KeyError as e:
-            raise ValueError('Unknown group: {}'.format(e))
+
+        srcg = self.cgroup(source)
+        dstg = self.cgroup(dest)
+
         self.target._execute_util(  # pylint: disable=protected-access
                     'cgroups_tasks_move {} {} \'{}\''.format(
                     srcg.directory, dstg.directory, exclude),
@@ -158,18 +157,18 @@ class Controller(object):
             raise ValueError('wrong type for "exclude" parameter, '
                              'it must be a str or a list')
 
-        logging.debug('Moving all tasks into %s', dest)
+        self.logger.debug('Moving all tasks into %s', dest)
 
         # Build list of tasks to exclude
         grep_filters = ''
         for comm in exclude:
             grep_filters += '-e {} '.format(comm)
-        logging.debug('   using grep filter: %s', grep_filters)
+        self.logger.debug('   using grep filter: %s', grep_filters)
         if grep_filters != '':
-            logging.debug('   excluding tasks which name matches:')
-            logging.debug('   %s', ', '.join(exclude))
+            self.logger.debug('   excluding tasks which name matches:')
+            self.logger.debug('   %s', ', '.join(exclude))
 
-        for cgroup in self._cgroups:
+        for cgroup in self.list_all():
             if cgroup != dest:
                 self.move_tasks(cgroup, dest, grep_filters)
 
@@ -262,8 +261,9 @@ class CGroup(object):
 
         # Control cgroup path
         self.directory = controller.mount_point
+
         if name != '/':
-            self.directory = self.target.path.join(controller.mount_point, name[1:])
+            self.directory = self.target.path.join(controller.mount_point, name.strip('/'))
 
         # Setup path for tasks file
         self.tasks_file = self.target.path.join(self.directory, 'tasks')
@@ -287,10 +287,8 @@ class CGroup(object):
     def get(self):
         conf = {}
 
-        logging.debug('Reading %s attributes from:',
-                self.controller.kind)
-        logging.debug('  %s',
-                self.directory)
+        self.logger.debug('Reading %s attributes from:', self.controller.kind)
+        self.logger.debug('  %s', self.directory)
         output = self.target._execute_util(  # pylint: disable=protected-access
                     'cgroups_get_attributes {} {}'.format(
                     self.directory, self.controller.kind),
@@ -329,7 +327,7 @@ class CGroup(object):
 
     def get_tasks(self):
         task_ids = self.target.read_value(self.tasks_file).split()
-        logging.debug('Tasks: %s', task_ids)
+        self.logger.debug('Tasks: %s', task_ids)
         return list(map(int, task_ids))
 
     def add_task(self, tid):

devlib/module/cpufreq.py

@@ -111,7 +111,7 @@ class CpufreqModule(Module):
         :Keyword Arguments: Governor tunables, See :meth:`set_governor_tunables`
         """
         if not cpus:
-            cpus = range(self.target.number_of_cpus)
+            cpus = self.target.list_online_cpus()
 
         # Setting a governor & tunables for a cpu will set them for all cpus
         # in the same clock domain, so only manipulating one cpu per domain

devlib/module/cpuidle.py

@@ -173,4 +173,7 @@ class Cpuidle(Module):
         return self.target.read_value(self.target.path.join(self.root_path, 'current_driver'))
 
     def get_governor(self):
-        return self.target.read_value(self.target.path.join(self.root_path, 'current_governor_ro'))
+        path = self.target.path.join(self.root_path, 'current_governor_ro')
+        if not self.target.path.exists(path):
+            path = self.target.path.join(self.root_path, 'current_governor')
+        return self.target.read_value(path)

devlib/module/hwmon.py

@@ -137,7 +137,7 @@ class HwmonModule(Module):
         self.scan()
 
     def scan(self):
-        values_tree = self.target.read_tree_values(self.root, depth=3)
+        values_tree = self.target.read_tree_values(self.root, depth=3, tar=True)
         for entry_id, fields in values_tree.items():
             path = self.target.path.join(self.root, entry_id)
             name = fields.pop('name', None)

devlib/module/sched.py

@@ -21,6 +21,7 @@ from past.builtins import basestring
 
 from devlib.module import Module
 from devlib.utils.misc import memoized
+from devlib.utils.types import boolean
 
 
 class SchedProcFSNode(object):
@@ -51,6 +52,12 @@ class SchedProcFSNode(object):
 
     _re_procfs_node = re.compile(r"(?P<name>.*\D)(?P<digits>\d+)$")
 
+    PACKABLE_ENTRIES = [
+        "cpu",
+        "domain",
+        "group"
+    ]
+
     @staticmethod
     def _ends_with_digits(node):
         if not isinstance(node, basestring):
@@ -70,18 +77,19 @@ class SchedProcFSNode(object):
         """
         :returns: The name of the procfs node
         """
-        return re.search(SchedProcFSNode._re_procfs_node, node).group("name")
+        match = re.search(SchedProcFSNode._re_procfs_node, node)
+        if match:
+            return match.group("name")
 
-    @staticmethod
-    def _packable(node, entries):
+        return node
+
+    @classmethod
+    def _packable(cls, node):
         """
         :returns: Whether it makes sense to pack a node into a common entry
         """
         return (SchedProcFSNode._ends_with_digits(node) and
-                any([SchedProcFSNode._ends_with_digits(x) and
-                     SchedProcFSNode._node_digits(x) != SchedProcFSNode._node_digits(node) and
-                     SchedProcFSNode._node_name(x) == SchedProcFSNode._node_name(node)
-                     for x in entries]))
+                SchedProcFSNode._node_name(node) in cls.PACKABLE_ENTRIES)
 
     @staticmethod
     def _build_directory(node_name, node_data):
@@ -118,7 +126,7 @@ class SchedProcFSNode(object):
         # Find which entries can be packed into a common entry
         packables = {
             node : SchedProcFSNode._node_name(node) + "s"
-            for node in list(nodes.keys()) if SchedProcFSNode._packable(node, list(nodes.keys()))
+            for node in list(nodes.keys()) if SchedProcFSNode._packable(node)
         }
 
         self._dyn_attrs = {}
@@ -227,13 +235,13 @@ class SchedProcFSData(SchedProcFSNode):
         # Even if we have a CPU entry, it can be empty (e.g. hotplugged out)
         # Make sure some data is there
         for cpu in cpus:
-            if target.file_exists(target.path.join(path, cpu, "domain0", "name")):
+            if target.file_exists(target.path.join(path, cpu, "domain0", "flags")):
                 return True
 
         return False
 
     def __init__(self, target, path=None):
-        if not path:
+        if path is None:
             path = self.sched_domain_root
 
         procfs = target.read_tree_values(path, depth=self._read_depth)
@@ -251,7 +259,128 @@ class SchedModule(Module):
         logger = logging.getLogger(SchedModule.name)
         SchedDomainFlag.check_version(target, logger)
 
-        return SchedProcFSData.available(target)
+        # It makes sense to load this module if at least one of those
+        # functionalities is enabled
+        schedproc = SchedProcFSData.available(target)
+        debug = SchedModule.target_has_debug(target)
+        dmips = any([target.file_exists(SchedModule.cpu_dmips_capacity_path(target, cpu))
+                     for cpu in target.list_online_cpus()])
+
+        logger.info("Scheduler sched_domain procfs entries %s",
+                    "found" if schedproc else "not found")
+        logger.info("Detected kernel compiled with SCHED_DEBUG=%s",
+                    "y" if debug else "n")
+        logger.info("CPU capacity sysfs entries %s",
+                    "found" if dmips else "not found")
+
+        return schedproc or debug or dmips
+
+    def get_kernel_attributes(self, matching=None, check_exit_code=True):
+        """
+        Get the value of scheduler attributes.
+
+        :param matching: an (optional) substring to filter the scheduler
+        attributes to be returned.
+
+        The scheduler exposes a list of tunable attributes under:
+            /proc/sys/kernel
+        all starting with the "sched_" prefix.
+
+        This method returns a dictionary of all the "sched_" attributes exposed
+        by the target kernel, within the prefix removed.
+        It's possible to restrict the list of attributes by specifying a
+        substring to be matched.
+
+        returns: a dictionary of scheduler tunables
+        """
+        command = 'sched_get_kernel_attributes {}'.format(
+            matching if matching else ''
+        )
+        output = self.target._execute_util(command, as_root=self.target.is_rooted,
+                                           check_exit_code=check_exit_code)
+        result = {}
+        for entry in output.strip().split('\n'):
+            if ':' not in entry:
+                continue
+            path, value = entry.strip().split(':', 1)
+            if value in ['0', '1']:
+                value = bool(int(value))
+            elif value.isdigit():
+                value = int(value)
+            result[path] = value
+        return result
+
+    def set_kernel_attribute(self, attr, value, verify=True):
+        """
+        Set the value of a scheduler attribute.
+
+        :param attr: the attribute to set, without the "sched_" prefix
+        :param value: the value to set
+        :param verify: true to check that the requested value has been set
+
+        :raise TargetError: if the attribute cannot be set
+        """
+        if isinstance(value, bool):
+            value = '1' if value else '0'
+        elif isinstance(value, int):
+            value = str(value)
+        path = '/proc/sys/kernel/sched_' + attr
+        self.target.write_value(path, value, verify)
+
+    @classmethod
+    def target_has_debug(cls, target):
+        if target.config.get('SCHED_DEBUG') != 'y':
+            return False
+        return target.file_exists('/sys/kernel/debug/sched_features')
+
+    @property
+    @memoized
+    def has_debug(self):
+        return self.target_has_debug(self.target)
+
+    def get_features(self):
+        """
+        Get the status of each sched feature
+
+        :returns: a dictionary of features and their "is enabled" status
+        """
+        if not self.has_debug:
+            raise RuntimeError("sched_features not available")
+        feats = self.target.read_value('/sys/kernel/debug/sched_features')
+        features = {}
+        for feat in feats.split():
+            value = True
+            if feat.startswith('NO'):
+                feat = feat.replace('NO_', '', 1)
+                value = False
+            features[feat] = value
+        return features
+
+    def set_feature(self, feature, enable, verify=True):
+        """
+        Set the status of a specified scheduler feature
+
+        :param feature: the feature name to set
+        :param enable: true to enable the feature, false otherwise
+
+        :raise ValueError: if the specified enable value is not bool
+        :raise RuntimeError: if the specified feature cannot be set
+        """
+        if not self.has_debug:
+            raise RuntimeError("sched_features not available")
+        feature = feature.upper()
+        feat_value = feature
+        if not boolean(enable):
+            feat_value = 'NO_' + feat_value
+        self.target.write_value('/sys/kernel/debug/sched_features',
+                                feat_value, verify=False)
+        if not verify:
+            return
+        msg = 'Failed to set {}, feature not supported?'.format(feat_value)
+        features = self.get_features()
+        feat_value = features.get(feature, not enable)
+        if feat_value != enable:
+            raise RuntimeError(msg)
 
     def get_cpu_sd_info(self, cpu):
         """
@@ -282,17 +411,26 @@ class SchedModule(Module):
         :returns: Whether energy model data is available for 'cpu'
         """
         if not sd:
-            sd = SchedProcFSData(self.target, cpu)
+            sd = self.get_cpu_sd_info(cpu)
 
         return sd.procfs["domain0"].get("group0", {}).get("energy", {}).get("cap_states") != None
 
+    @classmethod
+    def cpu_dmips_capacity_path(cls, target, cpu):
+        """
+        :returns: The target sysfs path where the dmips capacity data should be
+        """
+        return target.path.join(
+            cls.cpu_sysfs_root,
+            'cpu{}/cpu_capacity'.format(cpu))
+
     @memoized
     def has_dmips_capacity(self, cpu):
         """
         :returns: Whether dmips capacity data is available for 'cpu'
         """
         return self.target.file_exists(
-            self.target.path.join(self.cpu_sysfs_root, 'cpu{}/cpu_capacity'.format(cpu))
+            self.cpu_dmips_capacity_path(self.target, cpu)
         )
 
     @memoized
@@ -301,10 +439,13 @@ class SchedModule(Module):
         :returns: The maximum capacity value exposed by the EAS energy model
         """
         if not sd:
-            sd = SchedProcFSData(self.target, cpu)
+            sd = self.get_cpu_sd_info(cpu)
 
         cap_states = sd.domains[0].groups[0].energy.cap_states
-        return int(cap_states.split('\t')[-2])
+        cap_states_list = cap_states.split('\t')
+        num_cap_states = sd.domains[0].groups[0].energy.nr_cap_states
+        max_cap_index = -1 * int(len(cap_states_list) / num_cap_states)
+        return int(cap_states_list[max_cap_index])
 
     @memoized
     def get_dmips_capacity(self, cpu):
@@ -312,14 +453,9 @@ class SchedModule(Module):
         :returns: The capacity value generated from the capacity-dmips-mhz DT entry
         """
         return self.target.read_value(
-            self.target.path.join(
-                self.cpu_sysfs_root,
-                'cpu{}/cpu_capacity'.format(cpu)
-            ),
-            int
+            self.cpu_dmips_capacity_path(self.target, cpu), int
         )
 
-    @memoized
     def get_capacities(self, default=None):
         """
         :param default: Default capacity value to find if no data is
@@ -330,16 +466,30 @@ class SchedModule(Module):
         :raises RuntimeError: Raised when no capacity information is
         found and 'default' is None
         """
-        cpus = list(range(self.target.number_of_cpus))
+        cpus = self.target.list_online_cpus()
 
         capacities = {}
-        sd_info = self.get_sd_info()
 
         for cpu in cpus:
+            if self.has_dmips_capacity(cpu):
+                capacities[cpu] = self.get_dmips_capacity(cpu)
+
+        missing_cpus = set(cpus).difference(capacities.keys())
+        if not missing_cpus:
+            return capacities
+
+        if not SchedProcFSData.available(self.target):
+            if default != None:
+                capacities.update({cpu : default for cpu in missing_cpus})
+                return capacities
+            else:
+                raise RuntimeError(
+                    'No capacity data for cpus {}'.format(sorted(missing_cpus)))
+
+        sd_info = self.get_sd_info()
+        for cpu in missing_cpus:
             if self.has_em(cpu, sd_info.cpus[cpu]):
                 capacities[cpu] = self.get_em_capacity(cpu, sd_info.cpus[cpu])
-            elif self.has_dmips_capacity(cpu):
-                capacities[cpu] = self.get_dmips_capacity(cpu)
             else:
                 if default != None:
                     capacities[cpu] = default

devlib/module/thermal.py

@@ -48,7 +48,7 @@ class ThermalZone(object):
         self.path = target.path.join(root, self.name)
         self.trip_points = {}
 
-        for entry in self.target.list_directory(self.path):
+        for entry in self.target.list_directory(self.path, as_root=target.is_rooted):
             re_match = re.match('^trip_point_([0-9]+)_temp', entry)
             if re_match is not None:
                 self.add_trip_point(re_match.group(1))
@@ -88,6 +88,9 @@ class ThermalModule(Module):
 
         for entry in target.list_directory(self.thermal_root):
             re_match = re.match('^(thermal_zone|cooling_device)([0-9]+)', entry)
+            if not re_match:
+                self.logger.warning('unknown thermal entry: %s', entry)
+                continue
 
             if re_match.group(1) == 'thermal_zone':
                 self.add_thermal_zone(re_match.group(2))

devlib/module/vexpress.py

@@ -325,7 +325,7 @@ class VersatileExpressFlashModule(FlashModule):
         self.timeout = timeout
         self.short_delay = short_delay
 
-    def __call__(self, image_bundle=None, images=None, bootargs=None):
+    def __call__(self, image_bundle=None, images=None, bootargs=None, connect=True):
         self.target.hard_reset()
         with open_serial_connection(port=self.target.platform.serial_port,
                                     baudrate=self.target.platform.baudrate,
@@ -346,7 +346,8 @@ class VersatileExpressFlashModule(FlashModule):
             msg = 'Could not deploy images to {}; got: {}'
             raise TargetStableError(msg.format(self.vemsd_mount, e))
         self.target.boot()
-        self.target.connect(timeout=30)
+        if connect:
+            self.target.connect(timeout=30)
 
     def _deploy_image_bundle(self, bundle):
         self.logger.debug('Validating {}'.format(bundle))

devlib/platform/__init__.py

@@ -78,7 +78,16 @@ class Platform(object):
 
     def _set_model_from_target(self, target):
         if target.os == 'android':
-            self.model = target.getprop('ro.product.model')
+            try:
+                self.model = target.getprop(prop='ro.product.device')
+            except KeyError:
+                self.model = target.getprop('ro.product.model')
+        elif target.file_exists("/proc/device-tree/model"):
+            # There is currently no better way to do this cross platform.
+            # ARM does not have dmidecode
+            raw_model = target.execute("cat /proc/device-tree/model")
+            device_model_to_return = '_'.join(raw_model.split()[:2])
+            return device_model_to_return.rstrip(' \t\r\n\0')
         elif target.is_rooted:
             try:
                 self.model = target.execute('dmidecode -s system-version',

devlib/target.py

@@ -13,6 +13,9 @@
 # limitations under the License.
 #
 
+import io
+import base64
+import gzip
 import os
 import re
 import time
@@ -26,19 +29,31 @@ import threading
 import xml.dom.minidom
 import copy
 from collections import namedtuple, defaultdict
+from contextlib import contextmanager
 from pipes import quote
+from past.builtins import long
+from past.types import basestring
+from numbers import Number
+try:
+    from collections.abc import Mapping
+except ImportError:
+    from collections import Mapping
+
+from enum import Enum
 
 from devlib.host import LocalConnection, PACKAGE_BIN_DIRECTORY
 from devlib.module import get_module
 from devlib.platform import Platform
 from devlib.exception import (DevlibTransientError, TargetStableError,
                               TargetNotRespondingError, TimeoutError,
-                              TargetTransientError) # pylint: disable=redefined-builtin
+                              TargetTransientError, KernelConfigKeyError,
+                              TargetError) # pylint: disable=redefined-builtin
 from devlib.utils.ssh import SshConnection
 from devlib.utils.android import AdbConnection, AndroidProperties, LogcatMonitor, adb_command, adb_disconnect, INTENT_FLAGS
 from devlib.utils.misc import memoized, isiterable, convert_new_lines
 from devlib.utils.misc import commonprefix, merge_lists
 from devlib.utils.misc import ABI_MAP, get_cpu_name, ranges_to_list
+from devlib.utils.misc import batch_contextmanager
 from devlib.utils.types import integer, boolean, bitmask, identifier, caseless_string, bytes_regex
 
 
@@ -58,7 +73,6 @@ GOOGLE_DNS_SERVER_ADDRESS = '8.8.8.8'
 
 installed_package_info = namedtuple('installed_package_info', 'apk_path package')
 
-
 class Target(object):
 
     path = None
@@ -95,21 +109,18 @@ class Target(object):
 
     @property
     def connected_as_root(self):
-        if self._connected_as_root is None:
-            result = self.execute('id')
-            self._connected_as_root = 'uid=0(' in result
-        return self._connected_as_root
+        return self.conn and self.conn.connected_as_root
 
     @property
-    @memoized
     def is_rooted(self):
-        if self.connected_as_root:
-            return True
-        try:
-            self.execute('ls /', timeout=5, as_root=True)
-            return True
-        except (TargetStableError, TimeoutError):
-            return False
+        if self._is_rooted is None:
+            try:
+                self.execute('ls /', timeout=5, as_root=True)
+                self._is_rooted = True
+            except(TargetError, TimeoutError):
+                self._is_rooted = False
+
+        return self._is_rooted or self.connected_as_root
 
     @property
     @memoized
@@ -125,6 +136,10 @@ class Target(object):
     def os_version(self):  # pylint: disable=no-self-use
         return {}
 
+    @property
+    def model(self):
+        return self.platform.model
+
     @property
     def abi(self):  # pylint: disable=no-self-use
         return None
@@ -143,12 +158,33 @@ class Target(object):
     def number_of_cpus(self):
         num_cpus = 0
         corere = re.compile(r'^\s*cpu\d+\s*$')
-        output = self.execute('ls /sys/devices/system/cpu')
+        output = self.execute('ls /sys/devices/system/cpu', as_root=self.is_rooted)
         for entry in output.split():
             if corere.match(entry):
                 num_cpus += 1
         return num_cpus
 
+    @property
+    @memoized
+    def number_of_nodes(self):
+        num_nodes = 0
+        nodere = re.compile(r'^\s*node\d+\s*$')
+        output = self.execute('ls /sys/devices/system/node', as_root=self.is_rooted)
+        for entry in output.split():
+            if nodere.match(entry):
+                num_nodes += 1
+        return num_nodes
+
+    @property
+    @memoized
+    def list_nodes_cpus(self):
+        nodes_cpus = []
+        for node in range(self.number_of_nodes):
+            path = self.path.join('/sys/devices/system/node/node{}/cpulist'.format(node))
+            output = self.read_value(path)
+            nodes_cpus.append(ranges_to_list(output))
+        return nodes_cpus
+
     @property
     @memoized
     def config(self):
@@ -201,7 +237,7 @@ class Target(object):
                  conn_cls=None,
                  is_container=False
                  ):
-        self._connected_as_root = None
+        self._is_rooted = None
         self.connection_settings = connection_settings or {}
         # Set self.platform: either it's given directly (by platform argument)
         # or it's given in the connection_settings argument
@@ -310,7 +346,7 @@ class Target(object):
             timeout = max(timeout - reset_delay, 10)
         if self.has('boot'):
             self.boot()  # pylint: disable=no-member
-        self._connected_as_root = None
+        self.conn.connected_as_root = None
         if connect:
             self.connect(timeout=timeout)
 
@@ -372,7 +408,19 @@ class Target(object):
     # execution
 
     def execute(self, command, timeout=None, check_exit_code=True,
-                as_root=False, strip_colors=True, will_succeed=False):
+                as_root=False, strip_colors=True, will_succeed=False,
+                force_locale='C'):
+
+        # Force the locale if necessary for more predictable output
+        if force_locale:
+            # Use an explicit export so that the command is allowed to be any
+            # shell statement, rather than just a command invocation
+            command = 'export LC_ALL={} && {}'.format(quote(force_locale), command)
+
+        # Ensure to use deployed command when availables
+        if self.executables_directory:
+            command = "export PATH={}:$PATH && {}".format(quote(self.executables_directory), command)
+
         return self.conn.execute(command, timeout=timeout,
                 check_exit_code=check_exit_code, as_root=as_root,
                 strip_colors=strip_colors, will_succeed=will_succeed)
@@ -466,6 +514,18 @@ class Target(object):
     def read_bool(self, path):
         return self.read_value(path, kind=boolean)
 
+    @contextmanager
+    def revertable_write_value(self, path, value, verify=True):
+        orig_value = self.read_value(path)
+        try:
+            self.write_value(path, value, verify)
+            yield
+        finally:
+            self.write_value(path, orig_value, verify)
+
+    def batch_revertable_write_value(self, kwargs_list):
+        return batch_contextmanager(self.revertable_write_value, kwargs_list)
+
     def write_value(self, path, value, verify=True):
         value = str(value)
         self.execute('echo {} > {}'.format(quote(value), quote(path)), check_exit_code=False, as_root=True)
@@ -481,16 +541,16 @@ class Target(object):
         except (DevlibTransientError, subprocess.CalledProcessError):
             # on some targets "reboot" doesn't return gracefully
             pass
-        self._connected_as_root = None
+        self.conn.connected_as_root = None
 
     def check_responsive(self, explode=True):
         try:
             self.conn.execute('ls /', timeout=5)
-            return 1
+            return True
         except (DevlibTransientError, subprocess.CalledProcessError):
             if explode:
                 raise TargetNotRespondingError('Target {} is not responding'.format(self.conn.name))
-            return 0
+            return False
 
     # process management
 
@@ -607,12 +667,12 @@ class Target(object):
 
     which = get_installed
 
-    def install_if_needed(self, host_path, search_system_binaries=True):
+    def install_if_needed(self, host_path, search_system_binaries=True, timeout=None):
 
         binary_path = self.get_installed(os.path.split(host_path)[1],
                                          search_system_binaries=search_system_binaries)
         if not binary_path:
-            binary_path = self.install(host_path)
+            binary_path = self.install(host_path, timeout=timeout)
         return binary_path
 
     def is_installed(self, name):
@@ -684,6 +744,43 @@ class Target(object):
         timeout = duration + 10
         self.execute('sleep {}'.format(duration), timeout=timeout)
 
+    def read_tree_tar_flat(self, path, depth=1, check_exit_code=True,
+                              decode_unicode=True, strip_null_chars=True):
+        command = 'read_tree_tgz_b64 {} {} {}'.format(quote(path), depth,
+                                                  quote(self.working_directory))
+        output = self._execute_util(command, as_root=self.is_rooted,
+                                    check_exit_code=check_exit_code)
+
+        result = {}
+
+        # Unpack the archive in memory
+        tar_gz = base64.b64decode(output)
+        tar_gz_bytes = io.BytesIO(tar_gz)
+        tar_buf = gzip.GzipFile(fileobj=tar_gz_bytes).read()
+        tar_bytes = io.BytesIO(tar_buf)
+        with tarfile.open(fileobj=tar_bytes) as tar:
+            for member in tar.getmembers():
+                try:
+                    content_f = tar.extractfile(member)
+                # ignore exotic members like sockets
+                except Exception:
+                    continue
+                # if it is a file and not a folder
+                if content_f:
+                    content = content_f.read()
+                    if decode_unicode:
+                        try:
+                            content = content.decode('utf-8').strip()
+                            if strip_null_chars:
+                                content = content.replace('\x00', '').strip()
+                        except UnicodeDecodeError:
+                            content = ''
+
+                    name = self.path.join(path, member.name)
+                    result[name] = content
+
+        return result
+
     def read_tree_values_flat(self, path, depth=1, check_exit_code=True):
         command = 'read_tree_values {} {}'.format(quote(path), depth)
         output = self._execute_util(command, as_root=self.is_rooted,
@@ -699,10 +796,44 @@ class Target(object):
         result = {k: '\n'.join(v).strip() for k, v in accumulator.items()}
         return result
 
-    def read_tree_values(self, path, depth=1, dictcls=dict, check_exit_code=True):
-        value_map = self.read_tree_values_flat(path, depth, check_exit_code)
+    def read_tree_values(self, path, depth=1, dictcls=dict,
+                         check_exit_code=True, tar=False, decode_unicode=True,
+                         strip_null_chars=True):
+        """
+        Reads the content of all files under a given tree
+
+        :path: path to the tree
+        :depth: maximum tree depth to read
+        :dictcls: type of the dict used to store the results
+        :check_exit_code: raise an exception if the shutil command fails
+        :tar: fetch the entire tree using tar rather than just the value (more
+              robust but slower in some use-cases)
+        :decode_unicode: decode the content of tar-ed files as utf-8
+        :strip_null_chars: remove '\x00' chars from the content of utf-8
+                           decoded files
+
+        :returns: a tree-like dict with the content of files as leafs
+        """
+        if not tar:
+            value_map = self.read_tree_values_flat(path, depth, check_exit_code)
+        else:
+            value_map = self.read_tree_tar_flat(path, depth, check_exit_code,
+                                                decode_unicode,
+                                                strip_null_chars)
         return _build_path_tree(value_map, path, self.path.sep, dictcls)
 
+    def install_module(self, mod, **params):
+        mod = get_module(mod)
+        if mod.stage == 'early':
+            msg = 'Module {} cannot be installed after device setup has already occoured.'
+            raise TargetStableError(msg)
+
+        if mod.probe(self):
+            self._install_module(mod, **params)
+        else:
+            msg = 'Module {} is not supported by the target'.format(mod.name)
+            raise TargetStableError(msg)
+
     # internal methods
 
     def _setup_shutils(self):
@@ -772,7 +903,11 @@ class Target(object):
     def _install_module(self, mod, **params):
         if mod.name not in self._installed_modules:
             self.logger.debug('Installing module {}'.format(mod.name))
-            mod.install(self, **params)
+            try:
+                mod.install(self, **params)
+            except Exception as e:
+                self.logger.error('Module "{}" failed to install on target'.format(mod.name))
+                raise
             self._installed_modules[mod.name] = mod
         else:
             self.logger.debug('Module {} is already installed.'.format(mod.name))
@@ -849,17 +984,6 @@ class LinuxTarget(Target):
             os_version[name] = convert_new_lines(output.strip()).replace('\n', ' ')
         return os_version
 
-    @property
-    @memoized
-    # There is currently no better way to do this cross platform.
-    # ARM does not have dmidecode
-    def model(self):
-        if self.file_exists("/proc/device-tree/model"):
-            raw_model = self.execute("cat /proc/device-tree/model")
-            device_model_to_return = '_'.join(raw_model.split()[:2])
-            return device_model_to_return.rstrip(' \t\r\n\0')
-        return None
-
     @property
     @memoized
     def system_id(self):
@@ -932,7 +1056,7 @@ class LinuxTarget(Target):
     def install(self, filepath, timeout=None, with_name=None):  # pylint: disable=W0221
         destpath = self.path.join(self.executables_directory,
                                   with_name and with_name or self.path.basename(filepath))
-        self.push(filepath, destpath)
+        self.push(filepath, destpath, timeout=timeout)
         self.execute('chmod a+x {}'.format(quote(destpath)), timeout=timeout)
         self._installed_binaries[self.path.basename(destpath)] = destpath
         return destpath
@@ -1032,14 +1156,6 @@ class AndroidTarget(Target):
         output = self.execute('content query --uri content://settings/secure --projection value --where "name=\'android_id\'"').strip()
         return output.split('value=')[-1]
 
-    @property
-    @memoized
-    def model(self):
-        try:
-            return self.getprop(prop='ro.product.device')
-        except KeyError:
-            return None
-
     @property
     @memoized
     def system_id(self):
@@ -1094,7 +1210,7 @@ class AndroidTarget(Target):
         except (DevlibTransientError, subprocess.CalledProcessError):
             # on some targets "reboot" doesn't return gracefully
             pass
-        self._connected_as_root = None
+        self.conn.connected_as_root = None
 
     def wait_boot_complete(self, timeout=10):
         start = time.time()
@@ -1109,13 +1225,6 @@ class AndroidTarget(Target):
 
     def connect(self, timeout=30, check_boot_completed=True):  # pylint: disable=arguments-differ
         device = self.connection_settings.get('device')
-        if device and ':' in device:
-            # ADB does not automatically remove a network device from it's
-            # devices list when the connection is broken by the remote, so the
-            # adb connection may have gone "stale", resulting in adb blocking
-            # indefinitely when making calls to the device. To avoid this,
-            # always disconnect first.
-            adb_disconnect(device)
         super(AndroidTarget, self).connect(timeout=timeout, check_boot_completed=check_boot_completed)
 
     def kick_off(self, command, as_root=None):
@@ -1156,7 +1265,7 @@ class AndroidTarget(Target):
         if ext == '.apk':
             return self.install_apk(filepath, timeout)
         else:
-            return self.install_executable(filepath, with_name)
+            return self.install_executable(filepath, with_name, timeout)
 
     def uninstall(self, name):
         if self.package_is_installed(name):
@@ -1319,7 +1428,14 @@ class AndroidTarget(Target):
             if self.get_sdk_version() >= 23:
                 flags.append('-g')  # Grant all runtime permissions
             self.logger.debug("Replace APK = {}, ADB flags = '{}'".format(replace, ' '.join(flags)))
-            return adb_command(self.adb_name, "install {} {}".format(' '.join(flags), quote(filepath)), timeout=timeout)
+            if isinstance(self.conn, AdbConnection):
+                return adb_command(self.adb_name, "install {} {}".format(' '.join(flags), quote(filepath)), timeout=timeout)
+            else:
+                dev_path = self.get_workpath(filepath.rsplit(os.path.sep, 1)[-1])
+                self.push(quote(filepath), dev_path, timeout=timeout)
+                result = self.execute("pm install {} {}".format(' '.join(flags), quote(dev_path)), timeout=timeout)
+                self.remove(dev_path)
+                return result
         else:
             raise TargetStableError('Can\'t install {}: unsupported format.'.format(filepath))
 
@@ -1366,21 +1482,25 @@ class AndroidTarget(Target):
                   '-n com.android.providers.media/.MediaScannerReceiver'
         self.execute(command.format(quote('file://'+dirpath)), as_root=as_root)
 
-    def install_executable(self, filepath, with_name=None):
+    def install_executable(self, filepath, with_name=None, timeout=None):
         self._ensure_executables_directory_is_writable()
         executable_name = with_name or os.path.basename(filepath)
         on_device_file = self.path.join(self.working_directory, executable_name)
         on_device_executable = self.path.join(self.executables_directory, executable_name)
-        self.push(filepath, on_device_file)
+        self.push(filepath, on_device_file, timeout=timeout)
         if on_device_file != on_device_executable:
-            self.execute('cp {} {}'.format(quote(on_device_file), quote(on_device_executable)), as_root=self.needs_su)
+            self.execute('cp {} {}'.format(quote(on_device_file), quote(on_device_executable)),
+                         as_root=self.needs_su, timeout=timeout)
             self.remove(on_device_file, as_root=self.needs_su)
         self.execute("chmod 0777 {}".format(quote(on_device_executable)), as_root=self.needs_su)
         self._installed_binaries[executable_name] = on_device_executable
         return on_device_executable
 
     def uninstall_package(self, package):
-        adb_command(self.adb_name, "uninstall {}".format(quote(package)), timeout=30)
+        if isinstance(self.conn, AdbConnection):
+            adb_command(self.adb_name, "uninstall {}".format(quote(package)), timeout=30)
+        else:
+            self.execute("pm uninstall {}".format(quote(package)), timeout=30)
 
     def uninstall_executable(self, executable_name):
         on_device_executable = self.path.join(self.executables_directory, executable_name)
@@ -1390,34 +1510,31 @@ class AndroidTarget(Target):
     def dump_logcat(self, filepath, filter=None, append=False, timeout=30):  # pylint: disable=redefined-builtin
         op = '>>' if append else '>'
         filtstr = ' -s {}'.format(quote(filter)) if filter else ''
-        command = 'logcat -d{} {} {}'.format(filtstr, op, quote(filepath))
-        adb_command(self.adb_name, command, timeout=timeout)
+        if isinstance(self.conn, AdbConnection):
+            command = 'logcat -d{} {} {}'.format(filtstr, op, quote(filepath))
+            adb_command(self.adb_name, command, timeout=timeout)
+        else:
+            dev_path = self.get_workpath('logcat')
+            command = 'logcat -d{} {} {}'.format(filtstr, op, quote(dev_path))
+            self.execute(command, timeout=timeout)
+            self.pull(dev_path, filepath)
+            self.remove(dev_path)
 
     def clear_logcat(self):
         with self.clear_logcat_lock:
-            adb_command(self.adb_name, 'logcat -c', timeout=30)
+            if isinstance(self.conn, AdbConnection):
+                adb_command(self.adb_name, 'logcat -c', timeout=30)
+            else:
+                self.execute('logcat -c', timeout=30)
 
     def get_logcat_monitor(self, regexps=None):
         return LogcatMonitor(self, regexps)
 
-    def adb_kill_server(self, timeout=30):
-        adb_command(self.adb_name, 'kill-server', timeout)
+    def wait_for_device(self, timeout=30):
+        self.conn.wait_for_device()
 
-    def adb_wait_for_device(self, timeout=30):
-        adb_command(self.adb_name, 'wait-for-device', timeout)
-
-    def adb_reboot_bootloader(self, timeout=30):
-        adb_command(self.adb_name, 'reboot-bootloader', timeout)
-
-    def adb_root(self, enable=True, force=False):
-        if enable:
-            if self._connected_as_root and not force:
-                return
-            adb_command(self.adb_name, 'root', timeout=30)
-            self._connected_as_root = True
-            return
-        adb_command(self.adb_name, 'unroot', timeout=30)
-        self._connected_as_root = False
+    def reboot_bootloader(self, timeout=30):
+        self.conn.reboot_bootloader()
 
     def is_screen_on(self):
         output = self.execute('dumpsys power')
@@ -1722,8 +1839,56 @@ class KernelVersion(object):
     __repr__ = __str__
 
 
-class KernelConfig(object):
+class HexInt(long):
+    """
+    Subclass of :class:`int` that uses hexadecimal formatting by default.
+    """
 
+    def __new__(cls, val=0, base=16):
+        super_new = super(HexInt, cls).__new__
+        if isinstance(val, Number):
+            return super_new(cls, val)
+        else:
+            return super_new(cls, val, base=base)
+
+    def __str__(self):
+        return hex(self).strip('L')
+
+
+class KernelConfigTristate(Enum):
+    YES = 'y'
+    NO = 'n'
+    MODULE = 'm'
+
+    def __bool__(self):
+        """
+        Allow using this enum to represent bool Kconfig type, although it is
+        technically different from tristate.
+        """
+        return self in (self.YES, self.MODULE)
+
+    def __nonzero__(self):
+        """
+        For Python 2.x compatibility.
+        """
+        return self.__bool__()
+
+    @classmethod
+    def from_str(cls, str_):
+        for state in cls:
+            if state.value == str_:
+                return state
+        raise ValueError('No kernel config tristate value matches "{}"'.format(str_))
+
+
+class TypedKernelConfig(Mapping):
+    """
+    Mapping-like typed version of :class:`KernelConfig`.
+
+    Values are either :class:`str`, :class:`int`,
+    :class:`KernelConfigTristate`, or :class:`HexInt`. ``hex`` Kconfig type is
+    mapped to :class:`HexInt` and ``bool`` to :class:`KernelConfigTristate`.
+    """
     not_set_regex = re.compile(r'# (\S+) is not set')
 
     @staticmethod
@@ -1733,50 +1898,207 @@ class KernelConfig(object):
             name = 'CONFIG_' + name
         return name
 
-    def iteritems(self):
-        return iter(self._config.items())
+    def __init__(self, mapping=None):
+        mapping = mapping if mapping is not None else {}
+        self._config = {
+            # Ensure we use the canonical name of the config keys for internal
+            # representation
+            self.get_config_name(k): v
+            for k, v in dict(mapping).items()
+        }
 
-    def __init__(self, text):
-        self.text = text
-        self._config = {}
-        for line in text.split('\n'):
+    @classmethod
+    def from_str(cls, text):
+        """
+        Build a :class:`TypedKernelConfig` out of the string content of a
+        Kconfig file.
+        """
+        return cls(cls._parse_text(text))
+
+    @staticmethod
+    def _val_to_str(val):
+        "Convert back values to Kconfig-style string value"
+        # Special case the gracefully handle the output of get()
+        if val is None:
+            return None
+        elif isinstance(val, KernelConfigTristate):
+            return val.value
+        elif isinstance(val, basestring):
+            return '"{}"'.format(val.strip('"'))
+        else:
+            return str(val)
+
+    def __str__(self):
+        return '\n'.join(
+            '{}={}'.format(k, self._val_to_str(v))
+            for k, v in self.items()
+        )
+
+    @staticmethod
+    def _parse_val(k, v):
+        """
+        Parse a value of types handled by Kconfig:
+            * string
+            * bool
+            * tristate
+            * hex
+            * int
+
+        Since bool cannot be distinguished from tristate, tristate is
+        always used. :meth:`KernelConfigTristate.__bool__` will allow using
+        it as a bool though, so it should not impact user code.
+        """
+        if not v:
+            return None
+
+        # Handle "string" type
+        if v.startswith('"'):
+            # Strip enclosing "
+            return v[1:-1]
+
+        else:
+            try:
+                # Handles "bool" and "tristate" types
+                return KernelConfigTristate.from_str(v)
+            except ValueError:
+                pass
+
+            try:
+                # Handles "int" type
+                return int(v)
+            except ValueError:
+                pass
+
+            try:
+                # Handles "hex" type
+                return HexInt(v)
+            except ValueError:
+                pass
+
+            # If no type could be parsed
+            raise ValueError('Could not parse Kconfig key: {}={}'.format(
+                    k, v
+                ), k, v
+            )
+
+    @classmethod
+    def _parse_text(cls, text):
+        config = {}
+        for line in text.splitlines():
             line = line.strip()
+
+            # skip empty lines
+            if not line:
+                continue
+
             if line.startswith('#'):
-                match = self.not_set_regex.search(line)
+                match = cls.not_set_regex.search(line)
                 if match:
-                    self._config[match.group(1)] = 'n'
-            elif '=' in line:
+                    value = 'n'
+                    name = match.group(1)
+                else:
+                    continue
+            else:
                 name, value = line.split('=', 1)
-                self._config[name.strip()] = value.strip()
 
-    def get(self, name, strict=False):
+            name = cls.get_config_name(name.strip())
+            value = cls._parse_val(name, value.strip())
+            config[name] = value
+        return config
+
+    def __getitem__(self, name):
         name = self.get_config_name(name)
-        res = self._config.get(name)
+        try:
+            return self._config[name]
+        except KeyError:
+            raise KernelConfigKeyError(
+                "{} is not exposed in kernel config".format(name),
+                name
+            )
 
-        if not res and strict:
-            raise IndexError("{} is not exposed in target's config")
+    def __iter__(self):
+        return iter(self._config)
 
-        return self._config.get(name)
+    def __len__(self):
+        return len(self._config)
+
+    def __contains__(self, name):
+        name = self.get_config_name(name)
+        return name in self._config
 
     def like(self, name):
         regex = re.compile(name, re.I)
-        result = {}
-        for k, v in self._config.items():
-            if regex.search(k):
-                result[k] = v
-        return result
+        return {
+            k: v for k, v in self.items()
+            if regex.search(k)
+        }
 
     def is_enabled(self, name):
-        return self.get(name) == 'y'
+        return self.get(name) is KernelConfigTristate.YES
 
     def is_module(self, name):
-        return self.get(name) == 'm'
+        return self.get(name) is KernelConfigTristate.MODULE
 
     def is_not_set(self, name):
-        return self.get(name) == 'n'
+        return self.get(name) is KernelConfigTristate.NO
 
     def has(self, name):
-        return self.get(name) in ['m', 'y']
+        return self.is_enabled(name) or self.is_module(name)
+
+
+class KernelConfig(object):
+    """
+    Backward compatibility shim on top of :class:`TypedKernelConfig`.
+
+    This class does not provide a Mapping API and only return string values.
+    """
+    @staticmethod
+    def get_config_name(name):
+        return TypedKernelConfig.get_config_name(name)
+
+    def __init__(self, text):
+        # Expose typed_config as a non-private attribute, so that user code
+        # needing it can get it from any existing producer of KernelConfig.
+        self.typed_config = TypedKernelConfig.from_str(text)
+        # Expose the original text for backward compatibility
+        self.text = text
+
+    def __bool__(self):
+        return bool(self.typed_config)
+
+    not_set_regex = TypedKernelConfig.not_set_regex
+
+    def iteritems(self):
+        for k, v in self.typed_config.items():
+            yield (k, self.typed_config._val_to_str(v))
+
+    items = iteritems
+
+    def get(self, name, strict=False):
+        if strict:
+            val = self.typed_config[name]
+        else:
+            val = self.typed_config.get(name)
+
+        return self.typed_config._val_to_str(val)
+
+    def like(self, name):
+        return {
+            k: self.typed_config._val_to_str(v)
+            for k, v in self.typed_config.like(name).items()
+        }
+
+    def is_enabled(self, name):
+        return self.typed_config.is_enabled(name)
+
+    def is_module(self, name):
+        return self.typed_config.is_module(name)
+
+    def is_not_set(self, name):
+        return self.typed_config.is_not_set(name)
+
+    def has(self, name):
+        return self.typed_config.has(name)
 
 
 class LocalLinuxTarget(LinuxTarget):

devlib/trace/perf.py

@@ -1,137 +0,0 @@
-#    Copyright 2018 ARM Limited
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-import os
-import re
-from past.builtins import basestring, zip
-
-from devlib.host import PACKAGE_BIN_DIRECTORY
-from devlib.trace import TraceCollector
-from devlib.utils.misc import ensure_file_directory_exists as _f
-
-
-PERF_COMMAND_TEMPLATE = '{} stat {} {} sleep 1000 > {} 2>&1 '
-
-PERF_COUNT_REGEX = re.compile(r'^(CPU\d+)?\s*(\d+)\s*(.*?)\s*(\[\s*\d+\.\d+%\s*\])?\s*$')
-
-DEFAULT_EVENTS = [
-    'migrations',
-    'cs',
-]
-
-
-class PerfCollector(TraceCollector):
-    """
-    Perf is a Linux profiling with performance counters.
-
-    Performance counters are CPU hardware registers that count hardware events
-    such as instructions executed, cache-misses suffered, or branches
-    mispredicted. They form a basis for profiling applications to trace dynamic
-    control flow and identify hotspots.
-
-    pref accepts options and events. If no option is given the default '-a' is
-    used. For events, the default events are migrations and cs. They both can
-    be specified in the config file.
-
-    Events must be provided as a list that contains them and they will look like
-    this ::
-
-        perf_events = ['migrations', 'cs']
-
-    Events can be obtained by typing the following in the command line on the
-    device ::
-
-        perf list
-
-    Whereas options, they can be provided as a single string as following ::
-
-        perf_options = '-a -i'
-
-    Options can be obtained by running the following in the command line ::
-
-        man perf-stat
-    """
-
-    def __init__(self, target,
-                 events=None,
-                 optionstring=None,
-                 labels=None,
-                 force_install=False):
-        super(PerfCollector, self).__init__(target)
-        self.events = events if events else DEFAULT_EVENTS
-        self.force_install = force_install
-        self.labels = labels
-
-        # Validate parameters
-        if isinstance(optionstring, list):
-            self.optionstrings = optionstring
-        else:
-            self.optionstrings = [optionstring]
-        if self.events and isinstance(self.events, basestring):
-            self.events = [self.events]
-        if not self.labels:
-            self.labels = ['perf_{}'.format(i) for i in range(len(self.optionstrings))]
-        if len(self.labels) != len(self.optionstrings):
-            raise ValueError('The number of labels must match the number of optstrings provided for perf.')
-
-        self.binary = self.target.get_installed('perf')
-        if self.force_install or not self.binary:
-            self.binary = self._deploy_perf()
-
-        self.commands = self._build_commands()
-
-    def reset(self):
-        self.target.killall('perf', as_root=self.target.is_rooted)
-        for label in self.labels:
-            filepath = self._get_target_outfile(label)
-            self.target.remove(filepath)
-
-    def start(self):
-        for command in self.commands:
-            self.target.kick_off(command)
-
-    def stop(self):
-        self.target.killall('sleep', as_root=self.target.is_rooted)
-
-    # pylint: disable=arguments-differ
-    def get_trace(self, outdir):
-        for label in self.labels:
-            target_file = self._get_target_outfile(label)
-            host_relpath = os.path.basename(target_file)
-            host_file = _f(os.path.join(outdir, host_relpath))
-            self.target.pull(target_file, host_file)
-
-    def _deploy_perf(self):
-        host_executable = os.path.join(PACKAGE_BIN_DIRECTORY,
-                                       self.target.abi, 'perf')
-        return self.target.install(host_executable)
-
-    def _build_commands(self):
-        commands = []
-        for opts, label in zip(self.optionstrings, self.labels):
-            commands.append(self._build_perf_command(opts, self.events, label))
-        return commands
-
-    def _get_target_outfile(self, label):
-        return self.target.get_workpath('{}.out'.format(label))
-
-    def _build_perf_command(self, options, events, label):
-        event_string = ' '.join(['-e {}'.format(e) for e in events])
-        command = PERF_COMMAND_TEMPLATE.format(self.binary,
-                                               options or '',
-                                               event_string,
-                                               self._get_target_outfile(label))
-        return command

devlib/utils/android.py

@@ -28,7 +28,13 @@ import tempfile
 import subprocess
 from collections import defaultdict
 import pexpect
-from pipes import quote
+import xml.etree.ElementTree
+import zipfile
+
+try:
+    from shlex import quote
+except ImportError:
+    from pipes import quote
 
 from devlib.exception import TargetTransientError, TargetStableError, HostError
 from devlib.utils.misc import check_output, which, ABI_MAP
@@ -42,7 +48,8 @@ AM_START_ERROR = re.compile(r"Error: Activity.*")
 # See:
 # http://developer.android.com/guide/topics/manifest/uses-sdk-element.html#ApiLevels
 ANDROID_VERSION_MAP = {
-    28: 'P',
+    29: 'Q',
+    28: 'PIE',
     27: 'OREO_MR1',
     26: 'OREO',
     25: 'NOUGAT_MR1',
@@ -132,6 +139,7 @@ class ApkInfo(object):
     version_regex = re.compile(r"name='(?P<name>[^']+)' versionCode='(?P<vcode>[^']+)' versionName='(?P<vname>[^']+)'")
     name_regex = re.compile(r"name='(?P<name>[^']+)'")
     permission_regex = re.compile(r"name='(?P<permission>[^']+)'")
+    activity_regex = re.compile(r'\s*A:\s*android:name\(0x\d+\)=".(?P<name>\w+)"')
 
     def __init__(self, path=None):
         self.path = path
@@ -147,15 +155,7 @@ class ApkInfo(object):
     # pylint: disable=too-many-branches
     def parse(self, apk_path):
         _check_env()
-        command = [aapt, 'dump', 'badging', apk_path]
-        logger.debug(' '.join(command))
-        try:
-            output = subprocess.check_output(command, stderr=subprocess.STDOUT)
-            if sys.version_info[0] == 3:
-                output = output.decode(sys.stdout.encoding or 'utf-8', 'replace')
-        except subprocess.CalledProcessError as e:
-            raise HostError('Error parsing APK file {}. `aapt` says:\n{}'
-                            .format(apk_path, e.output))
+        output = self._run([aapt, 'dump', 'badging', apk_path])
         for line in output.split('\n'):
             if line.startswith('application-label:'):
                 self.label = line.split(':')[1].strip().replace('\'', '')
@@ -188,49 +188,93 @@ class ApkInfo(object):
             else:
                 pass  # not interested
 
+        self._apk_path = apk_path
+        self._activities = None
+        self._methods = None
+
+    @property
+    def activities(self):
+        if self._activities is None:
+            cmd = [aapt, 'dump', 'xmltree', self._apk_path,
+                   'AndroidManifest.xml']
+            matched_activities = self.activity_regex.finditer(self._run(cmd))
+            self._activities = [m.group('name') for m in matched_activities]
+        return self._activities
+
+    @property
+    def methods(self):
+        if self._methods is None:
+            with zipfile.ZipFile(self._apk_path, 'r') as z:
+                extracted = z.extract('classes.dex', tempfile.gettempdir())
+
+            dexdump = os.path.join(os.path.dirname(aapt), 'dexdump')
+            command = [dexdump, '-l', 'xml', extracted]
+            dump = self._run(command)
+
+            xml_tree = xml.etree.ElementTree.fromstring(dump)
+
+            package = next(i for i in xml_tree.iter('package')
+                           if i.attrib['name'] == self.package)
+
+            self._methods = [(meth.attrib['name'], klass.attrib['name'])
+                             for klass in package.iter('class')
+                             for meth in klass.iter('method')]
+        return self._methods
+
+    def _run(self, command):
+        logger.debug(' '.join(command))
+        try:
+            output = subprocess.check_output(command, stderr=subprocess.STDOUT)
+            if sys.version_info[0] == 3:
+                output = output.decode(sys.stdout.encoding or 'utf-8', 'replace')
+        except subprocess.CalledProcessError as e:
+            raise HostError('Error while running "{}":\n{}'
+                            .format(command, e.output))
+        return output
+
 
 class AdbConnection(object):
 
     # maintains the count of parallel active connections to a device, so that
     # adb disconnect is not invoked untill all connections are closed
     active_connections = defaultdict(int)
+    # Track connected as root status per device
+    _connected_as_root = defaultdict(lambda: None)
     default_timeout = 10
     ls_command = 'ls'
+    su_cmd = 'su -c {}'
 
     @property
     def name(self):
         return self.device
 
-    # Again, we need to handle boards where the default output format from ls is
-    # single column *and* boards where the default output is multi-column.
-    # We need to do this purely because the '-1' option causes errors on older
-    # versions of the ls tool in Android pre-v7.
-    def _setup_ls(self):
-        command = "shell '(ls -1); echo \"\n$?\"'"
-        try:
-            output = adb_command(self.device, command, timeout=self.timeout, adb_server=self.adb_server)
-        except subprocess.CalledProcessError as e:
-            raise HostError(
-                'Failed to set up ls command on Android device. Output:\n'
-                + e.output)
-        lines = output.splitlines()
-        retval = lines[-1].strip()
-        if int(retval) == 0:
-            self.ls_command = 'ls -1'
-        else:
-            self.ls_command = 'ls'
-        logger.debug("ls command is set to {}".format(self.ls_command))
+    @property
+    def connected_as_root(self):
+        if self._connected_as_root[self.device] is None:
+                result = self.execute('id')
+                self._connected_as_root[self.device] = 'uid=0(' in result
+        return self._connected_as_root[self.device]
+
+    @connected_as_root.setter
+    def connected_as_root(self, state):
+        self._connected_as_root[self.device] = state
+
 
     # pylint: disable=unused-argument
-    def __init__(self, device=None, timeout=None, platform=None, adb_server=None):
+    def __init__(self, device=None, timeout=None, platform=None, adb_server=None,
+                 adb_as_root=False):
         self.timeout = timeout if timeout is not None else self.default_timeout
         if device is None:
             device = adb_get_device(timeout=timeout, adb_server=adb_server)
         self.device = device
         self.adb_server = adb_server
+        self.adb_as_root = adb_as_root
+        if self.adb_as_root:
+            self.adb_root(enable=True)
         adb_connect(self.device)
         AdbConnection.active_connections[self.device] += 1
         self._setup_ls()
+        self._setup_su()
 
     def push(self, source, dest, timeout=None):
         if timeout is None:
@@ -260,7 +304,7 @@ class AdbConnection(object):
                 as_root=False, strip_colors=True, will_succeed=False):
         try:
             return adb_shell(self.device, command, timeout, check_exit_code,
-                             as_root, adb_server=self.adb_server)
+                             as_root, adb_server=self.adb_server, su_cmd=self.su_cmd)
         except TargetStableError as e:
             if will_succeed:
                 raise TargetTransientError(e)
@@ -268,11 +312,13 @@ class AdbConnection(object):
                 raise
 
     def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
-        return adb_background_shell(self.device, command, stdout, stderr, as_root)
+        return adb_background_shell(self.device, command, stdout, stderr, as_root, adb_server=self.adb_server)
 
     def close(self):
         AdbConnection.active_connections[self.device] -= 1
         if AdbConnection.active_connections[self.device] <= 0:
+            if self.adb_as_root:
+                self.adb_root(self.device, enable=False)
             adb_disconnect(self.device)
             del AdbConnection.active_connections[self.device]
 
@@ -282,6 +328,50 @@ class AdbConnection(object):
         # before the next one can be issued.
         pass
 
+    def adb_root(self, enable=True):
+        cmd = 'root' if enable else 'unroot'
+        output = adb_command(self.device, cmd, timeout=30)
+        if 'cannot run as root in production builds' in output:
+            raise TargetStableError(output)
+        AdbConnection._connected_as_root[self.device] = enable
+
+    def wait_for_device(self, timeout=30):
+        adb_command(self.device, 'wait-for-device', timeout)
+
+    def reboot_bootloader(self, timeout=30):
+        adb_command(self.device, 'reboot-bootloader', timeout)
+
+    # Again, we need to handle boards where the default output format from ls is
+    # single column *and* boards where the default output is multi-column.
+    # We need to do this purely because the '-1' option causes errors on older
+    # versions of the ls tool in Android pre-v7.
+    def _setup_ls(self):
+        command = "shell '(ls -1); echo \"\n$?\"'"
+        try:
+            output = adb_command(self.device, command, timeout=self.timeout, adb_server=self.adb_server)
+        except subprocess.CalledProcessError as e:
+            raise HostError(
+                'Failed to set up ls command on Android device. Output:\n'
+                + e.output)
+        lines = output.splitlines()
+        retval = lines[-1].strip()
+        if int(retval) == 0:
+            self.ls_command = 'ls -1'
+        else:
+            self.ls_command = 'ls'
+        logger.debug("ls command is set to {}".format(self.ls_command))
+
+    def _setup_su(self):
+        try:
+            # Try the new style of invoking `su`
+            self.execute('ls', timeout=self.timeout, as_root=True,
+                         check_exit_code=True)
+        # If failure assume either old style or unrooted. Here we will assume
+        # old style and root status will be verified later.
+        except (TargetStableError, TargetTransientError, TimeoutError):
+            self.su_cmd = 'echo {} | su'
+        logger.debug("su command is set to {}".format(quote(self.su_cmd)))
+
 
 def fastboot_command(command, timeout=None, device=None):
     _check_env()
@@ -341,6 +431,12 @@ def adb_connect(device, timeout=None, attempts=MAX_ATTEMPTS):
         tries += 1
         if device:
             if "." in device: # Connect is required only for ADB-over-IP
+                # ADB does not automatically remove a network device from it's
+                # devices list when the connection is broken by the remote, so the
+                # adb connection may have gone "stale", resulting in adb blocking
+                # indefinitely when making calls to the device. To avoid this,
+                # always disconnect first.
+                adb_disconnect(device)
                 command = 'adb connect {}'.format(quote(device))
                 logger.debug(command)
                 output, _ = check_output(command, shell=True, timeout=timeout)
@@ -380,25 +476,27 @@ def _ping(device):
 
 # pylint: disable=too-many-locals
 def adb_shell(device, command, timeout=None, check_exit_code=False,
-              as_root=False, adb_server=None):  # NOQA
+              as_root=False, adb_server=None, su_cmd='su -c {}'):  # NOQA
     _check_env()
-    if as_root:
-        command = 'echo {} | su'.format(quote(command))
-    device_part = []
-    if adb_server:
-        device_part = ['-H', adb_server]
-    device_part += ['-s', device] if device else []
 
     # On older combinations of ADB/Android versions, the adb host command always
     # exits with 0 if it was able to run the command on the target, even if the
     # command failed (https://code.google.com/p/android/issues/detail?id=3254).
     # Homogenise this behaviour by running the command then echoing the exit
-    # code.
-    adb_shell_command = '({}); echo \"\n$?\"'.format(command)
-    actual_command = ['adb'] + device_part + ['shell', adb_shell_command]
-    logger.debug('adb {} shell {}'.format(' '.join(device_part), command))
+    # code of the executed command itself.
+    command = r'({}); echo "\n$?"'.format(command)
+
+    parts = ['adb']
+    if adb_server is not None:
+        parts += ['-H', adb_server]
+    if device is not None:
+        parts += ['-s', device]
+    parts += ['shell',
+              command if not as_root else su_cmd.format(quote(command))]
+
+    logger.debug(' '.join(quote(part) for part in parts))
     try:
-        raw_output, _ = check_output(actual_command, timeout, shell=False, combined_output=True)
+        raw_output, _ = check_output(parts, timeout, shell=False, combined_output=True)
     except subprocess.CalledProcessError as e:
         raise TargetStableError(str(e))
 
@@ -439,16 +537,21 @@ def adb_shell(device, command, timeout=None, check_exit_code=False,
 def adb_background_shell(device, command,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE,
-                         as_root=False):
+                         as_root=False,
+                         adb_server=None):
     """Runs the sepcified command in a subprocess, returning the the Popen object."""
     _check_env()
     if as_root:
         command = 'echo {} | su'.format(quote(command))
-    device_string = ' -s {}'.format(device) if device else ''
+
+    device_string = ' -H {}'.format(adb_server) if adb_server else ''
+    device_string += ' -s {}'.format(device) if device else ''
     full_command = 'adb{} shell {}'.format(device_string, quote(command))
     logger.debug(full_command)
     return subprocess.Popen(full_command, stdout=stdout, stderr=stderr, shell=True)
 
+def adb_kill_server(self, timeout=30):
+    adb_command(None, 'kill-server', timeout)
 
 def adb_list_devices(adb_server=None):
     output = adb_command(None, 'devices', adb_server=adb_server)

devlib/utils/misc.py

@@ -19,11 +19,13 @@ Miscellaneous functions that don't fit anywhere else.
 
 """
 from __future__ import division
+from contextlib import contextmanager
 from functools import partial, reduce
 from itertools import groupby
 from operator import itemgetter
 
 import ctypes
+import functools
 import logging
 import os
 import pkgutil
@@ -38,6 +40,11 @@ import wrapt
 import warnings
 
 
+try:
+    from contextlib import ExitStack
+except AttributeError:
+    from contextlib2 import ExitStack
+
 from past.builtins import basestring
 
 # pylint: disable=redefined-builtin
@@ -695,3 +702,19 @@ def memoized(wrapped, instance, args, kwargs):  # pylint: disable=unused-argumen
         return __memo_cache[id_string]
 
     return memoize_wrapper(*args, **kwargs)
+
+@contextmanager
+def batch_contextmanager(f, kwargs_list):
+    """
+    Return a context manager that will call the ``f`` callable with the keyword
+    arguments dict in the given list, in one go.
+
+    :param f: Callable expected to return a context manager.
+
+    :param kwargs_list: list of kwargs dictionaries to be used to call ``f``.
+    :type kwargs_list: list(dict)
+    """
+    with ExitStack() as stack:
+        for kwargs in kwargs_list:
+            stack.enter_context(f(**kwargs))
+        yield

devlib/utils/rendering.py

@@ -49,12 +49,12 @@ class FrameCollector(threading.Thread):
         self.refresh_period = None
         self.drop_threshold = None
         self.unresponsive_count = 0
-        self.last_ready_time = None
+        self.last_ready_time = 0
         self.exc = None
         self.header = None
 
     def run(self):
-        logger.debug('Surface flinger frame data collection started.')
+        logger.debug('Frame data collection started.')
         try:
             self.stop_signal.clear()
             fd, self.temp_file = tempfile.mkstemp()
@@ -71,7 +71,7 @@ class FrameCollector(threading.Thread):
         except Exception as e:  # pylint: disable=W0703
             logger.warning('Exception on collector thread: {}({})'.format(e.__class__.__name__, e))
             self.exc = WorkerThreadError(self.name, sys.exc_info())
-        logger.debug('Surface flinger frame data collection stopped.')
+        logger.debug('Frame data collection stopped.')
 
     def stop(self):
         self.stop_signal.set()
@@ -133,7 +133,7 @@ class SurfaceFlingerFrameCollector(FrameCollector):
     def collect_frames(self, wfh):
         for activity in self.list():
             if activity == self.view:
-                wfh.write(self.get_latencies(activity))
+                wfh.write(self.get_latencies(activity).encode('utf-8'))
 
     def clear(self):
         self.target.execute('dumpsys SurfaceFlinger --latency-clear ')
@@ -147,32 +147,44 @@ class SurfaceFlingerFrameCollector(FrameCollector):
         return text.replace('\r\n', '\n').replace('\r', '\n').split('\n')
 
     def _process_raw_file(self, fh):
+        found = False
         text = fh.read().replace('\r\n', '\n').replace('\r', '\n')
         for line in text.split('\n'):
             line = line.strip()
-            if line:
-                self._process_trace_line(line)
+            if not line:
+                continue
+            if 'SurfaceFlinger appears to be unresponsive, dumping anyways' in line:
+                self.unresponsive_count += 1
+                continue
+            parts = line.split()
+            # We only want numerical data, ignore textual data.
+            try:
+                parts = list(map(int, parts))
+            except ValueError:
+                continue
+            found = True
+            self._process_trace_parts(parts)
+        if not found:
+            logger.warning('Could not find expected SurfaceFlinger output.')
 
-    def _process_trace_line(self, line):
-        parts = line.split()
+    def _process_trace_parts(self, parts):
         if len(parts) == 3:
-            frame = SurfaceFlingerFrame(*list(map(int, parts)))
+            frame = SurfaceFlingerFrame(*parts)
             if not frame.frame_ready_time:
                 return # "null" frame
             if frame.frame_ready_time <= self.last_ready_time:
                 return  # duplicate frame
             if (frame.frame_ready_time - frame.desired_present_time) > self.drop_threshold:
-                logger.debug('Dropping bogus frame {}.'.format(line))
+                logger.debug('Dropping bogus frame {}.'.format(' '.join(map(str, parts))))
                 return  # bogus data
             self.last_ready_time = frame.frame_ready_time
             self.frames.append(frame)
         elif len(parts) == 1:
-            self.refresh_period = int(parts[0])
+            self.refresh_period = parts[0]
             self.drop_threshold = self.refresh_period * 1000
-        elif 'SurfaceFlinger appears to be unresponsive, dumping anyways' in line:
-            self.unresponsive_count += 1
         else:
-            logger.warning('Unexpected SurfaceFlinger dump output: {}'.format(line))
+            msg = 'Unexpected SurfaceFlinger dump output: {}'.format(' '.join(map(str, parts)))
+            logger.warning(msg)
 
 
 def read_gfxinfo_columns(target):

devlib/utils/ssh.py

@@ -41,7 +41,8 @@ from pexpect import EOF, TIMEOUT, spawn
 # pylint: disable=redefined-builtin,wrong-import-position
 from devlib.exception import (HostError, TargetStableError, TargetNotRespondingError,
                               TimeoutError, TargetTransientError)
-from devlib.utils.misc import which, strip_bash_colors, check_output, sanitize_cmd_template
+from devlib.utils.misc import (which, strip_bash_colors, check_output,
+                               sanitize_cmd_template, memoized)
 from devlib.utils.types import boolean
 
 
@@ -53,7 +54,15 @@ sshpass = None
 logger = logging.getLogger('ssh')
 gem5_logger = logging.getLogger('gem5-connection')
 
-def ssh_get_shell(host, username, password=None, keyfile=None, port=None, timeout=10, telnet=False, original_prompt=None):
+def ssh_get_shell(host,
+                  username,
+                  password=None,
+                  keyfile=None,
+                  port=None,
+                  timeout=10,
+                  telnet=False,
+                  original_prompt=None,
+                  options=None):
     _check_env()
     start_time = time.time()
     while True:
@@ -62,7 +71,8 @@ def ssh_get_shell(host, username, password=None, keyfile=None, port=None, timeou
                 raise ValueError('keyfile may not be used with a telnet connection.')
             conn = TelnetPxssh(original_prompt=original_prompt)
         else:  # ssh
-            conn = pxssh.pxssh()
+            conn = pxssh.pxssh(options=options,
+                               echo=False)
 
         try:
             if keyfile:
@@ -157,6 +167,18 @@ class SshConnection(object):
     def name(self):
         return self.host
 
+    @property
+    def connected_as_root(self):
+        if self._connected_as_root is None:
+            # Execute directly to prevent deadlocking of connection
+            result = self._execute_and_wait_for_prompt('id', as_root=False)
+            self._connected_as_root = 'uid=0(' in result
+        return self._connected_as_root
+
+    @connected_as_root.setter
+    def connected_as_root(self, state):
+        self._connected_as_root = state
+
     # pylint: disable=unused-argument,super-init-not-called
     def __init__(self,
                  host,
@@ -169,8 +191,10 @@ class SshConnection(object):
                  password_prompt=None,
                  original_prompt=None,
                  platform=None,
-                 sudo_cmd="sudo -- sh -c {}"
+                 sudo_cmd="sudo -- sh -c {}",
+                 options=None
                  ):
+        self._connected_as_root = None
         self.host = host
         self.username = username
         self.password = password
@@ -181,7 +205,16 @@ class SshConnection(object):
         self.sudo_cmd = sanitize_cmd_template(sudo_cmd)
         logger.debug('Logging in {}@{}'.format(username, host))
         timeout = timeout if timeout is not None else self.default_timeout
-        self.conn = ssh_get_shell(host, username, password, self.keyfile, port, timeout, False, None)
+        self.options = options if options is not None else {}
+        self.conn = ssh_get_shell(host,
+                                  username,
+                                  password,
+                                  self.keyfile,
+                                  port,
+                                  timeout,
+                                  False,
+                                  None,
+                                  self.options)
         atexit.register(self.close)
 
     def push(self, source, dest, timeout=30):
@@ -231,9 +264,17 @@ class SshConnection(object):
         try:
             port_string = '-p {}'.format(self.port) if self.port else ''
             keyfile_string = '-i {}'.format(self.keyfile) if self.keyfile else ''
-            if as_root:
+            if as_root and not self.connected_as_root:
                 command = self.sudo_cmd.format(command)
-            command = '{} {} {} {}@{} {}'.format(ssh, keyfile_string, port_string, self.username, self.host, command)
+            options = " ".join([ "-o {}={}".format(key,val)
+                                for key,val in self.options.items()])
+            command = '{} {} {} {} {}@{} {}'.format(ssh,
+                                                    options,
+                                                    keyfile_string,
+                                                    port_string,
+                                                    self.username,
+                                                    self.host,
+                                                    command)
             logger.debug(command)
             if self.password:
                 command, _ = _give_password(self.password, command)
@@ -253,39 +294,41 @@ class SshConnection(object):
         # simulate impatiently hitting ^C until command prompt appears
         logger.debug('Sending ^C')
         for _ in range(self.max_cancel_attempts):
-            self.conn.sendline(chr(3))
+            self._sendline(chr(3))
             if self.conn.prompt(0.1):
                 return True
         return False
 
+    def wait_for_device(self, timeout=30):
+        return
+
+    def reboot_bootloader(self, timeout=30):
+        raise NotImplementedError()
+
     def _execute_and_wait_for_prompt(self, command, timeout=None, as_root=False, strip_colors=True, log=True):
         self.conn.prompt(0.1)  # clear an existing prompt if there is one.
-        if self.username == 'root':
+        if as_root and self.connected_as_root:
             # As we're already root, there is no need to use sudo.
             as_root = False
         if as_root:
             command = self.sudo_cmd.format(quote(command))
             if log:
                 logger.debug(command)
-            self.conn.sendline(command)
+            self._sendline(command)
             if self.password:
                 index = self.conn.expect_exact([self.password_prompt, TIMEOUT], timeout=0.5)
                 if index == 0:
-                    self.conn.sendline(self.password)
+                    self._sendline(self.password)
         else:  # not as_root
             if log:
                 logger.debug(command)
-            self.conn.sendline(command)
+            self._sendline(command)
         timed_out = self._wait_for_prompt(timeout)
-        # the regex removes line breaks potential introduced when writing
-        # command to shell.
         if sys.version_info[0] == 3:
             output = process_backspaces(self.conn.before.decode(sys.stdout.encoding or 'utf-8', 'replace'))
         else:
             output = process_backspaces(self.conn.before)
-        output = re.sub(r'\r([^\n])', r'\1', output)
-        if '\r\n' in output: # strip the echoed command
-            output = output.split('\r\n', 1)[1]
+
         if timed_out:
             self.cancel_running_command()
             raise TimeoutError(command, output)
@@ -308,7 +351,14 @@ class SshConnection(object):
         # only specify -P for scp if the port is *not* the default.
         port_string = '-P {}'.format(quote(str(self.port))) if (self.port and self.port != 22) else ''
         keyfile_string = '-i {}'.format(quote(self.keyfile)) if self.keyfile else ''
-        command = '{} -r {} {} {} {}'.format(scp, keyfile_string, port_string, quote(source), quote(dest))
+        options = " ".join(["-o {}={}".format(key,val)
+                            for key,val in self.options.items()])
+        command = '{} {} -r {} {} {} {}'.format(scp,
+                                                options,
+                                                keyfile_string,
+                                                port_string,
+                                                quote(source),
+                                                quote(dest))
         command_redacted = command
         logger.debug(command)
         if self.password:
@@ -321,6 +371,21 @@ class SshConnection(object):
         except TimeoutError as e:
             raise TimeoutError(command_redacted, e.output)
 
+    def _sendline(self, command):
+        # Workaround for https://github.com/pexpect/pexpect/issues/552
+        if len(command) == self._get_window_size()[1] - self._get_prompt_length():
+            command += ' '
+        self.conn.sendline(command)
+
+    @memoized
+    def _get_prompt_length(self):
+        self.conn.sendline()
+        self.conn.prompt()
+        return len(self.conn.after)
+
+    @memoized
+    def _get_window_size(self):
+        return self.conn.getwinsize()
 
 class TelnetConnection(SshConnection):
 
@@ -575,6 +640,19 @@ class Gem5Connection(TelnetConnection):
         # Delete the lock file
         os.remove(self.lock_file_name)
 
+    def wait_for_device(self, timeout=30):
+        """
+        Wait for Gem5 to be ready for interation with a timeout.
+        """
+        for _ in attempts(timeout):
+            if self.ready:
+                return
+            time.sleep(1)
+        raise TimeoutError('Gem5 is not ready for interaction')
+
+    def reboot_bootloader(self, timeout=30):
+        raise NotImplementedError()
+
     # Functions only to be called by the Gem5 connection itself
     def _connect_gem5_platform(self, platform):
         port = platform.gem5_port

devlib/utils/version.py

@@ -15,8 +15,23 @@
 
 import os
 import sys
+from collections import namedtuple
 from subprocess import Popen, PIPE
 
+
+VersionTuple = namedtuple('Version', ['major', 'minor', 'revision', 'dev'])
+
+version = VersionTuple(1, 2, 0, '')
+
+
+def get_devlib_version():
+    version_string = '{}.{}.{}'.format(
+        version.major, version.minor, version.revision)
+    if version.dev:
+        version_string += '.{}'.format(version.dev)
+    return version_string
+
+
 def get_commit():
     p = Popen(['git', 'rev-parse', 'HEAD'], cwd=os.path.dirname(__file__),
               stdout=PIPE, stderr=PIPE)

doc/collectors.rst

@@ -0,0 +1,153 @@
+.. _collector:
+
+Collectors
+==========
+
+The ``Collector`` API provide a consistent way of collecting arbitrary data from
+a target. Data is collected via an instance of a class derived from
+:class:`CollectorBase`.
+
+
+Example
+-------
+
+The following example shows how to use a collector to read the logcat output
+from an Android target.
+
+.. code-block:: python
+
+    # import and instantiate the Target and the collector
+    # (note: this assumes exactly one android target connected
+    #  to the host machine).
+    In [1]: from devlib import AndroidTarget, LogcatCollector
+
+    In [2]: t = AndroidTarget()
+
+    # Set up the collector on the Target.
+
+    In [3]: collector = LogcatCollector(t)
+
+    # Configure the output file path for the collector to use.
+    In [4]: collector.set_output('adb_log.txt')
+
+    # Reset the Collector to preform any required configuration or preparation.
+    In [5]: collector.reset()
+
+    # Start Collecting
+    In [6]: collector.start()
+
+    # Wait for some output to be generated
+    In [7]: sleep(10)
+
+    # Stop Collecting
+    In [8]: collector.stop()
+
+    # Retrieved the collected data
+    In [9]: output = collector.get_data()
+
+    # Display the returned ``CollectorOutput`` Object.
+    In [10]: output
+    Out[10]: [<adb_log.txt (file)>]
+
+    In [11] log_file = output[0]
+
+    # Get the path kind of the the returned CollectorOutputEntry.
+    In [12]: log_file.path_kind
+    Out[12]: 'file'
+
+    # Get the path of the returned CollectorOutputEntry.
+    In [13]: log_file.path
+    Out[13]: 'adb_log.txt'
+
+    # Find the full path to the log file.
+    In [14]: os.path.join(os.getcwd(), logfile)
+    Out[14]: '/tmp/adb_log.txt'
+
+
+API
+---
+.. collector:
+
+.. module:: devlib.collector
+
+
+CollectorBase
+~~~~~~~~~~~~~
+
+.. class:: CollectorBase(target, \*\*kwargs)
+
+   A ``CollectorBase`` is the the base class and API that should be
+   implemented to allowing collecting various data from a traget e.g. traces,
+   logs etc.
+
+.. method::  Collector.setup(\*args, \*\*kwargs)
+
+   This will set up the collector on the target. Parameters this method takes
+   are particular to subclasses (see documentation for specific collectors
+   below).  What actions are performed by this method are also
+   collector-specific.  Usually these will be things like  installing
+   executables, starting services, deploying assets, etc. Typically, this method
+   needs to be invoked at most once per reboot of the target (unless
+   ``teardown()`` has been called), but see documentation for the collector
+   you're interested in.
+
+.. method:: CollectorBase.reset()
+
+   This can be used to configure a collector for collection. This must be invoked
+   before ``start()`` is called to begin collection.
+
+.. method:: CollectorBase.start()
+
+   Starts collecting from the target.
+
+.. method:: CollectorBase.stop()
+
+   Stops collecting from target. Must be called after
+   :func:`start()`.
+
+
+.. method:: CollectorBase.set_output(output_path)
+
+   Configure the output path for the particular collector. This will be either
+   a directory or file path which will be used when storing the data. Please see
+   the individual Collector documentation for more information.
+
+
+.. method:: CollectorBase.get_data()
+
+    The collected data will be return via the previously specified output_path.
+    This method will return a ``CollectorOutput`` object which is a subclassed
+    list object containing individual ``CollectorOutputEntry`` objects with details
+    about the individual output entry.
+
+
+CollectorOutputEntry
+~~~~~~~~~~~~~~~~~~~~
+
+This object is designed to allow for the output of a collector to be processed
+generically. The object will behave as a regular string containing the path to
+underlying output path and can be used directly in ``os.path`` operations.
+
+.. attribute:: CollectorOutputEntry.path
+
+    The file path for the corresponding output item.
+
+.. attribute:: CollectorOutputEntry.path_kind
+
+    The type of output the is specified in the ``path`` attribute. Current valid
+    kinds are: ``file`` and ``directory``.
+
+.. method:: CollectorOutputEntry.__init__(path, path_kind)
+
+    Initialises a ``CollectorOutputEntry`` object with the desired file path and
+    kind of file path specified.
+
+
+.. collectors:
+
+Available Collectors
+---------------------
+
+This section lists collectors that are currently part of devlib.
+
+.. todo:: Add collectors

doc/connection.rst

@@ -100,7 +100,7 @@ class that implements the following methods.
 Connection Types
 ----------------
 
-.. class:: AdbConnection(device=None, timeout=None)
+.. class:: AdbConnection(device=None, timeout=None, adb_server=None, adb_as_root=False)
 
     A connection to an android device via ``adb`` (Android Debug Bridge).
     ``adb`` is part of the Android SDK (though stand-alone versions are also
@@ -113,10 +113,13 @@ Connection Types
     :param timeout: Connection timeout in seconds. If a connection to the device
                     is not established within this period, :class:`HostError`
                     is raised.
+    :param adb_server: Allows specifying the address of the adb server to use.
+    :param adb_as_root: Specify whether the adb server should be restarted in root mode.
 
 
 .. class:: SshConnection(host, username, password=None, keyfile=None, port=None,\
-                         timeout=None, password_prompt=None)
+                         timeout=None, password_prompt=None, \
+                         sudo_cmd="sudo -- sh -c {}", options=None)
 
     A connection to a device on the network over SSH.
 
@@ -141,6 +144,8 @@ Connection Types
     :param password_prompt: A string with the password prompt used by
                             ``sshpass``. Set this if your version of ``sshpass``
                             uses something other than ``"[sudo] password"``.
+    :param sudo_cmd: Specify the format of the command used to grant sudo access.
+    :param options: A dictionary with extra ssh configuration options.
 
 
 .. class:: TelnetConnection(host, username, password=None, port=None,\

doc/index.rst

@@ -19,6 +19,7 @@ Contents:
    target
    modules
    instrumentation
+   collectors
    derived_measurements
    platform
    connection

doc/instrumentation.rst

@@ -1,3 +1,5 @@
+.. _instrumentation:
+
 Instrumentation
 ===============
 
@@ -164,10 +166,21 @@ Instrument
 .. method:: Instrument.get_raw()
 
    Returns a list of paths to files containing raw output from the underlying
-   source(s) that is used to produce the data CSV. If now raw output is
+   source(s) that is used to produce the data CSV. If no raw output is
    generated or saved, an empty list will be returned. The format of the
    contents of the raw files is entirely source-dependent.
 
+  .. note:: This method is not guaranteed to return valid filepaths after the
+            :meth:`teardown` method has been invoked as the raw files may have
+            been deleted. Please ensure that copies are created manually
+            prior to calling :meth:`teardown` if the files are to be retained.
+
+.. method:: Instrument.teardown()
+
+   Performs any required clean up of the instrument. This usually includes
+   removing temporary and raw files (if ``keep_raw`` is set to ``False`` on relevant
+   instruments), stopping services etc.
+
 .. attribute:: Instrument.sample_rate_hz
 
    Sample rate of the instrument in Hz. Assumed to be the same for all channels.
@@ -400,7 +413,7 @@ For reference, the software stack on the host is roughly given by:
 
 Ethernet was the only IIO Interface used and tested during the development of
 this instrument. However,
-`USB seems to be supported<https://gitlab.com/baylibre-acme/ACME/issues/2>`_.
+`USB seems to be supported <https://gitlab.com/baylibre-acme/ACME/issues/2>`_.
 The IIO library also provides "Local" and "XML" connections but these are to be
 used when the IIO devices are directly connected to the host *i.e.* in our
 case, if we were to run Python and devlib on the BBB. These are also untested.

doc/modules.rst

@@ -322,7 +322,7 @@ FlashModule
 
     "flash"
 
-.. method:: __call__(image_bundle=None, images=None, boot_config=None)
+.. method:: __call__(image_bundle=None, images=None, boot_config=None, connect=True)
 
     Must be implemented by derived classes.
 
@@ -338,6 +338,7 @@ FlashModule
     :param boot_config: Some platforms require specifying boot arguments at the
                         time of flashing the images, rather than during each
                         reboot. For other platforms, this will be ignored.
+    :connect: Specifiy whether to try and connect to the target after flashing.
 
 
 Module Registration

doc/overview.rst

@@ -6,8 +6,7 @@ There are currently four target interfaces:
 
 - :class:`LinuxTarget` for interacting with Linux devices over SSH.
 - :class:`AndroidTarget` for interacting with Android devices over adb.
-- :class:`ChromeOsTarget`: for interacting with ChromeOS devices over SSH, and
-                           their Android containers over adb.
+- :class:`ChromeOsTarget`: for interacting with ChromeOS devices over SSH, and their Android containers over adb.
 - :class:`LocalLinuxTarget`: for interacting with the local Linux host.
 
 They all work in more-or-less the same way, with the major difference being in
@@ -240,17 +239,17 @@ complete. Retrying it or bailing out is therefore a responsability of the caller
 The hierarchy is as follows:
 
 - :class:`DevlibError`
-   
+
    - :class:`WorkerThreadError`
    - :class:`HostError`
    - :class:`TargetError`
-      
+
       - :class:`TargetStableError`
       - :class:`TargetTransientError`
       - :class:`TargetNotRespondingError`
-   
+
    - :class:`DevlibStableError`
-      
+
       - :class:`TargetStableError`
 
    - :class:`DevlibTransientError`
@@ -307,12 +306,22 @@ has been successfully installed on a target, you can use ``has()`` method, e.g.
 
 Please see the modules documentation for more detail.
 
+Instruments and Collectors
+--------------------------
 
-Measurement and Trace
----------------------
+You can retrieve multiple types of data from a target. There are two categories
+of classes that allow for this:
 
-You can collected traces (currently, just ftrace) using
-:class:`TraceCollector`\ s. For example
+
+- An :class:`Instrument` which may be used to collect measurements (such as power) from
+  targets that support it. Please see the
+  :ref:`instruments documentation <Instrumentation>` for more details.
+
+- A :class:`Collector` may be used to collect arbitary data from a ``Target`` varying
+  from screenshots to trace data. Please see the
+  :ref:`collectors documentation <collector>` for more details.
+
+An example workflow using :class:`FTraceCollector` is as follows:
 
 .. code:: python
 
@@ -326,23 +335,19 @@ You can collected traces (currently, just ftrace) using
    # As a context manager, clear ftrace buffer using trace.reset(),
    # start trace collection using trace.start(), then stop it Using
    # trace.stop(). Using a context manager brings the guarantee that
-   # tracing will stop even if an exception occurs, including 
+   # tracing will stop even if an exception occurs, including
    # KeyboardInterrupt (ctr-C) and SystemExit (sys.exit)
    with trace:
       # Perform the operations you want to trace here...
       import time; time.sleep(5)
 
    # extract the trace file from the target into a local file
-   trace.get_trace('/tmp/trace.bin')
+   trace.get_data('/tmp/trace.bin')
 
    # View trace file using Kernelshark (must be installed on the host).
    trace.view('/tmp/trace.bin')
 
    # Convert binary trace into text format. This would normally be done
-   # automatically during get_trace(), unless autoreport is set to False during
+   # automatically during get_data(), unless autoreport is set to False during
    # instantiation of the trace collector.
    trace.report('/tmp/trace.bin', '/tmp/trace.txt')
-
-In a similar way, :class:`Instrument` instances may be used to collect
-measurements (such as power) from targets that support it. Please see
-instruments documentation for more details.

doc/target.rst

@@ -232,7 +232,7 @@ Target
    :param timeout: timeout (in seconds) for the transfer; if the transfer does
        not  complete within this period, an exception will be raised.
 
-.. method:: Target.execute(command [, timeout [, check_exit_code [, as_root [, strip_colors [, will_succeed]]]]])
+.. method:: Target.execute(command [, timeout [, check_exit_code [, as_root [, strip_colors [, will_succeed [, force_locale]]]]]])
 
    Execute the specified command on the target device and return its output.
 
@@ -252,6 +252,9 @@ Target
        will make the method always raise an instance of a subclass of
        :class:`DevlibTransientError` when the command fails, instead of a
        :class:`DevlibStableError`.
+   :param force_locale: Prepend ``LC_ALL=<force_locale>`` in front of the
+      command to get predictable output that can be more safely parsed.
+      If ``None``, no locale is prepended.
 
 .. method:: Target.background(command [, stdout [, stderr [, as_root]]])
 
@@ -346,7 +349,19 @@ Target
        some sysfs entries silently failing to set the written value without
        returning an error code.
 
-.. method:: Target.read_tree_values(path, depth=1, dictcls=dict):
+.. method:: Target.revertable_write_value(path, value [, verify])
+
+   Same as :meth:`Target.write_value`, but as a context manager that will write
+   back the previous value on exit.
+
+.. method:: Target.batch_revertable_write_value(kwargs_list)
+
+   Calls :meth:`Target.revertable_write_value` with all the keyword arguments
+   dictionary given in the list. This is a convenience method to update
+   multiple files at once, leaving them in their original state on exit. If one
+   write fails, all the already-performed writes will be reverted as well.
+
+.. method:: Target.read_tree_values(path, depth=1, dictcls=dict, [, tar [, decode_unicode [, strip_null_char ]]]):
 
    Read values of all sysfs (or similar) file nodes under ``path``, traversing
    up to the maximum depth ``depth``.
@@ -358,9 +373,18 @@ Target
    value is a dict-line object  with a key for every entry under ``path``
    mapping onto its value or further dict-like objects as appropriate.
 
+   Although the default behaviour should suit most users, it is possible to
+   encounter issues when reading binary files, or files with colons in their
+   name for example. In such cases, the ``tar`` parameter can be set to force a
+   full archive of the tree using tar, hence providing a more robust behaviour.
+   This can, however, slow down the read process significantly.
+
    :param path: sysfs path to scan
    :param depth: maximum depth to descend
    :param dictcls: a dict-like type to be used for each level of the hierarchy.
+   :param tar: the files will be read using tar rather than grep
+   :param decode_unicode: decode the content of tar-ed files as utf-8
+   :param strip_null_char: remove null chars from utf-8 decoded files
 
 .. method:: Target.read_tree_values_flat(path, depth=1):
 
@@ -521,6 +545,15 @@ Target
 
    :returns: ``True`` if internet seems available, ``False`` otherwise.
 
+.. method:: Target.install_module(mod, **params)
+
+  :param mod: The module name or object to be installed to the target.
+  :param params: Keyword arguments used to instantiate the module.
+
+    Installs an additional module to the target after the initial setup has been
+    performed.
+
+
 Android Target
 ---------------
 
@@ -615,6 +648,14 @@ Android Target
    Returns ``True`` if the targets screen is currently on and ``False``
    otherwise.
 
+.. method:: AndroidTarget.wait_for_target(timeout=30)
+
+    Returns when the devices becomes available withing the given timeout
+    otherwise returns a ``TimeoutError``.
+
+.. method:: AndroidTarget.reboot_bootloader(timeout=30)
+    Attempts to reboot the target into it's bootloader.
+
 .. method:: AndroidTarget.homescreen()
 
    Returns the device to its home screen.
@@ -656,7 +697,7 @@ ChromeOS Target
 
     :param android_executables_directory: This is the location of the
         executables directory to be used for the android container. If not
-        specified will default to a ``bin`` subfolder in the
+        specified will default to a ``bin`` subdirectory in the
         ``android_working_directory.``
 
     :param package_data_directory: This is the location of the data stored

setup.py

@@ -41,23 +41,13 @@ except OSError:
     pass
 
 
-with open(os.path.join(devlib_dir, '__init__.py')) as fh:
-    # Extract the version by parsing the text of the file,
-    # as may not be able to load as a module yet.
-    for line in fh:
-        if '__version__' in line:
-            parts = line.split("'")
-            __version__ = parts[1]
-            break
-    else:
-        raise RuntimeError('Did not see __version__')
-
-    vh_path = os.path.join(devlib_dir, 'utils', 'version.py')
-    # can load this, as it does not have any devlib imports
-    version_helper = imp.load_source('version_helper', vh_path)
-    commit = version_helper.get_commit()
-    if commit:
-        __version__ = '{}+{}'.format(__version__, commit)
+vh_path = os.path.join(devlib_dir, 'utils', 'version.py')
+# can load this, as it does not have any devlib imports
+version_helper = imp.load_source('version_helper', vh_path)
+__version__ = version_helper.get_devlib_version()
+commit = version_helper.get_commit()
+if commit:
+    __version__ = '{}+{}'.format(__version__, commit)
 
 
 packages = []
@@ -95,21 +85,24 @@ params = dict(
         'wrapt',  # Basic for construction of decorator functions
         'future', # Python 2-3 compatibility
         'enum34;python_version<"3.4"', # Enums for Python < 3.4
-        'pandas',
-        'numpy',
+        'contextlib2;python_version<"3.0"', # Python 3 contextlib backport for Python 2
+        'numpy<=1.16.4; python_version<"3"',
+        'numpy; python_version>="3"',
+        'pandas<=0.24.2; python_version<"3"',
+        'pandas; python_version>"3"',
     ],
     extras_require={
-        'daq': ['daqpower'],
+        'daq': ['daqpower>=2'],
         'doc': ['sphinx'],
         'monsoon': ['python-gflags'],
         'acme': ['pandas', 'numpy'],
     },
     # https://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers=[
-        'Development Status :: 4 - Beta',
+        'Development Status :: 5 - Production/Stable',
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: POSIX :: Linux',
-        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
     ],
 )