utils/version: Bump release version

Some apks do not contain the file that we use to determine app methods
so return an empty list in this case.

README.rst

@@ -14,6 +14,16 @@ Installation
         sudo -H pip install devlib
 
 
+Dependencies
+------------
+
+``devlib`` should install all dependencies automatically, however if you run
+into issues please ensure you are using that latest version of pip.
+
+On some systems there may additional steps required to install the dependency
+``paramiko`` please consult the `module documentation <http://www.paramiko.org/installing.html>`_
+for more information.
+
 Usage
 -----
 

devlib/__init__.py

@@ -45,9 +45,11 @@ from devlib.derived import DerivedMeasurements, DerivedMetric
 from devlib.derived.energy import DerivedEnergyMeasurements
 from devlib.derived.fps import DerivedGfxInfoStats, DerivedSurfaceFlingerStats
 
-from devlib.trace.ftrace import FtraceCollector
+from devlib.collector.ftrace import FtraceCollector
-from devlib.trace.perf import PerfCollector
+from devlib.collector.perf import PerfCollector
-from devlib.trace.serial_trace import SerialTraceCollector
+from devlib.collector.serial_trace import SerialTraceCollector
+from devlib.collector.dmesg import DmesgCollector
+from devlib.collector.logcat import LogcatCollector
 
 from devlib.host import LocalConnection
 from devlib.utils.android import AdbConnection

devlib/collector/__init__.py

@@ -15,12 +15,14 @@
 
 import logging
 
+from devlib.utils.types import caseless_string
 
-class TraceCollector(object):
+class CollectorBase(object):
 
     def __init__(self, target):
         self.target = target
         self.logger = logging.getLogger(self.__class__.__name__)
+        self.output_path = None
 
     def reset(self):
         pass
@@ -31,6 +33,12 @@ class TraceCollector(object):
     def stop(self):
         pass
 
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
+        return CollectorOutput()
+
     def __enter__(self):
         self.reset()
         self.start()
@@ -39,5 +47,29 @@ class TraceCollector(object):
     def __exit__(self, exc_type, exc_value, traceback):
         self.stop()
 
-    def get_trace(self, outfile):
+class CollectorOutputEntry(object):
+
+    path_kinds = ['file', 'directory']
+
+    def __init__(self, path, path_kind):
+        self.path = path
+
+        path_kind = caseless_string(path_kind)
+        if path_kind not in self.path_kinds:
+            msg = '{} is not a valid path_kind [{}]'
+            raise ValueError(msg.format(path_kind, ' '.join(self.path_kinds)))
+        self.path_kind = path_kind
+
+    def __str__(self):
+        return self.path
+
+    def __repr__(self):
+        return '<{} ({})>'.format(self.path, self.path_kind)
+
+    def __fspath__(self):
+        """Allow using with os.path operations"""
+        return self.path
+
+
+class CollectorOutput(list):
     pass

devlib/collector/dmesg.py

@@ -0,0 +1,208 @@
+#    Copyright 2019 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import division
+import re
+from itertools import takewhile
+from datetime import timedelta
+
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
+
+
+class KernelLogEntry(object):
+    """
+    Entry of the kernel ring buffer.
+
+    :param facility: facility the entry comes from
+    :type facility: str
+
+    :param level: log level
+    :type level: str
+
+    :param timestamp: Timestamp of the entry
+    :type timestamp: datetime.timedelta
+
+    :param msg: Content of the entry
+    :type msg: str
+    """
+
+    _TIMESTAMP_MSG_REGEX = re.compile(r'\[(.*?)\] (.*)')
+    _RAW_LEVEL_REGEX = re.compile(r'<([0-9]+)>(.*)')
+    _PRETTY_LEVEL_REGEX = re.compile(r'\s*([a-z]+)\s*:([a-z]+)\s*:\s*(.*)')
+
+    def __init__(self, facility, level, timestamp, msg):
+        self.facility = facility
+        self.level = level
+        self.timestamp = timestamp
+        self.msg = msg
+
+    @classmethod
+    def from_str(cls, line):
+        """
+        Parses a "dmesg --decode" output line, formatted as following:
+        kern  :err   : [3618282.310743] nouveau 0000:01:00.0: systemd-logind[988]: nv50cal_space: -16
+
+        Or the more basic output given by "dmesg -r":
+        <3>[3618282.310743] nouveau 0000:01:00.0: systemd-logind[988]: nv50cal_space: -16
+
+        """
+
+        def parse_raw_level(line):
+            match = cls._RAW_LEVEL_REGEX.match(line)
+            if not match:
+                raise ValueError('dmesg entry format not recognized: {}'.format(line))
+            level, remainder = match.groups()
+            levels = DmesgCollector.LOG_LEVELS
+            # BusyBox dmesg can output numbers that need to wrap around
+            level = levels[int(level) % len(levels)]
+            return level, remainder
+
+        def parse_pretty_level(line):
+            match = cls._PRETTY_LEVEL_REGEX.match(line)
+            facility, level, remainder = match.groups()
+            return facility, level, remainder
+
+        def parse_timestamp_msg(line):
+            match = cls._TIMESTAMP_MSG_REGEX.match(line)
+            timestamp, msg = match.groups()
+            timestamp = timedelta(seconds=float(timestamp.strip()))
+            return timestamp, msg
+
+        line = line.strip()
+
+        # If we can parse the raw prio directly, that is a basic line
+        try:
+            level, remainder = parse_raw_level(line)
+            facility = None
+        except ValueError:
+            facility, level, remainder = parse_pretty_level(line)
+
+        timestamp, msg = parse_timestamp_msg(remainder)
+
+        return cls(
+            facility=facility,
+            level=level,
+            timestamp=timestamp,
+            msg=msg.strip(),
+        )
+
+    @classmethod
+    def from_dmesg_output(cls, dmesg_out):
+        """
+        Return a generator of :class:`KernelLogEntry` for each line of the
+        output of dmesg command.
+
+        .. note:: The same restrictions on the dmesg output format as for
+            :meth:`from_str` apply.
+        """
+        for line in dmesg_out.splitlines():
+            if line.strip():
+                yield cls.from_str(line)
+
+    def __str__(self):
+        facility = self.facility + ': ' if self.facility else ''
+        return '{facility}{level}: [{timestamp}] {msg}'.format(
+            facility=facility,
+            level=self.level,
+            timestamp=self.timestamp.total_seconds(),
+            msg=self.msg,
+        )
+
+
+class DmesgCollector(CollectorBase):
+    """
+    Dmesg output collector.
+
+    :param level: Minimum log level to enable. All levels that are more
+        critical will be collected as well.
+    :type level: str
+
+    :param facility: Facility to record, see dmesg --help for the list.
+    :type level: str
+
+    .. warning:: If BusyBox dmesg is used, facility and level will be ignored,
+        and the parsed entries will also lack that information.
+    """
+
+    # taken from "dmesg --help"
+    # This list needs to be ordered by priority
+    LOG_LEVELS = [
+        "emerg",        # system is unusable
+        "alert",        # action must be taken immediately
+        "crit",         # critical conditions
+        "err",          # error conditions
+        "warn",         # warning conditions
+        "notice",       # normal but significant condition
+        "info",         # informational
+        "debug",        # debug-level messages
+    ]
+
+    def __init__(self, target, level=LOG_LEVELS[-1], facility='kern'):
+        super(DmesgCollector, self).__init__(target)
+        self.output_path = None
+
+        if level not in self.LOG_LEVELS:
+            raise ValueError('level needs to be one of: {}'.format(
+                ', '.join(self.LOG_LEVELS)
+            ))
+        self.level = level
+
+        # Check if dmesg is the BusyBox one, or the one from util-linux in a
+        # recent version.
+        # Note: BusyBox dmesg does not support -h, but will still print the
+        # help with an exit code of 1
+        self.basic_dmesg = '--force-prefix' not in \
+                self.target.execute('dmesg -h', check_exit_code=False)
+        self.facility = facility
+        self.reset()
+
+    @property
+    def entries(self):
+        return KernelLogEntry.from_dmesg_output(self.dmesg_out)
+
+    def reset(self):
+        self.dmesg_out = None
+
+    def start(self):
+        self.reset()
+        # Empty the dmesg ring buffer
+        self.target.execute('dmesg -c', as_root=True)
+
+    def stop(self):
+        levels_list = list(takewhile(
+            lambda level: level != self.level,
+            self.LOG_LEVELS
+        ))
+        levels_list.append(self.level)
+        if self.basic_dmesg:
+            cmd = 'dmesg -r'
+        else:
+            cmd = 'dmesg --facility={facility} --force-prefix --decode --level={levels}'.format(
+                levels=','.join(levels_list),
+                facility=self.facility,
+            )
+
+        self.dmesg_out = self.target.execute(cmd)
+
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
+        with open(self.output_path, 'wt') as f:
+            f.write(self.dmesg_out + '\n')
+        return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')])

devlib/collector/ftrace.py

@@ -20,11 +20,14 @@ import time
 import re
 import subprocess
 import sys
+import contextlib
+from pipes import quote
 
-from devlib.trace import TraceCollector
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
 from devlib.host import PACKAGE_BIN_DIRECTORY
 from devlib.exception import TargetStableError, HostError
-from devlib.utils.misc import check_output, which
+from devlib.utils.misc import check_output, which, memoized
 
 
 TRACE_MARKER_START = 'TRACE_MARKER_START'
@@ -48,12 +51,14 @@ TIMEOUT = 180
 CPU_RE = re.compile(r'  Function \(CPU([0-9]+)\)')
 STATS_RE = re.compile(r'([^ ]*) +([0-9]+) +([0-9.]+) us +([0-9.]+) us +([0-9.]+) us')
 
-class FtraceCollector(TraceCollector):
+class FtraceCollector(CollectorBase):
 
     # pylint: disable=too-many-locals,too-many-branches,too-many-statements
     def __init__(self, target,
                  events=None,
                  functions=None,
+                 tracer=None,
+                 trace_children_functions=False,
                  buffer_size=None,
                  buffer_size_step=1000,
                  tracing_path='/sys/kernel/debug/tracing',
@@ -63,26 +68,34 @@ class FtraceCollector(TraceCollector):
                  no_install=False,
                  strict=False,
                  report_on_target=False,
+                 trace_clock='local',
+                 saved_cmdlines_nr=4096,
                  ):
         super(FtraceCollector, self).__init__(target)
         self.events = events if events is not None else DEFAULT_EVENTS
         self.functions = functions
+        self.tracer = tracer
+        self.trace_children_functions = trace_children_functions
         self.buffer_size = buffer_size
         self.buffer_size_step = buffer_size_step
         self.tracing_path = tracing_path
         self.automark = automark
         self.autoreport = autoreport
         self.autoview = autoview
+        self.strict = strict
         self.report_on_target = report_on_target
         self.target_output_file = target.path.join(self.target.working_directory, OUTPUT_TRACE_FILE)
         text_file_name = target.path.splitext(OUTPUT_TRACE_FILE)[0] + '.txt'
         self.target_text_file = target.path.join(self.target.working_directory, text_file_name)
+        self.output_path = None
         self.target_binary = None
         self.host_binary = None
         self.start_time = None
         self.stop_time = None
         self.event_string = None
         self.function_string = None
+        self.trace_clock = trace_clock
+        self.saved_cmdlines_nr = saved_cmdlines_nr
         self._reset_needed = True
 
         # pylint: disable=bad-whitespace
@@ -94,6 +107,9 @@ class FtraceCollector(TraceCollector):
         self.function_profile_file    = self.target.path.join(self.tracing_path, 'function_profile_enabled')
         self.marker_file              = self.target.path.join(self.tracing_path, 'trace_marker')
         self.ftrace_filter_file       = self.target.path.join(self.tracing_path, 'set_ftrace_filter')
+        self.trace_clock_file         = self.target.path.join(self.tracing_path, 'trace_clock')
+        self.save_cmdlines_size_file  = self.target.path.join(self.tracing_path, 'saved_cmdlines_size')
+        self.available_tracers_file  = self.target.path.join(self.tracing_path, 'available_tracers')
 
         self.host_binary = which('trace-cmd')
         self.kernelshark = which('kernelshark')
@@ -113,65 +129,146 @@ class FtraceCollector(TraceCollector):
             self.target_binary = 'trace-cmd'
 
         # Validate required events to be traced
-        available_events = self.target.execute(
+        def event_to_regex(event):
-                'cat {}'.format(self.available_events_file),
+            if not event.startswith('*'):
-                as_root=True).splitlines()
+                event = '*' + event
-        selected_events = []
+
-        for event in self.events:
+            return re.compile(event.replace('*', '.*'))
-            # Convert globs supported by FTrace into valid regexp globs
+
-            _event = event
+        def event_is_in_list(event, events):
-            if event[0] != '*':
+            return any(
-                _event = '*' + event
+                event_to_regex(event).match(_event)
-            event_re = re.compile(_event.replace('*', '.*'))
+                for _event in events
-            # Select events matching the required ones
+            )
-            if not list(filter(event_re.match, available_events)):
+
-                message = 'Event [{}] not available for tracing'.format(event)
+        unavailable_events = [
-                if strict:
+            event
+            for event in self.events
+            if not event_is_in_list(event, self.available_events)
+        ]
+        if unavailable_events:
+            message = 'Events not available for tracing: {}'.format(
+                ', '.join(unavailable_events)
+            )
+            if self.strict:
                 raise TargetStableError(message)
-                self.target.logger.warning(message)
             else:
-                selected_events.append(event)
+                self.target.logger.warning(message)
-        # If function profiling is enabled we always need at least one event.
+
-        # Thus, if not other events have been specified, try to add at least
+        selected_events = sorted(set(self.events) - set(unavailable_events))
-        # a tracepoint which is always available and possibly triggered few
+
-        # times.
+        if self.tracer and self.tracer not in self.available_tracers:
-        if self.functions and not selected_events:
+            raise TargetStableError('Unsupported tracer "{}". Available tracers: {}'.format(
-            selected_events = ['sched_wakeup_new']
+                self.tracer, ', '.join(self.available_tracers)))
-        self.event_string = _build_trace_events(selected_events)
 
         # Check for function tracing support
         if self.functions:
-            if not self.target.file_exists(self.function_profile_file):
-                raise TargetStableError('Function profiling not supported. '\
-                        'A kernel build with CONFIG_FUNCTION_PROFILER enable is required')
             # Validate required functions to be traced
-            available_functions = self.target.execute(
-                    'cat {}'.format(self.available_functions_file),
-                    as_root=True).splitlines()
             selected_functions = []
             for function in self.functions:
-                if function not in available_functions:
+                if function not in self.available_functions:
-                    message = 'Function [{}] not available for profiling'.format(function)
+                    message = 'Function [{}] not available for tracing/profiling'.format(function)
-                    if strict:
+                    if self.strict:
                         raise TargetStableError(message)
                     self.target.logger.warning(message)
                 else:
                     selected_functions.append(function)
+
+            # Function profiling
+            if self.tracer is None:
+                if not self.target.file_exists(self.function_profile_file):
+                    raise TargetStableError('Function profiling not supported. '\
+                                            'A kernel build with CONFIG_FUNCTION_PROFILER enable is required')
                 self.function_string = _build_trace_functions(selected_functions)
+                # If function profiling is enabled we always need at least one event.
+                # Thus, if not other events have been specified, try to add at least
+                # a tracepoint which is always available and possibly triggered few
+                # times.
+                if not selected_events:
+                    selected_events = ['sched_wakeup_new']
+
+            # Function tracing
+            elif self.tracer == 'function':
+                self.function_string = _build_graph_functions(selected_functions, False)
+
+            # Function graphing
+            elif self.tracer == 'function_graph':
+                self.function_string = _build_graph_functions(selected_functions, trace_children_functions)
+
+        self.event_string = _build_trace_events(selected_events)
+
+
+    @property
+    @memoized
+    def available_tracers(self):
+        """
+        List of ftrace tracers supported by the target's kernel.
+        """
+        return self.target.read_value(self.available_tracers_file).split(' ')
+
+    @property
+    @memoized
+    def available_events(self):
+        """
+        List of ftrace events supported by the target's kernel.
+        """
+        return self.target.read_value(self.available_events_file).splitlines()
+
+    @property
+    @memoized
+    def available_functions(self):
+        """
+        List of functions whose tracing/profiling is supported by the target's kernel.
+        """
+        return self.target.read_value(self.available_functions_file).splitlines()
 
     def reset(self):
         if self.buffer_size:
             self._set_buffer_size()
         self.target.execute('{} reset'.format(self.target_binary),
                             as_root=True, timeout=TIMEOUT)
+        if self.functions:
+            self.target.write_value(self.function_profile_file, 0, verify=False)
         self._reset_needed = False
 
     def start(self):
         self.start_time = time.time()
         if self._reset_needed:
             self.reset()
-        self.target.execute('{} start {}'.format(self.target_binary, self.event_string),
+
-                            as_root=True)
+        if self.tracer is not None and 'function' in self.tracer:
+            tracecmd_functions = self.function_string
+        else:
+            tracecmd_functions = ''
+
+        tracer_string = '-p {}'.format(self.tracer) if self.tracer else ''
+
+        # Ensure kallsyms contains addresses if possible, so that function the
+        # collected trace contains enough data for pretty printing
+        with contextlib.suppress(TargetStableError):
+            self.target.write_value('/proc/sys/kernel/kptr_restrict', 0)
+
+        self.target.write_value(self.trace_clock_file, self.trace_clock, verify=False)
+        try:
+            self.target.write_value(self.save_cmdlines_size_file, self.saved_cmdlines_nr)
+        except TargetStableError as e:
+            message = 'Could not set "save_cmdlines_size"'
+            if self.strict:
+                self.logger.error(message)
+                raise e
+            else:
+                self.logger.warning(message)
+                self.logger.debug(e)
+
+        self.target.execute(
+            '{} start {events} {tracer} {functions}'.format(
+                self.target_binary,
+                events=self.event_string,
+                tracer=tracer_string,
+                functions=tracecmd_functions,
+            ),
+            as_root=True,
+        )
         if self.automark:
             self.mark_start()
         if 'cpufreq' in self.target.modules:
@@ -181,7 +278,7 @@ class FtraceCollector(TraceCollector):
             self.logger.debug('Trace CPUIdle states')
             self.target.cpuidle.perturb_cpus()
         # Enable kernel function profiling
-        if self.functions:
+        if self.functions and self.tracer is None:
             self.target.execute('echo nop > {}'.format(self.current_tracer_file),
                                 as_root=True)
             self.target.execute('echo 0 > {}'.format(self.function_profile_file),
@@ -194,8 +291,8 @@ class FtraceCollector(TraceCollector):
 
     def stop(self):
         # Disable kernel function profiling
-        if self.functions:
+        if self.functions and self.tracer is None:
-            self.target.execute('echo 1 > {}'.format(self.function_profile_file),
+            self.target.execute('echo 0 > {}'.format(self.function_profile_file),
                                 as_root=True)
         if 'cpufreq' in self.target.modules:
             self.logger.debug('Trace CPUFreq frequencies')
@@ -207,9 +304,14 @@ class FtraceCollector(TraceCollector):
                             timeout=TIMEOUT, as_root=True)
         self._reset_needed = True
 
-    def get_trace(self, outfile):
+    def set_output(self, output_path):
-        if os.path.isdir(outfile):
+        if os.path.isdir(output_path):
-            outfile = os.path.join(outfile, os.path.basename(self.target_output_file))
+            output_path = os.path.join(output_path, os.path.basename(self.target_output_file))
+        self.output_path = output_path
+
+    def get_data(self):
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
         self.target.execute('{0} extract -o {1}; chmod 666 {1}'.format(self.target_binary,
                                                                        self.target_output_file),
                             timeout=TIMEOUT, as_root=True)
@@ -218,23 +320,27 @@ class FtraceCollector(TraceCollector):
         # Therefore timout for the pull command must also be adjusted
         # accordingly.
         pull_timeout = 10 * (self.stop_time - self.start_time)
-        self.target.pull(self.target_output_file, outfile, timeout=pull_timeout)
+        self.target.pull(self.target_output_file, self.output_path, timeout=pull_timeout)
-        if not os.path.isfile(outfile):
+        output = CollectorOutput()
+        if not os.path.isfile(self.output_path):
             self.logger.warning('Binary trace not pulled from device.')
         else:
+            output.append(CollectorOutputEntry(self.output_path, 'file'))
             if self.autoreport:
-                textfile = os.path.splitext(outfile)[0] + '.txt'
+                textfile = os.path.splitext(self.output_path)[0] + '.txt'
                 if self.report_on_target:
                     self.generate_report_on_target()
                     self.target.pull(self.target_text_file,
                                      textfile, timeout=pull_timeout)
                 else:
-                    self.report(outfile, textfile)
+                    self.report(self.output_path, textfile)
+                output.append(CollectorOutputEntry(textfile, 'file'))
             if self.autoview:
-                self.view(outfile)
+                self.view(self.output_path)
+        return output
 
     def get_stats(self, outfile):
-        if not self.functions:
+        if not (self.functions and self.tracer is None):
             return
 
         if os.path.isdir(outfile):
@@ -351,3 +457,10 @@ def _build_trace_events(events):
 def _build_trace_functions(functions):
     function_string = " ".join(functions)
     return function_string
+
+def _build_graph_functions(functions, trace_children_functions):
+    opt = 'g' if trace_children_functions else 'l'
+    return ' '.join(
+        '-{} {}'.format(opt, quote(f))
+        for f in functions
+    )

devlib/collector/logcat.py

@@ -16,14 +16,17 @@
 import os
 import shutil
 
-from devlib.trace import TraceCollector
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
 from devlib.utils.android import LogcatMonitor
 
-class LogcatCollector(TraceCollector):
+class LogcatCollector(CollectorBase):
 
-    def __init__(self, target, regexps=None):
+    def __init__(self, target, regexps=None, logcat_format=None):
         super(LogcatCollector, self).__init__(target)
         self.regexps = regexps
+        self.logcat_format = logcat_format
+        self.output_path = None
         self._collecting = False
         self._prev_log = None
         self._monitor = None
@@ -45,12 +48,14 @@ class LogcatCollector(TraceCollector):
         """
         Start collecting logcat lines
         """
-        self._monitor = LogcatMonitor(self.target, self.regexps)
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
+        self._monitor = LogcatMonitor(self.target, self.regexps, logcat_format=self.logcat_format)
         if self._prev_log:
             # Append new data collection to previous collection
             self._monitor.start(self._prev_log)
         else:
-            self._monitor.start()
+            self._monitor.start(self.output_path)
 
         self._collecting = True
 
@@ -65,9 +70,10 @@ class LogcatCollector(TraceCollector):
         self._collecting = False
         self._prev_log = self._monitor.logfile
 
-    def get_trace(self, outfile):
+    def set_output(self, output_path):
-        """
+        self.output_path = output_path
-        Output collected logcat lines to designated file
+
-        """
+    def get_data(self):
-        # copy self._monitor.logfile to outfile
+        if self.output_path is None:
-        shutil.copy(self._monitor.logfile, outfile)
+            raise RuntimeError("No data collected.")
+        return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')])

devlib/collector/perf.py

@@ -0,0 +1,253 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import time
+from past.builtins import basestring, zip
+
+from devlib.host import PACKAGE_BIN_DIRECTORY
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
+from devlib.utils.misc import ensure_file_directory_exists as _f
+
+
+PERF_COMMAND_TEMPLATE = '{binary} {command} {options} {events} sleep 1000 > {outfile} 2>&1 '
+PERF_REPORT_COMMAND_TEMPLATE= '{binary} report {options} -i {datafile} > {outfile} 2>&1 '
+PERF_RECORD_COMMAND_TEMPLATE= '{binary} record {options} {events} -o {outfile}'
+
+PERF_DEFAULT_EVENTS = [
+    'cpu-migrations',
+    'context-switches',
+]
+
+SIMPLEPERF_DEFAULT_EVENTS = [
+    'raw-cpu-cycles',
+    'raw-l1-dcache',
+    'raw-l1-dcache-refill',
+    'raw-br-mis-pred',
+    'raw-instruction-retired',
+]
+
+DEFAULT_EVENTS = {'perf':PERF_DEFAULT_EVENTS, 'simpleperf':SIMPLEPERF_DEFAULT_EVENTS}
+
+class PerfCollector(CollectorBase):
+    """
+    Perf is a Linux profiling with performance counters.
+    Simpleperf is an Android profiling tool with performance counters.
+
+    It is highly recomended to use perf_type = simpleperf when using this instrument
+    on android devices, since it recognises android symbols in record mode and is much more stable
+    when reporting record .data files. For more information see simpleperf documentation at:
+    https://android.googlesource.com/platform/system/extras/+/master/simpleperf/doc/README.md
+
+    Performance counters are CPU hardware registers that count hardware events
+    such as instructions executed, cache-misses suffered, or branches
+    mispredicted. They form a basis for profiling applications to trace dynamic
+    control flow and identify hotspots.
+
+    pref accepts options and events. If no option is given the default '-a' is
+    used. For events, the default events are migrations and cs for perf and raw-cpu-cycles,
+    raw-l1-dcache, raw-l1-dcache-refill, raw-instructions-retired. They both can
+    be specified in the config file.
+
+    Events must be provided as a list that contains them and they will look like
+    this ::
+
+        perf_events = ['migrations', 'cs']
+
+    Events can be obtained by typing the following in the command line on the
+    device ::
+
+        perf list
+        simpleperf list
+
+    Whereas options, they can be provided as a single string as following ::
+
+        perf_options = '-a -i'
+
+    Options can be obtained by running the following in the command line ::
+
+        man perf-stat
+    """
+
+    def __init__(self,
+                 target,
+                 perf_type='perf',
+                 command='stat',
+                 events=None,
+                 optionstring=None,
+                 report_options=None,
+                 labels=None,
+                 force_install=False):
+        super(PerfCollector, self).__init__(target)
+        self.force_install = force_install
+        self.labels = labels
+        self.report_options = report_options
+        self.output_path = None
+
+        # Validate parameters
+        if isinstance(optionstring, list):
+            self.optionstrings = optionstring
+        else:
+            self.optionstrings = [optionstring]
+        if perf_type in ['perf', 'simpleperf']:
+            self.perf_type = perf_type
+        else:
+            raise ValueError('Invalid perf type: {}, must be perf or simpleperf'.format(perf_type))
+        if not events:
+            self.events = DEFAULT_EVENTS[self.perf_type]
+        else:
+            self.events = events
+        if isinstance(self.events, basestring):
+            self.events = [self.events]
+        if not self.labels:
+            self.labels = ['perf_{}'.format(i) for i in range(len(self.optionstrings))]
+        if len(self.labels) != len(self.optionstrings):
+            raise ValueError('The number of labels must match the number of optstrings provided for perf.')
+        if command in ['stat', 'record']:
+            self.command = command
+        else:
+            raise ValueError('Unsupported perf command, must be stat or record')
+
+        self.binary = self.target.get_installed(self.perf_type)
+        if self.force_install or not self.binary:
+            self.binary = self._deploy_perf()
+
+        self._validate_events(self.events)
+
+        self.commands = self._build_commands()
+
+    def reset(self):
+        self.target.killall(self.perf_type, as_root=self.target.is_rooted)
+        self.target.remove(self.target.get_workpath('TemporaryFile*'))
+        for label in self.labels:
+            filepath = self._get_target_file(label, 'data')
+            self.target.remove(filepath)
+            filepath = self._get_target_file(label, 'rpt')
+            self.target.remove(filepath)
+
+    def start(self):
+        for command in self.commands:
+            self.target.kick_off(command)
+
+    def stop(self):
+        self.target.killall(self.perf_type, signal='SIGINT',
+                            as_root=self.target.is_rooted)
+        # perf doesn't transmit the signal to its sleep call so handled here:
+        self.target.killall('sleep', as_root=self.target.is_rooted)
+        # NB: we hope that no other "important" sleep is on-going
+
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
+
+        output = CollectorOutput()
+
+        for label in self.labels:
+            if self.command == 'record':
+                self._wait_for_data_file_write(label, self.output_path)
+                path = self._pull_target_file_to_host(label, 'rpt', self.output_path)
+                output.append(CollectorOutputEntry(path, 'file'))
+            else:
+                path = self._pull_target_file_to_host(label, 'out', self.output_path)
+                output.append(CollectorOutputEntry(path, 'file'))
+        return output
+
+    def _deploy_perf(self):
+        host_executable = os.path.join(PACKAGE_BIN_DIRECTORY,
+                                       self.target.abi, self.perf_type)
+        return self.target.install(host_executable)
+
+    def _get_target_file(self, label, extension):
+        return self.target.get_workpath('{}.{}'.format(label, extension))
+
+    def _build_commands(self):
+        commands = []
+        for opts, label in zip(self.optionstrings, self.labels):
+            if self.command == 'stat':
+                commands.append(self._build_perf_stat_command(opts, self.events, label))
+            else:
+                commands.append(self._build_perf_record_command(opts, label))
+        return commands
+
+    def _build_perf_stat_command(self, options, events, label):
+        event_string = ' '.join(['-e {}'.format(e) for e in events])
+        command = PERF_COMMAND_TEMPLATE.format(binary = self.binary,
+                                               command = self.command,
+                                               options = options or '',
+                                               events = event_string,
+                                               outfile = self._get_target_file(label, 'out'))
+        return command
+
+    def _build_perf_report_command(self, report_options, label):
+        command = PERF_REPORT_COMMAND_TEMPLATE.format(binary=self.binary,
+                                                      options=report_options or '',
+                                                      datafile=self._get_target_file(label, 'data'),
+                                                      outfile=self._get_target_file(label, 'rpt'))
+        return command
+
+    def _build_perf_record_command(self, options, label):
+        event_string = ' '.join(['-e {}'.format(e) for e in self.events])
+        command = PERF_RECORD_COMMAND_TEMPLATE.format(binary=self.binary,
+                                                      options=options or '',
+                                                      events=event_string,
+                                                      outfile=self._get_target_file(label, 'data'))
+        return command
+
+    def _pull_target_file_to_host(self, label, extension, output_path):
+        target_file = self._get_target_file(label, extension)
+        host_relpath = os.path.basename(target_file)
+        host_file = _f(os.path.join(output_path, host_relpath))
+        self.target.pull(target_file, host_file)
+        return host_file
+
+    def _wait_for_data_file_write(self, label, output_path):
+        data_file_finished_writing = False
+        max_tries = 80
+        current_tries = 0
+        while not data_file_finished_writing:
+            files = self.target.execute('cd {} && ls'.format(self.target.get_workpath('')))
+            # Perf stores data in tempory files whilst writing to data output file. Check if they have been removed.
+            if 'TemporaryFile' in files and current_tries <= max_tries:
+                time.sleep(0.25)
+                current_tries += 1
+            else:
+                if current_tries >= max_tries:
+                    self.logger.warning('''writing {}.data file took longer than expected,
+                                        file may not have written correctly'''.format(label))
+                data_file_finished_writing = True
+        report_command = self._build_perf_report_command(self.report_options, label)
+        self.target.execute(report_command)
+
+    def _validate_events(self, events):
+        available_events_string = self.target.execute('{} list | {} cat'.format(self.perf_type, self.target.busybox))
+        available_events = available_events_string.splitlines()
+        for available_event in available_events:
+            if available_event == '':
+                continue
+            if 'OR' in available_event:
+                available_events.append(available_event.split('OR')[1])
+            available_events[available_events.index(available_event)] = available_event.split()[0].strip()
+        # Raw hex event codes can also be passed in that do not appear on perf/simpleperf list, prefixed with 'r'
+        raw_event_code_regex = re.compile(r"^r(0x|0X)?[A-Fa-f0-9]+$")
+        for event in events:
+            if event in available_events or re.match(raw_event_code_regex, event):
+                continue
+            else:
+                raise ValueError('Event: {} is not in available event list for {}'.format(event, self.perf_type))

devlib/collector/screencapture.py

@@ -19,13 +19,14 @@ import sys
 import threading
 import time
 
-from devlib.trace import TraceCollector
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
 from devlib.exception import WorkerThreadError
 
 
 class ScreenCapturePoller(threading.Thread):
 
-    def __init__(self, target, period, output_path=None, timeout=30):
+    def __init__(self, target, period, timeout=30):
         super(ScreenCapturePoller, self).__init__()
         self.target = target
         self.logger = logging.getLogger('screencapture')
@@ -36,11 +37,16 @@ class ScreenCapturePoller(threading.Thread):
         self.last_poll = 0
         self.daemon = True
         self.exc = None
+        self.output_path = None
+
+    def set_output(self, output_path):
         self.output_path = output_path
 
     def run(self):
         self.logger.debug('Starting screen capture polling')
         try:
+            if self.output_path is None:
+                raise RuntimeError("Output path was not set.")
             while True:
                 if self.stop_signal.is_set():
                     break
@@ -66,24 +72,33 @@ class ScreenCapturePoller(threading.Thread):
         self.target.capture_screen(os.path.join(self.output_path, "screencap_{ts}.png"))
 
 
-class ScreenCaptureCollector(TraceCollector):
+class ScreenCaptureCollector(CollectorBase):
 
-    def __init__(self, target, output_path=None, period=None):
+    def __init__(self, target, period=None):
         super(ScreenCaptureCollector, self).__init__(target)
         self._collecting = False
-        self.output_path = output_path
+        self.output_path = None
         self.period = period
         self.target = target
-        self._poller = ScreenCapturePoller(self.target, self.period,
+
-                                           self.output_path)
+    def set_output(self, output_path):
+        self.output_path = output_path
 
     def reset(self):
-        pass
+        self._poller = ScreenCapturePoller(self.target, self.period)
+
+    def get_data(self):
+        if self.output_path is None:
+            raise RuntimeError("No data collected.")
+        return CollectorOutput([CollectorOutputEntry(self.output_path, 'directory')])
 
     def start(self):
         """
         Start collecting the screenshots
         """
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
+        self._poller.set_output(self.output_path)
         self._poller.start()
         self._collecting = True
 

devlib/collector/serial_trace.py

@@ -17,11 +17,12 @@ import shutil
 from tempfile import NamedTemporaryFile
 from pexpect.exceptions import TIMEOUT
 
-from devlib.trace import TraceCollector
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
 from devlib.utils.serial_port import get_connection
 
 
-class SerialTraceCollector(TraceCollector):
+class SerialTraceCollector(CollectorBase):
 
     @property
     def collecting(self):
@@ -32,33 +33,35 @@ class SerialTraceCollector(TraceCollector):
         self.serial_port = serial_port
         self.baudrate = baudrate
         self.timeout = timeout
+        self.output_path = None
 
         self._serial_target = None
         self._conn = None
-        self._tmpfile = None
+        self._outfile_fh = None
         self._collecting = False
 
     def reset(self):
         if self._collecting:
             raise RuntimeError("reset was called whilst collecting")
 
-        if self._tmpfile:
+        if self._outfile_fh:
-            self._tmpfile.close()
+            self._outfile_fh.close()
-            self._tmpfile = None
+            self._outfile_fh = None
 
     def start(self):
         if self._collecting:
             raise RuntimeError("start was called whilst collecting")
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
 
-
+        self._outfile_fh = open(self.output_path, 'wb')
-        self._tmpfile = NamedTemporaryFile()
         start_marker = "-------- Starting serial logging --------\n"
-        self._tmpfile.write(start_marker.encode('utf-8'))
+        self._outfile_fh.write(start_marker.encode('utf-8'))
 
         self._serial_target, self._conn = get_connection(port=self.serial_port,
                                                          baudrate=self.baudrate,
                                                          timeout=self.timeout,
-                                                         logfile=self._tmpfile,
+                                                         logfile=self._outfile_fh,
                                                          init_dtr=0)
         self._collecting = True
 
@@ -78,17 +81,19 @@ class SerialTraceCollector(TraceCollector):
         del self._conn
 
         stop_marker = "-------- Stopping serial logging --------\n"
-        self._tmpfile.write(stop_marker.encode('utf-8'))
+        self._outfile_fh.write(stop_marker.encode('utf-8'))
+        self._outfile_fh.flush()
+        self._outfile_fh.close()
+        self._outfile_fh = None
 
         self._collecting = False
 
-    def get_trace(self, outfile):
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
         if self._collecting:
-            raise RuntimeError("get_trace was called whilst collecting")
+            raise RuntimeError("get_data was called whilst collecting")
-
+        if self.output_path is None:
-        self._tmpfile.flush()
+            raise RuntimeError("No data collected.")
-
+        return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')])
-        shutil.copy(self._tmpfile.name, outfile)
-
-        self._tmpfile.close()
-        self._tmpfile = None

devlib/collector/systrace.py

@@ -19,8 +19,9 @@ import subprocess
 from shutil import copyfile
 from tempfile import NamedTemporaryFile
 
+from devlib.collector import (CollectorBase, CollectorOutput,
+                              CollectorOutputEntry)
 from devlib.exception import TargetStableError, HostError
-from devlib.trace import TraceCollector
 import devlib.utils.android
 from devlib.utils.misc import memoized
 
@@ -33,7 +34,7 @@ DEFAULT_CATEGORIES = [
     'idle'
 ]
 
-class SystraceCollector(TraceCollector):
+class SystraceCollector(CollectorBase):
     """
     A trace collector based on Systrace
 
@@ -74,9 +75,10 @@ class SystraceCollector(TraceCollector):
 
         self.categories = categories or DEFAULT_CATEGORIES
         self.buffer_size = buffer_size
+        self.output_path = None
 
         self._systrace_process = None
-        self._tmpfile = None
+        self._outfile_fh = None
 
         # Try to find a systrace binary
         self.systrace_binary = None
@@ -104,12 +106,12 @@ class SystraceCollector(TraceCollector):
         self.reset()
 
     def _build_cmd(self):
-        self._tmpfile = NamedTemporaryFile()
+        self._outfile_fh = open(self.output_path, 'w')
 
         # pylint: disable=attribute-defined-outside-init
-        self.systrace_cmd = '{} -o {} -e {}'.format(
+        self.systrace_cmd = 'python2 -u {} -o {} -e {}'.format(
             self.systrace_binary,
-            self._tmpfile.name,
+            self._outfile_fh.name,
             self.target.adb_name
         )
 
@@ -122,13 +124,11 @@ class SystraceCollector(TraceCollector):
         if self._systrace_process:
             self.stop()
 
-        if self._tmpfile:
-            self._tmpfile.close()
-            self._tmpfile = None
-
     def start(self):
         if self._systrace_process:
             raise RuntimeError("Tracing is already underway, call stop() first")
+        if self.output_path is None:
+            raise RuntimeError("Output path was not set.")
 
         self.reset()
 
@@ -137,9 +137,11 @@ class SystraceCollector(TraceCollector):
         self._systrace_process = subprocess.Popen(
             self.systrace_cmd,
             stdin=subprocess.PIPE,
+            stdout=subprocess.PIPE,
             shell=True,
             universal_newlines=True
         )
+        self._systrace_process.stdout.read(1)
 
     def stop(self):
         if not self._systrace_process:
@@ -149,11 +151,16 @@ class SystraceCollector(TraceCollector):
         self._systrace_process.communicate('\n')
         self._systrace_process = None
 
-    def get_trace(self, outfile):
+        if self._outfile_fh:
+            self._outfile_fh.close()
+            self._outfile_fh = None
+
+    def set_output(self, output_path):
+        self.output_path = output_path
+
+    def get_data(self):
         if self._systrace_process:
             raise RuntimeError("Tracing is underway, call stop() first")
-
+        if self.output_path is None:
-        if not self._tmpfile:
+            raise RuntimeError("No data collected.")
-            raise RuntimeError("No tracing data available")
+        return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')])
-
-        copyfile(self._tmpfile.name, outfile)

devlib/connection.py

@@ -0,0 +1,523 @@
+#    Copyright 2019 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from abc import ABC, abstractmethod
+from contextlib import contextmanager
+from datetime import datetime
+from functools import partial
+from weakref import WeakSet
+from shlex import quote
+from time import monotonic
+import os
+import signal
+import socket
+import subprocess
+import threading
+import time
+import logging
+
+from devlib.utils.misc import InitCheckpoint
+
+_KILL_TIMEOUT = 3
+
+
+def _kill_pgid_cmd(pgid, sig):
+    return 'kill -{} -{}'.format(sig.value, pgid)
+
+
+class ConnectionBase(InitCheckpoint):
+    """
+    Base class for all connections.
+    """
+    def __init__(self):
+        self._current_bg_cmds = WeakSet()
+        self._closed = False
+        self._close_lock = threading.Lock()
+        self.busybox = None
+
+    def cancel_running_command(self):
+        bg_cmds = set(self._current_bg_cmds)
+        for bg_cmd in bg_cmds:
+            bg_cmd.cancel()
+
+    @abstractmethod
+    def _close(self):
+        """
+        Close the connection.
+
+        The public :meth:`close` method makes sure that :meth:`_close` will
+        only be called once, and will serialize accesses to it if it happens to
+        be called from multiple threads at once.
+        """
+
+    def close(self):
+        # Locking the closing allows any thread to safely call close() as long
+        # as the connection can be closed from a thread that is not the one it
+        # started its life in.
+        with self._close_lock:
+            if not self._closed:
+                self._close()
+                self._closed = True
+
+    # Ideally, that should not be relied upon but that will improve the chances
+    # of the connection being properly cleaned up when it's not in use anymore.
+    def __del__(self):
+        # Since __del__ will be called if an exception is raised in __init__
+        # (e.g. we cannot connect), we only run close() when we are sure
+        # __init__ has completed successfully.
+        if self.initialized:
+            self.close()
+
+
+class BackgroundCommand(ABC):
+    """
+    Allows managing a running background command using a subset of the
+    :class:`subprocess.Popen` API.
+
+    Instances of this class can be used as context managers, with the same
+    semantic as :class:`subprocess.Popen`.
+    """
+    @abstractmethod
+    def send_signal(self, sig):
+        """
+        Send a POSIX signal to the background command's process group ID
+        (PGID).
+
+        :param signal: Signal to send.
+        :type signal: signal.Signals
+        """
+
+    def kill(self):
+        """
+        Send SIGKILL to the background command.
+        """
+        self.send_signal(signal.SIGKILL)
+
+    @abstractmethod
+    def cancel(self, kill_timeout=_KILL_TIMEOUT):
+        """
+        Try to gracefully terminate the process by sending ``SIGTERM``, then
+        waiting for ``kill_timeout`` to send ``SIGKILL``.
+        """
+
+    @abstractmethod
+    def wait(self):
+        """
+        Block until the background command completes, and return its exit code.
+        """
+
+    @abstractmethod
+    def poll(self):
+        """
+        Return exit code if the command has exited, None otherwise.
+        """
+
+    @property
+    @abstractmethod
+    def stdin(self):
+        """
+        File-like object connected to the background's command stdin.
+        """
+
+    @property
+    @abstractmethod
+    def stdout(self):
+        """
+        File-like object connected to the background's command stdout.
+        """
+
+    @property
+    @abstractmethod
+    def stderr(self):
+        """
+        File-like object connected to the background's command stderr.
+        """
+
+    @property
+    @abstractmethod
+    def pid(self):
+        """
+        Process Group ID (PGID) of the background command.
+
+        Since the command is usually wrapped in shell processes for IO
+        redirections, sudo etc, the PID cannot be assumed to be the actual PID
+        of the command passed by the user. It's is guaranteed to be a PGID
+        instead, which means signals sent to it as such will target all
+        subprocesses involved in executing that command.
+        """
+
+    @abstractmethod
+    def close(self):
+        """
+        Close all opened streams and then wait for command completion.
+
+        :returns: Exit code of the command.
+
+        .. note:: If the command is writing to its stdout/stderr, it might be
+            blocked on that and die when the streams are closed.
+        """
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.close()
+
+
+class PopenBackgroundCommand(BackgroundCommand):
+    """
+    :class:`subprocess.Popen`-based background command.
+    """
+
+    def __init__(self, popen):
+        self.popen = popen
+
+    def send_signal(self, sig):
+        return os.killpg(self.popen.pid, sig)
+
+    @property
+    def stdin(self):
+        return self.popen.stdin
+
+    @property
+    def stdout(self):
+        return self.popen.stdout
+
+    @property
+    def stderr(self):
+        return self.popen.stderr
+
+    @property
+    def pid(self):
+        return self.popen.pid
+
+    def wait(self):
+        return self.popen.wait()
+
+    def poll(self):
+        return self.popen.poll()
+
+    def cancel(self, kill_timeout=_KILL_TIMEOUT):
+        popen = self.popen
+        os.killpg(os.getpgid(popen.pid), signal.SIGTERM)
+        try:
+            popen.wait(timeout=_KILL_TIMEOUT)
+        except subprocess.TimeoutExpired:
+            os.killpg(os.getpgid(popen.pid), signal.SIGKILL)
+
+    def close(self):
+        self.popen.__exit__(None, None, None)
+        return self.popen.returncode
+
+    def __enter__(self):
+        self.popen.__enter__()
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.popen.__exit__(*args, **kwargs)
+
+
+class ParamikoBackgroundCommand(BackgroundCommand):
+    """
+    :mod:`paramiko`-based background command.
+    """
+    def __init__(self, conn, chan, pid, as_root, stdin, stdout, stderr, redirect_thread):
+        self.chan = chan
+        self.as_root = as_root
+        self.conn = conn
+        self._pid = pid
+        self._stdin = stdin
+        self._stdout = stdout
+        self._stderr = stderr
+        self.redirect_thread = redirect_thread
+
+    def send_signal(self, sig):
+        # If the command has already completed, we don't want to send a signal
+        # to another process that might have gotten that PID in the meantime.
+        if self.poll() is not None:
+            return
+        # Use -PGID to target a process group rather than just the process
+        # itself
+        cmd = _kill_pgid_cmd(self.pid, sig)
+        self.conn.execute(cmd, as_root=self.as_root)
+
+    @property
+    def pid(self):
+        return self._pid
+
+    def wait(self):
+        return self.chan.recv_exit_status()
+
+    def poll(self):
+        if self.chan.exit_status_ready():
+            return self.wait()
+        else:
+            return None
+
+    def cancel(self, kill_timeout=_KILL_TIMEOUT):
+        self.send_signal(signal.SIGTERM)
+        # Check if the command terminated quickly
+        time.sleep(10e-3)
+        # Otherwise wait for the full timeout and kill it
+        if self.poll() is None:
+            time.sleep(kill_timeout)
+            self.send_signal(signal.SIGKILL)
+            self.wait()
+
+    @property
+    def stdin(self):
+        return self._stdin
+
+    @property
+    def stdout(self):
+        return self._stdout
+
+    @property
+    def stderr(self):
+        return self._stderr
+
+    def close(self):
+        for x in (self.stdin, self.stdout, self.stderr):
+            if x is not None:
+                x.close()
+
+        exit_code = self.wait()
+        thread = self.redirect_thread
+        if thread:
+            thread.join()
+
+        return exit_code
+
+
+class AdbBackgroundCommand(BackgroundCommand):
+    """
+    ``adb``-based background command.
+    """
+
+    def __init__(self, conn, adb_popen, pid, as_root):
+        self.conn = conn
+        self.as_root = as_root
+        self.adb_popen = adb_popen
+        self._pid = pid
+
+    def send_signal(self, sig):
+        self.conn.execute(
+            _kill_pgid_cmd(self.pid, sig),
+            as_root=self.as_root,
+        )
+
+    @property
+    def stdin(self):
+        return self.adb_popen.stdin
+
+    @property
+    def stdout(self):
+        return self.adb_popen.stdout
+
+    @property
+    def stderr(self):
+        return self.adb_popen.stderr
+
+    @property
+    def pid(self):
+        return self._pid
+
+    def wait(self):
+        return self.adb_popen.wait()
+
+    def poll(self):
+        return self.adb_popen.poll()
+
+    def cancel(self, kill_timeout=_KILL_TIMEOUT):
+        self.send_signal(signal.SIGTERM)
+        try:
+            self.adb_popen.wait(timeout=_KILL_TIMEOUT)
+        except subprocess.TimeoutExpired:
+            self.send_signal(signal.SIGKILL)
+            self.adb_popen.kill()
+
+    def close(self):
+        self.adb_popen.__exit__(None, None, None)
+        return self.adb_popen.returncode
+
+    def __enter__(self):
+        self.adb_popen.__enter__()
+        return self
+
+    def __exit__(self, *args, **kwargs):
+        self.adb_popen.__exit__(*args, **kwargs)
+
+
+class TransferManagerBase(ABC):
+
+    def _pull_dest_size(self, dest):
+        if os.path.isdir(dest):
+            return sum(
+                os.stat(os.path.join(dirpath, f)).st_size
+	            for dirpath, _, fnames in os.walk(dest)
+	            for f in fnames
+            )
+        else:
+            return os.stat(dest).st_size
+        return 0
+
+    def _push_dest_size(self, dest):
+        cmd = '{} du -s {}'.format(quote(self.conn.busybox), quote(dest))
+        out = self.conn.execute(cmd)
+        try:
+            return int(out.split()[0])
+        except ValueError:
+            return 0
+
+    def __init__(self, conn, poll_period, start_transfer_poll_delay, total_timeout):
+        self.conn = conn
+        self.poll_period = poll_period
+        self.total_timeout = total_timeout
+        self.start_transfer_poll_delay = start_transfer_poll_delay
+
+        self.logger = logging.getLogger('FileTransfer')
+        self.managing = threading.Event()
+        self.transfer_started = threading.Event()
+        self.transfer_completed = threading.Event()
+        self.transfer_aborted = threading.Event()
+
+        self.monitor_thread = None
+        self.sources = None
+        self.dest = None
+        self.direction = None
+
+    @abstractmethod
+    def _cancel(self):
+        pass
+
+    def cancel(self, reason=None):
+        msg = 'Cancelling file transfer {} -> {}'.format(self.sources, self.dest)
+        if reason is not None:
+            msg += ' due to \'{}\''.format(reason)
+        self.logger.warning(msg)
+        self.transfer_aborted.set()
+        self._cancel()
+
+    @abstractmethod
+    def isactive(self):
+        pass
+
+    @contextmanager
+    def manage(self, sources, dest, direction):
+        try:
+            self.sources, self.dest, self.direction = sources, dest, direction
+            m_thread = threading.Thread(target=self._monitor)
+
+            self.transfer_completed.clear()
+            self.transfer_aborted.clear()
+            self.transfer_started.set()
+
+            m_thread.start()
+            yield self
+        except BaseException:
+            self.cancel(reason='exception during transfer')
+            raise
+        finally:
+            self.transfer_completed.set()
+            self.transfer_started.set()
+            m_thread.join()
+            self.transfer_started.clear()
+            self.transfer_completed.clear()
+            self.transfer_aborted.clear()
+    
+    def _monitor(self):
+        start_t = monotonic()
+        self.transfer_completed.wait(self.start_transfer_poll_delay)
+        while not self.transfer_completed.wait(self.poll_period):
+            if not self.isactive():
+                self.cancel(reason='transfer inactive')
+            elif monotonic() - start_t > self.total_timeout:
+                self.cancel(reason='transfer timed out')
+
+
+class PopenTransferManager(TransferManagerBase):
+
+    def __init__(self, conn, poll_period=30, start_transfer_poll_delay=30, total_timeout=3600):
+        super().__init__(conn, poll_period, start_transfer_poll_delay, total_timeout)
+        self.transfer = None
+        self.last_sample = None
+
+    def _cancel(self):
+        if self.transfer:
+            self.transfer.cancel()
+            self.transfer = None
+
+    def isactive(self):
+        size_fn = self._push_dest_size if self.direction == 'push' else self._pull_dest_size
+        curr_size = size_fn(self.dest)
+        self.logger.debug('Polled file transfer, destination size {}'.format(curr_size))
+        active = True if self.last_sample is None else curr_size > self.last_sample
+        self.last_sample = curr_size
+        return active
+
+    def set_transfer_and_wait(self, popen_bg_cmd):
+        self.transfer = popen_bg_cmd
+        ret = self.transfer.wait()
+      
+        if ret and not self.transfer_aborted.is_set():
+            raise subprocess.CalledProcessError(ret, self.transfer.popen.args)
+        elif self.transfer_aborted.is_set():
+            raise TimeoutError(self.transfer.popen.args)
+
+
+class SSHTransferManager(TransferManagerBase):
+
+    def __init__(self, conn, poll_period=30, start_transfer_poll_delay=30, total_timeout=3600):
+        super().__init__(conn, poll_period, start_transfer_poll_delay, total_timeout)
+        self.transferer = None
+        self.progressed = False
+        self.transferred = None
+        self.to_transfer = None
+
+    def _cancel(self):
+        self.transferer.close()
+
+    def isactive(self):
+        progressed = self.progressed
+        self.progressed = False
+        msg = 'Polled transfer: {}% [{}B/{}B]'
+        pc = format((self.transferred / self.to_transfer) * 100, '.2f')
+        self.logger.debug(msg.format(pc, self.transferred, self.to_transfer))
+        return progressed
+
+    @contextmanager
+    def manage(self, sources, dest, direction, transferer):
+        with super().manage(sources, dest, direction):
+            try:
+                self.progressed = False
+                self.transferer = transferer  # SFTPClient or SCPClient
+                yield self
+            except socket.error as e:
+                if self.transfer_aborted.is_set():
+                    self.transfer_aborted.clear()
+                    method = 'SCP' if self.conn.use_scp else 'SFTP'
+                    raise TimeoutError('{} {}: {} -> {}'.format(method, self.direction, sources, self.dest))
+                else:
+                    raise e
+
+    def progress_cb(self, *args):
+        if self.transfer_started.is_set():
+            self.progressed = True
+            if len(args) == 3:  # For SCPClient callbacks
+                self.transferred = args[2]
+                self.to_transfer = args[1]
+            elif len(args) == 2:  # For SFTPClient callbacks
+                self.transferred = args[0]
+                self.to_transfer = args[1]

devlib/exception.py

@@ -15,10 +15,16 @@
 
 class DevlibError(Exception):
     """Base class for all Devlib exceptions."""
+
+    def __init__(self, *args):
+        message = args[0] if args else None
+        self._message = message
+
     @property
     def message(self):
-        if self.args:
+        if self._message is not None:
-            return self.args[0]
+            return self._message
+        else:
             return str(self)
 
 
@@ -127,7 +133,7 @@ def get_traceback(exc=None):
     if not exc:
         return None
     tb = exc[2]
-    sio = io.BytesIO()
+    sio = io.StringIO()
     traceback.print_tb(tb, file=sio)
     del tb  # needs to be done explicitly see: http://docs.python.org/2/library/sys.html#sys.exc_info
     return sio.getvalue()

devlib/host.py

@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-from glob import iglob
+import glob
 import os
 import signal
 import shutil
@@ -24,9 +24,12 @@ from pipes import quote
 
 from devlib.exception import TargetTransientError, TargetStableError
 from devlib.utils.misc import check_output
+from devlib.connection import ConnectionBase, PopenBackgroundCommand
+
 
 PACKAGE_BIN_DIRECTORY = os.path.join(os.path.dirname(__file__), 'bin')
 
+
 # pylint: disable=redefined-outer-name
 def kill_children(pid, signal=signal.SIGKILL):
     with open('/proc/{0}/task/{0}/children'.format(pid), 'r') as fd:
@@ -34,47 +37,75 @@ def kill_children(pid, signal=signal.SIGKILL):
             kill_children(cpid, signal)
             os.kill(cpid, signal)
 
-class LocalConnection(object):
+
+class LocalConnection(ConnectionBase):
 
     name = 'local'
+    host = 'localhost'
+
+    @property
+    def connected_as_root(self):
+        if self._connected_as_root is None:
+            result = self.execute('id', as_root=False)
+            self._connected_as_root = 'uid=0(' in result
+        return self._connected_as_root
+
+    @connected_as_root.setter
+    def connected_as_root(self, state):
+        self._connected_as_root = state
 
     # pylint: disable=unused-argument
     def __init__(self, platform=None, keep_password=True, unrooted=False,
                  password=None, timeout=None):
+        super().__init__()
+        self._connected_as_root = None
         self.logger = logging.getLogger('local_connection')
         self.keep_password = keep_password
         self.unrooted = unrooted
         self.password = password
 
-    def push(self, source, dest, timeout=None, as_root=False):  # pylint: disable=unused-argument
+
-        self.logger.debug('cp {} {}'.format(source, dest))
+    def _copy_path(self, source, dest):
+        self.logger.debug('copying {} to {}'.format(source, dest))
+        if os.path.isdir(source):
+            # Behave similarly as cp, scp, adb push, etc. by creating a new
+            # folder instead of merging hierarchies
+            if os.path.exists(dest):
+                dest = os.path.join(dest, os.path.basename(os.path.normpath(src)))
+
+            # Use distutils copy_tree since it behaves the same as
+            # shutils.copytree except that it won't fail if some folders
+            # already exist.
+            #
+            # Mirror the behavior of all other targets which only copy the
+            # content without metadata
+            copy_tree(source, dest, preserve_mode=False, preserve_times=False)
+        else:
             shutil.copy(source, dest)
 
-    def pull(self, source, dest, timeout=None, as_root=False): # pylint: disable=unused-argument
+    def _copy_paths(self, sources, dest):
-        self.logger.debug('cp {} {}'.format(source, dest))
+        for source in sources:
-        if ('*' in source or '?' in source) and os.path.isdir(dest):
+            self._copy_path(source, dest)
-            # Pull all files matching a wildcard expression
+
-            for each_source in iglob(source):
+    def push(self, sources, dest, timeout=None, as_root=False):  # pylint: disable=unused-argument
-                shutil.copy(each_source, dest)
+        self._copy_paths(sources, dest)
-        else:
+
-            if os.path.isdir(source):
+    def pull(self, sources, dest, timeout=None, as_root=False): # pylint: disable=unused-argument
-                # Use distutils to allow copying into an existing directory structure.
+        self._copy_paths(sources, dest)
-                copy_tree(source, dest)
-            else:
-                shutil.copy(source, dest)
 
     # pylint: disable=unused-argument
     def execute(self, command, timeout=None, check_exit_code=True,
                 as_root=False, strip_colors=True, will_succeed=False):
         self.logger.debug(command)
-        if as_root:
+        use_sudo = as_root and not self.connected_as_root
+        if use_sudo:
             if self.unrooted:
                 raise TargetStableError('unrooted')
             password = self._get_password()
-            command = 'echo {} | sudo -S '.format(quote(password)) + command
+            command = "echo {} | sudo -p ' ' -S -- sh -c {}".format(quote(password), quote(command))
         ignore = None if check_exit_code else 'all'
         try:
-            return check_output(command, shell=True, timeout=timeout, ignore=ignore)[0]
+            stdout, stderr = check_output(command, shell=True, timeout=timeout, ignore=ignore)
         except subprocess.CalledProcessError as e:
             message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}'.format(
                 e.returncode, command, e.output)
@@ -83,20 +114,49 @@ class LocalConnection(object):
             else:
                 raise TargetStableError(message)
 
+        # Remove the one-character prompt of sudo -S -p
+        if use_sudo and stderr:
+            stderr = stderr[1:]
+
+        return stdout + stderr
+
     def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
-        if as_root:
+        if as_root and not self.connected_as_root:
             if self.unrooted:
                 raise TargetStableError('unrooted')
             password = self._get_password()
-            command = 'echo {} | sudo -S '.format(quote(password)) + command
+            # The sudo prompt will add a space on stderr, but we cannot filter
-        return subprocess.Popen(command, stdout=stdout, stderr=stderr, shell=True)
+            # it out here
+            command = "echo {} | sudo -p ' ' -S -- sh -c {}".format(quote(password), quote(command))
 
-    def close(self):
+        # Make sure to get a new PGID so PopenBackgroundCommand() can kill
+        # all sub processes that could be started without troubles.
+        def preexec_fn():
+            os.setpgrp()
+
+        popen = subprocess.Popen(
+            command,
+            stdout=stdout,
+            stderr=stderr,
+            shell=True,
+            preexec_fn=preexec_fn,
+        )
+        bg_cmd = PopenBackgroundCommand(popen)
+        self._current_bg_cmds.add(bg_cmd)
+        return bg_cmd
+
+    def _close(self):
         pass
 
     def cancel_running_command(self):
         pass
 
+    def wait_for_device(self, timeout=30):
+        return
+
+    def reboot_bootloader(self, timeout=30):
+        raise NotImplementedError()
+
     def _get_password(self):
         if self.password:
             return self.password

devlib/instrument/acmecape.py

@@ -58,12 +58,14 @@ class AcmeCapeInstrument(Instrument):
                  iio_capture=which('iio-capture'),
                  host='baylibre-acme.local',
                  iio_device='iio:device0',
-                 buffer_size=256):
+                 buffer_size=256,
+                 keep_raw=False):
         super(AcmeCapeInstrument, self).__init__(target)
         self.iio_capture = iio_capture
         self.host = host
         self.iio_device = iio_device
         self.buffer_size = buffer_size
+        self.keep_raw = keep_raw
         self.sample_rate_hz = 100
         if self.iio_capture is None:
             raise HostError('Missing iio-capture binary')
@@ -159,3 +161,8 @@ class AcmeCapeInstrument(Instrument):
 
     def get_raw(self):
         return [self.raw_data_file]
+
+    def teardown(self):
+        if not self.keep_raw:
+            if os.path.isfile(self.raw_data_file):
+                os.remove(self.raw_data_file)

devlib/instrument/arm_energy_probe.py

@@ -71,7 +71,7 @@ class ArmEnergyProbeInstrument(Instrument):
 
     MAX_CHANNELS = 12 # 4 Arm Energy Probes
 
-    def __init__(self, target, config_file='./config-aep', ):
+    def __init__(self, target, config_file='./config-aep', keep_raw=False):
         super(ArmEnergyProbeInstrument, self).__init__(target)
         self.arm_probe = which('arm-probe')
         if self.arm_probe is None:
@@ -80,6 +80,7 @@ class ArmEnergyProbeInstrument(Instrument):
         self.attributes = ['power', 'voltage', 'current']
         self.sample_rate_hz = 10000
         self.config_file = config_file
+        self.keep_raw = keep_raw
 
         self.parser = AepParser()
         #TODO make it generic
@@ -142,3 +143,8 @@ class ArmEnergyProbeInstrument(Instrument):
 
     def get_raw(self):
         return [self.output_file_raw]
+
+    def teardown(self):
+        if not self.keep_raw:
+            if os.path.isfile(self.output_file_raw):
+                os.remove(self.output_file_raw)

devlib/instrument/daq.py

@@ -14,20 +14,23 @@
 #
 
 import os
+import shutil
 import tempfile
-from itertools import chain
+import time
+from itertools import chain, zip_longest
 
+from devlib.host import PACKAGE_BIN_DIRECTORY
 from devlib.instrument import Instrument, MeasurementsCsv, CONTINUOUS
 from devlib.exception import HostError
 from devlib.utils.csvutil import csvwriter, create_reader
 from devlib.utils.misc import unique
 
 try:
-    from daqpower.client import execute_command, Status
+    from daqpower.client import DaqClient
-    from daqpower.config import DeviceConfiguration, ServerConfiguration
+    from daqpower.config import DeviceConfiguration
 except ImportError as e:
-    execute_command, Status = None, None
+    DaqClient = None
-    DeviceConfiguration, ServerConfiguration, ConfigurationError = None, None, None
+    DeviceConfiguration = None
     import_error_mesg = e.args[0] if e.args else str(e)
 
 
@@ -44,26 +47,30 @@ class DaqInstrument(Instrument):
                  dv_range=0.2,
                  sample_rate_hz=10000,
                  channel_map=(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23),
+                 keep_raw=False,
+                 time_as_clock_boottime=True
                  ):
         # pylint: disable=no-member
         super(DaqInstrument, self).__init__(target)
+        self.keep_raw = keep_raw
         self._need_reset = True
         self._raw_files = []
-        if execute_command is None:
+        self.tempdir = None
+        self.target_boottime_clock_at_start = 0.0
+        if DaqClient is None:
             raise HostError('Could not import "daqpower": {}'.format(import_error_mesg))
         if labels is None:
             labels = ['PORT_{}'.format(i) for i in range(len(resistor_values))]
         if len(labels) != len(resistor_values):
             raise ValueError('"labels" and "resistor_values" must be of the same length')
-        self.server_config = ServerConfiguration(host=host,
+        self.daq_client = DaqClient(host, port)
-                                                 port=port)
+        try:
-        result = self.execute('list_devices')
+            devices = self.daq_client.list_devices()
-        if result.status == Status.OK:
+            if device_id not in devices:
-            if device_id not in result.data:
                 msg = 'Device "{}" is not found on the DAQ server. Available devices are: "{}"'
-                raise ValueError(msg.format(device_id, ', '.join(result.data)))
+                raise ValueError(msg.format(device_id, ', '.join(devices)))
-        elif result.status != Status.OKISH:
+        except Exception as e:
-            raise HostError('Problem querying DAQ server: {}'.format(result.message))
+            raise HostError('Problem querying DAQ server: {}'.format(e))
 
         self.device_config = DeviceConfiguration(device_id=device_id,
                                                  v_range=v_range,
@@ -73,36 +80,63 @@ class DaqInstrument(Instrument):
                                                  channel_map=channel_map,
                                                  labels=labels)
         self.sample_rate_hz = sample_rate_hz
+        self.time_as_clock_boottime = time_as_clock_boottime
 
+        self.add_channel('Time', 'time')
         for label in labels:
             for kind in ['power', 'voltage']:
                 self.add_channel(label, kind)
 
+        if time_as_clock_boottime:
+            host_path = os.path.join(PACKAGE_BIN_DIRECTORY, self.target.abi,
+                                     'get_clock_boottime')
+            self.clock_boottime_cmd = self.target.install_if_needed(host_path,
+                                                                    search_system_binaries=False)
+
+    def calculate_boottime_offset(self):
+        time_before = time.time()
+        out = self.target.execute(self.clock_boottime_cmd)
+        time_after = time.time()
+
+        remote_clock_boottime = float(out)
+        propagation_delay = (time_after - time_before) / 2
+        boottime_at_end = remote_clock_boottime + propagation_delay
+
+        return time_after - boottime_at_end
+
     def reset(self, sites=None, kinds=None, channels=None):
         super(DaqInstrument, self).reset(sites, kinds, channels)
-        self.execute('close')
+        self.daq_client.close()
-        result = self.execute('configure', config=self.device_config)
+        self.daq_client.configure(self.device_config)
-        if not result.status == Status.OK:  # pylint: disable=no-member
-            raise HostError(result.message)
         self._need_reset = False
         self._raw_files = []
 
     def start(self):
         if self._need_reset:
-            self.reset()
+            # Preserve channel order
-        self.execute('start')
+            self.reset(channels=self.channels.keys())
+
+        if self.time_as_clock_boottime:
+            target_boottime_offset = self.calculate_boottime_offset()
+            time_start = time.time()
+
+        self.daq_client.start()
+
+        if self.time_as_clock_boottime:
+            time_end = time.time()
+            self.target_boottime_clock_at_start = (time_start + time_end) / 2 - target_boottime_offset
 
     def stop(self):
-        self.execute('stop')
+        self.daq_client.stop()
         self._need_reset = True
 
     def get_data(self, outfile):  # pylint: disable=R0914
-        tempdir = tempfile.mkdtemp(prefix='daq-raw-')
+        self.tempdir = tempfile.mkdtemp(prefix='daq-raw-')
-        self.execute('get_data', output_directory=tempdir)
+        self.daq_client.get_data(self.tempdir)
         raw_file_map = {}
-        for entry in os.listdir(tempdir):
+        for entry in os.listdir(self.tempdir):
             site = os.path.splitext(entry)[0]
-            path = os.path.join(tempdir, entry)
+            path = os.path.join(self.tempdir, entry)
             raw_file_map[site] = path
             self._raw_files.append(path)
 
@@ -117,32 +151,32 @@ class DaqInstrument(Instrument):
                     site_readers[site] = reader
                     file_handles.append(fh)
                 except KeyError:
+                    if not site.startswith("Time"):
                         message = 'Could not get DAQ trace for {}; Obtained traces are in {}'
-                    raise HostError(message.format(site, tempdir))
+                        raise HostError(message.format(site, self.tempdir))
 
             # The first row is the headers
-            channel_order = []
+            channel_order = ['Time_time']
             for site, reader in site_readers.items():
                 channel_order.extend(['{}_{}'.format(site, kind)
                                       for kind in next(reader)])
 
-            def _read_next_rows():
+            def _read_rows():
-                parts = []
+                row_iter = zip_longest(*site_readers.values(), fillvalue=(None, None))
-                for reader in site_readers.values():
+                for raw_row in row_iter:
-                    try:
+                    raw_row = list(chain.from_iterable(raw_row))
-                        parts.extend(next(reader))
+                    raw_row.insert(0, _read_rows.row_time_s)
-                    except StopIteration:
+                    yield raw_row
-                        parts.extend([None, None])
+                    _read_rows.row_time_s += 1.0 / self.sample_rate_hz
-                return list(chain(parts))
+
+            _read_rows.row_time_s = self.target_boottime_clock_at_start
 
             with csvwriter(outfile) as writer:
                 field_names = [c.label for c in self.active_channels]
                 writer.writerow(field_names)
-                raw_row = _read_next_rows()
+                for raw_row in _read_rows():
-                while any(raw_row):
                     row = [raw_row[channel_order.index(f)] for f in field_names]
                     writer.writerow(row)
-                    raw_row = _read_next_rows()
 
             return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
         finally:
@@ -153,7 +187,7 @@ class DaqInstrument(Instrument):
         return self._raw_files
 
     def teardown(self):
-        self.execute('close')
+        self.daq_client.close()
-
+        if not self.keep_raw:
-    def execute(self, command, **kwargs):
+            if self.tempdir and os.path.isdir(self.tempdir):
-        return execute_command(self.server_config, command, **kwargs)
+                shutil.rmtree(self.tempdir)

devlib/instrument/energy_probe.py

@@ -34,9 +34,11 @@ class EnergyProbeInstrument(Instrument):
     def __init__(self, target, resistor_values,
                  labels=None,
                  device_entry='/dev/ttyACM0',
+                 keep_raw=False
                  ):
         super(EnergyProbeInstrument, self).__init__(target)
         self.resistor_values = resistor_values
+        self.keep_raw = keep_raw
         if labels is not None:
             self.labels = labels
         else:
@@ -126,3 +128,8 @@ class EnergyProbeInstrument(Instrument):
 
     def get_raw(self):
         return [self.raw_data_file]
+
+    def teardown(self):
+        if self.keep_raw:
+            if os.path.isfile(self.raw_data_file):
+                os.remove(self.raw_data_file)

devlib/instrument/frames.py

@@ -14,6 +14,8 @@
 #
 
 from __future__ import division
+import os
+
 from devlib.instrument import (Instrument, CONTINUOUS,
                                MeasurementsCsv, MeasurementType)
 from devlib.utils.rendering import (GfxinfoFrameCollector,
@@ -70,6 +72,11 @@ class FramesInstrument(Instrument):
     def _init_channels(self):
         raise NotImplementedError()
 
+    def teardown(self):
+        if not self.keep_raw:
+            if os.path.isfile(self._raw_file):
+                os.remove(self._raw_file)
+
 
 class GfxInfoFramesInstrument(FramesInstrument):
 

devlib/module/__init__.py

@@ -91,7 +91,7 @@ class FlashModule(Module):
 
     kind = 'flash'
 
-    def __call__(self, image_bundle=None, images=None, boot_config=None):
+    def __call__(self, image_bundle=None, images=None, boot_config=None, connect=True):
         raise NotImplementedError()
 
 

devlib/module/android.py

@@ -54,7 +54,7 @@ class FastbootFlashModule(FlashModule):
     def probe(target):
         return target.os == 'android'
 
-    def __call__(self, image_bundle=None, images=None, bootargs=None):
+    def __call__(self, image_bundle=None, images=None, bootargs=None, connect=True):
         if bootargs:
             raise ValueError('{} does not support boot configuration'.format(self.name))
         self.prelude_done = False
@@ -67,6 +67,7 @@ class FastbootFlashModule(FlashModule):
             self.logger.debug('flashing {}'.format(partition))
             self._flash_image(self.target, partition, expand_path(image_path))
         fastboot_command('reboot')
+        if connect:
             self.target.connect(timeout=180)
 
     def _validate_image_bundle(self, image_bundle):

devlib/module/cgroups.py

@@ -124,11 +124,10 @@ class Controller(object):
     def move_tasks(self, source, dest, exclude=None):
         if exclude is None:
             exclude = []
-        try:
+
-            srcg = self._cgroups[source]
+        srcg = self.cgroup(source)
-            dstg = self._cgroups[dest]
+        dstg = self.cgroup(dest)
-        except KeyError as e:
+
-            raise ValueError('Unknown group: {}'.format(e))
         self.target._execute_util(  # pylint: disable=protected-access
                     'cgroups_tasks_move {} {} \'{}\''.format(
                     srcg.directory, dstg.directory, exclude),
@@ -158,18 +157,18 @@ class Controller(object):
             raise ValueError('wrong type for "exclude" parameter, '
                              'it must be a str or a list')
 
-        logging.debug('Moving all tasks into %s', dest)
+        self.logger.debug('Moving all tasks into %s', dest)
 
         # Build list of tasks to exclude
         grep_filters = ''
         for comm in exclude:
             grep_filters += '-e {} '.format(comm)
-        logging.debug('   using grep filter: %s', grep_filters)
+        self.logger.debug('   using grep filter: %s', grep_filters)
         if grep_filters != '':
-            logging.debug('   excluding tasks which name matches:')
+            self.logger.debug('   excluding tasks which name matches:')
-            logging.debug('   %s', ', '.join(exclude))
+            self.logger.debug('   %s', ', '.join(exclude))
 
-        for cgroup in self._cgroups:
+        for cgroup in self.list_all():
             if cgroup != dest:
                 self.move_tasks(cgroup, dest, grep_filters)
 
@@ -288,10 +287,8 @@ class CGroup(object):
     def get(self):
         conf = {}
 
-        logging.debug('Reading %s attributes from:',
+        self.logger.debug('Reading %s attributes from:', self.controller.kind)
-                self.controller.kind)
+        self.logger.debug('  %s', self.directory)
-        logging.debug('  %s',
-                self.directory)
         output = self.target._execute_util(  # pylint: disable=protected-access
                     'cgroups_get_attributes {} {}'.format(
                     self.directory, self.controller.kind),
@@ -330,7 +327,7 @@ class CGroup(object):
 
     def get_tasks(self):
         task_ids = self.target.read_value(self.tasks_file).split()
-        logging.debug('Tasks: %s', task_ids)
+        self.logger.debug('Tasks: %s', task_ids)
         return list(map(int, task_ids))
 
     def add_task(self, tid):

devlib/module/cpufreq.py

@@ -111,7 +111,7 @@ class CpufreqModule(Module):
         :Keyword Arguments: Governor tunables, See :meth:`set_governor_tunables`
         """
         if not cpus:
-            cpus = range(self.target.number_of_cpus)
+            cpus = self.target.list_online_cpus()
 
         # Setting a governor & tunables for a cpu will set them for all cpus
         # in the same clock domain, so only manipulating one cpu per domain
@@ -212,7 +212,7 @@ class CpufreqModule(Module):
 
     @memoized
     def list_frequencies(self, cpu):
-        """Returns a list of frequencies supported by the cpu or an empty list
+        """Returns a sorted list of frequencies supported by the cpu or an empty list
         if not could be found."""
         if isinstance(cpu, int):
             cpu = 'cpu{}'.format(cpu)
@@ -234,7 +234,7 @@ class CpufreqModule(Module):
                 raise
 
             available_frequencies = list(map(int, reversed([f for f, _ in zip(out_iter, out_iter)])))
-        return available_frequencies
+        return sorted(available_frequencies)
 
     @memoized
     def get_max_available_frequency(self, cpu):

devlib/module/cpuidle.py

@@ -15,6 +15,9 @@
 # pylint: disable=attribute-defined-outside-init
 from past.builtins import basestring
 
+from operator import attrgetter
+from pprint import pformat
+
 from devlib.module import Module
 from devlib.utils.types import integer, boolean
 
@@ -96,40 +99,35 @@ class Cpuidle(Module):
 
     def __init__(self, target):
         super(Cpuidle, self).__init__(target)
-        self._states = {}
 
         basepath = '/sys/devices/system/cpu/'
         values_tree = self.target.read_tree_values(basepath, depth=4, check_exit_code=False)
-        i = 0
-        cpu_id = 'cpu{}'.format(i)
-        while cpu_id in values_tree:
-            cpu_node = values_tree[cpu_id]
 
-            if 'cpuidle' in cpu_node:
+        self._states = {
-                idle_node = cpu_node['cpuidle']
+            cpu_name: sorted(
-                self._states[cpu_id] = []
+                (
-                j = 0
+                    CpuidleState(
-                state_id = 'state{}'.format(j)
-                while state_id in idle_node:
-                    state_node = idle_node[state_id]
-                    state = CpuidleState(
                         self.target,
-                        index=j,
+                        # state_name is formatted as "state42"
-                        path=self.target.path.join(basepath, cpu_id, 'cpuidle', state_id),
+                        index=int(state_name[len('state'):]),
+                        path=self.target.path.join(basepath, cpu_name, 'cpuidle', state_name),
                         name=state_node['name'],
                         desc=state_node['desc'],
                         power=int(state_node['power']),
                         latency=int(state_node['latency']),
                         residency=int(state_node['residency']) if 'residency' in state_node else None,
                     )
-                    msg = 'Adding {} state {}: {} {}'
+                    for state_name, state_node in cpu_node['cpuidle'].items()
-                    self.logger.debug(msg.format(cpu_id, j, state.name, state.desc))
+                    if state_name.startswith('state')
-                    self._states[cpu_id].append(state)
+                ),
-                    j += 1
+                key=attrgetter('index'),
-                    state_id = 'state{}'.format(j)
+            )
 
-            i += 1
+            for cpu_name, cpu_node in values_tree.items()
-            cpu_id = 'cpu{}'.format(i)
+            if cpu_name.startswith('cpu') and 'cpuidle' in cpu_node
+        }
+
+        self.logger.debug('Adding cpuidle states:\n{}'.format(pformat(self._states)))
 
     def get_states(self, cpu=0):
         if isinstance(cpu, int):
@@ -173,4 +171,7 @@ class Cpuidle(Module):
         return self.target.read_value(self.target.path.join(self.root_path, 'current_driver'))
 
     def get_governor(self):
-        return self.target.read_value(self.target.path.join(self.root_path, 'current_governor_ro'))
+        path = self.target.path.join(self.root_path, 'current_governor_ro')
+        if not self.target.file_exists(path):
+            path = self.target.path.join(self.root_path, 'current_governor')
+        return self.target.read_value(path)

devlib/module/sched.py

@@ -52,6 +52,12 @@ class SchedProcFSNode(object):
 
     _re_procfs_node = re.compile(r"(?P<name>.*\D)(?P<digits>\d+)$")
 
+    PACKABLE_ENTRIES = [
+        "cpu",
+        "domain",
+        "group"
+    ]
+
     @staticmethod
     def _ends_with_digits(node):
         if not isinstance(node, basestring):
@@ -71,18 +77,19 @@ class SchedProcFSNode(object):
         """
         :returns: The name of the procfs node
         """
-        return re.search(SchedProcFSNode._re_procfs_node, node).group("name")
+        match = re.search(SchedProcFSNode._re_procfs_node, node)
+        if match:
+            return match.group("name")
 
-    @staticmethod
+        return node
-    def _packable(node, entries):
+
+    @classmethod
+    def _packable(cls, node):
         """
         :returns: Whether it makes sense to pack a node into a common entry
         """
         return (SchedProcFSNode._ends_with_digits(node) and
-                any([SchedProcFSNode._ends_with_digits(x) and
+                SchedProcFSNode._node_name(node) in cls.PACKABLE_ENTRIES)
-                     SchedProcFSNode._node_digits(x) != SchedProcFSNode._node_digits(node) and
-                     SchedProcFSNode._node_name(x) == SchedProcFSNode._node_name(node)
-                     for x in entries]))
 
     @staticmethod
     def _build_directory(node_name, node_data):
@@ -119,7 +126,7 @@ class SchedProcFSNode(object):
         # Find which entries can be packed into a common entry
         packables = {
             node : SchedProcFSNode._node_name(node) + "s"
-            for node in list(nodes.keys()) if SchedProcFSNode._packable(node, list(nodes.keys()))
+            for node in list(nodes.keys()) if SchedProcFSNode._packable(node)
         }
 
         self._dyn_attrs = {}
@@ -228,13 +235,13 @@ class SchedProcFSData(SchedProcFSNode):
         # Even if we have a CPU entry, it can be empty (e.g. hotplugged out)
         # Make sure some data is there
         for cpu in cpus:
-            if target.file_exists(target.path.join(path, cpu, "domain0", "name")):
+            if target.file_exists(target.path.join(path, cpu, "domain0", "flags")):
                 return True
 
         return False
 
     def __init__(self, target, path=None):
-        if not path:
+        if path is None:
             path = self.sched_domain_root
 
         procfs = target.read_tree_values(path, depth=self._read_depth)
@@ -252,7 +259,21 @@ class SchedModule(Module):
         logger = logging.getLogger(SchedModule.name)
         SchedDomainFlag.check_version(target, logger)
 
-        return SchedProcFSData.available(target)
+        # It makes sense to load this module if at least one of those
+        # functionalities is enabled
+        schedproc = SchedProcFSData.available(target)
+        debug = SchedModule.target_has_debug(target)
+        dmips = any([target.file_exists(SchedModule.cpu_dmips_capacity_path(target, cpu))
+                     for cpu in target.list_online_cpus()])
+
+        logger.info("Scheduler sched_domain procfs entries %s",
+                    "found" if schedproc else "not found")
+        logger.info("Detected kernel compiled with SCHED_DEBUG=%s",
+                    "y" if debug else "n")
+        logger.info("CPU capacity sysfs entries %s",
+                    "found" if dmips else "not found")
+
+        return schedproc or debug or dmips
 
     def get_kernel_attributes(self, matching=None, check_exit_code=True):
         """
@@ -306,12 +327,16 @@ class SchedModule(Module):
         path = '/proc/sys/kernel/sched_' + attr
         self.target.write_value(path, value, verify)
 
+    @classmethod
+    def target_has_debug(cls, target):
+        if target.config.get('SCHED_DEBUG') != 'y':
+            return False
+        return target.file_exists('/sys/kernel/debug/sched_features')
+
     @property
     @memoized
     def has_debug(self):
-        if self.target.config.get('SCHED_DEBUG') != 'y':
+        return self.target_has_debug(self.target)
-            return False;
-        return self.target.file_exists('/sys/kernel/debug/sched_features')
 
     def get_features(self):
         """
@@ -386,17 +411,26 @@ class SchedModule(Module):
         :returns: Whether energy model data is available for 'cpu'
         """
         if not sd:
-            sd = SchedProcFSData(self.target, cpu)
+            sd = self.get_cpu_sd_info(cpu)
 
         return sd.procfs["domain0"].get("group0", {}).get("energy", {}).get("cap_states") != None
 
+    @classmethod
+    def cpu_dmips_capacity_path(cls, target, cpu):
+        """
+        :returns: The target sysfs path where the dmips capacity data should be
+        """
+        return target.path.join(
+            cls.cpu_sysfs_root,
+            'cpu{}/cpu_capacity'.format(cpu))
+
     @memoized
     def has_dmips_capacity(self, cpu):
         """
         :returns: Whether dmips capacity data is available for 'cpu'
         """
         return self.target.file_exists(
-            self.target.path.join(self.cpu_sysfs_root, 'cpu{}/cpu_capacity'.format(cpu))
+            self.cpu_dmips_capacity_path(self.target, cpu)
         )
 
     @memoized
@@ -405,10 +439,13 @@ class SchedModule(Module):
         :returns: The maximum capacity value exposed by the EAS energy model
         """
         if not sd:
-            sd = SchedProcFSData(self.target, cpu)
+            sd = self.get_cpu_sd_info(cpu)
 
         cap_states = sd.domains[0].groups[0].energy.cap_states
-        return int(cap_states.split('\t')[-2])
+        cap_states_list = cap_states.split('\t')
+        num_cap_states = sd.domains[0].groups[0].energy.nr_cap_states
+        max_cap_index = -1 * int(len(cap_states_list) / num_cap_states)
+        return int(cap_states_list[max_cap_index])
 
     @memoized
     def get_dmips_capacity(self, cpu):
@@ -416,14 +453,9 @@ class SchedModule(Module):
         :returns: The capacity value generated from the capacity-dmips-mhz DT entry
         """
         return self.target.read_value(
-            self.target.path.join(
+            self.cpu_dmips_capacity_path(self.target, cpu), int
-                self.cpu_sysfs_root,
-                'cpu{}/cpu_capacity'.format(cpu)
-            ),
-            int
         )
 
-    @memoized
     def get_capacities(self, default=None):
         """
         :param default: Default capacity value to find if no data is
@@ -434,16 +466,30 @@ class SchedModule(Module):
         :raises RuntimeError: Raised when no capacity information is
         found and 'default' is None
         """
-        cpus = list(range(self.target.number_of_cpus))
+        cpus = self.target.list_online_cpus()
 
         capacities = {}
-        sd_info = self.get_sd_info()
 
         for cpu in cpus:
+            if self.has_dmips_capacity(cpu):
+                capacities[cpu] = self.get_dmips_capacity(cpu)
+
+        missing_cpus = set(cpus).difference(capacities.keys())
+        if not missing_cpus:
+            return capacities
+
+        if not SchedProcFSData.available(self.target):
+            if default != None:
+                capacities.update({cpu : default for cpu in missing_cpus})
+                return capacities
+            else:
+                raise RuntimeError(
+                    'No capacity data for cpus {}'.format(sorted(missing_cpus)))
+
+        sd_info = self.get_sd_info()
+        for cpu in missing_cpus:
             if self.has_em(cpu, sd_info.cpus[cpu]):
                 capacities[cpu] = self.get_em_capacity(cpu, sd_info.cpus[cpu])
-            elif self.has_dmips_capacity(cpu):
-                capacities[cpu] = self.get_dmips_capacity(cpu)
             else:
                 if default != None:
                     capacities[cpu] = default

devlib/module/thermal.py

@@ -48,7 +48,7 @@ class ThermalZone(object):
         self.path = target.path.join(root, self.name)
         self.trip_points = {}
 
-        for entry in self.target.list_directory(self.path):
+        for entry in self.target.list_directory(self.path, as_root=target.is_rooted):
             re_match = re.match('^trip_point_([0-9]+)_temp', entry)
             if re_match is not None:
                 self.add_trip_point(re_match.group(1))
@@ -88,6 +88,9 @@ class ThermalModule(Module):
 
         for entry in target.list_directory(self.thermal_root):
             re_match = re.match('^(thermal_zone|cooling_device)([0-9]+)', entry)
+            if not re_match:
+                self.logger.warning('unknown thermal entry: %s', entry)
+                continue
 
             if re_match.group(1) == 'thermal_zone':
                 self.add_thermal_zone(re_match.group(2))

devlib/module/vexpress.py

@@ -130,7 +130,7 @@ class VexpressBootModule(BootModule):
                                     init_dtr=0) as tty:
             self.get_through_early_boot(tty)
             self.perform_boot_sequence(tty)
-            self.wait_for_android_prompt(tty)
+            self.wait_for_shell_prompt(tty)
 
     def perform_boot_sequence(self, tty):
         raise NotImplementedError()
@@ -159,8 +159,8 @@ class VexpressBootModule(BootModule):
         menu.wait(timeout=self.timeout)
         return menu
 
-    def wait_for_android_prompt(self, tty):
+    def wait_for_shell_prompt(self, tty):
-        self.logger.debug('Waiting for the Android prompt.')
+        self.logger.debug('Waiting for the shell prompt.')
         tty.expect(self.target.shell_prompt, timeout=self.timeout)
         # This delay is needed to allow the platform some time to finish
         # initilizing; querying the ip address too early from connect() may
@@ -325,7 +325,7 @@ class VersatileExpressFlashModule(FlashModule):
         self.timeout = timeout
         self.short_delay = short_delay
 
-    def __call__(self, image_bundle=None, images=None, bootargs=None):
+    def __call__(self, image_bundle=None, images=None, bootargs=None, connect=True):
         self.target.hard_reset()
         with open_serial_connection(port=self.target.platform.serial_port,
                                     baudrate=self.target.platform.baudrate,
@@ -346,6 +346,7 @@ class VersatileExpressFlashModule(FlashModule):
             msg = 'Could not deploy images to {}; got: {}'
             raise TargetStableError(msg.format(self.vemsd_mount, e))
         self.target.boot()
+        if connect:
             self.target.connect(timeout=30)
 
     def _deploy_image_bundle(self, bundle):

devlib/platform/__init__.py

@@ -78,7 +78,16 @@ class Platform(object):
 
     def _set_model_from_target(self, target):
         if target.os == 'android':
+            try:
+                self.model = target.getprop(prop='ro.product.device')
+            except KeyError:
                 self.model = target.getprop('ro.product.model')
+        elif target.file_exists("/proc/device-tree/model"):
+            # There is currently no better way to do this cross platform.
+            # ARM does not have dmidecode
+            raw_model = target.execute("cat /proc/device-tree/model")
+            device_model_to_return = '_'.join(raw_model.split()[:2])
+            return device_model_to_return.rstrip(' \t\r\n\0')
         elif target.is_rooted:
             try:
                 self.model = target.execute('dmidecode -s system-version',

devlib/platform/arm.py

@@ -90,9 +90,6 @@ class VersatileExpressPlatform(Platform):
     def _init_android_target(self, target):
         if target.connection_settings.get('device') is None:
             addr = self._get_target_ip_address(target)
-            if sys.version_info[0] == 3:
-                # Convert bytes to string for Python3 compatibility
-                addr = addr.decode("utf-8")
             target.connection_settings['device'] = addr + ':5555'
 
     def _init_linux_target(self, target):
@@ -108,7 +105,7 @@ class VersatileExpressPlatform(Platform):
                                     init_dtr=0) as tty:
             tty.sendline('su')  # this is, apprently, required to query network device
                                 # info by name on recent Juno builds...
-            self.logger.debug('Waiting for the Android shell prompt.')
+            self.logger.debug('Waiting for the shell prompt.')
             tty.expect(target.shell_prompt)
 
             self.logger.debug('Waiting for IP address...')
@@ -119,7 +116,7 @@ class VersatileExpressPlatform(Platform):
                     time.sleep(1)
                     try:
                         tty.expect(r'inet ([1-9]\d*.\d+.\d+.\d+)', timeout=10)
-                        return tty.match.group(1)
+                        return tty.match.group(1).decode('utf-8')
                     except pexpect.TIMEOUT:
                         pass  # We have our own timeout -- see below.
                     if (time.time() - wait_start_time) > self.ready_timeout:

devlib/target.py

@@ -15,7 +15,9 @@
 
 import io
 import base64
+import functools
 import gzip
+import glob
 import os
 import re
 import time
@@ -26,9 +28,11 @@ import sys
 import tarfile
 import tempfile
 import threading
+import uuid
 import xml.dom.minidom
 import copy
 from collections import namedtuple, defaultdict
+from contextlib import contextmanager
 from pipes import quote
 from past.builtins import long
 from past.types import basestring
@@ -45,24 +49,26 @@ from devlib.module import get_module
 from devlib.platform import Platform
 from devlib.exception import (DevlibTransientError, TargetStableError,
                               TargetNotRespondingError, TimeoutError,
-                              TargetTransientError, KernelConfigKeyError) # pylint: disable=redefined-builtin
+                              TargetTransientError, KernelConfigKeyError,
+                              TargetError, HostError) # pylint: disable=redefined-builtin
 from devlib.utils.ssh import SshConnection
 from devlib.utils.android import AdbConnection, AndroidProperties, LogcatMonitor, adb_command, adb_disconnect, INTENT_FLAGS
 from devlib.utils.misc import memoized, isiterable, convert_new_lines
 from devlib.utils.misc import commonprefix, merge_lists
 from devlib.utils.misc import ABI_MAP, get_cpu_name, ranges_to_list
+from devlib.utils.misc import batch_contextmanager, tls_property, nullcontext
 from devlib.utils.types import integer, boolean, bitmask, identifier, caseless_string, bytes_regex
 
 
 FSTAB_ENTRY_REGEX = re.compile(r'(\S+) on (.+) type (\S+) \((\S+)\)')
-ANDROID_SCREEN_STATE_REGEX = re.compile('(?:mPowerState|mScreenOn|Display Power: state)=([0-9]+|true|false|ON|OFF)',
+ANDROID_SCREEN_STATE_REGEX = re.compile('(?:mPowerState|mScreenOn|Display Power: state)=([0-9]+|true|false|ON|OFF|DOZE)',
                                         re.IGNORECASE)
 ANDROID_SCREEN_RESOLUTION_REGEX = re.compile(r'cur=(?P<width>\d+)x(?P<height>\d+)')
 ANDROID_SCREEN_ROTATION_REGEX = re.compile(r'orientation=(?P<rotation>[0-3])')
 DEFAULT_SHELL_PROMPT = re.compile(r'^.*(shell|root|juno)@?.*:[/~]\S* *[#$] ',
                                   re.MULTILINE)
 KVERSION_REGEX = re.compile(
-    r'(?P<version>\d+)(\.(?P<major>\d+)(\.(?P<minor>\d+)(-rc(?P<rc>\d+))?)?)?(.*-g(?P<sha1>[0-9a-fA-F]{7,}))?'
+    r'(?P<version>\d+)(\.(?P<major>\d+)(\.(?P<minor>\d+)(-rc(?P<rc>\d+))?)?)?(-(?P<commits>\d+)-g(?P<sha1>[0-9a-fA-F]{7,}))?'
 )
 
 GOOGLE_DNS_SERVER_ADDRESS = '8.8.8.8'
@@ -70,7 +76,6 @@ GOOGLE_DNS_SERVER_ADDRESS = '8.8.8.8'
 
 installed_package_info = namedtuple('installed_package_info', 'apk_path package')
 
-
 class Target(object):
 
     path = None
@@ -107,21 +112,18 @@ class Target(object):
 
     @property
     def connected_as_root(self):
-        if self._connected_as_root is None:
+        return self.conn and self.conn.connected_as_root
-            result = self.execute('id')
-            self._connected_as_root = 'uid=0(' in result
-        return self._connected_as_root
 
     @property
-    @memoized
     def is_rooted(self):
-        if self.connected_as_root:
+        if self._is_rooted is None:
-            return True
             try:
                 self.execute('ls /', timeout=5, as_root=True)
-            return True
+                self._is_rooted = True
-        except (TargetStableError, TimeoutError):
+            except(TargetError, TimeoutError):
-            return False
+                self._is_rooted = False
+
+        return self._is_rooted or self.connected_as_root
 
     @property
     @memoized
@@ -137,6 +139,10 @@ class Target(object):
     def os_version(self):  # pylint: disable=no-self-use
         return {}
 
+    @property
+    def model(self):
+        return self.platform.model
+
     @property
     def abi(self):  # pylint: disable=no-self-use
         return None
@@ -155,12 +161,38 @@ class Target(object):
     def number_of_cpus(self):
         num_cpus = 0
         corere = re.compile(r'^\s*cpu\d+\s*$')
-        output = self.execute('ls /sys/devices/system/cpu')
+        output = self.execute('ls /sys/devices/system/cpu', as_root=self.is_rooted)
         for entry in output.split():
             if corere.match(entry):
                 num_cpus += 1
         return num_cpus
 
+    @property
+    @memoized
+    def number_of_nodes(self):
+        cmd = 'cd /sys/devices/system/node && {busybox} find . -maxdepth 1'.format(busybox=quote(self.busybox))
+        try:
+            output = self.execute(cmd, as_root=self.is_rooted)
+        except TargetStableError:
+            return 1
+        else:
+            nodere = re.compile(r'^\./node\d+\s*$')
+            num_nodes = 0
+            for entry in output.splitlines():
+                if nodere.match(entry):
+                    num_nodes += 1
+            return num_nodes
+
+    @property
+    @memoized
+    def list_nodes_cpus(self):
+        nodes_cpus = []
+        for node in range(self.number_of_nodes):
+            path = self.path.join('/sys/devices/system/node/node{}/cpulist'.format(node))
+            output = self.read_value(path)
+            nodes_cpus.append(ranges_to_list(output))
+        return nodes_cpus
+
     @property
     @memoized
     def config(self):
@@ -183,17 +215,7 @@ class Target(object):
     @memoized
     def page_size_kb(self):
         cmd = "cat /proc/self/smaps | {0} grep KernelPageSize | {0} head -n 1 | {0} awk '{{ print $2 }}'"
-        return int(self.execute(cmd.format(self.busybox)))
+        return int(self.execute(cmd.format(self.busybox)) or 0)
-
-    @property
-    def conn(self):
-        if self._connections:
-            tid = id(threading.current_thread())
-            if tid not in self._connections:
-                self._connections[tid] = self.get_connection()
-            return self._connections[tid]
-        else:
-            return None
 
     @property
     def shutils(self):
@@ -201,6 +223,13 @@ class Target(object):
             self._setup_shutils()
         return self._shutils
 
+    @tls_property
+    def _conn(self):
+        return self.get_connection()
+
+    # Add a basic property that does not require calling to get the value
+    conn = _conn.basic_property
+
     def __init__(self,
                  connection_settings=None,
                  platform=None,
@@ -213,7 +242,8 @@ class Target(object):
                  conn_cls=None,
                  is_container=False
                  ):
-        self._connected_as_root = None
+
+        self._is_rooted = None
         self.connection_settings = connection_settings or {}
         # Set self.platform: either it's given directly (by platform argument)
         # or it's given in the connection_settings argument
@@ -242,7 +272,6 @@ class Target(object):
         self._installed_binaries = {}
         self._installed_modules = {}
         self._cache = {}
-        self._connections = {}
         self._shutils = None
         self._file_transfer_cache = None
         self.busybox = None
@@ -261,23 +290,34 @@ class Target(object):
 
     def connect(self, timeout=None, check_boot_completed=True):
         self.platform.init_target_connection(self)
-        tid = id(threading.current_thread())
+        # Forcefully set the thread-local value for the connection, with the
-        self._connections[tid] = self.get_connection(timeout=timeout)
+        # timeout we want
+        self.conn = self.get_connection(timeout=timeout)
         if check_boot_completed:
             self.wait_boot_complete(timeout)
+        self.check_connection()
         self._resolve_paths()
         self.execute('mkdir -p {}'.format(quote(self.working_directory)))
         self.execute('mkdir -p {}'.format(quote(self.executables_directory)))
-        self.busybox = self.install(os.path.join(PACKAGE_BIN_DIRECTORY, self.abi, 'busybox'))
+        self.busybox = self.install(os.path.join(PACKAGE_BIN_DIRECTORY, self.abi, 'busybox'), timeout=30)
+        self.conn.busybox = self.busybox
         self.platform.update_from_target(self)
         self._update_modules('connected')
         if self.platform.big_core and self.load_default_modules:
             self._install_module(get_module('bl'))
 
+    def check_connection(self):
+        """
+        Check that the connection works without obvious issues.
+        """
+        out = self.execute('true', as_root=False)
+        if out.strip():
+            raise TargetStableError('The shell seems to not be functional and adds content to stderr: {}'.format(out))
+
     def disconnect(self):
-        for conn in self._connections.values():
+        connections = self._conn.get_all_values()
+        for conn in connections:
             conn.close()
-        self._connections = {}
 
     def get_connection(self, timeout=None):
         if self.conn_cls is None:
@@ -322,31 +362,143 @@ class Target(object):
             timeout = max(timeout - reset_delay, 10)
         if self.has('boot'):
             self.boot()  # pylint: disable=no-member
-        self._connected_as_root = None
+        self.conn.connected_as_root = None
         if connect:
             self.connect(timeout=timeout)
 
     # file transfer
 
-    def push(self, source, dest, as_root=False, timeout=None):  # pylint: disable=arguments-differ
+    @contextmanager
-        if not as_root:
+    def _xfer_cache_path(self, name):
-            self.conn.push(source, dest, timeout=timeout)
+        """
-        else:
+        Context manager to provide a unique path in the transfer cache with the
-            device_tempfile = self.path.join(self._file_transfer_cache, source.lstrip(self.path.sep))
+        basename of the given name.
-            self.execute("mkdir -p {}".format(quote(self.path.dirname(device_tempfile))))
+        """
-            self.conn.push(source, device_tempfile, timeout=timeout)
+        # Use a UUID to avoid race conditions on the target side
-            self.execute("cp {} {}".format(quote(device_tempfile), quote(dest)), as_root=True)
+        xfer_uuid = uuid.uuid4().hex
+        folder = self.path.join(self._file_transfer_cache, xfer_uuid)
+        # Make sure basename will work on folders too
+        name = os.path.normpath(name)
+        # Ensure the name is relative so that os.path.join() will actually
+        # join the paths rather than ignoring the first one.
+        name = './{}'.format(os.path.basename(name))
 
-    def pull(self, source, dest, as_root=False, timeout=None):  # pylint: disable=arguments-differ
+        check_rm = False
-        if not as_root:
+        try:
-            self.conn.pull(source, dest, timeout=timeout)
+            self.makedirs(folder)
+            # Don't check the exit code as the folder might not even exist
+            # before this point, if creating it failed
+            check_rm = True
+            yield self.path.join(folder, name)
+        finally:
+            self.execute('rm -rf -- {}'.format(quote(folder)), check_exit_code=check_rm)
+
+    def _prepare_xfer(self, action, sources, dest):
+        """
+        Check the sanity of sources and destination and prepare the ground for
+        transfering multiple sources.
+        """
+        if action == 'push':
+            src_excep = HostError
+            dst_excep = TargetStableError
+            dst_path_exists = self.file_exists
+            dst_is_dir = self.directory_exists
+            dst_mkdir = self.makedirs
+
+            for source in sources:
+                if not os.path.exists(source):
+                    raise HostError('No such file "{}"'.format(source))
         else:
-            device_tempfile = self.path.join(self._file_transfer_cache, source.lstrip(self.path.sep))
+            src_excep = TargetStableError
-            self.execute("mkdir -p {}".format(quote(self.path.dirname(device_tempfile))))
+            dst_excep = HostError
-            self.execute("cp -r {} {}".format(quote(source), quote(device_tempfile)), as_root=True)
+            dst_path_exists = os.path.exists
-            self.execute("chmod 0644 {}".format(quote(device_tempfile)), as_root=True)
+            dst_is_dir = os.path.isdir
-            self.conn.pull(device_tempfile, dest, timeout=timeout)
+            dst_mkdir = functools.partial(os.makedirs, exist_ok=True)
-            self.execute("rm -r {}".format(quote(device_tempfile)), as_root=True)
+
+        if not sources:
+            raise src_excep('No file matching: {}'.format(source))
+        elif len(sources) > 1:
+            if dst_path_exists(dest):
+                if not dst_is_dir(dest):
+                    raise dst_excep('A folder dest is required for multiple matches but destination is a file: {}'.format(dest))
+            else:
+                dst_mkdir(dest)
+
+
+    def push(self, source, dest, as_root=False, timeout=None, globbing=False):  # pylint: disable=arguments-differ
+        sources = glob.glob(source) if globbing else [source]
+        self._prepare_xfer('push', sources, dest)
+
+        def do_push(sources, dest):
+            return self.conn.push(sources, dest, timeout=timeout)
+
+        if as_root:
+            for source in sources:
+                with self._xfer_cache_path(source) as device_tempfile:
+                    do_push([source], device_tempfile)
+                    self.execute("mv -f -- {} {}".format(quote(device_tempfile), quote(dest)), as_root=True)
+        else:
+            do_push(sources, dest)
+
+    def _expand_glob(self, pattern, **kwargs):
+        """
+        Expand the given path globbing pattern on the target using the shell
+        globbing.
+        """
+        # Since we split the results based on new lines, forbid them in the
+        # pattern
+        if '\n' in pattern:
+            raise ValueError(r'Newline character \n are not allowed in globbing patterns')
+
+        # If the pattern is in fact a plain filename, skip the expansion on the
+        # target to avoid an unncessary command execution.
+        #
+        # fnmatch char list from: https://docs.python.org/3/library/fnmatch.html
+        special_chars = ['*', '?', '[', ']']
+        if not any(char in pattern for char in special_chars):
+            return [pattern]
+
+        # Characters to escape that are impacting parameter splitting, since we
+        # want the pattern to be given in one piece. Unfortunately, there is no
+        # fool-proof way of doing that without also escaping globbing special
+        # characters such as wildcard which would defeat the entire purpose of
+        # that function.
+        for c in [' ', "'", '"']:
+            pattern = pattern.replace(c, '\\' + c)
+
+        cmd = "exec printf '%s\n' {}".format(pattern)
+        # Make sure to use the same shell everywhere for the path globbing,
+        # ensuring consistent results no matter what is the default platform
+        # shell
+        cmd = '{} sh -c {} 2>/dev/null'.format(quote(self.busybox), quote(cmd))
+        # On some shells, match failure will make the command "return" a
+        # non-zero code, even though the command was not actually called
+        result = self.execute(cmd, strip_colors=False, check_exit_code=False, **kwargs)
+        paths = result.splitlines()
+        if not paths:
+            raise TargetStableError('No file matching: {}'.format(pattern))
+
+        return paths
+
+    def pull(self, source, dest, as_root=False, timeout=None, globbing=False):  # pylint: disable=arguments-differ
+        if globbing:
+            sources = self._expand_glob(source, as_root=as_root)
+        else:
+            sources = [source]
+
+        self._prepare_xfer('pull', sources, dest)
+
+        def do_pull(sources, dest):
+            self.conn.pull(sources, dest, timeout=timeout)
+
+        if as_root:
+            for source in sources:
+                with self._xfer_cache_path(source) as device_tempfile:
+                    self.execute("cp -r -- {} {}".format(quote(source), quote(device_tempfile)), as_root=True)
+                    self.execute("{} chmod 0644 -- {}".format(self.busybox, quote(device_tempfile)), as_root=True)
+                    do_pull([device_tempfile], dest)
+        else:
+            do_pull(sources, dest)
 
     def get_directory(self, source_dir, dest, as_root=False):
         """ Pull a directory from the device, after compressing dir """
@@ -359,11 +511,12 @@ class Target(object):
         tmpfile = os.path.join(dest, tar_file_name)
 
         # If root is required, use tmp location for tar creation.
-        if as_root:
+        tar_file_cm = self._xfer_cache_path if as_root else nullcontext
-            tar_file_name = self.path.join(self._file_transfer_cache, tar_file_name)
 
         # Does the folder exist?
         self.execute('ls -la {}'.format(quote(source_dir)), as_root=as_root)
+
+        with tar_file_cm(tar_file_name) as tar_file_name:
             # Try compressing the folder
             try:
                 self.execute('{} tar -cvf {} {}'.format(
@@ -377,14 +530,26 @@ class Target(object):
                 os.mkdir(dest)
             self.pull(tar_file_name, tmpfile)
             # Decompress
-        f = tarfile.open(tmpfile, 'r')
+            with tarfile.open(tmpfile, 'r') as f:
                 f.extractall(outdir)
             os.remove(tmpfile)
 
     # execution
 
     def execute(self, command, timeout=None, check_exit_code=True,
-                as_root=False, strip_colors=True, will_succeed=False):
+                as_root=False, strip_colors=True, will_succeed=False,
+                force_locale='C'):
+
+        # Force the locale if necessary for more predictable output
+        if force_locale:
+            # Use an explicit export so that the command is allowed to be any
+            # shell statement, rather than just a command invocation
+            command = 'export LC_ALL={} && {}'.format(quote(force_locale), command)
+
+        # Ensure to use deployed command when availables
+        if self.executables_directory:
+            command = "export PATH={}:$PATH && {}".format(quote(self.executables_directory), command)
+
         return self.conn.execute(command, timeout=timeout,
                 check_exit_code=check_exit_code, as_root=as_root,
                 strip_colors=strip_colors, will_succeed=will_succeed)
@@ -478,6 +643,18 @@ class Target(object):
     def read_bool(self, path):
         return self.read_value(path, kind=boolean)
 
+    @contextmanager
+    def revertable_write_value(self, path, value, verify=True):
+        orig_value = self.read_value(path)
+        try:
+            self.write_value(path, value, verify)
+            yield
+        finally:
+            self.write_value(path, orig_value, verify)
+
+    def batch_revertable_write_value(self, kwargs_list):
+        return batch_contextmanager(self.revertable_write_value, kwargs_list)
+
     def write_value(self, path, value, verify=True):
         value = str(value)
         self.execute('echo {} > {}'.format(quote(value), quote(path)), check_exit_code=False, as_root=True)
@@ -490,19 +667,19 @@ class Target(object):
     def reset(self):
         try:
             self.execute('reboot', as_root=self.needs_su, timeout=2)
-        except (DevlibTransientError, subprocess.CalledProcessError):
+        except (TargetError, subprocess.CalledProcessError):
             # on some targets "reboot" doesn't return gracefully
             pass
-        self._connected_as_root = None
+        self.conn.connected_as_root = None
 
     def check_responsive(self, explode=True):
         try:
             self.conn.execute('ls /', timeout=5)
-            return 1
+            return True
         except (DevlibTransientError, subprocess.CalledProcessError):
             if explode:
                 raise TargetNotRespondingError('Target {} is not responding'.format(self.conn.name))
-            return 0
+            return False
 
     # process management
 
@@ -525,6 +702,9 @@ class Target(object):
 
     # files
 
+    def makedirs(self, path):
+        self.execute('mkdir -p {}'.format(quote(path)))
+
     def file_exists(self, filepath):
         command = 'if [ -e {} ]; then echo 1; else echo 0; fi'
         output = self.execute(command.format(quote(filepath)), as_root=self.is_rooted)
@@ -568,7 +748,7 @@ class Target(object):
         raise IOError('No usable temporary filename found')
 
     def remove(self, path, as_root=False):
-        self.execute('rm -rf {}'.format(quote(path)), as_root=as_root)
+        self.execute('rm -rf -- {}'.format(quote(path)), as_root=as_root)
 
     # misc
     def core_cpus(self, core):
@@ -619,12 +799,12 @@ class Target(object):
 
     which = get_installed
 
-    def install_if_needed(self, host_path, search_system_binaries=True):
+    def install_if_needed(self, host_path, search_system_binaries=True, timeout=None):
 
         binary_path = self.get_installed(os.path.split(host_path)[1],
                                          search_system_binaries=search_system_binaries)
         if not binary_path:
-            binary_path = self.install(host_path)
+            binary_path = self.install(host_path, timeout=timeout)
         return binary_path
 
     def is_installed(self, name):
@@ -774,6 +954,18 @@ class Target(object):
                                                 strip_null_chars)
         return _build_path_tree(value_map, path, self.path.sep, dictcls)
 
+    def install_module(self, mod, **params):
+        mod = get_module(mod)
+        if mod.stage == 'early':
+            msg = 'Module {} cannot be installed after device setup has already occoured.'
+            raise TargetStableError(msg)
+
+        if mod.probe(self):
+            self._install_module(mod, **params)
+        else:
+            msg = 'Module {} is not supported by the target'.format(mod.name)
+            raise TargetStableError(msg)
+
     # internal methods
 
     def _setup_shutils(self):
@@ -841,12 +1033,19 @@ class Target(object):
                     self.logger.warning(msg)
 
     def _install_module(self, mod, **params):
-        if mod.name not in self._installed_modules:
+        name = mod.name
-            self.logger.debug('Installing module {}'.format(mod.name))
+        if name not in self._installed_modules:
+            self.logger.debug('Installing module {}'.format(name))
+            try:
                 mod.install(self, **params)
-            self._installed_modules[mod.name] = mod
+            except Exception as e:
+                self.logger.error('Module "{}" failed to install on target'.format(name))
+                raise
+            self._installed_modules[name] = mod
+            if name not in self.modules:
+                self.modules.append(name)
         else:
-            self.logger.debug('Module {} is already installed.'.format(mod.name))
+            self.logger.debug('Module {} is already installed.'.format(name))
 
     def _resolve_paths(self):
         raise NotImplementedError()
@@ -920,17 +1119,6 @@ class LinuxTarget(Target):
             os_version[name] = convert_new_lines(output.strip()).replace('\n', ' ')
         return os_version
 
-    @property
-    @memoized
-    # There is currently no better way to do this cross platform.
-    # ARM does not have dmidecode
-    def model(self):
-        if self.file_exists("/proc/device-tree/model"):
-            raw_model = self.execute("cat /proc/device-tree/model")
-            device_model_to_return = '_'.join(raw_model.split()[:2])
-            return device_model_to_return.rstrip(' \t\r\n\0')
-        return None
-
     @property
     @memoized
     def system_id(self):
@@ -976,16 +1164,20 @@ class LinuxTarget(Target):
         else:
             return []
 
-    def ps(self, **kwargs):
+    def ps(self, threads=False, **kwargs):
-        command = 'ps -eo user,pid,ppid,vsize,rss,wchan,pcpu,state,fname'
+        ps_flags = '-eo'
+        if threads:
+            ps_flags = '-eLo'
+        command = 'ps {} user,pid,tid,ppid,vsize,rss,wchan,pcpu,state,fname'.format(ps_flags)
+
         lines = iter(convert_new_lines(self.execute(command)).split('\n'))
         next(lines)  # header
 
         result = []
         for line in lines:
-            parts = re.split(r'\s+', line, maxsplit=8)
+            parts = re.split(r'\s+', line, maxsplit=9)
             if parts and parts != ['']:
-                result.append(PsEntry(*(parts[0:1] + list(map(int, parts[1:5])) + parts[5:])))
+                result.append(PsEntry(*(parts[0:1] + list(map(int, parts[1:6])) + parts[6:])))
 
         if not kwargs:
             return result
@@ -1003,7 +1195,7 @@ class LinuxTarget(Target):
     def install(self, filepath, timeout=None, with_name=None):  # pylint: disable=W0221
         destpath = self.path.join(self.executables_directory,
                                   with_name and with_name or self.path.basename(filepath))
-        self.push(filepath, destpath)
+        self.push(filepath, destpath, timeout=timeout)
         self.execute('chmod a+x {}'.format(quote(destpath)), timeout=timeout)
         self._installed_binaries[self.path.basename(destpath)] = destpath
         return destpath
@@ -1085,7 +1277,11 @@ class AndroidTarget(Target):
 
     @property
     def adb_name(self):
-        return self.conn.device
+        return getattr(self.conn, 'device', None)
+
+    @property
+    def adb_server(self):
+        return getattr(self.conn, 'adb_server', None)
 
     @property
     @memoized
@@ -1103,14 +1299,6 @@ class AndroidTarget(Target):
         output = self.execute('content query --uri content://settings/secure --projection value --where "name=\'android_id\'"').strip()
         return output.split('value=')[-1]
 
-    @property
-    @memoized
-    def model(self):
-        try:
-            return self.getprop(prop='ro.product.device')
-        except KeyError:
-            return None
-
     @property
     @memoized
     def system_id(self):
@@ -1165,7 +1353,7 @@ class AndroidTarget(Target):
         except (DevlibTransientError, subprocess.CalledProcessError):
             # on some targets "reboot" doesn't return gracefully
             pass
-        self._connected_as_root = None
+        self.conn.connected_as_root = None
 
     def wait_boot_complete(self, timeout=10):
         start = time.time()
@@ -1180,13 +1368,6 @@ class AndroidTarget(Target):
 
     def connect(self, timeout=30, check_boot_completed=True):  # pylint: disable=arguments-differ
         device = self.connection_settings.get('device')
-        if device and ':' in device:
-            # ADB does not automatically remove a network device from it's
-            # devices list when the connection is broken by the remote, so the
-            # adb connection may have gone "stale", resulting in adb blocking
-            # indefinitely when making calls to the device. To avoid this,
-            # always disconnect first.
-            adb_disconnect(device)
         super(AndroidTarget, self).connect(timeout=timeout, check_boot_completed=check_boot_completed)
 
     def kick_off(self, command, as_root=None):
@@ -1227,7 +1408,7 @@ class AndroidTarget(Target):
         if ext == '.apk':
             return self.install_apk(filepath, timeout)
         else:
-            return self.install_executable(filepath, with_name)
+            return self.install_executable(filepath, with_name, timeout)
 
     def uninstall(self, name):
         if self.package_is_installed(name):
@@ -1243,18 +1424,32 @@ class AndroidTarget(Target):
                 result.append(entry.pid)
         return result
 
-    def ps(self, **kwargs):
+    def ps(self, threads=False, **kwargs):
-        lines = iter(convert_new_lines(self.execute('ps')).split('\n'))
+        maxsplit = 9 if threads else 8
+        command = 'ps'
+        if threads:
+            command = 'ps -AT'
+
+        lines = iter(convert_new_lines(self.execute(command)).split('\n'))
         next(lines)  # header
         result = []
         for line in lines:
-            parts = line.split(None, 8)
+            parts = line.split(None, maxsplit)
             if not parts:
                 continue
-            if len(parts) == 8:
+
+            wchan_missing = False
+            if len(parts) == maxsplit:
+                wchan_missing = True
+
+            if not threads:
+                # Duplicate PID into TID location.
+                parts.insert(2, parts[1])
+
+            if wchan_missing:
                 # wchan was blank; insert an empty field where it should be.
-                parts.insert(5, '')
+                parts.insert(6, '')
-            result.append(PsEntry(*(parts[0:1] + list(map(int, parts[1:5])) + parts[5:])))
+            result.append(PsEntry(*(parts[0:1] + list(map(int, parts[1:6])) + parts[6:])))
         if not kwargs:
             return result
         else:
@@ -1390,7 +1585,15 @@ class AndroidTarget(Target):
             if self.get_sdk_version() >= 23:
                 flags.append('-g')  # Grant all runtime permissions
             self.logger.debug("Replace APK = {}, ADB flags = '{}'".format(replace, ' '.join(flags)))
-            return adb_command(self.adb_name, "install {} {}".format(' '.join(flags), quote(filepath)), timeout=timeout)
+            if isinstance(self.conn, AdbConnection):
+                return adb_command(self.adb_name, "install {} {}".format(' '.join(flags), quote(filepath)),
+                                   timeout=timeout, adb_server=self.adb_server)
+            else:
+                dev_path = self.get_workpath(filepath.rsplit(os.path.sep, 1)[-1])
+                self.push(quote(filepath), dev_path, timeout=timeout)
+                result = self.execute("pm install {} {}".format(' '.join(flags), quote(dev_path)), timeout=timeout)
+                self.remove(dev_path)
+                return result
         else:
             raise TargetStableError('Can\'t install {}: unsupported format.'.format(filepath))
 
@@ -1437,74 +1640,94 @@ class AndroidTarget(Target):
                   '-n com.android.providers.media/.MediaScannerReceiver'
         self.execute(command.format(quote('file://'+dirpath)), as_root=as_root)
 
-    def install_executable(self, filepath, with_name=None):
+    def install_executable(self, filepath, with_name=None, timeout=None):
         self._ensure_executables_directory_is_writable()
         executable_name = with_name or os.path.basename(filepath)
         on_device_file = self.path.join(self.working_directory, executable_name)
         on_device_executable = self.path.join(self.executables_directory, executable_name)
-        self.push(filepath, on_device_file)
+        self.push(filepath, on_device_file, timeout=timeout)
         if on_device_file != on_device_executable:
-            self.execute('cp {} {}'.format(quote(on_device_file), quote(on_device_executable)), as_root=self.needs_su)
+            self.execute('cp {} {}'.format(quote(on_device_file), quote(on_device_executable)),
+                         as_root=self.needs_su, timeout=timeout)
             self.remove(on_device_file, as_root=self.needs_su)
         self.execute("chmod 0777 {}".format(quote(on_device_executable)), as_root=self.needs_su)
         self._installed_binaries[executable_name] = on_device_executable
         return on_device_executable
 
     def uninstall_package(self, package):
-        adb_command(self.adb_name, "uninstall {}".format(quote(package)), timeout=30)
+        if isinstance(self.conn, AdbConnection):
+            adb_command(self.adb_name, "uninstall {}".format(quote(package)), timeout=30,
+                        adb_server=self.adb_server)
+        else:
+            self.execute("pm uninstall {}".format(quote(package)), timeout=30)
 
     def uninstall_executable(self, executable_name):
         on_device_executable = self.path.join(self.executables_directory, executable_name)
         self._ensure_executables_directory_is_writable()
         self.remove(on_device_executable, as_root=self.needs_su)
 
-    def dump_logcat(self, filepath, filter=None, append=False, timeout=30):  # pylint: disable=redefined-builtin
+    def dump_logcat(self, filepath, filter=None, logcat_format=None, append=False,
+                    timeout=30):  # pylint: disable=redefined-builtin
         op = '>>' if append else '>'
         filtstr = ' -s {}'.format(quote(filter)) if filter else ''
-        command = 'logcat -d{} {} {}'.format(filtstr, op, quote(filepath))
+        formatstr = ' -v {}'.format(quote(logcat_format)) if logcat_format else ''
-        adb_command(self.adb_name, command, timeout=timeout)
+        logcat_opts = '-d' + formatstr + filtstr
+        if isinstance(self.conn, AdbConnection):
+            command = 'logcat {} {} {}'.format(logcat_opts, op, quote(filepath))
+            adb_command(self.adb_name, command, timeout=timeout, adb_server=self.adb_server)
+        else:
+            dev_path = self.get_workpath('logcat')
+            command = 'logcat {} {} {}'.format(logcat_opts, op, quote(dev_path))
+            self.execute(command, timeout=timeout)
+            self.pull(dev_path, filepath)
+            self.remove(dev_path)
 
     def clear_logcat(self):
         with self.clear_logcat_lock:
-            adb_command(self.adb_name, 'logcat -c', timeout=30)
+            if isinstance(self.conn, AdbConnection):
+                adb_command(self.adb_name, 'logcat -c', timeout=30, adb_server=self.adb_server)
+            else:
+                self.execute('logcat -c', timeout=30)
 
     def get_logcat_monitor(self, regexps=None):
         return LogcatMonitor(self, regexps)
 
-    def adb_kill_server(self, timeout=30):
+    def wait_for_device(self, timeout=30):
-        adb_command(self.adb_name, 'kill-server', timeout)
+        self.conn.wait_for_device()
 
-    def adb_wait_for_device(self, timeout=30):
+    def reboot_bootloader(self, timeout=30):
-        adb_command(self.adb_name, 'wait-for-device', timeout)
+        self.conn.reboot_bootloader()
-
-    def adb_reboot_bootloader(self, timeout=30):
-        adb_command(self.adb_name, 'reboot-bootloader', timeout)
-
-    def adb_root(self, enable=True, force=False):
-        if enable:
-            if self._connected_as_root and not force:
-                return
-            adb_command(self.adb_name, 'root', timeout=30)
-            self._connected_as_root = True
-            return
-        adb_command(self.adb_name, 'unroot', timeout=30)
-        self._connected_as_root = False
 
     def is_screen_on(self):
         output = self.execute('dumpsys power')
         match = ANDROID_SCREEN_STATE_REGEX.search(output)
         if match:
+            if 'DOZE' in match.group(1).upper():
+                return True
             return boolean(match.group(1))
         else:
             raise TargetStableError('Could not establish screen state.')
 
-    def ensure_screen_is_on(self):
+    def ensure_screen_is_on(self, verify=True):
         if not self.is_screen_on():
             self.execute('input keyevent 26')
+        if verify and not self.is_screen_on():
+             raise TargetStableError('Display cannot be turned on.')
 
-    def ensure_screen_is_off(self):
+    def ensure_screen_is_on_and_stays(self, verify=True, mode=7):
+        self.ensure_screen_is_on(verify=verify)
+        self.set_stay_on_mode(mode)
+
+    def ensure_screen_is_off(self, verify=True):
+        # Allow 2 attempts to help with cases of ambient display modes
+        # where the first attempt will switch the display fully on.
+        for _ in range(2):
             if self.is_screen_on():
                 self.execute('input keyevent 26')
+                time.sleep(0.5)
+        if verify and self.is_screen_on():
+             msg = 'Display cannot be turned off. Is always on display enabled?'
+             raise TargetStableError(msg)
 
     def set_auto_brightness(self, auto_brightness):
         cmd = 'settings put system screen_brightness_mode {}'
@@ -1538,6 +1761,10 @@ class AndroidTarget(Target):
         cmd = 'settings get global airplane_mode_on'
         return boolean(self.execute(cmd).strip())
 
+    def get_stay_on_mode(self):
+        cmd = 'settings get global stay_on_while_plugged_in'
+        return int(self.execute(cmd).strip())
+
     def set_airplane_mode(self, mode):
         root_required = self.get_sdk_version() > 23
         if root_required and not self.is_rooted:
@@ -1583,6 +1810,18 @@ class AndroidTarget(Target):
         cmd = 'settings put system user_rotation {}'
         self.execute(cmd.format(rotation))
 
+    def set_stay_on_never(self):
+        self.set_stay_on_mode(0)
+
+    def set_stay_on_while_powered(self):
+        self.set_stay_on_mode(7)
+
+    def set_stay_on_mode(self, mode):
+        if not 0 <= mode <= 7:
+            raise ValueError('Screen stay on mode must be between 0 and 7')
+        cmd = 'settings put global stay_on_while_plugged_in {}'
+        self.execute(cmd.format(mode))
+
     def open_url(self, url, force_new=False):
         """
         Start a view activity by specifying an URL
@@ -1654,7 +1893,7 @@ class AndroidTarget(Target):
         self.write_value(self._charging_enabled_path, int(bool(enabled)))
 
 FstabEntry = namedtuple('FstabEntry', ['device', 'mount_point', 'fs_type', 'options', 'dump_freq', 'pass_num'])
-PsEntry = namedtuple('PsEntry', 'user pid ppid vsize rss wchan pc state name')
+PsEntry = namedtuple('PsEntry', 'user pid tid ppid vsize rss wchan pc state name')
 LsmodEntry = namedtuple('LsmodEntry', ['name', 'size', 'use_count', 'used_by'])
 
 
@@ -1749,6 +1988,8 @@ class KernelVersion(object):
     :type minor: int
     :ivar rc: Release candidate number (e.g. 3 for Linux 4.9-rc3). May be None.
     :type rc: int
+    :ivar commits: Number of additional commits on the branch. May be None.
+    :type commits: int
     :ivar sha1: Kernel git revision hash, if available (otherwise None)
     :type sha1: str
 
@@ -1773,6 +2014,7 @@ class KernelVersion(object):
         self.minor = None
         self.sha1 = None
         self.rc = None
+        self.commits = None
         match = KVERSION_REGEX.match(version_string)
         if match:
             groups = match.groupdict()
@@ -1782,6 +2024,8 @@ class KernelVersion(object):
                 self.minor = int(groups['minor'])
             if groups['rc'] is not None:
                 self.rc = int(groups['rc'])
+            if groups['commits'] is not None:
+                self.commits = int(groups['commits'])
             if groups['sha1'] is not None:
                 self.sha1 = match.group('sha1')
 
@@ -2006,6 +2250,9 @@ class KernelConfig(object):
 
     This class does not provide a Mapping API and only return string values.
     """
+    @staticmethod
+    def get_config_name(name):
+        return TypedKernelConfig.get_config_name(name)
 
     def __init__(self, text):
         # Expose typed_config as a non-private attribute, so that user code
@@ -2014,7 +2261,9 @@ class KernelConfig(object):
         # Expose the original text for backward compatibility
         self.text = text
 
-    get_config_name = TypedKernelConfig.get_config_name
+    def __bool__(self):
+        return bool(self.typed_config)
+
     not_set_regex = TypedKernelConfig.not_set_regex
 
     def iteritems(self):
@@ -2154,7 +2403,10 @@ class ChromeOsTarget(LinuxTarget):
 
         # Pull out ssh connection settings
         ssh_conn_params = ['host', 'username', 'password', 'keyfile',
-                           'port', 'password_prompt', 'timeout', 'sudo_cmd']
+                           'port', 'timeout', 'sudo_cmd',
+                           'strict_host_check', 'use_scp',
+                           'total_timeout', 'poll_transfers',
+                           'start_transfer_poll_delay']
         self.ssh_connection_settings = {}
         for setting in ssh_conn_params:
             if connection_settings.get(setting, None):

devlib/trace/perf.py

@@ -1,137 +0,0 @@
-#    Copyright 2018 ARM Limited
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-
-import os
-import re
-from past.builtins import basestring, zip
-
-from devlib.host import PACKAGE_BIN_DIRECTORY
-from devlib.trace import TraceCollector
-from devlib.utils.misc import ensure_file_directory_exists as _f
-
-
-PERF_COMMAND_TEMPLATE = '{} stat {} {} sleep 1000 > {} 2>&1 '
-
-PERF_COUNT_REGEX = re.compile(r'^(CPU\d+)?\s*(\d+)\s*(.*?)\s*(\[\s*\d+\.\d+%\s*\])?\s*$')
-
-DEFAULT_EVENTS = [
-    'migrations',
-    'cs',
-]
-
-
-class PerfCollector(TraceCollector):
-    """
-    Perf is a Linux profiling with performance counters.
-
-    Performance counters are CPU hardware registers that count hardware events
-    such as instructions executed, cache-misses suffered, or branches
-    mispredicted. They form a basis for profiling applications to trace dynamic
-    control flow and identify hotspots.
-
-    pref accepts options and events. If no option is given the default '-a' is
-    used. For events, the default events are migrations and cs. They both can
-    be specified in the config file.
-
-    Events must be provided as a list that contains them and they will look like
-    this ::
-
-        perf_events = ['migrations', 'cs']
-
-    Events can be obtained by typing the following in the command line on the
-    device ::
-
-        perf list
-
-    Whereas options, they can be provided as a single string as following ::
-
-        perf_options = '-a -i'
-
-    Options can be obtained by running the following in the command line ::
-
-        man perf-stat
-    """
-
-    def __init__(self, target,
-                 events=None,
-                 optionstring=None,
-                 labels=None,
-                 force_install=False):
-        super(PerfCollector, self).__init__(target)
-        self.events = events if events else DEFAULT_EVENTS
-        self.force_install = force_install
-        self.labels = labels
-
-        # Validate parameters
-        if isinstance(optionstring, list):
-            self.optionstrings = optionstring
-        else:
-            self.optionstrings = [optionstring]
-        if self.events and isinstance(self.events, basestring):
-            self.events = [self.events]
-        if not self.labels:
-            self.labels = ['perf_{}'.format(i) for i in range(len(self.optionstrings))]
-        if len(self.labels) != len(self.optionstrings):
-            raise ValueError('The number of labels must match the number of optstrings provided for perf.')
-
-        self.binary = self.target.get_installed('perf')
-        if self.force_install or not self.binary:
-            self.binary = self._deploy_perf()
-
-        self.commands = self._build_commands()
-
-    def reset(self):
-        self.target.killall('perf', as_root=self.target.is_rooted)
-        for label in self.labels:
-            filepath = self._get_target_outfile(label)
-            self.target.remove(filepath)
-
-    def start(self):
-        for command in self.commands:
-            self.target.kick_off(command)
-
-    def stop(self):
-        self.target.killall('sleep', as_root=self.target.is_rooted)
-
-    # pylint: disable=arguments-differ
-    def get_trace(self, outdir):
-        for label in self.labels:
-            target_file = self._get_target_outfile(label)
-            host_relpath = os.path.basename(target_file)
-            host_file = _f(os.path.join(outdir, host_relpath))
-            self.target.pull(target_file, host_file)
-
-    def _deploy_perf(self):
-        host_executable = os.path.join(PACKAGE_BIN_DIRECTORY,
-                                       self.target.abi, 'perf')
-        return self.target.install(host_executable)
-
-    def _build_commands(self):
-        commands = []
-        for opts, label in zip(self.optionstrings, self.labels):
-            commands.append(self._build_perf_command(opts, self.events, label))
-        return commands
-
-    def _get_target_outfile(self, label):
-        return self.target.get_workpath('{}.out'.format(label))
-
-    def _build_perf_command(self, options, events, label):
-        event_string = ' '.join(['-e {}'.format(e) for e in events])
-        command = PERF_COMMAND_TEMPLATE.format(self.binary,
-                                               options or '',
-                                               event_string,
-                                               self._get_target_outfile(label))
-        return command

devlib/utils/android.py

@@ -19,6 +19,7 @@ Utility functions for working with Android devices through adb.
 
 """
 # pylint: disable=E1103
+import glob
 import os
 import re
 import sys
@@ -30,22 +31,29 @@ from collections import defaultdict
 import pexpect
 import xml.etree.ElementTree
 import zipfile
+import uuid
 
-from pipes import quote
+try:
+    from shlex import quote
+except ImportError:
+    from pipes import quote
 
 from devlib.exception import TargetTransientError, TargetStableError, HostError
-from devlib.utils.misc import check_output, which, ABI_MAP
+from devlib.utils.misc import check_output, which, ABI_MAP, redirect_streams, get_subprocess
+from devlib.connection import ConnectionBase, AdbBackgroundCommand, PopenBackgroundCommand, PopenTransferManager
 
 
 logger = logging.getLogger('android')
 
 MAX_ATTEMPTS = 5
 AM_START_ERROR = re.compile(r"Error: Activity.*")
+AAPT_BADGING_OUTPUT = re.compile(r"no dump ((file)|(apk)) specified", re.IGNORECASE)
 
 # See:
 # http://developer.android.com/guide/topics/manifest/uses-sdk-element.html#ApiLevels
 ANDROID_VERSION_MAP = {
-    28: 'P',
+    29: 'Q',
+    28: 'PIE',
     27: 'OREO_MR1',
     26: 'OREO',
     25: 'NOUGAT_MR1',
@@ -87,6 +95,7 @@ android_home = None
 platform_tools = None
 adb = None
 aapt = None
+aapt_version = None
 fastboot = None
 
 
@@ -146,6 +155,10 @@ class ApkInfo(object):
         self.version_code = None
         self.native_code = None
         self.permissions = []
+        self._apk_path = None
+        self._activities = None
+        self._methods = None
+        if path:
             self.parse(path)
 
     # pylint: disable=too-many-branches
@@ -191,8 +204,10 @@ class ApkInfo(object):
     @property
     def activities(self):
         if self._activities is None:
-            cmd = [aapt, 'dump', 'xmltree', self._apk_path,
+            cmd = [aapt, 'dump', 'xmltree', self._apk_path]
-                   'AndroidManifest.xml']
+            if aapt_version == 2:
+                cmd += ['--file']
+            cmd += ['AndroidManifest.xml']
             matched_activities = self.activity_regex.finditer(self._run(cmd))
             self._activities = [m.group('name') for m in matched_activities]
         return self._activities
@@ -200,21 +215,26 @@ class ApkInfo(object):
     @property
     def methods(self):
         if self._methods is None:
+            # Only try to extract once
+            self._methods = []
+            with tempfile.TemporaryDirectory() as tmp_dir:
                 with zipfile.ZipFile(self._apk_path, 'r') as z:
-                extracted = z.extract('classes.dex', tempfile.gettempdir())
+                    try:
-
+                        extracted = z.extract('classes.dex', tmp_dir)
+                    except KeyError:
+                        return []
                 dexdump = os.path.join(os.path.dirname(aapt), 'dexdump')
                 command = [dexdump, '-l', 'xml', extracted]
                 dump = self._run(command)
 
             xml_tree = xml.etree.ElementTree.fromstring(dump)
 
-            package = next(i for i in xml_tree.iter('package')
+            package = next((i for i in xml_tree.iter('package')
-                           if i.attrib['name'] == self.package)
+                           if i.attrib['name'] == self.package), None)
 
             self._methods = [(meth.attrib['name'], klass.attrib['name'])
                              for klass in package.iter('class')
-                             for meth in klass.iter('method')]
+                             for meth in klass.iter('method')] if package else []
         return self._methods
 
     def _run(self, command):
@@ -229,18 +249,135 @@ class ApkInfo(object):
         return output
 
 
-class AdbConnection(object):
+class AdbConnection(ConnectionBase):
 
     # maintains the count of parallel active connections to a device, so that
     # adb disconnect is not invoked untill all connections are closed
     active_connections = defaultdict(int)
+    # Track connected as root status per device
+    _connected_as_root = defaultdict(lambda: None)
     default_timeout = 10
     ls_command = 'ls'
+    su_cmd = 'su -c {}'
 
     @property
     def name(self):
         return self.device
 
+    @property
+    def connected_as_root(self):
+        if self._connected_as_root[self.device] is None:
+            result = self.execute('id')
+            self._connected_as_root[self.device] = 'uid=0(' in result
+        return self._connected_as_root[self.device]
+
+    @connected_as_root.setter
+    def connected_as_root(self, state):
+        self._connected_as_root[self.device] = state
+
+    # pylint: disable=unused-argument
+    def __init__(self, device=None, timeout=None, platform=None, adb_server=None,
+                 adb_as_root=False, connection_attempts=MAX_ATTEMPTS,
+                 poll_transfers=False,
+                 start_transfer_poll_delay=30,
+                 total_transfer_timeout=3600,
+                 transfer_poll_period=30,):
+        super().__init__()
+        self.timeout = timeout if timeout is not None else self.default_timeout
+        if device is None:
+            device = adb_get_device(timeout=timeout, adb_server=adb_server)
+        self.device = device
+        self.adb_server = adb_server
+        self.adb_as_root = adb_as_root
+        self.poll_transfers = poll_transfers
+        if poll_transfers:
+            transfer_opts = {'start_transfer_poll_delay': start_transfer_poll_delay,
+                            'total_timeout': total_transfer_timeout,
+                            'poll_period': transfer_poll_period,
+                            }
+        self.transfer_mgr = PopenTransferManager(self, **transfer_opts) if poll_transfers else None
+        if self.adb_as_root:
+            self.adb_root(enable=True)
+        adb_connect(self.device, adb_server=self.adb_server, attempts=connection_attempts)
+        AdbConnection.active_connections[self.device] += 1
+        self._setup_ls()
+        self._setup_su()
+
+    def push(self, sources, dest, timeout=None):
+        return self._push_pull('push', sources, dest, timeout)
+
+    def pull(self, sources, dest, timeout=None):
+        return self._push_pull('pull', sources, dest, timeout)
+
+    def _push_pull(self, action, sources, dest, timeout):
+        paths = sources + [dest]
+
+        # Quote twice to avoid expansion by host shell, then ADB globbing
+        do_quote = lambda x: quote(glob.escape(x))
+        paths = ' '.join(map(do_quote, paths))
+
+        command = "{} {}".format(action, paths)
+        if timeout or not self.poll_transfers:
+            adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
+        else:
+            with self.transfer_mgr.manage(sources, dest, action):
+                bg_cmd = adb_command_background(self.device, command, adb_server=self.adb_server)
+                self.transfer_mgr.set_transfer_and_wait(bg_cmd)
+
+    # pylint: disable=unused-argument
+    def execute(self, command, timeout=None, check_exit_code=False,
+                as_root=False, strip_colors=True, will_succeed=False):
+        try:
+            return adb_shell(self.device, command, timeout, check_exit_code,
+                             as_root, adb_server=self.adb_server, su_cmd=self.su_cmd)
+        except TargetStableError as e:
+            if will_succeed:
+                raise TargetTransientError(e)
+            else:
+                raise
+
+    def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
+        bg_cmd = self._background(command, stdout, stderr, as_root)
+        self._current_bg_cmds.add(bg_cmd)
+        return bg_cmd
+
+    def _background(self, command, stdout, stderr, as_root):
+        adb_shell, pid = adb_background_shell(self, command, stdout, stderr, as_root)
+        bg_cmd = AdbBackgroundCommand(
+            conn=self,
+            adb_popen=adb_shell,
+            pid=pid,
+            as_root=as_root
+        )
+        return bg_cmd
+
+    def _close(self):
+        AdbConnection.active_connections[self.device] -= 1
+        if AdbConnection.active_connections[self.device] <= 0:
+            if self.adb_as_root:
+                self.adb_root(enable=False)
+            adb_disconnect(self.device, self.adb_server)
+            del AdbConnection.active_connections[self.device]
+
+    def cancel_running_command(self):
+        # adbd multiplexes commands so that they don't interfer with each
+        # other, so there is no need to explicitly cancel a running command
+        # before the next one can be issued.
+        pass
+
+    def adb_root(self, enable=True):
+        cmd = 'root' if enable else 'unroot'
+        output = adb_command(self.device, cmd, timeout=30, adb_server=self.adb_server)
+        if 'cannot run as root in production builds' in output:
+            raise TargetStableError(output)
+        AdbConnection._connected_as_root[self.device] = enable
+
+    def wait_for_device(self, timeout=30):
+        adb_command(self.device, 'wait-for-device', timeout, self.adb_server)
+
+    def reboot_bootloader(self, timeout=30):
+        adb_command(self.device, 'reboot-bootloader', timeout, self.adb_server)
+
     # Again, we need to handle boards where the default output format from ls is
     # single column *and* boards where the default output is multi-column.
     # We need to do this purely because the '-1' option causes errors on older
@@ -261,66 +398,16 @@ class AdbConnection(object):
             self.ls_command = 'ls'
         logger.debug("ls command is set to {}".format(self.ls_command))
 
-    # pylint: disable=unused-argument
+    def _setup_su(self):
-    def __init__(self, device=None, timeout=None, platform=None, adb_server=None):
-        self.timeout = timeout if timeout is not None else self.default_timeout
-        if device is None:
-            device = adb_get_device(timeout=timeout, adb_server=adb_server)
-        self.device = device
-        self.adb_server = adb_server
-        adb_connect(self.device)
-        AdbConnection.active_connections[self.device] += 1
-        self._setup_ls()
-
-    def push(self, source, dest, timeout=None):
-        if timeout is None:
-            timeout = self.timeout
-        command = "push {} {}".format(quote(source), quote(dest))
-        if not os.path.exists(source):
-            raise HostError('No such file "{}"'.format(source))
-        return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
-
-    def pull(self, source, dest, timeout=None):
-        if timeout is None:
-            timeout = self.timeout
-        # Pull all files matching a wildcard expression
-        if os.path.isdir(dest) and \
-           ('*' in source or '?' in source):
-            command = 'shell {} {}'.format(self.ls_command, source)
-            output = adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
-            for line in output.splitlines():
-                command = "pull {} {}".format(quote(line.strip()), quote(dest))
-                adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
-            return
-        command = "pull {} {}".format(quote(source), quote(dest))
-        return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
-
-    # pylint: disable=unused-argument
-    def execute(self, command, timeout=None, check_exit_code=False,
-                as_root=False, strip_colors=True, will_succeed=False):
         try:
-            return adb_shell(self.device, command, timeout, check_exit_code,
+            # Try the new style of invoking `su`
-                             as_root, adb_server=self.adb_server)
+            self.execute('ls', timeout=self.timeout, as_root=True,
-        except TargetStableError as e:
+                         check_exit_code=True)
-            if will_succeed:
+        # If failure assume either old style or unrooted. Here we will assume
-                raise TargetTransientError(e)
+        # old style and root status will be verified later.
-            else:
+        except (TargetStableError, TargetTransientError, TimeoutError):
-                raise
+            self.su_cmd = 'echo {} | su'
-
+        logger.debug("su command is set to {}".format(quote(self.su_cmd)))
-    def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
-        return adb_background_shell(self.device, command, stdout, stderr, as_root, adb_server=self.adb_server)
-
-    def close(self):
-        AdbConnection.active_connections[self.device] -= 1
-        if AdbConnection.active_connections[self.device] <= 0:
-            adb_disconnect(self.device)
-            del AdbConnection.active_connections[self.device]
-
-    def cancel_running_command(self):
-        # adbd multiplexes commands so that they don't interfer with each
-        # other, so there is no need to explicitly cancel a running command
-        # before the next one can be issued.
-        pass
 
 
 def fastboot_command(command, timeout=None, device=None):
@@ -373,7 +460,7 @@ def adb_get_device(timeout=None, adb_server=None):
             time.sleep(1)
 
 
-def adb_connect(device, timeout=None, attempts=MAX_ATTEMPTS):
+def adb_connect(device, timeout=None, attempts=MAX_ATTEMPTS, adb_server=None):
     _check_env()
     tries = 0
     output = None
@@ -381,10 +468,17 @@ def adb_connect(device, timeout=None, attempts=MAX_ATTEMPTS):
         tries += 1
         if device:
             if "." in device: # Connect is required only for ADB-over-IP
-                command = 'adb connect {}'.format(quote(device))
+                # ADB does not automatically remove a network device from it's
+                # devices list when the connection is broken by the remote, so the
+                # adb connection may have gone "stale", resulting in adb blocking
+                # indefinitely when making calls to the device. To avoid this,
+                # always disconnect first.
+                adb_disconnect(device, adb_server)
+                adb_cmd = get_adb_command(None, 'connect', adb_server)
+                command = '{} {}'.format(adb_cmd, quote(device))
                 logger.debug(command)
                 output, _ = check_output(command, shell=True, timeout=timeout)
-        if _ping(device):
+        if _ping(device, adb_server):
             break
         time.sleep(10)
     else:  # did not connect to the device
@@ -394,22 +488,23 @@ def adb_connect(device, timeout=None, attempts=MAX_ATTEMPTS):
         raise HostError(message)
 
 
-def adb_disconnect(device):
+def adb_disconnect(device, adb_server=None):
     _check_env()
     if not device:
         return
-    if ":" in device and device in adb_list_devices():
+    if ":" in device and device in adb_list_devices(adb_server):
-        command = "adb disconnect " + device
+        adb_cmd = get_adb_command(None, 'disconnect', adb_server)
+        command = "{} {}".format(adb_cmd, device)
         logger.debug(command)
         retval = subprocess.call(command, stdout=open(os.devnull, 'wb'), shell=True)
         if retval:
             raise TargetTransientError('"{}" returned {}'.format(command, retval))
 
 
-def _ping(device):
+def _ping(device, adb_server=None):
     _check_env()
-    device_string = ' -s {}'.format(quote(device)) if device else ''
+    adb_cmd = get_adb_command(device, 'shell', adb_server)
-    command = "adb{} shell \"ls /data/local/tmp > /dev/null\"".format(device_string)
+    command = "{} {}".format(adb_cmd, quote('ls /data/local/tmp > /dev/null'))
     logger.debug(command)
     result = subprocess.call(command, stderr=subprocess.PIPE, shell=True)
     if not result:  # pylint: disable=simplifiable-if-statement
@@ -420,25 +515,27 @@ def _ping(device):
 
 # pylint: disable=too-many-locals
 def adb_shell(device, command, timeout=None, check_exit_code=False,
-              as_root=False, adb_server=None):  # NOQA
+              as_root=False, adb_server=None, su_cmd='su -c {}'):  # NOQA
     _check_env()
-    if as_root:
-        command = 'echo {} | su'.format(quote(command))
-    device_part = []
-    if adb_server:
-        device_part = ['-H', adb_server]
-    device_part += ['-s', device] if device else []
 
     # On older combinations of ADB/Android versions, the adb host command always
     # exits with 0 if it was able to run the command on the target, even if the
     # command failed (https://code.google.com/p/android/issues/detail?id=3254).
     # Homogenise this behaviour by running the command then echoing the exit
-    # code.
+    # code of the executed command itself.
-    adb_shell_command = '({}); echo \"\n$?\"'.format(command)
+    command = r'({}); echo "\n$?"'.format(command)
-    actual_command = ['adb'] + device_part + ['shell', adb_shell_command]
+
-    logger.debug('adb {} shell {}'.format(' '.join(device_part), command))
+    parts = ['adb']
+    if adb_server is not None:
+        parts += ['-H', adb_server]
+    if device is not None:
+        parts += ['-s', device]
+    parts += ['shell',
+              command if not as_root else su_cmd.format(quote(command))]
+
+    logger.debug(' '.join(quote(part) for part in parts))
     try:
-        raw_output, _ = check_output(actual_command, timeout, shell=False, combined_output=True)
+        raw_output, error = check_output(parts, timeout, shell=False)
     except subprocess.CalledProcessError as e:
         raise TargetStableError(str(e))
 
@@ -458,8 +555,8 @@ def adb_shell(device, command, timeout=None, check_exit_code=False,
         if exit_code.isdigit():
             if int(exit_code):
                 message = ('Got exit code {}\nfrom target command: {}\n'
-                           'OUTPUT: {}')
+                           'OUTPUT: {}\nSTDERR: {}\n')
-                raise TargetStableError(message.format(exit_code, command, output))
+                raise TargetStableError(message.format(exit_code, command, output, error))
             elif re_search:
                 message = 'Could not start activity; got the following:\n{}'
                 raise TargetStableError(message.format(re_search[0]))
@@ -470,28 +567,50 @@ def adb_shell(device, command, timeout=None, check_exit_code=False,
             else:
                 message = 'adb has returned early; did not get an exit code. '\
                           'Was kill-server invoked?\nOUTPUT:\n-----\n{}\n'\
-                          '-----'
+                          '-----\nSTDERR:\n-----\n{}\n-----'
-                raise TargetTransientError(message.format(raw_output))
+                raise TargetTransientError(message.format(raw_output, error))
 
-    return output
+    return output + error
 
 
-def adb_background_shell(device, command,
+def adb_background_shell(conn, command,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE,
-                         as_root=False,
+                         as_root=False):
-                         adb_server=None):
     """Runs the sepcified command in a subprocess, returning the the Popen object."""
+    device = conn.device
+    adb_server = conn.adb_server
+
     _check_env()
+    stdout, stderr, command = redirect_streams(stdout, stderr, command)
     if as_root:
         command = 'echo {} | su'.format(quote(command))
 
-    device_string = ' -H {}'.format(adb_server) if adb_server else ''
+    # Attach a unique UUID to the command line so it can be looked for without
-    device_string += ' -s {}'.format(device) if device else ''
+    # any ambiguity with ps
-    full_command = 'adb{} shell {}'.format(device_string, quote(command))
+    uuid_ = uuid.uuid4().hex
-    logger.debug(full_command)
+    uuid_var = 'BACKGROUND_COMMAND_UUID={}'.format(uuid_)
-    return subprocess.Popen(full_command, stdout=stdout, stderr=stderr, shell=True)
+    command = "{} sh -c {}".format(uuid_var, quote(command))
 
+    adb_cmd = get_adb_command(device, 'shell', adb_server)
+    full_command = '{} {}'.format(adb_cmd, quote(command))
+    logger.debug(full_command)
+    p = subprocess.Popen(full_command, stdout=stdout, stderr=stderr, shell=True)
+
+    # Out of band PID lookup, to avoid conflicting needs with stdout redirection
+    find_pid = 'ps -A -o pid,args | grep {}'.format(quote(uuid_var))
+    ps_out = conn.execute(find_pid)
+    pids = [
+        int(line.strip().split(' ', 1)[0])
+        for line in ps_out.splitlines()
+    ]
+    # The line we are looking for is the first one, since it was started before
+    # any look up command
+    pid = sorted(pids)[0]
+    return (p, pid)
+
+def adb_kill_server(timeout=30, adb_server=None):
+    adb_command(None, 'kill-server', timeout, adb_server)
 
 def adb_list_devices(adb_server=None):
     output = adb_command(None, 'devices', adb_server=adb_server)
@@ -511,12 +630,22 @@ def get_adb_command(device, command, adb_server=None):
     device_string += ' -s {}'.format(device) if device else ''
     return "adb{} {}".format(device_string, command)
 
+
 def adb_command(device, command, timeout=None, adb_server=None):
     full_command = get_adb_command(device, command, adb_server)
     logger.debug(full_command)
     output, _ = check_output(full_command, timeout, shell=True)
     return output
 
+
+def adb_command_background(device, command, adb_server=None):
+    full_command = get_adb_command(device, command, adb_server)
+    logger.debug(full_command)
+    proc = get_subprocess(full_command, shell=True)
+    cmd = PopenBackgroundCommand(proc)
+    return cmd
+
+
 def grant_app_permissions(target, package):
     """
     Grant an app all the permissions it may ask for
@@ -524,7 +653,7 @@ def grant_app_permissions(target, package):
     dumpsys = target.execute('dumpsys package {}'.format(package))
 
     permissions = re.search(
-        'requested permissions:\s*(?P<permissions>(android.permission.+\s*)+)', dumpsys
+        r'requested permissions:\s*(?P<permissions>(android.permission.+\s*)+)', dumpsys
     )
     if permissions is None:
         return
@@ -544,8 +673,10 @@ class _AndroidEnvironment(object):
     def __init__(self):
         self.android_home = None
         self.platform_tools = None
+        self.build_tools = None
         self.adb = None
         self.aapt = None
+        self.aapt_version = None
         self.fastboot = None
 
 
@@ -571,28 +702,73 @@ def _initialize_without_android_home(env):
     _init_common(env)
     return env
 
-
 def _init_common(env):
+    _discover_build_tools(env)
+    _discover_aapt(env)
+
+def _discover_build_tools(env):
     logger.debug('ANDROID_HOME: {}'.format(env.android_home))
     build_tools_directory = os.path.join(env.android_home, 'build-tools')
-    if not os.path.isdir(build_tools_directory):
+    if os.path.isdir(build_tools_directory):
-        msg = '''ANDROID_HOME ({}) does not appear to have valid Android SDK install
+        env.build_tools = build_tools_directory
-                 (cannot find build-tools)'''
-        raise HostError(msg.format(env.android_home))
-    versions = os.listdir(build_tools_directory)
-    for version in reversed(sorted(versions)):
-        aapt_path = os.path.join(build_tools_directory, version, 'aapt')
-        if os.path.isfile(aapt_path):
-            logger.debug('Using aapt for version {}'.format(version))
-            env.aapt = aapt_path
-            break
-    else:
-        raise HostError('aapt not found. Please make sure at least one Android '
-                        'platform is installed.')
 
+def _check_supported_aapt2(binary):
+    # At time of writing the version argument of aapt2 is not helpful as
+    # the output is only a placeholder that does not distinguish between versions
+    # with and without support for badging. Unfortunately aapt has been
+    # deprecated and fails to parse some valid apks so we will try to favour
+    # aapt2 if possible else will fall back to aapt.
+    # Try to execute the badging command and check if we get an expected error
+    # message as opposed to an unknown command error to determine if we have a
+    # suitable version.
+    cmd = '{} dump badging'.format(binary)
+    result = subprocess.run(cmd.encode('utf-8'), shell=True, stderr=subprocess.PIPE)
+    supported = bool(AAPT_BADGING_OUTPUT.search(result.stderr.decode('utf-8')))
+    msg = 'Found a {} aapt2 binary at: {}'
+    logger.debug(msg.format('supported' if supported else 'unsupported', binary))
+    return supported
+
+def _discover_aapt(env):
+    if env.build_tools:
+        aapt_path = ''
+        aapt2_path = ''
+        versions = os.listdir(env.build_tools)
+        for version in reversed(sorted(versions)):
+            if not aapt2_path and not os.path.isfile(aapt2_path):
+                aapt2_path = os.path.join(env.build_tools, version, 'aapt2')
+            if not aapt_path and not os.path.isfile(aapt_path):
+                aapt_path = os.path.join(env.build_tools, version, 'aapt')
+                aapt_version = 1
+                break
+
+        # Use aapt2 only if present and we have a suitable version
+        if aapt2_path and _check_supported_aapt2(aapt2_path):
+            aapt_path = aapt2_path
+            aapt_version = 2
+
+        # Use the aapt version discoverted from build tools.
+        if aapt_path:
+            logger.debug('Using {} for version {}'.format(aapt_path, version))
+            env.aapt = aapt_path
+            env.aapt_version = aapt_version
+            return
+
+    # Try detecting aapt2 and aapt from PATH
+    if not env.aapt:
+            aapt2_path = which('aapt2')
+            if _check_supported_aapt2(aapt2_path):
+                env.aapt = aapt2_path
+                env.aapt_version = 2
+            else:
+                env.aapt = which('aapt')
+                env.aapt_version = 1
+
+    if not env.aapt:
+        raise HostError('aapt/aapt2 not found. Please make sure it is avaliable in PATH'
+                        ' or at least one Android platform is installed')
 
 def _check_env():
-    global android_home, platform_tools, adb, aapt  # pylint: disable=W0603
+    global android_home, platform_tools, adb, aapt, aapt_version  # pylint: disable=W0603
     if not android_home:
         android_home = os.getenv('ANDROID_HOME')
         if android_home:
@@ -603,6 +779,7 @@ def _check_env():
         platform_tools = _env.platform_tools
         adb = _env.adb
         aapt = _env.aapt
+        aapt_version = _env.aapt_version
 
 class LogcatMonitor(object):
     """
@@ -621,11 +798,12 @@ class LogcatMonitor(object):
     def logfile(self):
         return self._logfile
 
-    def __init__(self, target, regexps=None):
+    def __init__(self, target, regexps=None, logcat_format=None):
         super(LogcatMonitor, self).__init__()
 
         self.target = target
         self._regexps = regexps
+        self._logcat_format = logcat_format
         self._logcat = None
         self._logfile = None
 
@@ -639,7 +817,7 @@ class LogcatMonitor(object):
         if outfile:
             self._logfile = open(outfile, 'w')
         else:
-            self._logfile = tempfile.NamedTemporaryFile()
+            self._logfile = tempfile.NamedTemporaryFile(mode='w')
 
         self.target.clear_logcat()
 
@@ -657,12 +835,16 @@ class LogcatMonitor(object):
             else:
                 logcat_cmd = '{} | grep {}'.format(logcat_cmd, quote(regexp))
 
-        logcat_cmd = get_adb_command(self.target.conn.device, logcat_cmd)
+        if self._logcat_format:
+            logcat_cmd = "{} -v {}".format(logcat_cmd, quote(self._logcat_format))
+
+        logcat_cmd = get_adb_command(self.target.conn.device, logcat_cmd, self.target.adb_server)
 
         logger.debug('logcat command ="{}"'.format(logcat_cmd))
-        self._logcat = pexpect.spawn(logcat_cmd, logfile=self._logfile)
+        self._logcat = pexpect.spawn(logcat_cmd, logfile=self._logfile, encoding='utf-8')
 
     def stop(self):
+        self.flush_log()
         self._logcat.terminate()
         self._logfile.close()
 
@@ -670,6 +852,12 @@ class LogcatMonitor(object):
         """
         Return the list of lines found by the monitor
         """
+        self.flush_log()
+
+        with open(self._logfile.name) as fh:
+            return [line for line in fh]
+
+    def flush_log(self):
         # Unless we tell pexect to 'expect' something, it won't read from
         # logcat's buffer or write into our logfile. We'll need to force it to
         # read any pending logcat output.
@@ -700,9 +888,6 @@ class LogcatMonitor(object):
                 # printed anything since pexpect last read from its buffer.
                 break
 
-        with open(self._logfile.name) as fh:
-            return [line for line in fh]
-
     def clear_log(self):
         with open(self._logfile.name, 'w') as _:
             pass

devlib/utils/gem5.py

@@ -18,7 +18,7 @@ import logging
 from devlib.utils.types import numeric
 
 
-GEM5STATS_FIELD_REGEX = re.compile("^(?P<key>[^- ]\S*) +(?P<value>[^#]+).+$")
+GEM5STATS_FIELD_REGEX = re.compile(r"^(?P<key>[^- ]\S*) +(?P<value>[^#]+).+$")
 GEM5STATS_DUMP_HEAD = '---------- Begin Simulation Statistics ----------'
 GEM5STATS_DUMP_TAIL = '---------- End Simulation Statistics   ----------'
 GEM5STATS_ROI_NUMBER = 8

devlib/utils/misc.py

@@ -19,11 +19,14 @@ Miscellaneous functions that don't fit anywhere else.
 
 """
 from __future__ import division
-from functools import partial, reduce
+from contextlib import contextmanager
+from functools import partial, reduce, wraps
 from itertools import groupby
 from operator import itemgetter
+from weakref import WeakKeyDictionary, WeakSet
 
 import ctypes
+import functools
 import logging
 import os
 import pkgutil
@@ -38,6 +41,16 @@ import wrapt
 import warnings
 
 
+try:
+    from contextlib import ExitStack
+except AttributeError:
+    from contextlib2 import ExitStack
+
+try:
+    from shlex import quote
+except ImportError:
+    from pipes import quote
+
 from past.builtins import basestring
 
 # pylint: disable=redefined-builtin
@@ -129,9 +142,6 @@ def get_cpu_name(implementer, part, variant):
 
 
 def preexec_function():
-    # Ignore the SIGINT signal by setting the handler to the standard
-    # signal handler SIG_IGN.
-    signal.signal(signal.SIGINT, signal.SIG_IGN)
     # Change process group in case we have to kill the subprocess and all of
     # its children later.
     # TODO: this is Unix-specific; would be good to find an OS-agnostic way
@@ -145,10 +155,22 @@ check_output_logger = logging.getLogger('check_output')
 check_output_lock = threading.Lock()
 
 
-def check_output(command, timeout=None, ignore=None, inputtext=None,
+def get_subprocess(command, **kwargs):
-                 combined_output=False, **kwargs):
+    if 'stdout' in kwargs:
-    """This is a version of subprocess.check_output that adds a timeout parameter to kill
+        raise ValueError('stdout argument not allowed, it will be overridden.')
-    the subprocess if it does not return within the specified time."""
+    with check_output_lock:
+        process = subprocess.Popen(command,
+                                   stdout=subprocess.PIPE,
+                                   stderr=subprocess.PIPE,
+                                   stdin=subprocess.PIPE,
+                                   preexec_fn=preexec_function,
+                                   **kwargs)
+    return process
+
+
+def check_subprocess_output(process, timeout=None, ignore=None, inputtext=None):
+    output = None
+    error = None
     # pylint: disable=too-many-branches
     if ignore is None:
         ignore = []
@@ -157,49 +179,35 @@ def check_output(command, timeout=None, ignore=None, inputtext=None,
     elif not isinstance(ignore, list) and ignore != 'all':
         message = 'Invalid value for ignore parameter: "{}"; must be an int or a list'
         raise ValueError(message.format(ignore))
-    if 'stdout' in kwargs:
-        raise ValueError('stdout argument not allowed, it will be overridden.')
-
-    def callback(pid):
-        try:
-            check_output_logger.debug('{} timed out; sending SIGKILL'.format(pid))
-            os.killpg(pid, signal.SIGKILL)
-        except OSError:
-            pass  # process may have already terminated.
-
-    with check_output_lock:
-        stderr = subprocess.STDOUT if combined_output else subprocess.PIPE
-        process = subprocess.Popen(command,
-                                   stdout=subprocess.PIPE,
-                                   stderr=stderr,
-                                   stdin=subprocess.PIPE,
-                                   preexec_fn=preexec_function,
-                                   **kwargs)
-
-    if timeout:
-        timer = threading.Timer(timeout, callback, [process.pid, ])
-        timer.start()
 
     try:
-        output, error = process.communicate(inputtext)
+        output, error = process.communicate(inputtext, timeout=timeout)
-        if sys.version_info[0] == 3:
+    except subprocess.TimeoutExpired as e:
+        timeout_expired = e
+    else:
+        timeout_expired = None
+
     # Currently errors=replace is needed as 0x8c throws an error
-            output = output.decode(sys.stdout.encoding or 'utf-8', "replace")
+    output = output.decode(sys.stdout.encoding or 'utf-8', "replace") if output else ''
-            if error:
+    error = error.decode(sys.stderr.encoding or 'utf-8', "replace") if error else ''
-                error = error.decode(sys.stderr.encoding or 'utf-8', "replace")
+
-    finally:
+    if timeout_expired:
-        if timeout:
+        raise TimeoutError(process.args, output='\n'.join([output, error]))
-            timer.cancel()
 
     retcode = process.poll()
-    if retcode:
+    if retcode and ignore != 'all' and retcode not in ignore:
-        if retcode == -9:  # killed, assume due to timeout callback
+        raise subprocess.CalledProcessError(retcode, process.args, output='\n'.join([output, error]))
-            raise TimeoutError(command, output='\n'.join([output or '', error or '']))
+
-        elif ignore != 'all' and retcode not in ignore:
-            raise subprocess.CalledProcessError(retcode, command, output='\n'.join([output or '', error or '']))
     return output, error
 
 
+def check_output(command, timeout=None, ignore=None, inputtext=None, **kwargs):
+    """This is a version of subprocess.check_output that adds a timeout parameter to kill
+    the subprocess if it does not return within the specified time."""
+    process = get_subprocess(command, **kwargs)
+    return check_subprocess_output(process, timeout=timeout, ignore=ignore, inputtext=inputtext)
+
+
 def walk_modules(path):
     """
     Given package name, return a list of all modules (including submodules, etc)
@@ -237,6 +245,32 @@ def walk_modules(path):
             mods.append(submod)
     return mods
 
+def redirect_streams(stdout, stderr, command):
+    """
+    Update a command to redirect a given stream to /dev/null if it's
+    ``subprocess.DEVNULL``.
+
+    :return: A tuple (stdout, stderr, command) with stream set to ``subprocess.PIPE``
+        if the `stream` parameter was set to ``subprocess.DEVNULL``.
+    """
+    def redirect(stream, redirection):
+        if stream == subprocess.DEVNULL:
+            suffix = '{}/dev/null'.format(redirection)
+        elif stream == subprocess.STDOUT:
+            suffix = '{}&1'.format(redirection)
+            # Indicate that there is nothing to monitor for stderr anymore
+            # since it's merged into stdout
+            stream = subprocess.DEVNULL
+        else:
+            suffix = ''
+
+        return (stream, suffix)
+
+    stdout, suffix1 = redirect(stdout, '>')
+    stderr, suffix2 = redirect(stderr, '2>')
+
+    command = 'sh -c {} {} {}'.format(quote(command), suffix1, suffix2)
+    return (stdout, stderr, command)
 
 def ensure_directory_exists(dirpath):
     """A filter for directory paths to ensure they exist."""
@@ -461,7 +495,7 @@ def escape_spaces(text):
 
     .. note:: :func:`pipes.quote` should be favored where possible.
     """
-    return text.replace(' ', '\ ')
+    return text.replace(' ', '\\ ')
 
 
 def getch(count=1):
@@ -695,3 +729,200 @@ def memoized(wrapped, instance, args, kwargs):  # pylint: disable=unused-argumen
         return __memo_cache[id_string]
 
     return memoize_wrapper(*args, **kwargs)
+
+@contextmanager
+def batch_contextmanager(f, kwargs_list):
+    """
+    Return a context manager that will call the ``f`` callable with the keyword
+    arguments dict in the given list, in one go.
+
+    :param f: Callable expected to return a context manager.
+
+    :param kwargs_list: list of kwargs dictionaries to be used to call ``f``.
+    :type kwargs_list: list(dict)
+    """
+    with ExitStack() as stack:
+        for kwargs in kwargs_list:
+            stack.enter_context(f(**kwargs))
+        yield
+
+
+@contextmanager
+def nullcontext(enter_result=None):
+    """
+    Backport of Python 3.7 ``contextlib.nullcontext``
+
+    This context manager does nothing, so it can be used as a default
+    placeholder for code that needs to select at runtime what context manager
+    to use.
+
+    :param enter_result: Object that will be bound to the target of the with
+        statement, or `None` if nothing is specified.
+    :type enter_result: object
+    """
+    yield enter_result
+
+
+class tls_property:
+    """
+    Use it like `property` decorator, but the result will be memoized per
+    thread. When the owning thread dies, the values for that thread will be
+    destroyed.
+
+    In order to get the values, it's necessary to call the object
+    given by the property. This is necessary in order to be able to add methods
+    to that object, like :meth:`_BoundTLSProperty.get_all_values`.
+
+    Values can be set and deleted as well, which will be a thread-local set.
+    """
+
+    @property
+    def name(self):
+        return self.factory.__name__
+
+    def __init__(self, factory):
+        self.factory = factory
+        # Lock accesses to shared WeakKeyDictionary and WeakSet
+        self.lock = threading.Lock()
+
+    def __get__(self, instance, owner=None):
+        return _BoundTLSProperty(self, instance, owner)
+
+    def _get_value(self, instance, owner):
+        tls, values = self._get_tls(instance)
+        try:
+            return tls.value
+        except AttributeError:
+            # Bind the method to `instance`
+            f = self.factory.__get__(instance, owner)
+            obj = f()
+            tls.value = obj
+            # Since that's a WeakSet, values will be removed automatically once
+            # the threading.local variable that holds them is destroyed
+            with self.lock:
+                values.add(obj)
+            return obj
+
+    def _get_all_values(self, instance, owner):
+        with self.lock:
+            # Grab a reference to all the objects at the time of the call by
+            # using a regular set
+            tls, values = self._get_tls(instance=instance)
+            return set(values)
+
+    def __set__(self, instance, value):
+        tls, values = self._get_tls(instance)
+        tls.value = value
+        with self.lock:
+            values.add(value)
+
+    def __delete__(self, instance):
+        tls, values = self._get_tls(instance)
+        with self.lock:
+            values.discard(tls.value)
+        del tls.value
+
+    def _get_tls(self, instance):
+        dct = instance.__dict__
+        name = self.name
+        try:
+            # Using instance.__dict__[self.name] is safe as
+            # getattr(instance, name) will return the property instead, as
+            # the property is a descriptor
+            tls = dct[name]
+        except KeyError:
+            with self.lock:
+                # Double check after taking the lock to avoid a race
+                if name not in dct:
+                    tls = (threading.local(), WeakSet())
+                    dct[name] = tls
+
+        return tls
+
+    @property
+    def basic_property(self):
+        """
+        Return a basic property that can be used to access the TLS value
+        without having to call it first.
+
+        The drawback is that it's not possible to do anything over than
+        getting/setting/deleting.
+        """
+        def getter(instance, owner=None):
+            prop = self.__get__(instance, owner)
+            return prop()
+
+        return property(getter, self.__set__, self.__delete__)
+
+class _BoundTLSProperty:
+    """
+    Simple proxy object to allow either calling it to get the TLS value, or get
+    some other informations by calling methods.
+    """
+    def __init__(self, tls_property, instance, owner):
+        self.tls_property = tls_property
+        self.instance = instance
+        self.owner = owner
+
+    def __call__(self):
+        return self.tls_property._get_value(
+            instance=self.instance,
+            owner=self.owner,
+        )
+
+    def get_all_values(self):
+        """
+        Returns all the thread-local values currently in use in the process for
+        that property for that instance.
+        """
+        return self.tls_property._get_all_values(
+            instance=self.instance,
+            owner=self.owner,
+        )
+
+
+class InitCheckpointMeta(type):
+    """
+    Metaclass providing an ``initialized`` boolean attributes on instances.
+
+    ``initialized`` is set to ``True`` once the ``__init__`` constructor has
+    returned. It will deal cleanly with nested calls to ``super().__init__``.
+    """
+    def __new__(metacls, name, bases, dct, **kwargs):
+        cls = super().__new__(metacls, name, bases, dct, **kwargs)
+        init_f = cls.__init__
+
+        @wraps(init_f)
+        def init_wrapper(self, *args, **kwargs):
+            self.initialized = False
+
+            # Track the nesting of super()__init__ to set initialized=True only
+            # when the outer level is finished
+            try:
+                stack = self._init_stack
+            except AttributeError:
+                stack = []
+                self._init_stack = stack
+
+            stack.append(init_f)
+            try:
+                x = init_f(self, *args, **kwargs)
+            finally:
+                stack.pop()
+
+            if not stack:
+                self.initialized = True
+                del self._init_stack
+
+            return x
+
+        cls.__init__ = init_wrapper
+
+        return cls
+
+
+class InitCheckpoint(metaclass=InitCheckpointMeta):
+    """
+    Inherit from this class to set the :class:`InitCheckpointMeta` metaclass.
+    """
+    pass

devlib/utils/rendering.py

@@ -147,32 +147,44 @@ class SurfaceFlingerFrameCollector(FrameCollector):
         return text.replace('\r\n', '\n').replace('\r', '\n').split('\n')
 
     def _process_raw_file(self, fh):
+        found = False
         text = fh.read().replace('\r\n', '\n').replace('\r', '\n')
         for line in text.split('\n'):
             line = line.strip()
-            if line:
+            if not line:
-                self._process_trace_line(line)
+                continue
-
+            if 'SurfaceFlinger appears to be unresponsive, dumping anyways' in line:
-    def _process_trace_line(self, line):
+                self.unresponsive_count += 1
+                continue
             parts = line.split()
+            # We only want numerical data, ignore textual data.
+            try:
+                parts = list(map(int, parts))
+            except ValueError:
+                continue
+            found = True
+            self._process_trace_parts(parts)
+        if not found:
+            logger.warning('Could not find expected SurfaceFlinger output.')
+
+    def _process_trace_parts(self, parts):
         if len(parts) == 3:
-            frame = SurfaceFlingerFrame(*list(map(int, parts)))
+            frame = SurfaceFlingerFrame(*parts)
             if not frame.frame_ready_time:
                 return # "null" frame
             if frame.frame_ready_time <= self.last_ready_time:
                 return  # duplicate frame
             if (frame.frame_ready_time - frame.desired_present_time) > self.drop_threshold:
-                logger.debug('Dropping bogus frame {}.'.format(line))
+                logger.debug('Dropping bogus frame {}.'.format(' '.join(map(str, parts))))
                 return  # bogus data
             self.last_ready_time = frame.frame_ready_time
             self.frames.append(frame)
         elif len(parts) == 1:
-            self.refresh_period = int(parts[0])
+            self.refresh_period = parts[0]
             self.drop_threshold = self.refresh_period * 1000
-        elif 'SurfaceFlinger appears to be unresponsive, dumping anyways' in line:
-            self.unresponsive_count += 1
         else:
-            logger.warning('Unexpected SurfaceFlinger dump output: {}'.format(line))
+            msg = 'Unexpected SurfaceFlinger dump output: {}'.format(' '.join(map(str, parts)))
+            logger.warning(msg)
 
 
 def read_gfxinfo_columns(target):

devlib/utils/ssh.py

@@ -14,6 +14,7 @@
 #
 
 
+import glob
 import os
 import stat
 import logging
@@ -26,9 +27,19 @@ import socket
 import sys
 import time
 import atexit
+import contextlib
+import weakref
+import select
+import copy
 from pipes import quote
 from future.utils import raise_from
 
+from paramiko.client import SSHClient, AutoAddPolicy, RejectPolicy
+import paramiko.ssh_exception
+from scp import SCPClient
+# By default paramiko is very verbose, including at the INFO level
+logging.getLogger("paramiko").setLevel(logging.WARNING)
+
 # pylint: disable=import-error,wrong-import-position,ungrouped-imports,wrong-import-order
 import pexpect
 from distutils.version import StrictVersion as V
@@ -42,8 +53,13 @@ from pexpect import EOF, TIMEOUT, spawn
 from devlib.exception import (HostError, TargetStableError, TargetNotRespondingError,
                               TimeoutError, TargetTransientError)
 from devlib.utils.misc import (which, strip_bash_colors, check_output,
-                               sanitize_cmd_template, memoized)
+                               sanitize_cmd_template, memoized, redirect_streams)
 from devlib.utils.types import boolean
+from devlib.connection import (ConnectionBase, ParamikoBackgroundCommand, PopenBackgroundCommand,
+                               SSHTransferManager)
+
+
+DEFAULT_SSH_SUDO_COMMAND = "sudo -p ' ' -S -- sh -c {}"
 
 
 ssh = None
@@ -54,21 +70,113 @@ sshpass = None
 logger = logging.getLogger('ssh')
 gem5_logger = logging.getLogger('gem5-connection')
 
-def ssh_get_shell(host, username, password=None, keyfile=None, port=None, timeout=10, telnet=False, original_prompt=None):
+
+@contextlib.contextmanager
+def _handle_paramiko_exceptions(command=None):
+    try:
+        yield
+    except paramiko.ssh_exception.NoValidConnectionsError as e:
+        raise TargetNotRespondingError('Connection lost: {}'.format(e))
+    except paramiko.ssh_exception.AuthenticationException as e:
+        raise TargetStableError('Could not authenticate: {}'.format(e))
+    except paramiko.ssh_exception.BadAuthenticationType as e:
+        raise TargetStableError('Bad authentication type: {}'.format(e))
+    except paramiko.ssh_exception.BadHostKeyException as e:
+        raise TargetStableError('Bad host key: {}'.format(e))
+    except paramiko.ssh_exception.ChannelException as e:
+        raise TargetStableError('Could not open an SSH channel: {}'.format(e))
+    except paramiko.ssh_exception.PasswordRequiredException as e:
+        raise TargetStableError('Please unlock the private key file: {}'.format(e))
+    except paramiko.ssh_exception.ProxyCommandFailure as e:
+        raise TargetStableError('Proxy command failure: {}'.format(e))
+    except paramiko.ssh_exception.SSHException as e:
+        raise TargetTransientError('SSH logic error: {}'.format(e))
+    except socket.timeout:
+        raise TimeoutError(command, output=None)
+
+
+def _read_paramiko_streams(stdout, stderr, select_timeout, callback, init, chunk_size=int(1e42)):
+    try:
+        return _read_paramiko_streams_internal(stdout, stderr, select_timeout, callback, init, chunk_size)
+    finally:
+        # Close the channel to make sure the remove process will receive
+        # SIGPIPE when writing on its streams. That could happen if the
+        # user closed the out_streams but the remote process has not
+        # finished yet.
+        assert stdout.channel is stderr.channel
+        stdout.channel.close()
+
+
+def _read_paramiko_streams_internal(stdout, stderr, select_timeout, callback, init, chunk_size):
+    channel = stdout.channel
+    assert stdout.channel is stderr.channel
+
+    def read_channel(callback_state):
+        read_list, _, _ = select.select([channel], [], [], select_timeout)
+        for desc in read_list:
+            for ready, recv, name in (
+                (desc.recv_ready(), desc.recv, 'stdout'),
+                (desc.recv_stderr_ready(), desc.recv_stderr, 'stderr')
+            ):
+                if ready:
+                    chunk = recv(chunk_size)
+                    if chunk:
+                        try:
+                            callback_state = callback(callback_state, name, chunk)
+                        except Exception as e:
+                            return (e, callback_state)
+
+        return (None, callback_state)
+
+    def read_all_channel(callback=None, callback_state=None):
+        for stream, name in ((stdout, 'stdout'), (stderr, 'stderr')):
+            try:
+                chunk = stream.read()
+            except Exception:
+                continue
+
+            if callback is not None and chunk:
+                callback_state = callback(callback_state, name, chunk)
+
+        return callback_state
+
+    callback_excep = None
+    try:
+        callback_state = init
+        while not channel.exit_status_ready():
+            callback_excep, callback_state = read_channel(callback_state)
+            if callback_excep is not None:
+                raise callback_excep
+    # Make sure to always empty the streams to unblock the remote process on
+    # the way to exit, in case something bad happened. For example, the
+    # callback could raise an exception to signal it does not want to do
+    # anything anymore, or only reading from one of the stream might have
+    # raised an exception, leaving the other one non-empty.
+    except Exception as e:
+        if callback_excep is None:
+            # Only call the callback if there was no exception originally, as
+            # we don't want to reenter it if it raised an exception
+            read_all_channel(callback, callback_state)
+        raise e
+    else:
+        # Finish emptying the buffers
+        callback_state = read_all_channel(callback, callback_state)
+        exit_code = channel.recv_exit_status()
+        return (callback_state, exit_code)
+
+
+def telnet_get_shell(host,
+                  username,
+                  password=None,
+                  port=None,
+                  timeout=10,
+                  original_prompt=None):
     _check_env()
     start_time = time.time()
     while True:
-        if telnet:
-            if keyfile:
-                raise ValueError('keyfile may not be used with a telnet connection.')
         conn = TelnetPxssh(original_prompt=original_prompt)
-        else:  # ssh
-            conn = pxssh.pxssh(echo=False)
 
         try:
-            if keyfile:
-                conn.login(host, username, ssh_key=keyfile, port=port, login_timeout=timeout)
-            else:
             conn.login(host, username, password, port=port, login_timeout=timeout)
             break
         except EOF:
@@ -148,50 +256,613 @@ def check_keyfile(keyfile):
         return keyfile
 
 
-class SshConnection(object):
+class SshConnectionBase(ConnectionBase):
+    """
+    Base class for SSH connections.
+    """
 
-    default_password_prompt = '[sudo] password'
-    max_cancel_attempts = 5
     default_timeout = 10
 
     @property
     def name(self):
         return self.host
 
-    # pylint: disable=unused-argument,super-init-not-called
+    @property
+    def connected_as_root(self):
+        if self._connected_as_root is None:
+            try:
+                result = self.execute('id', as_root=False)
+            except TargetStableError:
+                is_root = False
+            else:
+                is_root = 'uid=0(' in result
+            self._connected_as_root = is_root
+        return self._connected_as_root
+
+    @connected_as_root.setter
+    def connected_as_root(self, state):
+        self._connected_as_root = state
+
     def __init__(self,
                  host,
                  username,
                  password=None,
                  keyfile=None,
                  port=None,
-                 timeout=None,
-                 telnet=False,
-                 password_prompt=None,
-                 original_prompt=None,
                  platform=None,
-                 sudo_cmd="sudo -- sh -c {}"
+                 sudo_cmd=DEFAULT_SSH_SUDO_COMMAND,
+                 strict_host_check=True,
                  ):
+        super().__init__()
+        self._connected_as_root = None
         self.host = host
         self.username = username
         self.password = password
         self.keyfile = check_keyfile(keyfile) if keyfile else keyfile
         self.port = port
-        self.lock = threading.Lock()
-        self.password_prompt = password_prompt if password_prompt is not None else self.default_password_prompt
         self.sudo_cmd = sanitize_cmd_template(sudo_cmd)
+        self.platform = platform
+        self.strict_host_check = strict_host_check
+        self.options = {}
         logger.debug('Logging in {}@{}'.format(username, host))
-        timeout = timeout if timeout is not None else self.default_timeout
+
-        self.conn = ssh_get_shell(host, username, password, self.keyfile, port, timeout, False, None)
+    def fmt_remote_path(self, path):
+        return '{}@{}:{}'.format(self.username, self.host, path)
+
+    def push(self, sources, dest, timeout=30):
+        # Quote the destination as SCP would apply globbing too
+        dest = self.fmt_remote_path(quote(dest))
+        paths = sources + [dest]
+        return self._scp(paths, timeout)
+
+    def pull(self, sources, dest, timeout=30):
+        # First level of escaping for the remote shell
+        sources = ' '.join(map(quote, sources))
+        # All the sources are merged into one scp parameter
+        sources = self.fmt_remote_path(sources)
+        paths = [sources, dest]
+        self._scp(paths, timeout)
+
+    def _scp(self, paths, timeout=30):
+        # NOTE: the version of scp in Ubuntu 12.04 occasionally (and bizarrely)
+        # fails to connect to a device if port is explicitly specified using -P
+        # option, even if it is the default port, 22. To minimize this problem,
+        # only specify -P for scp if the port is *not* the default.
+        port_string = '-P {}'.format(quote(str(self.port))) if (self.port and self.port != 22) else ''
+        keyfile_string = '-i {}'.format(quote(self.keyfile)) if self.keyfile else ''
+        options = " ".join(["-o {}={}".format(key, val)
+                            for key, val in self.options.items()])
+        paths = ' '.join(map(quote, paths))
+        command = '{} {} -r {} {} {}'.format(scp,
+                                                options,
+                                                keyfile_string,
+                                                port_string,
+                                                paths)
+        command_redacted = command
+        logger.debug(command)
+        if self.password:
+            command, command_redacted = _give_password(self.password, command)
+        try:
+            check_output(command, timeout=timeout, shell=True)
+        except subprocess.CalledProcessError as e:
+            raise_from(HostError("Failed to copy file with '{}'. Output:\n{}".format(
+                command_redacted, e.output)), None)
+        except TimeoutError as e:
+            raise TimeoutError(command_redacted, e.output)
+
+    def _get_default_options(self):
+        if self.strict_host_check:
+            options = {
+                'StrictHostKeyChecking': 'yes',
+            }
+        else:
+            options = {
+                'StrictHostKeyChecking': 'no',
+                'UserKnownHostsFile': '/dev/null',
+            }
+        return options
+
+
+class SshConnection(SshConnectionBase):
+    # pylint: disable=unused-argument,super-init-not-called
+    def __init__(self,
+                 host,
+                 username,
+                 password=None,
+                 keyfile=None,
+                 port=22,
+                 timeout=None,
+                 platform=None,
+                 sudo_cmd=DEFAULT_SSH_SUDO_COMMAND,
+                 strict_host_check=True,
+                 use_scp=False,
+                 poll_transfers=False,
+                 start_transfer_poll_delay=30,
+                 total_transfer_timeout=3600,
+                 transfer_poll_period=30,
+                 ):
+
+        super().__init__(
+            host=host,
+            username=username,
+            password=password,
+            keyfile=keyfile,
+            port=port,
+            platform=platform,
+            sudo_cmd=sudo_cmd,
+            strict_host_check=strict_host_check,
+        )
+        self.timeout = timeout if timeout is not None else self.default_timeout
+
+        # Allow using scp for file transfer if sftp is not supported
+        self.use_scp = use_scp
+        self.poll_transfers=poll_transfers
+        if poll_transfers:
+            transfer_opts = {'start_transfer_poll_delay': start_transfer_poll_delay,
+                            'total_timeout': total_transfer_timeout,
+                            'poll_period': transfer_poll_period,
+                            }
+
+        if self.use_scp:
+            logger.debug('Using SCP for file transfer')
+            _check_env()
+            self.options = self._get_default_options()
+        else:
+            logger.debug('Using SFTP for file transfer')
+
+        self.transfer_mgr = SSHTransferManager(self, **transfer_opts) if poll_transfers else None
+        self.client = self._make_client()
         atexit.register(self.close)
 
-    def push(self, source, dest, timeout=30):
+        # Use a marker in the output so that we will be able to differentiate
-        dest = '{}@{}:{}'.format(self.username, self.host, dest)
+        # target connection issues with "password needed".
-        return self._scp(source, dest, timeout)
+        # Also, sudo might not be installed at all on the target (but
+        # everything will work as long as we login as root). If sudo is still
+        # needed, it will explode when someone tries to use it. After all, the
+        # user might not be interested in being root at all.
+        self._sudo_needs_password = (
+            'NEED_PASSWORD' in
+            self.execute(
+                # sudo -n is broken on some versions on MacOSX, revisit that if
+                # someone ever cares
+                'sudo -n true || echo NEED_PASSWORD',
+                as_root=False,
+                check_exit_code=False,
+            )
+        )
 
-    def pull(self, source, dest, timeout=30):
+    def _make_client(self):
-        source = '{}@{}:{}'.format(self.username, self.host, source)
+        if self.strict_host_check:
-        return self._scp(source, dest, timeout)
+            policy = RejectPolicy
+        else:
+            policy = AutoAddPolicy
+        # Only try using SSH keys if we're not using a password
+        check_ssh_keys = self.password is None
+
+        with _handle_paramiko_exceptions():
+            client = SSHClient()
+            client.load_system_host_keys()
+            client.set_missing_host_key_policy(policy)
+            client.connect(
+                hostname=self.host,
+                port=self.port,
+                username=self.username,
+                password=self.password,
+                key_filename=self.keyfile,
+                timeout=self.timeout,
+                look_for_keys=check_ssh_keys,
+                allow_agent=check_ssh_keys
+            )
+
+            return client
+
+    def _make_channel(self):
+        with _handle_paramiko_exceptions():
+            transport = self.client.get_transport()
+            channel = transport.open_session()
+            return channel
+
+    def _get_progress_cb(self):
+        return self.transfer_mgr.progress_cb if self.transfer_mgr is not None else None
+
+    def _get_sftp(self, timeout):
+        sftp = self.client.open_sftp()
+        sftp.get_channel().settimeout(timeout)
+        return sftp
+
+    def _get_scp(self, timeout):
+        return SCPClient(self.client.get_transport(), socket_timeout=timeout, progress=self._get_progress_cb())
+
+    def _push_file(self, sftp, src, dst):
+        try:
+            sftp.put(src, dst, callback=self._get_progress_cb())
+        # Maybe the dst was a folder
+        except OSError as orig_excep:
+            # If dst was an existing folder, we add the src basename to create
+            # a new destination for the file as cp would do
+            new_dst = os.path.join(
+                dst,
+                os.path.basename(src),
+            )
+            logger.debug('Trying: {} -> {}'.format(src, new_dst))
+            try:
+                sftp.put(src, new_dst, callback=self._get_progress_cb())
+            # This still failed, which either means:
+            # * There are some missing folders in the dirnames
+            # * Something else SFTP-related is wrong
+            except OSError as e:
+                # Raise the original exception, as it is closer to what the
+                # user asked in the first place
+                raise orig_excep
+
+    @classmethod
+    def _path_exists(cls, sftp, path):
+        try:
+            sftp.lstat(path)
+        except FileNotFoundError:
+            return False
+        else:
+            return True
+
+    def _push_folder(self, sftp, src, dst):
+        # Behave like the "mv" command or adb push: a new folder is created
+        # inside the destination folder, rather than merging the trees, but
+        # only if the destination already exists. Otherwise, it is use as-is as
+        # the new hierarchy name.
+        if self._path_exists(sftp, dst):
+            dst = os.path.join(
+                dst,
+                os.path.basename(os.path.normpath(src)),
+            )
+
+        return self._push_folder_internal(sftp, src, dst)
+
+    def _push_folder_internal(self, sftp, src, dst):
+        # This might fail if the folder already exists
+        with contextlib.suppress(IOError):
+            sftp.mkdir(dst)
+
+        for entry in os.scandir(src):
+            name = entry.name
+            src_path = os.path.join(src, name)
+            dst_path = os.path.join(dst, name)
+            if entry.is_dir():
+                push = self._push_folder_internal
+            else:
+                push = self._push_file
+
+            push(sftp, src_path, dst_path)
+
+    def _push_path(self, sftp, src, dst):
+        logger.debug('Pushing via sftp: {} -> {}'.format(src, dst))
+        push = self._push_folder if os.path.isdir(src) else self._push_file
+        push(sftp, src, dst)
+
+    def _pull_file(self, sftp, src, dst):
+        # Pulling a file into a folder will use the source basename
+        if os.path.isdir(dst):
+            dst = os.path.join(
+                dst,
+                os.path.basename(src),
+            )
+
+        with contextlib.suppress(FileNotFoundError):
+            os.remove(dst)
+
+        sftp.get(src, dst, callback=self._get_progress_cb())
+
+    def _pull_folder(self, sftp, src, dst):
+        with contextlib.suppress(FileNotFoundError):
+            try:
+                shutil.rmtree(dst)
+            except OSError:
+                os.remove(dst)
+
+        os.makedirs(dst)
+        for fileattr in sftp.listdir_attr(src):
+            filename = fileattr.filename
+            src_path = os.path.join(src, filename)
+            dst_path = os.path.join(dst, filename)
+            if stat.S_ISDIR(fileattr.st_mode):
+                pull = self._pull_folder
+            else:
+                pull = self._pull_file
+
+            pull(sftp, src_path, dst_path)
+
+    def _pull_path(self, sftp, src, dst):
+        logger.debug('Pulling via sftp: {} -> {}'.format(src, dst))
+        try:
+            self._pull_file(sftp, src, dst)
+        except IOError:
+            # Maybe that was a directory, so retry as such
+            self._pull_folder(sftp, src, dst)
+
+    def push(self, sources, dest, timeout=None):
+        self._push_pull('push', sources, dest, timeout)
+
+    def pull(self, sources, dest, timeout=None):
+        self._push_pull('pull', sources, dest, timeout)
+
+    def _push_pull(self, action, sources, dest, timeout):
+        if action not in ['push', 'pull']:
+            raise ValueError("Action must be either `push` or `pull`")
+
+        # If timeout is set, or told not to poll
+        if timeout is not None or not self.poll_transfers:
+            if self.use_scp:
+                scp = self._get_scp(timeout)
+                scp_cmd = getattr(scp, 'put' if action == 'push' else 'get')
+                scp_msg = '{}ing via scp: {} -> {}'.format(action, sources, dest)
+                logger.debug(scp_msg.capitalize())
+                scp_cmd(sources, dest, recursive=True)
+            else:
+                sftp = self._get_sftp(timeout)
+                sftp_cmd = getattr(self, '_' + action + '_path')
+                with _handle_paramiko_exceptions():
+                    for source in sources:
+                        sftp_cmd(sftp, source, dest)
+
+        # No timeout, and polling is set
+        elif self.use_scp:
+            scp = self._get_scp(timeout)
+            scp_cmd = getattr(scp, 'put' if action == 'push' else 'get')
+            with _handle_paramiko_exceptions(), self.transfer_mgr.manage(sources, dest, action, scp):
+                scp_msg = '{}ing via scp: {} -> {}'.format(action, sources, dest)
+                logger.debug(scp_msg.capitalize())
+                scp_cmd(sources, dest, recursive=True)
+        else:
+            sftp = self._get_sftp(timeout)
+            sftp_cmd = getattr(self, '_' + action + '_path')
+            with _handle_paramiko_exceptions(), self.transfer_mgr.manage(sources, dest, action, sftp):
+                for source in sources:
+                    sftp_cmd(sftp, source, dest)
+
+    def execute(self, command, timeout=None, check_exit_code=True,
+                as_root=False, strip_colors=True, will_succeed=False): #pylint: disable=unused-argument
+        if command == '':
+            return ''
+        try:
+            with _handle_paramiko_exceptions(command):
+                exit_code, output = self._execute(command, timeout, as_root, strip_colors)
+        except TargetStableError as e:
+            if will_succeed:
+                raise TargetTransientError(e)
+            else:
+                raise
+        else:
+            if check_exit_code and exit_code:
+                message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}'
+                raise TargetStableError(message.format(exit_code, command, output))
+            return output
+
+    def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
+        with _handle_paramiko_exceptions(command):
+            bg_cmd = self._background(command, stdout, stderr, as_root)
+
+        self._current_bg_cmds.add(bg_cmd)
+        return bg_cmd
+
+    def _background(self, command, stdout, stderr, as_root):
+        stdout, stderr, command = redirect_streams(stdout, stderr, command)
+
+        command = "printf '%s\n' $$; exec sh -c {}".format(quote(command))
+        channel = self._make_channel()
+
+        def executor(cmd, timeout):
+            channel.exec_command(cmd)
+            # Read are not buffered so we will always get the data as soon as
+            # they arrive
+            return (
+                channel.makefile_stdin(),
+                channel.makefile(),
+                channel.makefile_stderr(),
+            )
+
+        stdin, stdout_in, stderr_in = self._execute_command(
+            command,
+            as_root=as_root,
+            log=False,
+            timeout=None,
+            executor=executor,
+        )
+        pid = int(stdout_in.readline())
+
+        def create_out_stream(stream_in, stream_out):
+            """
+            Create a pair of file-like objects. The first one is used to read
+            data and the second one to write.
+            """
+
+            if stream_out == subprocess.DEVNULL:
+                r, w = None, None
+            # When asked for a pipe, we just give the file-like object as the
+            # reading end and no writing end, since paramiko already writes to
+            # it
+            elif stream_out == subprocess.PIPE:
+                r, w = os.pipe()
+                r = os.fdopen(r, 'rb')
+                w = os.fdopen(w, 'wb')
+            # Turn a file descriptor into a file-like object
+            elif isinstance(stream_out, int) and stream_out >= 0:
+                r = os.fdopen(stream_out, 'rb')
+                w = os.fdopen(stream_out, 'wb')
+            # file-like object
+            else:
+                r = stream_out
+                w = stream_out
+
+            return (r, w)
+
+        out_streams = {
+            name: create_out_stream(stream_in, stream_out)
+            for stream_in, stream_out, name in (
+                (stdout_in, stdout, 'stdout'),
+                (stderr_in, stderr, 'stderr'),
+            )
+        }
+
+        def redirect_thread_f(stdout_in, stderr_in, out_streams, select_timeout):
+            def callback(out_streams, name, chunk):
+                try:
+                    r, w = out_streams[name]
+                except KeyError:
+                    return out_streams
+
+                try:
+                    w.write(chunk)
+                # Write failed
+                except ValueError:
+                    # Since that stream is now closed, stop trying to write to it
+                    del out_streams[name]
+                    # If that was the last open stream, we raise an
+                    # exception so the thread can terminate.
+                    if not out_streams:
+                        raise
+
+                return out_streams
+
+            try:
+                _read_paramiko_streams(stdout_in, stderr_in, select_timeout, callback, copy.copy(out_streams))
+            # The streams closed while we were writing to it, the job is done here
+            except ValueError:
+                pass
+
+            # Make sure the writing end are closed proper since we are not
+            # going to write anything anymore
+            for r, w in out_streams.values():
+                if r is not w and w is not None:
+                    w.close()
+
+        # If there is anything we need to redirect to, spawn a thread taking
+        # care of that
+        select_timeout = 1
+        thread_out_streams = {
+            name: (r, w)
+            for name, (r, w) in out_streams.items()
+            if w is not None
+        }
+        redirect_thread = threading.Thread(
+            target=redirect_thread_f,
+            args=(stdout_in, stderr_in, thread_out_streams, select_timeout),
+            # The thread will die when the main thread dies
+            daemon=True,
+        )
+        redirect_thread.start()
+
+        return ParamikoBackgroundCommand(
+            conn=self,
+            as_root=as_root,
+            chan=channel,
+            pid=pid,
+            stdin=stdin,
+            # We give the reading end to the consumer of the data
+            stdout=out_streams['stdout'][0],
+            stderr=out_streams['stderr'][0],
+            redirect_thread=redirect_thread,
+        )
+
+    def _close(self):
+        logger.debug('Logging out {}@{}'.format(self.username, self.host))
+        with _handle_paramiko_exceptions():
+            bg_cmds = set(self._current_bg_cmds)
+            for bg_cmd in bg_cmds:
+                bg_cmd.close()
+            self.client.close()
+
+    def _execute_command(self, command, as_root, log, timeout, executor):
+        # As we're already root, there is no need to use sudo.
+        log_debug = logger.debug if log else lambda msg: None
+        use_sudo = as_root and not self.connected_as_root
+
+        if use_sudo:
+            if self._sudo_needs_password and not self.password:
+                raise TargetStableError('Attempt to use sudo but no password was specified')
+
+            command = self.sudo_cmd.format(quote(command))
+
+            log_debug(command)
+            streams = executor(command, timeout=timeout)
+            if self._sudo_needs_password:
+                stdin = streams[0]
+                stdin.write(self.password + '\n')
+                stdin.flush()
+        else:
+            log_debug(command)
+            streams = executor(command, timeout=timeout)
+
+        return streams
+
+    def _execute(self, command, timeout=None, as_root=False, strip_colors=True, log=True):
+        # Merge stderr into stdout since we are going without a TTY
+        command = '({}) 2>&1'.format(command)
+
+        stdin, stdout, stderr = self._execute_command(
+            command,
+            as_root=as_root,
+            log=log,
+            timeout=timeout,
+            executor=self.client.exec_command,
+        )
+        stdin.close()
+
+        # Empty the stdout buffer of the command, allowing it to carry on to
+        # completion
+        def callback(output_chunks, name, chunk):
+            output_chunks.append(chunk)
+            return output_chunks
+
+        select_timeout = 1
+        output_chunks, exit_code = _read_paramiko_streams(stdout, stderr, select_timeout, callback, [])
+        # Join in one go to avoid O(N^2) concatenation
+        output = b''.join(output_chunks)
+
+        if sys.version_info[0] == 3:
+            output = output.decode(sys.stdout.encoding or 'utf-8', 'replace')
+        if strip_colors:
+            output = strip_bash_colors(output)
+
+        return (exit_code, output)
+
+
+class TelnetConnection(SshConnectionBase):
+
+    default_password_prompt = '[sudo] password'
+    max_cancel_attempts = 5
+
+    # pylint: disable=unused-argument,super-init-not-called
+    def __init__(self,
+                 host,
+                 username,
+                 password=None,
+                 port=None,
+                 timeout=None,
+                 password_prompt=None,
+                 original_prompt=None,
+                 sudo_cmd="sudo -- sh -c {}",
+                 strict_host_check=True,
+                 platform=None):
+
+        super().__init__(
+            host=host,
+            username=username,
+            password=password,
+            keyfile=None,
+            port=port,
+            platform=platform,
+            sudo_cmd=sudo_cmd,
+            strict_host_check=strict_host_check,
+        )
+
+        self.options = self._get_default_options()
+
+        self.lock = threading.Lock()
+        self.password_prompt = password_prompt if password_prompt is not None else self.default_password_prompt
+        logger.debug('Logging in {}@{}'.format(username, host))
+        timeout = timeout if timeout is not None else self.default_timeout
+
+        self.conn = telnet_get_shell(host, username, password, port, timeout, original_prompt)
+        atexit.register(self.close)
 
     def execute(self, command, timeout=None, check_exit_code=True,
                 as_root=False, strip_colors=True, will_succeed=False): #pylint: disable=unused-argument
@@ -232,9 +903,17 @@ class SshConnection(object):
         try:
             port_string = '-p {}'.format(self.port) if self.port else ''
             keyfile_string = '-i {}'.format(self.keyfile) if self.keyfile else ''
-            if as_root:
+            if as_root and not self.connected_as_root:
                 command = self.sudo_cmd.format(command)
-            command = '{} {} {} {}@{} {}'.format(ssh, keyfile_string, port_string, self.username, self.host, command)
+            options = " ".join([ "-o {}={}".format(key,val)
+                                for key,val in self.options.items()])
+            command = '{} {} {} {} {}@{} {}'.format(ssh,
+                                                    options,
+                                                    keyfile_string,
+                                                    port_string,
+                                                    self.username,
+                                                    self.host,
+                                                    command)
             logger.debug(command)
             if self.password:
                 command, _ = _give_password(self.password, command)
@@ -242,7 +921,7 @@ class SshConnection(object):
         except EOF:
             raise TargetNotRespondingError('Connection lost.')
 
-    def close(self):
+    def _close(self):
         logger.debug('Logging out {}@{}'.format(self.username, self.host))
         try:
             self.conn.logout()
@@ -259,9 +938,15 @@ class SshConnection(object):
                 return True
         return False
 
+    def wait_for_device(self, timeout=30):
+        return
+
+    def reboot_bootloader(self, timeout=30):
+        raise NotImplementedError()
+
     def _execute_and_wait_for_prompt(self, command, timeout=None, as_root=False, strip_colors=True, log=True):
         self.conn.prompt(0.1)  # clear an existing prompt if there is one.
-        if self.username == 'root':
+        if as_root and self.connected_as_root:
             # As we're already root, there is no need to use sudo.
             as_root = False
         if as_root:
@@ -298,26 +983,6 @@ class SshConnection(object):
                 pass
             return False
 
-    def _scp(self, source, dest, timeout=30):
-        # NOTE: the version of scp in Ubuntu 12.04 occasionally (and bizarrely)
-        # fails to connect to a device if port is explicitly specified using -P
-        # option, even if it is the default port, 22. To minimize this problem,
-        # only specify -P for scp if the port is *not* the default.
-        port_string = '-P {}'.format(quote(str(self.port))) if (self.port and self.port != 22) else ''
-        keyfile_string = '-i {}'.format(quote(self.keyfile)) if self.keyfile else ''
-        command = '{} -r {} {} {} {}'.format(scp, keyfile_string, port_string, quote(source), quote(dest))
-        command_redacted = command
-        logger.debug(command)
-        if self.password:
-            command, command_redacted = _give_password(self.password, command)
-        try:
-            check_output(command, timeout=timeout, shell=True)
-        except subprocess.CalledProcessError as e:
-            raise_from(HostError("Failed to copy file with '{}'. Output:\n{}".format(
-                command_redacted, e.output)), None)
-        except TimeoutError as e:
-            raise TimeoutError(command_redacted, e.output)
-
     def _sendline(self, command):
         # Workaround for https://github.com/pexpect/pexpect/issues/552
         if len(command) == self._get_window_size()[1] - self._get_prompt_length():
@@ -334,29 +999,6 @@ class SshConnection(object):
     def _get_window_size(self):
         return self.conn.getwinsize()
 
-class TelnetConnection(SshConnection):
-
-    # pylint: disable=super-init-not-called
-    def __init__(self,
-                 host,
-                 username,
-                 password=None,
-                 port=None,
-                 timeout=None,
-                 password_prompt=None,
-                 original_prompt=None,
-                 platform=None):
-        self.host = host
-        self.username = username
-        self.password = password
-        self.port = port
-        self.keyfile = None
-        self.lock = threading.Lock()
-        self.password_prompt = password_prompt if password_prompt is not None else self.default_password_prompt
-        logger.debug('Logging in {}@{}'.format(username, host))
-        timeout = timeout if timeout is not None else self.default_timeout
-        self.conn = ssh_get_shell(host, username, password, None, port, timeout, True, original_prompt)
-
 
 class Gem5Connection(TelnetConnection):
 
@@ -438,7 +1080,7 @@ class Gem5Connection(TelnetConnection):
                     .format(self.gem5_input_dir, indir))
         self.gem5_input_dir = indir
 
-    def push(self, source, dest, timeout=None):
+    def push(self, sources, dest, timeout=None):
         """
         Push a file to the gem5 device using VirtIO
 
@@ -450,6 +1092,7 @@ class Gem5Connection(TelnetConnection):
         # First check if the connection is set up to interact with gem5
         self._check_ready()
 
+        for source in sources:
             filename = os.path.basename(source)
             logger.debug("Pushing {} to device.".format(source))
             logger.debug("gem5interactdir: {}".format(self.gem5_interact_dir))
@@ -471,7 +1114,7 @@ class Gem5Connection(TelnetConnection):
             self._gem5_shell("ls -al {}".format(quote(self.gem5_input_dir)))
             logger.debug("Push complete.")
 
-    def pull(self, source, dest, timeout=0): #pylint: disable=unused-argument
+    def pull(self, sources, dest, timeout=0): #pylint: disable=unused-argument
         """
         Pull a file from the gem5 device using m5 writefile
 
@@ -483,6 +1126,7 @@ class Gem5Connection(TelnetConnection):
         # First check if the connection is set up to interact with gem5
         self._check_ready()
 
+        for source in sources:
             result = self._gem5_shell("ls {}".format(source))
             files = strip_bash_colors(result).split()
 
@@ -563,7 +1207,7 @@ class Gem5Connection(TelnetConnection):
                          'get this file'.format(redirection_file))
         return output
 
-    def close(self):
+    def _close(self):
         """
         Close and disconnect from the gem5 simulation. Additionally, we remove
         the temporary directory used to pass files into the simulation.
@@ -587,6 +1231,19 @@ class Gem5Connection(TelnetConnection):
         # Delete the lock file
         os.remove(self.lock_file_name)
 
+    def wait_for_device(self, timeout=30):
+        """
+        Wait for Gem5 to be ready for interation with a timeout.
+        """
+        for _ in attempts(timeout):
+            if self.ready:
+                return
+            time.sleep(1)
+        raise TimeoutError('Gem5 is not ready for interaction')
+
+    def reboot_bootloader(self, timeout=30):
+        raise NotImplementedError()
+
     # Functions only to be called by the Gem5 connection itself
     def _connect_gem5_platform(self, platform):
         port = platform.gem5_port
@@ -739,7 +1396,7 @@ class Gem5Connection(TelnetConnection):
         both of these.
         """
         gem5_logger.debug("Sending Sync")
-        self.conn.send("echo \*\*sync\*\*\n")
+        self.conn.send("echo \\*\\*sync\\*\\*\n")
         self.conn.expect(r"\*\*sync\*\*", timeout=self.default_timeout)
         self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)
         self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)

devlib/utils/version.py

@@ -21,7 +21,7 @@ from subprocess import Popen, PIPE
 
 VersionTuple = namedtuple('Version', ['major', 'minor', 'revision', 'dev'])
 
-version = VersionTuple(1, 1, 1, '')
+version = VersionTuple(1, 3, 0, '')
 
 
 def get_devlib_version():

doc/collectors.rst

@@ -0,0 +1,153 @@
+.. _collector:
+
+Collectors
+==========
+
+The ``Collector`` API provide a consistent way of collecting arbitrary data from
+a target. Data is collected via an instance of a class derived from
+:class:`CollectorBase`.
+
+
+Example
+-------
+
+The following example shows how to use a collector to read the logcat output
+from an Android target.
+
+.. code-block:: python
+
+    # import and instantiate the Target and the collector
+    # (note: this assumes exactly one android target connected
+    #  to the host machine).
+    In [1]: from devlib import AndroidTarget, LogcatCollector
+
+    In [2]: t = AndroidTarget()
+
+    # Set up the collector on the Target.
+
+    In [3]: collector = LogcatCollector(t)
+
+    # Configure the output file path for the collector to use.
+    In [4]: collector.set_output('adb_log.txt')
+
+    # Reset the Collector to preform any required configuration or preparation.
+    In [5]: collector.reset()
+
+    # Start Collecting
+    In [6]: collector.start()
+
+    # Wait for some output to be generated
+    In [7]: sleep(10)
+
+    # Stop Collecting
+    In [8]: collector.stop()
+
+    # Retrieved the collected data
+    In [9]: output = collector.get_data()
+
+    # Display the returned ``CollectorOutput`` Object.
+    In [10]: output
+    Out[10]: [<adb_log.txt (file)>]
+
+    In [11] log_file = output[0]
+
+    # Get the path kind of the the returned CollectorOutputEntry.
+    In [12]: log_file.path_kind
+    Out[12]: 'file'
+
+    # Get the path of the returned CollectorOutputEntry.
+    In [13]: log_file.path
+    Out[13]: 'adb_log.txt'
+
+    # Find the full path to the log file.
+    In [14]: os.path.join(os.getcwd(), logfile)
+    Out[14]: '/tmp/adb_log.txt'
+
+
+API
+---
+.. collector:
+
+.. module:: devlib.collector
+
+
+CollectorBase
+~~~~~~~~~~~~~
+
+.. class:: CollectorBase(target, \*\*kwargs)
+
+   A ``CollectorBase`` is the the base class and API that should be
+   implemented to allowing collecting various data from a traget e.g. traces,
+   logs etc.
+
+.. method::  Collector.setup(\*args, \*\*kwargs)
+
+   This will set up the collector on the target. Parameters this method takes
+   are particular to subclasses (see documentation for specific collectors
+   below).  What actions are performed by this method are also
+   collector-specific.  Usually these will be things like  installing
+   executables, starting services, deploying assets, etc. Typically, this method
+   needs to be invoked at most once per reboot of the target (unless
+   ``teardown()`` has been called), but see documentation for the collector
+   you're interested in.
+
+.. method:: CollectorBase.reset()
+
+   This can be used to configure a collector for collection. This must be invoked
+   before ``start()`` is called to begin collection.
+
+.. method:: CollectorBase.start()
+
+   Starts collecting from the target.
+
+.. method:: CollectorBase.stop()
+
+   Stops collecting from target. Must be called after
+   :func:`start()`.
+
+
+.. method:: CollectorBase.set_output(output_path)
+
+   Configure the output path for the particular collector. This will be either
+   a directory or file path which will be used when storing the data. Please see
+   the individual Collector documentation for more information.
+
+
+.. method:: CollectorBase.get_data()
+
+    The collected data will be return via the previously specified output_path.
+    This method will return a ``CollectorOutput`` object which is a subclassed
+    list object containing individual ``CollectorOutputEntry`` objects with details
+    about the individual output entry.
+
+
+CollectorOutputEntry
+~~~~~~~~~~~~~~~~~~~~
+
+This object is designed to allow for the output of a collector to be processed
+generically. The object will behave as a regular string containing the path to
+underlying output path and can be used directly in ``os.path`` operations.
+
+.. attribute:: CollectorOutputEntry.path
+
+    The file path for the corresponding output item.
+
+.. attribute:: CollectorOutputEntry.path_kind
+
+    The type of output the is specified in the ``path`` attribute. Current valid
+    kinds are: ``file`` and ``directory``.
+
+.. method:: CollectorOutputEntry.__init__(path, path_kind)
+
+    Initialises a ``CollectorOutputEntry`` object with the desired file path and
+    kind of file path specified.
+
+
+.. collectors:
+
+Available Collectors
+---------------------
+
+This section lists collectors that are currently part of devlib.
+
+.. todo:: Add collectors

doc/connection.rst

@@ -3,16 +3,17 @@ Connection
 
 A :class:`Connection` abstracts an actual physical connection to a device. The
 first connection is created when :func:`Target.connect` method is called. If a
-:class:`Target` is used in a multi-threaded environment, it will maintain a
+:class:`~devlib.target.Target` is used in a multi-threaded environment, it will
-connection for each thread in which it is invoked. This allows the same target
+maintain a connection for each thread in which it is invoked. This allows
-object to be used in parallel in multiple threads.
+the same target object to be used in parallel in multiple threads.
 
 :class:`Connection`\ s will be automatically created and managed by
-:class:`Target`\ s, so there is usually no reason to create one manually.
+:class:`~devlib.target.Target`\ s, so there is usually no reason to create one
-Instead, configuration for a :class:`Connection` is passed as
+manually. Instead, configuration for a :class:`Connection` is passed as
-`connection_settings` parameter when creating a :class:`Target`. The connection
+`connection_settings` parameter when creating a
-to be used target is also specified on instantiation by `conn_cls` parameter,
+:class:`~devlib.target.Target`. The connection to be used target is also
-though all concrete :class:`Target` implementations will set an appropriate
+specified on instantiation by `conn_cls` parameter, though all concrete
+:class:`~devlib.target.Target` implementations will set an appropriate
 default, so there is typically no need to specify this explicitly.
 
 :class:`Connection` classes are not a part of an inheritance hierarchy, i.e.
@@ -20,25 +21,25 @@ they do not derive from a common base. Instead, a :class:`Connection` is any
 class that implements the following methods.
 
 
-.. method:: push(self, source, dest, timeout=None)
+.. method:: push(self, sources, dest, timeout=None)
 
-   Transfer a file from the host machine to the connected device.
+   Transfer a list of files from the host machine to the connected device.
 
-   :param source: path of to the file on the host
+   :param sources: list of paths on the host
-   :param dest: path of to the file on the connected device.
+   :param dest: path to the file or folder on the connected device.
-   :param timeout: timeout (in seconds) for the transfer; if the transfer does
+   :param timeout: timeout (in seconds) for the transfer of each file; if the
-       not  complete within this period, an exception will be raised.
+       transfer does not complete within this period, an exception will be
+       raised.
 
-.. method:: pull(self, source, dest, timeout=None)
+.. method:: pull(self, sources, dest, timeout=None)
 
-   Transfer a file, or files matching a glob pattern, from the connected device
+   Transfer a list of files from the connected device to the host machine.
-   to the host machine.
 
-   :param source: path of to the file on the connected device. If ``dest`` is a
+   :param sources: list of paths on the connected device.
-       directory, may be a glob pattern.
+   :param dest: path to the file or folder on the host
-   :param dest: path of to the file on the host
+   :param timeout: timeout (in seconds) for the transfer for each file; if the
-   :param timeout: timeout (in seconds) for the transfer; if the transfer does
+       transfer does not complete within this period, an exception will be
-       not  complete within this period, an exception will be raised.
+       raised.
 
 .. method:: execute(self, command, timeout=None, check_exit_code=False, as_root=False, strip_colors=True, will_succeed=False)
 
@@ -58,7 +59,7 @@ class that implements the following methods.
    :param will_succeed: The command is assumed to always succeed, unless there is
        an issue in the environment like the loss of network connectivity. That
        will make the method always raise an instance of a subclass of
-       :class:`DevlibTransientError' when the command fails, instead of a
+       :class:`DevlibTransientError` when the command fails, instead of a
        :class:`DevlibStableError`.
 
 .. method:: background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False)
@@ -76,7 +77,7 @@ class that implements the following methods.
 
    .. note:: This **will block the connection** until the command completes.
 
-.. note:: The above methods are directly wrapped by :class:`Target` methods,
+.. note:: The above methods are directly wrapped by :class:`~devlib.target.Target` methods,
           however note that some of the defaults are different.
 
 .. method:: cancel_running_command(self)
@@ -100,7 +101,12 @@ class that implements the following methods.
 Connection Types
 ----------------
 
-.. class:: AdbConnection(device=None, timeout=None)
+
+.. module:: devlib.utils.android
+
+.. class:: AdbConnection(device=None, timeout=None, adb_server=None, adb_as_root=False, connection_attempts=MAX_ATTEMPTS,\
+                         poll_transfers=False, start_transfer_poll_delay=30, total_transfer_timeout=3600,\
+                         transfer_poll_period=30)
 
     A connection to an android device via ``adb`` (Android Debug Bridge).
     ``adb`` is part of the Android SDK (though stand-alone versions are also
@@ -113,10 +119,37 @@ Connection Types
     :param timeout: Connection timeout in seconds. If a connection to the device
                     is not established within this period, :class:`HostError`
                     is raised.
+    :param adb_server: Allows specifying the address of the adb server to use.
+    :param adb_as_root: Specify whether the adb server should be restarted in root mode.
+    :param connection_attempts: Specify how many connection attempts, 10 seconds
+                                apart, should be attempted to connect to the device.
+                                Defaults to 5.
+    :param poll_transfers: Specify whether file transfers should be polled. Polling
+                           monitors the progress of file transfers and periodically
+                           checks whether they have stalled, attempting to cancel
+                           the transfers prematurely if so.
+    :param start_transfer_poll_delay: If transfers are polled, specify the length of
+                                      time after a transfer has started before polling
+                                      should start.
+    :param total_transfer_timeout: If transfers are polled, specify the total amount of time
+                                   to elapse before the transfer is cancelled, regardless
+                                   of its activity.
+    :param transfer_poll_period: If transfers are polled, specify the period at which
+                                 the transfers are sampled for activity. Too small values
+                                 may cause the destination size to appear the same over
+                                 one or more sample periods, causing improper transfer
+                                 cancellation.
 
 
-.. class:: SshConnection(host, username, password=None, keyfile=None, port=None,\
+
-                         timeout=None, password_prompt=None)
+.. module:: devlib.utils.ssh
+
+.. class:: SshConnection(host, username, password=None, keyfile=None, port=22,\
+                         timeout=None, platform=None, \
+                         sudo_cmd="sudo -- sh -c {}", strict_host_check=True, \
+                         use_scp=False, poll_transfers=False,
+                         start_transfer_poll_delay=30, total_transfer_timeout=3600,\
+                         transfer_poll_period=30)
 
     A connection to a device on the network over SSH.
 
@@ -124,6 +157,9 @@ Connection Types
     :param username: username for SSH login
     :param password: password for the SSH connection
 
+                     .. note:: To connect to a system without a password this
+                               parameter should be set to an empty string otherwise
+                               ssh key authentication will be attempted.
                      .. note:: In order to user password-based authentication,
                                ``sshpass`` utility must be installed on the
                                system.
@@ -138,10 +174,26 @@ Connection Types
     :param timeout: Timeout for the connection in seconds. If a connection
                     cannot be established within this time, an error will be
                     raised.
-    :param password_prompt: A string with the password prompt used by
+    :param platform: Specify the platform to be used. The generic :class:`~devlib.platform.Platform`
-                            ``sshpass``. Set this if your version of ``sshpass``
+                     class is used by default.
-                            uses something other than ``"[sudo] password"``.
+    :param sudo_cmd: Specify the format of the command used to grant sudo access.
-
+    :param strict_host_check: Specify the ssh connection parameter ``StrictHostKeyChecking``,              
+    :param use_scp: Use SCP for file transfers, defaults to SFTP.
+    :param poll_transfers: Specify whether file transfers should be polled. Polling
+                           monitors the progress of file transfers and periodically
+                           checks whether they have stalled, attempting to cancel
+                           the transfers prematurely if so.
+    :param start_transfer_poll_delay: If transfers are polled, specify the length of
+                                      time after a transfer has started before polling
+                                      should start.
+    :param total_transfer_timeout: If transfers are polled, specify the total amount of time
+                                   to elapse before the transfer is cancelled, regardless
+                                   of its activity.
+    :param transfer_poll_period: If transfers are polled, specify the period at which
+                                 the transfers are sampled for activity. Too small values
+                                 may cause the destination size to appear the same over
+                                 one or more sample periods, causing improper transfer
+                                 cancellation.
 
 .. class:: TelnetConnection(host, username, password=None, port=None,\
                             timeout=None, password_prompt=None,\
@@ -174,6 +226,7 @@ Connection Types
                             connection to reduce the possibility of clashes).
                             This parameter is ignored for SSH connections.
 
+.. module:: devlib.host
 
 .. class:: LocalConnection(keep_password=True, unrooted=False, password=None)
 
@@ -189,6 +242,9 @@ Connection Types
                      prompting for it.
 
 
+.. module:: devlib.utils.ssh
+   :noindex:
+
 .. class:: Gem5Connection(platform, host=None, username=None, password=None,\
                           timeout=None, password_prompt=None,\
                           original_prompt=None)
@@ -197,7 +253,7 @@ Connection Types
 
     .. note:: Some of the following input parameters are optional and will be ignored during
               initialisation. They were kept to keep the analogy with a :class:`TelnetConnection`
-              (i.e. ``host``, `username``, ``password``, ``port``,
+              (i.e. ``host``, ``username``, ``password``, ``port``,
               ``password_prompt`` and ``original_promp``)
 
 
@@ -207,7 +263,7 @@ Connection Types
                                will be ignored, the gem5 simulation needs to be
                                on the same host the user is currently on, so if
                                the host given as input parameter is not the
-                               same as the actual host, a ``TargetStableError``
+                               same as the actual host, a :class:`TargetStableError`
                                will be raised to prevent confusion.
 
     :param username: Username in the simulated system
@@ -233,7 +289,7 @@ The only methods discussed below are those that will be overwritten by the
 
     A connection to a gem5 simulation that emulates a Linux system.
 
-.. method:: _login_to_device(self)
+    .. method:: _login_to_device(self)
 
         Login to the gem5 simulated system.
 
@@ -241,6 +297,6 @@ The only methods discussed below are those that will be overwritten by the
 
     A connection to a gem5 simulation that emulates an Android system.
 
-.. method:: _wait_for_boot(self)
+    .. method:: _wait_for_boot(self)
 
         Wait for the gem5 simulated system to have booted and finished the booting animation.

doc/derived_measurements.rst

@@ -1,7 +1,6 @@
 Derived Measurements
 =====================
 
-
 The ``DerivedMeasurements`` API provides a consistent way of performing post
 processing on a provided :class:`MeasurementCsv` file.
 
@@ -35,6 +34,8 @@ API
 Derived Measurements
 ~~~~~~~~~~~~~~~~~~~~
 
+.. module:: devlib.derived
+
 .. class:: DerivedMeasurements
 
    The ``DerivedMeasurements`` class provides an API for post-processing
@@ -102,17 +103,20 @@ Available Derived Measurements
 Energy
 ~~~~~~
 
+.. module:: devlib.derived.energy
+
 .. class:: DerivedEnergyMeasurements
 
-   The ``DerivedEnergyMeasurements`` class is used to calculate average power and
+   The ``DerivedEnergyMeasurements`` class is used to calculate average power
-   cumulative energy for each site if the required data is present.
+   and cumulative energy for each site if the required data is present.
 
-   The calculation of cumulative energy can occur in 3 ways. If a
+   The calculation of cumulative energy can occur in 3 ways. If a ``site``
-   ``site`` contains ``energy`` results, the first and last measurements are extracted
+   contains ``energy`` results, the first and last measurements are extracted
-   and the delta calculated. If not, a ``timestamp`` channel will be used to calculate
+   and the delta calculated. If not, a ``timestamp`` channel will be used to
-   the energy from the power channel, failing back to using the sample rate attribute
+   calculate the energy from the power channel, failing back to using the sample
-   of the :class:`MeasurementCsv` file if timestamps are not available. If neither
+   rate attribute of the :class:`MeasurementCsv` file if timestamps are not
-   timestamps or a sample rate are available then an error will be raised.
+   available. If neither timestamps or a sample rate are available then an error
+   will be raised.
 
 
 .. method:: DerivedEnergyMeasurements.process(measurement_csv)
@@ -128,6 +132,8 @@ Energy
 FPS / Rendering
 ~~~~~~~~~~~~~~~
 
+.. module:: devlib.derived.fps
+
 .. class:: DerivedGfxInfoStats(drop_threshold=5, suffix='-fps', filename=None, outdir=None)
 
    Produces FPS (frames-per-second) and other derived statistics from

doc/index.rst

@@ -3,6 +3,8 @@
    You can adapt this file completely to your liking, but it should at least
    contain the root `toctree` directive.
 
+.. module:: devlib
+
 Welcome to devlib documentation
 ===============================
 
@@ -19,6 +21,7 @@ Contents:
    target
    modules
    instrumentation
+   collectors
    derived_measurements
    platform
    connection

doc/instrumentation.rst

@@ -1,11 +1,13 @@
+.. _instrumentation:
+
 Instrumentation
 ===============
 
 The ``Instrument`` API provide a consistent way of collecting measurements from
 a target. Measurements are collected via an instance of a class derived from
-:class:`Instrument`. An ``Instrument`` allows collection of measurement from one
+:class:`~devlib.instrument.Instrument`. An ``Instrument`` allows collection of
-or more channels. An ``Instrument`` may support ``INSTANTANEOUS`` or
+measurement from one or more channels. An ``Instrument`` may support
-``CONTINUOUS`` collection, or both.
+``INSTANTANEOUS`` or ``CONTINUOUS`` collection, or both.
 
 Example
 -------
@@ -48,6 +50,8 @@ Android target.
 API
 ---
 
+.. module:: devlib.instrument
+
 Instrument
 ~~~~~~~~~~
 
@@ -120,14 +124,16 @@ Instrument
    Take a single measurement from ``active_channels``. Returns a list of
    :class:`Measurement` objects (one for each active channel).
 
-   .. note:: This method is only implemented by :class:`Instrument`\ s that
+   .. note:: This method is only implemented by
+             :class:`~devlib.instrument.Instrument`\ s that
              support ``INSTANTANEOUS`` measurement.
 
 .. method:: Instrument.start()
 
    Starts collecting measurements from ``active_channels``.
 
-   .. note:: This method is only implemented by :class:`Instrument`\ s that
+   .. note:: This method is only implemented by
+             :class:`~devlib.instrument.Instrument`\ s that
              support ``CONTINUOUS`` measurement.
 
 .. method:: Instrument.stop()
@@ -135,7 +141,8 @@ Instrument
    Stops collecting measurements from ``active_channels``. Must be called after
    :func:`start()`.
 
-   .. note:: This method is only implemented by :class:`Instrument`\ s that
+   .. note:: This method is only implemented by
+             :class:`~devlib.instrument.Instrument`\ s that
              support ``CONTINUOUS`` measurement.
 
 .. method:: Instrument.get_data(outfile)
@@ -146,9 +153,9 @@ Instrument
    ``<site>_<kind>`` (see :class:`InstrumentChannel`). The order of the columns
    will be the same as the order of channels in ``Instrument.active_channels``.
 
-   If reporting timestamps, one channel must have a ``site`` named ``"timestamp"``
+   If reporting timestamps, one channel must have a ``site`` named
-   and a ``kind`` of a :class:`MeasurmentType` of an appropriate time unit which will
+   ``"timestamp"`` and a ``kind`` of a :class:`MeasurmentType` of an appropriate
-   be used, if appropriate, during any post processing.
+   time unit which will be used, if appropriate, during any post processing.
 
    .. note:: Currently supported time units are seconds, milliseconds and
              microseconds, other units can also be used if an appropriate
@@ -158,21 +165,34 @@ Instrument
    that can be used to stream :class:`Measurement`\ s lists (similar to what is
    returned by ``take_measurement()``.
 
-   .. note:: This method is only implemented by :class:`Instrument`\ s that
+   .. note:: This method is only implemented by
+             :class:`~devlib.instrument.Instrument`\ s that
              support ``CONTINUOUS`` measurement.
 
 .. method:: Instrument.get_raw()
 
    Returns a list of paths to files containing raw output from the underlying
-   source(s) that is used to produce the data CSV. If now raw output is
+   source(s) that is used to produce the data CSV. If no raw output is
    generated or saved, an empty list will be returned. The format of the
    contents of the raw files is entirely source-dependent.
 
+  .. note:: This method is not guaranteed to return valid filepaths after the
+            :meth:`teardown` method has been invoked as the raw files may have
+            been deleted. Please ensure that copies are created manually
+            prior to calling :meth:`teardown` if the files are to be retained.
+
+.. method:: Instrument.teardown()
+
+   Performs any required clean up of the instrument. This usually includes
+   removing temporary and raw files (if ``keep_raw`` is set to ``False`` on relevant
+   instruments), stopping services etc.
+
 .. attribute:: Instrument.sample_rate_hz
 
    Sample rate of the instrument in Hz. Assumed to be the same for all channels.
 
-   .. note:: This attribute is only provided by :class:`Instrument`\ s that
+   .. note:: This attribute is only provided by
+             :class:`~devlib.instrument.Instrument`\ s that
              support ``CONTINUOUS`` measurement.
 
 Instrument Channel
@@ -181,8 +201,8 @@ Instrument Channel
 .. class:: InstrumentChannel(name, site, measurement_type, \*\*attrs)
 
    An :class:`InstrumentChannel` describes a single type of measurement that may
-   be collected by an :class:`Instrument`. A channel is primarily defined by a
+   be collected by an :class:`~devlib.instrument.Instrument`. A channel is
-   ``site`` and a ``measurement_type``.
+   primarily defined by a ``site`` and a ``measurement_type``.
 
    A ``site`` indicates where  on the target a measurement is collected from
    (e.g. a voltage rail or location of a sensor).
@@ -400,7 +420,7 @@ For reference, the software stack on the host is roughly given by:
 
 Ethernet was the only IIO Interface used and tested during the development of
 this instrument. However,
-`USB seems to be supported<https://gitlab.com/baylibre-acme/ACME/issues/2>`_.
+`USB seems to be supported <https://gitlab.com/baylibre-acme/ACME/issues/2>`_.
 The IIO library also provides "Local" and "XML" connections but these are to be
 used when the IIO devices are directly connected to the host *i.e.* in our
 case, if we were to run Python and devlib on the BBB. These are also untested.
@@ -475,12 +495,13 @@ voltage (see previous figure), samples are retrieved at a frequency of
 
 where :math:`T_X` is the integration time for the :math:`X` voltage.
 
-As described below (:meth:`BaylibreAcmeInstrument.reset`), the integration
+As described below (:meth:`BaylibreAcmeInstrument.reset
-times for the bus and shunt voltage can be set separately which allows a
+<devlib.instrument.baylibre_acme.BaylibreAcmeInstrument.reset>`), the
-tradeoff of accuracy between signals. This is particularly useful as the shunt
+integration times for the bus and shunt voltage can be set separately which
-voltage returned by the INA226 has a higher resolution than the bus voltage
+allows a tradeoff of accuracy between signals. This is particularly useful as
-(2.5 μV and 1.25 mV LSB, respectively) and therefore would benefit more from a
+the shunt voltage returned by the INA226 has a higher resolution than the bus
-longer integration time.
+voltage (2.5 μV and 1.25 mV LSB, respectively) and therefore would benefit more
+from a longer integration time.
 
 As an illustration, consider the following sampled sine wave and notice how
 increasing the integration time (of the bus voltage in this case) "smoothes"
@@ -588,8 +609,9 @@ Buffer-based transactions
 
 Samples made available by the INA226 are retrieved by the BBB and stored in a
 buffer which is sent back to the host once it is full (see
-``buffer_samples_count`` in :meth:`BaylibreAcmeInstrument.setup` for setting
+``buffer_samples_count`` in :meth:`BaylibreAcmeInstrument.setup
-its size). Therefore, the larger the buffer is, the longer it takes to be
+<devlib.instrument.baylibre_acme.BaylibreAcmeInstrument.setup>` for setting its
+size). Therefore, the larger the buffer is, the longer it takes to be
 transmitted back but the less often it has to be transmitted. To illustrate
 this, consider the following graphs showing the time difference between
 successive samples in a retrieved signal when the size of the buffer changes:
@@ -611,6 +633,8 @@ given by `libiio (the Linux IIO interface)`_ however only the network-based one
 has been tested. For the other classes, please refer to the official IIO
 documentation for the meaning of their constructor parameters.
 
+.. module:: devlib.instrument.baylibre_acme
+
 .. class:: BaylibreAcmeInstrument(target=None, iio_context=None, use_base_iio_context=False, probe_names=None)
 
    Base class wrapper for the ACME instrument which itself is a wrapper for the

doc/modules.rst

@@ -1,11 +1,13 @@
+.. module:: devlib.module
+
 .. _modules:
 
 Modules
 =======
 
-Modules add additional functionality to the core :class:`Target` interface.
+Modules add additional functionality to the core :class:`~devlib.target.Target`
-Usually, it is support for specific subsystems on the target. Modules are
+interface. Usually, it is support for specific subsystems on the target. Modules
-instantiated as attributes of the :class:`Target` instance.
+are instantiated as attributes of the :class:`~devlib.target.Target` instance.
 
 hotplug
 -------
@@ -28,6 +30,8 @@ interface to this subsystem
    # Make sure all cpus are online
    target.hotplug.online_all()
 
+.. module:: devlib.module.cpufreq
+
 cpufreq
 -------
 
@@ -132,6 +136,9 @@ policies (governors). The ``devlib`` module exposes the following interface
        ``1`` or ``"cpu1"``).
    :param frequency: Frequency to set.
 
+
+.. module:: devlib.module.cupidle
+
 cpuidle
 -------
 
@@ -167,11 +174,15 @@ cpuidle
 You can also call ``enable()`` or ``disable()`` on :class:`CpuidleState` objects
 returned by get_state(s).
 
+.. module:: devlib.module.cgroups
+
 cgroups
 -------
 
 TODO
 
+.. module:: devlib.module.hwmon
+
 hwmon
 -----
 
@@ -187,8 +198,8 @@ Modules implement discrete, optional pieces of functionality ("optional" in the
 sense that the functionality may or may not be present on the target device, or
 that it may or may not be necessary for a particular application).
 
-Every module (ultimately) derives from :class:`Module` class.  A module must
+Every module (ultimately) derives from :class:`devlib.module.Module` class.  A
-define the following class attributes:
+module must define the following class attributes:
 
 :name: A unique name for the module. This cannot clash with any of the existing
        names and must be a valid Python identifier, but is otherwise free-form.
@@ -204,14 +215,16 @@ define the following class attributes:
                  which case the module's ``name`` will be treated as its
                  ``kind`` as well.
 
-:stage: This defines when the module will be installed into a :class:`Target`.
+:stage: This defines when the module will be installed into a
-        Currently, the following values are allowed:
+        :class:`~devlib.target.Target`. Currently, the following values are
+        allowed:
 
         :connected: The module is installed after a connection to the target has
                     been established. This is the default.
-        :early: The module will be installed when a :class:`Target` is first
+        :early: The module will be installed when a
-                created. This should be used for modules that do not rely on a
+                :class:`~devlib.target.Target` is first created. This should be
-                live connection to the target.
+                used for modules that do not rely on a live connection to the
+                target.
         :setup: The module will be installed after initial setup of the device
                 has been performed. This allows the module to utilize assets
                 deployed during the setup stage for example 'Busybox'.
@@ -220,8 +233,8 @@ Additionally, a module must implement a static (or class) method :func:`probe`:
 
 .. method:: Module.probe(target)
 
-    This method takes a :class:`Target` instance and returns ``True`` if this
+    This method takes a :class:`~devlib.target.Target` instance and returns
-    module is supported by that target, or ``False`` otherwise.
+    ``True`` if this module is supported by that target, or ``False`` otherwise.
 
     .. note:: If the module ``stage`` is ``"early"``, this method cannot assume
               that a connection has been established (i.e. it can only access
@@ -231,9 +244,9 @@ Installation and invocation
 ***************************
 
 The default installation method will create an instance of a module (the
-:class:`Target` instance being the sole argument) and assign it to the target
+:class:`~devlib.target.Target` instance being the sole argument) and assign it
-instance attribute named after the module's ``kind`` (or ``name`` if ``kind`` is
+to the target instance attribute named after the module's ``kind`` (or
-``None``).
+``name`` if ``kind`` is ``None``).
 
 It is possible to change the installation procedure for a module by overriding
 the default :func:`install` method. The method must have the following
@@ -322,7 +335,7 @@ FlashModule
 
     "flash"
 
-.. method:: __call__(image_bundle=None, images=None, boot_config=None)
+.. method:: __call__(image_bundle=None, images=None, boot_config=None, connect=True)
 
     Must be implemented by derived classes.
 
@@ -338,15 +351,17 @@ FlashModule
     :param boot_config: Some platforms require specifying boot arguments at the
                         time of flashing the images, rather than during each
                         reboot. For other platforms, this will be ignored.
+    :connect: Specifiy whether to try and connect to the target after flashing.
 
 
 Module Registration
 ~~~~~~~~~~~~~~~~~~~
 
-Modules are specified on :class:`Target` or :class:`Platform` creation by name.
+Modules are specified on :class:`~devlib.target.Target` or
-In order to find the class associated with the name, the module needs to be
+:class:`~devlib.platform.Platform` creation by name. In order to find the class
-registered with ``devlib``. This is accomplished by passing the module class
+associated with the name, the module needs to be registered with ``devlib``.
-into :func:`register_module` method once it is defined.
+This is accomplished by passing the module class into :func:`register_module`
+method once it is defined.
 
 .. note:: If you're wiring a module to be included as part of ``devlib`` code
           base, you can place the file with the module class under

doc/overview.rst

@@ -1,26 +1,26 @@
 Overview
 ========
 
-A :class:`Target` instance serves as the main interface to the target device.
+A :class:`~devlib.target.Target` instance serves as the main interface to the target device.
 There are currently four target interfaces:
 
-- :class:`LinuxTarget` for interacting with Linux devices over SSH.
+- :class:`~devlib.target.LinuxTarget` for interacting with Linux devices over SSH.
-- :class:`AndroidTarget` for interacting with Android devices over adb.
+- :class:`~devlib.target.AndroidTarget` for interacting with Android devices over adb.
-- :class:`ChromeOsTarget`: for interacting with ChromeOS devices over SSH, and
+- :class:`~devlib.target.ChromeOsTarget`: for interacting with ChromeOS devices
-                           their Android containers over adb.
+  over SSH, and their Android containers over adb.
-- :class:`LocalLinuxTarget`: for interacting with the local Linux host.
+- :class:`~devlib.target.LocalLinuxTarget`: for interacting with the local Linux host.
 
 They all work in more-or-less the same way, with the major difference being in
 how connection settings are specified; though there may also be a few APIs
-specific to a particular target type (e.g. :class:`AndroidTarget` exposes
+specific to a particular target type (e.g. :class:`~devlib.target.AndroidTarget`
-methods for working with logcat).
+exposes methods for working with logcat).
 
 
 Acquiring a Target
 ------------------
 
 To create an interface to your device, you just need to instantiate one of the
-:class:`Target` derivatives listed above, and pass it the right
+:class:`~devlib.target.Target` derivatives listed above, and pass it the right
 ``connection_settings``. Code snippet below gives a typical example of
 instantiating each of the three target types.
 
@@ -47,21 +47,22 @@ instantiating each of the three target types.
    t3 = AndroidTarget(connection_settings={'device': '0123456789abcde'})
 
 Instantiating a target may take a second or two as the remote device will be
-queried to initialize :class:`Target`'s internal state. If you would like to
+queried to initialize :class:`~devlib.target.Target`'s internal state. If you
-create a :class:`Target` instance but not immediately connect to the remote
+would like to create a :class:`~devlib.target.Target` instance but not
-device, you can pass ``connect=False`` parameter. If you do that, you would have
+immediately connect to the remote device, you can pass ``connect=False``
-to then explicitly call ``t.connect()`` before you can interact with the device.
+parameter. If you do that, you would have to then explicitly call
+``t.connect()`` before you can interact with the device.
 
 There are a few additional parameters you can pass in instantiation besides
 ``connection_settings``, but they are usually unnecessary. Please see
-:class:`Target` API documentation for more details.
+:class:`~devlib.target.Target` API documentation for more details.
 
 Target Interface
 ----------------
 
 This is a quick overview of the basic interface to the device. See
-:class:`Target` API documentation for the full list of supported methods and
+:class:`~devlib.target.Target` API documentation for the full list of supported
-more detailed documentation.
+methods and more detailed documentation.
 
 One-time Setup
 ~~~~~~~~~~~~~~
@@ -167,15 +168,16 @@ Process Control
    # PsEntry records.
    entries = t.ps()
    # e.g.  print virtual memory sizes of all running sshd processes:
-   print ', '.join(str(e.vsize) for e in entries if e.name == 'sshd')
+   print(', '.join(str(e.vsize) for e in entries if e.name == 'sshd'))
 
 
 More...
 ~~~~~~~
 
 As mentioned previously, the above is not intended to be exhaustive
-documentation of the :class:`Target` interface. Please refer to the API
+documentation of the :class:`~devlib.target.Target` interface. Please refer to
-documentation for the full list of attributes and methods and their parameters.
+the API documentation for the full list of attributes and methods and their
+parameters.
 
 Super User Privileges
 ---------------------
@@ -239,6 +241,8 @@ complete. Retrying it or bailing out is therefore a responsability of the caller
 
 The hierarchy is as follows:
 
+.. module:: devlib.exception
+
 - :class:`DevlibError`
 
    - :class:`WorkerThreadError`
@@ -288,7 +292,7 @@ Modules
 Additional functionality is exposed via modules. Modules are initialized as
 attributes of a target instance. By default, ``hotplug``, ``cpufreq``,
 ``cpuidle``, ``cgroups`` and ``hwmon`` will attempt to load on target; additional
-modules may be specified when creating a :class:`Target` instance.
+modules may be specified when creating a :class:`~devlib.target.Target` instance.
 
 A module will probe the target for support before attempting to load. So if the
 underlying platform does not support particular functionality (e.g. the kernel
@@ -307,12 +311,22 @@ has been successfully installed on a target, you can use ``has()`` method, e.g.
 
 Please see the modules documentation for more detail.
 
+Instruments and Collectors
+--------------------------
 
-Measurement and Trace
+You can retrieve multiple types of data from a target. There are two categories
----------------------
+of classes that allow for this:
 
-You can collected traces (currently, just ftrace) using
+
-:class:`TraceCollector`\ s. For example
+- An :class:`Instrument` which may be used to collect measurements (such as power) from
+  targets that support it. Please see the
+  :ref:`instruments documentation <Instrumentation>` for more details.
+
+- A :class:`Collector` may be used to collect arbitary data from a ``Target`` varying
+  from screenshots to trace data. Please see the
+  :ref:`collectors documentation <collector>` for more details.
+
+An example workflow using :class:`FTraceCollector` is as follows:
 
 .. code:: python
 
@@ -333,16 +347,12 @@ You can collected traces (currently, just ftrace) using
       import time; time.sleep(5)
 
    # extract the trace file from the target into a local file
-   trace.get_trace('/tmp/trace.bin')
+   trace.get_data('/tmp/trace.bin')
 
    # View trace file using Kernelshark (must be installed on the host).
    trace.view('/tmp/trace.bin')
 
    # Convert binary trace into text format. This would normally be done
-   # automatically during get_trace(), unless autoreport is set to False during
+   # automatically during get_data(), unless autoreport is set to False during
    # instantiation of the trace collector.
    trace.report('/tmp/trace.bin', '/tmp/trace.txt')
-
-In a similar way, :class:`Instrument` instances may be used to collect
-measurements (such as power) from targets that support it. Please see
-instruments documentation for more details.

doc/platform.rst

@@ -1,14 +1,17 @@
+.. module:: devlib.platform
+
 .. _platform:
 
 Platform
 ========
 
-:class:`Platform`\ s describe the system underlying the OS. They encapsulate
+:class:`~devlib.platform.Platform`\ s describe the system underlying the OS.
-hardware- and firmware-specific details. In most cases, the generic
+They encapsulate hardware- and firmware-specific details. In most cases, the
-:class:`Platform` class, which gets used if a platform is not explicitly
+generic :class:`~devlib.platform.Platform` class, which gets used if a
-specified on :class:`Target` creation, will be sufficient. It will automatically
+platform is not explicitly specified on :class:`~devlib.target.Target`
-query as much platform information (such CPU topology, hardware model, etc) if
+creation, will be sufficient. It will automatically query as much platform
-it was not specified explicitly by the user.
+information (such CPU topology, hardware model, etc) if it was not specified
+explicitly by the user.
 
 
 .. class:: Platform(name=None, core_names=None, core_clusters=None,\
@@ -31,6 +34,7 @@ it was not specified explicitly by the user.
                     platform (e.g. for handling flashing, rebooting, etc). These
                     would be added to the Target's modules. (See :ref:`modules`\ ).
 
+.. module:: devlib.platform.arm
 
 Versatile Express
 -----------------
@@ -38,8 +42,8 @@ Versatile Express
 The generic platform may be extended to support hardware- or
 infrastructure-specific functionality. Platforms exist for ARM
 VersatileExpress-based :class:`Juno` and :class:`TC2` development boards. In
-addition to the standard :class:`Platform` parameters above, these platforms
+addition to the standard :class:`~devlib.platform.Platform` parameters above,
-support additional configuration:
+these platforms support additional configuration:
 
 
 .. class:: VersatileExpressPlatform
@@ -116,43 +120,53 @@ support additional configuration:
 Gem5 Simulation Platform
 ------------------------
 
-By initialising a Gem5SimulationPlatform, devlib will start a gem5 simulation (based upon the
+By initialising a Gem5SimulationPlatform, devlib will start a gem5 simulation
-arguments the user provided) and then connect to it using :class:`Gem5Connection`.
+(based upon the arguments the user provided) and then connect to it using
-Using the methods discussed above, some methods of the :class:`Target` will be altered
+:class:`~devlib.utils.ssh.Gem5Connection`. Using the methods discussed above,
-slightly to better suit gem5.
+some methods of the :class:`~devlib.target.Target` will be altered slightly to
+better suit gem5.
+
+.. module:: devlib.platform.gem5
 
 .. class:: Gem5SimulationPlatform(name, host_output_dir, gem5_bin, gem5_args, gem5_virtio, gem5_telnet_port=None)
 
-    During initialisation the gem5 simulation will be kicked off (based upon the arguments
+    During initialisation the gem5 simulation will be kicked off (based upon the
-    provided by the user) and the telnet port used by the gem5 simulation will be intercepted
+    arguments provided by the user) and the telnet port used by the gem5
-    and stored for use by the :class:`Gem5Connection`.
+    simulation will be intercepted and stored for use by the
+    :class:`~devlib.utils.ssh.Gem5Connection`.
 
     :param name: Platform name
 
-    :param host_output_dir: Path on the host where the gem5 outputs will be placed (e.g. stats file)
+    :param host_output_dir: Path on the host where the gem5 outputs will be
+        placed (e.g. stats file)
 
     :param gem5_bin: gem5 binary
 
     :param gem5_args: Arguments to be passed onto gem5 such as config file etc.
 
-    :param gem5_virtio: Arguments to be passed onto gem5 in terms of the virtIO device used
+    :param gem5_virtio: Arguments to be passed onto gem5 in terms of the virtIO
-                        to transfer files between the host and the gem5 simulated system.
+        device used to transfer files between the host and the gem5 simulated
+        system.
 
-    :param gem5_telnet_port: Not yet in use as it would be used in future implementations
+    :param gem5_telnet_port: Not yet in use as it would be used in future
-                             of devlib in which the user could use the platform to pick
+                             implementations of devlib in which the user could
-                             up an existing and running simulation.
+                             use the platform to pick up an existing and running
+                             simulation.
 
 
 .. method:: Gem5SimulationPlatform.init_target_connection([target])
 
-    Based upon the OS defined in the :class:`Target`, the type of :class:`Gem5Connection`
+    Based upon the OS defined in the :class:`~devlib.target.Target`, the type of
-    will be set (:class:`AndroidGem5Connection` or :class:`AndroidGem5Connection`).
+    :class:`~devlib.utils.ssh.Gem5Connection` will be set
+    (:class:`~devlib.utils.ssh.AndroidGem5Connection` or
+    :class:`~devlib.utils.ssh.AndroidGem5Connection`).
 
 .. method:: Gem5SimulationPlatform.update_from_target([target])
 
-    This method provides specific setup procedures for a gem5 simulation. First of all, the m5
+    This method provides specific setup procedures for a gem5 simulation. First
-    binary will be installed on the guest (if it is not present). Secondly, three methods
+    of all, the m5 binary will be installed on the guest (if it is not present).
-    in the :class:`Target` will be monkey-patched:
+    Secondly, three methods in the :class:`~devlib.target.Target` will be
+    monkey-patched:
 
             - **reboot**: this is not supported in gem5
             - **reset**: this is not supported in gem5
@@ -160,7 +174,7 @@ slightly to better suit gem5.
               monkey-patched method will first try to
               transfer the existing screencaps.
               In case that does not work, it will fall back
-              to the original :class:`Target` implementation
+              to the original :class:`~devlib.target.Target` implementation
               of :func:`capture_screen`.
 
     Finally, it will call the parent implementation of :func:`update_from_target`.

doc/target.rst

@@ -1,57 +1,62 @@
+.. module:: devlib.target
+
 Target
 ======
 
-
 .. class:: Target(connection_settings=None, platform=None, working_directory=None, executables_directory=None, connect=True, modules=None, load_default_modules=True, shell_prompt=DEFAULT_SHELL_PROMPT, conn_cls=None)
 
-    :class:`Target` is the primary interface to the remote device. All interactions
+    :class:`~devlib.target.Target` is the primary interface to the remote
-    with the device are performed via a :class:`Target` instance, either
+    device. All interactions with the device are performed via a
-    directly, or via its modules or a wrapper interface (such as an
+    :class:`~devlib.target.Target` instance, either directly, or via its
-    :class:`Instrument`).
+    modules or a wrapper interface (such as an
+    :class:`~devlib.instrument.Instrument`).
 
-    :param connection_settings: A ``dict`` that specifies how to connect to the remote
+    :param connection_settings: A ``dict`` that specifies how to connect to the
-       device. Its contents depend on the specific :class:`Target` type (used see
+       remote device. Its contents depend on the specific
+       :class:`~devlib.target.Target` type (used see
        :ref:`connection-types`\ ).
 
-    :param platform: A :class:`Target` defines interactions at Operating System level. A
+    :param platform: A :class:`~devlib.target.Target` defines interactions at
-        :class:`Platform` describes the underlying hardware (such as CPUs
+        Operating System level. A :class:`~devlib.platform.Platform` describes
-        available). If a :class:`Platform` instance is not specified on
+        the underlying hardware (such as CPUs available). If a
-        :class:`Target` creation, one will be created automatically and it will
+        :class:`~devlib.platform.Platform` instance is not specified on
-        dynamically probe the device to discover as much about the underlying
+        :class:`~devlib.target.Target` creation, one will be created
-        hardware as it can. See also :ref:`platform`\ .
+        automatically and it will dynamically probe the device to discover
+        as much about the underlying hardware as it can. See also
+        :ref:`platform`\ .
 
     :param working_directory: This is primary location for on-target file system
-        interactions performed by ``devlib``. This location *must* be readable and
+        interactions performed by ``devlib``. This location *must* be readable
-        writable directly (i.e. without sudo) by the connection's user account.
+        and writable directly (i.e. without sudo) by the connection's user
-        It may or may not allow execution. This location will be created,
+        account. It may or may not allow execution. This location will be
-        if necessary, during ``setup()``.
+        created, if necessary, during :meth:`setup()`.
 
         If not explicitly specified, this will be set to a default value
-        depending on the type of :class:`Target`
+        depending on the type of :class:`~devlib.target.Target`
 
     :param executables_directory: This is the location to which ``devlib`` will
-        install executable binaries (either during ``setup()`` or via an
+        install executable binaries (either during :meth:`setup()` or via an
-        explicit ``install()`` call). This location *must* support execution
+        explicit :meth:`install()` call). This location *must* support execution
         (obviously). It should also be possible to write to this location,
         possibly with elevated privileges (i.e. on a rooted Linux target, it
         should be possible to write here with sudo, but not necessarily directly
-        by the connection's account). This location will be created,
+        by the connection's account). This location will be created, if
-        if necessary, during ``setup()``.
+        necessary, during :meth:`setup()`.
 
         This location does *not* need to be same as the system's executables
         location. In fact, to prevent devlib from overwriting system's defaults,
         it better if this is a separate location, if possible.
 
         If not explicitly specified, this will be set to a default value
-        depending on the type of :class:`Target`
+        depending on the type of :class:`~devlib.target.Target`
 
     :param connect: Specifies whether a connections should be established to the
-        target. If this is set to ``False``, then ``connect()`` must be
+        target. If this is set to ``False``, then :meth:`connect()` must be
-        explicitly called later on before the :class:`Target` instance can be
+        explicitly called later on before the :class:`~devlib.target.Target`
-        used.
+        instance can be used.
 
-    :param modules: a list of additional modules to be installed. Some modules will
+    :param modules: a list of additional modules to be installed. Some modules
-        try to install by default (if supported by the underlying target).
+        will try to install by default (if supported by the underlying target).
         Current default modules are ``hotplug``, ``cpufreq``, ``cpuidle``,
         ``cgroups``, and ``hwmon`` (See :ref:`modules`\ ).
 
@@ -59,40 +64,40 @@ Target
 
     :param load_default_modules: If set to ``False``,  default modules listed
          above will *not* attempt to load. This may be used to either speed up
-         target instantiation (probing for initializing modules takes a bit of time)
+         target instantiation (probing for initializing modules takes a bit of
-         or if there is an issue with one of the modules on a particular device
+         time) or if there is an issue with one of the modules on a particular
-         (the rest of the modules will then have to be explicitly specified in
+         device (the rest of the modules will then have to be explicitly
-         the ``modules``).
+         specified in the ``modules``).
 
     :param shell_prompt: This is a regular expression that matches the shell
          prompted on the target. This may be used by some modules that establish
          auxiliary connections to a target over UART.
 
-    :param conn_cls: This is the type of connection that will be used to communicate
+    :param conn_cls: This is the type of connection that will be used to
-        with the device.
+        communicate with the device.
 
 .. attribute:: Target.core_names
 
    This is a list containing names of CPU cores on the target, in the order in
    which they are index by the kernel. This is obtained via the underlying
-   :class:`Platform`.
+   :class:`~devlib.platform.Platform`.
 
 .. attribute:: Target.core_clusters
 
    Some devices feature heterogeneous core configurations (such as ARM
    big.LITTLE).  This is a list that maps CPUs onto underlying clusters.
    (Usually, but not always, clusters correspond to groups of CPUs with the same
-   name). This is obtained via the underlying :class:`Platform`.
+   name). This is obtained via the underlying :class:`~devlib.platform.Platform`.
 
 .. attribute:: Target.big_core
 
    This is the name of the cores that are the "big"s in an ARM big.LITTLE
-   configuration. This is obtained via the underlying :class:`Platform`.
+   configuration. This is obtained via the underlying :class:`~devlib.platform.Platform`.
 
 .. attribute:: Target.little_core
 
    This is the name of the cores that are the "little"s in an ARM big.LITTLE
-   configuration. This is obtained via the underlying :class:`Platform`.
+   configuration. This is obtained via the underlying :class:`~devlib.platform.Platform`.
 
 .. attribute:: Target.is_connected
 
@@ -152,11 +157,11 @@ Target
 
    The underlying connection object. This will be ``None`` if an active
    connection does not exist (e.g. if ``connect=False`` as passed on
-   initialization and ``connect()`` has not been called).
+   initialization and :meth:`connect()` has not been called).
 
-   .. note:: a :class:`Target` will automatically create a connection per
+   .. note:: a :class:`~devlib.target.Target` will automatically create a
-             thread. This will always be set to the connection for the current
+             connection per thread. This will always be set to the connection
-             thread.
+             for the current thread.
 
 .. method:: Target.connect([timeout])
 
@@ -176,19 +181,20 @@ Target
    being executed.
 
    This should *not* be used to establish an initial connection; use
-   ``connect()`` instead.
+   :meth:`connect()` instead.
 
-   .. note:: :class:`Target` will automatically create a connection per
+   .. note:: :class:`~devlib.target.Target` will automatically create a connection
-             thread, so you don't normally need to use this explicitly in
+             per thread, so you don't normally need to use this explicitly in
              threaded code. This is generally useful if you want to perform a
-             blocking operation (e.g. using ``background()``) while at the same
+             blocking operation (e.g. using :class:`background()`) while at the same
              time doing something else in the same host-side thread.
 
 .. method:: Target.setup([executables])
 
    This will perform an initial one-time set up of a device for devlib
-   interaction. This involves deployment of tools relied on the :class:`Target`,
+   interaction. This involves deployment of tools relied on the
-   creation of working locations on the device, etc.
+   :class:`~devlib.target.Target`, creation of working locations on the device,
+   etc.
 
    Usually, it is enough to call this method once per new device, as its effects
    will persist across reboots. However, it is safe to call this method multiple
@@ -212,27 +218,45 @@ Target
         operations during reboot process to detect if the reboot has failed and
         the device has hung.
 
-.. method:: Target.push(source, dest [,as_root , timeout])
+.. method:: Target.push(source, dest [,as_root , timeout, globbing])
 
    Transfer a file from the host machine to the target device.
 
-   :param source: path of to the file on the host
+   If transfer polling is supported (ADB connections and SSH connections),
-   :param dest: path of to the file on the target
+   ``poll_transfers`` is set in the connection, and a timeout is not specified,
+   the push will be polled for activity. Inactive transfers will be
+   cancelled. (See :ref:`connection-types`\ for more information on polling).
+
+   :param source: path on the host
+   :param dest: path on the target
    :param as_root: whether root is required. Defaults to false.
    :param timeout: timeout (in seconds) for the transfer; if the transfer does
        not  complete within this period, an exception will be raised.
+   :param globbing: If ``True``, the ``source`` is interpreted as a globbing
+        pattern instead of being take as-is. If the pattern has mulitple
+        matches, ``dest`` must be a folder (or will be created as such if it
+        does not exists yet).
 
-.. method:: Target.pull(source, dest [, as_root, timeout])
+.. method:: Target.pull(source, dest [, as_root, timeout, globbing])
 
    Transfer a file from the target device to the host machine.
 
-   :param source: path of to the file on the target
+   If transfer polling is supported (ADB connections and SSH connections),
-   :param dest: path of to the file on the host
+   ``poll_transfers`` is set in the connection, and a timeout is not specified,
+   the pull will be polled for activity. Inactive transfers will be
+   cancelled. (See :ref:`connection-types`\ for more information on polling).
+
+   :param source: path on the target
+   :param dest: path on the host
    :param as_root: whether root is required. Defaults to false.
    :param timeout: timeout (in seconds) for the transfer; if the transfer does
        not  complete within this period, an exception will be raised.
+   :param globbing: If ``True``, the ``source`` is interpreted as a globbing
+        pattern instead of being take as-is. If the pattern has mulitple
+        matches, ``dest`` must be a folder (or will be created as such if it
+        does not exists yet).
 
-.. method:: Target.execute(command [, timeout [, check_exit_code [, as_root [, strip_colors [, will_succeed]]]]])
+.. method:: Target.execute(command [, timeout [, check_exit_code [, as_root [, strip_colors [, will_succeed [, force_locale]]]]]])
 
    Execute the specified command on the target device and return its output.
 
@@ -252,6 +276,9 @@ Target
        will make the method always raise an instance of a subclass of
        :class:`DevlibTransientError` when the command fails, instead of a
        :class:`DevlibStableError`.
+   :param force_locale: Prepend ``LC_ALL=<force_locale>`` in front of the
+      command to get predictable output that can be more safely parsed.
+      If ``None``, no locale is prepended.
 
 .. method:: Target.background(command [, stdout [, stderr [, as_root]]])
 
@@ -278,31 +305,31 @@ Target
           a string.
    :param in_directory:  execute the binary in the  specified directory. This must
                    be an absolute path.
-   :param on_cpus:  taskset the binary to these CPUs. This may be a single ``int`` (in which
+   :param on_cpus:  taskset the binary to these CPUs. This may be a single
-          case, it will be interpreted as the mask), a list of ``ints``, in which
+          ``int`` (in which case, it will be interpreted as the mask), a list of
-          case this will be interpreted as the list of cpus, or string, which
+          ``ints``, in which case this will be interpreted as the list of cpus,
-          will be interpreted as a comma-separated list of cpu ranges, e.g.
+          or string, which will be interpreted as a comma-separated list of cpu
-          ``"0,4-7"``.
+          ranges, e.g. ``"0,4-7"``.
    :param as_root: Specify whether the command should be run as root
    :param timeout: If this is specified and invocation does not terminate within this number
            of seconds, an exception will be raised.
 
 .. method:: Target.background_invoke(binary [, args [, in_directory [, on_cpus [, as_root ]]]])
 
-   Execute the specified binary on target (must already be installed) as a background
+   Execute the specified binary on target (must already be installed) as a
-   task, under the specified conditions and return the :class:`subprocess.Popen`
+   background task, under the specified conditions and return the
-   instance for the command.
+   :class:`subprocess.Popen` instance for the command.
 
    :param binary: binary to execute. Must be present and executable on the device.
    :param args: arguments to be passed to the binary. The can be either a list or
           a string.
    :param in_directory:  execute the binary in the  specified directory. This must
                    be an absolute path.
-   :param on_cpus:  taskset the binary to these CPUs. This may be a single ``int`` (in which
+   :param on_cpus:  taskset the binary to these CPUs. This may be a single
-          case, it will be interpreted as the mask), a list of ``ints``, in which
+          ``int`` (in which case, it will be interpreted as the mask), a list of
-          case this will be interpreted as the list of cpus, or string, which
+          ``ints``, in which case this will be interpreted as the list of cpus,
-          will be interpreted as a comma-separated list of cpu ranges, e.g.
+          or string, which will be interpreted as a comma-separated list of cpu
-          ``"0,4-7"``.
+          ranges, e.g. ``"0,4-7"``.
    :param as_root: Specify whether the command should be run as root
 
 .. method:: Target.kick_off(command [, as_root])
@@ -346,7 +373,19 @@ Target
        some sysfs entries silently failing to set the written value without
        returning an error code.
 
-.. method:: Target.read_tree_values(path, depth=1, dictcls=dict, [, tar [, decode_unicode [, strip_null_char ]]]):
+.. method:: Target.revertable_write_value(path, value [, verify])
+
+   Same as :meth:`Target.write_value`, but as a context manager that will write
+   back the previous value on exit.
+
+.. method:: Target.batch_revertable_write_value(kwargs_list)
+
+   Calls :meth:`Target.revertable_write_value` with all the keyword arguments
+   dictionary given in the list. This is a convenience method to update
+   multiple files at once, leaving them in their original state on exit. If one
+   write fails, all the already-performed writes will be reverted as well.
+
+.. method:: Target.read_tree_values(path, depth=1, dictcls=dict, [, tar [, decode_unicode [, strip_null_char ]]])
 
    Read values of all sysfs (or similar) file nodes under ``path``, traversing
    up to the maximum depth ``depth``.
@@ -371,7 +410,7 @@ Target
    :param decode_unicode: decode the content of tar-ed files as utf-8
    :param strip_null_char: remove null chars from utf-8 decoded files
 
-.. method:: Target.read_tree_values_flat(path, depth=1):
+.. method:: Target.read_tree_values_flat(path, depth=1)
 
    Read values of all sysfs (or similar) file nodes under ``path``, traversing
    up to the maximum depth ``depth``.
@@ -415,6 +454,10 @@ Target
    Return a list of :class:`PsEntry` instances for all running processes on the
    system.
 
+.. method:: Target.makedirs(self, path)
+
+   Create a directory at the given path and all its ancestors if needed.
+
 .. method:: Target.file_exists(self, filepath)
 
    Returns ``True`` if the specified path exists on the target and ``False``
@@ -530,15 +573,43 @@ Target
 
    :returns: ``True`` if internet seems available, ``False`` otherwise.
 
+.. method:: Target.install_module(mod, **params)
+
+  :param mod: The module name or object to be installed to the target.
+  :param params: Keyword arguments used to instantiate the module.
+
+    Installs an additional module to the target after the initial setup has been
+    performed.
+
+Linux Target
+------------
+
+.. class:: LinuxTarget(connection_settings=None, platform=None, working_directory=None, executables_directory=None, connect=True, modules=None, load_default_modules=True, shell_prompt=DEFAULT_SHELL_PROMPT, conn_cls=SshConnection, is_container=False,)
+
+    :class:`LinuxTarget` is a subclass of :class:`~devlib.target.Target`
+    with customisations specific to a device running linux.
+
+
+Local Linux Target
+------------------
+
+.. class:: LocalLinuxTarget(connection_settings=None, platform=None, working_directory=None, executables_directory=None, connect=True, modules=None, load_default_modules=True, shell_prompt=DEFAULT_SHELL_PROMPT, conn_cls=SshConnection, is_container=False,)
+
+    :class:`LocalLinuxTarget` is a subclass of
+    :class:`~devlib.target.LinuxTarget` with customisations specific to using
+    the host machine running linux as the target.
+
+
 Android Target
 ---------------
 
 .. class:: AndroidTarget(connection_settings=None, platform=None, working_directory=None, executables_directory=None, connect=True, modules=None, load_default_modules=True, shell_prompt=DEFAULT_SHELL_PROMPT, conn_cls=AdbConnection, package_data_directory="/data/data")
 
-    :class:`AndroidTarget` is a subclass of :class:`Target` with additional features specific to a device running Android.
+    :class:`AndroidTarget` is a subclass of :class:`~devlib.target.Target` with
+    additional features specific to a device running Android.
 
-    :param package_data_directory: This is the location of the data stored
+    :param package_data_directory: This is the location of the data stored for
-        for installed Android packages on the device.
+                                   installed Android packages on the device.
 
 .. method:: AndroidTarget.set_rotation(rotation)
 
@@ -611,18 +682,58 @@ Android Target
    Returns ``True`` if the targets auto brightness is currently
    enabled and ``False`` otherwise.
 
-.. method:: AndroidTarget.ensure_screen_is_off()
+.. method:: AndroidTarget.set_stay_on_never()
+
+   Sets the stay-on mode to ``0``, where the screen will turn off
+   as standard after the timeout.
+
+.. method:: AndroidTarget.set_stay_on_while_powered()
+
+   Sets the stay-on mode to ``7``, where the screen will stay on
+   while the device is charging
+
+.. method:: AndroidTarget.set_stay_on_mode(mode)
+
+   Sets the stay-on mode to the specified number between ``0`` and
+   ``7`` (inclusive).
+
+.. method:: AndroidTarget.get_stay_on_mode()
+
+   Returns an integer between ``0`` and ``7`` representing the current
+   stay-on mode of the device. 
+
+.. method:: AndroidTarget.ensure_screen_is_off(verify=True)
 
    Checks if the devices screen is on and if so turns it off.
+   If ``verify`` is set to ``True`` then a ``TargetStableError``
+   will be raise if the display cannot be turned off. E.g. if
+   always on mode is enabled.
 
-.. method:: AndroidTarget.ensure_screen_is_on()
+.. method:: AndroidTarget.ensure_screen_is_on(verify=True)
 
    Checks if the devices screen is off and if so turns it on.
+   If ``verify`` is set to ``True`` then a ``TargetStableError``
+   will be raise if the display cannot be turned on.
+
+.. method:: AndroidTarget.ensure_screen_is_on_and_stays(verify=True, mode=7)
+
+   Calls ``AndroidTarget.ensure_screen_is_on(verify)`` then additionally
+   sets the screen stay on mode to ``mode``.
 
 .. method:: AndroidTarget.is_screen_on()
 
    Returns ``True`` if the targets screen is currently on and ``False``
-   otherwise.
+   otherwise. If the display is in a "Doze" mode or similar always on state,
+   this will return ``True``.
+
+.. method:: AndroidTarget.wait_for_device(timeout=30)
+
+    Returns when the devices becomes available withing the given timeout
+    otherwise returns a ``TimeoutError``.
+
+.. method:: AndroidTarget.reboot_bootloader(timeout=30)
+
+    Attempts to reboot the target into it's bootloader.
 
 .. method:: AndroidTarget.homescreen()
 
@@ -655,9 +766,9 @@ ChromeOS Target
     :class:`ChromeOsTarget` if the device supports android otherwise only the
     :class:`LinuxTarget` methods will be available.
 
-    :param working_directory: This is the location of the working
+    :param working_directory: This is the location of the working directory to
-        directory to be used for the Linux target container. If not specified will
+        be used for the Linux target container. If not specified will default to
-        default to ``"/mnt/stateful_partition/devlib-target"``.
+        ``"/mnt/stateful_partition/devlib-target"``.
 
     :param android_working_directory: This is the location of the working
         directory to be used for the android container. If not specified it will
@@ -665,7 +776,7 @@ ChromeOS Target
 
     :param android_executables_directory: This is the location of the
         executables directory to be used for the android container. If not
-        specified will default to a ``bin`` subfolder in the
+        specified will default to a ``bin`` subdirectory in the
         ``android_working_directory.``
 
     :param package_data_directory: This is the location of the data stored

setup.py

@@ -82,24 +82,29 @@ params = dict(
         'python-dateutil',  # converting between UTC and local time.
         'pexpect>=3.3',  # Send/recieve to/from device
         'pyserial',  # Serial port interface
+        'paramiko', # SSH connection
+        'scp', # SSH connection file transfers
         'wrapt',  # Basic for construction of decorator functions
         'future', # Python 2-3 compatibility
         'enum34;python_version<"3.4"', # Enums for Python < 3.4
-        'pandas',
+        'contextlib2;python_version<"3.0"', # Python 3 contextlib backport for Python 2
-        'numpy',
+        'numpy<=1.16.4; python_version<"3"',
+        'numpy; python_version>="3"',
+        'pandas<=0.24.2; python_version<"3"',
+        'pandas; python_version>"3"',
     ],
     extras_require={
-        'daq': ['daqpower'],
+        'daq': ['daqpower>=2'],
         'doc': ['sphinx'],
         'monsoon': ['python-gflags'],
         'acme': ['pandas', 'numpy'],
     },
     # https://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers=[
-        'Development Status :: 4 - Beta',
+        'Development Status :: 5 - Production/Stable',
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: POSIX :: Linux',
-        'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
     ],
 )
 

src/get_clock_boottime/Makefile

@@ -0,0 +1,6 @@
+CFLAGS=-Wall --pedantic-errors -O2 -static
+
+all: get_clock_boottime
+
+get_clock_boottime: get_clock_boottime.c
+	$(CC) $(CFLAGS) $^ -o $@

src/get_clock_boottime/get_clock_boottime.c

@@ -0,0 +1,18 @@
+#include <stdio.h>
+#include <stdlib.h>
+#include <time.h>
+
+int main(void) {
+    int ret;
+    struct timespec tp;
+
+    ret = clock_gettime(CLOCK_BOOTTIME, &tp);
+    if (ret) {
+        perror("clock_gettime()");
+        return EXIT_FAILURE;
+    }
+
+    printf("%ld.%ld\n", tp.tv_sec, tp.tv_nsec);
+
+    return EXIT_SUCCESS;
+}