version: Bump revision number

Use the readme as the long description, this will be
displayed on the PyPi page.

devlib/collector/dmesg.py

@@ -20,6 +20,8 @@ from datetime import timedelta
 
 from devlib.collector import (CollectorBase, CollectorOutput,
                               CollectorOutputEntry)
+from devlib.target import KernelConfigTristate
+from devlib.exception import TargetStableError
 
 
 class KernelLogEntry(object):
@@ -152,6 +154,10 @@ class DmesgCollector(CollectorBase):
 
     def __init__(self, target, level=LOG_LEVELS[-1], facility='kern'):
         super(DmesgCollector, self).__init__(target)
+
+        if not target.is_rooted:
+            raise TargetStableError('Cannot collect dmesg on non-rooted target')
+
         self.output_path = None
 
         if level not in self.LOG_LEVELS:
@@ -167,6 +173,8 @@ class DmesgCollector(CollectorBase):
         self.basic_dmesg = '--force-prefix' not in \
                 self.target.execute('dmesg -h', check_exit_code=False)
         self.facility = facility
+        self.needs_root = bool(target.config.typed_config.get(
+            'CONFIG_SECURITY_DMESG_RESTRICT', KernelConfigTristate.NO))
         self.reset()
 
     @property
@@ -178,7 +186,7 @@ class DmesgCollector(CollectorBase):
 
     def start(self):
         self.reset()
-        # Empty the dmesg ring buffer
+        # Empty the dmesg ring buffer. This requires root in all cases
         self.target.execute('dmesg -c', as_root=True)
 
     def stop(self):
@@ -195,7 +203,7 @@ class DmesgCollector(CollectorBase):
                 facility=self.facility,
             )
 
-        self.dmesg_out = self.target.execute(cmd)
+        self.dmesg_out = self.target.execute(cmd, as_root=self.needs_root)
 
     def set_output(self, output_path):
         self.output_path = output_path

devlib/collector/perf.py

@@ -24,8 +24,9 @@ from devlib.collector import (CollectorBase, CollectorOutput,
 from devlib.utils.misc import ensure_file_directory_exists as _f
 
 
-PERF_COMMAND_TEMPLATE = '{binary} {command} {options} {events} sleep 1000 > {outfile} 2>&1 '
+PERF_STAT_COMMAND_TEMPLATE = '{binary} {command} {options} {events} {sleep_cmd} > {outfile} 2>&1 '
 PERF_REPORT_COMMAND_TEMPLATE= '{binary} report {options} -i {datafile} > {outfile} 2>&1 '
+PERF_REPORT_SAMPLE_COMMAND_TEMPLATE= '{binary} report-sample {options} -i {datafile} > {outfile} '
 PERF_RECORD_COMMAND_TEMPLATE= '{binary} record {options} {events} -o {outfile}'
 
 PERF_DEFAULT_EVENTS = [
@@ -90,12 +91,16 @@ class PerfCollector(CollectorBase):
                  events=None,
                  optionstring=None,
                  report_options=None,
+                 run_report_sample=False,
+                 report_sample_options=None,
                  labels=None,
                  force_install=False):
         super(PerfCollector, self).__init__(target)
         self.force_install = force_install
         self.labels = labels
         self.report_options = report_options
+        self.run_report_sample = run_report_sample
+        self.report_sample_options = report_sample_options
         self.output_path = None
 
         # Validate parameters
@@ -138,14 +143,17 @@ class PerfCollector(CollectorBase):
             self.target.remove(filepath)
             filepath = self._get_target_file(label, 'rpt')
             self.target.remove(filepath)
+            filepath = self._get_target_file(label, 'rptsamples')
+            self.target.remove(filepath)
 
     def start(self):
         for command in self.commands:
-            self.target.kick_off(command)
+            self.target.background(command, as_root=self.target.is_rooted)
 
     def stop(self):
         self.target.killall(self.perf_type, signal='SIGINT',
                             as_root=self.target.is_rooted)
+        if self.perf_type == "perf" and self.command == "stat":
             # perf doesn't transmit the signal to its sleep call so handled here:
             self.target.killall('sleep', as_root=self.target.is_rooted)
             # NB: we hope that no other "important" sleep is on-going
@@ -164,6 +172,9 @@ class PerfCollector(CollectorBase):
                 self._wait_for_data_file_write(label, self.output_path)
                 path = self._pull_target_file_to_host(label, 'rpt', self.output_path)
                 output.append(CollectorOutputEntry(path, 'file'))
+                if self.run_report_sample:
+                    report_samples_path = self._pull_target_file_to_host(label, 'rptsamples', self.output_path)
+                    output.append(CollectorOutputEntry(report_samples_path, 'file'))
             else:
                 path = self._pull_target_file_to_host(label, 'out', self.output_path)
                 output.append(CollectorOutputEntry(path, 'file'))
@@ -188,10 +199,12 @@ class PerfCollector(CollectorBase):
 
     def _build_perf_stat_command(self, options, events, label):
         event_string = ' '.join(['-e {}'.format(e) for e in events])
-        command = PERF_COMMAND_TEMPLATE.format(binary = self.binary,
+        sleep_cmd = 'sleep 1000' if self.perf_type == 'perf' else ''
+        command = PERF_STAT_COMMAND_TEMPLATE.format(binary = self.binary,
                                                command = self.command,
                                                options = options or '',
                                                events = event_string,
+                                               sleep_cmd = sleep_cmd,
                                                outfile = self._get_target_file(label, 'out'))
         return command
 
@@ -202,6 +215,13 @@ class PerfCollector(CollectorBase):
                                                       outfile=self._get_target_file(label, 'rpt'))
         return command
 
+    def _build_perf_report_sample_command(self, label):
+        command = PERF_REPORT_SAMPLE_COMMAND_TEMPLATE.format(binary=self.binary,
+                                                      options=self.report_sample_options or '',
+                                                      datafile=self._get_target_file(label, 'data'),
+                                                      outfile=self._get_target_file(label, 'rptsamples'))
+        return command
+
     def _build_perf_record_command(self, options, label):
         event_string = ' '.join(['-e {}'.format(e) for e in self.events])
         command = PERF_RECORD_COMMAND_TEMPLATE.format(binary=self.binary,
@@ -234,6 +254,9 @@ class PerfCollector(CollectorBase):
                 data_file_finished_writing = True
         report_command = self._build_perf_report_command(self.report_options, label)
         self.target.execute(report_command)
+        if self.run_report_sample:
+            report_sample_command = self._build_perf_report_sample_command(label)
+            self.target.execute(report_sample_command)
 
     def _validate_events(self, events):
         available_events_string = self.target.execute('{} list | {} cat'.format(self.perf_type, self.target.busybox))

devlib/connection.py

@@ -105,12 +105,20 @@ class BackgroundCommand(ABC):
         """
         self.send_signal(signal.SIGKILL)
 
-    @abstractmethod
     def cancel(self, kill_timeout=_KILL_TIMEOUT):
         """
         Try to gracefully terminate the process by sending ``SIGTERM``, then
         waiting for ``kill_timeout`` to send ``SIGKILL``.
         """
+        if self.poll() is None:
+            self._cancel(kill_timeout=kill_timeout)
+
+    @abstractmethod
+    def _cancel(self, kill_timeout):
+        """
+        Method to override in subclasses to implement :meth:`cancel`.
+        """
+        pass
 
     @abstractmethod
     def wait(self):
@@ -209,11 +217,11 @@ class PopenBackgroundCommand(BackgroundCommand):
     def poll(self):
         return self.popen.poll()
 
-    def cancel(self, kill_timeout=_KILL_TIMEOUT):
+    def _cancel(self, kill_timeout):
         popen = self.popen
         os.killpg(os.getpgid(popen.pid), signal.SIGTERM)
         try:
-            popen.wait(timeout=_KILL_TIMEOUT)
+            popen.wait(timeout=kill_timeout)
         except subprocess.TimeoutExpired:
             os.killpg(os.getpgid(popen.pid), signal.SIGKILL)
 
@@ -266,7 +274,7 @@ class ParamikoBackgroundCommand(BackgroundCommand):
         else:
             return None
 
-    def cancel(self, kill_timeout=_KILL_TIMEOUT):
+    def _cancel(self, kill_timeout):
         self.send_signal(signal.SIGTERM)
         # Check if the command terminated quickly
         time.sleep(10e-3)
@@ -340,10 +348,10 @@ class AdbBackgroundCommand(BackgroundCommand):
     def poll(self):
         return self.adb_popen.poll()
 
-    def cancel(self, kill_timeout=_KILL_TIMEOUT):
+    def _cancel(self, kill_timeout):
         self.send_signal(signal.SIGTERM)
         try:
-            self.adb_popen.wait(timeout=_KILL_TIMEOUT)
+            self.adb_popen.wait(timeout=kill_timeout)
         except subprocess.TimeoutExpired:
             self.send_signal(signal.SIGKILL)
             self.adb_popen.kill()
@@ -458,6 +466,7 @@ class PopenTransferManager(TransferManagerBase):
         if self.transfer:
             self.transfer.cancel()
             self.transfer = None
+            self.last_sample = None
 
     def isactive(self):
         size_fn = self._push_dest_size if self.direction == 'push' else self._pull_dest_size
@@ -469,6 +478,7 @@ class PopenTransferManager(TransferManagerBase):
 
     def set_transfer_and_wait(self, popen_bg_cmd):
         self.transfer = popen_bg_cmd
+        self.last_sample = None
         ret = self.transfer.wait()
 
         if ret and not self.transfer_aborted.is_set():

devlib/host.py

@@ -102,7 +102,7 @@ class LocalConnection(ConnectionBase):
             if self.unrooted:
                 raise TargetStableError('unrooted')
             password = self._get_password()
-            command = "echo {} | sudo -p ' ' -S -- sh -c {}".format(quote(password), quote(command))
+            command = "echo {} | sudo -k -p ' ' -S -- sh -c {}".format(quote(password), quote(command))
         ignore = None if check_exit_code else 'all'
         try:
             stdout, stderr = check_output(command, shell=True, timeout=timeout, ignore=ignore)
@@ -127,7 +127,7 @@ class LocalConnection(ConnectionBase):
             password = self._get_password()
             # The sudo prompt will add a space on stderr, but we cannot filter
             # it out here
-            command = "echo {} | sudo -p ' ' -S -- sh -c {}".format(quote(password), quote(command))
+            command = "echo {} | sudo -k -p ' ' -S -- sh -c {}".format(quote(password), quote(command))
 
         # Make sure to get a new PGID so PopenBackgroundCommand() can kill
         # all sub processes that could be started without troubles.

devlib/module/cpufreq.py

@@ -301,7 +301,7 @@ class CpufreqModule(Module):
         except ValueError:
             raise ValueError('Frequency must be an integer; got: "{}"'.format(frequency))
 
-    def get_frequency(self, cpu):
+    def get_frequency(self, cpu, cpuinfo=False):
         """
         Returns the current frequency currently set for the specified CPU.
 
@@ -309,12 +309,18 @@ class CpufreqModule(Module):
         try to read the current frequency and the following exception will be
         raised ::
 
+        :param cpuinfo: Read the value in the cpuinfo interface that reflects
+                        the actual running frequency.
+
         :raises: TargetStableError if for some reason the frequency could not be read.
 
         """
         if isinstance(cpu, int):
             cpu = 'cpu{}'.format(cpu)
-        sysfile = '/sys/devices/system/cpu/{}/cpufreq/scaling_cur_freq'.format(cpu)
+
+        sysfile = '/sys/devices/system/cpu/{}/cpufreq/{}'.format(
+                cpu,
+                'cpuinfo_cur_freq' if cpuinfo else 'scaling_cur_freq')
         return self.target.read_int(sysfile)
 
     def set_frequency(self, cpu, frequency, exact=True):
@@ -350,6 +356,10 @@ class CpufreqModule(Module):
                 raise TargetStableError('Can\'t set {} frequency; governor must be "userspace"'.format(cpu))
             sysfile = '/sys/devices/system/cpu/{}/cpufreq/scaling_setspeed'.format(cpu)
             self.target.write_value(sysfile, value, verify=False)
+            cpuinfo = self.get_frequency(cpu, cpuinfo=True)
+            if cpuinfo != value:
+                self.logger.warning(
+                    'The cpufreq value has not been applied properly cpuinfo={} request={}'.format(cpuinfo, value))
         except ValueError:
             raise ValueError('Frequency must be an integer; got: "{}"'.format(frequency))
 

devlib/module/hotplug.py

@@ -57,3 +57,23 @@ class HotplugModule(Module):
             return
         value = 1 if online else 0
         self.target.write_value(path, value)
+
+    def _get_path(self, path):
+        return self.target.path.join(self.base_path,
+            path)
+
+    def fail(self, cpu, state):
+        path = self._get_path('cpu{}/hotplug/fail'.format(cpu))
+        return self.target.write_value(path, state)
+
+    def get_state(self, cpu):
+        path = self._get_path('cpu{}/hotplug/state'.format(cpu))
+        return self.target.read_value(path)
+
+    def get_states(self):
+        path = self._get_path('hotplug/states')
+        states_string = self.target.read_value(path)
+        return dict(
+                map(str.strip, string.split(':', 1))
+                for string in states_string.strip().splitlines()
+        )

devlib/module/sched.py

@@ -15,7 +15,6 @@
 
 import logging
 import re
-from enum import Enum
 
 from past.builtins import basestring
 
@@ -147,43 +146,44 @@ class SchedProcFSNode(object):
             self._dyn_attrs[key] = self._build_node(key, nodes[key])
 
 
-class DocInt(int):
+class _SchedDomainFlag:
-
-    # See https://stackoverflow.com/a/50473952/5096023
-    def __new__(cls, value, doc):
-        new = super(DocInt, cls).__new__(cls, value)
-        new.__doc__ = doc
-        return new
-
-
-class SchedDomainFlag(DocInt, Enum):
     """
-    Represents a sched domain flag
+    Backward-compatible emulation of the former :class:`enum.Enum` that will
+    work on recent kernels with dynamic sched domain flags name and no value
+    exposed.
     """
-    # pylint: disable=bad-whitespace
-    # Domain flags obtained from include/linux/sched/topology.h on v4.17
-    # https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+/v4.17/include/linux/sched/topology.h#20
-    SD_LOAD_BALANCE =        0x0001, "Do load balancing on this domain"
-    SD_BALANCE_NEWIDLE =     0x0002, "Balance when about to become idle"
-    SD_BALANCE_EXEC =        0x0004, "Balance on exec"
-    SD_BALANCE_FORK =        0x0008, "Balance on fork, clone"
-    SD_BALANCE_WAKE =        0x0010, "Balance on wakeup"
-    SD_WAKE_AFFINE =         0x0020, "Wake task to waking CPU"
-    SD_ASYM_CPUCAPACITY =    0x0040, "Groups have different max cpu capacities"
-    SD_SHARE_CPUCAPACITY =   0x0080, "Domain members share cpu capacity"
-    SD_SHARE_POWERDOMAIN =   0x0100, "Domain members share power domain"
-    SD_SHARE_PKG_RESOURCES = 0x0200, "Domain members share cpu pkg resources"
-    SD_SERIALIZE =           0x0400, "Only a single load balancing instance"
-    SD_ASYM_PACKING =        0x0800, "Place busy groups earlier in the domain"
-    SD_PREFER_SIBLING =      0x1000, "Prefer to place tasks in a sibling domain"
-    SD_OVERLAP =             0x2000, "Sched_domains of this level overlap"
-    SD_NUMA =                0x4000, "Cross-node balancing"
-    # Only defined in Android
-    # https://android.googlesource.com/kernel/common/+/android-4.14/include/linux/sched/topology.h#29
-    SD_SHARE_CAP_STATES =    0x8000, "(Android only) Domain members share capacity state"
 
-    @classmethod
+    _INSTANCES = {}
-    def check_version(cls, target, logger):
+    """
+    Dictionary storing the instances so that they can be compared with ``is``
+    operator.
+    """
+
+    def __new__(cls, name, value, doc=None):
+        self = super().__new__(cls)
+        self.name = name
+        self._value = value
+        self.__doc__ = doc
+        return cls._INSTANCES.setdefault(self, self)
+
+    def __eq__(self, other):
+        # We *have to* check for "value" as well, otherwise it will be
+        # impossible to keep in the same set 2 instances with differing values.
+        return self.name == other.name and self._value == other._value
+
+    def __hash__(self):
+        return hash((self.name, self._value))
+
+    @property
+    def value(self):
+        value = self._value
+        if value is None:
+            raise AttributeError('The kernel does not expose the sched domain flag values')
+        else:
+            return value
+
+    @staticmethod
+    def check_version(target, logger):
         """
         Check the target and see if its kernel version matches our view of the world
         """
@@ -197,24 +197,111 @@ class SchedDomainFlag(DocInt, Enum):
                 "but target is running v{}".format(ref_parts, parts)
             )
 
-
     def __str__(self):
         return self.name
 
+    def __repr__(self):
+        return '<SchedDomainFlag: {}>'.format(self.name)
+
+
+class _SchedDomainFlagMeta(type):
+    """
+    Metaclass of :class:`SchedDomainFlag`.
+
+    Provides some level of emulation of :class:`enum.Enum` behavior for
+    backward compatibility.
+    """
+    @property
+    def _flags(self):
+        return [
+            attr
+            for name, attr in self.__dict__.items()
+            if name.startswith('SD_')
+        ]
+
+    def __getitem__(self, i):
+        return self._flags[i]
+
+    def __len__(self):
+        return len(self._flags)
+
+    # These would be provided by collections.abc.Sequence, but using it on a
+    # metaclass seems to have issues around __init_subclass__
+    def __iter__(self):
+        return iter(self._flags)
+
+    def __reversed__(self):
+        return reversed(self._flags)
+
+    def __contains__(self, x):
+        return x in self._flags
+
+    @property
+    def __members__(self):
+        return {flag.name: flag for flag in self._flags}
+
+
+class SchedDomainFlag(_SchedDomainFlag, metaclass=_SchedDomainFlagMeta):
+    """
+    Represents a sched domain flag.
+
+    .. note:: ``SD_*`` class attributes are deprecated, new code should never
+        test a given flag against one of these attributes with ``is`` (.e.g ``x
+        is SchedDomainFlag.SD_LOAD_BALANCE``. This is because the
+        ``SD_LOAD_BALANCE`` flag exists in two flavors that are not equal: one
+        with a value (the class attribute) and one without (dynamically created
+        when parsing flags for new kernels). Old code ran on old kernels should
+        work fine though.
+    """
+    # pylint: disable=bad-whitespace
+    # Domain flags obtained from include/linux/sched/topology.h on v4.17
+    # https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+/v4.17/include/linux/sched/topology.h#20
+    SD_LOAD_BALANCE =        _SchedDomainFlag("SD_LOAD_BALANCE", 0x0001, "Do load balancing on this domain")
+    SD_BALANCE_NEWIDLE =     _SchedDomainFlag("SD_BALANCE_NEWIDLE", 0x0002, "Balance when about to become idle")
+    SD_BALANCE_EXEC =        _SchedDomainFlag("SD_BALANCE_EXEC", 0x0004, "Balance on exec")
+    SD_BALANCE_FORK =        _SchedDomainFlag("SD_BALANCE_FORK", 0x0008, "Balance on fork, clone")
+    SD_BALANCE_WAKE =        _SchedDomainFlag("SD_BALANCE_WAKE", 0x0010, "Balance on wakeup")
+    SD_WAKE_AFFINE =         _SchedDomainFlag("SD_WAKE_AFFINE", 0x0020, "Wake task to waking CPU")
+    SD_ASYM_CPUCAPACITY =    _SchedDomainFlag("SD_ASYM_CPUCAPACITY", 0x0040, "Groups have different max cpu capacities")
+    SD_SHARE_CPUCAPACITY =   _SchedDomainFlag("SD_SHARE_CPUCAPACITY", 0x0080, "Domain members share cpu capacity")
+    SD_SHARE_POWERDOMAIN =   _SchedDomainFlag("SD_SHARE_POWERDOMAIN", 0x0100, "Domain members share power domain")
+    SD_SHARE_PKG_RESOURCES = _SchedDomainFlag("SD_SHARE_PKG_RESOURCES", 0x0200, "Domain members share cpu pkg resources")
+    SD_SERIALIZE =           _SchedDomainFlag("SD_SERIALIZE", 0x0400, "Only a single load balancing instance")
+    SD_ASYM_PACKING =        _SchedDomainFlag("SD_ASYM_PACKING", 0x0800, "Place busy groups earlier in the domain")
+    SD_PREFER_SIBLING =      _SchedDomainFlag("SD_PREFER_SIBLING", 0x1000, "Prefer to place tasks in a sibling domain")
+    SD_OVERLAP =             _SchedDomainFlag("SD_OVERLAP", 0x2000, "Sched_domains of this level overlap")
+    SD_NUMA =                _SchedDomainFlag("SD_NUMA", 0x4000, "Cross-node balancing")
+    # Only defined in Android
+    # https://android.googlesource.com/kernel/common/+/android-4.14/include/linux/sched/topology.h#29
+    SD_SHARE_CAP_STATES =    _SchedDomainFlag("SD_SHARE_CAP_STATES", 0x8000, "(Android only) Domain members share capacity state")
+
 
 class SchedDomain(SchedProcFSNode):
     """
     Represents a sched domain as seen through procfs
     """
     def __init__(self, nodes):
-        super(SchedDomain, self).__init__(nodes)
+        super().__init__(nodes)
 
-        obj_flags = set()
+        flags = self.flags
-        for flag in list(SchedDomainFlag):
+        # Recent kernels now have a space-separated list of flags instead of a
-            if self.flags & flag.value == flag.value:
+        # packed bitfield
-                obj_flags.add(flag)
+        if isinstance(flags, str):
+            flags = {
+                _SchedDomainFlag(name=name, value=None)
+                for name in flags.split()
+            }
+        else:
+            def has_flag(flags, flag):
+                return flags & flag.value == flag.value
 
-        self.flags = obj_flags
+            flags = {
+                flag
+                for flag in SchedDomainFlag
+                if has_flag(flags, flag)
+            }
+
+        self.flags = flags
 
 
 class SchedProcFSData(SchedProcFSNode):

devlib/target.py

@@ -135,6 +135,14 @@ class Target(object):
     def kernel_version(self):
         return KernelVersion(self.execute('{} uname -r -v'.format(quote(self.busybox))).strip())
 
+    @property
+    def hostid(self):
+        return int(self.execute('{} hostid'.format(self.busybox)).strip(), 16)
+
+    @property
+    def hostname(self):
+        return self.execute('{} hostname'.format(self.busybox)).strip()
+
     @property
     def os_version(self):  # pylint: disable=no-self-use
         return {}
@@ -536,10 +544,7 @@ class Target(object):
 
     # execution
 
-    def execute(self, command, timeout=None, check_exit_code=True,
+    def _prepare_cmd(self, command, force_locale):
-                as_root=False, strip_colors=True, will_succeed=False,
-                force_locale='C'):
-
         # Force the locale if necessary for more predictable output
         if force_locale:
             # Use an explicit export so that the command is allowed to be any
@@ -550,12 +555,26 @@ class Target(object):
         if self.executables_directory:
             command = "export PATH={}:$PATH && {}".format(quote(self.executables_directory), command)
 
+        return command
+
+    def execute(self, command, timeout=None, check_exit_code=True,
+                as_root=False, strip_colors=True, will_succeed=False,
+                force_locale='C'):
+
+        command = self._prepare_cmd(command, force_locale)
         return self.conn.execute(command, timeout=timeout,
                 check_exit_code=check_exit_code, as_root=as_root,
                 strip_colors=strip_colors, will_succeed=will_succeed)
 
-    def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
+    def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False,
-        return self.conn.background(command, stdout, stderr, as_root)
+                   force_locale='C', timeout=None):
+        command = self._prepare_cmd(command, force_locale)
+        bg_cmd = self.conn.background(command, stdout, stderr, as_root)
+        if timeout is not None:
+            timer = threading.Timer(timeout, function=bg_cmd.cancel)
+            timer.daemon = True
+            timer.start()
+        return bg_cmd
 
     def invoke(self, binary, args=None, in_directory=None, on_cpus=None,
                redirect_stderr=False, as_root=False, timeout=30):
@@ -1667,7 +1686,7 @@ class AndroidTarget(Target):
         self.remove(on_device_executable, as_root=self.needs_su)
 
     def dump_logcat(self, filepath, filter=None, logcat_format=None, append=False,
-                    timeout=30):  # pylint: disable=redefined-builtin
+                    timeout=60):  # pylint: disable=redefined-builtin
         op = '>>' if append else '>'
         filtstr = ' -s {}'.format(quote(filter)) if filter else ''
         formatstr = ' -v {}'.format(quote(logcat_format)) if logcat_format else ''

devlib/utils/android.py

@@ -20,18 +20,20 @@ Utility functions for working with Android devices through adb.
 """
 # pylint: disable=E1103
 import glob
-import os
-import re
-import sys
-import time
 import logging
-import tempfile
+import os
-import subprocess
-from collections import defaultdict
 import pexpect
-import xml.etree.ElementTree
+import re
-import zipfile
+import subprocess
+import sys
+import tempfile
+import time
 import uuid
+import zipfile
+
+from collections import defaultdict
+from io import StringIO
+from lxml import etree
 
 try:
     from shlex import quote
@@ -227,7 +229,10 @@ class ApkInfo(object):
                 command = [dexdump, '-l', 'xml', extracted]
                 dump = self._run(command)
 
-            xml_tree = xml.etree.ElementTree.fromstring(dump)
+            # Dexdump from build tools v30.0.X does not seem to produce
+            # valid xml from certain APKs so ignore errors and attempt to recover.
+            parser = etree.XMLParser(encoding='utf-8', recover=True)
+            xml_tree = etree.parse(StringIO(dump), parser)
 
             package = next((i for i in xml_tree.iter('package')
                            if i.attrib['name'] == self.package), None)
@@ -577,7 +582,7 @@ def adb_background_shell(conn, command,
                          stdout=subprocess.PIPE,
                          stderr=subprocess.PIPE,
                          as_root=False):
-    """Runs the sepcified command in a subprocess, returning the the Popen object."""
+    """Runs the specified command in a subprocess, returning the the Popen object."""
     device = conn.device
     adb_server = conn.adb_server
 
@@ -598,7 +603,7 @@ def adb_background_shell(conn, command,
     p = subprocess.Popen(full_command, stdout=stdout, stderr=stderr, shell=True)
 
     # Out of band PID lookup, to avoid conflicting needs with stdout redirection
-    find_pid = 'ps -A -o pid,args | grep {}'.format(quote(uuid_var))
+    find_pid = '{} ps -A -o pid,args | grep {}'.format(conn.busybox, quote(uuid_var))
     ps_out = conn.execute(find_pid)
     pids = [
         int(line.strip().split(' ', 1)[0])
@@ -734,11 +739,13 @@ def _discover_aapt(env):
         aapt2_path = ''
         versions = os.listdir(env.build_tools)
         for version in reversed(sorted(versions)):
-            if not aapt2_path and not os.path.isfile(aapt2_path):
+            if not os.path.isfile(aapt2_path):
                 aapt2_path = os.path.join(env.build_tools, version, 'aapt2')
-            if not aapt_path and not os.path.isfile(aapt_path):
+            if not os.path.isfile(aapt_path):
                 aapt_path = os.path.join(env.build_tools, version, 'aapt')
                 aapt_version = 1
+            # Use latest available version for aapt/appt2 but ensure at least one is valid.
+            if os.path.isfile(aapt2_path) or os.path.isfile(aapt_path):
                 break
 
         # Use aapt2 only if present and we have a suitable version

devlib/utils/ssh.py

@@ -59,7 +59,7 @@ from devlib.connection import (ConnectionBase, ParamikoBackgroundCommand, PopenB
                                SSHTransferManager)
 
 
-DEFAULT_SSH_SUDO_COMMAND = "sudo -p ' ' -S -- sh -c {}"
+DEFAULT_SSH_SUDO_COMMAND = "sudo -k -p ' ' -S -- sh -c {}"
 
 
 ssh = None

devlib/utils/version.py

@@ -21,7 +21,7 @@ from subprocess import Popen, PIPE
 
 VersionTuple = namedtuple('Version', ['major', 'minor', 'revision', 'dev'])
 
-version = VersionTuple(1, 3, 0, '')
+version = VersionTuple(1, 3, 1, '')
 
 
 def get_devlib_version():
@@ -33,8 +33,11 @@ def get_devlib_version():
 
 
 def get_commit():
+    try:
         p = Popen(['git', 'rev-parse', 'HEAD'], cwd=os.path.dirname(__file__),
                   stdout=PIPE, stderr=PIPE)
+    except FileNotFoundError:
+        return None
     std, _ = p.communicate()
     p.wait()
     if p.returncode:

doc/connection.rst

@@ -147,7 +147,7 @@ Connection Types
 .. class:: SshConnection(host, username, password=None, keyfile=None, port=22,\
                          timeout=None, platform=None, \
                          sudo_cmd="sudo -- sh -c {}", strict_host_check=True, \
-                         use_scp=False, poll_transfers=False,
+                         use_scp=False, poll_transfers=False, \
                          start_transfer_poll_delay=30, total_transfer_timeout=3600,\
                          transfer_poll_period=30)
 

doc/target.rst

@@ -125,10 +125,21 @@ Target
    This is a dict that contains a mapping of OS version elements to their
    values. This mapping is OS-specific.
 
+.. attribute:: Target.hostname
+
+   A string containing the hostname of the target.
+
+.. attribute:: Target.hostid
+
+   A numerical id used to represent the identity of the target.
+
+   .. note:: Currently on 64-bit PowerPC devices this id will always be 0. This is
+             due to the included busybox binary being linked with musl.
+
 .. attribute:: Target.system_id
 
    A unique identifier for the system running on the target. This identifier is
-   intended to be uninque for the combination of hardware, kernel, and file
+   intended to be unique for the combination of hardware, kernel, and file
    system.
 
 .. attribute:: Target.model
@@ -225,15 +236,16 @@ Target
    If transfer polling is supported (ADB connections and SSH connections),
    ``poll_transfers`` is set in the connection, and a timeout is not specified,
    the push will be polled for activity. Inactive transfers will be
-   cancelled. (See :ref:`connection-types`\ for more information on polling).
+   cancelled. (See :ref:`connection-types` for more information on polling).
 
    :param source: path on the host
    :param dest: path on the target
    :param as_root: whether root is required. Defaults to false.
    :param timeout: timeout (in seconds) for the transfer; if the transfer does
-       not  complete within this period, an exception will be raised.
+       not complete within this period, an exception will be raised. Leave unset
+       to utilise transfer polling if enabled.
    :param globbing: If ``True``, the ``source`` is interpreted as a globbing
-        pattern instead of being take as-is. If the pattern has mulitple
+        pattern instead of being take as-is. If the pattern has multiple
         matches, ``dest`` must be a folder (or will be created as such if it
         does not exists yet).
 
@@ -244,7 +256,7 @@ Target
    If transfer polling is supported (ADB connections and SSH connections),
    ``poll_transfers`` is set in the connection, and a timeout is not specified,
    the pull will be polled for activity. Inactive transfers will be
-   cancelled. (See :ref:`connection-types`\ for more information on polling).
+   cancelled. (See :ref:`connection-types` for more information on polling).
 
    :param source: path on the target
    :param dest: path on the host
@@ -252,7 +264,7 @@ Target
    :param timeout: timeout (in seconds) for the transfer; if the transfer does
        not  complete within this period, an exception will be raised.
    :param globbing: If ``True``, the ``source`` is interpreted as a globbing
-        pattern instead of being take as-is. If the pattern has mulitple
+        pattern instead of being take as-is. If the pattern has multiple
         matches, ``dest`` must be a folder (or will be created as such if it
         does not exists yet).
 
@@ -280,7 +292,7 @@ Target
       command to get predictable output that can be more safely parsed.
       If ``None``, no locale is prepended.
 
-.. method:: Target.background(command [, stdout [, stderr [, as_root]]])
+.. method:: Target.background(command [, stdout [, stderr [, as_root, [, force_locale [, timeout]]])
 
    Execute the command on the target, invoking it via subprocess on the host.
    This will return :class:`subprocess.Popen` instance for the command.
@@ -292,6 +304,12 @@ Target
       this may be used to redirect it to an alternative file handle.
    :param as_root: The command will be executed as root. This will fail on
        unrooted targets.
+   :param force_locale: Prepend ``LC_ALL=<force_locale>`` in front of the
+      command to get predictable output that can be more safely parsed.
+      If ``None``, no locale is prepended.
+   :param timeout: Timeout (in seconds) for the execution of the command. When
+      the timeout expires, :meth:`BackgroundCommand.cancel` is executed to
+      terminate the command.
 
    .. note:: This **will block the connection** until the command completes.
 

setup.py

@@ -69,9 +69,13 @@ for root, dirs, files in os.walk(devlib_dir):
         filepaths = [os.path.join(root, f) for f in files]
         data_files[package_name].extend([os.path.relpath(f, package_dir) for f in filepaths])
 
+with open("README.rst", "r") as fh:
+    long_description = fh.read()
+
 params = dict(
     name='devlib',
-    description='A framework for automating workload execution and measurment collection on ARM devices.',
+    description='A library for interacting with and instrumentation of remote devices.',
+    long_description=long_description,
     version=__version__,
     packages=packages,
     package_data=data_files,
@@ -92,6 +96,7 @@ params = dict(
         'numpy; python_version>="3"',
         'pandas<=0.24.2; python_version<"3"',
         'pandas; python_version>"3"',
+        'lxml', # More robust xml parsing
     ],
     extras_require={
         'daq': ['daqpower>=2'],