Version Bump

Move from the current implementation (piping the command to su) which
has unexpected behaviours to the '-c' su flag (which then becomes
required).

devlib/__init__.py

@@ -48,6 +48,7 @@ from devlib.derived.fps import DerivedGfxInfoStats, DerivedSurfaceFlingerStats
 from devlib.trace.ftrace import FtraceCollector
 from devlib.trace.perf import PerfCollector
 from devlib.trace.serial_trace import SerialTraceCollector
+from devlib.trace.dmesg import DmesgCollector
 
 from devlib.host import LocalConnection
 from devlib.utils.android import AdbConnection

devlib/host.py

@@ -71,7 +71,7 @@ class LocalConnection(object):
             if self.unrooted:
                 raise TargetStableError('unrooted')
             password = self._get_password()
-            command = 'echo {} | sudo -S '.format(quote(password)) + command
+            command = 'echo {} | sudo -S -- sh -c '.format(quote(password)) + quote(command)
         ignore = None if check_exit_code else 'all'
         try:
             return check_output(command, shell=True, timeout=timeout, ignore=ignore)[0]

devlib/module/sched.py

@@ -52,6 +52,12 @@ class SchedProcFSNode(object):
 
     _re_procfs_node = re.compile(r"(?P<name>.*\D)(?P<digits>\d+)$")
 
+    PACKABLE_ENTRIES = [
+        "cpu",
+        "domain",
+        "group"
+    ]
+
     @staticmethod
     def _ends_with_digits(node):
         if not isinstance(node, basestring):
@@ -71,18 +77,19 @@ class SchedProcFSNode(object):
         """
         :returns: The name of the procfs node
         """
-        return re.search(SchedProcFSNode._re_procfs_node, node).group("name")
+        match = re.search(SchedProcFSNode._re_procfs_node, node)
+        if match:
+            return match.group("name")
 
-    @staticmethod
-    def _packable(node, entries):
+        return node
+
+    @classmethod
+    def _packable(cls, node):
         """
         :returns: Whether it makes sense to pack a node into a common entry
         """
         return (SchedProcFSNode._ends_with_digits(node) and
-                any([SchedProcFSNode._ends_with_digits(x) and
-                     SchedProcFSNode._node_digits(x) != SchedProcFSNode._node_digits(node) and
-                     SchedProcFSNode._node_name(x) == SchedProcFSNode._node_name(node)
-                     for x in entries]))
+                SchedProcFSNode._node_name(node) in cls.PACKABLE_ENTRIES)
 
     @staticmethod
     def _build_directory(node_name, node_data):
@@ -119,7 +126,7 @@ class SchedProcFSNode(object):
         # Find which entries can be packed into a common entry
         packables = {
             node : SchedProcFSNode._node_name(node) + "s"
-            for node in list(nodes.keys()) if SchedProcFSNode._packable(node, list(nodes.keys()))
+            for node in list(nodes.keys()) if SchedProcFSNode._packable(node)
         }
 
         self._dyn_attrs = {}
@@ -228,13 +235,13 @@ class SchedProcFSData(SchedProcFSNode):
         # Even if we have a CPU entry, it can be empty (e.g. hotplugged out)
         # Make sure some data is there
         for cpu in cpus:
-            if target.file_exists(target.path.join(path, cpu, "domain0", "name")):
+            if target.file_exists(target.path.join(path, cpu, "domain0", "flags")):
                 return True
 
         return False
 
     def __init__(self, target, path=None):
-        if not path:
+        if path is None:
             path = self.sched_domain_root
 
         procfs = target.read_tree_values(path, depth=self._read_depth)
@@ -252,7 +259,21 @@ class SchedModule(Module):
         logger = logging.getLogger(SchedModule.name)
         SchedDomainFlag.check_version(target, logger)
 
-        return SchedProcFSData.available(target)
+        # It makes sense to load this module if at least one of those
+        # functionalities is enabled
+        schedproc = SchedProcFSData.available(target)
+        debug = SchedModule.target_has_debug(target)
+        dmips = any([target.file_exists(SchedModule.cpu_dmips_capacity_path(target, cpu))
+                     for cpu in target.list_online_cpus()])
+
+        logger.info("Scheduler sched_domain procfs entries %s",
+                    "found" if schedproc else "not found")
+        logger.info("Detected kernel compiled with SCHED_DEBUG=%s",
+                    "y" if debug else "n")
+        logger.info("CPU capacity sysfs entries %s",
+                    "found" if dmips else "not found")
+
+        return schedproc or debug or dmips
 
     def get_kernel_attributes(self, matching=None, check_exit_code=True):
         """
@@ -306,12 +327,16 @@ class SchedModule(Module):
         path = '/proc/sys/kernel/sched_' + attr
         self.target.write_value(path, value, verify)
 
+    @classmethod
+    def target_has_debug(cls, target):
+        if target.config.get('SCHED_DEBUG') != 'y':
+            return False
+        return target.file_exists('/sys/kernel/debug/sched_features')
+
     @property
     @memoized
     def has_debug(self):
-        if self.target.config.get('SCHED_DEBUG') != 'y':
-            return False;
-        return self.target.file_exists('/sys/kernel/debug/sched_features')
+        return self.target_has_debug(self.target)
 
     def get_features(self):
         """
@@ -386,17 +411,26 @@ class SchedModule(Module):
         :returns: Whether energy model data is available for 'cpu'
         """
         if not sd:
-            sd = SchedProcFSData(self.target, cpu)
+            sd = self.get_cpu_sd_info(cpu)
 
         return sd.procfs["domain0"].get("group0", {}).get("energy", {}).get("cap_states") != None
 
+    @classmethod
+    def cpu_dmips_capacity_path(cls, target, cpu):
+        """
+        :returns: The target sysfs path where the dmips capacity data should be
+        """
+        return target.path.join(
+            cls.cpu_sysfs_root,
+            'cpu{}/cpu_capacity'.format(cpu))
+
     @memoized
     def has_dmips_capacity(self, cpu):
         """
         :returns: Whether dmips capacity data is available for 'cpu'
         """
         return self.target.file_exists(
-            self.target.path.join(self.cpu_sysfs_root, 'cpu{}/cpu_capacity'.format(cpu))
+            self.cpu_dmips_capacity_path(self.target, cpu)
         )
 
     @memoized
@@ -405,10 +439,13 @@ class SchedModule(Module):
         :returns: The maximum capacity value exposed by the EAS energy model
         """
         if not sd:
-            sd = SchedProcFSData(self.target, cpu)
+            sd = self.get_cpu_sd_info(cpu)
 
         cap_states = sd.domains[0].groups[0].energy.cap_states
-        return int(cap_states.split('\t')[-2])
+        cap_states_list = cap_states.split('\t')
+        num_cap_states = sd.domains[0].groups[0].energy.nr_cap_states
+        max_cap_index = -1 * int(len(cap_states_list) / num_cap_states)
+        return int(cap_states_list[max_cap_index])
 
     @memoized
     def get_dmips_capacity(self, cpu):
@@ -416,14 +453,9 @@ class SchedModule(Module):
         :returns: The capacity value generated from the capacity-dmips-mhz DT entry
         """
         return self.target.read_value(
-            self.target.path.join(
-                self.cpu_sysfs_root,
-                'cpu{}/cpu_capacity'.format(cpu)
-            ),
-            int
+            self.cpu_dmips_capacity_path(self.target, cpu), int
         )
 
-    @memoized
     def get_capacities(self, default=None):
         """
         :param default: Default capacity value to find if no data is
@@ -434,16 +466,16 @@ class SchedModule(Module):
         :raises RuntimeError: Raised when no capacity information is
         found and 'default' is None
         """
-        cpus = list(range(self.target.number_of_cpus))
+        cpus = self.target.list_online_cpus()
 
         capacities = {}
         sd_info = self.get_sd_info()
 
         for cpu in cpus:
-            if self.has_em(cpu, sd_info.cpus[cpu]):
-                capacities[cpu] = self.get_em_capacity(cpu, sd_info.cpus[cpu])
-            elif self.has_dmips_capacity(cpu):
+            if self.has_dmips_capacity(cpu):
                 capacities[cpu] = self.get_dmips_capacity(cpu)
+            elif self.has_em(cpu, sd_info.cpus[cpu]):
+                capacities[cpu] = self.get_em_capacity(cpu, sd_info.cpus[cpu])
             else:
                 if default != None:
                     capacities[cpu] = default

devlib/module/thermal.py

@@ -88,6 +88,9 @@ class ThermalModule(Module):
 
         for entry in target.list_directory(self.thermal_root):
             re_match = re.match('^(thermal_zone|cooling_device)([0-9]+)', entry)
+            if not re_match:
+                self.logger.warning('unknown thermal entry: %s', entry)
+                continue
 
             if re_match.group(1) == 'thermal_zone':
                 self.add_thermal_zone(re_match.group(2))

devlib/target.py

@@ -155,7 +155,7 @@ class Target(object):
     def number_of_cpus(self):
         num_cpus = 0
         corere = re.compile(r'^\s*cpu\d+\s*$')
-        output = self.execute('ls /sys/devices/system/cpu')
+        output = self.execute('ls /sys/devices/system/cpu', as_root=self.is_rooted)
         for entry in output.split():
             if corere.match(entry):
                 num_cpus += 1
@@ -385,6 +385,9 @@ class Target(object):
 
     def execute(self, command, timeout=None, check_exit_code=True,
                 as_root=False, strip_colors=True, will_succeed=False):
+        # Ensure to use deployed command when availables
+        if self.executables_directory:
+            command = "PATH={}:$PATH && {}".format(self.executables_directory, command)
         return self.conn.execute(command, timeout=timeout,
                 check_exit_code=check_exit_code, as_root=as_root,
                 strip_colors=strip_colors, will_succeed=will_succeed)
@@ -2006,6 +2009,9 @@ class KernelConfig(object):
 
     This class does not provide a Mapping API and only return string values.
     """
+    @staticmethod
+    def get_config_name(name):
+        return TypedKernelConfig.get_config_name(name)
 
     def __init__(self, text):
         # Expose typed_config as a non-private attribute, so that user code
@@ -2014,7 +2020,9 @@ class KernelConfig(object):
         # Expose the original text for backward compatibility
         self.text = text
 
-    get_config_name = TypedKernelConfig.get_config_name
+    def __bool__(self):
+        return bool(self.typed_config)
+
     not_set_regex = TypedKernelConfig.not_set_regex
 
     def iteritems(self):

devlib/trace/dmesg.py

@@ -0,0 +1,198 @@
+#    Copyright 2019 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import division
+import re
+from itertools import takewhile
+from datetime import timedelta
+
+from devlib.trace import TraceCollector
+
+
+class KernelLogEntry(object):
+    """
+    Entry of the kernel ring buffer.
+
+    :param facility: facility the entry comes from
+    :type facility: str
+
+    :param level: log level
+    :type level: str
+
+    :param timestamp: Timestamp of the entry
+    :type timestamp: datetime.timedelta
+
+    :param msg: Content of the entry
+    :type msg: str
+    """
+
+    _TIMESTAMP_MSG_REGEX = re.compile(r'\[(.*?)\] (.*)')
+    _RAW_LEVEL_REGEX = re.compile(r'<([0-9]+)>(.*)')
+    _PRETTY_LEVEL_REGEX = re.compile(r'\s*([a-z]+)\s*:([a-z]+)\s*:\s*(.*)')
+
+    def __init__(self, facility, level, timestamp, msg):
+        self.facility = facility
+        self.level = level
+        self.timestamp = timestamp
+        self.msg = msg
+
+    @classmethod
+    def from_str(cls, line):
+        """
+        Parses a "dmesg --decode" output line, formatted as following:
+        kern  :err   : [3618282.310743] nouveau 0000:01:00.0: systemd-logind[988]: nv50cal_space: -16
+
+        Or the more basic output given by "dmesg -r":
+        <3>[3618282.310743] nouveau 0000:01:00.0: systemd-logind[988]: nv50cal_space: -16
+
+        """
+
+        def parse_raw_level(line):
+            match = cls._RAW_LEVEL_REGEX.match(line)
+            if not match:
+                raise ValueError('dmesg entry format not recognized: {}'.format(line))
+            level, remainder = match.groups()
+            levels = DmesgCollector.LOG_LEVELS
+            # BusyBox dmesg can output numbers that need to wrap around
+            level = levels[int(level) % len(levels)]
+            return level, remainder
+
+        def parse_pretty_level(line):
+            match = cls._PRETTY_LEVEL_REGEX.match(line)
+            facility, level, remainder = match.groups()
+            return facility, level, remainder
+
+        def parse_timestamp_msg(line):
+            match = cls._TIMESTAMP_MSG_REGEX.match(line)
+            timestamp, msg = match.groups()
+            timestamp = timedelta(seconds=float(timestamp.strip()))
+            return timestamp, msg
+
+        line = line.strip()
+
+        # If we can parse the raw prio directly, that is a basic line
+        try:
+            level, remainder = parse_raw_level(line)
+            facility = None
+        except ValueError:
+            facility, level, remainder = parse_pretty_level(line)
+
+        timestamp, msg = parse_timestamp_msg(remainder)
+
+        return cls(
+            facility=facility,
+            level=level,
+            timestamp=timestamp,
+            msg=msg.strip(),
+        )
+
+    def __str__(self):
+        facility = self.facility + ': ' if self.facility else ''
+        return '{facility}{level}: [{timestamp}] {msg}'.format(
+            facility=facility,
+            level=self.level,
+            timestamp=self.timestamp.total_seconds(),
+            msg=self.msg,
+        )
+
+
+class DmesgCollector(TraceCollector):
+    """
+    Dmesg output collector.
+
+    :param level: Minimum log level to enable. All levels that are more
+        critical will be collected as well.
+    :type level: str
+
+    :param facility: Facility to record, see dmesg --help for the list.
+    :type level: str
+
+    .. warning:: If BusyBox dmesg is used, facility and level will be ignored,
+        and the parsed entries will also lack that information.
+    """
+
+    # taken from "dmesg --help"
+    # This list needs to be ordered by priority
+    LOG_LEVELS = [
+        "emerg",        # system is unusable
+        "alert",        # action must be taken immediately
+        "crit",         # critical conditions
+        "err",          # error conditions
+        "warn",         # warning conditions
+        "notice",       # normal but significant condition
+        "info",         # informational
+        "debug",        # debug-level messages
+    ]
+
+    def __init__(self, target, level=LOG_LEVELS[-1], facility='kern'):
+        super(DmesgCollector, self).__init__(target)
+
+        if level not in self.LOG_LEVELS:
+            raise ValueError('level needs to be one of: {}'.format(
+                ', '.join(self.LOG_LEVELS)
+            ))
+        self.level = level
+
+        # Check if dmesg is the BusyBox one, or the one from util-linux in a
+        # recent version.
+        # Note: BusyBox dmesg does not support -h, but will still print the
+        # help with an exit code of 1
+        self.basic_dmesg = '--force-prefix' not in \
+                self.target.execute('dmesg -h', check_exit_code=False)
+        self.facility = facility
+        self.reset()
+
+    @property
+    def entries(self):
+        return self._parse_entries(self.dmesg_out)
+
+    @classmethod
+    def _parse_entries(cls, dmesg_out):
+        if not dmesg_out:
+            return []
+        else:
+            return [
+                KernelLogEntry.from_str(line)
+                for line in dmesg_out.splitlines()
+            ]
+
+    def reset(self):
+        self.dmesg_out = None
+
+    def start(self):
+        self.reset()
+        # Empty the dmesg ring buffer
+        self.target.execute('dmesg -c', as_root=True)
+
+    def stop(self):
+        levels_list = list(takewhile(
+            lambda level: level != self.level,
+            self.LOG_LEVELS
+        ))
+        levels_list.append(self.level)
+        if self.basic_dmesg:
+            cmd = 'dmesg -r'
+        else:
+            cmd = 'dmesg --facility={facility} --force-prefix --decode --level={levels}'.format(
+                levels=','.join(levels_list),
+                facility=self.facility,
+            )
+
+        self.dmesg_out = self.target.execute(cmd)
+
+    def get_trace(self, outfile):
+        with open(outfile, 'wt') as f:
+            f.write(self.dmesg_out + '\n')
+

devlib/trace/perf.py

@@ -104,7 +104,11 @@ class PerfCollector(TraceCollector):
             self.target.kick_off(command)
 
     def stop(self):
+        self.target.killall('perf', signal='SIGINT',
+                            as_root=self.target.is_rooted)
+        # perf doesn't transmit the signal to its sleep call so handled here:
         self.target.killall('sleep', as_root=self.target.is_rooted)
+        # NB: we hope that no other "important" sleep is on-going
 
     # pylint: disable=arguments-differ
     def get_trace(self, outdir):

devlib/utils/android.py

@@ -31,7 +31,10 @@ import pexpect
 import xml.etree.ElementTree
 import zipfile
 
-from pipes import quote
+try:
+    from shlex import quote
+except ImportError:
+    from pipes import quote
 
 from devlib.exception import TargetTransientError, TargetStableError, HostError
 from devlib.utils.misc import check_output, which, ABI_MAP
@@ -422,23 +425,23 @@ def _ping(device):
 def adb_shell(device, command, timeout=None, check_exit_code=False,
               as_root=False, adb_server=None):  # NOQA
     _check_env()
-    if as_root:
-        command = 'echo {} | su'.format(quote(command))
-    device_part = []
-    if adb_server:
-        device_part = ['-H', adb_server]
-    device_part += ['-s', device] if device else []
+    parts = ['adb']
+    if adb_server is not None:
+        parts += ['-H', adb_server]
+    if device is not None:
+        parts += ['-s', device]
+    parts += ['shell',
+              command if not as_root else 'su -c {}'.format(quote(command))]
 
+    logger.debug(' '.join(quote(part) for part in parts))
     # On older combinations of ADB/Android versions, the adb host command always
     # exits with 0 if it was able to run the command on the target, even if the
     # command failed (https://code.google.com/p/android/issues/detail?id=3254).
     # Homogenise this behaviour by running the command then echoing the exit
     # code.
-    adb_shell_command = '({}); echo \"\n$?\"'.format(command)
-    actual_command = ['adb'] + device_part + ['shell', adb_shell_command]
-    logger.debug('adb {} shell {}'.format(' '.join(device_part), command))
+    parts[-1] += ' ; echo "\n$?"'
     try:
-        raw_output, _ = check_output(actual_command, timeout, shell=False, combined_output=True)
+        raw_output, _ = check_output(parts, timeout, shell=False, combined_output=True)
     except subprocess.CalledProcessError as e:
         raise TargetStableError(str(e))
 

devlib/utils/version.py

@@ -21,7 +21,7 @@ from subprocess import Popen, PIPE
 
 VersionTuple = namedtuple('Version', ['major', 'minor', 'revision', 'dev'])
 
-version = VersionTuple(1, 1, 1, '')
+version = VersionTuple(1, 1, 2, '')
 
 
 def get_devlib_version():

setup.py

@@ -85,8 +85,10 @@ params = dict(
         'wrapt',  # Basic for construction of decorator functions
         'future', # Python 2-3 compatibility
         'enum34;python_version<"3.4"', # Enums for Python < 3.4
-        'pandas',
-        'numpy',
+        'numpy<=1.16.4; python_version<"3"',
+        'numpy; python_version>="3"',
+        'pandas<=0.24.2; python_version<"3"',
+        'pandas; python_version>"3"',
     ],
     extras_require={
         'daq': ['daqpower'],
@@ -96,10 +98,11 @@ params = dict(
     },
     # https://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers=[
-        'Development Status :: 4 - Beta',
+        'Development Status :: 5 - Production/Stable',
         'License :: OSI Approved :: Apache Software License',
         'Operating System :: POSIX :: Linux',
         'Programming Language :: Python :: 2.7',
+        'Programming Language :: Python :: 3',
     ],
 )