setup.py: add -s flag to sdist

Add -s flag to sdist command, which, when used, strips away the git
commit hash from the package version. This is needed for upload to PyPI.

README.rst

@@ -17,7 +17,7 @@ Installation
 Usage
 -----
 
-Please refer  to the "Overview" section of the documentation.
+Please refer  to the "Overview" section of the `documentation <http://devlib.readthedocs.io/en/latest/>`_.
 
 
 License

devlib/__init__.py

@@ -1,4 +1,19 @@
-from devlib.target import Target, LinuxTarget, AndroidTarget, LocalLinuxTarget
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from devlib.target import Target, LinuxTarget, AndroidTarget, LocalLinuxTarget, ChromeOsTarget
 from devlib.host import PACKAGE_BIN_DIRECTORY
 from devlib.exception import DevlibError, TargetError, HostError, TargetNotRespondingError
 
@@ -13,12 +28,31 @@ from devlib.instrument import Instrument, InstrumentChannel, Measurement, Measur
 from devlib.instrument import MEASUREMENT_TYPES, INSTANTANEOUS, CONTINUOUS
 from devlib.instrument.daq import DaqInstrument
 from devlib.instrument.energy_probe import EnergyProbeInstrument
+from devlib.instrument.arm_energy_probe import ArmEnergyProbeInstrument
+from devlib.instrument.frames import GfxInfoFramesInstrument, SurfaceFlingerFramesInstrument
 from devlib.instrument.hwmon import HwmonInstrument
 from devlib.instrument.monsoon import MonsoonInstrument
 from devlib.instrument.netstats import NetstatsInstrument
+from devlib.instrument.gem5power import Gem5PowerInstrument
+
+from devlib.derived import DerivedMeasurements, DerivedMetric
+from devlib.derived.energy import DerivedEnergyMeasurements
+from devlib.derived.fps import DerivedGfxInfoStats, DerivedSurfaceFlingerStats
 
 from devlib.trace.ftrace import FtraceCollector
+from devlib.trace.serial_trace import SerialTraceCollector
 
 from devlib.host import LocalConnection
 from devlib.utils.android import AdbConnection
 from devlib.utils.ssh import SshConnection, TelnetConnection, Gem5Connection
+
+from devlib.utils.version import get_commit as __get_commit
+
+
+__version__ = '1.0.0'
+
+__commit = __get_commit()
+if __commit:
+    __full_version__ = '{}-{}'.format(__version__, __commit)
+else:
+    __full_version__ = __version__

devlib/bin/scripts/shutils.in

@@ -47,6 +47,37 @@ cpufreq_trace_all_frequencies() {
 	done
 }
 
+################################################################################
+# DevFrequency Utility Functions
+################################################################################
+
+devfreq_set_all_frequencies() {
+	FREQ=$1
+	for DEV in /sys/class/devfreq/*; do
+		echo $FREQ > $DEV/min_freq
+		echo $FREQ > $DEV/max_freq
+	done
+}
+
+devfreq_get_all_frequencies() {
+	for DEV in /sys/class/devfreq/*; do
+		echo "`basename $DEV` `cat $DEV/cur_freq`"
+	done
+}
+
+devfreq_set_all_governors() {
+	GOV=$1
+	for DEV in /sys/class/devfreq/*; do
+		echo $GOV > $DEV/governor
+	done
+}
+
+devfreq_get_all_governors() {
+	for DEV in /sys/class/devfreq/*; do
+		echo "`basename $DEV` `cat $DEV/governor`"
+	done
+}
+
 ################################################################################
 # CPUIdle Utility Functions
 ################################################################################
@@ -124,14 +155,14 @@ cgroups_run_into() {
 
 	  # Check if the required CGroup exists
 	  $FIND $CGMOUNT -type d -mindepth 1 | \
-	  $GREP "$CGP" &>/dev/null
+	  $GREP -E "^$CGMOUNT/devlib_cgh[0-9]{1,2}$CGP" &>/dev/null
 	  if [ $? -ne 0 ]; then
 		echo "ERROR: could not find any $CGP cgroup under $CGMOUNT"
 		exit 1
 	  fi
 
 	  $FIND $CGMOUNT -type d -mindepth 1 | \
-	  $GREP "$CGP" | \
+	  $GREP -E "^$CGMOUNT/devlib_cgh[0-9]{1,2}$CGP$" | \
 	  while read CGPATH; do
 		  # Move this shell into that control group
 		  echo $$ > $CGPATH/cgroup.procs
@@ -177,6 +208,61 @@ cgroups_tasks_in() {
 	exit 0
 }
 
+cgroups_freezer_set_state() {
+    STATE=${1}
+    SYSFS_ENTRY=${2}/freezer.state
+
+    # Set the state of the freezer
+    echo $STATE > $SYSFS_ENTRY
+    
+    # And check it applied cleanly
+    for i in `seq 1 10`; do
+        [ $($CAT $SYSFS_ENTRY) = $STATE ] && exit 0
+        sleep 1
+    done
+
+    # We have an issue
+    echo "ERROR: Freezer stalled while changing state to \"$STATE\"." >&2
+    exit 1
+}
+
+################################################################################
+# Hotplug
+################################################################################
+
+hotplug_online_all() {
+    for path in /sys/devices/system/cpu/cpu[0-9]*; do
+        if [ $(cat $path/online) -eq 0 ]; then
+            echo 1 > $path/online
+        fi
+    done
+}
+
+################################################################################
+# Misc
+################################################################################
+
+read_tree_values() {
+    BASEPATH=$1
+    MAXDEPTH=$2
+
+    if [ ! -e $BASEPATH ]; then
+        echo "ERROR: $BASEPATH does not exist"
+        exit 1
+    fi
+
+    PATHS=$($BUSYBOX find $BASEPATH -follow -maxdepth $MAXDEPTH)
+    i=0
+    for path in $PATHS; do
+        i=$(expr $i + 1)
+        if [ $i -gt 1 ]; then
+            break;
+        fi
+    done
+    if [ $i -gt 1 ]; then
+        $BUSYBOX grep -s '' $PATHS
+    fi
+}
 
 ################################################################################
 # Main Function Dispatcher
@@ -198,6 +284,18 @@ cpufreq_get_all_governors)
 cpufreq_trace_all_frequencies)
     cpufreq_trace_all_frequencies $*
     ;;
+devfreq_set_all_frequencies)
+    devfreq_set_all_frequencies $*
+    ;;
+devfreq_get_all_frequencies)
+    devfreq_get_all_frequencies
+    ;;
+devfreq_set_all_governors)
+    devfreq_set_all_governors $*
+    ;;
+devfreq_get_all_governors)
+    devfreq_get_all_governors
+    ;;
 cpuidle_wake_all_cpus)
     cpuidle_wake_all_cpus $*
     ;;
@@ -213,9 +311,18 @@ cgroups_tasks_move)
 cgroups_tasks_in)
 	cgroups_tasks_in $*
 	;;
+cgroups_freezer_set_state)
+	cgroups_freezer_set_state $*
+	;;
 ftrace_get_function_stats)
     ftrace_get_function_stats
     ;;
+hotplug_online_all)
+	hotplug_online_all
+    ;;
+read_tree_values)
+	read_tree_values $*
+    ;;
 *)
     echo "Command [$CMD] not supported"
     exit -1

devlib/derived/__init__.py

@@ -0,0 +1,60 @@
+#    Copyright 2015-2017 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from devlib.instrument import MeasurementType, MEASUREMENT_TYPES
+
+
+class DerivedMetric(object):
+
+    __slots__ = ['name', 'value', 'measurement_type']
+
+    @property
+    def units(self):
+        return self.measurement_type.units
+
+    def __init__(self, name, value, measurement_type):
+        self.name = name
+        self.value = value
+        if isinstance(measurement_type, MeasurementType):
+            self.measurement_type = measurement_type
+        else:
+            try:
+                self.measurement_type = MEASUREMENT_TYPES[measurement_type]
+            except KeyError:
+                msg = 'Unknown measurement type:  {}'
+                raise ValueError(msg.format(measurement_type))
+
+    def __cmp__(self, other):
+        if hasattr(other, 'value'):
+            return cmp(self.value, other.value)
+        else:
+            return cmp(self.value, other)
+
+    def __str__(self):
+        if self.units:
+            return '{}: {} {}'.format(self.name, self.value, self.units)
+        else:
+            return '{}: {}'.format(self.name, self.value)
+
+    __repr__ = __str__
+
+
+class DerivedMeasurements(object):
+
+    def process(self, measurements_csv):
+        return []
+
+    def process_raw(self, *args):
+        return []

devlib/derived/energy.py

@@ -0,0 +1,97 @@
+#    Copyright 2013-2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from __future__ import division
+from collections import defaultdict
+
+from devlib import DerivedMeasurements, DerivedMetric
+from devlib.instrument import  MEASUREMENT_TYPES, InstrumentChannel
+
+
+class DerivedEnergyMeasurements(DerivedMeasurements):
+
+    @staticmethod
+    def process(measurements_csv):
+
+        should_calculate_energy = []
+        use_timestamp = False
+
+        # Determine sites to calculate energy for
+        channel_map = defaultdict(list)
+        for channel in measurements_csv.channels:
+            channel_map[channel.site].append(channel.kind)
+            if channel.site == 'timestamp':
+                use_timestamp = True
+                time_measurment = channel.measurement_type
+        for site, kinds in channel_map.items():
+            if 'power' in kinds and not 'energy' in kinds:
+                should_calculate_energy.append(site)
+
+        if measurements_csv.sample_rate_hz is None and not use_timestamp:
+            msg = 'Timestamp data is unavailable, please provide a sample rate'
+            raise ValueError(msg)
+
+        if use_timestamp:
+            # Find index of timestamp column
+            ts_index = [i for i, chan in enumerate(measurements_csv.channels)
+                        if chan.site == 'timestamp']
+            if len(ts_index) > 1:
+                raise ValueError('Multiple timestamps detected')
+            ts_index = ts_index[0]
+
+        row_ts = 0
+        last_ts = 0
+        energy_results = defaultdict(dict)
+        power_results = defaultdict(float)
+
+        # Process data
+        for count, row in enumerate(measurements_csv.iter_measurements()):
+            if use_timestamp:
+                last_ts = row_ts
+                row_ts = time_measurment.convert(float(row[ts_index].value), 'time')
+            for entry in row:
+                channel = entry.channel
+                site = channel.site
+                if channel.kind == 'energy':
+                    if count == 0:
+                        energy_results[site]['start'] = entry.value
+                    else:
+                        energy_results[site]['end'] = entry.value
+
+                if channel.kind == 'power':
+                    power_results[site] += entry.value
+
+                    if site in should_calculate_energy:
+                        if count == 0:
+                            energy_results[site]['start'] = 0
+                            energy_results[site]['end'] = 0
+                        elif use_timestamp:
+                            energy_results[site]['end'] += entry.value * (row_ts - last_ts)
+                        else:
+                            energy_results[site]['end'] += entry.value * (1 /
+                                                           measurements_csv.sample_rate_hz)
+
+        # Calculate final measurements
+        derived_measurements = []
+        for site in energy_results:
+            total_energy = energy_results[site]['end'] - energy_results[site]['start']
+            name = '{}_total_energy'.format(site)
+            derived_measurements.append(DerivedMetric(name, total_energy, MEASUREMENT_TYPES['energy']))
+
+        for site in power_results:
+            power = power_results[site] / (count + 1)  #pylint: disable=undefined-loop-variable
+            name = '{}_average_power'.format(site)
+            derived_measurements.append(DerivedMetric(name, power, MEASUREMENT_TYPES['power']))
+
+        return derived_measurements

devlib/derived/fps.py

@@ -0,0 +1,232 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import division
+import os
+import re
+
+try:
+    import pandas as pd
+except ImportError:
+    pd = None
+
+from past.builtins import basestring
+
+from devlib import DerivedMeasurements, DerivedMetric, MeasurementsCsv, InstrumentChannel
+from devlib.exception import HostError
+from devlib.utils.csvutil import csvwriter
+from devlib.utils.rendering import gfxinfo_get_last_dump, VSYNC_INTERVAL
+from devlib.utils.types import numeric
+
+
+class DerivedFpsStats(DerivedMeasurements):
+
+    def __init__(self, drop_threshold=5, suffix=None, filename=None, outdir=None):
+        self.drop_threshold = drop_threshold
+        self.suffix = suffix
+        self.filename = filename
+        self.outdir = outdir
+        if (filename is None) and (suffix is None):
+            self.suffix = '-fps'
+        elif (filename is not None) and (suffix is not None):
+            raise ValueError('suffix and filename cannot be specified at the same time.')
+        if filename is not None and os.sep in filename:
+            raise ValueError('filename cannot be a path (cannot countain "{}"'.format(os.sep))
+
+    def process(self, measurements_csv):
+        if isinstance(measurements_csv, basestring):
+            measurements_csv = MeasurementsCsv(measurements_csv)
+        if pd is not None:
+            return self._process_with_pandas(measurements_csv)
+        return self._process_without_pandas(measurements_csv)
+
+    def _get_csv_file_name(self, frames_file):
+        outdir = self.outdir or os.path.dirname(frames_file)
+        if self.filename:
+            return os.path.join(outdir, self.filename)
+
+        frames_basename = os.path.basename(frames_file)
+        rest, ext = os.path.splitext(frames_basename)
+        csv_basename = rest + self.suffix + ext
+        return os.path.join(outdir, csv_basename)
+
+
+class DerivedGfxInfoStats(DerivedFpsStats):
+
+    @staticmethod
+    def process_raw(filepath, *args):
+        metrics = []
+        dump = gfxinfo_get_last_dump(filepath)
+        seen_stats = False
+        for line in dump.split('\n'):
+            if seen_stats and not line.strip():
+                break
+            elif line.startswith('Janky frames:'):
+                text = line.split(': ')[-1]
+                val_text, pc_text = text.split('(')
+                metrics.append(DerivedMetric('janks', numeric(val_text.strip()), 'count'))
+                metrics.append(DerivedMetric('janks_pc', numeric(pc_text[:-3]), 'percent'))
+            elif ' percentile: ' in line:
+                ptile, val_text = line.split(' percentile: ')
+                name = 'render_time_{}_ptile'.format(ptile)
+                value = numeric(val_text.strip()[:-2])
+                metrics.append(DerivedMetric(name, value, 'time_ms'))
+            elif line.startswith('Number '):
+                name_text, val_text = line.strip().split(': ')
+                name = name_text[7:].lower().replace(' ', '_')
+                value = numeric(val_text)
+                metrics.append(DerivedMetric(name, value, 'count'))
+            else:
+                continue
+            seen_stats = True
+        return metrics
+
+    def _process_without_pandas(self, measurements_csv):
+        per_frame_fps = []
+        start_vsync, end_vsync = None, None
+        frame_count = 0
+
+        for frame_data in measurements_csv.iter_values():
+            if frame_data.Flags_flags != 0:
+                continue
+            frame_count += 1
+
+            if start_vsync is None:
+                start_vsync = frame_data.Vsync_time_us
+            end_vsync = frame_data.Vsync_time_us
+
+            frame_time = frame_data.FrameCompleted_time_us - frame_data.IntendedVsync_time_us
+            pff = 1e9 / frame_time
+            if pff > self.drop_threshold:
+                per_frame_fps.append([pff])
+
+        if frame_count:
+            duration = end_vsync - start_vsync
+            fps = (1e6 * frame_count) / float(duration)
+        else:
+            duration = 0
+            fps = 0
+
+        csv_file = self._get_csv_file_name(measurements_csv.path)
+        with csvwriter(csv_file) as writer:
+            writer.writerow(['fps'])
+            writer.writerows(per_frame_fps)
+
+        return [DerivedMetric('fps', fps, 'fps'),
+                DerivedMetric('total_frames', frame_count, 'frames'),
+                MeasurementsCsv(csv_file)]
+
+    def _process_with_pandas(self, measurements_csv):
+        data = pd.read_csv(measurements_csv.path)
+        data = data[data.Flags_flags == 0]
+        frame_time = data.FrameCompleted_time_us - data.IntendedVsync_time_us
+        per_frame_fps = (1e6 / frame_time)
+        keep_filter = per_frame_fps > self.drop_threshold
+        per_frame_fps = per_frame_fps[keep_filter]
+        per_frame_fps.name = 'fps'
+
+        frame_count = data.index.size
+        if frame_count > 1:
+            duration = data.Vsync_time_us.iloc[-1] - data.Vsync_time_us.iloc[0]
+            fps = (1e9 * frame_count) / float(duration)
+        else:
+            duration = 0
+            fps = 0
+
+        csv_file = self._get_csv_file_name(measurements_csv.path)
+        per_frame_fps.to_csv(csv_file, index=False, header=True)
+
+        return [DerivedMetric('fps', fps, 'fps'),
+                DerivedMetric('total_frames', frame_count, 'frames'),
+                MeasurementsCsv(csv_file)]
+
+
+class DerivedSurfaceFlingerStats(DerivedFpsStats):
+
+    def _process_with_pandas(self, measurements_csv):
+        data = pd.read_csv(measurements_csv.path)
+
+        # fiter out bogus frames.
+        bogus_frames_filter = data.actual_present_time_us != 0x7fffffffffffffff
+        actual_present_times = data.actual_present_time_us[bogus_frames_filter]
+        actual_present_time_deltas = actual_present_times.diff().dropna()
+
+        vsyncs_to_compose = actual_present_time_deltas.div(VSYNC_INTERVAL)
+        vsyncs_to_compose.apply(lambda x: int(round(x, 0)))
+
+        # drop values lower than drop_threshold FPS as real in-game frame
+        # rate is unlikely to drop below that (except on loading screens
+        # etc, which should not be factored in frame rate calculation).
+        per_frame_fps = (1.0 / (vsyncs_to_compose.multiply(VSYNC_INTERVAL / 1e9)))
+        keep_filter = per_frame_fps > self.drop_threshold
+        filtered_vsyncs_to_compose = vsyncs_to_compose[keep_filter]
+        per_frame_fps.name = 'fps'
+
+        csv_file = self._get_csv_file_name(measurements_csv.path)
+        per_frame_fps.to_csv(csv_file, index=False, header=True)
+
+        if not filtered_vsyncs_to_compose.empty:
+            fps = 0
+            total_vsyncs = filtered_vsyncs_to_compose.sum()
+            frame_count = filtered_vsyncs_to_compose.size
+
+            if total_vsyncs:
+                fps = 1e9 * frame_count / (VSYNC_INTERVAL * total_vsyncs)
+
+            janks = self._calc_janks(filtered_vsyncs_to_compose)
+            not_at_vsync = self._calc_not_at_vsync(vsyncs_to_compose)
+        else:
+            fps = 0
+            frame_count = 0
+            janks = 0
+            not_at_vsync = 0
+
+        janks_pc =  0 if frame_count == 0 else janks * 100 / frame_count
+
+        return [DerivedMetric('fps', fps, 'fps'),
+                DerivedMetric('total_frames', frame_count, 'frames'),
+                MeasurementsCsv(csv_file),
+                DerivedMetric('janks', janks, 'count'),
+                DerivedMetric('janks_pc', janks_pc, 'percent'),
+                DerivedMetric('missed_vsync', not_at_vsync, 'count')]
+
+    def _process_without_pandas(self, measurements_csv):
+        # Given that SurfaceFlinger has been deprecated in favor of GfxInfo,
+        # it does not seem worth it implementing this.
+        raise HostError('Please install "pandas" Python package to process SurfaceFlinger frames')
+
+    @staticmethod
+    def _calc_janks(filtered_vsyncs_to_compose):
+        """
+        Internal method for calculating jank frames.
+        """
+        pause_latency = 20
+        vtc_deltas = filtered_vsyncs_to_compose.diff().dropna()
+        vtc_deltas = vtc_deltas.abs()
+        janks = vtc_deltas.apply(lambda x: (pause_latency > x > 1.5) and 1 or 0).sum()
+
+        return janks
+
+    @staticmethod
+    def _calc_not_at_vsync(vsyncs_to_compose):
+        """
+        Internal method for calculating the number of frames that did not
+        render in a single vsync cycle.
+        """
+        epsilon = 0.0001
+        func = lambda x: (abs(x - 1.0) > epsilon) and 1 or 0
+        not_at_vsync = vsyncs_to_compose.apply(func).sum()
+
+        return not_at_vsync

devlib/exception.py

@@ -1,4 +1,4 @@
-#    Copyright 2013-2015 ARM Limited
+#    Copyright 2013-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,10 +13,13 @@
 # limitations under the License.
 #
 
-
 class DevlibError(Exception):
     """Base class for all Devlib exceptions."""
-    pass
+    @property
+    def message(self):
+        if self.args:
+            return self.args[0]
+        return str(self)
 
 
 class TargetError(DevlibError):
@@ -26,9 +29,7 @@ class TargetError(DevlibError):
 
 class TargetNotRespondingError(DevlibError):
     """The target is unresponsive."""
-
+    pass
-    def __init__(self, target):
-        super(TargetNotRespondingError, self).__init__('Target {} is not responding.'.format(target))
 
 
 class HostError(DevlibError):
@@ -49,3 +50,42 @@ class TimeoutError(DevlibError):
 
     def __str__(self):
         return '\n'.join([self.message, 'OUTPUT:', self.output or ''])
+
+
+class WorkerThreadError(DevlibError):
+    """
+    This should get raised  in the main thread if a non-WAError-derived
+    exception occurs on a worker/background thread. If a WAError-derived
+    exception is raised in the worker, then it that exception should be
+    re-raised on the main thread directly -- the main point of this is to
+    preserve the backtrace in the output, and backtrace doesn't get output for
+    WAErrors.
+
+    """
+
+    def __init__(self, thread, exc_info):
+        self.thread = thread
+        self.exc_info = exc_info
+        orig = self.exc_info[1]
+        orig_name = type(orig).__name__
+        message = 'Exception of type {} occured on thread {}:\n'.format(orig_name, thread)
+        message += '{}\n{}: {}'.format(get_traceback(self.exc_info), orig_name, orig)
+        super(WorkerThreadError, self).__init__(message)
+
+
+def get_traceback(exc=None):
+    """
+    Returns the string with the traceback for the specifiec exc
+    object, or for the current exception exc is not specified.
+
+    """
+    import io, traceback, sys
+    if exc is None:
+        exc = sys.exc_info()
+    if not exc:
+        return None
+    tb = exc[2]
+    sio = io.BytesIO()
+    traceback.print_tb(tb, file=sio)
+    del tb  # needs to be done explicitly see: http://docs.python.org/2/library/sys.html#sys.exc_info
+    return sio.getvalue()

devlib/host.py

@@ -1,4 +1,4 @@
-#    Copyright 2015 ARM Limited
+#    Copyright 2015-2017 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 #
 from glob import iglob
 import os
+import signal
 import shutil
 import subprocess
 import logging
@@ -24,6 +25,11 @@ from devlib.utils.misc import check_output
 
 PACKAGE_BIN_DIRECTORY = os.path.join(os.path.dirname(__file__), 'bin')
 
+def kill_children(pid, signal=signal.SIGKILL):
+    with open('/proc/{0}/task/{0}/children'.format(pid), 'r') as fd:
+        for cpid in map(int, fd.read().strip().split()):
+            kill_children(cpid, signal)
+            os.kill(cpid, signal)
 
 class LocalConnection(object):
 

devlib/instrument/__init__.py

@@ -1,4 +1,4 @@
-#    Copyright 2015 ARM Limited
+#    Copyright 2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,11 +12,15 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-import csv
+from __future__ import division
 import logging
 import collections
 
+from past.builtins import basestring
+
+from devlib.utils.csvutil import csvreader
 from devlib.utils.types import numeric
+from devlib.utils.types import identifier
 
 
 # Channel modes describe what sort of measurement the instrument supports.
@@ -24,28 +28,35 @@ from devlib.utils.types import numeric
 INSTANTANEOUS = 1
 CONTINUOUS = 2
 
+MEASUREMENT_TYPES = {}  # populated further down
 
-class MeasurementType(tuple):
 
-    __slots__ = []
+class MeasurementType(object):
 
-    def __new__(cls, name, units, category=None):
+    def __init__(self, name, units, category=None, conversions=None):
-        return tuple.__new__(cls, (name, units, category))
+        self.name = name
+        self.units = units
+        self.category = category
+        self.conversions = {}
+        if conversions is not None:
+            for key, value in conversions.items():
+                if not callable(value):
+                    msg = 'Converter must be callable; got {} "{}"'
+                    raise ValueError(msg.format(type(value), value))
+                self.conversions[key] = value
 
-    @property
+    def convert(self, value, to):
-    def name(self):
+        if isinstance(to, basestring) and to in MEASUREMENT_TYPES:
-        return tuple.__getitem__(self, 0)
+            to = MEASUREMENT_TYPES[to]
-
+        if not isinstance(to, MeasurementType):
-    @property
+            msg = 'Unexpected conversion target: "{}"'
-    def units(self):
+            raise ValueError(msg.format(to))
-        return tuple.__getitem__(self, 1)
+        if to.name == self.name:
-
+            return value
-    @property
+        if not to.name in self.conversions:
-    def category(self):
+            msg = 'No conversion from {} to {} available'
-        return tuple.__getitem__(self, 2)
+            raise ValueError(msg.format(self.name, to.name))
-
+        return self.conversions[to.name](value)
-    def __getitem__(self, item):
-        raise TypeError()
 
     def __cmp__(self, other):
         if isinstance(other, MeasurementType):
@@ -55,24 +66,73 @@ class MeasurementType(tuple):
     def __str__(self):
         return self.name
 
-    __repr__ = __str__
+    def __repr__(self):
+        if self.category:
+            text = 'MeasurementType({}, {}, {})'
+            return text.format(self.name, self.units, self.category)
+        else:
+            text = 'MeasurementType({}, {})'
+            return text.format(self.name, self.units)
 
 
-# Standard measures
+# Standard measures. In order to make sure that downstream data processing is not tied
+# to particular insturments (e.g. a particular method of mearuing power), instruments
+# must, where possible, resport their measurments formatted as on of the standard types
+# defined here.
 _measurement_types = [
-    MeasurementType('time', 'seconds'),
+    # For whatever reason, the type of measurement could not be established.
-    MeasurementType('temperature', 'degrees'),
+    MeasurementType('unknown', None),
 
+    # Generic measurements
+    MeasurementType('count', 'count'),
+    MeasurementType('percent', 'percent'),
+
+    # Time measurement. While there is typically a single "canonical" unit
+    # used for each type of measurmenent, time may be measured to a wide variety
+    # of events occuring at a wide range of scales. Forcing everying into a
+    # single scale will lead to inefficient and awkward to work with result tables.
+    # Coversion functions between the formats are specified, so that downstream
+    # processors that expect all times time be at a particular scale can automatically
+    # covert without being familar with individual instruments.
+    MeasurementType('time', 'seconds', 'time',
+        conversions={
+            'time_us': lambda x: x * 1000000,
+            'time_ms': lambda x: x * 1000,
+        }
+    ),
+    MeasurementType('time_us', 'microseconds', 'time',
+        conversions={
+            'time': lambda x: x / 1000000,
+            'time_ms': lambda x: x / 1000,
+        }
+    ),
+    MeasurementType('time_ms', 'milliseconds', 'time',
+        conversions={
+            'time': lambda x: x / 1000,
+            'time_us': lambda x: x * 1000,
+        }
+    ),
+
+    # Measurements related to thermals.
+    MeasurementType('temperature', 'degrees', 'thermal'),
+
+    # Measurements related to power end energy consumption.
     MeasurementType('power', 'watts', 'power/energy'),
     MeasurementType('voltage', 'volts', 'power/energy'),
     MeasurementType('current', 'amps', 'power/energy'),
     MeasurementType('energy', 'joules', 'power/energy'),
 
+    # Measurments realted to data transfer, e.g. neworking,
+    # memory, or backing storage.
     MeasurementType('tx', 'bytes', 'data transfer'),
     MeasurementType('rx', 'bytes', 'data transfer'),
     MeasurementType('tx/rx', 'bytes', 'data transfer'),
+
+    MeasurementType('fps', 'fps', 'ui render'),
+    MeasurementType('frames', 'frames', 'ui render'),
 ]
-MEASUREMENT_TYPES = {m.name: m for m in _measurement_types}
+for m in _measurement_types:
+    MEASUREMENT_TYPES[m.name] = m
 
 
 class Measurement(object):
@@ -92,7 +152,7 @@ class Measurement(object):
         self.channel = channel
 
     def __cmp__(self, other):
-        if isinstance(other, Measurement):
+        if hasattr(other, 'value'):
             return cmp(self.value, other.value)
         else:
             return cmp(self.value, other)
@@ -108,28 +168,72 @@ class Measurement(object):
 
 class MeasurementsCsv(object):
 
-    def __init__(self, path, channels):
+    def __init__(self, path, channels=None, sample_rate_hz=None):
         self.path = path
         self.channels = channels
-        self._fh = open(path, 'rb')
+        self.sample_rate_hz = sample_rate_hz
+        if self.channels is None:
+            self._load_channels()
+        headings = [chan.label for chan in self.channels]
+        self.data_tuple = collections.namedtuple('csv_entry',
+                                                 map(identifier, headings))
 
     def measurements(self):
-        return list(self.itermeasurements())
+        return list(self.iter_measurements())
 
-    def itermeasurements(self):
+    def iter_measurements(self):
-        self._fh.seek(0)
+        for row in self._iter_rows():
-        reader = csv.reader(self._fh)
-        reader.next()  # headings
-        for row in reader:
             values = map(numeric, row)
             yield [Measurement(v, c) for (v, c) in zip(values, self.channels)]
 
+    def values(self):
+        return list(self.iter_values())
+
+    def iter_values(self):
+        for row in self._iter_rows():
+            values = list(map(numeric, row))
+            yield self.data_tuple(*values)
+
+    def _load_channels(self):
+        header = []
+        with csvreader(self.path) as reader:
+            header = next(reader)
+
+        self.channels = []
+        for entry in header:
+            for mt in MEASUREMENT_TYPES:
+                suffix = '_{}'.format(mt)
+                if entry.endswith(suffix):
+                    site =  entry[:-len(suffix)]
+                    measure = mt
+                    break
+            else:
+                if entry in MEASUREMENT_TYPES:
+                    site = None
+                    measure = entry
+                else:
+                    site = entry
+                    measure = 'unknown'
+
+            chan = InstrumentChannel(site, measure)
+            self.channels.append(chan)
+
+    def _iter_rows(self):
+        with csvreader(self.path) as reader:
+            next(reader)  # headings
+            for row in reader:
+                yield row
+
 
 class InstrumentChannel(object):
 
     @property
     def label(self):
+        if self.site is not None:
             return '{}_{}'.format(self.site, self.kind)
+        return self.kind
+
+    name = label
 
     @property
     def kind(self):
@@ -139,8 +243,7 @@ class InstrumentChannel(object):
     def units(self):
         return self.measurement_type.units
 
-    def __init__(self, name, site, measurement_type, **attrs):
+    def __init__(self, site, measurement_type, **attrs):
-        self.name = name
         self.site = site
         if isinstance(measurement_type, MeasurementType):
             self.measurement_type = measurement_type
@@ -149,7 +252,7 @@ class InstrumentChannel(object):
                 self.measurement_type = MEASUREMENT_TYPES[measurement_type]
             except KeyError:
                 raise ValueError('Unknown measurement type:  {}'.format(measurement_type))
-        for atname, atvalue in attrs.iteritems():
+        for atname, atvalue in attrs.items():
             setattr(self, atname, atvalue)
 
     def __str__(self):
@@ -175,17 +278,15 @@ class Instrument(object):
     # channel management
 
     def list_channels(self):
-        return self.channels.values()
+        return list(self.channels.values())
 
     def get_channels(self, measure):
         if hasattr(measure, 'name'):
             measure = measure.name
         return [c for c in self.list_channels() if c.kind == measure]
 
-    def add_channel(self, site, measure, name=None, **attrs):
+    def add_channel(self, site, measure, **attrs):
-        if name is None:
+        chan = InstrumentChannel(site, measure, **attrs)
-            name = '{}_{}'.format(site, measure)
-        chan = InstrumentChannel(name, site, measure, **attrs)
         self.channels[chan.label] = chan
 
     # initialization and teardown
@@ -197,24 +298,26 @@ class Instrument(object):
         pass
 
     def reset(self, sites=None, kinds=None, channels=None):
-        if kinds is None and sites is None and channels is None:
+        if channels is not None:
-            self.active_channels = sorted(self.channels.values(), key=lambda x: x.label)
+            if sites is not None or kinds is not None:
+                raise ValueError('sites and kinds should not be set if channels is set')
+
+            try:
+                self.active_channels = [self.channels[ch] for ch in channels]
+            except KeyError as e:
+                msg = 'Unexpected channel "{}"; must be in {}'
+                raise ValueError(msg.format(e, self.channels.keys()))
+        elif sites is None and kinds is None:
+            self.active_channels = sorted(self.channels.itervalues(), key=lambda x: x.label)
         else:
             if isinstance(sites, basestring):
                 sites = [sites]
             if isinstance(kinds, basestring):
                 kinds = [kinds]
-            self.active_channels = []
+
-            for chan_name in (channels or []):
+            wanted = lambda ch : ((kinds is None or ch.kind in kinds) and
-                try:
+                                  (sites is None or ch.site in sites))
-                    self.active_channels.append(self.channels[chan_name])
+            self.active_channels = filter(wanted, self.channels.itervalues())
-                except KeyError:
-                    msg = 'Unexpected channel "{}"; must be in {}'
-                    raise ValueError(msg.format(chan_name, self.channels.keys()))
-            for chan in self.channels.values():
-                if (kinds is None or chan.kind in kinds) and \
-                   (sites is None or chan.site in sites):
-                    self.active_channels.append(chan)
 
     # instantaneous
 
@@ -231,3 +334,6 @@ class Instrument(object):
 
     def get_data(self, outfile):
         pass
+
+    def get_raw(self):
+        return []

devlib/instrument/acmecape.py

@@ -0,0 +1,157 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#pylint: disable=attribute-defined-outside-init
+from __future__ import division
+import os
+import sys
+import time
+import tempfile
+from fcntl import fcntl, F_GETFL, F_SETFL
+from string import Template
+from subprocess import Popen, PIPE, STDOUT
+
+from devlib import Instrument, CONTINUOUS, MeasurementsCsv
+from devlib.exception import HostError
+from devlib.utils.csvutil import csvreader, csvwriter
+from devlib.utils.misc import which
+
+OUTPUT_CAPTURE_FILE = 'acme-cape.csv'
+IIOCAP_CMD_TEMPLATE = Template("""
+${iio_capture} -n ${host} -b ${buffer_size} -c -f ${outfile} ${iio_device}
+""")
+
+def _read_nonblock(pipe, size=1024):
+    fd = pipe.fileno()
+    flags = fcntl(fd, F_GETFL)
+    flags |= os.O_NONBLOCK
+    fcntl(fd, F_SETFL, flags)
+
+    output = ''
+    try:
+        while True:
+            output += pipe.read(size)
+    except IOError:
+        pass
+    return output
+
+
+class AcmeCapeInstrument(Instrument):
+
+    mode = CONTINUOUS
+
+    def __init__(self, target,
+                 iio_capture=which('iio-capture'),
+                 host='baylibre-acme.local',
+                 iio_device='iio:device0',
+                 buffer_size=256):
+        super(AcmeCapeInstrument, self).__init__(target)
+        self.iio_capture = iio_capture
+        self.host = host
+        self.iio_device = iio_device
+        self.buffer_size = buffer_size
+        self.sample_rate_hz = 100
+        if self.iio_capture is None:
+            raise HostError('Missing iio-capture binary')
+        self.command = None
+        self.process = None
+
+        self.add_channel('shunt', 'voltage')
+        self.add_channel('bus', 'voltage')
+        self.add_channel('device', 'power')
+        self.add_channel('device', 'current')
+        self.add_channel('timestamp', 'time_ms')
+
+    def __del__(self):
+        if self.process and self.process.pid:
+            self.logger.warning('killing iio-capture process [{}]...'.format(self.process.pid))
+            self.process.kill()
+
+    def reset(self, sites=None, kinds=None, channels=None):
+        super(AcmeCapeInstrument, self).reset(sites, kinds, channels)
+        self.raw_data_file = tempfile.mkstemp('.csv')[1]
+        params = dict(
+            iio_capture=self.iio_capture,
+            host=self.host,
+            buffer_size=self.buffer_size,
+            iio_device=self.iio_device,
+            outfile=self.raw_data_file
+        )
+        self.command = IIOCAP_CMD_TEMPLATE.substitute(**params)
+        self.logger.debug('ACME cape command: {}'.format(self.command))
+
+    def start(self):
+        self.process = Popen(self.command.split(), stdout=PIPE, stderr=STDOUT)
+
+    def stop(self):
+        self.process.terminate()
+        timeout_secs = 10
+        output = ''
+        for _ in range(timeout_secs):
+            if self.process.poll() is not None:
+                break
+            time.sleep(1)
+        else:
+            output += _read_nonblock(self.process.stdout)
+            self.process.kill()
+            self.logger.error('iio-capture did not terminate gracefully')
+            if self.process.poll() is None:
+                msg = 'Could not terminate iio-capture:\n{}'
+                raise HostError(msg.format(output))
+        if self.process.returncode != 15: # iio-capture exits with 15 when killed
+            if sys.version_info[0] == 3:
+                output += self.process.stdout.read().decode(sys.stdout.encoding, 'replace')
+            else:
+                output += self.process.stdout.read()
+            self.logger.info('ACME instrument encountered an error, '
+                             'you may want to try rebooting the ACME device:\n'
+                             '  ssh root@{} reboot'.format(self.host))
+            raise HostError('iio-capture exited with an error ({}), output:\n{}'
+                            .format(self.process.returncode, output))
+        if not os.path.isfile(self.raw_data_file):
+            raise HostError('Output CSV not generated.')
+        self.process = None
+
+    def get_data(self, outfile):
+        if os.stat(self.raw_data_file).st_size == 0:
+            self.logger.warning('"{}" appears to be empty'.format(self.raw_data_file))
+            return
+
+        all_channels = [c.label for c in self.list_channels()]
+        active_channels = [c.label for c in self.active_channels]
+        active_indexes = [all_channels.index(ac) for ac in active_channels]
+
+        with csvreader(self.raw_data_file, skipinitialspace=True) as reader:
+            with csvwriter(outfile) as writer:
+                writer.writerow(active_channels)
+
+                header = next(reader)
+                ts_index = header.index('timestamp ms')
+
+
+                for row in reader:
+                    output_row = []
+                    for i in active_indexes:
+                        if i == ts_index:
+                            # Leave time in ms
+                            output_row.append(float(row[i]))
+                        else:
+                            # Convert rest into standard units.
+                            output_row.append(float(row[i])/1000)
+                    writer.writerow(output_row)
+        return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
+
+    def get_raw(self):
+        return [self.raw_data_file]

devlib/instrument/arm_energy_probe.py

@@ -0,0 +1,145 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#    Copyright 2018 Linaro Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+# pylint: disable=W0613,E1101,access-member-before-definition,attribute-defined-outside-init
+from __future__ import division
+import os
+import subprocess
+import signal
+import struct
+import sys
+
+import tempfile
+import shutil
+
+from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv
+from devlib.exception import HostError
+from devlib.utils.csvutil import csvreader, csvwriter
+from devlib.utils.misc import which
+
+from devlib.utils.parse_aep import AepParser
+
+class ArmEnergyProbeInstrument(Instrument):
+    """
+    Collects power traces using the ARM Energy Probe.
+
+    This instrument requires ``arm-probe`` utility to be installed on the host and be in the PATH.
+    arm-probe is available here:
+    ``https://git.linaro.org/tools/arm-probe.git``.
+
+    Details about how to build and use it is available here:
+    ``https://git.linaro.org/tools/arm-probe.git/tree/README``
+
+    ARM energy probe (AEP) device can simultaneously collect power from up to 3 power rails and
+    arm-probe utility can record data from several AEP devices simultaneously.
+
+    To connect the energy probe on a rail, connect the white wire to the pin that is closer to the
+    Voltage source and the black wire to the pin that is closer to the load (the SoC or the device
+    you are probing). Between the pins there should be a shunt resistor of known resistance in the
+    range of 5 to 500 mOhm but the voltage on the shunt resistor must stay smaller than 165mV.
+    The resistance of the shunt resistors is a mandatory parameter to be set in the ``config`` file.
+    """
+
+    mode = CONTINUOUS
+
+    MAX_CHANNELS = 12 # 4 Arm Energy Probes
+
+    def __init__(self, target, config_file='./config-aep', ):
+        super(ArmEnergyProbeInstrument, self).__init__(target)
+        self.arm_probe = which('arm-probe')
+        if self.arm_probe is None:
+            raise HostError('arm-probe must be installed on the host')
+        #todo detect is config file exist
+        self.attributes = ['power', 'voltage', 'current']
+        self.sample_rate_hz = 10000
+        self.config_file = config_file
+
+        self.parser = AepParser()
+        #TODO make it generic
+        topo = self.parser.topology_from_config(self.config_file)
+        for item in topo:
+            if item == 'time':
+                self.add_channel('timestamp', 'time')
+            else:
+                self.add_channel(item, 'power')
+
+    def reset(self, sites=None, kinds=None, channels=None):
+        super(ArmEnergyProbeInstrument, self).reset(sites, kinds, channels)
+        self.output_directory = tempfile.mkdtemp(prefix='energy_probe')
+        self.output_file_raw = os.path.join(self.output_directory, 'data_raw')
+        self.output_file = os.path.join(self.output_directory, 'data')
+        self.output_file_figure = os.path.join(self.output_directory, 'summary.txt')
+        self.output_file_error = os.path.join(self.output_directory, 'error.log')
+        self.output_fd_error = open(self.output_file_error, 'w')
+        self.command = 'arm-probe --config {} > {}'.format(self.config_file, self.output_file_raw)
+
+    def start(self):
+        self.logger.debug(self.command)
+        self.armprobe = subprocess.Popen(self.command,
+                                       stderr=self.output_fd_error,
+                                       preexec_fn=os.setpgrp,
+                                       shell=True)
+
+    def stop(self):
+       self.logger.debug("kill running arm-probe")
+       os.killpg(self.armprobe.pid, signal.SIGTERM)
+
+    def get_data(self, outfile):  # pylint: disable=R0914
+        self.logger.debug("Parse data and compute consumed energy")
+        self.parser.prepare(self.output_file_raw, self.output_file, self.output_file_figure)
+        self.parser.parse_aep()
+        self.parser.unprepare()
+        skip_header = 1
+
+        all_channels = [c.label for c in self.list_channels()]
+        active_channels = [c.label for c in self.active_channels]
+        active_indexes = [all_channels.index(ac) for ac in active_channels]
+
+        with csvreader(self.output_file, delimiter=' ') as reader:
+            with csvwriter(outfile) as writer:
+                for row in reader:
+                    if skip_header == 1:
+                        writer.writerow(active_channels)
+                        skip_header = 0
+                        continue
+                    if len(row) < len(active_channels):
+                        continue
+                    # all data are in micro (seconds/watt)
+                    new = [ float(row[i])/1000000 for i in active_indexes ]
+                    writer.writerow(new)
+
+        self.output_fd_error.close()
+        shutil.rmtree(self.output_directory)
+
+        return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
+
+    def get_raw(self):
+        return [self.output_file_raw]

devlib/instrument/daq.py

@@ -1,19 +1,34 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import os
-import csv
 import tempfile
 from itertools import chain
 
 from devlib.instrument import Instrument, MeasurementsCsv, CONTINUOUS
 from devlib.exception import HostError
+from devlib.utils.csvutil import csvwriter, create_reader
 from devlib.utils.misc import unique
 
 try:
     from daqpower.client import execute_command, Status
     from daqpower.config import DeviceConfiguration, ServerConfiguration
-except ImportError, e:
+except ImportError as e:
     execute_command, Status = None, None
     DeviceConfiguration, ServerConfiguration, ConfigurationError = None, None, None
-    import_error_mesg = e.message
+    import_error_mesg = e.args[0] if e.args else str(e)
 
 
 class DaqInstrument(Instrument):
@@ -33,10 +48,11 @@ class DaqInstrument(Instrument):
         # pylint: disable=no-member
         super(DaqInstrument, self).__init__(target)
         self._need_reset = True
+        self._raw_files = []
         if execute_command is None:
             raise HostError('Could not import "daqpower": {}'.format(import_error_mesg))
         if labels is None:
-            labels = ['PORT_{}'.format(i) for i in xrange(len(resistor_values))]
+            labels = ['PORT_{}'.format(i) for i in range(len(resistor_values))]
         if len(labels) != len(resistor_values):
             raise ValueError('"labels" and "resistor_values" must be of the same length')
         self.server_config = ServerConfiguration(host=host,
@@ -68,6 +84,7 @@ class DaqInstrument(Instrument):
         if not result.status == Status.OK:  # pylint: disable=no-member
             raise HostError(result.message)
         self._need_reset = False
+        self._raw_files = []
 
     def start(self):
         if self._need_reset:
@@ -86,6 +103,7 @@ class DaqInstrument(Instrument):
             site = os.path.splitext(entry)[0]
             path = os.path.join(tempdir, entry)
             raw_file_map[site] = path
+            self._raw_files.append(path)
 
         active_sites = unique([c.site for c in self.active_channels])
         file_handles = []
@@ -94,8 +112,8 @@ class DaqInstrument(Instrument):
             for site in active_sites:
                 try:
                     site_file = raw_file_map[site]
-                    fh = open(site_file, 'rb')
+                    reader, fh = create_reader(site_file)
-                    site_readers[site] = csv.reader(fh)
+                    site_readers[site] = reader
                     file_handles.append(fh)
                 except KeyError:
                     message = 'Could not get DAQ trace for {}; Obtained traces are in {}'
@@ -103,22 +121,21 @@ class DaqInstrument(Instrument):
 
             # The first row is the headers
             channel_order = []
-            for site, reader in site_readers.iteritems():
+            for site, reader in site_readers.items():
                 channel_order.extend(['{}_{}'.format(site, kind)
-                                      for kind in reader.next()])
+                                      for kind in next(reader)])
 
             def _read_next_rows():
                 parts = []
-                for reader in site_readers.itervalues():
+                for reader in site_readers.values():
                     try:
-                        parts.extend(reader.next())
+                        parts.extend(next(reader))
                     except StopIteration:
                         parts.extend([None, None])
                 return list(chain(parts))
 
-            with open(outfile, 'wb') as wfh:
+            with csvwriter(outfile) as writer:
                 field_names = [c.label for c in self.active_channels]
-                writer = csv.writer(wfh)
                 writer.writerow(field_names)
                 raw_row = _read_next_rows()
                 while any(raw_row):
@@ -126,11 +143,14 @@ class DaqInstrument(Instrument):
                     writer.writerow(row)
                     raw_row = _read_next_rows()
 
-            return MeasurementsCsv(outfile, self.active_channels)
+            return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
         finally:
             for fh in file_handles:
                 fh.close()
 
+    def get_raw(self):
+        return self._raw_files
+
     def teardown(self):
         self.execute('close')
 

devlib/instrument/energy_probe.py

@@ -1,4 +1,4 @@
-#    Copyright 2015 ARM Limited
+#    Copyright 2015-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,14 +14,15 @@
 #
 from __future__ import division
 import os
-import csv
 import signal
 import tempfile
 import struct
 import subprocess
+import sys
 
 from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv
 from devlib.exception import HostError
+from devlib.utils.csvutil import csvwriter
 from devlib.utils.misc import which
 
 
@@ -39,7 +40,7 @@ class EnergyProbeInstrument(Instrument):
             self.labels = labels
         else:
             self.labels = ['PORT_{}'.format(i)
-                           for i in xrange(len(resistor_values))]
+                           for i in range(len(resistor_values))]
         self.device_entry = device_entry
         self.caiman = which('caiman')
         if self.caiman is None:
@@ -52,6 +53,7 @@ class EnergyProbeInstrument(Instrument):
         self.raw_output_directory = None
         self.process = None
         self.sample_rate_hz = 10000 # Determined empirically
+        self.raw_data_file = None
 
         for label in self.labels:
             for kind in self.attributes:
@@ -64,6 +66,7 @@ class EnergyProbeInstrument(Instrument):
                  for i, rval in enumerate(self.resistor_values)]
         rstring = ''.join(parts)
         self.command = '{} -d {} -l {} {}'.format(self.caiman, self.device_entry, rstring, self.raw_output_directory)
+        self.raw_data_file = None
 
     def start(self):
         self.logger.debug(self.command)
@@ -78,11 +81,14 @@ class EnergyProbeInstrument(Instrument):
         self.process.poll()
         if self.process.returncode is not None:
             stdout, stderr = self.process.communicate()
+            if sys.version_info[0] == 3:
+                stdout = stdout.decode(sys.stdout.encoding, 'replace')
+                stderr = stderr.decode(sys.stdout.encoding, 'replace')
             raise HostError(
                 'Energy Probe: Caiman exited unexpectedly with exit code {}.\n'
                 'stdout:\n{}\nstderr:\n{}'.format(self.process.returncode,
                                                   stdout, stderr))
-        os.killpg(self.process.pid, signal.SIGTERM)
+        os.killpg(self.process.pid, signal.SIGINT)
 
     def get_data(self, outfile):  # pylint: disable=R0914
         all_channels = [c.label for c in self.list_channels()]
@@ -92,12 +98,11 @@ class EnergyProbeInstrument(Instrument):
         num_of_ports = len(self.resistor_values)
         struct_format = '{}I'.format(num_of_ports * self.attributes_per_sample)
         not_a_full_row_seen = False
-        raw_data_file = os.path.join(self.raw_output_directory, '0000000000')
+        self.raw_data_file = os.path.join(self.raw_output_directory, '0000000000')
 
-        self.logger.debug('Parsing raw data file: {}'.format(raw_data_file))
+        self.logger.debug('Parsing raw data file: {}'.format(self.raw_data_file))
-        with open(raw_data_file, 'rb') as bfile:
+        with open(self.raw_data_file, 'rb') as bfile:
-            with open(outfile, 'wb') as wfh:
+            with csvwriter(outfile) as writer:
-                writer = csv.writer(wfh)
                 writer.writerow(active_channels)
                 while True:
                     data = bfile.read(num_of_ports * self.bytes_per_sample)
@@ -113,4 +118,7 @@ class EnergyProbeInstrument(Instrument):
                             continue
                         else:
                             not_a_full_row_seen = True
-        return MeasurementsCsv(outfile, self.active_channels)
+        return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
+
+    def get_raw(self):
+        return [self.raw_data_file]

devlib/instrument/frames.py

@@ -0,0 +1,97 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from __future__ import division
+from devlib.instrument import (Instrument, CONTINUOUS,
+                               MeasurementsCsv, MeasurementType)
+from devlib.utils.rendering import (GfxinfoFrameCollector,
+                                    SurfaceFlingerFrameCollector,
+                                    SurfaceFlingerFrame,
+                                    read_gfxinfo_columns)
+
+
+class FramesInstrument(Instrument):
+
+    mode = CONTINUOUS
+    collector_cls = None
+
+    def __init__(self, target, collector_target, period=2, keep_raw=True):
+        super(FramesInstrument, self).__init__(target)
+        self.collector_target = collector_target
+        self.period = period
+        self.keep_raw = keep_raw
+        self.sample_rate_hz = 1 / self.period
+        self.collector = None
+        self.header = None
+        self._need_reset = True
+        self._raw_file = None
+        self._init_channels()
+
+    def reset(self, sites=None, kinds=None, channels=None):
+        super(FramesInstrument, self).reset(sites, kinds, channels)
+        self.collector = self.collector_cls(self.target, self.period,
+                                            self.collector_target, self.header)
+        self._need_reset = False
+        self._raw_file = None
+
+    def start(self):
+        if self._need_reset:
+            self.reset()
+        self.collector.start()
+
+    def stop(self):
+        self.collector.stop()
+        self._need_reset = True
+
+    def get_data(self, outfile):
+        if self.keep_raw:
+            self._raw_file = outfile + '.raw'
+        self.collector.process_frames(self._raw_file)
+        active_sites = [chan.label for chan in self.active_channels]
+        self.collector.write_frames(outfile, columns=active_sites)
+        return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
+
+    def get_raw(self):
+        return [self._raw_file] if self._raw_file else []
+
+    def _init_channels(self):
+        raise NotImplementedError()
+
+
+class GfxInfoFramesInstrument(FramesInstrument):
+
+    mode = CONTINUOUS
+    collector_cls = GfxinfoFrameCollector
+
+    def _init_channels(self):
+        columns = read_gfxinfo_columns(self.target)
+        for entry in columns:
+            if entry == 'Flags':
+                self.add_channel('Flags', MeasurementType('flags', 'flags'))
+            else:
+                self.add_channel(entry, 'time_us')
+        self.header = [chan.label for chan in self.channels.values()]
+
+
+class SurfaceFlingerFramesInstrument(FramesInstrument):
+
+    mode = CONTINUOUS
+    collector_cls = SurfaceFlingerFrameCollector
+
+    def _init_channels(self):
+        for field in SurfaceFlingerFrame._fields:
+            # remove the "_time" from filed names to avoid duplication
+            self.add_channel(field[:-5], 'time_us')
+        self.header = [chan.label for chan in self.channels.values()]

devlib/instrument/gem5power.py

@@ -0,0 +1,80 @@
+#    Copyright 2017-2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from __future__ import division
+import re
+
+from devlib.platform.gem5 import Gem5SimulationPlatform
+from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv
+from devlib.exception import TargetError, HostError
+from devlib.utils.csvutil import csvwriter
+
+
+class Gem5PowerInstrument(Instrument):
+    '''
+    Instrument enabling power monitoring in gem5
+    '''
+
+    mode = CONTINUOUS
+    roi_label = 'power_instrument'
+    site_mapping = {'timestamp': 'sim_seconds'}
+
+    def __init__(self, target, power_sites):
+        '''
+        Parameter power_sites is a list of gem5 identifiers for power values.
+        One example of such a field:
+            system.cluster0.cores0.power_model.static_power
+        '''
+        if not isinstance(target.platform, Gem5SimulationPlatform):
+            raise TargetError('Gem5PowerInstrument requires a gem5 platform')
+        if not target.has('gem5stats'):
+            raise TargetError('Gem5StatsModule is not loaded')
+        super(Gem5PowerInstrument, self).__init__(target)
+
+        # power_sites is assumed to be a list later
+        if isinstance(power_sites, list):
+            self.power_sites = power_sites
+        else:
+            self.power_sites = [power_sites]
+        self.add_channel('timestamp', 'time')
+        for field in self.power_sites:
+            self.add_channel(field, 'power')
+        self.target.gem5stats.book_roi(self.roi_label)
+        self.sample_period_ns = 10000000
+        # Sample rate must remain unset as gem5 does not provide samples
+        # at regular intervals therefore the reported timestamp should be used.
+        self.sample_rate_hz = None
+        self.target.gem5stats.start_periodic_dump(0, self.sample_period_ns)
+        self._base_stats_dump = 0
+
+    def start(self):
+        self.target.gem5stats.roi_start(self.roi_label)
+
+    def stop(self):
+        self.target.gem5stats.roi_end(self.roi_label)
+
+    def get_data(self, outfile):
+        active_sites = [c.site for c in self.active_channels]
+        with csvwriter(outfile) as writer:
+            writer.writerow([c.label for c in self.active_channels]) # headers
+            sites_to_match = [self.site_mapping.get(s, s) for s in active_sites]
+            for rec, rois in self.target.gem5stats.match_iter(sites_to_match,
+                    [self.roi_label], self._base_stats_dump):
+                writer.writerow([rec[s] for s in sites_to_match])
+        return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
+
+    def reset(self, sites=None, kinds=None, channels=None):
+        super(Gem5PowerInstrument, self).reset(sites, kinds, channels)
+        self._base_stats_dump = self.target.gem5stats.next_dump_no()
+

devlib/instrument/hwmon.py

@@ -1,4 +1,4 @@
-#    Copyright 2015 ARM Limited
+#    Copyright 2015-2017 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -45,7 +45,7 @@ class HwmonInstrument(Instrument):
                 measure = self.measure_map.get(ts.kind)[0]
                 if measure:
                     self.logger.debug('\tAdding sensor {}'.format(ts.name))
-                    self.add_channel(_guess_site(ts), measure, name=ts.name, sensor=ts)
+                    self.add_channel(_guess_site(ts), measure, sensor=ts)
                 else:
                     self.logger.debug('\tSkipping sensor {} (unknown kind "{}")'.format(ts.name, ts.kind))
             except ValueError:

devlib/instrument/monsoon.py

@@ -1,23 +1,42 @@
-import csv
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import os
 import signal
+import sys
 from subprocess import Popen, PIPE
 from tempfile import NamedTemporaryFile
+
 from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv
 from devlib.exception import HostError
 from devlib.host import PACKAGE_BIN_DIRECTORY
+from devlib.utils.csvutil import csvwriter
 from devlib.utils.misc import which
 
+
 INSTALL_INSTRUCTIONS="""
 MonsoonInstrument requires the monsoon.py tool, available from AOSP:
 
 https://android.googlesource.com/platform/cts/+/master/tools/utils/monsoon.py
 
 Download this script and put it in your $PATH (or pass it as the monsoon_bin
-parameter to MonsoonInstrument). `pip install gflags pyserial` to install the
+parameter to MonsoonInstrument). `pip install python-gflags pyserial` to install
-dependencies.
+the dependencies.
 """
 
+
 class MonsoonInstrument(Instrument):
     """Instrument for Monsoon Solutions power monitor
 
@@ -81,6 +100,9 @@ class MonsoonInstrument(Instrument):
         process.poll()
         if process.returncode is not None:
             stdout, stderr = process.communicate()
+            if sys.version_info[0] == 3:
+                stdout = stdout.encode(sys.stdout.encoding)
+                stderr = stderr.encode(sys.stdout.encoding)
             raise HostError(
                 'Monsoon script exited unexpectedly with exit code {}.\n'
                 'stdout:\n{}\nstderr:\n{}'.format(process.returncode,
@@ -104,8 +126,7 @@ class MonsoonInstrument(Instrument):
 
         stdout, stderr = self.output
 
-        with open(outfile, 'wb') as f:
+        with csvwriter(outfile) as writer:
-            writer = csv.writer(f)
             active_sites = [c.site for c in self.active_channels]
 
             # Write column headers
@@ -129,4 +150,4 @@ class MonsoonInstrument(Instrument):
                     row.append(usb)
                 writer.writerow(row)
 
-        return MeasurementsCsv(outfile, self.active_channels)
+        return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)

devlib/instrument/netstats/__init__.py

@@ -1,14 +1,30 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import os
 import re
-import csv
 import tempfile
 from datetime import datetime
 from collections import defaultdict
-from itertools import izip_longest
+
+from future.moves.itertools import zip_longest
 
 from devlib.instrument import Instrument, MeasurementsCsv, CONTINUOUS
 from devlib.exception import TargetError, HostError
 from devlib.utils.android import ApkInfo
+from devlib.utils.csvutil import csvwriter
 
 
 THIS_DIR = os.path.dirname(__file__)
@@ -46,10 +62,9 @@ def netstats_to_measurements(netstats):
 def write_measurements_csv(measurements, filepath):
     headers = sorted(measurements.keys())
     columns = [measurements[h] for h in headers]
-    with open(filepath, 'wb') as wfh:
+    with csvwriter(filepath) as writer:
-        writer = csv.writer(wfh)
         writer.writerow(headers)
-        writer.writerows(izip_longest(*columns))
+        writer.writerows(zip_longest(*columns))
 
 
 class NetstatsInstrument(Instrument):

devlib/module/__init__.py

@@ -1,4 +1,4 @@
-#    Copyright 2014-2015 ARM Limited
+#    Copyright 2014-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -15,6 +15,8 @@
 import logging
 from inspect import isclass
 
+from past.builtins import basestring
+
 from devlib.utils.misc import walk_modules
 from devlib.utils.types import identifier
 
@@ -56,7 +58,7 @@ class Module(object):
 
     def __init__(self, target):
         self.target = target
-        self.logger = logging.getLogger(self.__class__.__name__)
+        self.logger = logging.getLogger(self.name)
 
 
 class HardRestModule(Module):  # pylint: disable=R0921
@@ -75,7 +77,7 @@ class BootModule(Module):  # pylint: disable=R0921
         raise NotImplementedError()
 
     def update(self, **kwargs):
-        for name, value in kwargs.iteritems():
+        for name, value in kwargs.items():
             if not hasattr(self, name):
                 raise ValueError('Unknown parameter "{}" for {}'.format(name, self.name))
             self.logger.debug('Updating "{}" to "{}"'.format(name, value))
@@ -117,6 +119,6 @@ def register_module(mod):
 
 def __load_cache():
     for module in walk_modules('devlib.module'):
-        for obj in vars(module).itervalues():
+        for obj in vars(module).values():
             if isclass(obj) and issubclass(obj, Module) and obj.name:
                 register_module(obj)

devlib/module/android.py

@@ -1,4 +1,4 @@
-#    Copyright 2014-2015 ARM Limited
+#    Copyright 2014-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -63,7 +63,7 @@ class FastbootFlashModule(FlashModule):
             image_bundle = expand_path(image_bundle)
             to_flash = self._bundle_to_images(image_bundle)
         to_flash = merge_dicts(to_flash, images or {}, should_normalize=False)
-        for partition, image_path in to_flash.iteritems():
+        for partition, image_path in to_flash.items():
             self.logger.debug('flashing {}'.format(partition))
             self._flash_image(self.target, partition, expand_path(image_path))
         fastboot_command('reboot')

devlib/module/biglittle.py

@@ -1,3 +1,18 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 from devlib.module import Module
 
 
@@ -44,79 +59,151 @@ class BigLittleModule(Module):
     # cpufreq
 
     def list_bigs_frequencies(self):
-        return self.target.cpufreq.list_frequencies(self.bigs_online[0])
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            return self.target.cpufreq.list_frequencies(bigs_online[0])
 
     def list_bigs_governors(self):
-        return self.target.cpufreq.list_governors(self.bigs_online[0])
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            return self.target.cpufreq.list_governors(bigs_online[0])
 
     def list_bigs_governor_tunables(self):
-        return self.target.cpufreq.list_governor_tunables(self.bigs_online[0])
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            return self.target.cpufreq.list_governor_tunables(bigs_online[0])
 
     def list_littles_frequencies(self):
-        return self.target.cpufreq.list_frequencies(self.littles_online[0])
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            return self.target.cpufreq.list_frequencies(littles_online[0])
 
     def list_littles_governors(self):
-        return self.target.cpufreq.list_governors(self.littles_online[0])
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            return self.target.cpufreq.list_governors(littles_online[0])
 
     def list_littles_governor_tunables(self):
-        return self.target.cpufreq.list_governor_tunables(self.littles_online[0])
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            return self.target.cpufreq.list_governor_tunables(littles_online[0])
 
     def get_bigs_governor(self):
-        return self.target.cpufreq.get_governor(self.bigs_online[0])
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            return self.target.cpufreq.get_governor(bigs_online[0])
 
     def get_bigs_governor_tunables(self):
-        return self.target.cpufreq.get_governor_tunables(self.bigs_online[0])
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            return self.target.cpufreq.get_governor_tunables(bigs_online[0])
 
     def get_bigs_frequency(self):
-        return self.target.cpufreq.get_frequency(self.bigs_online[0])
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            return self.target.cpufreq.get_frequency(bigs_online[0])
 
     def get_bigs_min_frequency(self):
-        return self.target.cpufreq.get_min_frequency(self.bigs_online[0])
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            return self.target.cpufreq.get_min_frequency(bigs_online[0])
 
     def get_bigs_max_frequency(self):
-        return self.target.cpufreq.get_max_frequency(self.bigs_online[0])
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            return self.target.cpufreq.get_max_frequency(bigs_online[0])
 
     def get_littles_governor(self):
-        return self.target.cpufreq.get_governor(self.littles_online[0])
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            return self.target.cpufreq.get_governor(littles_online[0])
 
     def get_littles_governor_tunables(self):
-        return self.target.cpufreq.get_governor_tunables(self.littles_online[0])
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            return self.target.cpufreq.get_governor_tunables(littles_online[0])
 
     def get_littles_frequency(self):
-        return self.target.cpufreq.get_frequency(self.littles_online[0])
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            return self.target.cpufreq.get_frequency(littles_online[0])
 
     def get_littles_min_frequency(self):
-        return self.target.cpufreq.get_min_frequency(self.littles_online[0])
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            return self.target.cpufreq.get_min_frequency(littles_online[0])
 
     def get_littles_max_frequency(self):
-        return self.target.cpufreq.get_max_frequency(self.littles_online[0])
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            return self.target.cpufreq.get_max_frequency(littles_online[0])
 
     def set_bigs_governor(self, governor, **kwargs):
-        self.target.cpufreq.set_governor(self.bigs_online[0], governor, **kwargs)
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            self.target.cpufreq.set_governor(bigs_online[0], governor, **kwargs)
+        else:
+            raise ValueError("All bigs appear to be offline")
 
     def set_bigs_governor_tunables(self, governor, **kwargs):
-        self.target.cpufreq.set_governor_tunables(self.bigs_online[0], governor, **kwargs)
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            self.target.cpufreq.set_governor_tunables(bigs_online[0], governor, **kwargs)
+        else:
+            raise ValueError("All bigs appear to be offline")
 
     def set_bigs_frequency(self, frequency, exact=True):
-        self.target.cpufreq.set_frequency(self.bigs_online[0], frequency, exact)
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            self.target.cpufreq.set_frequency(bigs_online[0], frequency, exact)
+        else:
+            raise ValueError("All bigs appear to be offline")
 
     def set_bigs_min_frequency(self, frequency, exact=True):
-        self.target.cpufreq.set_min_frequency(self.bigs_online[0], frequency, exact)
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            self.target.cpufreq.set_min_frequency(bigs_online[0], frequency, exact)
+        else:
+            raise ValueError("All bigs appear to be offline")
 
     def set_bigs_max_frequency(self, frequency, exact=True):
-        self.target.cpufreq.set_max_frequency(self.bigs_online[0], frequency, exact)
+        bigs_online = self.bigs_online
+        if len(bigs_online) > 0:
+            self.target.cpufreq.set_max_frequency(bigs_online[0], frequency, exact)
+        else:
+            raise ValueError("All bigs appear to be offline")
 
     def set_littles_governor(self, governor, **kwargs):
-        self.target.cpufreq.set_governor(self.littles_online[0], governor, **kwargs)
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            self.target.cpufreq.set_governor(littles_online[0], governor, **kwargs)
+        else:
+            raise ValueError("All littles appear to be offline")
 
     def set_littles_governor_tunables(self, governor, **kwargs):
-        self.target.cpufreq.set_governor_tunables(self.littles_online[0], governor, **kwargs)
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            self.target.cpufreq.set_governor_tunables(littles_online[0], governor, **kwargs)
+        else:
+            raise ValueError("All littles appear to be offline")
 
     def set_littles_frequency(self, frequency, exact=True):
-        self.target.cpufreq.set_frequency(self.littles_online[0], frequency, exact)
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            self.target.cpufreq.set_frequency(littles_online[0], frequency, exact)
+        else:
+            raise ValueError("All littles appear to be offline")
 
     def set_littles_min_frequency(self, frequency, exact=True):
-        self.target.cpufreq.set_min_frequency(self.littles_online[0], frequency, exact)
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            self.target.cpufreq.set_min_frequency(littles_online[0], frequency, exact)
+        else:
+            raise ValueError("All littles appear to be offline")
 
     def set_littles_max_frequency(self, frequency, exact=True):
-        self.target.cpufreq.set_max_frequency(self.littles_online[0], frequency, exact)
+        littles_online = self.littles_online
+        if len(littles_online) > 0:
+            self.target.cpufreq.set_max_frequency(littles_online[0], frequency, exact)
+        else:
+            raise ValueError("All littles appear to be offline")

devlib/module/cgroups.py

@@ -1,4 +1,4 @@
-#    Copyright 2014-2015 ARM Limited
+#    Copyright 2014-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +14,7 @@
 #
 # pylint: disable=attribute-defined-outside-init
 import logging
+import re
 from collections import namedtuple
 
 from devlib.module import Module
@@ -102,7 +103,7 @@ class Controller(object):
                     .format(self.kind))
         if name not in self._cgroups:
             self._cgroups[name] = CGroup(self, name, create=False)
-        return self._cgroups[name].existe()
+        return self._cgroups[name].exists()
 
     def list_all(self):
         self.logger.debug('Listing groups for %s controller', self.kind)
@@ -168,7 +169,37 @@ class Controller(object):
             if cgroup != dest:
                 self.move_tasks(cgroup, dest, grep_filters)
 
-    def tasks(self, cgroup):
+    def tasks(self, cgroup,
+              filter_tid='',
+              filter_tname='',
+              filter_tcmdline=''):
+        """
+        Report the tasks that are included in a cgroup. The tasks can be
+        filtered by their tid, tname or tcmdline if filter_tid, filter_tname or
+        filter_tcmdline are defined respectively. In this case, the reported
+        tasks are the ones in the cgroup that match these patterns.
+
+        Example of tasks format:
+        TID,tname,tcmdline
+        903,cameraserver,/system/bin/cameraserver
+
+        :params filter_tid: regexp pattern to filter by TID
+        :type filter_tid: str
+
+        :params filter_tname: regexp pattern to filter by tname
+        :type filter_tname: str
+
+        :params filter_tcmdline: regexp pattern to filter by tcmdline
+        :type filter_tcmdline: str
+
+        :returns: a dictionary in the form: {tid:(tname, tcmdline)}
+        """
+        if not isinstance(filter_tid, str):
+            raise TypeError('filter_tid should be a str')
+        if not isinstance(filter_tname, str):
+            raise TypeError('filter_tname should be a str')
+        if not isinstance(filter_tcmdline, str):
+            raise TypeError('filter_tcmdline should be a str')
         try:
             cg = self._cgroups[cgroup]
         except KeyError as e:
@@ -179,15 +210,24 @@ class Controller(object):
         entries = output.splitlines()
         tasks = {}
         for task in entries:
-            tid = task.split(',')[0]
+            fields = task.split(',', 2)
-            try:
+            nr_fields = len(fields)
-                tname = task.split(',')[1]
+            if nr_fields < 2:
-            except: continue
+                continue
-            try:
+            elif nr_fields == 2:
-                tcmdline = task.split(',')[2]
+                tid_str, tname = fields
-            except:
                 tcmdline = ''
-            tasks[int(tid)] = (tname, tcmdline)
+            else:
+                tid_str, tname, tcmdline = fields
+
+            if not re.search(filter_tid, tid_str):
+                continue
+            if not re.search(filter_tname, tname):
+                continue
+            if not re.search(filter_tcmdline, tcmdline):
+                continue
+
+            tasks[int(tid_str)] = (tname, tcmdline)
         return tasks
 
     def tasks_count(self, cgroup):
@@ -285,7 +325,7 @@ class CGroup(object):
     def get_tasks(self):
         task_ids = self.target.read_value(self.tasks_file).split()
         logging.debug('Tasks: %s', task_ids)
-        return map(int, task_ids)
+        return list(map(int, task_ids))
 
     def add_task(self, tid):
         self.target.write_value(self.tasks_file, tid, verify=False)
@@ -354,7 +394,7 @@ class CgroupsModule(Module):
     def list_subsystems(self):
         subsystems = []
         for line in self.target.execute('{} cat /proc/cgroups'\
-                .format(self.target.busybox)).splitlines()[1:]:
+                .format(self.target.busybox), as_root=self.target.is_rooted).splitlines()[1:]:
             line = line.strip()
             if not line or line.startswith('#'):
                 continue
@@ -466,11 +506,11 @@ class CgroupsModule(Module):
         if freezer is None:
             raise RuntimeError('freezer cgroup controller not present')
         freezer_cg = freezer.cgroup('/DEVLIB_FREEZER')
-        thawed_cg = freezer.cgroup('/')
+        cmd = 'cgroups_freezer_set_state {{}} {}'.format(freezer_cg.directory)
 
         if thaw:
             # Restart froozen tasks
-            freezer_cg.set(state='THAWED')
+            freezer.target._execute_util(cmd.format('THAWED'), as_root=True)
             # Remove all tasks from freezer
             freezer.move_all_tasks_to('/')
             return
@@ -482,7 +522,7 @@ class CgroupsModule(Module):
         tasks = freezer.tasks('/')
 
         # Freeze all tasks
-        freezer_cg.set(state='FROZEN')
+        freezer.target._execute_util(cmd.format('FROZEN'), as_root=True)
 
         return tasks
 

devlib/module/cpufreq.py

@@ -1,4 +1,4 @@
-#    Copyright 2014-2015 ARM Limited
+#    Copyright 2014-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -37,7 +37,7 @@ class CpufreqModule(Module):
                 return True
 
         # Generic CPUFreq support (single policy)
-        path = '/sys/devices/system/cpu/cpufreq'
+        path = '/sys/devices/system/cpu/cpufreq/policy0'
         if target.file_exists(path):
             return True
 
@@ -150,7 +150,7 @@ class CpufreqModule(Module):
         if governor is None:
             governor = self.get_governor(cpu)
         valid_tunables = self.list_governor_tunables(cpu)
-        for tunable, value in kwargs.iteritems():
+        for tunable, value in kwargs.items():
             if tunable in valid_tunables:
                 path = '/sys/devices/system/cpu/{}/cpufreq/{}/{}'.format(cpu, governor, tunable)
                 try:
@@ -176,16 +176,41 @@ class CpufreqModule(Module):
         try:
             cmd = 'cat /sys/devices/system/cpu/{}/cpufreq/scaling_available_frequencies'.format(cpu)
             output = self.target.execute(cmd)
-            available_frequencies = map(int, output.strip().split())  # pylint: disable=E1103
+            available_frequencies = list(map(int, output.strip().split()))  # pylint: disable=E1103
         except TargetError:
             # On some devices scaling_frequencies  is not generated.
             # http://adrynalyne-teachtofish.blogspot.co.uk/2011/11/how-to-enable-scalingavailablefrequenci.html
             # Fall back to parsing stats/time_in_state
-            cmd = 'cat /sys/devices/system/cpu/{}/cpufreq/stats/time_in_state'.format(cpu)
+            path = '/sys/devices/system/cpu/{}/cpufreq/stats/time_in_state'.format(cpu)
-            out_iter = iter(self.target.execute(cmd).strip().split())
+            try:
-            available_frequencies = map(int, reversed([f for f, _ in zip(out_iter, out_iter)]))
+                out_iter = iter(self.target.read_value(path).split())
+            except TargetError:
+                if not self.target.file_exists(path):
+                    # Probably intel_pstate. Can't get available freqs.
+                    return []
+                raise
+
+            available_frequencies = list(map(int, reversed([f for f, _ in zip(out_iter, out_iter)])))
         return available_frequencies
 
+    @memoized
+    def get_max_available_frequency(self, cpu):
+        """
+        Returns the maximum available frequency for a given core or None if
+        could not be found.
+        """
+        freqs = self.list_frequencies(cpu)
+        return freqs and max(freqs) or None
+
+    @memoized
+    def get_min_available_frequency(self, cpu):
+        """
+        Returns the minimum available frequency for a given core or None if
+        could not be found.
+        """
+        freqs = self.list_frequencies(cpu)
+        return freqs and min(freqs) or None
+
     def get_min_frequency(self, cpu):
         """
         Returns the min frequency currently set for the specified CPU.
@@ -382,7 +407,9 @@ class CpufreqModule(Module):
                 'cpufreq_set_all_governors {}'.format(governor),
                 as_root=True)
         except TargetError as e:
-            if "echo: I/O error" in str(e):
+            if ("echo: I/O error" in str(e) or
+                "write error: Invalid argument" in str(e)):
+
                 cpus_unsupported = [c for c in self.target.list_online_cpus()
                                     if governor not in self.list_governors(c)]
                 raise TargetError("Governor {} unsupported for CPUs {}".format(
@@ -410,10 +437,9 @@ class CpufreqModule(Module):
         """
         return self.target._execute_util('cpufreq_trace_all_frequencies', as_root=True)
 
-    @memoized
+    def get_affected_cpus(self, cpu):
-    def get_domain_cpus(self, cpu):
         """
-        Get the CPUs that share a frequency domain with the given CPU
+        Get the online CPUs that share a frequency domain with the given CPU
         """
         if isinstance(cpu, int):
             cpu = 'cpu{}'.format(cpu)
@@ -421,3 +447,38 @@ class CpufreqModule(Module):
         sysfile = '/sys/devices/system/cpu/{}/cpufreq/affected_cpus'.format(cpu)
 
         return [int(c) for c in self.target.read_value(sysfile).split()]
+
+    @memoized
+    def get_related_cpus(self, cpu):
+        """
+        Get the CPUs that share a frequency domain with the given CPU
+        """
+        if isinstance(cpu, int):
+            cpu = 'cpu{}'.format(cpu)
+
+        sysfile = '/sys/devices/system/cpu/{}/cpufreq/related_cpus'.format(cpu)
+
+        return [int(c) for c in self.target.read_value(sysfile).split()]
+
+    @memoized
+    def get_driver(self, cpu):
+        """
+        Get the name of the driver used by this cpufreq policy.
+        """
+        if isinstance(cpu, int):
+            cpu = 'cpu{}'.format(cpu)
+
+        sysfile = '/sys/devices/system/cpu/{}/cpufreq/scaling_driver'.format(cpu)
+
+        return self.target.read_value(sysfile).strip()
+
+    def iter_domains(self):
+        """
+        Iterate over the frequency domains in the system
+        """
+        cpus = set(range(self.target.number_of_cpus))
+        while cpus:
+            cpu = next(iter(cpus))
+            domain = self.target.cpufreq.get_related_cpus(cpu)
+            yield domain
+            cpus = cpus.difference(domain)

devlib/module/cpuidle.py

@@ -1,4 +1,4 @@
-#    Copyright 2014-2015 ARM Limited
+#    Copyright 2014-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -13,6 +13,8 @@
 # limitations under the License.
 #
 # pylint: disable=attribute-defined-outside-init
+from past.builtins import basestring
+
 from devlib.module import Module
 from devlib.utils.misc import memoized
 from devlib.utils.types import integer, boolean
@@ -41,51 +43,17 @@ class CpuidleState(object):
                 raise ValueError('invalid idle state name: "{}"'.format(self.id))
         return int(self.id[i:])
 
-    def __init__(self, target, index, path):
+    def __init__(self, target, index, path, name, desc, power, latency, residency):
         self.target = target
         self.index = index
         self.path = path
+        self.name = name
+        self.desc = desc
+        self.power = power
+        self.latency = latency
         self.id = self.target.path.basename(self.path)
         self.cpu = self.target.path.basename(self.target.path.dirname(path))
 
-    @property
-    @memoized
-    def desc(self):
-        return self.get('desc')
-
-    @property
-    @memoized
-    def name(self):
-        return self.get('name')
-
-    @property
-    @memoized
-    def latency(self):
-        """Exit latency in uS"""
-        return self.get('latency')
-
-    @property
-    @memoized
-    def power(self):
-        """Power usage in mW
-
-        ..note::
-
-            This value is not always populated by the kernel and may be garbage.
-        """
-        return self.get('power')
-
-    @property
-    @memoized
-    def target_residency(self):
-        """Target residency in uS
-
-        This is the amount of time in the state required to 'break even' on
-        power - the system should avoid entering the state for less time than
-        this.
-        """
-        return self.get('residency')
-
     def enable(self):
         self.set('disable', 0)
 
@@ -126,23 +94,47 @@ class Cpuidle(Module):
     def probe(target):
         return target.file_exists(Cpuidle.root_path)
 
-    def get_driver(self):
+    def __init__(self, target):
-        return self.target.read_value(self.target.path.join(self.root_path, 'current_driver'))
+        super(Cpuidle, self).__init__(target)
+        self._states = {}
 
-    def get_governor(self):
+        basepath = '/sys/devices/system/cpu/'
-        return self.target.read_value(self.target.path.join(self.root_path, 'current_governor_ro'))
+        values_tree = self.target.read_tree_values(basepath, depth=4, check_exit_code=False)
+        i = 0
+        cpu_id = 'cpu{}'.format(i)
+        while cpu_id in values_tree:
+            cpu_node = values_tree[cpu_id]
+
+            if 'cpuidle' in cpu_node:
+                idle_node = cpu_node['cpuidle']
+                self._states[cpu_id] = []
+                j = 0
+                state_id = 'state{}'.format(j)
+                while state_id in idle_node:
+                    state_node = idle_node[state_id]
+                    state = CpuidleState(
+                        self.target,
+                        index=j,
+                        path=self.target.path.join(basepath, cpu_id, 'cpuidle', state_id),
+                        name=state_node['name'],
+                        desc=state_node['desc'],
+                        power=int(state_node['power']),
+                        latency=int(state_node['latency']),
+                        residency=int(state_node['residency']) if 'residency' in state_node else None,
+                    )
+                    msg = 'Adding {} state {}: {} {}'
+                    self.logger.debug(msg.format(cpu_id, j, state.name, state.desc))
+                    self._states[cpu_id].append(state)
+                    j += 1
+                    state_id = 'state{}'.format(j)
+
+            i += 1
+            cpu_id = 'cpu{}'.format(i)
 
-    @memoized
     def get_states(self, cpu=0):
         if isinstance(cpu, int):
             cpu = 'cpu{}'.format(cpu)
-        states_dir = self.target.path.join(self.target.path.dirname(self.root_path), cpu, 'cpuidle')
+        return self._states.get(cpu, [])
-        idle_states = []
-        for state in self.target.list_directory(states_dir):
-            if state.startswith('state'):
-                index = int(state[5:])
-                idle_states.append(CpuidleState(self.target, index, self.target.path.join(states_dir, state)))
-        return idle_states
 
     def get_state(self, state, cpu=0):
         if isinstance(state, int):
@@ -175,4 +167,9 @@ class Cpuidle(Module):
         Momentarily wake each CPU. Ensures cpu_idle events in trace file.
         """
         output = self.target._execute_util('cpuidle_wake_all_cpus')
-        print(output)
+
+    def get_driver(self):
+        return self.target.read_value(self.target.path.join(self.root_path, 'current_driver'))
+
+    def get_governor(self):
+        return self.target.read_value(self.target.path.join(self.root_path, 'current_governor_ro'))

devlib/module/devfreq.py

@@ -0,0 +1,261 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+from devlib.module import Module
+from devlib.exception import TargetError
+from devlib.utils.misc import memoized
+
+class DevfreqModule(Module):
+
+    name = 'devfreq'
+
+    @staticmethod
+    def probe(target):
+        path = '/sys/class/devfreq/'
+        if not target.file_exists(path):
+            return False
+
+        # Check that at least one policy is implemented
+        if not target.list_directory(path):
+            return False
+
+        return True
+
+    @memoized
+    def list_devices(self):
+        """Returns a list of devfreq devices supported by the target platform."""
+        sysfile = '/sys/class/devfreq/'
+        return self.target.list_directory(sysfile)
+
+    @memoized
+    def list_governors(self, device):
+        """Returns a list of governors supported by the device."""
+        sysfile = '/sys/class/devfreq/{}/available_governors'.format(device)
+        output = self.target.read_value(sysfile)
+        return output.strip().split()
+
+    def get_governor(self, device):
+        """Returns the governor currently set for the specified device."""
+        if isinstance(device, int):
+            device = 'device{}'.format(device)
+        sysfile = '/sys/class/devfreq/{}/governor'.format(device)
+        return self.target.read_value(sysfile)
+
+    def set_governor(self, device, governor):
+        """
+        Set the governor for the specified device.
+
+        :param device: The device for which the governor is to be set. This must be
+                    the full name as it appears in sysfs, e.g. "e82c0000.mali".
+        :param governor: The name of the governor to be used. This must be
+                         supported by the specific device.
+
+        Additional keyword arguments can be used to specify governor tunables for
+        governors that support them.
+
+        :raises: TargetError if governor is not supported by the device, or if,
+                 for some reason, the governor could not be set.
+
+        """
+        supported = self.list_governors(device)
+        if governor not in supported:
+            raise TargetError('Governor {} not supported for device {}'.format(governor, device))
+        sysfile = '/sys/class/devfreq/{}/governor'.format(device)
+        self.target.write_value(sysfile, governor)
+
+    @memoized
+    def list_frequencies(self, device):
+        """
+        Returns a list of frequencies supported by the device or an empty list
+        if could not be found.
+        """
+        cmd = 'cat /sys/class/devfreq/{}/available_frequencies'.format(device)
+        output = self.target.execute(cmd)
+        available_frequencies = [int(freq) for freq in output.strip().split()]
+
+        return available_frequencies
+
+    def get_min_frequency(self, device):
+        """
+        Returns the min frequency currently set for the specified device.
+
+        Warning, this method does not check if the device is present or not. It
+        will try to read the minimum frequency and the following exception will
+        be raised ::
+
+        :raises: TargetError if for some reason the frequency could not be read.
+
+        """
+        sysfile = '/sys/class/devfreq/{}/min_freq'.format(device)
+        return self.target.read_int(sysfile)
+
+    def set_min_frequency(self, device, frequency, exact=True):
+        """
+        Sets the minimum value for device frequency. Actual frequency will
+        depend on the thermal governor used and may vary during execution. The
+        value should be either an int or a string representing an integer. The
+        Value must also be supported by the device. The available frequencies
+        can be obtained by calling list_frequencies() or examining
+
+        /sys/class/devfreq/<device_name>/available_frequencies
+
+        on the device.
+
+        :raises: TargetError if the frequency is not supported by the device, or if, for
+                 some reason, frequency could not be set.
+        :raises: ValueError if ``frequency`` is not an integer.
+
+        """
+        available_frequencies = self.list_frequencies(device)
+        try:
+            value = int(frequency)
+            if exact and available_frequencies and value not in available_frequencies:
+                raise TargetError('Can\'t set {} frequency to {}\nmust be in {}'.format(device,
+                                                                                        value,
+                                                                                        available_frequencies))
+            sysfile = '/sys/class/devfreq/{}/min_freq'.format(device)
+            self.target.write_value(sysfile, value)
+        except ValueError:
+            raise ValueError('Frequency must be an integer; got: "{}"'.format(frequency))
+
+    def get_frequency(self, device):
+        """
+        Returns the current frequency currently set for the specified device.
+
+        Warning, this method does not check if the device is present or not. It
+        will try to read the current frequency and the following exception will
+        be raised ::
+
+        :raises: TargetError if for some reason the frequency could not be read.
+
+        """
+        sysfile = '/sys/class/devfreq/{}/cur_freq'.format(device)
+        return self.target.read_int(sysfile)
+
+    def get_max_frequency(self, device):
+        """
+        Returns the max frequency currently set for the specified device.
+
+        Warning, this method does not check if the device is online or not. It will
+        try to read the maximum frequency and the following exception will be
+        raised ::
+
+        :raises: TargetError if for some reason the frequency could not be read.
+        """
+        sysfile = '/sys/class/devfreq/{}/max_freq'.format(device)
+        return self.target.read_int(sysfile)
+
+    def set_max_frequency(self, device, frequency, exact=True):
+        """
+        Sets the maximum value for device frequency. Actual frequency will
+        depend on the Governor used and may vary during execution. The value
+        should be either an int or a string representing an integer. The Value
+        must also be supported by the device. The available frequencies can be
+        obtained by calling get_frequencies() or examining
+
+        /sys/class/devfreq/<device_name>/available_frequencies
+
+        on the device.
+
+        :raises: TargetError if the frequency is not supported by the device, or
+                 if, for some reason, frequency could not be set.
+        :raises: ValueError if ``frequency`` is not an integer.
+
+        """
+        available_frequencies = self.list_frequencies(device)
+        try:
+            value = int(frequency)
+        except ValueError:
+            raise ValueError('Frequency must be an integer; got: "{}"'.format(frequency))
+
+        if exact and value not in available_frequencies:
+            raise TargetError('Can\'t set {} frequency to {}\nmust be in {}'.format(device,
+                                                                                    value,
+                                                                                    available_frequencies))
+        sysfile = '/sys/class/devfreq/{}/max_freq'.format(device)
+        self.target.write_value(sysfile, value)
+
+    def set_governor_for_devices(self, devices, governor):
+        """
+        Set the governor for the specified list of devices.
+
+        :param devices: The list of device for which the governor is to be set.
+        """
+        for device in devices:
+            self.set_governor(device, governor)
+
+    def set_all_governors(self, governor):
+        """
+        Set the specified governor for all the (available) devices
+        """
+        try:
+            return self.target._execute_util(
+                'devfreq_set_all_governors {}'.format(governor), as_root=True)
+        except TargetError as e:
+            if ("echo: I/O error" in str(e) or
+                "write error: Invalid argument" in str(e)):
+
+                devs_unsupported = [d for d in self.target.list_devices()
+                                    if governor not in self.list_governors(d)]
+                raise TargetError("Governor {} unsupported for devices {}".format(
+                    governor, devs_unsupported))
+            else:
+                raise
+
+    def get_all_governors(self):
+        """
+        Get the current governor for all the (online) CPUs
+        """
+        output = self.target._execute_util(
+                'devfreq_get_all_governors', as_root=True)
+        governors = {}
+        for x in output.splitlines():
+            kv = x.split(' ')
+            if kv[0] == '':
+                break
+            governors[kv[0]] = kv[1]
+        return governors
+
+    def set_frequency_for_devices(self, devices, freq, exact=False):
+        """
+        Set the frequency for the specified list of devices.
+
+        :param devices: The list of device for which the frequency has to be set.
+        """
+        for device in devices:
+            self.set_max_frequency(device, freq, exact)
+            self.set_min_frequency(device, freq, exact)
+
+    def set_all_frequencies(self, freq):
+        """
+        Set the specified (minimum) frequency for all the (available) devices
+        """
+        return self.target._execute_util(
+                'devfreq_set_all_frequencies {}'.format(freq),
+                as_root=True)
+
+    def get_all_frequencies(self):
+        """
+        Get the current frequency for all the (available) devices
+        """
+        output = self.target._execute_util(
+                'devfreq_get_all_frequencies', as_root=True)
+        frequencies = {}
+        for x in output.splitlines():
+            kv = x.split(' ')
+            if kv[0] == '':
+                break
+            frequencies[kv[0]] = kv[1]
+        return frequencies
+

devlib/module/gem5stats.py

@@ -0,0 +1,254 @@
+#    Copyright 2017-2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import sys
+import logging
+import os.path
+from collections import defaultdict
+
+import devlib
+from devlib.exception import TargetError
+from devlib.module import Module
+from devlib.platform import Platform
+from devlib.platform.gem5 import Gem5SimulationPlatform
+from devlib.utils.gem5 import iter_statistics_dump, GEM5STATS_ROI_NUMBER, GEM5STATS_DUMP_TAIL
+
+
+class Gem5ROI:
+    def __init__(self, number, target):
+        self.target = target
+        self.number = number
+        self.running = False
+        self.field = 'ROI::{}'.format(number)
+
+    def start(self):
+        if self.running:
+            return False
+        self.target.execute('m5 roistart {}'.format(self.number))
+        self.running = True
+        return True
+    
+    def stop(self):
+        if not self.running:
+            return False
+        self.target.execute('m5 roiend {}'.format(self.number))
+        self.running = False
+        return True
+
+class Gem5StatsModule(Module):
+    '''
+    Module controlling Region of Interest (ROIs) markers, satistics dump 
+    frequency and parsing statistics log file when using gem5 platforms.
+
+    ROIs are identified by user-defined labels and need to be booked prior to
+    use. The translation of labels into gem5 ROI numbers will be performed
+    internally in order to avoid conflicts between multiple clients.
+    '''
+    name = 'gem5stats'
+
+    @staticmethod
+    def probe(target):
+        return isinstance(target.platform, Gem5SimulationPlatform)
+
+    def __init__(self, target):
+        super(Gem5StatsModule, self).__init__(target)
+        self._current_origin = 0
+        self._stats_file_path = os.path.join(target.platform.gem5_out_dir,
+                                            'stats.txt')
+        self.rois = {}
+        self._dump_pos_cache = {0: 0}
+
+    def book_roi(self, label):
+        if label in self.rois:
+            raise KeyError('ROI label {} already used'.format(label))
+        if len(self.rois) >= GEM5STATS_ROI_NUMBER:
+            raise RuntimeError('Too many ROIs reserved')
+        all_rois = set(range(GEM5STATS_ROI_NUMBER))
+        used_rois = set([roi.number for roi in self.rois.values()])
+        avail_rois = all_rois - used_rois
+        self.rois[label] = Gem5ROI(list(avail_rois)[0], self.target)
+
+    def free_roi(self, label):
+        if label not in self.rois:
+            raise KeyError('ROI label {} not reserved yet'.format(label))
+        self.rois[label].stop()
+        del self.rois[label]
+
+    def roi_start(self, label):
+        if label not in self.rois:
+            raise KeyError('Incorrect ROI label: {}'.format(label))
+        if not self.rois[label].start():
+            raise TargetError('ROI {} was already running'.format(label))
+    
+    def roi_end(self, label):
+        if label not in self.rois:
+            raise KeyError('Incorrect ROI label: {}'.format(label))
+        if not self.rois[label].stop():
+            raise TargetError('ROI {} was not running'.format(label))
+
+    def start_periodic_dump(self, delay_ns=0, period_ns=10000000):
+        # Default period is 10ms because it's roughly what's needed to have
+        # accurate power estimations
+        if delay_ns < 0 or period_ns < 0:
+            msg = 'Delay ({}) and period ({}) for periodic dumps must be positive'
+            raise ValueError(msg.format(delay_ns, period_ns))
+        self.target.execute('m5 dumpresetstats {} {}'.format(delay_ns, period_ns))
+    
+    def match(self, keys, rois_labels, base_dump=0):
+        '''
+        Extract specific values from the statistics log file of gem5
+
+        :param keys: a list of key name or regular expression patterns that
+            will be matched in the fields of the statistics file. ``match()``
+            returns only the values of fields matching at least one these
+            keys.
+        :type keys: list
+
+        :param rois_labels: list of ROIs labels. ``match()`` returns the 
+            values of the specified fields only during dumps spanned by at
+            least one of these ROIs.
+        :type rois_label: list
+
+        :param base_dump: dump number from which ``match()`` should operate. By 
+            specifying a non-zero dump number, one can virtually truncate 
+            the head of the stats file and ignore all dumps before a specific
+            instant. The value of ``base_dump`` will typically (but not 
+            necessarily) be the result of a previous call to ``next_dump_no``.
+            Default value is 0.
+        :type base_dump: int
+
+        :returns: a dict indexed by key parameters containing a dict indexed by
+        ROI labels containing an in-order list of records for the key under
+        consideration during the active intervals of the ROI. 
+        
+        Example of return value:
+         * Result of match(['sim_'],['roi_1']):
+            {
+                'sim_inst': 
+                {
+                    'roi_1': [265300176, 267975881]
+                }
+                'sim_ops': 
+                {
+                    'roi_1': [324395787, 327699419]
+                }
+                'sim_seconds': 
+                {
+                    'roi_1': [0.199960, 0.199897]
+                }
+                'sim_freq': 
+                {
+                    'roi_1': [1000000000000, 1000000000000]
+                }
+                'sim_ticks': 
+                {
+                    'roi_1': [199960234227, 199896897330]
+                }
+            }
+        '''
+        records = defaultdict(lambda : defaultdict(list))
+        for record, active_rois in self.match_iter(keys, rois_labels, base_dump):
+            for key in record:
+                for roi_label in active_rois:
+                    records[key][roi_label].append(record[key])
+        return records
+
+    def match_iter(self, keys, rois_labels, base_dump=0):
+        '''
+        Yield specific values dump-by-dump from the statistics log file of gem5
+
+        :param keys: same as ``match()``
+        :param rois_labels: same as ``match()``
+        :param base_dump: same as ``match()``
+        :returns: a pair containing:
+            1. a dict storing the values corresponding to each of the found keys
+            2. the list of currently active ROIs among those passed as parameters
+
+        Example of return value:
+         * Result of match_iter(['sim_'],['roi_1', 'roi_2']).next()
+            ( 
+                { 
+                    'sim_inst': 265300176,
+                    'sim_ops': 324395787,
+                    'sim_seconds': 0.199960, 
+                    'sim_freq': 1000000000000,
+                    'sim_ticks': 199960234227,
+                },
+                [ 'roi_1 ' ] 
+            )
+        '''
+        for label in rois_labels:
+            if label not in self.rois:
+                raise KeyError('Impossible to match ROI label {}'.format(label))
+            if self.rois[label].running:
+                self.logger.warning('Trying to match records in statistics file'
+                        ' while ROI {} is running'.format(label))
+        
+        # Construct one large regex that concatenates all keys because
+        # matching one large expression is more efficient than several smaller
+        all_keys_re = re.compile('|'.join(keys))
+        
+        def roi_active(roi_label, dump):
+            roi = self.rois[roi_label]
+            return (roi.field in dump) and (int(dump[roi.field]) == 1)
+
+        with open(self._stats_file_path, 'r') as stats_file:
+            self._goto_dump(stats_file, base_dump)
+            for dump in iter_statistics_dump(stats_file):
+                active_rois = [l for l in rois_labels if roi_active(l, dump)]
+                if active_rois:
+                    rec = {k: dump[k] for k in dump if all_keys_re.search(k)}
+                    yield (rec, active_rois)
+
+    def next_dump_no(self):
+        '''
+        Returns the number of the next dump to be written to the stats file.
+        
+        For example, if next_dump_no is called while there are 5 (0 to 4) full 
+        dumps in the stats file, it will return 5. This will be usefull to know
+        from which dump one should match() in the future to get only data from
+        now on.
+        '''
+        with open(self._stats_file_path, 'r') as stats_file:
+            # _goto_dump reach EOF and returns the total number of dumps + 1
+            return self._goto_dump(stats_file, sys.maxsize)
+    
+    def _goto_dump(self, stats_file, target_dump):
+        if target_dump < 0:
+            raise HostError('Cannot go to dump {}'.format(target_dump))
+
+        # Go to required dump quickly if it was visited before
+        if target_dump in self._dump_pos_cache:
+            stats_file.seek(self._dump_pos_cache[target_dump])
+            return target_dump
+        # Or start from the closest dump already visited before the required one
+        prev_dumps = filter(lambda x: x < target_dump, self._dump_pos_cache.keys())
+        curr_dump = max(prev_dumps)
+        curr_pos = self._dump_pos_cache[curr_dump]
+        stats_file.seek(curr_pos)
+        
+        # And iterate until target_dump
+        dump_iterator = iter_statistics_dump(stats_file)
+        while curr_dump < target_dump:
+            try:
+                dump = next(dump_iterator)
+            except StopIteration:
+                break
+            # End of passed dump is beginning og next one
+            curr_pos = stats_file.tell()
+            curr_dump += 1
+        self._dump_pos_cache[curr_dump] = curr_pos
+        return curr_dump
+

devlib/module/gpufreq.py

@@ -0,0 +1,90 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#    Copyright 2017 Google, ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import json
+from devlib.module import Module
+from devlib.exception import TargetError
+from devlib.utils.misc import memoized
+
+class GpufreqModule(Module):
+
+    name = 'gpufreq'
+    path = ''
+
+    def __init__(self, target):
+        super(GpufreqModule, self).__init__(target)
+        frequencies_str = self.target.read_value("/sys/kernel/gpu/gpu_freq_table")
+        self.frequencies = list(map(int, frequencies_str.split(" ")))
+        self.frequencies.sort()
+        self.governors = self.target.read_value("/sys/kernel/gpu/gpu_available_governor").split(" ")
+
+    @staticmethod
+    def probe(target):
+        # kgsl/Adreno
+        probe_path = '/sys/kernel/gpu/'
+        if target.file_exists(probe_path):
+            model = target.read_value(probe_path + "gpu_model")
+            if re.search('adreno', model, re.IGNORECASE):
+                return True
+        return False
+
+    def set_governor(self, governor):
+        if governor not in self.governors:
+            raise TargetError('Governor {} not supported for gpu {}'.format(governor, cpu))
+        self.target.write_value("/sys/kernel/gpu/gpu_governor", governor)
+
+    def get_frequencies(self):
+        """
+        Returns the list of frequencies that the GPU can have
+        """
+        return self.frequencies
+
+    def get_current_frequency(self):
+        """
+        Returns the current frequency currently set for the GPU.
+
+        Warning, this method does not check if the gpu is online or not. It will
+        try to read the current frequency and the following exception will be
+        raised ::
+
+        :raises: TargetError if for some reason the frequency could not be read.
+
+        """
+        return int(self.target.read_value("/sys/kernel/gpu/gpu_clock"))
+
+    @memoized
+    def get_model_name(self):
+        """
+        Returns the model name reported by the GPU.
+        """
+        try:
+          return self.target.read_value("/sys/kernel/gpu/gpu_model")
+        except:
+          return "unknown"

devlib/module/hotplug.py

@@ -1,3 +1,18 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 from devlib.module import Module
 
 
@@ -21,7 +36,8 @@ class HotplugModule(Module):
         return target.path.join(cls.base_path, cpu, 'online')
 
     def online_all(self):
-        self.online(*range(self.target.number_of_cpus))
+        self.target._execute_util('hotplug_online_all',
+                                  as_root=self.target.is_rooted)
 
     def online(self, *args):
         for cpu in args:

devlib/module/hwmon.py

@@ -1,4 +1,4 @@
-#    Copyright 2015 ARM Limited
+#    Copyright 2015-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,9 +12,11 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import os
 import re
 from collections import defaultdict
 
+from devlib import TargetError
 from devlib.module import Module
 from devlib.utils.types import integer
 
@@ -73,20 +75,19 @@ class HwmonDevice(object):
     @property
     def sensors(self):
         all_sensors = []
-        for sensors_of_kind in self._sensors.itervalues():
+        for sensors_of_kind in self._sensors.values():
-            all_sensors.extend(sensors_of_kind.values())
+            all_sensors.extend(list(sensors_of_kind.values()))
         return all_sensors
 
-    def __init__(self, target, path):
+    def __init__(self, target, path, name, fields):
         self.target = target
         self.path = path
-        self.name = self.target.read_value(self.target.path.join(self.path, 'name'))
+        self.name = name
         self._sensors = defaultdict(dict)
         path = self.path
         if not path.endswith(self.target.path.sep):
             path += self.target.path.sep
-        for entry in self.target.list_directory(path,
+        for entry in fields:
-                                                as_root=self.target.is_rooted):
             match = HWMON_FILE_REGEX.search(entry)
             if match:
                 kind = match.group('kind')
@@ -99,7 +100,7 @@ class HwmonDevice(object):
 
     def get(self, kind, number=None):
         if number is None:
-            return [s for _, s in sorted(self._sensors[kind].iteritems(),
+            return [s for _, s in sorted(self._sensors[kind].items(),
                                          key=lambda x: x[0])]
         else:
             return self._sensors[kind].get(number)
@@ -116,7 +117,12 @@ class HwmonModule(Module):
 
     @staticmethod
     def probe(target):
-        return target.file_exists(HWMON_ROOT)
+        try:
+            target.list_directory(HWMON_ROOT, as_root=target.is_rooted)
+        except TargetError:
+            # Doesn't exist or no permissions
+            return False
+        return True
 
     @property
     def sensors(self):
@@ -132,11 +138,13 @@ class HwmonModule(Module):
         self.scan()
 
     def scan(self):
-        for entry in self.target.list_directory(self.root,
+        values_tree = self.target.read_tree_values(self.root, depth=3)
-                                                as_root=self.target.is_rooted):
+        for entry_id, fields in values_tree.items():
-            if entry.startswith('hwmon'):
+            path = self.target.path.join(self.root, entry_id)
-                entry_path = self.target.path.join(self.root, entry)
+            name = fields.pop('name', None)
-                if self.target.file_exists(self.target.path.join(entry_path, 'name')):
+            if name is None:
-                    device = HwmonDevice(self.target, entry_path)
+                continue
+            self.logger.debug('Adding device {}'.format(name))
+            device = HwmonDevice(self.target, path, name, fields)
             self.devices.append(device)
 

devlib/module/sched.py

@@ -0,0 +1,334 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#    Copyright 2018 Arm Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import logging
+import re
+
+from devlib.module import Module
+from devlib.utils.misc import memoized
+
+from past.builtins import basestring
+
+
+class SchedProcFSNode(object):
+    """
+    Represents a sched_domain procfs node
+
+    :param nodes: Dictionnary view of the underlying procfs nodes
+        (as returned by devlib.read_tree_values())
+    :type nodes: dict
+
+
+    Say you want to represent this path/data:
+    $ cat /proc/sys/kernel/sched_domain/cpu0/domain*/name
+    MC
+    DIE
+
+    Taking cpu0 as a root, this can be defined as:
+    >>> data = {"domain0" : {"name" : "MC"}, "domain1" : {"name" : "DIE"}}
+
+    >>> repr = SchedProcFSNode(data)
+    >>> print repr.domains[0].name
+    MC
+
+    The "raw" dict remains available under the `procfs` field:
+    >>> print repr.procfs["domain0"]["name"]
+    MC
+    """
+
+    _re_procfs_node = re.compile(r"(?P<name>.*)(?P<digits>\d+)$")
+
+    @staticmethod
+    def _ends_with_digits(node):
+        if not isinstance(node, basestring):
+            return False
+
+        return re.search(SchedProcFSNode._re_procfs_node, node) != None
+
+    @staticmethod
+    def _node_digits(node):
+        """
+        :returns: The ending digits of the procfs node
+        """
+        return int(re.search(SchedProcFSNode._re_procfs_node, node).group("digits"))
+
+    @staticmethod
+    def _node_name(node):
+        """
+        :returns: The name of the procfs node
+        """
+        return re.search(SchedProcFSNode._re_procfs_node, node).group("name")
+
+    @staticmethod
+    def _packable(node, entries):
+        """
+        :returns: Whether it makes sense to pack a node into a common entry
+        """
+        return (SchedProcFSNode._ends_with_digits(node) and
+                any([SchedProcFSNode._ends_with_digits(x) and
+                     SchedProcFSNode._node_digits(x) != SchedProcFSNode._node_digits(node) and
+                     SchedProcFSNode._node_name(x) == SchedProcFSNode._node_name(node)
+                     for x in entries]))
+
+    @staticmethod
+    def _build_directory(node_name, node_data):
+        if node_name.startswith("domain"):
+            return SchedDomain(node_data)
+        else:
+            return SchedProcFSNode(node_data)
+
+    @staticmethod
+    def _build_entry(node_name, node_data):
+        value = node_data
+
+        # Most nodes just contain numerical data, try to convert
+        try:
+            value = int(value)
+        except ValueError:
+            pass
+
+        return value
+
+    @staticmethod
+    def _build_node(node_name, node_data):
+        if isinstance(node_data, dict):
+            return SchedProcFSNode._build_directory(node_name, node_data)
+        else:
+            return SchedProcFSNode._build_entry(node_name, node_data)
+
+    def __getattr__(self, name):
+        return self._dyn_attrs[name]
+
+    def __init__(self, nodes):
+        self.procfs = nodes
+        # First, reduce the procs fields by packing them if possible
+        # Find which entries can be packed into a common entry
+        packables = {
+            node : SchedProcFSNode._node_name(node) + "s"
+            for node in list(nodes.keys()) if SchedProcFSNode._packable(node, list(nodes.keys()))
+        }
+
+        self._dyn_attrs = {}
+
+        for dest in set(packables.values()):
+            self._dyn_attrs[dest] = {}
+
+        # Pack common entries
+        for key, dest in packables.items():
+            i = SchedProcFSNode._node_digits(key)
+            self._dyn_attrs[dest][i] = self._build_node(key, nodes[key])
+
+        # Build the other nodes
+        for key in nodes.keys():
+            if key in packables:
+                continue
+
+            self._dyn_attrs[key] = self._build_node(key, nodes[key])
+
+
+class SchedDomain(SchedProcFSNode):
+    """
+    Represents a sched domain as seen through procfs
+    """
+    # Domain flags obtained from include/linux/sched/topology.h on v4.17
+    # https://kernel.googlesource.com/pub/scm/linux/kernel/git/torvalds/linux/+/v4.17/include/linux/sched/topology.h#20
+    SD_LOAD_BALANCE        = 0x0001  # Do load balancing on this domain.
+    SD_BALANCE_NEWIDLE     = 0x0002  # Balance when about to become idle
+    SD_BALANCE_EXEC        = 0x0004  # Balance on exec
+    SD_BALANCE_FORK        = 0x0008  # Balance on fork, clone
+    SD_BALANCE_WAKE        = 0x0010  # Balance on wakeup
+    SD_WAKE_AFFINE         = 0x0020  # Wake task to waking CPU
+    SD_ASYM_CPUCAPACITY    = 0x0040  # Groups have different max cpu capacities
+    SD_SHARE_CPUCAPACITY   = 0x0080  # Domain members share cpu capacity
+    SD_SHARE_POWERDOMAIN   = 0x0100  # Domain members share power domain
+    SD_SHARE_PKG_RESOURCES = 0x0200  # Domain members share cpu pkg resources
+    SD_SERIALIZE           = 0x0400  # Only a single load balancing instance
+    SD_ASYM_PACKING        = 0x0800  # Place busy groups earlier in the domain
+    SD_PREFER_SIBLING      = 0x1000  # Prefer to place tasks in a sibling domain
+    SD_OVERLAP             = 0x2000  # sched_domains of this level overlap
+    SD_NUMA                = 0x4000  # cross-node balancing
+    # Only defined in Android
+    # https://android.googlesource.com/kernel/common/+/android-4.14/include/linux/sched/topology.h#29
+    SD_SHARE_CAP_STATES    = 0x8000  # Domain members share capacity state
+
+    # Checked to be valid from v4.4
+    SD_FLAGS_REF_PARTS = (4, 4, 0)
+
+    @staticmethod
+    def check_version(target, logger):
+        """
+        Check the target and see if its kernel version matches our view of the world
+        """
+        parts = target.kernel_version.parts
+        if parts < SchedDomain.SD_FLAGS_REF_PARTS:
+            logger.warn(
+                "Sched domain flags are defined for kernels v{} and up, "
+                "but target is running v{}".format(SchedDomain.SD_FLAGS_REF_PARTS, parts)
+            )
+
+    def has_flags(self, flags):
+        """
+        :returns: Whether 'flags' are set on this sched domain
+        """
+        return self.flags & flags == flags
+
+
+class SchedProcFSData(SchedProcFSNode):
+    """
+    Root class for creating & storing SchedProcFSNode instances
+    """
+    _read_depth = 6
+    sched_domain_root = '/proc/sys/kernel/sched_domain'
+
+    @staticmethod
+    def available(target):
+        return target.directory_exists(SchedProcFSData.sched_domain_root)
+
+    def __init__(self, target, path=None):
+        if not path:
+            path = self.sched_domain_root
+
+        procfs = target.read_tree_values(path, depth=self._read_depth)
+        super(SchedProcFSData, self).__init__(procfs)
+
+
+class SchedModule(Module):
+
+    name = 'sched'
+
+    cpu_sysfs_root = '/sys/devices/system/cpu'
+
+    @staticmethod
+    def probe(target):
+        logger = logging.getLogger(SchedModule.name)
+        SchedDomain.check_version(target, logger)
+
+        return SchedProcFSData.available(target)
+
+    def get_cpu_sd_info(self, cpu):
+        """
+        :returns: An object view of /proc/sys/kernel/sched_domain/cpu<cpu>/*
+        """
+        path = self.target.path.join(
+            SchedProcFSData.sched_domain_root,
+            "cpu{}".format(cpu)
+        )
+
+        return SchedProcFSData(self.target, path)
+
+    def get_sd_info(self):
+        """
+        :returns: An object view of /proc/sys/kernel/sched_domain/*
+        """
+        return SchedProcFSData(self.target)
+
+    def get_capacity(self, cpu):
+        """
+        :returns: The capacity of 'cpu'
+        """
+        return self.get_capacities()[cpu]
+
+    @memoized
+    def has_em(self, cpu, sd=None):
+        """
+        :returns: Whether energy model data is available for 'cpu'
+        """
+        if not sd:
+            sd = SchedProcFSData(self.target, cpu)
+
+        return sd.procfs["domain0"].get("group0", {}).get("energy", {}).get("cap_states") != None
+
+    @memoized
+    def has_dmips_capacity(self, cpu):
+        """
+        :returns: Whether dmips capacity data is available for 'cpu'
+        """
+        return self.target.file_exists(
+            self.target.path.join(self.cpu_sysfs_root, 'cpu{}/cpu_capacity'.format(cpu))
+        )
+
+    @memoized
+    def get_em_capacity(self, cpu, sd=None):
+        """
+        :returns: The maximum capacity value exposed by the EAS energy model
+        """
+        if not sd:
+            sd = SchedProcFSData(self.target, cpu)
+
+        cap_states = sd.domains[0].groups[0].energy.cap_states
+        return int(cap_states.split('\t')[-2])
+
+    @memoized
+    def get_dmips_capacity(self, cpu):
+        """
+        :returns: The capacity value generated from the capacity-dmips-mhz DT entry
+        """
+        return self.target.read_value(
+            self.target.path.join(
+                self.cpu_sysfs_root,
+                'cpu{}/cpu_capacity'.format(cpu)
+            ),
+            int
+        )
+
+    @memoized
+    def get_capacities(self, default=None):
+        """
+        :param default: Default capacity value to find if no data is
+        found in procfs
+
+        :returns: a dictionnary of the shape {cpu : capacity}
+
+        :raises RuntimeError: Raised when no capacity information is
+        found and 'default' is None
+        """
+        cpus = list(range(self.target.number_of_cpus))
+
+        capacities = {}
+        sd_info = self.get_sd_info()
+
+        for cpu in cpus:
+            if self.has_em(cpu, sd_info.cpus[cpu]):
+                capacities[cpu] = self.get_em_capacity(cpu, sd_info.cpus[cpu])
+            elif self.has_dmips_capacity(cpu):
+                capacities[cpu] = self.get_dmips_capacity(cpu)
+            else:
+                if default != None:
+                    capacities[cpu] = default
+                else:
+                    raise RuntimeError('No capacity data for cpu{}'.format(cpu))
+
+        return capacities
+
+    @memoized
+    def get_hz(self):
+        """
+        :returns: The scheduler tick frequency on the target
+        """
+        return int(self.target.config.get('CONFIG_HZ', strict=True))

devlib/module/thermal.py

@@ -1,4 +1,4 @@
-#    Copyright 2015 ARM Limited
+#    Copyright 2015-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -61,8 +61,8 @@ class ThermalZone(object):
         value = self.target.read_value(self.target.path.join(self.path, 'mode'))
         return value == 'enabled'
 
-    def set_mode(self, enable):
+    def set_enabled(self, enabled=True):
-        value = 'enabled' if enable else 'disabled'
+        value = 'enabled' if enabled else 'disabled'
         self.target.write_value(self.target.path.join(self.path, 'mode'), value)
 
     def get_temperature(self):
@@ -100,5 +100,5 @@ class ThermalModule(Module):
 
     def disable_all_zones(self):
         """Disables all the thermal zones in the target"""
-        for zone in self.zones:
+        for zone in self.zones.values():
-            zone.set_mode('disabled')
+            zone.set_enabled(False)

devlib/module/vexpress.py

@@ -1,5 +1,5 @@
 #
-#    Copyright 2015 ARM Limited
+#    Copyright 2015-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,6 +17,7 @@ import os
 import time
 import tarfile
 import shutil
+from subprocess import CalledProcessError
 
 from devlib.module import HardRestModule, BootModule, FlashModule
 from devlib.exception import TargetError, HostError
@@ -25,7 +26,8 @@ from devlib.utils.uefi import UefiMenu, UefiConfig
 from devlib.utils.uboot import UbootMenu
 
 
-AUTOSTART_MESSAGE = 'Press Enter to stop auto boot...'
+OLD_AUTOSTART_MESSAGE = 'Press Enter to stop auto boot...'
+AUTOSTART_MESSAGE = 'Hit any key to stop autoboot:'
 POWERUP_MESSAGE = 'Powering up system...'
 DEFAULT_MCC_PROMPT = 'Cmd>'
 
@@ -51,7 +53,7 @@ class VexpressDtrHardReset(HardRestModule):
         try:
             if self.target.is_connected:
                 self.target.execute('sync')
-        except TargetError:
+        except (TargetError, CalledProcessError):
             pass
         with open_serial_connection(port=self.port,
                                     baudrate=self.baudrate,
@@ -136,18 +138,20 @@ class VexpressBootModule(BootModule):
     def get_through_early_boot(self, tty):
         self.logger.debug('Establishing initial state...')
         tty.sendline('')
-        i = tty.expect([AUTOSTART_MESSAGE, POWERUP_MESSAGE, self.mcc_prompt])
+        i = tty.expect([AUTOSTART_MESSAGE, OLD_AUTOSTART_MESSAGE, POWERUP_MESSAGE, self.mcc_prompt])
-        if i == 2:
+        if i == 3:
             self.logger.debug('Saw MCC prompt.')
             time.sleep(self.short_delay)
             tty.sendline('reboot')
-        elif i == 1:
+        elif i == 2:
             self.logger.debug('Saw powering up message (assuming soft reboot).')
         else:
             self.logger.debug('Saw auto boot message.')
             tty.sendline('')
             time.sleep(self.short_delay)
+            # could be either depending on where in the boot we are
             tty.sendline('reboot')
+            tty.sendline('reset')
 
     def get_uefi_menu(self, tty):
         menu = UefiMenu(tty)
@@ -247,7 +251,7 @@ class VexpressUBoot(VexpressBootModule):
         menu = UbootMenu(tty)
         self.logger.debug('Waiting for U-Boot prompt...')
         menu.open(timeout=120)
-        for var, value in self.env.iteritems():
+        for var, value in self.env.items():
             menu.setenv(var, value)
         menu.boot()
 
@@ -324,7 +328,7 @@ class VersatileExpressFlashModule(FlashModule):
                                     baudrate=self.target.platform.baudrate,
                                     timeout=self.timeout,
                                     init_dtr=0) as tty:
-            i = tty.expect([self.mcc_prompt, AUTOSTART_MESSAGE])
+            i = tty.expect([self.mcc_prompt, AUTOSTART_MESSAGE, OLD_AUTOSTART_MESSAGE])
             if i:
                 tty.sendline('')
             wait_for_vemsd(self.vemsd_mount, tty, self.mcc_prompt, self.short_delay)
@@ -334,7 +338,7 @@ class VersatileExpressFlashModule(FlashModule):
             if images:
                 self._overlay_images(images)
             os.system('sync')
-        except (IOError, OSError), e:
+        except (IOError, OSError) as e:
             msg = 'Could not deploy images to {}; got: {}'
             raise TargetError(msg.format(self.vemsd_mount, e))
         self.target.boot()
@@ -348,7 +352,7 @@ class VersatileExpressFlashModule(FlashModule):
             tar.extractall(self.vemsd_mount)
 
     def _overlay_images(self, images):
-        for dest, src in images.iteritems():
+        for dest, src in images.items():
             dest = os.path.join(self.vemsd_mount, dest)
             self.logger.debug('Copying {} to {}'.format(src, dest))
             shutil.copy(src, dest)
@@ -375,7 +379,7 @@ def wait_for_vemsd(vemsd_mount, tty, mcc_prompt=DEFAULT_MCC_PROMPT, short_delay=
     path = os.path.join(vemsd_mount, 'config.txt')
     if os.path.exists(path):
         return
-    for _ in xrange(attempts):
+    for _ in range(attempts):
         tty.sendline('')  # clear any garbage
         tty.expect(mcc_prompt, timeout=short_delay)
         tty.sendline('usb_on')

devlib/platform/__init__.py

@@ -1,7 +1,22 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import logging
 
 
-BIG_CPUS = ['A15', 'A57', 'A72']
+BIG_CPUS = ['A15', 'A57', 'A72', 'A73']
 
 
 class Platform(object):

devlib/platform/arm.py

@@ -1,4 +1,4 @@
-#    Copyright 2015 ARM Limited
+#    Copyright 2015-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,8 +14,8 @@
 #
 from __future__ import division
 import os
+import sys
 import tempfile
-import csv
 import time
 import pexpect
 
@@ -23,6 +23,7 @@ from devlib.platform import Platform
 from devlib.instrument import Instrument, InstrumentChannel, MeasurementsCsv, Measurement,  CONTINUOUS,  INSTANTANEOUS
 from devlib.exception import TargetError, HostError
 from devlib.host import PACKAGE_BIN_DIRECTORY
+from devlib.utils.csvutil import csvreader, csvwriter
 from devlib.utils.serial_port import open_serial_connection
 
 
@@ -33,6 +34,7 @@ class VersatileExpressPlatform(Platform):
                  core_names=None,
                  core_clusters=None,
                  big_core=None,
+                 model=None,
                  modules=None,
 
                  # serial settings
@@ -61,6 +63,7 @@ class VersatileExpressPlatform(Platform):
                                                        core_names,
                                                        core_clusters,
                                                        big_core,
+                                                       model,
                                                        modules)
         self.serial_port = serial_port
         self.baudrate = baudrate
@@ -86,6 +89,9 @@ class VersatileExpressPlatform(Platform):
     def _init_android_target(self, target):
         if target.connection_settings.get('device') is None:
             addr = self._get_target_ip_address(target)
+            if sys.version_info[0] == 3:
+                # Convert bytes to string for Python3 compatibility
+                addr = addr.decode("utf-8")
             target.connection_settings['device'] = addr + ':5555'
 
     def _init_linux_target(self, target):
@@ -98,12 +104,14 @@ class VersatileExpressPlatform(Platform):
                                     baudrate=self.baudrate,
                                     timeout=30,
                                     init_dtr=0) as tty:
-            tty.sendline('')
+            tty.sendline('su')  # this is, apprently, required to query network device
+                                # info by name on recent Juno builds...
             self.logger.debug('Waiting for the Android shell prompt.')
             tty.expect(target.shell_prompt)
 
             self.logger.debug('Waiting for IP address...')
             wait_start_time = time.time()
+            try:
                 while True:
                     tty.sendline('ip addr list eth0')
                     time.sleep(1)
@@ -114,6 +122,8 @@ class VersatileExpressPlatform(Platform):
                         pass  # We have our own timeout -- see below.
                     if (time.time() - wait_start_time) > self.ready_timeout:
                         raise TargetError('Could not acquire IP address.')
+            finally:
+                tty.sendline('exit')  # exit shell created by "su" call at the start
 
     def _set_hard_reset_method(self, hard_reset_method):
         if hard_reset_method == 'dtr':
@@ -210,22 +220,22 @@ class JunoEnergyInstrument(Instrument):
     mode = CONTINUOUS | INSTANTANEOUS
 
     _channels = [
-        InstrumentChannel('sys_curr', 'sys', 'current'),
+        InstrumentChannel('sys', 'current'),
-        InstrumentChannel('a57_curr', 'a57', 'current'),
+        InstrumentChannel('a57', 'current'),
-        InstrumentChannel('a53_curr', 'a53', 'current'),
+        InstrumentChannel('a53', 'current'),
-        InstrumentChannel('gpu_curr', 'gpu', 'current'),
+        InstrumentChannel('gpu', 'current'),
-        InstrumentChannel('sys_volt', 'sys', 'voltage'),
+        InstrumentChannel('sys', 'voltage'),
-        InstrumentChannel('a57_volt', 'a57', 'voltage'),
+        InstrumentChannel('a57', 'voltage'),
-        InstrumentChannel('a53_volt', 'a53', 'voltage'),
+        InstrumentChannel('a53', 'voltage'),
-        InstrumentChannel('gpu_volt', 'gpu', 'voltage'),
+        InstrumentChannel('gpu', 'voltage'),
-        InstrumentChannel('sys_pow', 'sys', 'power'),
+        InstrumentChannel('sys', 'power'),
-        InstrumentChannel('a57_pow', 'a57', 'power'),
+        InstrumentChannel('a57', 'power'),
-        InstrumentChannel('a53_pow', 'a53', 'power'),
+        InstrumentChannel('a53', 'power'),
-        InstrumentChannel('gpu_pow', 'gpu', 'power'),
+        InstrumentChannel('gpu', 'power'),
-        InstrumentChannel('sys_cenr', 'sys', 'energy'),
+        InstrumentChannel('sys', 'energy'),
-        InstrumentChannel('a57_cenr', 'a57', 'energy'),
+        InstrumentChannel('a57', 'energy'),
-        InstrumentChannel('a53_cenr', 'a53', 'energy'),
+        InstrumentChannel('a53', 'energy'),
-        InstrumentChannel('gpu_cenr', 'gpu', 'energy'),
+        InstrumentChannel('gpu', 'energy'),
     ]
 
     def __init__(self, target):
@@ -243,9 +253,11 @@ class JunoEnergyInstrument(Instrument):
     def setup(self):
         self.binary = self.target.install(os.path.join(PACKAGE_BIN_DIRECTORY,
                                                        self.target.abi, self.binname))
+        self.command = '{} -o {}'.format(self.binary, self.on_target_file)
+        self.command2 = '{}'.format(self.binary)
 
-    def reset(self, sites=None, kinds=None):
+    def reset(self, sites=None, kinds=None, channels=None):
-        super(JunoEnergyInstrument, self).reset(sites, kinds)
+        super(JunoEnergyInstrument, self).reset(sites, kinds, channels)
         self.target.killall(self.binname, as_root=True)
 
     def start(self):
@@ -259,9 +271,8 @@ class JunoEnergyInstrument(Instrument):
         self.target.pull(self.on_target_file, temp_file)
         self.target.remove(self.on_target_file)
 
-        with open(temp_file, 'rb') as fh:
+        with csvreader(temp_file) as reader:
-            reader = csv.reader(fh)
+            headings = next(reader)
-            headings = reader.next()
 
             # Figure out which columns from the collected csv we actually want
             select_columns = []
@@ -271,23 +282,22 @@ class JunoEnergyInstrument(Instrument):
                 except ValueError:
                     raise HostError('Channel "{}" is not in {}'.format(chan.name, temp_file))
 
-            with open(output_file, 'wb') as wfh:
+            with csvwriter(output_file) as writer:
                 write_headings = ['{}_{}'.format(c.site, c.kind)
                                   for c in self.active_channels]
-                writer = csv.writer(wfh)
                 writer.writerow(write_headings)
                 for row in reader:
                     write_row = [row[c] for c in select_columns]
                     writer.writerow(write_row)
 
-        return MeasurementsCsv(output_file, self.active_channels)
+        return MeasurementsCsv(output_file, self.active_channels, sample_rate_hz=10)
 
     def take_measurement(self):
         result = []
         output = self.target.execute(self.command2).split()
-        reader=csv.reader(output)
+        with csvreader(output) as reader:
-        headings=reader.next()
+            headings=next(reader)
-        values = reader.next()
+            values = next(reader)
             for chan in self.active_channels:
                 value = values[headings.index(chan.name)]
                 result.append(Measurement(value, chan))

devlib/platform/gem5.py

@@ -1,4 +1,4 @@
-#    Copyright 2016 ARM Limited
+#    Copyright 2016-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -63,13 +63,12 @@ class Gem5SimulationPlatform(Platform):
 
         # Find the first one that does not exist. Ensures that we do not re-use
         # the directory used by someone else.
-        for i in xrange(sys.maxint):
+        i = 0
         directory = os.path.join(self.gem5_interact_dir, "wa_{}".format(i))
-            try:
+        while os.path.exists(directory):
-                os.stat(directory)
+            i += 1
-                continue
+            directory = os.path.join(self.gem5_interact_dir, "wa_{}".format(i))
-            except OSError:
+
-                break
         self.gem5_interact_dir = directory
         self.logger.debug("Using {} as the temporary directory."
                           .format(self.gem5_interact_dir))
@@ -168,12 +167,17 @@ class Gem5SimulationPlatform(Platform):
         while self.gem5_port is None:
             # Check that gem5 is running!
             if self.gem5.poll():
-                raise TargetError("The gem5 process has crashed with error code {}!".format(self.gem5.poll()))
+                message = "The gem5 process has crashed with error code {}!\n\tPlease see {} for details."
+                raise TargetError(message.format(self.gem5.poll(), self.stderr_file.name))
 
             # Open the stderr file
             with open(self.stderr_filename, 'r') as f:
                 for line in f:
+                    # Look for two different strings, exact wording depends on
+                    # version of gem5
                     m = re.search(r"Listening for system connection on port (?P<port>\d+)", line)
+                    if not m:
+                        m = re.search(r"Listening for connections on port (?P<port>\d+)", line)
                     if m:
                         port = int(m.group('port'))
                         if port >= 3456 and port < 5900:
@@ -200,8 +204,6 @@ class Gem5SimulationPlatform(Platform):
         """
         Deploy m5 if not yet installed
         """
-        m5_path = target.get_installed('m5')
-        if m5_path is None:
         m5_path = self._deploy_m5(target)
         target.conn.m5_path = m5_path
 
@@ -239,6 +241,11 @@ class Gem5SimulationPlatform(Platform):
             if '.bmp' in f:
                 screen_caps.append(f)
 
+        if '{ts}' in filepath:
+            cmd = '{} date -u -Iseconds'
+            ts = self.target.execute(cmd.format(self.target.busybox)).strip()
+            filepath = filepath.format(ts=ts)
+
         successful_capture = False
         if len(screen_caps) == 1:
             # Bail out if we do not have image, and resort to the slower, built

devlib/trace/__init__.py

@@ -1,3 +1,18 @@
+#    Copyright 2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
 import logging
 
 

devlib/trace/ftrace.py

@@ -1,4 +1,4 @@
-#    Copyright 2015 ARM Limited
+#    Copyright 2015-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ import json
 import time
 import re
 import subprocess
+import sys
 
 from devlib.trace import TraceCollector
 from devlib.host import PACKAGE_BIN_DIRECTORY
@@ -60,6 +61,7 @@ class FtraceCollector(TraceCollector):
                  autoview=False,
                  no_install=False,
                  strict=False,
+                 report_on_target=False,
                  ):
         super(FtraceCollector, self).__init__(target)
         self.events = events if events is not None else DEFAULT_EVENTS
@@ -70,7 +72,10 @@ class FtraceCollector(TraceCollector):
         self.automark = automark
         self.autoreport = autoreport
         self.autoview = autoview
-        self.target_output_file = os.path.join(self.target.working_directory, OUTPUT_TRACE_FILE)
+        self.report_on_target = report_on_target
+        self.target_output_file = target.path.join(self.target.working_directory, OUTPUT_TRACE_FILE)
+        text_file_name = target.path.splitext(OUTPUT_TRACE_FILE)[0] + '.txt'
+        self.target_text_file = target.path.join(self.target.working_directory, text_file_name)
         self.target_binary = None
         self.host_binary = None
         self.start_time = None
@@ -93,7 +98,7 @@ class FtraceCollector(TraceCollector):
 
         if not self.target.is_rooted:
             raise TargetError('trace-cmd instrument cannot be used on an unrooted device.')
-        if self.autoreport and self.host_binary is None:
+        if self.autoreport and not self.report_on_target and self.host_binary is None:
             raise HostError('trace-cmd binary must be installed on the host if autoreport=True.')
         if self.autoview and self.kernelshark is None:
             raise HostError('kernelshark binary must be installed on the host if autoview=True.')
@@ -117,7 +122,7 @@ class FtraceCollector(TraceCollector):
                 _event = '*' + event
             event_re = re.compile(_event.replace('*', '.*'))
             # Select events matching the required ones
-            if len(filter(event_re.match, available_events)) == 0:
+            if len(list(filter(event_re.match, available_events))) == 0:
                 message = 'Event [{}] not available for tracing'.format(event)
                 if strict:
                     raise TargetError(message)
@@ -202,20 +207,26 @@ class FtraceCollector(TraceCollector):
 
     def get_trace(self, outfile):
         if os.path.isdir(outfile):
-            outfile = os.path.join(outfile, os.path.dirname(self.target_output_file))
+            outfile = os.path.join(outfile, os.path.basename(self.target_output_file))
-        self.target.execute('{} extract -o {}'.format(self.target_binary, self.target_output_file),
+        self.target.execute('{0} extract -o {1}; chmod 666 {1}'.format(self.target_binary,
+                                                                       self.target_output_file),
                             timeout=TIMEOUT, as_root=True)
 
         # The size of trace.dat will depend on how long trace-cmd was running.
         # Therefore timout for the pull command must also be adjusted
         # accordingly.
-        pull_timeout = 5 * (self.stop_time - self.start_time)
+        pull_timeout = 10 * (self.stop_time - self.start_time)
         self.target.pull(self.target_output_file, outfile, timeout=pull_timeout)
         if not os.path.isfile(outfile):
             self.logger.warning('Binary trace not pulled from device.')
         else:
             if self.autoreport:
                 textfile = os.path.splitext(outfile)[0] + '.txt'
+                if self.report_on_target:
+                    self.generate_report_on_target()
+                    self.target.pull(self.target_text_file,
+                                     textfile, timeout=pull_timeout)
+                else:
                     self.report(outfile, textfile)
             if self.autoview:
                 self.view(outfile)
@@ -266,6 +277,8 @@ class FtraceCollector(TraceCollector):
             self.logger.debug(command)
             process = subprocess.Popen(command, stderr=subprocess.PIPE, shell=True)
             _, error = process.communicate()
+            if sys.version_info[0] == 3:
+                error = error.decode(sys.stdout.encoding, 'replace')
             if process.returncode:
                 raise TargetError('trace-cmd returned non-zero exit code {}'.format(process.returncode))
             if error:
@@ -286,6 +299,12 @@ class FtraceCollector(TraceCollector):
         except OSError:
             raise HostError('Could not find trace-cmd. Please make sure it is installed and is in PATH.')
 
+    def generate_report_on_target(self):
+        command = '{} report {} > {}'.format(self.target_binary,
+                                             self.target_output_file,
+                                             self.target_text_file)
+        self.target.execute(command, timeout=TIMEOUT)
+
     def view(self, binfile):
         check_output('{} {}'.format(self.kernelshark, binfile), shell=True)
 

devlib/trace/logcat.py

@@ -0,0 +1,73 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import re
+import shutil
+
+from devlib.trace import TraceCollector
+from devlib.utils.android import LogcatMonitor
+
+class LogcatCollector(TraceCollector):
+
+    def __init__(self, target, regexps=None):
+        super(LogcatCollector, self).__init__(target)
+        self.regexps = regexps
+        self._collecting = False
+        self._prev_log = None
+
+    def reset(self):
+        """
+        Clear Collector data but do not interrupt collection
+        """
+        if not self._monitor:
+            return
+
+        if self._collecting:
+            self._monitor.clear_log()
+        elif self._prev_log:
+            os.remove(self._prev_log)
+            self._prev_log = None
+
+    def start(self):
+        """
+        Start collecting logcat lines
+        """
+        self._monitor = LogcatMonitor(self.target, self.regexps)
+        if self._prev_log:
+            # Append new data collection to previous collection
+            self._monitor.start(self._prev_log)
+        else:
+            self._monitor.start()
+
+        self._collecting = True
+
+    def stop(self):
+        """
+        Stop collecting logcat lines
+        """
+        if not self._collecting:
+            raise RuntimeError('Logcat monitor not running, nothing to stop')
+
+        self._monitor.stop()
+        self._collecting = False
+        self._prev_log = self._monitor.logfile
+
+    def get_trace(self, outfile):
+        """
+        Output collected logcat lines to designated file
+        """
+        # copy self._monitor.logfile to outfile
+        shutil.copy(self._monitor.logfile, outfile)

devlib/trace/screencapture.py

@@ -0,0 +1,98 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import os
+import sys
+import threading
+import time
+
+from devlib.trace import TraceCollector
+from devlib.exception import WorkerThreadError
+
+
+class ScreenCapturePoller(threading.Thread):
+
+    def __init__(self, target, period, output_path=None, timeout=30):
+        super(ScreenCapturePoller, self).__init__()
+        self.target = target
+        self.logger = logging.getLogger('screencapture')
+        self.period = period
+        self.timeout = timeout
+        self.stop_signal = threading.Event()
+        self.lock = threading.Lock()
+        self.last_poll = 0
+        self.daemon = True
+        self.exc = None
+        self.output_path = output_path
+
+    def run(self):
+        self.logger.debug('Starting screen capture polling')
+        try:
+            while True:
+                if self.stop_signal.is_set():
+                    break
+                with self.lock:
+                    current_time = time.time()
+                    if (current_time - self.last_poll) >= self.period:
+                        self.poll()
+                time.sleep(0.5)
+        except Exception:  # pylint: disable=W0703
+            self.exc = WorkerThreadError(self.name, sys.exc_info())
+
+    def stop(self):
+        self.logger.debug('Stopping screen capture polling')
+        self.stop_signal.set()
+        self.join(self.timeout)
+        if self.is_alive():
+            self.logger.error('Could not join screen capture poller thread.')
+        if self.exc:
+            raise self.exc  # pylint: disable=E0702
+
+    def poll(self):
+        self.last_poll = time.time()
+        self.target.capture_screen(os.path.join(self.output_path, "screencap_{ts}.png"))
+
+
+class ScreenCaptureCollector(TraceCollector):
+
+    def __init__(self, target, output_path=None, period=None):
+        super(ScreenCaptureCollector, self).__init__(target)
+        self._collecting = False
+        self.output_path = output_path
+        self.period = period
+        self.target = target
+        self._poller = ScreenCapturePoller(self.target, self.period,
+                                           self.output_path)
+
+    def reset(self):
+        pass
+
+    def start(self):
+        """
+        Start collecting the screenshots
+        """
+        self._poller.start()
+        self._collecting = True
+
+    def stop(self):
+        """
+        Stop collecting the screenshots
+        """
+        if not self._collecting:
+            raise RuntimeError('Screen capture collector is not running, nothing to stop')
+
+        self._poller.stop()
+        self._collecting = False

devlib/trace/serial_trace.py

@@ -0,0 +1,92 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+from pexpect.exceptions import TIMEOUT
+import shutil
+from tempfile import NamedTemporaryFile
+
+from devlib.trace import TraceCollector
+from devlib.utils.serial_port import get_connection
+
+
+class SerialTraceCollector(TraceCollector):
+
+    @property
+    def collecting(self):
+        return self._collecting
+
+    def __init__(self, target, serial_port, baudrate, timeout=20):
+        super(SerialTraceCollector, self).__init__(target)
+        self.serial_port = serial_port
+        self.baudrate = baudrate
+        self.timeout = timeout
+
+        self._serial_target = None
+        self._conn = None
+        self._tmpfile = None
+        self._collecting = False
+
+    def reset(self):
+        if self._collecting:
+            raise RuntimeError("reset was called whilst collecting")
+
+        if self._tmpfile:
+            self._tmpfile.close()
+            self._tmpfile = None
+
+    def start(self):
+        if self._collecting:
+            raise RuntimeError("start was called whilst collecting")
+
+
+        self._tmpfile = NamedTemporaryFile()
+        self._tmpfile.write("-------- Starting serial logging --------\n")
+
+        self._serial_target, self._conn = get_connection(port=self.serial_port,
+                                                         baudrate=self.baudrate,
+                                                         timeout=self.timeout,
+                                                         logfile=self._tmpfile,
+                                                         init_dtr=0)
+        self._collecting = True
+
+    def stop(self):
+        if not self._collecting:
+            raise RuntimeError("stop was called whilst not collecting")
+
+        # We expect the below to fail, but we need to get pexpect to
+        # do something so that it interacts with the serial device,
+        # and hence updates the logfile.
+        try:
+            self._serial_target.expect(".", timeout=1)
+        except TIMEOUT:
+            pass
+
+        self._serial_target.close()
+        del self._conn
+
+        self._tmpfile.write("-------- Stopping serial logging --------\n")
+
+        self._collecting = False
+
+    def get_trace(self, outfile):
+        if self._collecting:
+            raise RuntimeError("get_trace was called whilst collecting")
+
+        self._tmpfile.flush()
+
+        shutil.copy(self._tmpfile.name, outfile)
+
+        self._tmpfile.close()
+        self._tmpfile = None

devlib/trace/systrace.py

@@ -0,0 +1,173 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#    Copyright 2018 Arm Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import subprocess
+
+from shutil import copyfile
+from tempfile import NamedTemporaryFile
+
+from devlib.exception import TargetError, HostError
+from devlib.trace import TraceCollector
+from devlib.utils.android import platform_tools
+from devlib.utils.misc import memoized
+
+
+DEFAULT_CATEGORIES = [
+    'gfx',
+    'view',
+    'sched',
+    'freq',
+    'idle'
+]
+
+class SystraceCollector(TraceCollector):
+    """
+    A trace collector based on Systrace
+
+    For more details, see https://developer.android.com/studio/command-line/systrace
+
+    :param target: Devlib target
+    :type target: AndroidTarget
+
+    :param outdir: Working directory to use on the host
+    :type outdir: str
+
+    :param categories: Systrace categories to trace. See `available_categories`
+    :type categories: list(str)
+
+    :param buffer_size: Buffer size in kb
+    :type buffer_size: int
+
+    :param strict: Raise an exception if any of the requested categories
+        are not available
+    :type strict: bool
+    """
+
+    @property
+    @memoized
+    def available_categories(self):
+        lines = subprocess.check_output([self.systrace_binary, '-l']).splitlines()
+
+        categories = []
+        for line in lines:
+            categories.append(line.split()[0])
+
+        return categories
+
+    def __init__(self, target,
+                 categories=None,
+                 buffer_size=None,
+                 strict=False):
+
+        super(SystraceCollector, self).__init__(target)
+
+        self.categories = categories or DEFAULT_CATEGORIES
+        self.buffer_size = buffer_size
+
+        self._systrace_process = None
+        self._tmpfile = None
+
+        # Try to find a systrace binary
+        self.systrace_binary = None
+
+        systrace_binary_path = os.path.join(platform_tools, 'systrace', 'systrace.py')
+        if not os.path.isfile(systrace_binary_path):
+            raise HostError('Could not find any systrace binary under {}'.format(platform_tools))
+
+        self.systrace_binary = systrace_binary_path
+
+        # Filter the requested categories
+        for category in self.categories:
+            if category not in self.available_categories:
+                message = 'Category [{}] not available for tracing'.format(category)
+                if strict:
+                    raise TargetError(message)
+                self.logger.warning(message)
+
+        self.categories = list(set(self.categories) & set(self.available_categories))
+        if not self.categories:
+            raise TargetError('None of the requested categories are available')
+
+    def __del__(self):
+        self.reset()
+
+    def _build_cmd(self):
+        self._tmpfile = NamedTemporaryFile()
+
+        self.systrace_cmd = '{} -o {} -e {}'.format(
+            self.systrace_binary,
+            self._tmpfile.name,
+            self.target.adb_name
+        )
+
+        if self.buffer_size:
+            self.systrace_cmd += ' -b {}'.format(self.buffer_size)
+
+        self.systrace_cmd += ' {}'.format(' '.join(self.categories))
+
+    def reset(self):
+        if self._systrace_process:
+            self.stop()
+
+        if self._tmpfile:
+            self._tmpfile.close()
+            self._tmpfile = None
+
+    def start(self):
+        if self._systrace_process:
+            raise RuntimeError("Tracing is already underway, call stop() first")
+
+        self.reset()
+
+        self._build_cmd()
+
+        self._systrace_process = subprocess.Popen(
+            self.systrace_cmd,
+            stdin=subprocess.PIPE,
+            shell=True
+        )
+
+    def stop(self):
+        if not self._systrace_process:
+            raise RuntimeError("No tracing to stop, call start() first")
+
+        # Systrace expects <enter> to stop
+        self._systrace_process.communicate('\n')
+        self._systrace_process = None
+
+    def get_trace(self, outfile):
+        if self._systrace_process:
+            raise RuntimeError("Tracing is underway, call stop() first")
+
+        if not self._tmpfile:
+            raise RuntimeError("No tracing data available")
+
+        copyfile(self._tmpfile.name, outfile)

devlib/utils/android.py

@@ -1,4 +1,4 @@
-#    Copyright 2013-2015 ARM Limited
+#    Copyright 2013-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -20,25 +20,36 @@ Utility functions for working with Android devices through adb.
 """
 # pylint: disable=E1103
 import os
+import pexpect
 import time
 import subprocess
 import logging
 import re
+import threading
+import tempfile
+import queue
+import sys
 from collections import defaultdict
 
 from devlib.exception import TargetError, HostError, DevlibError
-from devlib.utils.misc import check_output, which, memoized
+from devlib.utils.misc import check_output, which, memoized, ABI_MAP
 from devlib.utils.misc import escape_single_quotes, escape_double_quotes
+from devlib import host
 
 
 logger = logging.getLogger('android')
 
 MAX_ATTEMPTS = 5
-AM_START_ERROR = re.compile(r"Error: Activity class {[\w|.|/]*} does not exist")
+AM_START_ERROR = re.compile(r"Error: Activity.*")
 
 # See:
 # http://developer.android.com/guide/topics/manifest/uses-sdk-element.html#ApiLevels
 ANDROID_VERSION_MAP = {
+    28: 'P',
+    27: 'OREO_MR1',
+    26: 'OREO',
+    25: 'NOUGAT_MR1',
+    24: 'NOUGAT',
     23: 'MARSHMALLOW',
     22: 'LOLLYPOP_MR1',
     21: 'LOLLYPOP',
@@ -64,6 +75,12 @@ ANDROID_VERSION_MAP = {
     1: 'BASE',
 }
 
+# See https://developer.android.com/reference/android/content/Intent.html#setFlags(int)
+INTENT_FLAGS = {
+    'ACTIVITY_NEW_TASK' : 0x10000000,
+    'ACTIVITY_CLEAR_TASK' : 0x00008000
+}
+
 
 # Initialized in functions near the botton of the file
 android_home = None
@@ -83,7 +100,7 @@ class AndroidProperties(object):
         self._properties = dict(re.findall(r'\[(.*?)\]:\s+\[(.*?)\]', text))
 
     def iteritems(self):
-        return self._properties.iteritems()
+        return iter(self._properties.items())
 
     def __iter__(self):
         return iter(self._properties)
@@ -116,6 +133,7 @@ class ApkInfo(object):
 
     version_regex = re.compile(r"name='(?P<name>[^']+)' versionCode='(?P<vcode>[^']+)' versionName='(?P<vname>[^']+)'")
     name_regex = re.compile(r"name='(?P<name>[^']+)'")
+    permission_regex = re.compile(r"name='(?P<permission>[^']+)'")
 
     def __init__(self, path=None):
         self.path = path
@@ -124,13 +142,21 @@ class ApkInfo(object):
         self.label = None
         self.version_name = None
         self.version_code = None
+        self.native_code = None
+        self.permissions = []
         self.parse(path)
 
     def parse(self, apk_path):
         _check_env()
         command = [aapt, 'dump', 'badging', apk_path]
         logger.debug(' '.join(command))
-        output = subprocess.check_output(command)
+        try:
+            output = subprocess.check_output(command, stderr=subprocess.STDOUT)
+            if sys.version_info[0] == 3:
+                output = output.decode(sys.stdout.encoding, 'replace')
+        except subprocess.CalledProcessError as e:
+            raise HostError('Error parsing APK file {}. `aapt` says:\n{}'
+                            .format(apk_path, e.output))
         for line in output.split('\n'):
             if line.startswith('application-label:'):
                 self.label = line.split(':')[1].strip().replace('\'', '')
@@ -143,6 +169,23 @@ class ApkInfo(object):
             elif line.startswith('launchable-activity:'):
                 match = self.name_regex.search(line)
                 self.activity = match.group('name')
+            elif line.startswith('native-code'):
+                apk_abis = [entry.strip() for entry in line.split(':')[1].split("'") if entry.strip()]
+                mapped_abis = []
+                for apk_abi in apk_abis:
+                    found = False
+                    for abi, architectures in ABI_MAP.items():
+                        if apk_abi in architectures:
+                            mapped_abis.append(abi)
+                            found = True
+                            break
+                    if not found:
+                        mapped_abis.append(apk_abi)
+                self.native_code = mapped_abis
+            elif line.startswith('uses-permission:'):
+                match = self.permission_regex.search(line)
+                if match:
+                    self.permissions.append(match.group('permission'))
             else:
                 pass  # not interested
 
@@ -159,18 +202,6 @@ class AdbConnection(object):
     def name(self):
         return self.device
 
-    @property
-    @memoized
-    def newline_separator(self):
-        output = adb_command(self.device,
-                             "shell '({}); echo \"\n$?\"'".format(self.ls_command))
-        if output.endswith('\r\n'):
-            return '\r\n'
-        elif output.endswith('\n'):
-            return '\n'
-        else:
-            raise DevlibError("Unknown line ending")
-
     # Again, we need to handle boards where the default output format from ls is
     # single column *and* boards where the default output is multi-column.
     # We need to do this purely because the '-1' option causes errors on older
@@ -178,7 +209,7 @@ class AdbConnection(object):
     def _setup_ls(self):
         command = "shell '(ls -1); echo \"\n$?\"'"
         try:
-            output = adb_command(self.device, command, timeout=self.timeout)
+            output = adb_command(self.device, command, timeout=self.timeout, adb_server=self.adb_server)
         except subprocess.CalledProcessError as e:
             raise HostError(
                 'Failed to set up ls command on Android device. Output:\n'
@@ -189,13 +220,14 @@ class AdbConnection(object):
             self.ls_command = 'ls -1'
         else:
             self.ls_command = 'ls'
-        logger.info("ls command is set to {}".format(self.ls_command))
+        logger.debug("ls command is set to {}".format(self.ls_command))
 
-    def __init__(self, device=None, timeout=None, platform=None):
+    def __init__(self, device=None, timeout=None, platform=None, adb_server=None):
         self.timeout = timeout if timeout is not None else self.default_timeout
         if device is None:
-            device = adb_get_device(timeout=timeout)
+            device = adb_get_device(timeout=timeout, adb_server=adb_server)
         self.device = device
+        self.adb_server = adb_server
         adb_connect(self.device)
         AdbConnection.active_connections[self.device] += 1
         self._setup_ls()
@@ -206,7 +238,7 @@ class AdbConnection(object):
         command = "push '{}' '{}'".format(source, dest)
         if not os.path.exists(source):
             raise HostError('No such file "{}"'.format(source))
-        return adb_command(self.device, command, timeout=timeout)
+        return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
 
     def pull(self, source, dest, timeout=None):
         if timeout is None:
@@ -215,18 +247,18 @@ class AdbConnection(object):
         if os.path.isdir(dest) and \
            ('*' in source or '?' in source):
             command = 'shell {} {}'.format(self.ls_command, source)
-            output = adb_command(self.device, command, timeout=timeout)
+            output = adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
             for line in output.splitlines():
                 command = "pull '{}' '{}'".format(line.strip(), dest)
-                adb_command(self.device, command, timeout=timeout)
+                adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
             return
         command = "pull '{}' '{}'".format(source, dest)
-        return adb_command(self.device, command, timeout=timeout)
+        return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
 
     def execute(self, command, timeout=None, check_exit_code=False,
                 as_root=False, strip_colors=True):
         return adb_shell(self.device, command, timeout, check_exit_code,
-                         as_root, self.newline_separator)
+                         as_root, adb_server=self.adb_server)
 
     def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
         return adb_background_shell(self.device, command, stdout, stderr, as_root)
@@ -258,7 +290,7 @@ def fastboot_flash_partition(partition, path_to_image):
     fastboot_command(command)
 
 
-def adb_get_device(timeout=None):
+def adb_get_device(timeout=None, adb_server=None):
     """
     Returns the serial number of a connected android device.
 
@@ -267,13 +299,17 @@ def adb_get_device(timeout=None):
     """
     # TODO this is a hacky way to issue a adb command to all listed devices
 
+    # Ensure server is started so the 'daemon started successfully' message
+    # doesn't confuse the parsing below
+    adb_command(None, 'start-server', adb_server=adb_server)
+
     # The output of calling adb devices consists of a heading line then
     # a list of the devices sperated by new line
     # The last line is a blank new line. in otherwords, if there is a device found
     # then the output length is 2 + (1 for each device)
     start = time.time()
     while True:
-        output = adb_command(None, "devices").splitlines()  # pylint: disable=E1103
+        output = adb_command(None, "devices", adb_server=adb_server).splitlines()  # pylint: disable=E1103
         output_length = len(output)
         if output_length == 3:
             # output[1] is the 2nd line in the output which has the device name
@@ -292,15 +328,12 @@ def adb_get_device(timeout=None):
 
 def adb_connect(device, timeout=None, attempts=MAX_ATTEMPTS):
     _check_env()
-    # Connect is required only for ADB-over-IP
-    if "." not in device:
-        logger.debug('Device connected via USB, connect not required')
-        return
     tries = 0
     output = None
     while tries <= attempts:
         tries += 1
         if device:
+            if "." in device: # Connect is required only for ADB-over-IP
                 command = 'adb connect {}'.format(device)
                 logger.debug(command)
                 output, _ = check_output(command, shell=True, timeout=timeout)
@@ -329,7 +362,7 @@ def adb_disconnect(device):
 def _ping(device):
     _check_env()
     device_string = ' -s {}'.format(device) if device else ''
-    command = "adb{} shell \"ls / > /dev/null\"".format(device_string)
+    command = "adb{} shell \"ls /data/local/tmp > /dev/null\"".format(device_string)
     logger.debug(command)
     result = subprocess.call(command, stderr=subprocess.PIPE, shell=True)
     if not result:
@@ -339,11 +372,14 @@ def _ping(device):
 
 
 def adb_shell(device, command, timeout=None, check_exit_code=False,
-              as_root=False, newline_separator='\r\n'):  # NOQA
+              as_root=False, adb_server=None):  # NOQA
     _check_env()
     if as_root:
         command = 'echo \'{}\' | su'.format(escape_single_quotes(command))
-    device_part = ['-s', device] if device else []
+    device_part = []
+    if adb_server:
+        device_part = ['-H', adb_server]
+    device_part += ['-s', device] if device else []
 
     # On older combinations of ADB/Android versions, the adb host command always
     # exits with 0 if it was able to run the command on the target, even if the
@@ -353,12 +389,16 @@ def adb_shell(device, command, timeout=None, check_exit_code=False,
     adb_shell_command = '({}); echo \"\n$?\"'.format(command)
     actual_command = ['adb'] + device_part + ['shell', adb_shell_command]
     logger.debug('adb {} shell {}'.format(' '.join(device_part), command))
-    raw_output, error = check_output(actual_command, timeout, shell=False)
+    try:
+        raw_output, _ = check_output(actual_command, timeout, shell=False, combined_output=True)
+    except subprocess.CalledProcessError as e:
+        raise TargetError(str(e))
+
     if raw_output:
         try:
-            output, exit_code, _ = raw_output.rsplit(newline_separator, 2)
+            output, exit_code, _ = raw_output.replace('\r\n', '\n').replace('\r', '\n').rsplit('\n', 2)
         except ValueError:
-            exit_code, _ = raw_output.rsplit(newline_separator, 1)
+            exit_code, _ = raw_output.replace('\r\n', '\n').replace('\r', '\n').rsplit('\n', 1)
             output = ''
     else:  # raw_output is empty
         exit_code = '969696'  # just because
@@ -366,23 +406,24 @@ def adb_shell(device, command, timeout=None, check_exit_code=False,
 
     if check_exit_code:
         exit_code = exit_code.strip()
+        re_search = AM_START_ERROR.findall(output)
         if exit_code.isdigit():
             if int(exit_code):
                 message = ('Got exit code {}\nfrom target command: {}\n'
-                           'STDOUT: {}\nSTDERR: {}')
+                           'OUTPUT: {}')
-                raise TargetError(message.format(exit_code, command, output, error))
+                raise TargetError(message.format(exit_code, command, output))
-            elif AM_START_ERROR.findall(output):
+            elif re_search:
-                message = 'Could not start activity; got the following:'
-                message += '\n{}'.format(AM_START_ERROR.findall(output)[0])
-                raise TargetError(message)
-        else:  # not all digits
-            if AM_START_ERROR.findall(output):
                 message = 'Could not start activity; got the following:\n{}'
-                raise TargetError(message.format(AM_START_ERROR.findall(output)[0]))
+                raise TargetError(message.format(re_search[0]))
+        else:  # not all digits
+            if re_search:
+                message = 'Could not start activity; got the following:\n{}'
+                raise TargetError(message.format(re_search[0]))
             else:
                 message = 'adb has returned early; did not get an exit code. '\
-                          'Was kill-server invoked?'
+                          'Was kill-server invoked?\nOUTPUT:\n-----\n{}\n'\
-                raise TargetError(message)
+                          '-----'
+                raise TargetError(message.format(raw_output))
 
     return output
 
@@ -401,8 +442,8 @@ def adb_background_shell(device, command,
     return subprocess.Popen(full_command, stdout=stdout, stderr=stderr, shell=True)
 
 
-def adb_list_devices():
+def adb_list_devices(adb_server=None):
-    output = adb_command(None, 'devices')
+    output = adb_command(None, 'devices',adb_server=adb_server)
     devices = []
     for line in output.splitlines():
         parts = [p.strip() for p in line.split()]
@@ -411,14 +452,39 @@ def adb_list_devices():
     return devices
 
 
-def adb_command(device, command, timeout=None):
+def get_adb_command(device, command, timeout=None,adb_server=None):
     _check_env()
-    device_string = ' -s {}'.format(device) if device else ''
+    device_string = ""
-    full_command = "adb{} {}".format(device_string, command)
+    if adb_server != None:
+        device_string = ' -H {}'.format(adb_server)
+    device_string += ' -s {}'.format(device) if device else ''
+    return "adb{} {}".format(device_string, command)
+
+def adb_command(device, command, timeout=None,adb_server=None):
+    full_command = get_adb_command(device, command, timeout, adb_server)
     logger.debug(full_command)
     output, _ = check_output(full_command, timeout, shell=True)
     return output
 
+def grant_app_permissions(target, package):
+    """
+    Grant an app all the permissions it may ask for
+    """
+    dumpsys = target.execute('dumpsys package {}'.format(package))
+
+    permissions = re.search(
+        'requested permissions:\s*(?P<permissions>(android.permission.+\s*)+)', dumpsys
+    )
+    if permissions is None:
+        return
+    permissions = permissions.group('permissions').replace(" ", "").splitlines()
+
+    for permission in permissions:
+        try:
+            target.execute('pm grant {} {}'.format(package, permission))
+        except TargetError:
+            logger.debug('Cannot grant {}'.format(permission))
+
 
 # Messy environment initialisation stuff...
 
@@ -486,3 +552,144 @@ def _check_env():
         platform_tools = _env.platform_tools
         adb = _env.adb
         aapt = _env.aapt
+
+class LogcatMonitor(object):
+    """
+    Helper class for monitoring Anroid's logcat
+
+    :param target: Android target to monitor
+    :type target: :class:`AndroidTarget`
+
+    :param regexps: List of uncompiled regular expressions to filter on the
+                    device. Logcat entries that don't match any will not be
+                    seen. If omitted, all entries will be sent to host.
+    :type regexps: list(str)
+    """
+
+    @property
+    def logfile(self):
+        return self._logfile
+
+    def __init__(self, target, regexps=None):
+        super(LogcatMonitor, self).__init__()
+
+        self.target = target
+        self._regexps = regexps
+
+    def start(self, outfile=None):
+        """
+        Start logcat and begin monitoring
+
+        :param outfile: Optional path to file to store all logcat entries
+        :type outfile: str
+        """
+        if outfile:
+            self._logfile = open(outfile, 'w')
+        else:
+            self._logfile = tempfile.NamedTemporaryFile()
+
+        self.target.clear_logcat()
+
+        logcat_cmd = 'logcat'
+
+        # Join all requested regexps with an 'or'
+        if self._regexps:
+            regexp = '{}'.format('|'.join(self._regexps))
+            if len(self._regexps) > 1:
+                regexp = '({})'.format(regexp)
+            # Logcat on older version of android do not support the -e argument
+            # so fall back to using grep.
+            if self.target.get_sdk_version() > 23:
+                logcat_cmd = '{} -e "{}"'.format(logcat_cmd, regexp)
+            else:
+                logcat_cmd = '{} | grep "{}"'.format(logcat_cmd, regexp)
+
+        logcat_cmd = get_adb_command(self.target.conn.device, logcat_cmd)
+
+        logger.debug('logcat command ="{}"'.format(logcat_cmd))
+        self._logcat = pexpect.spawn(logcat_cmd, logfile=self._logfile)
+
+    def stop(self):
+        self._logcat.terminate()
+        self._logfile.close()
+
+    def get_log(self):
+        """
+        Return the list of lines found by the monitor
+        """
+        # Unless we tell pexect to 'expect' something, it won't read from
+        # logcat's buffer or write into our logfile. We'll need to force it to
+        # read any pending logcat output.
+        while True:
+            try:
+                read_size = 1024 * 8
+                # This will read up to read_size bytes, but only those that are
+                # already ready (i.e. it won't block). If there aren't any bytes
+                # already available it raises pexpect.TIMEOUT.
+                buf = self._logcat.read_nonblocking(read_size, timeout=0)
+
+                # We can't just keep calling read_nonblocking until we get a
+                # pexpect.TIMEOUT (i.e. until we don't find any available
+                # bytes), because logcat might be writing bytes the whole time -
+                # in that case we might never return from this function. In
+                # fact, we only care about bytes that were written before we
+                # entered this function. So, if we read read_size bytes (as many
+                # as we were allowed to), then we'll assume there are more bytes
+                # that have already been sitting in the output buffer of the
+                # logcat command. If not, we'll assume we read everything that
+                # had already been written.
+                if len(buf) == read_size:
+                    continue
+                else:
+                    break
+            except pexpect.TIMEOUT:
+                # No available bytes to read. No prob, logcat just hasn't
+                # printed anything since pexpect last read from its buffer.
+                break
+
+        with open(self._logfile.name) as fh:
+            return [line for line in fh]
+
+    def clear_log(self):
+        with open(self._logfile.name, 'w') as fh:
+            pass
+
+    def search(self, regexp):
+        """
+        Search a line that matches a regexp in the logcat log
+        Return immediatly
+        """
+        return [line for line in self.get_log() if re.match(regexp, line)]
+
+    def wait_for(self, regexp, timeout=30):
+        """
+        Search a line that matches a regexp in the logcat log
+        Wait for it to appear if it's not found
+
+        :param regexp: regexp to search
+        :type regexp: str
+
+        :param timeout: Timeout in seconds, before rasing RuntimeError.
+                        ``None`` means wait indefinitely
+        :type timeout: number
+
+        :returns: List of matched strings
+        """
+        log = self.get_log()
+        res = [line for line in log if re.match(regexp, line)]
+
+        # Found some matches, return them
+        if len(res) > 0:
+            return res
+
+        # Store the number of lines we've searched already, so we don't have to
+        # re-grep them after 'expect' returns
+        next_line_num = len(log)
+
+        try:
+            self._logcat.expect(regexp, timeout=timeout)
+        except pexpect.TIMEOUT:
+            raise RuntimeError('Logcat monitor timeout ({}s)'.format(timeout))
+
+        return [line for line in self.get_log()[next_line_num:]
+                if re.match(regexp, line)]

devlib/utils/csvutil.py

@@ -0,0 +1,100 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+'''
+Due to the change in the nature of "binary mode" when opening files in
+Python 3, the way files need to be opened for ``csv.reader`` and ``csv.writer``
+is different from Python 2.
+
+The functions in this module are intended to hide these differences allowing
+the rest of the code to create csv readers/writers without worrying about which
+Python version it is running under.
+
+First up are ``csvwriter`` and ``csvreader`` context mangers that handle the
+opening and closing of the underlying file. These are intended to replace the
+most common usage pattern
+
+.. code-block:: python
+
+    with open(filepath, 'wb') as wfh:  # or open(filepath, 'w', newline='') in Python 3
+        writer = csv.writer(wfh)
+        writer.writerows(data)
+
+
+with
+
+.. code-block:: python
+
+    with csvwriter(filepath) as writer:
+        writer.writerows(data)
+
+
+``csvreader`` works in an analogous way. ``csvreader`` and ``writer`` can take
+additional arguments which will be passed directly to the
+``csv.reader``/``csv.writer`` calls.
+
+In some cases, it is desirable not to use a context manager (e.g. if the
+reader/writer is intended to be returned from the function that creates it. For
+such cases, alternative functions, ``create_reader`` and ``create_writer``,
+exit. These return a two-tuple, with the created reader/writer as the first
+element, and the corresponding ``FileObject`` as the second. It is the
+responsibility of the calling code to ensure that the file is closed properly.
+
+'''
+import csv
+import sys
+from contextlib import contextmanager
+
+
+@contextmanager
+def csvwriter(filepath, *args, **kwargs):
+    if sys.version_info[0] == 3:
+        wfh = open(filepath, 'w', newline='')
+    else:
+        wfh = open(filepath, 'wb')
+
+    try:
+        yield csv.writer(wfh, *args, **kwargs)
+    finally:
+        wfh.close()
+
+
+@contextmanager
+def csvreader(filepath, *args, **kwargs):
+    if sys.version_info[0] == 3:
+        fh = open(filepath, 'r', newline='')
+    else:
+        fh = open(filepath, 'rb')
+
+    try:
+        yield csv.reader(fh, *args, **kwargs)
+    finally:
+        fh.close()
+
+
+def create_writer(filepath, *args, **kwargs):
+    if sys.version_info[0] == 3:
+        wfh = open(filepath, 'w', newline='')
+    else:
+        wfh = open(filepath, 'wb')
+    return csv.writer(wfh, *args, **kwargs), wfh
+
+
+def create_reader(filepath, *args, **kwargs):
+    if sys.version_info[0] == 3:
+        fh = open(filepath, 'r', newline='')
+    else:
+        fh = open(filepath, 'rb')
+    return csv.reader(fh, *args, **kwargs), fh

devlib/utils/gem5.py

@@ -0,0 +1,53 @@
+#    Copyright 2017-2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import logging
+
+from devlib.utils.types import numeric
+
+
+GEM5STATS_FIELD_REGEX = re.compile("^(?P<key>[^- ]\S*) +(?P<value>[^#]+).+$")
+GEM5STATS_DUMP_HEAD = '---------- Begin Simulation Statistics ----------'
+GEM5STATS_DUMP_TAIL = '---------- End Simulation Statistics   ----------'
+GEM5STATS_ROI_NUMBER = 8
+
+logger = logging.getLogger('gem5')
+
+
+def iter_statistics_dump(stats_file):
+    '''
+    Yields statistics dumps as dicts. The parameter is assumed to be a stream 
+    reading from the statistics log file.
+    '''
+    cur_dump = {}
+    while True:
+        line = stats_file.readline()
+        if not line:
+            break
+        if GEM5STATS_DUMP_TAIL in line:
+            yield cur_dump
+            cur_dump = {}
+        else:
+            res = GEM5STATS_FIELD_REGEX.match(line) 
+            if res:
+                k = res.group("key")
+                vtext = res.group("value")
+                try:
+                    v = list(map(numeric, vtext.split()))
+                    cur_dump[k] = v[0] if len(v)==1 else set(v)
+                except ValueError:
+                    msg = 'Found non-numeric entry in gem5 stats ({}: {})'
+                    logger.warning(msg.format(k, vtext))
+

devlib/utils/misc.py

@@ -1,4 +1,4 @@
-#    Copyright 2013-2015 ARM Limited
+#    Copyright 2013-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -30,18 +30,21 @@ import pkgutil
 import logging
 import random
 import ctypes
+import threading
 from operator import itemgetter
 from itertools import groupby
 from functools import partial
 
 import wrapt
+from past.builtins import basestring
 
 from devlib.exception import HostError, TimeoutError
+from functools import reduce
 
 
 # ABI --> architectures list
 ABI_MAP = {
-    'armeabi': ['armeabi', 'armv7', 'armv7l', 'armv7el', 'armv7lh'],
+    'armeabi': ['armeabi', 'armv7', 'armv7l', 'armv7el', 'armv7lh', 'armeabi-v7a'],
     'arm64': ['arm64', 'armv8', 'arm64-v8a', 'aarch64'],
 }
 
@@ -79,9 +82,19 @@ CPU_PART_MAP = {
         0xd08: {None: 'A72'},
         0xd09: {None: 'A73'},
     },
+    0x42: {  # Broadcom
+        0x516: {None: 'Vulcan'},
+    },
+    0x43: {  # Cavium
+        0x0a1: {None: 'Thunderx'},
+        0x0a2: {None: 'Thunderx81xx'},
+    },
     0x4e: {  # Nvidia
         0x0: {None: 'Denver'},
     },
+    0x50: {  # AppliedMicro
+        0x0: {None: 'xgene'},
+    },
     0x51: {  # Qualcomm
         0x02d: {None: 'Scorpion'},
         0x04d: {None: 'MSM8960'},
@@ -91,6 +104,10 @@ CPU_PART_MAP = {
         },
         0x205: {0x1: 'KryoSilver'},
         0x211: {0x1: 'KryoGold'},
+        0x800: {None: 'Falkor'},
+    },
+    0x53: {  # Samsung LSI
+        0x001: {0x1: 'MongooseM1'},
     },
     0x56: {  # Marvell
         0x131: {
@@ -121,9 +138,13 @@ def preexec_function():
 
 
 check_output_logger = logging.getLogger('check_output')
+# Popen is not thread safe. If two threads attempt to call it at the same time,
+# one may lock up. See https://bugs.python.org/issue12739.
+check_output_lock = threading.Lock()
 
 
-def check_output(command, timeout=None, ignore=None, inputtext=None, **kwargs):
+def check_output(command, timeout=None, ignore=None, inputtext=None,
+                 combined_output=False, **kwargs):
     """This is a version of subprocess.check_output that adds a timeout parameter to kill
     the subprocess if it does not return within the specified time."""
     # pylint: disable=too-many-branches
@@ -144,9 +165,14 @@ def check_output(command, timeout=None, ignore=None, inputtext=None, **kwargs):
         except OSError:
             pass  # process may have already terminated.
 
-    process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+    with check_output_lock:
+        stderr = subprocess.STDOUT if combined_output else subprocess.PIPE
+        process = subprocess.Popen(command,
+                                   stdout=subprocess.PIPE,
+                                   stderr=stderr,
                                    stdin=subprocess.PIPE,
-                               preexec_fn=preexec_function, **kwargs)
+                                   preexec_fn=preexec_function,
+                                   **kwargs)
 
     if timeout:
         timer = threading.Timer(timeout, callback, [process.pid, ])
@@ -154,6 +180,11 @@ def check_output(command, timeout=None, ignore=None, inputtext=None, **kwargs):
 
     try:
         output, error = process.communicate(inputtext)
+        if sys.version_info[0] == 3:
+            # Currently errors=replace is needed as 0x8c throws an error
+            output = output.decode(sys.stdout.encoding, "replace")
+            if error:
+                error =  error.decode(sys.stderr.encoding, "replace")
     finally:
         if timeout:
             timer.cancel()
@@ -161,9 +192,9 @@ def check_output(command, timeout=None, ignore=None, inputtext=None, **kwargs):
     retcode = process.poll()
     if retcode:
         if retcode == -9:  # killed, assume due to timeout callback
-            raise TimeoutError(command, output='\n'.join([output, error]))
+            raise TimeoutError(command, output='\n'.join([output or '', error or '']))
         elif ignore != 'all' and retcode not in ignore:
-            raise subprocess.CalledProcessError(retcode, command, output='\n'.join([output, error]))
+            raise subprocess.CalledProcessError(retcode, command, output='\n'.join([output or '', error or '']))
     return output, error
 
 
@@ -235,8 +266,8 @@ def _merge_two_dicts(base, other, list_duplicates='all', match_types=False,  # p
                      dict_type=dict, should_normalize=True, should_merge_lists=True):
     """Merge dicts normalizing their keys."""
     merged = dict_type()
-    base_keys = base.keys()
+    base_keys = list(base.keys())
-    other_keys = other.keys()
+    other_keys = list(other.keys())
     norm = normalize if should_normalize else lambda x, y: x
 
     base_only = []
@@ -368,7 +399,7 @@ def normalize(value, dict_type=dict):
     no surrounding whitespace, underscore-delimited strings."""
     if isinstance(value, dict):
         normalized = dict_type()
-        for k, v in value.iteritems():
+        for k, v in value.items():
             key = k.strip().lower().replace(' ', '_')
             normalized[key] = normalize(v, dict_type)
         return normalized
@@ -400,11 +431,16 @@ def escape_double_quotes(text):
     return re.sub(r'\\("|\')', r'\\\\\1', text).replace('\"', '\\\"')
 
 
+def escape_spaces(text):
+    """Escape spaces in the specified text"""
+    return text.replace(' ', '\ ')
+
+
 def getch(count=1):
     """Read ``count`` characters from standard input."""
     if os.name == 'nt':
         import msvcrt  # pylint: disable=F0401
-        return ''.join([msvcrt.getch() for _ in xrange(count)])
+        return ''.join([msvcrt.getch() for _ in range(count)])
     else:  # assume Unix
         import tty  # NOQA
         import termios  # NOQA
@@ -431,6 +467,19 @@ def as_relative(path):
     return path.lstrip(os.sep)
 
 
+def commonprefix(file_list, sep=os.sep):
+    """
+    Find the lowest common base folder of a passed list of files.
+    """
+    common_path = os.path.commonprefix(file_list)
+    cp_split = common_path.split(sep)
+    other_split = file_list[0].split(sep)
+    last = len(cp_split) - 1
+    if cp_split[last] != other_split[last]:
+        cp_split = cp_split[:-1]
+    return sep.join(cp_split)
+
+
 def get_cpu_mask(cores):
     """Return a string with the hex for the cpu mask for the specified core numbers."""
     mask = 0
@@ -460,8 +509,8 @@ def which(name):
             return None
 
 
-_bash_color_regex = re.compile('\x1b\\[[0-9;]+m')
+# This matches most ANSI escape sequences, not just colors
-
+_bash_color_regex = re.compile(r'\x1b\[[0-9;]*[a-zA-Z]')
 
 def strip_bash_colors(text):
     return _bash_color_regex.sub('', text)
@@ -469,7 +518,7 @@ def strip_bash_colors(text):
 
 def get_random_string(length):
     """Returns a random ASCII string of the specified length)."""
-    return ''.join(random.choice(string.ascii_letters + string.digits) for _ in xrange(length))
+    return ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(length))
 
 
 class LoadSyntaxError(Exception):
@@ -486,13 +535,18 @@ class LoadSyntaxError(Exception):
 
 RAND_MOD_NAME_LEN = 30
 BAD_CHARS = string.punctuation + string.whitespace
+if sys.version_info[0] == 3:
+    TRANS_TABLE = str.maketrans(BAD_CHARS, '_' * len(BAD_CHARS))
+else:
     TRANS_TABLE = string.maketrans(BAD_CHARS, '_' * len(BAD_CHARS))
 
 
 def to_identifier(text):
     """Converts text to a valid Python identifier by replacing all
-    whitespace and punctuation."""
+    whitespace and punctuation and adding a prefix if starting with a digit"""
-    return re.sub('_+', '_', text.translate(TRANS_TABLE))
+    if text[:1].isdigit():
+        text = '_' + text
+    return re.sub('_+', '_', str(text).translate(TRANS_TABLE))
 
 
 def unique(alist):
@@ -513,8 +567,8 @@ def ranges_to_list(ranges_string):
     values = []
     for rg in ranges_string.split(','):
         if '-' in rg:
-            first, last = map(int, rg.split('-'))
+            first, last = list(map(int, rg.split('-')))
-            values.extend(xrange(first, last + 1))
+            values.extend(range(first, last + 1))
         else:
             values.append(int(rg))
     return values
@@ -523,8 +577,8 @@ def ranges_to_list(ranges_string):
 def list_to_ranges(values):
     """Converts a list, e.g ``[0,2,3,4]``, into a sysfs-style ranges string, e.g. ``"0,2-4"``"""
     range_groups = []
-    for _, g in groupby(enumerate(values), lambda (i, x): i - x):
+    for _, g in groupby(enumerate(values), lambda i_x: i_x[0] - i_x[1]):
-        range_groups.append(map(itemgetter(1), g))
+        range_groups.append(list(map(itemgetter(1), g)))
     range_strings = []
     for group in range_groups:
         if len(group) == 1:
@@ -547,7 +601,7 @@ def mask_to_list(mask):
     """Converts the specfied integer bitmask into a list of
     indexes of bits that are set in the mask."""
     size = len(bin(mask)) - 2  # because of "0b"
-    return [size - i - 1 for i in xrange(size)
+    return [size - i - 1 for i in range(size)
             if mask & (1 << size - i - 1)]
 
 
@@ -592,7 +646,7 @@ def memoized(wrapped, instance, args, kwargs):
     def memoize_wrapper(*args, **kwargs):
         id_string = func_id + ','.join([__get_memo_id(a) for a in  args])
         id_string += ','.join('{}={}'.format(k, v)
-                              for k, v in kwargs.iteritems())
+                              for k, v in kwargs.items())
         if id_string not in __memo_cache:
             __memo_cache[id_string] = wrapped(*args, **kwargs)
         return __memo_cache[id_string]

devlib/utils/parse_aep.py

@@ -0,0 +1,536 @@
+#!/usr/bin/env python
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+#    Copyright 2018 Linaro Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import getopt
+import subprocess
+import logging
+import signal
+import serial
+import time
+import math
+
+logger = logging.getLogger('aep-parser')
+
+class AepParser(object):
+    prepared = False
+
+    @staticmethod
+    def topology_from_data(array, topo):
+    # Extract topology information for the data file
+    # The header of a data file looks like this ('#' included):
+    # configuration: <file path>
+    # config_name: <file name>
+    # trigger: 0.400000V (hyst 0.200000V) 0.000000W (hyst 0.200000W) 400us
+    # date: Fri, 10 Jun 2016 11:25:07 +0200
+    # host: <host name>
+    #
+    # CHN_0	    Pretty_name_0   PARENT_0    Color0  Class0
+    # CHN_1	    Pretty_name_1   PARENT_1    Color1  Class1
+    # CHN_2	    Pretty_name_2   PARENT_2    Color2  Class2
+    # CHN_3	    Pretty_name_3   PARENT_3    Color3  Class3
+    # ..
+    # CHN_N	    Pretty_name_N   PARENT_N    ColorN  ClassN
+    #
+
+        info = {}
+
+        if len(array) == 6:
+            info['name'] = array[1]
+            info['parent'] = array[3]
+            info['pretty'] = array[2]
+            # add an entry for both name and pretty name in order to not parse
+            # the whole dict when looking for a parent and the parent of parent
+            topo[array[1]] = info
+            topo[array[2]] = info
+        return topo
+
+    @staticmethod
+    def create_virtual(topo, label, hide, duplicate):
+    # Create a list of virtual power domain that are the sum of others
+    # A virtual domain is the parent of several channels but is not sampled by a
+    # channel
+    # This can be useful if a power domain is supplied by 2 power rails
+        virtual = {}
+
+        # Create an entry for each virtual parent
+        for supply in topo.keys():
+            index = topo[supply]['index']
+            # Don't care of hidden columns
+            if hide[index]:
+                continue
+
+            # Parent is in the topology
+            parent = topo[supply]['parent']
+            if parent in topo:
+                continue
+
+            if parent not in virtual:
+                virtual[parent] = { supply : index }
+
+            virtual[parent][supply] = index
+
+        # Remove parent with 1 child as they don't give more information than their
+        # child
+        for supply in list(virtual.keys()):
+            if len(virtual[supply]) == 1:
+                del virtual[supply];
+
+        for supply in list(virtual.keys()):
+            # Add label, hide and duplicate columns for virtual domains
+            hide.append(0)
+            duplicate.append(1)
+            label.append(supply)
+
+        return virtual
+
+    @staticmethod
+    def get_label(array):
+    # Get the label of each column
+    # Remove unit '(X)' from the end of the label
+        label = [""]*len(array)
+        unit = [""]*len(array)
+
+        label[0] = array[0]
+        unit[0] = "(S)"
+        for i in range(1,len(array)):
+            label[i] = array[i][:-3]
+            unit[i] = array[i][-3:]
+
+        return label, unit
+
+    @staticmethod
+    def filter_column(label, unit, topo):
+    # Filter columns
+    # We don't parse Volt and Amper columns: put in hide list
+    # We don't add in Total a column that is the child of another one: put in duplicate list
+
+        # By default we hide all columns
+        hide = [1] * len(label)
+        # By default we assume that there is no child
+        duplicate = [0] * len(label)
+
+        for i in range(len(label)):
+            # We only care about time and Watt
+            if label[i] == 'time':
+                hide[i] = 0
+                continue
+
+            if '(W)' not in unit[i]:
+                continue
+
+            hide[i] = 0
+
+            #label is pretty name
+            pretty = label[i]
+
+            # We don't add a power domain that is already accounted by its parent
+            if topo[pretty]['parent'] in topo:
+                duplicate[i] = 1
+
+            # Set index, that will be used by virtual domain
+            topo[topo[pretty]['name']]['index'] = i
+
+            # remove pretty element that is useless now
+            del topo[pretty]
+
+        return hide, duplicate
+
+    @staticmethod
+    def parse_text(array, hide):
+        data = [0]*len(array)
+        for i in range(len(array)):
+            if hide[i]:
+                continue
+
+            try:
+                data[i] = int(float(array[i])*1000000)
+            except ValueError:
+                continue
+
+        return data
+
+    @staticmethod
+    def add_virtual_data(data, virtual):
+        # write virtual domain
+        for parent in virtual.keys():
+            power = 0
+            for child in list(virtual[parent].values()):
+                try:
+                    power += data[child]
+                except IndexError:
+                    continue
+            data.append(power)
+
+        return data
+
+    @staticmethod
+    def delta_nrj(array, delta, min, max, hide):
+    # Compute the energy consumed in this time slice and add it
+    # delta[0] is used to save the last time stamp
+
+        if (delta[0] < 0):
+            delta[0] = array[0]
+
+        time = array[0] - delta[0]
+        if (time <= 0):
+            return delta
+
+        for i in range(len(array)):
+            if hide[i]:
+                continue
+
+            try:
+                data = array[i]
+            except ValueError:
+                continue
+
+            if (data < min[i]):
+                min[i] = data
+            if (data > max[i]):
+                max[i] = data
+            delta[i] += time * data
+
+        # save last time stamp
+        delta[0] = array[0]
+
+        return delta
+
+    def output_label(self, label, hide):
+        self.fo.write(label[0]+"(uS)")
+        for i in range(1, len(label)):
+            if hide[i]:
+                continue
+            self.fo.write(" "+label[i]+"(uW)")
+
+        self.fo.write("\n")
+
+    def output_power(self, array, hide):
+        #skip partial line. Most probably the last one
+        if len(array) < len(hide):
+            return
+
+        # write not hidden colums
+        self.fo.write(str(array[0]))
+        for i in range(1, len(array)):
+            if hide[i]:
+                continue
+
+            self.fo.write(" "+str(array[i]))
+
+        self.fo.write("\n")
+
+    def prepare(self, infile, outfile, summaryfile):
+
+        try:
+            self.fi = open(infile, "r")
+        except IOError:
+            logger.warn('Unable to open input file {}'.format(infile))
+            logger.warn('Usage: parse_arp.py -i <inputfile> [-o <outputfile>]')
+            sys.exit(2)
+
+        self.parse = True
+        if len(outfile) > 0:
+            try:
+                self.fo = open(outfile, "w")
+            except IOError:
+                logger.warn('Unable to create {}'.format(outfile))
+                self.parse = False
+        else:
+                self.parse = False
+
+        self.summary = True
+        if len(summaryfile) > 0:
+            try:
+                self.fs = open(summaryfile, "w")
+            except IOError:
+                logger.warn('Unable to create {}'.format(summaryfile))
+                self.fs = sys.stdout
+        else:
+                self.fs = sys.stdout
+
+        self.prepared = True
+
+    def unprepare(self):
+        if not self.prepared:
+            # nothing has been prepared
+            return
+
+        self.fi.close()
+
+        if self.parse:
+            self.fo.close()
+
+        self.prepared = False
+
+    def parse_aep(self, start=0, lenght=-1):
+    # Parse aep data and calculate the energy consumed
+        begin = 0
+
+        label_line = 1
+
+        topo = {}
+
+        lines = self.fi.readlines()
+
+        for myline in lines:
+            array  = myline.split()
+
+            if "#" in myline:
+                # update power topology
+                topo = self.topology_from_data(array, topo)
+                continue
+
+            if label_line:
+                label_line = 0
+                # 1st line not starting with # gives label of each column
+                label, unit = self.get_label(array)
+                # hide useless columns and detect channels that are children
+                # of other channels
+                hide, duplicate = self.filter_column(label, unit, topo)
+
+                # Create virtual power domains
+                virtual = self.create_virtual(topo, label, hide, duplicate)
+                if self.parse:
+                    self.output_label(label, hide)
+
+                logger.debug('Topology : {}'.format(topo))
+                logger.debug('Virtual power domain : {}'.format(virtual))
+                logger.debug('Duplicated power domain : : {}'.format(duplicate))
+                logger.debug('Name of columns : {}'.format(label))
+                logger.debug('Hidden columns : {}'.format(hide))
+                logger.debug('Unit of columns : {}'.format(unit))
+
+                # Init arrays
+                nrj = [0]*len(label)
+                min = [100000000]*len(label)
+                max = [0]*len(label)
+                offset = [0]*len(label)
+
+                continue
+
+            # convert text to int and unit to micro-unit
+            data = self.parse_text(array, hide)
+
+            # get 1st time stamp
+            if begin <= 0:
+                being = data[0]
+
+            # skip data before start
+            if (data[0]-begin) < start:
+                continue
+
+            # stop after lenght
+            if lenght >= 0 and (data[0]-begin) > (start + lenght):
+                continue
+
+            # add virtual domains
+            data = self.add_virtual_data(data, virtual)
+
+            # extract power figures
+            self.delta_nrj(data, nrj, min, max, hide)
+
+            # write data into new file
+            if self.parse:
+                self.output_power(data, hide)
+
+        # if there is no data just return
+        if label_line or len(nrj) == 1:
+            raise ValueError('No data found in the data file. Please check the Arm Energy Probe')
+            return
+
+        # display energy consumption of each channel and total energy consumption
+        total = 0
+        results_table = {}
+        for i in range(1, len(nrj)):
+            if hide[i]:
+                continue
+
+            nrj[i] -= offset[i] * nrj[0]
+
+            total_nrj = nrj[i]/1000000000000.0
+            duration = (max[0]-min[0])/1000000.0
+            channel_name = label[i]
+            average_power = total_nrj/duration
+
+            self.fs.write("Total nrj: %8.3f J for %s -- duration %8.3f sec -- min %8.3f W -- max %8.3f W\n" % (nrj[i]/1000000000000.0, label[i], (max[0]-min[0])/1000000.0, min[i]/1000000.0, max[i]/1000000.0))
+
+            # store each AEP channel info  except Platform in the results table
+            results_table[channel_name] = total_nrj, average_power
+
+            if (min[i] < offset[i]):
+                self.fs.write ("!!! Min below offset\n")
+
+            if duplicate[i]:
+                continue
+
+            total += nrj[i]
+
+        self.fs.write ("Total nrj: %8.3f J for %s -- duration %8.3f sec\n" % (total/1000000000000.0, "Platform ", (max[0]-min[0])/1000000.0))
+
+        total_nrj = total/1000000000000.0
+        duration = (max[0]-min[0])/1000000.0
+        average_power = total_nrj/duration
+
+        # store AEP Platform channel info in the results table
+        results_table["Platform"] = total_nrj, average_power
+
+        return results_table
+
+    def topology_from_config(self, topofile):
+        try:
+            ft = open(topofile, "r")
+        except IOError:
+            logger.warn('Unable to open config file {}'.format(topofile))
+            return
+        lines = ft.readlines()
+
+        topo = {}
+        virtual = {}
+        name = ""
+        offset = 0
+        index = 0
+        #parse config file
+        for myline in lines:
+            if myline.startswith("#"):
+                # skip comment
+                continue
+
+            if myline == "\n":
+                # skip empty line
+                continue
+
+            if name == "":
+                # 1st valid line is the config's name
+                name = myline
+                continue
+
+            if not myline.startswith((' ', '\t')):
+                # new device path
+                offset = index
+                continue
+
+            # Get parameters of channel configuration
+            items = myline.split()
+
+            info = {}
+            info['name'] = items[0]
+            info['parent'] = items[9]
+            info['pretty'] = items[8]
+            info['index'] = int(items[2])+offset
+
+            # Add channel
+            topo[items[0]] = info
+
+            # Increase index
+            index +=1
+
+
+        # Create an entry for each virtual parent
+        for supply in topo.keys():
+            # Parent is in the topology
+            parent = topo[supply]['parent']
+            if parent in topo:
+                continue
+
+            if parent not in virtual:
+                virtual[parent] = { supply : topo[supply]['index'] }
+
+            virtual[parent][supply] = topo[supply]['index']
+
+
+        # Remove parent with 1 child as they don't give more information than their
+        # child
+        for supply in list(virtual.keys()):
+            if len(virtual[supply]) == 1:
+                del virtual[supply];
+
+        topo_list = ['']*(1+len(topo)+len(virtual))
+        topo_list[0] = 'time'
+        for chnl in topo.keys():
+            topo_list[topo[chnl]['index']] = chnl
+        for chnl in virtual.keys():
+            index +=1
+            topo_list[index] = chnl
+
+        ft.close()
+
+        return topo_list
+
+    def __del__(self):
+        self.unprepare()
+
+if __name__ == '__main__':
+
+    def handleSigTERM(signum, frame):
+        sys.exit(2)
+
+    signal.signal(signal.SIGTERM, handleSigTERM)
+    signal.signal(signal.SIGINT, handleSigTERM)
+
+    logger.setLevel(logging.WARN)
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+    logger.addHandler(ch)
+
+    infile = ""
+    outfile = ""
+    figurefile = ""
+    start = 0
+    lenght = -1
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "i:vo:s:l:t:")
+    except getopt.GetoptError as err:
+        print(str(err)) # will print something like "option -a not recognized"
+        sys.exit(2)
+
+    for o, a in opts:
+        if o == "-i":
+            infile = a
+        if o == "-v":
+            logger.setLevel(logging.DEBUG)
+        if o == "-o":
+            parse = True
+            outfile = a
+        if o == "-s":
+            start = int(float(a)*1000000)
+        if o == "-l":
+            lenght = int(float(a)*1000000)
+        if o == "-t":
+            topofile = a
+            parser = AepParser()
+            print(parser.topology_from_config(topofile))
+            exit(0)
+
+    parser = AepParser()
+    parser.prepare(infile, outfile, figurefile)
+    parser.parse_aep(start, lenght)

devlib/utils/rendering.py

@@ -0,0 +1,272 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import logging
+import os
+import re
+import shutil
+import sys
+import tempfile
+import threading
+import time
+from collections import namedtuple, OrderedDict
+from distutils.version import LooseVersion
+
+from devlib.exception  import WorkerThreadError, TargetNotRespondingError, TimeoutError
+from devlib.utils.csvutil import csvwriter
+
+
+logger = logging.getLogger('rendering')
+
+SurfaceFlingerFrame = namedtuple('SurfaceFlingerFrame',
+                                 'desired_present_time actual_present_time frame_ready_time')
+
+VSYNC_INTERVAL = 16666667
+
+
+class FrameCollector(threading.Thread):
+
+    def __init__(self, target, period):
+        super(FrameCollector, self).__init__()
+        self.target = target
+        self.period = period
+        self.stop_signal = threading.Event()
+        self.frames = []
+
+        self.temp_file = None
+        self.refresh_period = None
+        self.drop_threshold = None
+        self.unresponsive_count = 0
+        self.last_ready_time = None
+        self.exc = None
+        self.header = None
+
+    def run(self):
+        logger.debug('Surface flinger frame data collection started.')
+        try:
+            self.stop_signal.clear()
+            fd, self.temp_file = tempfile.mkstemp()
+            logger.debug('temp file: {}'.format(self.temp_file))
+            wfh = os.fdopen(fd, 'wb')
+            try:
+                while not self.stop_signal.is_set():
+                    self.collect_frames(wfh)
+                    time.sleep(self.period)
+            finally:
+                wfh.close()
+        except (TargetNotRespondingError, TimeoutError):  # pylint: disable=W0703
+            raise
+        except Exception as e:  # pylint: disable=W0703
+            logger.warning('Exception on collector thread: {}({})'.format(e.__class__.__name__, e))
+            self.exc = WorkerThreadError(self.name, sys.exc_info())
+        logger.debug('Surface flinger frame data collection stopped.')
+
+    def stop(self):
+        self.stop_signal.set()
+        self.join()
+        if self.unresponsive_count:
+            message = 'FrameCollector was unrepsonsive {} times.'.format(self.unresponsive_count)
+            if self.unresponsive_count > 10:
+                logger.warning(message)
+            else:
+                logger.debug(message)
+        if self.exc:
+            raise self.exc  # pylint: disable=E0702
+
+    def process_frames(self, outfile=None):
+        if not self.temp_file:
+            raise RuntimeError('Attempting to process frames before running the collector')
+        with open(self.temp_file) as fh:
+            self._process_raw_file(fh)
+        if outfile:
+            shutil.copy(self.temp_file, outfile)
+        os.unlink(self.temp_file)
+        self.temp_file = None
+
+    def write_frames(self, outfile, columns=None):
+        if columns is None:
+            header = self.header
+            frames = self.frames
+        else:
+            indexes = []
+            for c in columns:
+                if c not in self.header:
+                    msg = 'Invalid column "{}"; must be in {}'
+                    raise ValueError(msg.format(c, self.header))
+                indexes.append(self.header.index(c))
+            frames = [[f[i] for i in indexes] for f in self.frames]
+            header = columns
+        with csvwriter(outfile) as writer:
+            if header:
+                writer.writerow(header)
+            writer.writerows(frames)
+
+    def collect_frames(self, wfh):
+        raise NotImplementedError()
+
+    def clear(self):
+        raise NotImplementedError()
+
+    def _process_raw_file(self, fh):
+        raise NotImplementedError()
+
+
+class SurfaceFlingerFrameCollector(FrameCollector):
+
+    def __init__(self, target, period, view, header=None):
+        super(SurfaceFlingerFrameCollector, self).__init__(target, period)
+        self.view = view
+        self.header = header or SurfaceFlingerFrame._fields
+
+    def collect_frames(self, wfh):
+        for activity in self.list():
+            if activity == self.view:
+                wfh.write(self.get_latencies(activity))
+
+    def clear(self):
+        self.target.execute('dumpsys SurfaceFlinger --latency-clear ')
+
+    def get_latencies(self, activity):
+        cmd = 'dumpsys SurfaceFlinger --latency "{}"'
+        return self.target.execute(cmd.format(activity))
+
+    def list(self):
+        text = self.target.execute('dumpsys SurfaceFlinger --list')
+        return text.replace('\r\n', '\n').replace('\r', '\n').split('\n')
+
+    def _process_raw_file(self, fh):
+        text = fh.read().replace('\r\n', '\n').replace('\r', '\n')
+        for line in text.split('\n'):
+            line = line.strip()
+            if line:
+                self._process_trace_line(line)
+
+    def _process_trace_line(self, line):
+        parts = line.split()
+        if len(parts) == 3:
+            frame = SurfaceFlingerFrame(*list(map(int, parts)))
+            if not frame.frame_ready_time:
+                return # "null" frame
+            if frame.frame_ready_time <= self.last_ready_time:
+                return  # duplicate frame
+            if (frame.frame_ready_time - frame.desired_present_time) > self.drop_threshold:
+                logger.debug('Dropping bogus frame {}.'.format(line))
+                return  # bogus data
+            self.last_ready_time = frame.frame_ready_time
+            self.frames.append(frame)
+        elif len(parts) == 1:
+            self.refresh_period = int(parts[0])
+            self.drop_threshold = self.refresh_period * 1000
+        elif 'SurfaceFlinger appears to be unresponsive, dumping anyways' in line:
+            self.unresponsive_count += 1
+        else:
+            logger.warning('Unexpected SurfaceFlinger dump output: {}'.format(line))
+
+
+def read_gfxinfo_columns(target):
+    output = target.execute('dumpsys gfxinfo --list framestats')
+    lines = iter(output.split('\n'))
+    for line in lines:
+        if line.startswith('---PROFILEDATA---'):
+            break
+    columns_line = next(lines)
+    return columns_line.split(',')[:-1]  # has a trailing ','
+
+
+class GfxinfoFrameCollector(FrameCollector):
+
+    def __init__(self, target, period, package, header=None):
+        super(GfxinfoFrameCollector, self).__init__(target, period)
+        self.package = package
+        self.header =  None
+        self._init_header(header)
+
+    def collect_frames(self, wfh):
+        cmd = 'dumpsys gfxinfo {} framestats'
+        wfh.write(self.target.execute(cmd.format(self.package)))
+
+    def clear(self):
+        pass
+
+    def _init_header(self, header):
+        if header is not None:
+            self.header = header
+        else:
+            self.header = read_gfxinfo_columns(self.target)
+
+    def _process_raw_file(self, fh):
+        found = False
+        try:
+            last_vsync = 0
+            while True:
+                for line in fh:
+                    if line.startswith('---PROFILEDATA---'):
+                        found = True
+                        break
+
+                next(fh)  # headers
+                for line in fh:
+                    if line.startswith('---PROFILEDATA---'):
+                        break
+                    entries = list(map(int, line.strip().split(',')[:-1]))  # has a trailing ','
+                    if entries[1] <= last_vsync:
+                        continue  # repeat frame
+                    last_vsync = entries[1]
+                    self.frames.append(entries)
+        except StopIteration:
+            pass
+        if not found:
+            logger.warning('Could not find frames data in gfxinfo output')
+            return
+
+
+def _file_reverse_iter(fh, buf_size=1024):
+    fh.seek(0, os.SEEK_END)
+    offset = 0
+    file_size = remaining_size = fh.tell()
+    while remaining_size > 0:
+        offset = min(file_size, offset + buf_size)
+        fh.seek(file_size - offset)
+        buf = fh.read(min(remaining_size, buf_size))
+        remaining_size -= buf_size
+        yield buf
+
+
+def gfxinfo_get_last_dump(filepath):
+    """
+    Return the last gfxinfo dump from the frame collector's raw output.
+
+    """
+    record = ''
+    with open(filepath, 'r') as fh:
+        fh_iter = _file_reverse_iter(fh)
+        try:
+            while True:
+                buf = next(fh_iter)
+                ix = buf.find('** Graphics')
+                if ix >= 0:
+                    return buf[ix:] + record
+
+                ix = buf.find(' **\n')
+                if ix >= 0:
+                    buf =  next(fh_iter) + buf
+                    ix = buf.find('** Graphics')
+                    if ix < 0:
+                        msg = '"{}" appears to be corrupted'
+                        raise RuntimeError(msg.format(filepath))
+                    return buf[ix:] + record
+                record = buf + record
+        except StopIteration:
+            pass

devlib/utils/serial_port.py

@@ -1,4 +1,4 @@
-#    Copyright 2013-2015 ARM Limited
+#    Copyright 2013-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -32,6 +32,14 @@ from pexpect import EOF, TIMEOUT  # NOQA pylint: disable=W0611
 from devlib.exception import HostError
 
 
+class SerialLogger(Logger):
+
+    write = Logger.debug
+
+    def flush(self):
+        pass
+
+
 def pulse_dtr(conn, state=True, duration=0.1):
     """Set the DTR line of the specified serial connection to the specified state
     for the specified duration (note: the initial state of the line is *not* checked."""
@@ -40,19 +48,19 @@ def pulse_dtr(conn, state=True, duration=0.1):
     conn.setDTR(not state)
 
 
-def get_connection(timeout, init_dtr=None, logcls=Logger,
+def get_connection(timeout, init_dtr=None, logcls=SerialLogger,
-                   *args, **kwargs):
+                   logfile=None, *args, **kwargs):
     if init_dtr is not None:
         kwargs['dsrdtr'] = True
     try:
         conn = serial.Serial(*args, **kwargs)
     except serial.SerialException as e:
-        raise HostError(e.message)
+        raise HostError(str(e))
     if init_dtr is not None:
         conn.setDTR(init_dtr)
     conn.nonblocking()
     conn.flushOutput()
-    target = fdpexpect.fdspawn(conn.fileno(), timeout=timeout)
+    target = fdpexpect.fdspawn(conn.fileno(), timeout=timeout, logfile=logfile)
     target.logfile_read = logcls('read')
     target.logfile_send = logcls('send')
 
@@ -83,7 +91,7 @@ def write_characters(conn, line, delay=0.05):
 
 @contextmanager
 def open_serial_connection(timeout, get_conn=False, init_dtr=None,
-                           logcls=Logger, *args, **kwargs):
+                           logcls=SerialLogger, *args, **kwargs):
     """
     Opens a serial connection to a device.
 

devlib/utils/ssh.py

@@ -1,4 +1,4 @@
-#    Copyright 2014-2015 ARM Limited
+#    Copyright 2014-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -23,6 +23,7 @@ import threading
 import tempfile
 import shutil
 import socket
+import sys
 import time
 
 import pexpect
@@ -34,7 +35,9 @@ else:
 from pexpect import EOF, TIMEOUT, spawn
 
 from devlib.exception import HostError, TargetError, TimeoutError
-from devlib.utils.misc import which, strip_bash_colors, escape_single_quotes, check_output
+from devlib.utils.misc import which, strip_bash_colors, check_output
+from devlib.utils.misc import (escape_single_quotes, escape_double_quotes,
+                               escape_spaces)
 from devlib.utils.types import boolean
 
 
@@ -160,7 +163,8 @@ class SshConnection(object):
                  telnet=False,
                  password_prompt=None,
                  original_prompt=None,
-                 platform=None
+                 platform=None,
+                 sudo_cmd="sudo -- sh -c '{}'"
                  ):
         self.host = host
         self.username = username
@@ -169,16 +173,21 @@ class SshConnection(object):
         self.port = port
         self.lock = threading.Lock()
         self.password_prompt = password_prompt if password_prompt is not None else self.default_password_prompt
+        self.sudo_cmd = sudo_cmd
         logger.debug('Logging in {}@{}'.format(username, host))
         timeout = timeout if timeout is not None else self.default_timeout
         self.conn = ssh_get_shell(host, username, password, self.keyfile, port, timeout, False, None)
 
     def push(self, source, dest, timeout=30):
-        dest = '{}@{}:{}'.format(self.username, self.host, dest)
+        dest = '"{}"@"{}":"{}"'.format(escape_double_quotes(self.username),
+                                       escape_spaces(escape_double_quotes(self.host)),
+                                       escape_spaces(escape_double_quotes(dest)))
         return self._scp(source, dest, timeout)
 
     def pull(self, source, dest, timeout=30):
-        source = '{}@{}:{}'.format(self.username, self.host, source)
+        source = '"{}"@"{}":"{}"'.format(escape_double_quotes(self.username),
+                                         escape_spaces(escape_double_quotes(self.host)),
+                                         escape_spaces(escape_double_quotes(source)))
         return self._scp(source, dest, timeout)
 
     def execute(self, command, timeout=None, check_exit_code=True,
@@ -212,7 +221,7 @@ class SshConnection(object):
             port_string = '-p {}'.format(self.port) if self.port else ''
             keyfile_string = '-i {}'.format(self.keyfile) if self.keyfile else ''
             if as_root:
-                command = "sudo -- sh -c '{}'".format(command)
+                command = self.sudo_cmd.format(command)
             command = '{} {} {} {}@{} {}'.format(ssh, keyfile_string, port_string, self.username, self.host, command)
             logger.debug(command)
             if self.password:
@@ -228,7 +237,7 @@ class SshConnection(object):
     def cancel_running_command(self):
         # simulate impatiently hitting ^C until command prompt appears
         logger.debug('Sending ^C')
-        for _ in xrange(self.max_cancel_attempts):
+        for _ in range(self.max_cancel_attempts):
             self.conn.sendline(chr(3))
             if self.conn.prompt(0.1):
                 return True
@@ -240,7 +249,7 @@ class SshConnection(object):
             # As we're already root, there is no need to use sudo.
             as_root = False
         if as_root:
-            command = "sudo -- sh -c '{}'".format(escape_single_quotes(command))
+            command = self.sudo_cmd.format(escape_single_quotes(command))
             if log:
                 logger.debug(command)
             self.conn.sendline(command)
@@ -255,6 +264,9 @@ class SshConnection(object):
         timed_out = self._wait_for_prompt(timeout)
         # the regex removes line breaks potential introduced when writing
         # command to shell.
+        if sys.version_info[0] == 3:
+            output = process_backspaces(self.conn.before.decode(sys.stdout.encoding, 'replace'))
+        else:
             output = process_backspaces(self.conn.before)
         output = re.sub(r'\r([^\n])', r'\1', output)
         if '\r\n' in output: # strip the echoed command
@@ -282,16 +294,18 @@ class SshConnection(object):
         port_string = '-P {}'.format(self.port) if (self.port and self.port != 22) else ''
         keyfile_string = '-i {}'.format(self.keyfile) if self.keyfile else ''
         command = '{} -r {} {} {} {}'.format(scp, keyfile_string, port_string, source, dest)
-        pass_string = ''
+        command_redacted = command
         logger.debug(command)
         if self.password:
             command = _give_password(self.password, command)
+            command_redacted = command.replace(self.password, '<redacted>')
         try:
             check_output(command, timeout=timeout, shell=True)
         except subprocess.CalledProcessError as e:
-            raise subprocess.CalledProcessError(e.returncode, e.cmd.replace(pass_string, ''), e.output)
+            raise HostError("Failed to copy file with '{}'. Output:\n{}".format(
+                command_redacted, e.output))
         except TimeoutError as e:
-            raise TimeoutError(e.command.replace(pass_string, ''), e.output)
+            raise TimeoutError(command_redacted, e.output)
 
 
 class TelnetConnection(SshConnection):
@@ -328,6 +342,7 @@ class Gem5Connection(TelnetConnection):
                  timeout=None,
                  password_prompt=None,
                  original_prompt=None,
+                 strip_echoed_commands=False,
                  ):
         if host is not None:
             host_system = socket.gethostname()
@@ -344,6 +359,8 @@ class Gem5Connection(TelnetConnection):
         self.is_rooted = True
         self.password = None
         self.port = None
+        # Flag to indicate whether commands are echoed by the simulated system
+        self.strip_echoed_commands = strip_echoed_commands
         # Long timeouts to account for gem5 being slow
         # Can be overriden if the given timeout is longer
         self.default_timeout = 3600
@@ -439,25 +456,39 @@ class Gem5Connection(TelnetConnection):
         # First check if the connection is set up to interact with gem5
         self._check_ready()
 
-        filename = os.path.basename(source)
+        result = self._gem5_shell("ls {}".format(source))
+        files = strip_bash_colors(result).split()
 
-        logger.debug("pull_file {} {}".format(source, filename))
+        for filename in files:
-        # We don't check the exit code here because it is non-zero if the source
+            dest_file = os.path.basename(filename)
-        # and destination are the same. The ls below will cause an error if the
+            logger.debug("pull_file {} {}".format(filename, dest_file))
-        # file was not where we expected it to be.
+            # writefile needs the file to be copied to be in the current
-        if os.path.dirname(source) != os.getcwd():
+            # working directory so if needed, copy to the working directory
-            self._gem5_shell("cat '{}' > '{}'".format(source, filename))
+            # We don't check the exit code here because it is non-zero if the
+            # source and destination are the same. The ls below will cause an
+            # error if the file was not where we expected it to be.
+            if os.path.isabs(source):
+                if os.path.dirname(source) != self.execute('pwd',
+                                              check_exit_code=False):
+                    self._gem5_shell("cat '{}' > '{}'".format(filename,
+                                                              dest_file))
             self._gem5_shell("sync")
-        self._gem5_shell("ls -la {}".format(filename))
+            self._gem5_shell("ls -la {}".format(dest_file))
             logger.debug('Finished the copy in the simulator')
-        self._gem5_util("writefile {}".format(filename))
+            self._gem5_util("writefile {}".format(dest_file))
 
             if 'cpu' not in filename:
-            while not os.path.exists(os.path.join(self.gem5_out_dir, filename)):
+                while not os.path.exists(os.path.join(self.gem5_out_dir,
+                                                      dest_file)):
                     time.sleep(1)
 
             # Perform the local move
-        shutil.move(os.path.join(self.gem5_out_dir, filename), dest)
+            if os.path.exists(os.path.join(dest, dest_file)):
+                logger.warning(
+                            'Destination file {} already exists!'\
+                            .format(dest_file))
+            else:
+                shutil.move(os.path.join(self.gem5_out_dir, dest_file), dest)
             logger.debug("Pull complete.")
 
     def execute(self, command, timeout=1000, check_exit_code=True,
@@ -468,7 +499,9 @@ class Gem5Connection(TelnetConnection):
         # First check if the connection is set up to interact with gem5
         self._check_ready()
 
-        output = self._gem5_shell(command, as_root=as_root)
+        output = self._gem5_shell(command,
+                                  check_exit_code=check_exit_code,
+                                  as_root=as_root)
         if strip_colors:
             output = strip_bash_colors(output)
         return output
@@ -503,6 +536,10 @@ class Gem5Connection(TelnetConnection):
         """
         gem5_logger.info("Gracefully terminating the gem5 simulation.")
         try:
+            # Unmount the virtio device BEFORE we kill the
+            # simulation. This is done to simplify checkpointing at
+            # the end of a simulation!
+            self._unmount_virtio()
             self._gem5_util("exit")
             self.gem5simulation.wait()
         except EOF:
@@ -525,6 +562,19 @@ class Gem5Connection(TelnetConnection):
 
         self.connect_gem5(port, gem5_simulation, gem5_interact_dir, gem5_out_dir)
 
+    # Handle the EOF exception raised by pexpect
+    def _gem5_EOF_handler(self, gem5_simulation, gem5_out_dir, err):
+        # If we have reached the "EOF", it typically means
+        # that gem5 crashed and closed the connection. Let's
+        # check and actually tell the user what happened here,
+        # rather than spewing out pexpect errors.
+        if gem5_simulation.poll():
+            message = "The gem5 process has crashed with error code {}!\n\tPlease see {} for details."
+            raise TargetError(message.format(gem5_simulation.poll(), gem5_out_dir))
+        else:
+            # Let's re-throw the exception in this case.
+            raise err
+
     # This function connects to the gem5 simulation
     def connect_gem5(self, port, gem5_simulation, gem5_interact_dir,
                       gem5_out_dir):
@@ -562,6 +612,8 @@ class Gem5Connection(TelnetConnection):
                 break
             except pxssh.ExceptionPxssh:
                 pass
+            except EOF as err:
+                self._gem5_EOF_handler(gem5_simulation, gem5_out_dir, err)
         else:
             gem5_simulation.kill()
             raise TargetError("Failed to connect to the gem5 telnet session.")
@@ -582,13 +634,18 @@ class Gem5Connection(TelnetConnection):
                 self._login_to_device()
             except TIMEOUT:
                 pass
+            except EOF as err:
+                self._gem5_EOF_handler(gem5_simulation, gem5_out_dir, err)
+
             try:
                 # Try and force a prompt to be shown
                 self.conn.send('\n')
-                self.conn.expect([r'# ', self.conn.UNIQUE_PROMPT, r'\[PEXPECT\][\\\$\#]+ '], timeout=60)
+                self.conn.expect([r'# ', r'\$ ', self.conn.UNIQUE_PROMPT, r'\[PEXPECT\][\\\$\#]+ '], timeout=60)
                 prompt_found = True
             except TIMEOUT:
                 pass
+            except EOF as err:
+                self._gem5_EOF_handler(gem5_simulation, gem5_out_dir, err)
 
         gem5_logger.info("Successfully logged in")
         gem5_logger.info("Setting unique prompt...")
@@ -675,6 +732,9 @@ class Gem5Connection(TelnetConnection):
 
         gem5_logger.debug("gem5_shell command: {}".format(command))
 
+        if as_root:
+            command = 'echo "{}" | su'.format(escape_double_quotes(command))
+
         # Send the actual command
         self.conn.send("{}\n".format(command))
 
@@ -700,8 +760,8 @@ class Gem5Connection(TelnetConnection):
 
         output = output[command_index + len(command):].strip()
 
-        # It is possible that gem5 will echo the command. Therefore, we need to
+        # If the gem5 system echoes the executed command, we need to remove that too!
-        # remove that too!
+        if self.strip_echoed_commands:
             command_index = output.find(command)
             if command_index != -1:
                 output = output[command_index + len(command):].strip()
@@ -733,9 +793,33 @@ class Gem5Connection(TelnetConnection):
         """
         gem5_logger.info("Mounting VirtIO device in simulated system")
 
-        self._gem5_shell('su -c "mkdir -p {}" root'.format(self.gem5_input_dir))
+        self._gem5_shell('mkdir -p {}'.format(self.gem5_input_dir), as_root=True)
         mount_command = "mount -t 9p -o trans=virtio,version=9p2000.L,aname={} gem5 {}".format(self.gem5_interact_dir, self.gem5_input_dir)
-        self._gem5_shell(mount_command)
+        self._gem5_shell(mount_command, as_root=True)
+
+    def _unmount_virtio(self):
+        """
+        Unmount the VirtIO device in the simulated system.
+        """
+        gem5_logger.info("Unmounting VirtIO device in simulated system")
+
+        unmount_command = "umount {}".format(self.gem5_input_dir)
+        self._gem5_shell(unmount_command, as_root=True)
+
+    def take_checkpoint(self):
+        """
+        Take a checkpoint of the simulated system.
+
+        In order to take a checkpoint we first unmount the virtio
+        device, take then checkpoint, and then remount the device to
+        allow us to continue the current run. This needs to be done to
+        ensure that future gem5 simulations are able to utilise the
+        virtio device (i.e., we need to drop the current state
+        information that the device has).
+        """
+        self._unmount_virtio()
+        self._gem5_util("checkpoint")
+        self._mount_virtio()
 
     def _move_to_temp_dir(self, source):
         """

devlib/utils/types.py

@@ -1,4 +1,4 @@
-#    Copyright 2014-2015 ARM Limited
+#    Copyright 2014-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -26,6 +26,11 @@ is not the best language to use for configuration.
 
 """
 import math
+import re
+import sys
+from functools import total_ordering
+
+from past.builtins import basestring
 
 from devlib.utils.misc import isiterable, to_identifier, ranges_to_list, list_to_mask
 
@@ -68,6 +73,15 @@ def numeric(value):
     """
     if isinstance(value, int):
         return value
+
+    if isinstance(value, basestring):
+        value = value.strip()
+        if value.endswith('%'):
+            try:
+                return float(value.rstrip('%')) / 100
+            except ValueError:
+                raise ValueError('Not numeric: {}'.format(value))
+
     try:
         fvalue = float(value)
     except ValueError:
@@ -79,6 +93,7 @@ def numeric(value):
     return fvalue
 
 
+@total_ordering
 class caseless_string(str):
     """
     Just like built-in Python string except case-insensitive on comparisons. However, the
@@ -92,12 +107,17 @@ class caseless_string(str):
         return self.lower() == other
 
     def __ne__(self, other):
-        return not self.__eq__(other)
+        if isinstance(other, basestring):
-
-    def __cmp__(self, other):
-        if isinstance(basestring, other):
             other = other.lower()
-        return cmp(self.lower(), other)
+        return self.lower() != other
+
+    def __lt__(self, other):
+        if isinstance(other, basestring):
+            other = other.lower()
+        return self.lower() < other
+
+    def __hash__(self):
+        return hash(self.lower())
 
     def format(self, *args, **kwargs):
         return caseless_string(super(caseless_string, self).format(*args, **kwargs))
@@ -111,3 +131,40 @@ def bitmask(value):
     if not isinstance(value, int):
         raise ValueError(value)
     return value
+
+
+regex_type = type(re.compile(''))
+
+
+if sys.version_info[0] == 3:
+    def regex(value):
+        if isinstance(value, regex_type):
+            if isinstance(value.pattern, str):
+                return value
+            return re.compile(value.pattern.decode(),
+                              value.flags | re.UNICODE)
+        else:
+            if isinstance(value, bytes):
+                value = value.decode()
+            return re.compile(value)
+
+
+    def bytes_regex(value):
+        if isinstance(value, regex_type):
+            if isinstance(value.pattern, bytes):
+                return value
+            return re.compile(value.pattern.encode(sys.stdout.encoding),
+                              value.flags & ~re.UNICODE)
+        else:
+            if isinstance(value, str):
+                value = value.encode(sys.stdout.encoding)
+            return re.compile(value)
+else:
+    def regex(value):
+        if isinstance(value, regex_type):
+            return value
+        else:
+            return re.compile(value)
+
+
+    bytes_regex = regex

devlib/utils/uefi.py

@@ -1,4 +1,4 @@
-#    Copyright 2014-2015 ARM Limited
+#    Copyright 2014-2018 ARM Limited
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -19,6 +19,8 @@ import time
 import logging
 from copy import copy
 
+from past.builtins import basestring
+
 from devlib.utils.serial_port import write_characters, TIMEOUT
 from devlib.utils.types import boolean
 
@@ -193,14 +195,14 @@ class UefiMenu(object):
         is not in the current menu, ``LookupError`` will be raised."""
         if not self.prompt:
             self.read_menu(timeout)
-        return self.options.items()
+        return list(self.options.items())
 
     def get_option_index(self, text, timeout=default_timeout):
         """Returns the menu index of the specified option text (uses regex matching). If the option
         is not in the current menu, ``LookupError`` will be raised."""
         if not self.prompt:
             self.read_menu(timeout)
-        for k, v in self.options.iteritems():
+        for k, v in self.options.items():
             if re.search(text, v):
                 return k
         raise LookupError(text)

devlib/utils/version.py

@@ -0,0 +1,30 @@
+#    Copyright 2018 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+import sys
+from subprocess import Popen, PIPE
+
+def get_commit():
+    p = Popen(['git', 'rev-parse', 'HEAD'], cwd=os.path.dirname(__file__),
+              stdout=PIPE, stderr=PIPE)
+    std, _ = p.communicate()
+    p.wait()
+    if p.returncode:
+        return None
+    if sys.version_info[0] == 3:
+        return std[:8].decode(sys.stdout.encoding, 'replace')
+    else:
+        return std[:8]

doc/connection.rst

@@ -99,19 +99,19 @@ Connection Types
     ``adb`` is part of the Android SDK (though stand-alone versions are also
     available).
 
-    :param device: The name of the adb divice. This is usually a unique hex
+    :param device: The name of the adb device. This is usually a unique hex
                    string for USB-connected devices, or an ip address/port
                    combination. To see connected devices, you can run ``adb
                    devices`` on the host.
     :param timeout: Connection timeout in seconds. If a connection to the device
-                    is not esblished within this period, :class:`HostError`
+                    is not established within this period, :class:`HostError`
                     is raised.
 
 
 .. class:: SshConnection(host, username, password=None, keyfile=None, port=None,\
                          timeout=None, password_prompt=None)
 
-    A connectioned to a device on the network over SSH.
+    A connection to a device on the network over SSH.
 
     :param host: SSH host to which to connect
     :param username: username for SSH login
@@ -126,21 +126,21 @@ Connection Types
                     .. note:: ``keyfile`` and ``password`` can't be specified
                               at the same time.
 
-    :param port: TCP port on which SSH server is litening on the remoted device.
+    :param port: TCP port on which SSH server is listening on the remote device.
                  Omit to use the default port.
     :param timeout: Timeout for the connection in seconds. If a connection
                     cannot be established within this time, an error will be
                     raised.
     :param password_prompt: A string with the password prompt used by
                             ``sshpass``. Set this if your version of ``sshpass``
-                            uses somethin other than ``"[sudo] password"``.
+                            uses something other than ``"[sudo] password"``.
 
 
 .. class:: TelnetConnection(host, username, password=None, port=None,\
                             timeout=None, password_prompt=None,\
                             original_prompt=None)
 
-    A connectioned to a device on the network over Telenet.
+    A connection to a device on the network over Telenet.
 
     .. note:: Since Telenet protocol is does not support file transfer, scp is
               used for that purpose.
@@ -153,19 +153,19 @@ Connection Types
                                ``sshpass`` utility must be installed on the
                                system.
 
-    :param port: TCP port on which SSH server is litening on the remoted device.
+    :param port: TCP port on which SSH server is listening on the remote device.
                  Omit to use the default port.
     :param timeout: Timeout for the connection in seconds. If a connection
                     cannot be established within this time, an error will be
                     raised.
     :param password_prompt: A string with the password prompt used by
                             ``sshpass``. Set this if your version of ``sshpass``
-                            uses somethin other than ``"[sudo] password"``.
+                            uses something other than ``"[sudo] password"``.
     :param original_prompt: A regex for the shell prompted presented in the Telenet
                             connection (the prompt will be reset to a
                             randomly-generated pattern for the duration of the
                             connection to reduce the possibility of clashes).
-                            This paramer is ignored for SSH connections.
+                            This parameter is ignored for SSH connections.
 
 
 .. class:: LocalConnection(keep_password=True, unrooted=False, password=None)
@@ -189,7 +189,7 @@ Connection Types
     A connection to a gem5 simulation using a local Telnet connection.
 
     .. note:: Some of the following input parameters are optional and will be ignored during
-              initialisation. They were kept to keep the anology with a :class:`TelnetConnection`
+              initialisation. They were kept to keep the analogy with a :class:`TelnetConnection`
               (i.e. ``host``, `username``, ``password``, ``port``,
               ``password_prompt`` and ``original_promp``)
 
@@ -220,7 +220,7 @@ Connection Types
 There are two classes that inherit from :class:`Gem5Connection`:
 :class:`AndroidGem5Connection` and :class:`LinuxGem5Connection`.
 They inherit *almost* all methods from the parent class, without altering them.
-The only methods discussed belows are those that will be overwritten by the
+The only methods discussed below are those that will be overwritten by the
 :class:`LinuxGem5Connection` and :class:`AndroidGem5Connection` respectively.
 
 .. class:: LinuxGem5Connection

doc/derived_measurements.rst

@@ -0,0 +1,221 @@
+Derived Measurements
+=====================
+
+
+The ``DerivedMeasurements`` API provides a consistent way of performing post
+processing on a provided :class:`MeasurementCsv` file.
+
+Example
+-------
+
+The following example shows how to use an implementation of a
+:class:`DerivedMeasurement` to obtain a list of calculated ``DerivedMetric``'s.
+
+.. code-block:: ipython
+
+    # Import the relevant derived measurement module
+    # in this example the derived energy module is used.
+    In [1]: from devlib import DerivedEnergyMeasurements
+
+    # Obtain a MeasurementCsv file from an instrument or create from
+    # existing .csv file. In this example an existing csv file is used which was
+    # created with a sampling rate of 100Hz
+    In [2]: from devlib import MeasurementsCsv
+    In [3]: measurement_csv = MeasurementsCsv('/example/measurements.csv', sample_rate_hz=100)
+
+    # Process the file and obtain a list of the derived measurements
+    In [4]: derived_measurements = DerivedEnergyMeasurements.process(measurement_csv)
+
+    In [5]: derived_measurements
+    Out[5]: [device_energy: 239.1854075 joules, device_power: 5.5494089227 watts]
+
+API
+---
+
+Derived Measurements
+~~~~~~~~~~~~~~~~~~~~
+
+.. class:: DerivedMeasurements
+
+   The ``DerivedMeasurements`` class provides an API for post-processing
+   instrument output offline (i.e. without a connection to the target device) to
+   generate additional metrics.
+
+.. method:: DerivedMeasurements.process(measurement_csv)
+
+   Process a :class:`MeasurementsCsv`, returning  a list of
+   :class:`DerivedMetric` and/or :class:`MeasurementsCsv` objects that have been
+   derived from the input. The exact nature and ordering of the list members
+   is specific to individual 'class'`DerivedMeasurements` implementations.
+
+.. method:: DerivedMeasurements.process_raw(\*args)
+
+   Process raw output from an instrument, returning a list :class:`DerivedMetric`
+   and/or :class:`MeasurementsCsv` objects that have been derived from the
+   input. The exact nature and ordering of the list members is specific to
+   individual 'class'`DerivedMeasurements` implementations.
+
+   The arguments to this method should be paths to raw output files generated by
+   an instrument. The number and order of expected arguments is specific to
+   particular implementations.
+
+
+Derived Metric
+~~~~~~~~~~~~~~
+
+.. class:: DerivedMetric
+
+  Represents a metric derived from previously collected ``Measurement``s.
+  Unlike, a ``Measurement``, this was not measured directly from the target.
+
+
+.. attribute:: DerivedMetric.name
+
+   The name of the derived metric. This uniquely defines a metric -- two
+   ``DerivedMetric`` objects with the same ``name`` represent to instances of
+   the same metric (e.g. computed from two different inputs).
+
+.. attribute:: DerivedMetric.value
+
+   The ``numeric`` value of the metric that has been computed for a particular
+   input.
+
+.. attribute:: DerivedMetric.measurement_type
+
+   The ``MeasurementType`` of the metric. This indicates which conceptual
+   category the metric falls into, its units, and conversions to other
+   measurement types.
+
+.. attribute:: DerivedMetric.units
+
+   The units in which the metric's value is expressed.
+
+
+Available Derived Measurements
+-------------------------------
+
+.. note:: If a method of the API is not documented for a particular
+          implementation, that means that it s not overridden by that
+          implementation. It is still safe to call it -- an empty list will be
+          returned.
+
+Energy
+~~~~~~
+
+.. class:: DerivedEnergyMeasurements
+
+   The ``DerivedEnergyMeasurements`` class is used to calculate average power and
+   cumulative energy for each site if the required data is present.
+
+   The calculation of cumulative energy can occur in 3 ways. If a
+   ``site`` contains ``energy`` results, the first and last measurements are extracted
+   and the delta calculated. If not, a ``timestamp`` channel will be used to calculate
+   the energy from the power channel, failing back to using the sample rate attribute
+   of the :class:`MeasurementCsv` file if timestamps are not available. If neither
+   timestamps or a sample rate are available then an error will be raised.
+
+
+.. method:: DerivedEnergyMeasurements.process(measurement_csv)
+
+   This will return total cumulative energy for each energy channel, and the
+   average power for each power channel in the input CSV. The output will contain
+   all energy metrics followed by power metrics. The ordering of both will match
+   the ordering of channels in the input. The metrics will by named based on the
+   sites of the corresponding channels according to the following patters:
+   ``"<site>_total_energy"`` and ``"<site>_average_power"``.
+
+
+FPS / Rendering
+~~~~~~~~~~~~~~~
+
+.. class:: DerivedGfxInfoStats(drop_threshold=5, suffix='-fps', filename=None, outdir=None)
+
+   Produces FPS (frames-per-second) and other derived statistics from
+   :class:`GfxInfoFramesInstrument` output. This takes several optional
+   parameters in creation:
+
+   :param drop_threshold: FPS in an application, such as a game, which this
+                          processor is primarily targeted at, cannot reasonably
+                          drop to a very low value. This is specified to this
+                          threshold. If an FPS for a frame is computed to be
+                          lower than this threshold, it will be dropped on the
+                          assumption that frame rendering was suspended by the
+                          system (e.g. when idling), or there was some sort of
+                          error, and therefore this should be used in
+                          performance calculations. defaults to ``5``.
+   :param  suffix: The name of the generated per-frame FPS csv file will be
+                   derived from the input frames csv file by appending this
+                   suffix. This cannot be specified at the same time as
+                   a ``filename``.
+   :param filename: As an alternative to the suffix, a complete file name for
+                    FPS csv can be specified. This cannot be used at the same
+                    time as the ``suffix``.
+   :param outdir: By default, the FPS csv file will be placed in the same
+                  directory as the input frames csv file. This can be changed
+                  by specifying an alternate directory here
+
+   .. warning:: Specifying both ``filename`` and ``oudir`` will mean that exactly
+                the same file will be used for FPS output on each invocation of
+                ``process()`` (even for different inputs) resulting in previous
+                results being overwritten.
+
+.. method:: DerivedGfxInfoStats.process(measurement_csv)
+
+   Process the fames csv generated by :class:`GfxInfoFramesInstrument` and
+   returns a list containing exactly three entries: :class:`DerivedMetric`\ s
+   ``fps`` and ``total_frames``, followed by a :class:`MeasurentCsv` containing
+   per-frame FPSs values.
+
+.. method:: DerivedGfxInfoStats.process_raw(gfxinfo_frame_raw_file)
+
+   As input, this takes a single argument, which should be the path to the raw
+   output file of  :class:`GfxInfoFramesInstrument`. The returns stats
+   accumulated by gfxinfo. At the time of writing, the stats (in order) are:
+   ``janks``, ``janks_pc`` (percentage of all frames),
+   ``render_time_50th_ptile`` (50th percentile, or median, for time to render a
+   frame), ``render_time_90th_ptile``, ``render_time_95th_ptile``,
+   ``render_time_99th_ptile``, ``missed_vsync``, ``hight_input_latency``,
+   ``slow_ui_thread``, ``slow_bitmap_uploads``, ``slow_issue_draw_commands``.
+   Please see the `gfxinfo documentation`_ for details.
+
+.. _gfxinfo documentation: https://developer.android.com/training/testing/performance.html
+
+
+.. class:: DerivedSurfaceFlingerStats(drop_threshold=5, suffix='-fps', filename=None, outdir=None)
+
+   Produces FPS (frames-per-second) and other derived statistics from
+   :class:`SurfaceFlingerFramesInstrument` output. This takes several optional
+   parameters in creation:
+
+   :param drop_threshold: FPS in an application, such as a game, which this
+                          processor is primarily targeted at, cannot reasonably
+                          drop to a very low value. This is specified to this
+                          threshold. If an FPS for a frame is computed to be
+                          lower than this threshold, it will be dropped on the
+                          assumption that frame rendering was suspended by the
+                          system (e.g. when idling), or there was some sort of
+                          error, and therefore this should be used in
+                          performance calculations. defaults to ``5``.
+   :param  suffix: The name of the generated per-frame FPS csv file will be
+                   derived from the input frames csv file by appending this
+                   suffix. This cannot be specified at the same time as
+                   a ``filename``.
+   :param filename: As an alternative to the suffix, a complete file name for
+                    FPS csv can be specified. This cannot be used at the same
+                    time as the ``suffix``.
+   :param outdir: By default, the FPS csv file will be placed in the same
+                  directory as the input frames csv file. This can be changed
+                  by specifying an alternate directory here
+
+   .. warning:: Specifying both ``filename`` and ``oudir`` will mean that exactly
+                the same file will be used for FPS output on each invocation of
+                ``process()`` (even for different inputs) resulting in previous
+                results being overwritten.
+
+.. method:: DerivedSurfaceFlingerStats.process(measurement_csv)
+
+   Process the fames csv generated by :class:`SurfaceFlingerFramesInstrument` and
+   returns a list containing exactly three entries: :class:`DerivedMetric`\ s
+   ``fps`` and ``total_frames``, followed by a :class:`MeasurentCsv` containing
+   per-frame FPSs values, followed by ``janks`` ``janks_pc``, and
+   ``missed_vsync`` metrics.

doc/index.rst

@@ -19,6 +19,7 @@ Contents:
    target
    modules
    instrumentation
+   derived_measurements
    platform
    connection
 

doc/instrumentation.rst

@@ -65,8 +65,8 @@ Instrument
    :INSTANTANEOUS: The instrument supports taking a single sample via
                    ``take_measurement()``.
    :CONTINUOUS: The instrument supports collecting measurements over a
-                period of time via ``start()``, ``stop()``, and
+                period of time via ``start()``, ``stop()``, ``get_data()``,
-                ``get_data()`` methods.
+		and (optionally) ``get_raw`` methods.
 
    .. note:: It's possible for one instrument to support more than a single
              mode.
@@ -99,14 +99,21 @@ Instrument
    ``teardown()`` has been called), but see documentation for the instrument
    you're interested in.
 
-.. method:: Instrument.reset([sites, [kinds]])
+.. method:: Instrument.reset(sites=None, kinds=None, channels=None)
 
    This is used to configure an instrument for collection. This must be invoked
-   before ``start()`` is called to begin collection. ``sites`` and ``kinds``
+   before ``start()`` is called to begin collection. This methods sets the
-   parameters may be used to specify which channels measurements should be
+   ``active_channels`` attribute of the ``Instrument``.
-   collected from (if omitted, then measurements will be collected for all
+
-   available sites/kinds). This methods sets the ``active_channels`` attribute
+   If ``channels`` is provided, it is a list of names of channels to enable and
-   of the ``Instrument``.
+   ``sites`` and ``kinds`` must both be ``None``.
+
+   Otherwise, if one of ``sites`` or ``kinds`` is provided, all channels
+   matching the given sites or kinds are enabled. If both are provided then all
+   channels of the given kinds at the given sites are enabled.
+
+   If none of ``sites``, ``kinds`` or ``channels`` are provided then all
+   available channels are enabled.
 
 .. method:: Instrument.take_measurment()
 
@@ -114,14 +121,14 @@ Instrument
    :class:`Measurement` objects (one for each active channel).
 
    .. note:: This method is only implemented by :class:`Instrument`\ s that
-             support ``INSTANTANEOUS`` measurment.
+             support ``INSTANTANEOUS`` measurement.
 
 .. method:: Instrument.start()
 
    Starts collecting measurements from ``active_channels``.
 
    .. note:: This method is only implemented by :class:`Instrument`\ s that
-             support ``CONTINUOUS`` measurment.
+             support ``CONTINUOUS`` measurement.
 
 .. method:: Instrument.stop()
 
@@ -129,29 +136,44 @@ Instrument
    :func:`start()`.
 
    .. note:: This method is only implemented by :class:`Instrument`\ s that
-             support ``CONTINUOUS`` measurment.
+             support ``CONTINUOUS`` measurement.
 
 .. method:: Instrument.get_data(outfile)
 
    Write collected data into ``outfile``. Must be called after :func:`stop()`.
    Data will be written in CSV format with a column for each channel and a row
    for each sample. Column heading will be channel, labels in the form
-   ``<site>_<kind>`` (see :class:`InstrumentChannel`). The order of the coluns
+   ``<site>_<kind>`` (see :class:`InstrumentChannel`). The order of the columns
    will be the same as the order of channels in ``Instrument.active_channels``.
 
+   If reporting timestamps, one channel must have a ``site`` named ``"timestamp"``
+   and a ``kind`` of a :class:`MeasurmentType` of an appropriate time unit which will
+   be used, if appropriate, during any post processing.
+
+   .. note:: Currently supported time units are seconds, milliseconds and
+             microseconds, other units can also be used if an appropriate
+             conversion is provided.
+
    This returns a :class:`MeasurementCsv` instance associated with the outfile
    that can be used to stream :class:`Measurement`\ s lists (similar to what is
    returned by ``take_measurement()``.
 
    .. note:: This method is only implemented by :class:`Instrument`\ s that
-             support ``CONTINUOUS`` measurment.
+             support ``CONTINUOUS`` measurement.
+
+.. method:: Instrument.get_raw()
+
+   Returns a list of paths to files containing raw output from the underlying
+   source(s) that is used to produce the data CSV. If now raw output is
+   generated or saved, an empty list will be returned. The format of the
+   contents of the raw files is entirely source-dependent.
 
 .. attribute:: Instrument.sample_rate_hz
 
    Sample rate of the instrument in Hz. Assumed to be the same for all channels.
 
    .. note:: This attribute is only provided by :class:`Instrument`\ s that
-             support ``CONTINUOUS`` measurment.
+             support ``CONTINUOUS`` measurement.
 
 Instrument Channel
 ~~~~~~~~~~~~~~~~~~
@@ -163,16 +185,16 @@ Instrument Channel
    ``site`` and a ``measurement_type``.
 
    A ``site`` indicates where  on the target a measurement is collected from
-   (e.g. a volage rail or location of a sensor).
+   (e.g. a voltage rail or location of a sensor).
 
    A ``measurement_type`` is an instance of :class:`MeasurmentType` that
-   describes what sort of measurment this is (power, temperature, etc). Each
+   describes what sort of measurement this is (power, temperature, etc). Each
-   mesurement type has a standard unit it is reported in, regardless of an
+   measurement type has a standard unit it is reported in, regardless of an
    instrument used to collect it.
 
    A channel (i.e. site/measurement_type combination) is unique per instrument,
    however there may be more than one channel associated with one site (e.g. for
-   both volatage and power).
+   both voltage and power).
 
    It should not be assumed that any site/measurement_type combination is valid.
    The list of available channels can queried with
@@ -180,22 +202,22 @@ Instrument Channel
 
 .. attribute:: InstrumentChannel.site
 
-   The name of the "site" from which the measurments are collected (e.g. voltage
+   The name of the "site" from which the measurements are collected (e.g. voltage
    rail, sensor, etc).
 
 .. attribute:: InstrumentChannel.kind
 
-   A string indingcating the type of measrument that will be collted. This is
+   A string indicating the type of measurement that will be collected. This is
    the ``name`` of the :class:`MeasurmentType` associated with this channel.
 
 .. attribute:: InstrumentChannel.units
 
-   Units in which measurment will be reported. this is determined by the
+   Units in which measurement will be reported. this is determined by the
    underlying :class:`MeasurmentType`.
 
 .. attribute:: InstrumentChannel.label
 
-   A label that can be attached to measurments associated with with channel.
+   A label that can be attached to measurements associated with with channel.
    This is constructed with ::
 
        '{}_{}'.format(self.site, self.kind)
@@ -211,27 +233,33 @@ be reported as "power" in Watts, and never as "pwr" in milliWatts. Currently
 defined measurement types are
 
 
-+-------------+---------+---------------+
++-------------+-------------+---------------+
 | name        | units       | category      |
-+=============+=========+===============+
++=============+=============+===============+
-| time        | seconds |               |
+| count       | count       |               |
-+-------------+---------+---------------+
++-------------+-------------+---------------+
-| temperature | degrees |               |
+| percent     | percent     |               |
-+-------------+---------+---------------+
++-------------+-------------+---------------+
+| time_us     | microseconds|  time         |
++-------------+-------------+---------------+
+| time_ms     | milliseconds|  time         |
++-------------+-------------+---------------+
+| temperature | degrees     |  thermal      |
++-------------+-------------+---------------+
 | power       | watts       | power/energy  |
-+-------------+---------+---------------+
++-------------+-------------+---------------+
 | voltage     | volts       | power/energy  |
-+-------------+---------+---------------+
++-------------+-------------+---------------+
 | current     | amps        | power/energy  |
-+-------------+---------+---------------+
++-------------+-------------+---------------+
 | energy      | joules      | power/energy  |
-+-------------+---------+---------------+
++-------------+-------------+---------------+
 | tx          | bytes       | data transfer |
-+-------------+---------+---------------+
++-------------+-------------+---------------+
 | rx          | bytes       | data transfer |
-+-------------+---------+---------------+
++-------------+-------------+---------------+
 | tx/rx       | bytes       | data transfer |
-+-------------+---------+---------------+
++-------------+-------------+---------------+
 
 
 .. instruments:

doc/modules.rst

@@ -106,11 +106,20 @@ policies (governors). The ``devlib`` module exposes the following interface
             target.cpufreq.set_min_frequency(cpu, frequency[, exact=True])
             target.cpufreq.set_max_frequency(cpu, frequency[, exact=True])
 
-   Get and set min and max frequencies on the specified CPU. "set" functions are
+   Get the currently set, or set new min and max frequencies for the specified
-   available with all governors other than ``userspace``.
+   CPU. "set" functions are available with all governors other than
+   ``userspace``.
 
    :param cpu: The cpu; could be a numeric or the corresponding string (e.g.
        ``1`` or ``"cpu1"``).
+
+.. method:: target.cpufreq.get_min_available_frequency(cpu)
+            target.cpufreq.get_max_available_frequency(cpu)
+
+    Retrieve the min or max DVFS frequency that is supported (as opposed to
+    currently enforced) for a given CPU. Returns an int or None if could not be
+    determined.
+
    :param frequency: Frequency to set.
 
 .. method:: target.cpufreq.get_frequency(cpu)
@@ -126,7 +135,7 @@ policies (governors). The ``devlib`` module exposes the following interface
 cpuidle
 -------
 
-``cpufreq`` is the kernel subsystem for managing CPU low power (idle) states.
+``cpuidle`` is the kernel subsystem for managing CPU low power (idle) states.
 
 .. method:: target.cpuidle.get_driver()
 
@@ -182,7 +191,7 @@ Every module (ultimately) derives from :class:`Module` class.  A module must
 define the following class attributes:
 
 :name: A unique name for the module. This cannot clash with any of the existing
-       names and must be a valid Python identifier, but is otherwise free-from.
+       names and must be a valid Python identifier, but is otherwise free-form.
 :kind: This identifies the type of functionality a module implements, which in
        turn determines the interface implemented by the module (all modules of
        the same kind must expose a consistent interface). This must be a valid

doc/overview.rst

@@ -74,13 +74,13 @@ This sets up the target for ``devlib`` interaction. This includes creating
 working directories, deploying busybox, etc. It's usually enough to do this once
 for a new device, as the changes this makes will persist across reboots.
 However, there is no issue with calling this multiple times, so, to be on the
-safe site, it's a good idea to call this once at the beginning of your scripts.
+safe side, it's a good idea to call this once at the beginning of your scripts.
 
 Command Execution
 ~~~~~~~~~~~~~~~~~
 
 There are several ways to execute a command on the target. In each case, a
-:class:`TargetError` will be raised if something goes wrong. In very case, it is
+:class:`TargetError` will be raised if something goes wrong. In each case, it is
 also possible to specify ``as_root=True`` if the specified command should be
 executed as root.
 
@@ -154,7 +154,7 @@ Process Control
    # kill all running instances of a process.
    t.killall('badexe', signal=signal.SIGKILL)
 
-   # List processes running on the target. This retruns a list of parsed
+   # List processes running on the target. This returns a list of parsed
    # PsEntry records.
    entries = t.ps()
    # e.g.  print virtual memory sizes of all running sshd processes:
@@ -173,7 +173,7 @@ Super User Privileges
 
 It is not necessary for the account logged in on the target to have super user
 privileges, however the functionality will obviously be diminished, if that is
-not the case. ``devilib`` will determine if the logged in user has root
+not the case. ``devlib`` will determine if the logged in user has root
 privileges and the correct way to invoke it. You should avoid including "sudo"
 directly in your commands, instead, specify ``as_root=True`` where needed. This
 will make your scripts portable across multiple devices and OS's.

doc/platform.rst

@@ -24,11 +24,11 @@ it was not specified explicitly by the user.
                           assumed to be in a cluster).
     :param big_core: The name of the big core in a big.LITTLE system. If this is
                      not specified it will be inferred (on systems with exactly
-                     two clasters).
+                     two clusters).
     :param model: Model name of the hardware system. If this is not specified it
                   will be queried at run time.
     :param modules: Modules with additional functionality supported by the
-                    platfrom (e.g. for handling flashing, rebooting, etc). These
+                    platform (e.g. for handling flashing, rebooting, etc). These
                     would be added to the Target's modules. (See :ref:`modules`\ ).
 
 
@@ -38,13 +38,13 @@ Versatile Express
 The generic platform may be extended to support hardware- or
 infrastructure-specific functionality. Platforms exist for ARM
 VersatileExpress-based :class:`Juno` and :class:`TC2` development boards. In
-addition to the standard :class:`Platform` parameters above, these platfroms
+addition to the standard :class:`Platform` parameters above, these platforms
 support additional configuration:
 
 
 .. class:: VersatileExpressPlatform
 
-    Normally, this would be instatiated via one of its derived classes
+    Normally, this would be instantiated via one of its derived classes
     (:class:`Juno` or :class:`TC2`) that set appropriate defaults for some of
     the parameters.
 
@@ -63,7 +63,7 @@ support additional configuration:
                         mounted on the host system.
     :param hard_reset_method: Specifies the method for hard-resetting the devices
                             (e.g. if it becomes unresponsive and normal reboot
-                            method doesn not work). Currently supported methods
+                            method doesn't not work). Currently supported methods
                             are:
 
                             :dtr: reboot by toggling DTR line on the serial
@@ -80,7 +80,7 @@ support additional configuration:
                       The following values are currently supported:
 
                        :uefi: Boot via UEFI menu, by selecting the entry
-                              specified by ``uefi_entry`` paramter. If this
+                              specified by ``uefi_entry`` parameter. If this
                               entry does not exist, it will be automatically
                               created based on values provided for ``image``,
                               ``initrd``, ``fdt``, and ``bootargs`` parameters.

doc/target.rst

@@ -2,7 +2,7 @@ Target
 ======
 
 
-.. class:: Target(connection_settings=None, platform=None, working_directory=None, executables_directory=None, connect=True, modules=None, load_default_modules=True, shell_prompt=DEFAULT_SHELL_PROMPT)
+.. class:: Target(connection_settings=None, platform=None, working_directory=None, executables_directory=None, connect=True, modules=None, load_default_modules=True, shell_prompt=DEFAULT_SHELL_PROMPT, conn_cls=None)
 
     :class:`Target` is the primary interface to the remote device. All interactions
     with the device are performed via a :class:`Target` instance, either
@@ -38,7 +38,7 @@ Target
         by the connection's account). This location will be created,
         if necessary, during ``setup()``.
 
-        This location does *not* to be same as the system's executables
+        This location does *not* need to be same as the system's executables
         location. In fact, to prevent devlib from overwriting system's defaults,
         it better if this is a separate location, if possible.
 
@@ -68,6 +68,9 @@ Target
          prompted on the target. This may be used by some modules that establish
          auxiliary connections to a target over UART.
 
+    :param conn_cls: This is the type of connection that will be used to communicate
+        with the device.
+
 .. attribute:: Target.core_names
 
    This is a list containing names of CPU cores on the target, in the order in
@@ -83,12 +86,12 @@ Target
 
 .. attribute:: Target.big_core
 
-   This is the name of the cores that the "big"s in an ARM big.LITTLE
+   This is the name of the cores that are the "big"s in an ARM big.LITTLE
    configuration. This is obtained via the underlying :class:`Platform`.
 
 .. attribute:: Target.little_core
 
-   This is the name of the cores that the "little"s in an ARM big.LITTLE
+   This is the name of the cores that are the "little"s in an ARM big.LITTLE
    configuration. This is obtained via the underlying :class:`Platform`.
 
 .. attribute:: Target.is_connected
@@ -199,21 +202,23 @@ Target
         operations during reboot process to detect if the reboot has failed and
         the device has hung.
 
-.. method:: Target.push(source, dest [, timeout])
+.. method:: Target.push(source, dest [,as_root , timeout])
 
    Transfer a file from the host machine to the target device.
 
    :param source: path of to the file on the host
    :param dest: path of to the file on the target
+   :param as_root: whether root is required. Defaults to false.
    :param timeout: timeout (in seconds) for the transfer; if the transfer does
        not  complete within this period, an exception will be raised.
 
-.. method:: Target.pull(source, dest [, timeout])
+.. method:: Target.pull(source, dest [, as_root, timeout])
 
    Transfer a file from the target device to the host machine.
 
    :param source: path of to the file on the target
    :param dest: path of to the file on the host
+   :param as_root: whether root is required. Defaults to false.
    :param timeout: timeout (in seconds) for the transfer; if the transfer does
        not  complete within this period, an exception will be raised.
 
@@ -265,6 +270,24 @@ Target
    :param timeout: If this is specified and invocation does not terminate within this number
            of seconds, an exception will be raised.
 
+.. method:: Target.background_invoke(binary [, args [, in_directory [, on_cpus [, as_root ]]]])
+
+   Execute the specified binary on target (must already be installed) as a background
+   task, under the specified conditions and return the :class:`subprocess.Popen`
+   instance for the command.
+
+   :param binary: binary to execute. Must be present and executable on the device.
+   :param args: arguments to be passed to the binary. The can be either a list or
+          a string.
+   :param in_directory:  execute the binary in the  specified directory. This must
+                   be an absolute path.
+   :param on_cpus:  taskset the binary to these CPUs. This may be a single ``int`` (in which
+          case, it will be interpreted as the mask), a list of ``ints``, in which
+          case this will be interpreted as the list of cpus, or string, which
+          will be interpreted as a comma-separated list of cpu ranges, e.g.
+          ``"0,4-7"``.
+   :param as_root: Specify whether the command should be run as root
+
 .. method:: Target.kick_off(command [, as_root])
 
    Kick off the specified command on the target and return immediately. Unlike
@@ -288,11 +311,11 @@ Target
 
 .. method:: Target.read_int(self, path)
 
-   Equivalent to ``Target.read_value(path, kind=devlab.utils.types.integer)``
+   Equivalent to ``Target.read_value(path, kind=devlib.utils.types.integer)``
 
 .. method:: Target.read_bool(self, path)
 
-   Equivalent to ``Target.read_value(path, kind=devlab.utils.types.boolean)``
+   Equivalent to ``Target.read_value(path, kind=devlib.utils.types.boolean)``
 
 .. method:: Target.write_value(path, value [, verify])
 
@@ -306,6 +329,32 @@ Target
        some sysfs entries silently failing to set the written value without
        returning an error code.
 
+.. method:: Target.read_tree_values(path, depth=1, dictcls=dict):
+
+   Read values of all sysfs (or similar) file nodes under ``path``, traversing
+   up to the maximum depth ``depth``.
+
+   Returns a nested structure of dict-like objects (``dict``\ s by default) that
+   follows the structure of the scanned sub-directory tree. The top-level entry
+   has a single item who's key is ``path``. If ``path`` points to a single file,
+   the value of the entry is the value ready from that file node. Otherwise, the
+   value is a dict-line object  with a key for every entry under ``path``
+   mapping onto its value or further dict-like objects as appropriate.
+
+   :param path: sysfs path to scan
+   :param depth: maximum depth to descend
+   :param dictcls: a dict-like type to be used for each level of the hierarchy.
+
+.. method:: Target.read_tree_values_flat(path, depth=1):
+
+   Read values of all sysfs (or similar) file nodes under ``path``, traversing
+   up to the maximum depth ``depth``.
+
+   Returns a dict mapping paths of file nodes to corresponding values.
+
+   :param path: sysfs path to scan
+   :param depth: maximum depth to descend
+
 .. method:: Target.reset()
 
    Soft reset the target. Typically, this means executing ``reboot`` on the
@@ -392,7 +441,9 @@ Target
 .. method:: Target.capture_screen(filepath)
 
    Take a screenshot on the device and save it to the specified file on the
-   host. This may not be supported by the target.
+   host. This may not be supported by the target. You can optionally insert a
+   ``{ts}`` tag into the file name, in which case it will be substituted with
+   on-target timestamp of the screen shot in ISO8601 format.
 
 .. method:: Target.install(filepath[, timeout[, with_name]])
 
@@ -402,6 +453,17 @@ Target
    :param timeout: Optional timeout (in seconds) for the installation
    :param with_name: This may be used to rename the executable on the target
 
+
+.. method:: Target.install_if_needed(host_path, search_system_binaries=True)
+
+   Check to see if the binary is already installed on the device and if not,
+   install it.
+
+   :param host_path: path to the executable on the host
+   :param search_system_binaries: Specify whether to search the devices PATH
+       when checking to see if the executable is installed, otherwise only check
+       user installed binaries.
+
 .. method:: Target.uninstall(name)
 
    Uninstall the specified executable from the target
@@ -422,13 +484,163 @@ Target
 
 .. method:: Target.extract(path, dest=None)
 
-   Extracts the specified archive/file and returns the path to the extrated
+   Extracts the specified archive/file and returns the path to the extracted
    contents. The extraction method is determined based on the file extension.
    ``zip``, ``tar``, ``gzip``, and ``bzip2`` are supported.
 
    :param dest: Specified an on-target destination directory (which must exist)
-                for the extrated contents.
+                for the extracted contents.
 
     Returns the path to the extracted contents. In case of files (gzip and
     bzip2), the path to the decompressed file is returned; for archives, the
     path to the directory with the archive's contents is returned.
+
+.. method:: Target.is_network_connected()
+
+   Checks for internet connectivity on the device. This doesn't actually
+   guarantee that the internet connection is "working" (which is rather
+   nebulous), it's intended just for failing early when definitively _not_
+   connected to the internet.
+
+   :returns: ``True`` if internet seems available, ``False`` otherwise.
+
+Android Target
+---------------
+
+.. class:: AndroidTarget(connection_settings=None, platform=None, working_directory=None, executables_directory=None, connect=True, modules=None, load_default_modules=True, shell_prompt=DEFAULT_SHELL_PROMPT, conn_cls=AdbConnection, package_data_directory="/data/data")
+
+    :class:`AndroidTarget` is a subclass of :class:`Target` with additional features specific to a device running Android.
+
+    :param package_data_directory: This is the location of the data stored
+        for installed Android packages on the device.
+
+.. method:: AndroidTarget.set_rotation(rotation)
+
+   Specify an integer representing the desired screen rotation with the
+   following mappings: Natural: ``0``, Rotated Left: ``1``, Inverted : ``2``
+   and Rotated Right : ``3``.
+
+.. method:: AndroidTarget.get_rotation(rotation)
+
+   Returns an integer value representing the orientation of the devices
+   screen. ``0`` : Natural, ``1`` : Rotated Left, ``2`` : Inverted
+   and ``3`` : Rotated Right.
+
+.. method:: AndroidTarget.set_natural_rotation()
+
+   Sets the screen orientation of the device to its natural (0 degrees)
+   orientation.
+
+.. method:: AndroidTarget.set_left_rotation()
+
+   Sets the screen orientation of the device to 90 degrees.
+
+.. method:: AndroidTarget.set_inverted_rotation()
+
+   Sets the screen orientation of the device to its inverted (180 degrees)
+   orientation.
+
+.. method:: AndroidTarget.set_right_rotation()
+
+   Sets the screen orientation of the device to 270 degrees.
+
+.. method:: AndroidTarget.set_auto_rotation(autorotate)
+
+   Specify a boolean value for whether the devices auto-rotation should
+   be enabled.
+
+.. method:: AndroidTarget.get_auto_rotation()
+
+   Returns ``True`` if the targets auto rotation is currently enabled and
+   ``False`` otherwise.
+
+.. method:: AndroidTarget.set_airplane_mode(mode)
+
+   Specify a boolean value for whether the device should be in airplane mode.
+
+   .. note:: Requires the device to be rooted if the device is running Android 7+.
+
+.. method:: AndroidTarget.get_airplane_mode()
+
+   Returns ``True`` if the target is currently in airplane mode and
+   ``False`` otherwise.
+
+.. method:: AndroidTarget.set_brightness(value)
+
+   Sets the devices screen brightness to a specified integer between ``0`` and
+   ``255``.
+
+.. method:: AndroidTarget.get_brightness()
+
+   Returns an integer between ``0`` and ``255`` representing the devices
+   current screen brightness.
+
+.. method:: AndroidTarget.set_auto_brightness(auto_brightness)
+
+   Specify a boolean value for whether the devices auto brightness
+   should be enabled.
+
+.. method:: AndroidTarget.get_auto_brightness()
+
+   Returns ``True`` if the targets auto brightness is currently
+   enabled and ``False`` otherwise.
+
+.. method:: AndroidTarget.ensure_screen_is_off()
+
+   Checks if the devices screen is on and if so turns it off.
+
+.. method:: AndroidTarget.ensure_screen_is_on()
+
+   Checks if the devices screen is off and if so turns it on.
+
+.. method:: AndroidTarget.is_screen_on()
+
+   Returns ``True`` if the targets screen is currently on and ``False``
+   otherwise.
+
+.. method:: AndroidTarget.homescreen()
+
+   Returns the device to its home screen.
+
+.. method:: AndroidTarget.swipe_to_unlock(direction="diagonal")
+
+   Performs a swipe input on the device to try and unlock the device.
+   A direction of ``"horizontal"``, ``"vertical"`` or ``"diagonal"``
+   can be supplied to specify in which direction the swipe should be
+   performed. By default ``"diagonal"`` will be used to try and
+   support the majority of newer devices.
+
+
+ChromeOS Target
+---------------
+
+.. class:: ChromeOsTarget(connection_settings=None, platform=None, working_directory=None, executables_directory=None, android_working_directory=None, android_executables_directory=None, connect=True, modules=None, load_default_modules=True, shell_prompt=DEFAULT_SHELL_PROMPT, package_data_directory="/data/data")
+
+    :class:`ChromeOsTarget` is a subclass of :class:`LinuxTarget` with
+    additional features specific to a device running ChromeOS for example,
+    if supported, its own android container which can be accessed via the
+    ``android_container`` attribute. When making calls to or accessing
+    properties and attributes of the ChromeOS target, by default they will
+    be applied to Linux target as this is where the majority of device
+    configuration will be performed and if not available, will fall back to
+    using the android container if available. This means that all the
+    available methods from
+    :class:`LinuxTarget` and :class:`AndroidTarget` are available for
+    :class:`ChromeOsTarget` if the device supports android otherwise only the
+    :class:`LinuxTarget` methods will be available.
+
+    :param working_directory: This is the location of the working
+        directory to be used for the Linux target container. If not specified will
+        default to ``"/mnt/stateful_partition/devlib-target"``.
+
+    :param android_working_directory: This is the location of the working
+        directory to be used for the android container. If not specified it will
+        use the working directory default for :class:`AndroidTarget.`.
+
+    :param android_executables_directory: This is the location of the
+        executables directory to be used for the android container. If not
+        specified will default to a ``bin`` subfolder in the
+        ``android_working_directory.``
+
+    :param package_data_directory: This is the location of the data stored
+        for installed Android packages on the device.

setup.py

@@ -13,6 +13,7 @@
 # limitations under the License.
 #
 
+import imp
 import os
 import sys
 import warnings
@@ -20,8 +21,10 @@ from itertools import chain
 
 try:
     from setuptools import setup
+    from setuptools.command.sdist import sdist as orig_sdist
 except ImportError:
     from distutils.core import setup
+    from distutils.command.sdist import sdist as orig_sdist
 
 
 devlib_dir = os.path.join(os.path.dirname(__file__), 'devlib')
@@ -37,6 +40,26 @@ try:
 except OSError:
     pass
 
+
+with open(os.path.join(devlib_dir, '__init__.py')) as fh:
+    # Extract the version by parsing the text of the file,
+    # as may not be able to load as a module yet.
+    for line in fh:
+        if '__version__' in line:
+            parts = line.split("'")
+            __version__ = parts[1]
+            break
+    else:
+        raise RuntimeError('Did not see __version__')
+
+    vh_path = os.path.join(devlib_dir, 'utils', 'version.py')
+    # can load this, as it does not have any devlib imports
+    version_helper = imp.load_source('version_helper', vh_path)
+    commit = version_helper.get_commit()
+    if commit:
+        __version__ = '{}+{}'.format(__version__, commit)
+
+
 packages = []
 data_files = {}
 source_dir = os.path.dirname(__file__)
@@ -59,10 +82,10 @@ for root, dirs, files in os.walk(devlib_dir):
 params = dict(
     name='devlib',
     description='A framework for automating workload execution and measurment collection on ARM devices.',
-    version='0.0.4',
+    version=__version__,
     packages=packages,
     package_data=data_files,
-    url='N/A',
+    url='https://github.com/ARM-software/devlib',
     license='Apache v2',
     maintainer='ARM Ltd.',
     install_requires=[
@@ -70,10 +93,12 @@ params = dict(
         'pexpect>=3.3',  # Send/recieve to/from device
         'pyserial',  # Serial port interface
         'wrapt',  # Basic for construction of decorator functions
+        'future', # Python 2-3 compatibility
     ],
     extras_require={
         'daq': ['daqpower'],
         'doc': ['sphinx'],
+        'monsoon': ['python-gflags'],
     },
     # https://pypi.python.org/pypi?%3Aaction=list_classifiers
     classifiers=[
@@ -84,7 +109,28 @@ params = dict(
     ],
 )
 
-all_extras = list(chain(params['extras_require'].itervalues()))
+all_extras = list(chain(iter(params['extras_require'].values())))
 params['extras_require']['full'] = all_extras
 
+
+class sdist(orig_sdist):
+
+    user_options = orig_sdist.user_options + [
+        ('strip-commit', 's',
+         "Strip git commit hash from package version ")
+    ]
+
+    def initialize_options(self):
+        orig_sdist.initialize_options(self)
+        self.strip_commit = False
+
+
+    def run(self):
+        if self.strip_commit:
+            self.distribution.get_version = lambda : __version__.split('+')[0]
+        orig_sdist.run(self)
+
+
+params['cmdclass'] = {'sdist': sdist}
+
 setup(**params)

src/readenergy/readenergy.c

@@ -114,7 +114,7 @@ struct reading
 	double sys_enm_ch0_gpu;
 };
 
-inline uint64_t join_64bit_register(uint32_t *buffer, int index)
+static inline uint64_t join_64bit_register(uint32_t *buffer, int index)
 {
 	uint64_t result = 0;
 	result |= buffer[index];
@@ -254,10 +254,10 @@ void emeter_init(struct emeter *this, char *outfile)
 	}
 
 	if(this->out) {
-		fprintf(this->out, "sys_curr,a57_curr,a53_curr,gpu_curr,"
+		fprintf(this->out, "sys_current,a57_current,a53_current,gpu_current,"
-				   "sys_volt,a57_volt,a53_volt,gpu_volt,"
+				   "sys_voltage,a57_voltage,a53_voltage,gpu_voltage,"
-				   "sys_pow,a57_pow,a53_pow,gpu_pow,"
+				   "sys_power,a57_power,a53_power,gpu_power,"
-				   "sys_cenr,a57_cenr,a53_cenr,gpu_cenr\n");
+				   "sys_energy,a57_energy,a53_energy,gpu_energy\n");
 	}
 }