Version bump.

ssh: Combine exit code and main command

.gitignore

@@ -3,3 +3,7 @@
 *.orig
 .ropeproject
 *.egg-info
+devlib/bin/scripts/shutils
+doc/_build/
+build/
+dist/

devlib/__init__.py

@@ -7,12 +7,18 @@ from devlib.module import get_module, register_module
 
 from devlib.platform import Platform
 from devlib.platform.arm import TC2, Juno, JunoEnergyInstrument
+from devlib.platform.gem5 import Gem5SimulationPlatform
 
 from devlib.instrument import Instrument, InstrumentChannel, Measurement, MeasurementsCsv
 from devlib.instrument import MEASUREMENT_TYPES, INSTANTANEOUS, CONTINUOUS
 from devlib.instrument.daq import DaqInstrument
 from devlib.instrument.energy_probe import EnergyProbeInstrument
 from devlib.instrument.hwmon import HwmonInstrument
+from devlib.instrument.monsoon import MonsoonInstrument
 from devlib.instrument.netstats import NetstatsInstrument
 
 from devlib.trace.ftrace import FtraceCollector
+
+from devlib.host import LocalConnection
+from devlib.utils.android import AdbConnection
+from devlib.utils.ssh import SshConnection, TelnetConnection, Gem5Connection

devlib/bin/scripts/shutils.in

@@ -0,0 +1,224 @@
+#!__DEVLIB_SHELL__
+
+CMD=$1
+shift
+
+BUSYBOX=${BUSYBOX:-__DEVLIB_BUSYBOX__}
+FIND=${FIND:-$BUSYBOX find}
+GREP=${GREP:-$BUSYBOX grep}
+SED=${SED:-$BUSYBOX sed}
+CAT=${CAT:-$BUSYBOX cat}
+AWK=${AWK:-$BUSYBOX awk}
+PS=${PS:-$BUSYBOX ps}
+
+################################################################################
+# CPUFrequency Utility Functions
+################################################################################
+
+cpufreq_set_all_frequencies() {
+	FREQ=$1
+	for CPU in /sys/devices/system/cpu/cpu[0-9]*; do
+		echo $FREQ > $CPU/cpufreq/scaling_cur_freq
+	done
+}
+
+cpufreq_get_all_frequencies() {
+	$GREP '' /sys/devices/system/cpu/cpu*/cpufreq/scaling_cur_freq | \
+		$SED -e 's|/sys/devices/system/cpu/cpu||' -e 's|/cpufreq/scaling_cur_freq:| |'
+}
+
+cpufreq_set_all_governors() {
+	GOV=$1
+	for CPU in /sys/devices/system/cpu/cpu[0-9]*; do
+		echo $GOV > $CPU/cpufreq/scaling_governor
+	done
+}
+
+cpufreq_get_all_governors() {
+	$GREP '' /sys/devices/system/cpu/cpu*/cpufreq/scaling_governor | \
+		$SED -e 's|/sys/devices/system/cpu/cpu||' -e 's|/cpufreq/scaling_governor:| |'
+}
+
+cpufreq_trace_all_frequencies() {
+	FREQS=$($CAT /sys/devices/system/cpu/cpu*/cpufreq/scaling_cur_freq)
+	CPU=0; for F in $FREQS; do
+		echo "cpu_frequency_devlib:        state=$F cpu_id=$CPU" > /sys/kernel/debug/tracing/trace_marker
+		CPU=$((CPU + 1))
+	done
+}
+
+################################################################################
+# CPUIdle Utility Functions
+################################################################################
+
+cpuidle_wake_all_cpus() {
+	CPU_PATHS=/sys/devices/system/cpu/cpu[0-9]*
+	MASK=0x1; for F in $CPU_PATHS; do
+		$BUSYBOX taskset $MASK true &
+		MASK=$($BUSYBOX printf '0x%x' $((MASK * 2)))
+	done
+}
+
+################################################################################
+# FTrace Utility Functions
+################################################################################
+
+ftrace_get_function_stats() {
+    for CPU in $(ls /sys/kernel/debug/tracing/trace_stat | sed 's/function//'); do
+        REPLACE_STRING="s/  Function/\n  Function (CPU$CPU)/"
+        $CAT /sys/kernel/debug/tracing/trace_stat/function$CPU \
+            | sed "$REPLACE_STRING"
+    done
+}
+
+
+################################################################################
+# CGroups Utility Functions
+################################################################################
+
+cgroups_get_attributes() {
+	test $# -eq 2 || exit -1
+	CGROUP="$1"
+	CONTROLLER="$2"
+	# Check if controller is mounted with "noprefix" option, which is quite
+	# common on Android for backward compatibility
+	ls $CGROUP/$CONTROLLER\.* 2>&1 >/dev/null
+	if [ $? -eq 0 ]; then
+		# no "noprefix" option, attributes format is:
+		#   mnt_point/controller.attribute_name
+		$GREP '' $CGROUP/* | \
+			$GREP "$CONTROLLER\." | \
+			$SED -e "s|$CONTROLLER\.||" -e "s|$CGROUP/||"
+	else
+		# "noprefix" option, attribute format is:
+		#   mnt_point/attribute_name
+		$GREP '' $(\
+			$FIND $CGROUP -type f -maxdepth 1 |
+			$GREP -v -e ".*tasks" -e ".*cgroup\..*") | \
+		$SED "s|$CGROUP/||"
+	fi
+}
+
+cgroups_run_into() {
+
+	# Control groups mount point
+	CGMOUNT=${CGMOUNT:-/sys/fs/cgroup}
+	# The control group we want to run into
+	CGP=${1}
+	shift 1
+	# The command to run
+	CMD="${@}"
+
+	# Execution under root CGgroup
+	if [ "x/" == "x$CGP" ]; then
+
+	  $FIND $CGMOUNT -type d -maxdepth 0 | \
+	  while read CGPATH; do
+		# Move this shell into that control group
+		echo $$ > $CGPATH/cgroup.procs
+		echo "Moving task into root CGroup ($CGPATH)"
+	  done
+
+	# Execution under specified CGroup
+	else
+
+	  # Check if the required CGroup exists
+	  $FIND $CGMOUNT -type d -mindepth 1 | \
+	  $GREP "$CGP" &>/dev/null
+	  if [ $? -ne 0 ]; then
+		echo "ERROR: could not find any $CGP cgroup under $CGMOUNT"
+		exit 1
+	  fi
+
+	  $FIND $CGMOUNT -type d -mindepth 1 | \
+	  $GREP "$CGP" | \
+	  while read CGPATH; do
+		  # Move this shell into that control group
+		  echo $$ > $CGPATH/cgroup.procs
+		  echo "Moving task into $CGPATH"
+	  done
+
+	fi
+
+	# Execute the command
+	exec $CMD
+
+}
+
+cgroups_tasks_move() {
+	SRC_GRP=${1}
+	DST_GRP=${2}
+	shift 2
+	FILTERS=$*
+
+	$CAT $SRC_GRP/tasks | while read TID; do
+	  echo $TID > $DST_GRP/cgroup.procs
+	done
+
+	[ "x$FILTERS" = "x" ] && exit 0
+
+	PIDS=`$PS -o comm,pid | $GREP $FILTERS | $AWK '{print $2}'`
+	PIDS=`echo $PIDS`
+	echo "PIDs to save: [$PIDS]"
+	for TID in $PIDS; do
+	  COMM=`$CAT /proc/$TID/comm`
+	  echo "$TID : $COMM"
+	  echo $TID > $SRC_GRP/cgroup.procs || true
+	done
+}
+
+cgroups_tasks_in() {
+	GRP=${1}
+	for TID in $($CAT $GRP/tasks); do
+		COMM=`$CAT /proc/$TID/comm 2>/dev/null`
+		[ "$COMM" != "" ] && CMDL=`$CAT /proc/$TID/cmdline 2>/dev/null`
+		[ "$COMM" != "" ] && echo "$TID,$COMM,$CMDL"
+	done
+	exit 0
+}
+
+
+################################################################################
+# Main Function Dispatcher
+################################################################################
+
+case $CMD in
+cpufreq_set_all_frequencies)
+    cpufreq_set_all_frequencies $*
+    ;;
+cpufreq_get_all_frequencies)
+    cpufreq_get_all_frequencies
+    ;;
+cpufreq_set_all_governors)
+    cpufreq_set_all_governors $*
+    ;;
+cpufreq_get_all_governors)
+    cpufreq_get_all_governors
+    ;;
+cpufreq_trace_all_frequencies)
+    cpufreq_trace_all_frequencies $*
+    ;;
+cpuidle_wake_all_cpus)
+    cpuidle_wake_all_cpus $*
+    ;;
+cgroups_get_attributes)
+	cgroups_get_attributes $*
+	;;
+cgroups_run_into)
+    cgroups_run_into $*
+    ;;
+cgroups_tasks_move)
+	cgroups_tasks_move $*
+	;;
+cgroups_tasks_in)
+	cgroups_tasks_in $*
+	;;
+ftrace_get_function_stats)
+    ftrace_get_function_stats
+    ;;
+*)
+    echo "Command [$CMD] not supported"
+    exit -1
+esac
+
+# vim: tabstop=4 shiftwidth=4

devlib/exception.py

@@ -14,11 +14,8 @@
 #
 
 
-from devlib.utils.misc import TimeoutError  # NOQA pylint: disable=W0611
-
-
 class DevlibError(Exception):
-    """Base class for all Workload Automation exceptions."""
+    """Base class for all Devlib exceptions."""
     pass
 
 
@@ -38,3 +35,17 @@ class HostError(DevlibError):
     """An error has occured on the host"""
     pass
 
+
+class TimeoutError(DevlibError):
+    """Raised when a subprocess command times out. This is basically a ``DevlibError``-derived version
+    of ``subprocess.CalledProcessError``, the thinking being that while a timeout could be due to
+    programming error (e.g. not setting long enough timers), it is often due to some failure in the
+    environment, and there fore should be classed as a "user error"."""
+
+    def __init__(self, command, output):
+        super(TimeoutError, self).__init__('Timed out: {}'.format(command))
+        self.command = command
+        self.output = output
+
+    def __str__(self):
+        return '\n'.join([self.message, 'OUTPUT:', self.output or ''])

devlib/host.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+from glob import iglob
 import os
 import shutil
 import subprocess
@@ -28,12 +29,12 @@ class LocalConnection(object):
 
     name = 'local'
 
-    def __init__(self, timeout=10, keep_password=True, unrooted=False):
+    def __init__(self, platform=None, keep_password=True, unrooted=False,
+                 password=None, timeout=None):
         self.logger = logging.getLogger('local_connection')
-        self.timeout = timeout
         self.keep_password = keep_password
         self.unrooted = unrooted
-        self.password = None
+        self.password = password
 
     def push(self, source, dest, timeout=None, as_root=False):  # pylint: disable=unused-argument
         self.logger.debug('cp {} {}'.format(source, dest))
@@ -41,9 +42,15 @@ class LocalConnection(object):
 
     def pull(self, source, dest, timeout=None, as_root=False): # pylint: disable=unused-argument
         self.logger.debug('cp {} {}'.format(source, dest))
-        shutil.copy(source, dest)
+        if ('*' in source or '?' in source) and os.path.isdir(dest):
+            # Pull all files matching a wildcard expression
+            for each_source in iglob(source):
+                shutil.copy(each_source, dest)
+        else:
+            shutil.copy(source, dest)
 
-    def execute(self, command, timeout=None, check_exit_code=True, as_root=False):
+    def execute(self, command, timeout=None, check_exit_code=True,
+                as_root=False, strip_colors=True):
         self.logger.debug(command)
         if as_root:
             if self.unrooted:
@@ -54,7 +61,9 @@ class LocalConnection(object):
         try:
             return check_output(command, shell=True, timeout=timeout, ignore=ignore)[0]
         except subprocess.CalledProcessError as e:
-            raise TargetError(e)
+            message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}'.format(
+                e.returncode, command, e.output)
+            raise TargetError(message)
 
     def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
         if as_root:
@@ -77,4 +86,3 @@ class LocalConnection(object):
         if self.keep_password:
             self.password = password
         return password
-

devlib/instrument/__init__.py

@@ -14,6 +14,7 @@
 #
 import csv
 import logging
+import collections
 
 from devlib.utils.types import numeric
 
@@ -167,8 +168,9 @@ class Instrument(object):
     def __init__(self, target):
         self.target = target
         self.logger = logging.getLogger(self.__class__.__name__)
-        self.channels = {}
+        self.channels = collections.OrderedDict()
         self.active_channels = []
+        self.sample_rate_hz = None
 
     # channel management
 
@@ -178,7 +180,7 @@ class Instrument(object):
     def get_channels(self, measure):
         if hasattr(measure, 'name'):
             measure = measure.name
-        return [c for c in self.channels if c.measure.name == measure]
+        return [c for c in self.list_channels() if c.kind == measure]
 
     def add_channel(self, site, measure, name=None, **attrs):
         if name is None:
@@ -194,8 +196,8 @@ class Instrument(object):
     def teardown(self):
         pass
 
-    def reset(self, sites=None, kinds=None):
-        if kinds is None and sites is None:
+    def reset(self, sites=None, kinds=None, channels=None):
+        if kinds is None and sites is None and channels is None:
             self.active_channels = sorted(self.channels.values(), key=lambda x: x.label)
         else:
             if isinstance(sites, basestring):
@@ -203,6 +205,12 @@ class Instrument(object):
             if isinstance(kinds, basestring):
                 kinds = [kinds]
             self.active_channels = []
+            for chan_name in (channels or []):
+                try:
+                    self.active_channels.append(self.channels[chan_name])
+                except KeyError:
+                    msg = 'Unexpected channel "{}"; must be in {}'
+                    raise ValueError(msg.format(chan_name, self.channels.keys()))
             for chan in self.channels.values():
                 if (kinds is None or chan.kind in kinds) and \
                    (sites is None or chan.site in sites):

devlib/instrument/daq.py

@@ -27,7 +27,7 @@ class DaqInstrument(Instrument):
                  device_id='Dev1',
                  v_range=2.5,
                  dv_range=0.2,
-                 sampling_rate=10000,
+                 sample_rate_hz=10000,
                  channel_map=(0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23),
                  ):
         # pylint: disable=no-member
@@ -51,17 +51,18 @@ class DaqInstrument(Instrument):
         self.device_config = DeviceConfiguration(device_id=device_id,
                                                  v_range=v_range,
                                                  dv_range=dv_range,
-                                                 sampling_rate=sampling_rate,
+                                                 sampling_rate=sample_rate_hz,
                                                  resistor_values=resistor_values,
                                                  channel_map=channel_map,
                                                  labels=labels)
+        self.sample_rate_hz = sample_rate_hz
 
         for label in labels:
             for kind in ['power', 'voltage']:
                 self.add_channel(label, kind)
 
-    def reset(self, sites=None, kinds=None):
-        super(DaqInstrument, self).reset(sites, kinds)
+    def reset(self, sites=None, kinds=None, channels=None):
+        super(DaqInstrument, self).reset(sites, kinds, channels)
         self.execute('close')
         result = self.execute('configure', config=self.device_config)
         if not result.status == Status.OK:  # pylint: disable=no-member

devlib/instrument/energy_probe.py

@@ -20,11 +20,6 @@ import tempfile
 import struct
 import subprocess
 
-try:
-    import pandas
-except ImportError:
-    pandas = None
-
 from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv
 from devlib.exception import HostError
 from devlib.utils.misc import which
@@ -50,22 +45,20 @@ class EnergyProbeInstrument(Instrument):
         if self.caiman is None:
             raise HostError('caiman must be installed on the host '
                             '(see https://github.com/ARM-software/caiman)')
-        if pandas is None:
-            self.logger.info("pandas package will significantly speed up this instrument")
-            self.logger.info("to install it try: pip install pandas")
         self.attributes_per_sample = 3
         self.bytes_per_sample = self.attributes_per_sample * 4
         self.attributes = ['power', 'voltage', 'current']
         self.command = None
         self.raw_output_directory = None
         self.process = None
+        self.sample_rate_hz = 10000 # Determined empirically
 
         for label in self.labels:
             for kind in self.attributes:
                 self.add_channel(label, kind)
 
-    def reset(self, sites=None, kinds=None):
-        super(EnergyProbeInstrument, self).reset(sites, kinds)
+    def reset(self, sites=None, kinds=None, channels=None):
+        super(EnergyProbeInstrument, self).reset(sites, kinds, channels)
         self.raw_output_directory = tempfile.mkdtemp(prefix='eprobe-caiman-')
         parts = ['-r {}:{} '.format(i, int(1000 * rval))
                  for i, rval in enumerate(self.resistor_values)]
@@ -82,6 +75,13 @@ class EnergyProbeInstrument(Instrument):
                                         shell=True)
 
     def stop(self):
+        self.process.poll()
+        if self.process.returncode is not None:
+            stdout, stderr = self.process.communicate()
+            raise HostError(
+                'Energy Probe: Caiman exited unexpectedly with exit code {}.\n'
+                'stdout:\n{}\nstderr:\n{}'.format(self.process.returncode,
+                                                  stdout, stderr))
         os.killpg(self.process.pid, signal.SIGTERM)
 
     def get_data(self, outfile):  # pylint: disable=R0914

devlib/instrument/monsoon.py

@@ -0,0 +1,132 @@
+import csv
+import os
+import signal
+from subprocess import Popen, PIPE
+from tempfile import NamedTemporaryFile
+from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv
+from devlib.exception import HostError
+from devlib.host import PACKAGE_BIN_DIRECTORY
+from devlib.utils.misc import which
+
+INSTALL_INSTRUCTIONS="""
+MonsoonInstrument requires the monsoon.py tool, available from AOSP:
+
+https://android.googlesource.com/platform/cts/+/master/tools/utils/monsoon.py
+
+Download this script and put it in your $PATH (or pass it as the monsoon_bin
+parameter to MonsoonInstrument). `pip install gflags pyserial` to install the
+dependencies.
+"""
+
+class MonsoonInstrument(Instrument):
+    """Instrument for Monsoon Solutions power monitor
+
+    To use this instrument, you need to install the monsoon.py script available
+    from the Android Open Source Project. As of May 2017 this is under the CTS
+    repository:
+
+    https://android.googlesource.com/platform/cts/+/master/tools/utils/monsoon.py
+
+    Collects power measurements only, from a selection of two channels, the USB
+    passthrough channel and the main output channel.
+
+    :param target: Ignored
+    :param monsoon_bin: Path to monsoon.py executable. If not provided,
+                        ``$PATH`` is searched.
+    :param tty_device: TTY device to use to communicate with the Power
+                       Monitor. If not provided, a sane default is used.
+    """
+
+    mode = CONTINUOUS
+
+    def __init__(self, target, monsoon_bin=None, tty_device=None):
+        super(MonsoonInstrument, self).__init__(target)
+        self.monsoon_bin = monsoon_bin or which('monsoon.py')
+        if not self.monsoon_bin:
+            raise HostError(INSTALL_INSTRUCTIONS)
+
+        self.tty_device = tty_device
+
+        self.process = None
+        self.output = None
+
+        self.sample_rate_hz = 500
+        self.add_channel('output', 'power')
+        self.add_channel('USB', 'power')
+
+    def reset(self, sites=None, kinds=None, channels=None):
+        super(MonsoonInstrument, self).reset(sites, kinds)
+
+    def start(self):
+        if self.process:
+            self.process.kill()
+
+        cmd = [self.monsoon_bin,
+               '--hz', str(self.sample_rate_hz),
+               '--samples', '-1', # -1 means sample indefinitely
+               '--includeusb']
+        if self.tty_device:
+            cmd += ['--device', self.tty_device]
+
+        self.logger.debug(' '.join(cmd))
+        self.buffer_file = NamedTemporaryFile(prefix='monsoon', delete=False)
+        self.process = Popen(cmd, stdout=self.buffer_file, stderr=PIPE)
+
+    def stop(self):
+        process = self.process
+        self.process = None
+        if not process:
+            raise RuntimeError('Monsoon script not started')
+
+        process.poll()
+        if process.returncode is not None:
+            stdout, stderr = process.communicate()
+            raise HostError(
+                'Monsoon script exited unexpectedly with exit code {}.\n'
+                'stdout:\n{}\nstderr:\n{}'.format(process.returncode,
+                                                  stdout, stderr))
+
+        process.send_signal(signal.SIGINT)
+
+        stderr =  process.stderr.read()
+
+        self.buffer_file.close()
+        with open(self.buffer_file.name) as f:
+            stdout = f.read()
+        os.remove(self.buffer_file.name)
+        self.buffer_file = None
+
+        self.output = (stdout, stderr)
+
+    def get_data(self, outfile):
+        if self.process:
+            raise RuntimeError('`get_data` called before `stop`')
+
+        stdout, stderr = self.output
+
+        with open(outfile, 'wb') as f:
+            writer = csv.writer(f)
+            active_sites = [c.site for c in self.active_channels]
+
+            # Write column headers
+            row = []
+            if 'output' in active_sites:
+                row.append('output_power')
+            if 'USB' in active_sites:
+                row.append('USB_power')
+            writer.writerow(row)
+
+            # Write data
+            for line in stdout.splitlines():
+                # Each output line is a main_output, usb_output measurement pair.
+                # (If our user only requested one channel we still collect both,
+                # and just ignore one of them)
+                output, usb = line.split()
+                row = []
+                if 'output' in active_sites:
+                    row.append(output)
+                if 'USB' in active_sites:
+                    row.append(usb)
+                writer.writerow(row)
+
+        return MeasurementsCsv(outfile, self.active_channels)

devlib/instrument/netstats/__init__.py

@@ -98,8 +98,8 @@ class NetstatsInstrument(Instrument):
             self.logger.debug('Deploying {} to target'.format(self.package))
             self.target.install(self.apk)
 
-    def reset(self, sites=None, kinds=None, period=None):  # pylint: disable=arguments-differ
-        super(NetstatsInstrument, self).reset(sites, kinds)
+    def reset(self, sites=None, kinds=None, channels=None, period=None):  # pylint: disable=arguments-differ
+        super(NetstatsInstrument, self).reset(sites, kinds, channels)
         period_arg, packages_arg = '', ''
         self.tag = 'netstats-{}'.format(datetime.now().strftime('%Y%m%d%H%M%s'))
         tag_arg = ' --es tag {}'.format(self.tag)

devlib/module/cgroups.py

@@ -24,29 +24,33 @@ from devlib.utils.types import boolean
 
 class Controller(object):
 
-    def __new__(cls, arg):
-        if isinstance(arg, cls):
-            return arg
-        else:
-            return object.__new__(cls, arg)
+    def __init__(self, kind, hid, clist):
+        """
+        Initialize a controller given the hierarchy it belongs to.
 
-    def __init__(self, kind):
-        self.mount_name = 'devlib_'+kind
+        :param kind: the name of the controller
+        :type kind: str
+
+        :param hid: the Hierarchy ID this controller is mounted on
+        :type hid: int
+
+        :param clist: the list of controller mounted in the same hierarchy
+        :type clist: list(str)
+        """
+        self.mount_name = 'devlib_cgh{}'.format(hid)
         self.kind = kind
+        self.hid = hid
+        self.clist = clist
         self.target = None
+        self._noprefix = False
+
+        self.logger = logging.getLogger('CGroup.'+self.kind)
+        self.logger.debug('Initialized [%s, %d, %s]',
+                          self.kind, self.hid, self.clist)
 
-        self.logger = logging.getLogger('cgroups.'+self.kind)
         self.mount_point = None
         self._cgroups = {}
 
-    def probe(self, target):
-        try:
-            exists = target.execute('{} grep {} /proc/cgroups'\
-                    .format(target.busybox, self.kind))
-        except TargetError:
-            return False
-        return True
-
     def mount(self, target, mount_root):
 
         mounted = target.list_file_systems()
@@ -63,13 +67,20 @@ class Controller(object):
             target.execute('mkdir -p {} 2>/dev/null'\
                     .format(self.mount_point), as_root=True)
             target.execute('mount -t cgroup -o {} {} {}'\
-                    .format(self.kind,
+                    .format(','.join(self.clist),
                             self.mount_name,
                             self.mount_point),
                             as_root=True)
 
-        self.logger.info('Controller %s mounted under: %s',
-            self.kind, self.mount_point)
+        # Check if this controller uses "noprefix" option
+        output = target.execute('mount | grep "{} "'.format(self.mount_name))
+        if 'noprefix' in output:
+            self._noprefix = True
+            # self.logger.debug('Controller %s using "noprefix" option',
+            #                   self.kind)
+
+        self.logger.debug('Controller %s mounted under: %s (noprefix=%s)',
+            self.kind, self.mount_point, self._noprefix)
 
         # Mark this contoller as available
         self.target = target
@@ -96,9 +107,10 @@ class Controller(object):
     def list_all(self):
         self.logger.debug('Listing groups for %s controller', self.kind)
         output = self.target.execute('{} find {} -type d'\
-                .format(self.target.busybox, self.mount_point))
+                .format(self.target.busybox, self.mount_point),
+                as_root=True)
         cgroups = []
-        for cg in output.split('\n'):
+        for cg in output.splitlines():
             cg = cg.replace(self.mount_point + '/', '/')
             cg = cg.replace(self.mount_point, '/')
             cg = cg.strip()
@@ -108,24 +120,92 @@ class Controller(object):
             cgroups.append(cg)
         return cgroups
 
-    def move_tasks(self, source, dest):
+    def move_tasks(self, source, dest, exclude=[]):
         try:
             srcg = self._cgroups[source]
             dstg = self._cgroups[dest]
-            command = 'for task in $(cat {}); do echo $task>{}; done'
-            self.target.execute(command.format(srcg.tasks_file, dstg.tasks_file),
-                                # this will always fail as some of the tasks
-                                # are kthreads that cannot be migrated, but we
-                                # don't care about those, so don't check exit
-                                # code.
-                                check_exit_code=False, as_root=True)
         except KeyError as e:
             raise ValueError('Unkown group: {}'.format(e))
+        output = self.target._execute_util(
+                    'cgroups_tasks_move {} {} \'{}\''.format(
+                    srcg.directory, dstg.directory, exclude),
+                    as_root=True)
+
+    def move_all_tasks_to(self, dest, exclude=[]):
+        """
+        Move all the tasks to the specified CGroup
+
+        Tasks are moved from all their original CGroup the the specified on.
+        The tasks which name matches one of the string in exclude are moved
+        instead in the root CGroup for the controller.
+        The name of a tasks to exclude must be a substring of the task named as
+        reported by the "ps" command. Indeed, this list will be translated into
+        a: "ps | grep -e name1 -e name2..." in order to obtain the PID of these
+        tasks.
+
+        :param exclude: list of commands to keep in the root CGroup
+        :type exlude: list(str)
+        """
+
+        if isinstance(exclude, str):
+            exclude = [exclude]
+        if not isinstance(exclude, list):
+            raise ValueError('wrong type for "exclude" parameter, '
+                             'it must be a str or a list')
+
+        logging.debug('Moving all tasks into %s', dest)
+
+        # Build list of tasks to exclude
+        grep_filters = ''
+        for comm in exclude:
+            grep_filters += '-e {} '.format(comm)
+        logging.debug('   using grep filter: %s', grep_filters)
+        if grep_filters != '':
+            logging.debug('   excluding tasks which name matches:')
+            logging.debug('   %s', ', '.join(exclude))
 
-    def move_all_tasks_to(self, dest):
         for cgroup in self._cgroups:
             if cgroup != dest:
-                self.move_tasks(cgroup, dest)
+                self.move_tasks(cgroup, dest, grep_filters)
+
+    def tasks(self, cgroup):
+        try:
+            cg = self._cgroups[cgroup]
+        except KeyError as e:
+            raise ValueError('Unkown group: {}'.format(e))
+        output = self.target._execute_util(
+                    'cgroups_tasks_in {}'.format(cg.directory),
+                    as_root=True)
+        entries = output.splitlines()
+        tasks = {}
+        for task in entries:
+            tid = task.split(',')[0]
+            try:
+                tname = task.split(',')[1]
+            except: continue
+            try:
+                tcmdline = task.split(',')[2]
+            except:
+                tcmdline = ''
+            tasks[int(tid)] = (tname, tcmdline)
+        return tasks
+
+    def tasks_count(self, cgroup):
+        try:
+            cg = self._cgroups[cgroup]
+        except KeyError as e:
+            raise ValueError('Unkown group: {}'.format(e))
+        output = self.target.execute(
+                    '{} wc -l {}/tasks'.format(
+                    self.target.busybox, cg.directory),
+                    as_root=True)
+        return int(output.split()[0])
+
+    def tasks_per_group(self):
+        tasks = {}
+        for cg in self.list_all():
+            tasks[cg] = self.tasks_count(cg)
+        return tasks
 
 class CGroup(object):
 
@@ -147,14 +227,14 @@ class CGroup(object):
         if not create:
             return
 
-        self.logger.info('Creating cgroup %s', self.directory)
+        self.logger.debug('Creating cgroup %s', self.directory)
         self.target.execute('[ -d {0} ] || mkdir -p {0}'\
                 .format(self.directory), as_root=True)
 
     def exists(self):
         try:
             self.target.execute('[ -d {0} ]'\
-                .format(self.directory))
+                .format(self.directory), as_root=True)
             return True
         except TargetError:
             return False
@@ -166,14 +246,11 @@ class CGroup(object):
                 self.controller.kind)
         logging.debug('  %s',
                 self.directory)
-        output = self.target.execute('{} grep \'\' {}/{}.*'.format(
-                    self.target.busybox,
-                    self.directory,
-                    self.controller.kind))
-        for res in output.split('\n'):
-            if res.find(self.controller.kind) < 0:
-                continue
-            res = res.split('.')[1]
+        output = self.target._execute_util(
+                    'cgroups_get_attributes {} {}'.format(
+                    self.directory, self.controller.kind),
+                    as_root=True)
+        for res in output.splitlines():
             attr = res.split(':')[0]
             value = res.split(':')[1]
             conf[attr] = value
@@ -185,14 +262,25 @@ class CGroup(object):
             if isiterable(attrs[idx]):
                 attrs[idx] = list_to_ranges(attrs[idx])
             # Build attribute path
-            path = '{}.{}'.format(self.controller.kind, idx)
-            path = self.target.path.join(self.directory, path)
+            if self.controller._noprefix:
+                attr_name = '{}'.format(idx)
+            else:
+                attr_name = '{}.{}'.format(self.controller.kind, idx)
+            path = self.target.path.join(self.directory, attr_name)
 
             self.logger.debug('Set attribute [%s] to: %s"',
                     path, attrs[idx])
 
             # Set the attribute value
-            self.target.write_value(path, attrs[idx])
+            try:
+                self.target.write_value(path, attrs[idx])
+            except TargetError:
+                # Check if the error is due to a non-existing attribute
+                attrs = self.get()
+                if idx not in attrs:
+                    raise ValueError('Controller [{}] does not provide attribute [{}]'\
+                                     .format(self.controller.kind, attr_name))
+                raise
 
     def get_tasks(self):
         task_ids = self.target.read_value(self.tasks_file).split()
@@ -214,54 +302,59 @@ CgroupSubsystemEntry = namedtuple('CgroupSubsystemEntry', 'name hierarchy num_cg
 class CgroupsModule(Module):
 
     name = 'cgroups'
-    cgroup_root = '/sys/fs/cgroup'
+    stage = 'setup'
 
     @staticmethod
     def probe(target):
-        return target.config.has('cgroups') and target.is_rooted
+        if not target.is_rooted:
+            return False
+        if target.file_exists('/proc/cgroups'):
+            return True
+        return target.config.has('cgroups')
 
     def __init__(self, target):
         super(CgroupsModule, self).__init__(target)
 
         self.logger = logging.getLogger('CGroups')
 
-        # Initialize controllers mount point
-        mounted = self.target.list_file_systems()
-        if self.cgroup_root not in [e.mount_point for e in mounted]:
-            self.target.execute('mount -t tmpfs {} {}'\
-                    .format('cgroup_root',
-                            self.cgroup_root),
-                            as_root=True)
-        else:
-            self.logger.debug('cgroup_root already mounted at %s',
-                    self.cgroup_root)
+        # Set Devlib's CGroups mount point
+        self.cgroup_root = target.path.join(
+            target.working_directory, 'cgroups')
 
-        # Load list of available controllers
-        controllers = []
+        # Get the list of the available controllers
         subsys = self.list_subsystems()
-        for (n, h, c, e) in subsys:
-            controllers.append(n)
-        self.logger.info('Available controllers: %s', controllers)
+        if len(subsys) == 0:
+            self.logger.warning('No CGroups controller available')
+            return
+
+        # Map hierarchy IDs into a list of controllers
+        hierarchy = {}
+        for ss in subsys:
+            try:
+                hierarchy[ss.hierarchy].append(ss.name)
+            except KeyError:
+                hierarchy[ss.hierarchy] = [ss.name]
+        self.logger.debug('Available hierarchies: %s', hierarchy)
 
         # Initialize controllers
+        self.logger.info('Available controllers:')
         self.controllers = {}
-        for idx in controllers:
-            controller = Controller(idx)
-            self.logger.debug('Init %s controller...', controller.kind)
-            if not controller.probe(self.target):
-                continue
+        for ss in subsys:
+            hid = ss.hierarchy
+            controller = Controller(ss.name, hid, hierarchy[hid])
             try:
                 controller.mount(self.target, self.cgroup_root)
             except TargetError:
-                message = 'cgroups {} controller is not supported by the target'
+                message = 'Failed to mount "{}" controller'
                 raise TargetError(message.format(controller.kind))
-            self.logger.debug('Controller %s enabled', controller.kind)
-            self.controllers[idx] = controller
+            self.logger.info('  %-12s : %s', controller.kind,
+                             controller.mount_point)
+            self.controllers[ss.name] = controller
 
     def list_subsystems(self):
         subsystems = []
         for line in self.target.execute('{} cat /proc/cgroups'\
-                .format(self.target.busybox)).split('\n')[1:]:
+                .format(self.target.busybox)).splitlines()[1:]:
             line = line.strip()
             if not line or line.startswith('#'):
                 continue
@@ -279,3 +372,117 @@ class CgroupsModule(Module):
             return None
         return self.controllers[kind]
 
+    def run_into_cmd(self, cgroup, cmdline):
+        """
+        Get the command to run a command into a given cgroup
+
+        :param cmdline: Commdand to be run into cgroup
+        :param cgroup: Name of cgroup to run command into
+        :returns: A command to run `cmdline` into `cgroup`
+        """
+        return 'CGMOUNT={} {} cgroups_run_into {} {}'\
+                .format(self.cgroup_root, self.target.shutils,
+                        cgroup, cmdline)
+
+    def run_into(self, cgroup, cmdline):
+        """
+        Run the specified command into the specified CGroup
+
+        :param cmdline: Command to be run into cgroup
+        :param cgroup: Name of cgroup to run command into
+        :returns: Output of command.
+        """
+        cmd = self.run_into_cmd(cgroup, cmdline)
+        raw_output = self.target.execute(cmd)
+
+        # First line of output comes from shutils; strip it out.
+        return raw_output.split('\n', 1)[1]
+
+    def cgroups_tasks_move(self, srcg, dstg, exclude=''):
+        """
+        Move all the tasks from the srcg CGroup to the dstg one.
+        A regexps of tasks names can be used to defined tasks which should not
+        be moved.
+        """
+        return self.target._execute_util(
+            'cgroups_tasks_move {} {} {}'.format(srcg, dstg, exclude),
+            as_root=True)
+
+    def isolate(self, cpus, exclude=[]):
+        """
+        Remove all userspace tasks from specified CPUs.
+
+        A list of CPUs can be specified where we do not want userspace tasks
+        running. This functions creates a sandbox cpuset CGroup where all
+        user-space tasks and not-pinned kernel-space tasks are moved into.
+        This should allows to isolate the specified CPUs which will not get
+        tasks running unless explicitely moved into the isolated group.
+
+        :param cpus: the list of CPUs to isolate
+        :type cpus: list(int)
+
+        :return: the (sandbox, isolated) tuple, where:
+                 sandbox is the CGroup of sandboxed CPUs
+                 isolated is the CGroup of isolated CPUs
+        """
+        all_cpus = set(range(self.target.number_of_cpus))
+        sbox_cpus = list(all_cpus - set(cpus))
+        isol_cpus = list(all_cpus - set(sbox_cpus))
+
+        # Create Sandbox and Isolated cpuset CGroups
+        cpuset = self.controller('cpuset')
+        sbox_cg = cpuset.cgroup('/DEVLIB_SBOX')
+        isol_cg = cpuset.cgroup('/DEVLIB_ISOL')
+
+        # Set CPUs for Sandbox and Isolated CGroups
+        sbox_cg.set(cpus=sbox_cpus, mems=0)
+        isol_cg.set(cpus=isol_cpus, mems=0)
+
+        # Move all currently running tasks to the Sandbox CGroup
+        cpuset.move_all_tasks_to('/DEVLIB_SBOX', exclude)
+
+        return sbox_cg, isol_cg
+
+    def freeze(self, exclude=[], thaw=False):
+        """
+        Freeze all user-space tasks but the specified ones
+
+        A freezer cgroup is used to stop all the tasks in the target system but
+        the ones which name match one of the path specified by the exclude
+        paramater. The name of a tasks to exclude must be a substring of the
+        task named as reported by the "ps" command. Indeed, this list will be
+        translated into a: "ps | grep -e name1 -e name2..." in order to obtain
+        the PID of these tasks.
+
+        :param exclude: list of commands paths to exclude from freezer
+        :type exclude: list(str)
+
+        :param thaw: if true thaw tasks instead
+        :type thaw: bool
+        """
+
+        # Create Freezer CGroup
+        freezer = self.controller('freezer')
+        if freezer is None:
+            raise RuntimeError('freezer cgroup controller not present')
+        freezer_cg = freezer.cgroup('/DEVLIB_FREEZER')
+        thawed_cg = freezer.cgroup('/')
+
+        if thaw:
+            # Restart froozen tasks
+            freezer_cg.set(state='THAWED')
+            # Remove all tasks from freezer
+            freezer.move_all_tasks_to('/')
+            return
+
+        # Move all tasks into the freezer group
+        freezer.move_all_tasks_to('/DEVLIB_FREEZER', exclude)
+
+        # Get list of not frozen tasks, which is reported as output
+        tasks = freezer.tasks('/')
+
+        # Freeze all tasks
+        freezer_cg.set(state='FROZEN')
+
+        return tasks
+

devlib/module/cpufreq.py

@@ -133,7 +133,7 @@ class CpufreqModule(Module):
         keyword arguments. Which tunables and values are valid depends on the
         governor.
 
-        :param cpu: The cpu for which the governor will be set. This must be the
+        :param cpu: The cpu for which the governor will be set. ``int`` or
                     full cpu name as it appears in sysfs, e.g. ``cpu0``.
         :param governor: The name of the governor. Must be all lower case.
 
@@ -152,10 +152,14 @@ class CpufreqModule(Module):
         valid_tunables = self.list_governor_tunables(cpu)
         for tunable, value in kwargs.iteritems():
             if tunable in valid_tunables:
+                path = '/sys/devices/system/cpu/{}/cpufreq/{}/{}'.format(cpu, governor, tunable)
                 try:
-                    path = '/sys/devices/system/cpu/{}/cpufreq/{}/{}'.format(cpu, governor, tunable)
                     self.target.write_value(path, value)
-                except TargetError:  # May be an older kernel
+                except TargetError:
+                    if self.target.file_exists(path):
+                        # File exists but we did something wrong
+                        raise
+                    # Expected file doesn't exist, try older sysfs layout.
                     path = '/sys/devices/system/cpu/cpufreq/{}/{}'.format(governor, tunable)
                     self.target.write_value(path, value)
             else:
@@ -334,9 +338,8 @@ class CpufreqModule(Module):
 
         :param cpus: The list of CPU for which the governor is to be set.
         """
-        online_cpus = self.target.list_online_cpus()
-        for cpu in online_cpus:
-            self.set_governor(cpu, governor, kwargs)
+        for cpu in cpus:
+            self.set_governor(cpu, governor, **kwargs)
 
     def set_frequency_for_cpus(self, cpus, freq, exact=False):
         """
@@ -345,21 +348,76 @@ class CpufreqModule(Module):
 
         :param cpus: The list of CPU for which the frequency has to be set.
         """
-        online_cpus = self.target.list_online_cpus()
-        for cpu in online_cpus:
+        for cpu in cpus:
             self.set_frequency(cpu, freq, exact)
 
-    def set_all_frequencies(self, freq, exact=False):
-        self.target.execute(
-                "for CPU in /sys/devices/system/cpu/cpu[0-9]*; do "\
-                        "echo {} > $CPU/cpufreq/scaling_cur_freq; "\
-                "done"\
-                .format(freq), as_root=True)
+    def set_all_frequencies(self, freq):
+        """
+        Set the specified (minimum) frequency for all the (online) CPUs
+        """
+        return self.target._execute_util(
+                'cpufreq_set_all_frequencies {}'.format(freq),
+                as_root=True)
+
+    def get_all_frequencies(self):
+        """
+        Get the current frequency for all the (online) CPUs
+        """
+        output = self.target._execute_util(
+                'cpufreq_get_all_frequencies', as_root=True)
+        frequencies = {}
+        for x in output.splitlines():
+            kv = x.split(' ')
+            if kv[0] == '':
+                break
+            frequencies[kv[0]] = kv[1]
+        return frequencies
 
     def set_all_governors(self, governor):
-        self.target.execute(
-                "for CPU in /sys/devices/system/cpu/cpu[0-9]*; do "\
-                        "echo {} > $CPU/cpufreq/scaling_governor; "\
-                "done"\
-                .format(governor), as_root=True)
+        """
+        Set the specified governor for all the (online) CPUs
+        """
+        try:
+            return self.target._execute_util(
+                'cpufreq_set_all_governors {}'.format(governor),
+                as_root=True)
+        except TargetError as e:
+            if "echo: I/O error" in str(e):
+                cpus_unsupported = [c for c in self.target.list_online_cpus()
+                                    if governor not in self.list_governors(c)]
+                raise TargetError("Governor {} unsupported for CPUs {}".format(
+                    governor, cpus_unsupported))
+            else:
+                raise
 
+    def get_all_governors(self):
+        """
+        Get the current governor for all the (online) CPUs
+        """
+        output = self.target._execute_util(
+                'cpufreq_get_all_governors', as_root=True)
+        governors = {}
+        for x in output.splitlines():
+            kv = x.split(' ')
+            if kv[0] == '':
+                break
+            governors[kv[0]] = kv[1]
+        return governors
+
+    def trace_frequencies(self):
+        """
+        Report current frequencies on trace file
+        """
+        return self.target._execute_util('cpufreq_trace_all_frequencies', as_root=True)
+
+    @memoized
+    def get_domain_cpus(self, cpu):
+        """
+        Get the CPUs that share a frequency domain with the given CPU
+        """
+        if isinstance(cpu, int):
+            cpu = 'cpu{}'.format(cpu)
+
+        sysfile = '/sys/devices/system/cpu/{}/cpufreq/affected_cpus'.format(cpu)
+
+        return [int(c) for c in self.target.read_value(sysfile).split()]

devlib/module/cpuidle.py

@@ -47,10 +47,44 @@ class CpuidleState(object):
         self.path = path
         self.id = self.target.path.basename(self.path)
         self.cpu = self.target.path.basename(self.target.path.dirname(path))
-        self.desc = self.get('desc')
-        self.name = self.get('name')
-        self.latency = self.get('latency')
-        self.power = self.get('power')
+
+    @property
+    @memoized
+    def desc(self):
+        return self.get('desc')
+
+    @property
+    @memoized
+    def name(self):
+        return self.get('name')
+
+    @property
+    @memoized
+    def latency(self):
+        """Exit latency in uS"""
+        return self.get('latency')
+
+    @property
+    @memoized
+    def power(self):
+        """Power usage in mW
+
+        ..note::
+
+            This value is not always populated by the kernel and may be garbage.
+        """
+        return self.get('power')
+
+    @property
+    @memoized
+    def target_residency(self):
+        """Target residency in uS
+
+        This is the amount of time in the state required to 'break even' on
+        power - the system should avoid entering the state for less time than
+        this.
+        """
+        return self.get('residency')
 
     def enable(self):
         self.set('disable', 0)
@@ -113,7 +147,7 @@ class Cpuidle(Module):
     def get_state(self, state, cpu=0):
         if isinstance(state, int):
             try:
-                self.get_states(cpu)[state].enable()
+                return self.get_states(cpu)[state]
             except IndexError:
                 raise ValueError('Cpuidle state {} does not exist'.format(state))
         else:  # assume string-like
@@ -136,3 +170,9 @@ class Cpuidle(Module):
         for state in self.get_states(cpu):
             state.disable()
 
+    def perturb_cpus(self):
+        """
+        Momentarily wake each CPU. Ensures cpu_idle events in trace file.
+        """
+        output = self.target._execute_util('cpuidle_wake_all_cpus')
+        print(output)

devlib/module/hwmon.py

@@ -85,7 +85,8 @@ class HwmonDevice(object):
         path = self.path
         if not path.endswith(self.target.path.sep):
             path += self.target.path.sep
-        for entry in self.target.list_directory(path):
+        for entry in self.target.list_directory(path,
+                                                as_root=self.target.is_rooted):
             match = HWMON_FILE_REGEX.search(entry)
             if match:
                 kind = match.group('kind')
@@ -131,7 +132,8 @@ class HwmonModule(Module):
         self.scan()
 
     def scan(self):
-        for entry in self.target.list_directory(self.root):
+        for entry in self.target.list_directory(self.root,
+                                                as_root=self.target.is_rooted):
             if entry.startswith('hwmon'):
                 entry_path = self.target.path.join(self.root, entry)
                 if self.target.file_exists(self.target.path.join(entry_path, 'name')):

devlib/module/thermal.py

@@ -0,0 +1,104 @@
+#    Copyright 2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+
+from devlib.module import Module
+
+class TripPoint(object):
+    def __init__(self, zone, _id):
+        self._id = _id
+        self.zone = zone
+        self.temp_node = 'trip_point_' + _id + '_temp'
+        self.type_node = 'trip_point_' + _id + '_type'
+
+    @property
+    def target(self):
+        return self.zone.target
+
+    def get_temperature(self):
+        """Returns the currently configured temperature of the trip point"""
+        temp_file = self.target.path.join(self.zone.path, self.temp_node)
+        return self.target.read_int(temp_file)
+
+    def set_temperature(self, temperature):
+        temp_file = self.target.path.join(self.zone.path, self.temp_node)
+        self.target.write_value(temp_file, temperature)
+
+    def get_type(self):
+        """Returns the type of trip point"""
+        type_file = self.target.path.join(self.zone.path, self.type_node)
+        return self.target.read_value(type_file)
+
+class ThermalZone(object):
+    def __init__(self, target, root, _id):
+        self.target = target
+        self.name = 'thermal_zone' + _id
+        self.path = target.path.join(root, self.name)
+        self.trip_points = {}
+
+        for entry in self.target.list_directory(self.path):
+            re_match = re.match('^trip_point_([0-9]+)_temp', entry)
+            if re_match is not None:
+                self.add_trip_point(re_match.group(1))
+
+    def add_trip_point(self, _id):
+        self.trip_points[int(_id)] = TripPoint(self, _id)
+
+    def is_enabled(self):
+        """Returns a boolean representing the 'mode' of the thermal zone"""
+        value = self.target.read_value(self.target.path.join(self.path, 'mode'))
+        return value == 'enabled'
+
+    def set_mode(self, enable):
+        value = 'enabled' if enable else 'disabled'
+        self.target.write_value(self.target.path.join(self.path, 'mode'), value)
+
+    def get_temperature(self):
+        """Returns the temperature of the thermal zone"""
+        temp_file = self.target.path.join(self.path, 'temp')
+        return self.target.read_int(temp_file)
+
+class ThermalModule(Module):
+    name = 'thermal'
+    thermal_root = '/sys/class/thermal'
+
+    @staticmethod
+    def probe(target):
+
+        if target.file_exists(ThermalModule.thermal_root):
+            return True
+
+    def __init__(self, target):
+        super(ThermalModule, self).__init__(target)
+
+        self.zones = {}
+        self.cdevs = []
+
+        for entry in target.list_directory(self.thermal_root):
+            re_match = re.match('^(thermal_zone|cooling_device)([0-9]+)', entry)
+
+            if re_match.group(1) == 'thermal_zone':
+                self.add_thermal_zone(re_match.group(2))
+            elif re_match.group(1) == 'cooling_device':
+                # TODO
+                pass
+
+    def add_thermal_zone(self, _id):
+        self.zones[int(_id)] = ThermalZone(self.target, self.thermal_root, _id)
+
+    def disable_all_zones(self):
+        """Disables all the thermal zones in the target"""
+        for zone in self.zones:
+            zone.set_mode('disabled')

devlib/platform/__init__.py

@@ -1,6 +1,9 @@
 import logging
 
 
+BIG_CPUS = ['A15', 'A57', 'A72']
+
+
 class Platform(object):
 
     @property
@@ -25,7 +28,6 @@ class Platform(object):
         self.logger = logging.getLogger(self.name)
         if not self.core_clusters and self.core_names:
             self._set_core_clusters_from_core_names()
-        self._validate()
 
     def init_target_connection(self, target):
         # May be ovewritten by subclasses to provide target-specific
@@ -37,8 +39,7 @@ class Platform(object):
             self.core_names = target.cpuinfo.cpu_names
             self._set_core_clusters_from_core_names()
         if not self.big_core and self.number_of_clusters == 2:
-            big_idx = self.core_clusters.index(max(self.core_clusters))
-            self.big_core = self.core_names[big_idx]
+            self.big_core = self._identify_big_core()
         if not self.core_clusters and self.core_names:
             self._set_core_clusters_from_core_names()
         if not self.model:
@@ -47,6 +48,11 @@ class Platform(object):
             self.name = self.model
         self._validate()
 
+    def setup(self, target):
+        # May be overwritten by subclasses to provide platform-specific
+        # setup procedures.
+        pass
+
     def _set_core_clusters_from_core_names(self):
         self.core_clusters = []
         clusters = []
@@ -65,6 +71,13 @@ class Platform(object):
             except Exception:  # pylint: disable=broad-except
                 pass  # this is best-effort
 
+    def _identify_big_core(self):
+        for core in self.core_names:
+            if core.upper() in BIG_CPUS:
+                return core
+        big_idx = self.core_clusters.index(max(self.core_clusters))
+        return self.core_names[big_idx]
+
     def _validate(self):
         if len(self.core_names) != len(self.core_clusters):
             raise ValueError('core_names and core_clusters are of different lengths.')
@@ -76,6 +89,7 @@ class Platform(object):
             raise ValueError(message.format(self.big_core,
                                             ', '.join(set(self.core_names))))
         if self.big_core:
-            little_idx = self.core_clusters.index(min(self.core_clusters))
-            self.little_core = self.core_names[little_idx]
-
+            for core in self.core_names:
+                if core != self.big_core:
+                    self.little_core = core
+                    break

devlib/platform/arm.py

@@ -20,7 +20,7 @@ import time
 import pexpect
 
 from devlib.platform import Platform
-from devlib.instrument import Instrument, InstrumentChannel, MeasurementsCsv, CONTINUOUS
+from devlib.instrument import Instrument, InstrumentChannel, MeasurementsCsv, Measurement,  CONTINUOUS,  INSTANTANEOUS
 from devlib.exception import TargetError, HostError
 from devlib.host import PACKAGE_BIN_DIRECTORY
 from devlib.utils.serial_port import open_serial_connection
@@ -145,9 +145,12 @@ class VersatileExpressPlatform(Platform):
                                                          'bootargs': self.bootargs,
                                                          }})
         elif self.bootloader == 'u-boot':
+            uboot_env = None
+            if self.bootargs:
+                uboot_env = {'bootargs': self.bootargs}
             self.modules.append({'vexpress-u-boot': {'port': self.serial_port,
                                                      'baudrate': self.baudrate,
-                                                     'env': {'bootargs': self.bootargs},
+                                                     'env': uboot_env,
                                                      }})
         elif self.bootloader == 'bootmon':
             self.modules.append({'vexpress-bootmon': {'port': self.serial_port,
@@ -204,7 +207,7 @@ class TC2(VersatileExpressPlatform):
 class JunoEnergyInstrument(Instrument):
 
     binname = 'readenergy'
-    mode = CONTINUOUS
+    mode = CONTINUOUS | INSTANTANEOUS
 
     _channels = [
         InstrumentChannel('sys_curr', 'sys', 'current'),
@@ -233,7 +236,9 @@ class JunoEnergyInstrument(Instrument):
         for chan in self._channels:
             self.channels[chan.name] = chan
         self.on_target_file = self.target.tempfile('energy', '.csv')
+        self.sample_rate_hz = 10 # DEFAULT_PERIOD is 100[ms] in readenergy.c
         self.command = '{} -o {}'.format(self.binary, self.on_target_file)
+        self.command2 = '{}'.format(self.binary)
 
     def setup(self):
         self.binary = self.target.install(os.path.join(PACKAGE_BIN_DIRECTORY,
@@ -277,4 +282,14 @@ class JunoEnergyInstrument(Instrument):
 
         return MeasurementsCsv(output_file, self.active_channels)
 
+    def take_measurement(self):
+        result = []
+        output = self.target.execute(self.command2).split()
+        reader=csv.reader(output)
+        headings=reader.next()
+        values = reader.next()
+        for chan in self.active_channels:
+            value = values[headings.index(chan.name)]
+            result.append(Measurement(value, chan))
+        return result
 

devlib/platform/gem5.py

@@ -0,0 +1,292 @@
+#    Copyright 2016 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import subprocess
+import sys
+import shutil
+import time
+import types
+
+from devlib.exception import TargetError
+from devlib.host import PACKAGE_BIN_DIRECTORY
+from devlib.platform import Platform
+from devlib.utils.ssh import AndroidGem5Connection, LinuxGem5Connection
+
+class Gem5SimulationPlatform(Platform):
+
+    def __init__(self, name,
+                 host_output_dir,
+                 gem5_bin,
+                 gem5_args,
+                 gem5_virtio,
+                 core_names=None,
+                 core_clusters=None,
+                 big_core=None,
+                 model=None,
+                 modules=None,
+                 gem5_telnet_port=None):
+
+        # First call the parent class
+        super(Gem5SimulationPlatform, self).__init__(name, core_names, core_clusters,
+                                                     big_core, model, modules)
+
+        # Start setting up the gem5 parameters/directories
+        # The gem5 subprocess
+        self.gem5 = None
+        self.gem5_port = gem5_telnet_port or None
+        self.stats_directory = host_output_dir
+        self.gem5_out_dir = os.path.join(self.stats_directory, "gem5")
+        self.gem5_interact_dir = '/tmp' # Host directory
+        self.executable_dir = None # Device directory
+        self.working_dir = None # Device directory
+        self.stdout_file = None
+        self.stderr_file = None
+        self.stderr_filename = None
+        if self.gem5_port is None:
+            # Allows devlib to pick up already running simulations
+            self.start_gem5_simulation = True
+        else:
+            self.start_gem5_simulation = False
+
+        # Find the first one that does not exist. Ensures that we do not re-use
+        # the directory used by someone else.
+        for i in xrange(sys.maxint):
+            directory = os.path.join(self.gem5_interact_dir, "wa_{}".format(i))
+            try:
+                os.stat(directory)
+                continue
+            except OSError:
+                break
+        self.gem5_interact_dir = directory
+        self.logger.debug("Using {} as the temporary directory."
+                          .format(self.gem5_interact_dir))
+
+        # Parameters passed onto gem5
+        self.gem5args_binary = gem5_bin
+        self.gem5args_args = gem5_args
+        self.gem5args_virtio = gem5_virtio
+        self._check_gem5_command()
+
+        # Start the interaction with gem5
+        self._start_interaction_gem5()
+
+    def _check_gem5_command(self):
+        """
+        Check if the command to start gem5 makes sense
+        """
+        if self.gem5args_binary is None:
+            raise TargetError('Please specify a gem5 binary.')
+        if self.gem5args_args is None:
+            raise TargetError('Please specify the arguments passed on to gem5.')
+        self.gem5args_virtio = str(self.gem5args_virtio).format(self.gem5_interact_dir)
+        if self.gem5args_virtio is None:
+            raise TargetError('Please specify arguments needed for virtIO.')
+
+    def _start_interaction_gem5(self):
+        """
+        Starts the interaction of devlib with gem5.
+        """
+
+        # First create the input and output directories for gem5
+        if self.start_gem5_simulation:
+            # Create the directory to send data to/from gem5 system
+            self.logger.info("Creating temporary directory for interaction "
+                             " with gem5 via virtIO: {}"
+                             .format(self.gem5_interact_dir))
+            os.mkdir(self.gem5_interact_dir)
+
+            # Create the directory for gem5 output (stats files etc)
+            if not os.path.exists(self.stats_directory):
+                os.mkdir(self.stats_directory)
+            if os.path.exists(self.gem5_out_dir):
+                raise TargetError("The gem5 stats directory {} already "
+                                  "exists.".format(self.gem5_out_dir))
+            else:
+                os.mkdir(self.gem5_out_dir)
+
+            # We need to redirect the standard output and standard error for the
+            # gem5 process to a file so that we can debug when things go wrong.
+            f = os.path.join(self.gem5_out_dir, 'stdout')
+            self.stdout_file = open(f, 'w')
+            f = os.path.join(self.gem5_out_dir, 'stderr')
+            self.stderr_file = open(f, 'w')
+            # We need to keep this so we can check which port to use for the
+            # telnet connection.
+            self.stderr_filename = f
+
+            # Start gem5 simulation
+            self.logger.info("Starting the gem5 simulator")
+
+            command_line = "{} --outdir={} {} {}".format(self.gem5args_binary,
+                                                         self.gem5_out_dir,
+                                                         self.gem5args_args,
+                                                         self.gem5args_virtio)
+            self.logger.debug("gem5 command line: {}".format(command_line))
+            self.gem5 = subprocess.Popen(command_line.split(),
+                                         stdout=self.stdout_file,
+                                         stderr=self.stderr_file)
+
+        else:
+            # The simulation should already be running
+            # Need to dig up the (1) gem5 simulation in question (2) its input
+            # and output directories (3) virtio setting
+            self._intercept_existing_gem5()
+
+        # As the gem5 simulation is running now or was already running
+        # we now need to find out which telnet port it uses
+        self._intercept_telnet_port()
+
+    def _intercept_existing_gem5(self):
+        """
+        Intercept the information about a running gem5 simulation
+        e.g. pid, input directory etc
+        """
+        self.logger("This functionality is not yet implemented")
+        raise TargetError()
+
+    def _intercept_telnet_port(self):
+        """
+        Intercept the telnet port of a running gem5 simulation
+        """
+
+        if self.gem5 is None:
+            raise TargetError('The platform has no gem5 simulation! '
+                              'Something went wrong')
+        while self.gem5_port is None:
+            # Check that gem5 is running!
+            if self.gem5.poll():
+                raise TargetError("The gem5 process has crashed with error code {}!".format(self.gem5.poll()))
+
+            # Open the stderr file
+            with open(self.stderr_filename, 'r') as f:
+                for line in f:
+                    m = re.search(r"Listening for system connection on port (?P<port>\d+)", line)
+                    if m:
+                        port = int(m.group('port'))
+                        if port >= 3456 and port < 5900:
+                            self.gem5_port = port
+                            break
+                    # Check if the sockets are not disabled
+                    m = re.search(r"Sockets disabled, not accepting terminal connections", line)
+                    if m:
+                        raise TargetError("The sockets have been disabled!"
+                                          "Pass --listener-mode=on to gem5")
+                else:
+                    time.sleep(1)
+
+    def init_target_connection(self, target):
+        """
+        Update the type of connection in the target from here
+        """
+        if target.os == 'linux':
+            target.conn_cls = LinuxGem5Connection
+        else:
+            target.conn_cls = AndroidGem5Connection
+
+    def setup(self, target):
+        """
+        Deploy m5 if not yet installed
+        """
+        m5_path = target.get_installed('m5')
+        if m5_path is None:
+            m5_path = self._deploy_m5(target)
+        target.conn.m5_path = m5_path
+
+        # Set the terminal settings for the connection to gem5
+        self._resize_shell(target)
+
+    def update_from_target(self, target):
+        """
+        Set the m5 path and if not yet installed, deploy m5
+        Overwrite certain methods in the target that either can be done
+        more efficiently by gem5 or don't exist in gem5
+        """
+        m5_path = target.get_installed('m5')
+        if m5_path is None:
+            m5_path = self._deploy_m5(target)
+        target.conn.m5_path = m5_path
+
+        # Overwrite the following methods (monkey-patching)
+        self.logger.debug("Overwriting the 'capture_screen' method in target")
+        # Housekeeping to prevent recursion
+        setattr(target, 'target_impl_capture_screen', target.capture_screen)
+        target.capture_screen = types.MethodType(_overwritten_capture_screen, target)
+        self.logger.debug("Overwriting the 'reset' method in target")
+        target.reset = types.MethodType(_overwritten_reset, target)
+        self.logger.debug("Overwriting the 'reboot' method in target")
+        target.reboot = types.MethodType(_overwritten_reboot, target)
+
+        # Call the general update_from_target implementation
+        super(Gem5SimulationPlatform, self).update_from_target(target)
+
+    def gem5_capture_screen(self, filepath):
+        file_list = os.listdir(self.gem5_out_dir)
+        screen_caps = []
+        for f in file_list:
+            if '.bmp' in f:
+                screen_caps.append(f)
+
+        successful_capture = False
+        if len(screen_caps) == 1:
+            # Bail out if we do not have image, and resort to the slower, built
+            # in method.
+            try:
+                import Image
+                gem5_image = os.path.join(self.gem5_out_dir, screen_caps[0])
+                temp_image = os.path.join(self.gem5_out_dir, "file.png")
+                im = Image.open(gem5_image)
+                im.save(temp_image, "PNG")
+                shutil.copy(temp_image, filepath)
+                os.remove(temp_image)
+                gem5_logger.info("capture_screen: using gem5 screencap")
+                successful_capture = True
+
+            except (shutil.Error, ImportError, IOError):
+                pass
+
+        return successful_capture
+
+    def _deploy_m5(self, target):
+        # m5 is not yet installed so install it
+        host_executable = os.path.join(PACKAGE_BIN_DIRECTORY,
+                                       target.abi, 'm5')
+        return target.install(host_executable)
+
+    def _resize_shell(self, target):
+        """
+        Resize the shell to avoid line wrapping issues.
+
+        """
+        # Try and avoid line wrapping as much as possible.
+        target.execute('{} stty columns 1024'.format(target.busybox))
+        target.execute('reset', check_exit_code=False)
+
+# Methods that will be monkey-patched onto the target
+def _overwritten_reset(self):
+    raise TargetError('Resetting is not allowed on gem5 platforms!')
+
+def _overwritten_reboot(self):
+    raise TargetError('Rebooting is not allowed on gem5 platforms!')
+
+def _overwritten_capture_screen(self, filepath):
+    connection_screencapped = self.platform.gem5_capture_screen(filepath)
+    if connection_screencapped == False:
+        # The connection was not able to capture the screen so use the target
+        # implementation
+        self.logger.debug('{} was not able to screen cap, using the original target implementation'.format(self.platform.__class__.__name__))
+        self.target_impl_capture_screen(filepath)
+
+

devlib/target.py

@@ -19,18 +19,20 @@ from devlib.utils.misc import ABI_MAP, get_cpu_name, ranges_to_list, escape_doub
 from devlib.utils.types import integer, boolean, bitmask, identifier, caseless_string
 
 
-FSTAB_ENTRY_REGEX = re.compile(r'(\S+) on (\S+) type (\S+) \((\S+)\)')
-ANDROID_SCREEN_STATE_REGEX = re.compile('(?:mPowerState|mScreenOn)=([0-9]+|true|false)',
+FSTAB_ENTRY_REGEX = re.compile(r'(\S+) on (.+) type (\S+) \((\S+)\)')
+ANDROID_SCREEN_STATE_REGEX = re.compile('(?:mPowerState|mScreenOn|Display Power: state)=([0-9]+|true|false|ON|OFF)',
                                         re.IGNORECASE)
 ANDROID_SCREEN_RESOLUTION_REGEX = re.compile(r'mUnrestrictedScreen=\(\d+,\d+\)'
                                              r'\s+(?P<width>\d+)x(?P<height>\d+)')
 DEFAULT_SHELL_PROMPT = re.compile(r'^.*(shell|root)@.*:/\S* [#$] ',
                                   re.MULTILINE)
+KVERSION_REGEX =re.compile(
+    r'(?P<version>\d+)(\.(?P<major>\d+)(\.(?P<minor>\d+)(-rc(?P<rc>\d+))?)?)?(.*-g(?P<sha1>[0-9a-fA-F]{7,}))?'
+)
 
 
 class Target(object):
 
-    conn_cls = None
     path = None
     os = None
 
@@ -63,10 +65,11 @@ class Target(object):
         return self.conn is not None
 
     @property
-    @memoized
     def connected_as_root(self):
-        result = self.execute('id')
-        return 'uid=0(' in result
+        if self._connected_as_root is None:
+            result = self.execute('id')
+            self._connected_as_root = 'uid=0(' in result
+        return self._connected_as_root
 
     @property
     @memoized
@@ -79,10 +82,15 @@ class Target(object):
         except (TargetError, TimeoutError):
             return False
 
+    @property
+    @memoized
+    def needs_su(self):
+        return not self.connected_as_root and self.is_rooted
+
     @property
     @memoized
     def kernel_version(self):
-        return KernelVersion(self.execute('uname -r -v').strip())
+        return KernelVersion(self.execute('{} uname -r -v'.format(self.busybox)).strip())
 
     @property
     def os_version(self):  # pylint: disable=no-self-use
@@ -145,14 +153,32 @@ class Target(object):
                  modules=None,
                  load_default_modules=True,
                  shell_prompt=DEFAULT_SHELL_PROMPT,
+                 conn_cls=None,
                  ):
+        self._connected_as_root = None
         self.connection_settings = connection_settings or {}
-        self.platform = platform or Platform()
+        # Set self.platform: either it's given directly (by platform argument)
+        # or it's given in the connection_settings argument
+        # If neither, create default Platform()
+        if platform is None:
+            self.platform = self.connection_settings.get('platform', Platform())
+        else:
+            self.platform = platform
+        # Check if the user hasn't given two different platforms
+        if 'platform' in self.connection_settings:
+            if connection_settings['platform'] is not platform:
+                raise TargetError('Platform specified in connection_settings '
+                                   '({}) differs from that directly passed '
+                                   '({})!)'
+                                   .format(connection_settings['platform'],
+                                    self.platform))
+        self.connection_settings['platform'] = self.platform
         self.working_directory = working_directory
         self.executables_directory = executables_directory
         self.modules = modules or []
         self.load_default_modules = load_default_modules
         self.shell_prompt = shell_prompt
+        self.conn_cls = conn_cls
         self.logger = logging.getLogger(self.__class__.__name__)
         self._installed_binaries = {}
         self._installed_modules = {}
@@ -176,6 +202,7 @@ class Target(object):
         self.platform.init_target_connection(self)
         tid = id(threading.current_thread())
         self._connections[tid] = self.get_connection(timeout=timeout)
+        self._resolve_paths()
         self.busybox = self.get_installed('busybox')
         self.platform.update_from_target(self)
         self._update_modules('connected')
@@ -188,17 +215,39 @@ class Target(object):
         self._connections = {}
 
     def get_connection(self, timeout=None):
-        if self.conn_cls is None:
-            raise NotImplementedError('conn_cls must be set by the subclass of Target')
+        if self.conn_cls == None:
+            raise ValueError('Connection class not specified on Target creation.')
         return self.conn_cls(timeout=timeout, **self.connection_settings)  # pylint: disable=not-callable
 
     def setup(self, executables=None):
         self.execute('mkdir -p {}'.format(self.working_directory))
         self.execute('mkdir -p {}'.format(self.executables_directory))
         self.busybox = self.install(os.path.join(PACKAGE_BIN_DIRECTORY, self.abi, 'busybox'))
+
+        # Setup shutils script for the target
+        shutils_ifile = os.path.join(PACKAGE_BIN_DIRECTORY, 'scripts', 'shutils.in')
+        shutils_ofile = os.path.join(PACKAGE_BIN_DIRECTORY, 'scripts', 'shutils')
+        shell_path = '/bin/sh'
+        if self.os == 'android':
+            shell_path = '/system/bin/sh'
+        with open(shutils_ifile) as fh:
+            lines = fh.readlines()
+        with open(shutils_ofile, 'w') as ofile:
+            for line in lines:
+                line = line.replace("__DEVLIB_SHELL__", shell_path)
+                line = line.replace("__DEVLIB_BUSYBOX__", self.busybox)
+                ofile.write(line)
+        self.shutils = self.install(os.path.join(PACKAGE_BIN_DIRECTORY, 'scripts', 'shutils'))
+
         for host_exe in (executables or []):  # pylint: disable=superfluous-parens
             self.install(host_exe)
 
+        # Check for platform dependent setup procedures
+        self.platform.setup(self)
+
+        # Initialize modules which requires Buxybox (e.g. shutil dependent tasks)
+        self._update_modules('setup')
+
     def reboot(self, hard=False, connect=True, timeout=180):
         if hard:
             if not self.has('hard_reset'):
@@ -211,8 +260,16 @@ class Target(object):
                           '(in which case, a hard_reset module must be installed)'
                 raise TargetError(message)
             self.reset()
+            # Wait a fixed delay before starting polling to give the target time to
+            # shut down, otherwise, might create the connection while it's still shutting
+            # down resulting in subsequenct connection failing.
+            self.logger.debug('Waiting for target to power down...')
+            reset_delay = 20
+            time.sleep(reset_delay)
+            timeout = max(timeout - reset_delay, 10)
         if self.has('boot'):
             self.boot()  # pylint: disable=no-member
+        self._connected_as_root = None
         if connect:
             self.connect(timeout=timeout)
 
@@ -226,6 +283,10 @@ class Target(object):
 
     # execution
 
+    def _execute_util(self, command, timeout=None, check_exit_code=True, as_root=False):
+        command = '{} {}'.format(self.shutils, command)
+        return self.conn.execute(command, timeout, check_exit_code, as_root)
+
     def execute(self, command, timeout=None, check_exit_code=True, as_root=False):
         return self.conn.execute(command, timeout, check_exit_code, as_root)
 
@@ -252,6 +313,7 @@ class Target(object):
                   a ``TimeoutError`` exception will be raised. Set to ``None`` if the
                   invocation should not timeout.
 
+        :returns: output of command.
         """
         command = binary
         if args:
@@ -271,7 +333,7 @@ class Target(object):
     # sysfs interaction
 
     def read_value(self, path, kind=None):
-        output = self.execute('cat \'{}\''.format(path), as_root=self.is_rooted).strip()  # pylint: disable=E1103
+        output = self.execute('cat \'{}\''.format(path), as_root=self.needs_su).strip()  # pylint: disable=E1103
         if kind:
             return kind(output)
         else:
@@ -294,10 +356,11 @@ class Target(object):
 
     def reset(self):
         try:
-            self.execute('reboot', as_root=self.is_rooted, timeout=2)
+            self.execute('reboot', as_root=self.needs_su, timeout=2)
         except (TargetError, TimeoutError, subprocess.CalledProcessError):
             # on some targets "reboot" doesn't return gracefully
             pass
+        self._connected_as_root = None
 
     def check_responsive(self):
         try:
@@ -313,7 +376,10 @@ class Target(object):
 
     def killall(self, process_name, signal=None, as_root=False):
         for pid in self.get_pids_of(process_name):
-            self.kill(pid, signal=signal, as_root=as_root)
+            try:
+                self.kill(pid, signal=signal, as_root=as_root)
+            except TargetError:
+                pass
 
     def get_pids_of(self, process_name):
         raise NotImplementedError()
@@ -325,7 +391,14 @@ class Target(object):
 
     def file_exists(self, filepath):
         command = 'if [ -e \'{}\' ]; then echo 1; else echo 0; fi'
-        return boolean(self.execute(command.format(filepath)).strip())
+        output = self.execute(command.format(filepath), as_root=self.is_rooted)
+        return boolean(output.strip())
+
+    def directory_exists(self, filepath):
+        output = self.execute('if [ -d \'{}\' ]; then echo 1; else echo 0; fi'.format(filepath))
+        # output from ssh my contain part of the expression in the buffer,
+        # split out everything except the last word.
+        return boolean(output.split()[-1])  # pylint: disable=maybe-no-member
 
     def list_file_systems(self):
         output = self.execute('mount')
@@ -394,18 +467,30 @@ class Target(object):
     def uninstall(self, name):
         raise NotImplementedError()
 
-    def get_installed(self, name):
-        for path in self.getenv('PATH').split(self.path.pathsep):
-            try:
-                if  name in self.list_directory(path):
-                    return self.path.join(path, name)
-            except TargetError:
-                pass  # directory does not exist or no executable premssions
-        if name in self.list_directory(self.executables_directory):
-            return self.path.join(self.executables_directory, name)
+    def get_installed(self, name, search_system_binaries=True):
+        # Check user installed binaries first
+        if self.file_exists(self.executables_directory):
+            if name in self.list_directory(self.executables_directory):
+                return self.path.join(self.executables_directory, name)
+        # Fall back to binaries in PATH
+        if search_system_binaries:
+            for path in self.getenv('PATH').split(self.path.pathsep):
+                try:
+                    if name in self.list_directory(path):
+                        return self.path.join(path, name)
+                except TargetError:
+                    pass  # directory does not exist or no executable premssions
 
     which = get_installed
 
+    def install_if_needed(self, host_path, search_system_binaries=True):
+
+        binary_path = self.get_installed(os.path.split(host_path)[1],
+                                         search_system_binaries=search_system_binaries)
+        if not binary_path:
+            binary_path = self.install(host_path)
+        return binary_path
+
     def is_installed(self, name):
         return bool(self.get_installed(name))
 
@@ -415,6 +500,87 @@ class Target(object):
     def has(self, modname):
         return hasattr(self, identifier(modname))
 
+    def lsmod(self):
+        lines = self.execute('lsmod').splitlines()
+        entries = []
+        for line in lines[1:]:  # first line is the header
+            if not line.strip():
+                continue
+            parts = line.split()
+            name = parts[0]
+            size = int(parts[1])
+            use_count = int(parts[2])
+            if len(parts) > 3:
+                used_by = ''.join(parts[3:]).split(',')
+            else:
+                used_by = []
+            entries.append(LsmodEntry(name, size, use_count, used_by))
+        return entries
+
+    def insmod(self, path):
+        target_path = self.get_workpath(os.path.basename(path))
+        self.push(path, target_path)
+        self.execute('insmod {}'.format(target_path), as_root=True)
+
+
+    def extract(self, path, dest=None):
+        """
+        Extact the specified on-target file. The extraction method to be used
+        (unzip, gunzip, bunzip2, or tar) will be based on the file's extension.
+        If ``dest`` is specified, it must be an existing directory on target;
+        the extracted contents will be placed there.
+
+        Note that, depending on the archive file format (and therfore the
+        extraction method used), the original archive file may or may not exist
+        after the extraction.
+
+        The return value is the path to the extracted contents.  In case of
+        gunzip and bunzip2, this will be path to the extracted file; for tar
+        and uzip, this will be the directory with the extracted file(s)
+        (``dest`` if it was specified otherwise, the directory that cotained
+        the archive).
+
+        """
+        for ending in ['.tar.gz', '.tar.bz', '.tar.bz2',
+                       '.tgz', '.tbz', '.tbz2']:
+            if path.endswith(ending):
+                return self._extract_archive(path, 'tar xf {} -C {}', dest)
+
+        ext = self.path.splitext(path)[1]
+        if ext in ['.bz', '.bz2']:
+            return self._extract_file(path, 'bunzip2 -f {}', dest)
+        elif ext == '.gz':
+            return self._extract_file(path, 'gunzip -f {}', dest)
+        elif ext == '.zip':
+            return self._extract_archive(path, 'unzip {} -d {}', dest)
+        else:
+            raise ValueError('Unknown compression format: {}'.format(ext))
+
+    # internal methods
+
+    def _extract_archive(self, path, cmd, dest=None):
+        cmd = '{} ' + cmd  # busybox
+        if dest:
+            extracted = dest
+        else:
+            extracted = self.path.dirname(path)
+        cmdtext = cmd.format(self.busybox, path, extracted)
+        self.execute(cmdtext)
+        return extracted
+
+    def _extract_file(self, path, cmd, dest=None):
+        cmd = '{} ' + cmd  # busybox
+        cmdtext = cmd.format(self.busybox, path)
+        self.execute(cmdtext)
+        extracted = self.path.splitext(path)[0]
+        if dest:
+            self.execute('mv -f {} {}'.format(extracted, dest))
+            if dest.endswith('/'):
+                extracted = self.path.join(dest, self.path.basename(extracted))
+            else:
+                extracted = dest
+        return extracted
+
     def _update_modules(self, stage):
         for mod in self.modules:
             if isinstance(mod, dict):
@@ -427,7 +593,11 @@ class Target(object):
             if mod.probe(self):
                 self._install_module(mod, **params)
             else:
-                self.logger.debug('Module {} is not supported by the target'.format(mod.name))
+                msg = 'Module {} is not supported by the target'.format(mod.name)
+                if self.load_default_modules:
+                    self.logger.debug(msg)
+                else:
+                    self.logger.warning(msg)
 
     def _install_module(self, mod, **params):
         if mod.name not in self._installed_modules:
@@ -437,10 +607,12 @@ class Target(object):
         else:
             self.logger.debug('Module {} is already installed.'.format(mod.name))
 
+    def _resolve_paths(self):
+        raise NotImplementedError()
+
 
 class LinuxTarget(Target):
 
-    conn_cls = SshConnection
     path = posixpath
     os = 'linux'
 
@@ -471,15 +643,39 @@ class LinuxTarget(Target):
             raise
         return os_version
 
+    @property
+    @memoized
+    # There is currently no better way to do this cross platform.
+    # ARM does not have dmidecode
+    def model(self):
+        if self.file_exists("/proc/device-tree/model"):
+            raw_model = self.execute("cat /proc/device-tree/model")
+            return '_'.join(raw_model.split()[:2])
+        return None
+
+    def __init__(self,
+                 connection_settings=None,
+                 platform=None,
+                 working_directory=None,
+                 executables_directory=None,
+                 connect=True,
+                 modules=None,
+                 load_default_modules=True,
+                 shell_prompt=DEFAULT_SHELL_PROMPT,
+                 conn_cls=SshConnection,
+                 ):
+        super(LinuxTarget, self).__init__(connection_settings=connection_settings,
+                                          platform=platform,
+                                          working_directory=working_directory,
+                                          executables_directory=executables_directory,
+                                          connect=connect,
+                                          modules=modules,
+                                          load_default_modules=load_default_modules,
+                                          shell_prompt=shell_prompt,
+                                          conn_cls=conn_cls)
+
     def connect(self, timeout=None):
         super(LinuxTarget, self).connect(timeout=timeout)
-        if self.working_directory is None:
-            if self.connected_as_root:
-                self.working_directory = '/root/devlib-target'
-            else:
-                self.working_directory = '/home/{}/devlib-target'.format(self.user)
-        if self.executables_directory is None:
-            self.executables_directory = self.path.join(self.working_directory, 'bin')
 
     def kick_off(self, command, as_root=False):
         command = 'sh -c "{}" 1>/dev/null 2>/dev/null &'.format(escape_double_quotes(command))
@@ -547,12 +743,21 @@ class LinuxTarget(Target):
             message = e.message.split('OUTPUT:', 1)[1].strip()  # pylint: disable=no-member
             self.logger.debug('Could not take screenshot: {}'.format(message))
 
+    def _resolve_paths(self):
+        if self.working_directory is None:
+            if self.connected_as_root:
+                self.working_directory = '/root/devlib-target'
+            else:
+                self.working_directory = '/home/{}/devlib-target'.format(self.user)
+        if self.executables_directory is None:
+            self.executables_directory = self.path.join(self.working_directory, 'bin')
+
 
 class AndroidTarget(Target):
 
-    conn_cls = AdbConnection
     path = posixpath
     os = 'android'
+    ls_command = ''
 
     @property
     @memoized
@@ -573,6 +778,30 @@ class AndroidTarget(Target):
     def adb_name(self):
         return self.conn.device
 
+    @property
+    @memoized
+    def android_id(self):
+        """
+        Get the device's ANDROID_ID. Which is
+
+            "A 64-bit number (as a hex string) that is randomly generated when the user
+            first sets up the device and should remain constant for the lifetime of the
+            user's device."
+
+        .. note:: This will get reset on userdata erasure.
+
+        """
+        output = self.execute('content query --uri content://settings/secure --projection value --where "name=\'android_id\'"').strip()
+        return output.split('value=')[-1]
+
+    @property
+    @memoized
+    def model(self):
+        try:
+            return self.getprop(prop='ro.product.device')
+        except KeyError:
+            return None
+
     @property
     @memoized
     def screen_resolution(self):
@@ -584,18 +813,37 @@ class AndroidTarget(Target):
         else:
             return (0, 0)
 
-
-    def __init__(self, *args, **kwargs):
-        super(AndroidTarget, self).__init__(*args, **kwargs)
-        self._file_transfer_cache = None
+    def __init__(self,
+                 connection_settings=None,
+                 platform=None,
+                 working_directory=None,
+                 executables_directory=None,
+                 connect=True,
+                 modules=None,
+                 load_default_modules=True,
+                 shell_prompt=DEFAULT_SHELL_PROMPT,
+                 conn_cls=AdbConnection,
+                 package_data_directory="/data/data",
+                 ):
+        super(AndroidTarget, self).__init__(connection_settings=connection_settings,
+                                            platform=platform,
+                                            working_directory=working_directory,
+                                            executables_directory=executables_directory,
+                                            connect=connect,
+                                            modules=modules,
+                                            load_default_modules=load_default_modules,
+                                            shell_prompt=shell_prompt,
+                                            conn_cls=conn_cls)
+        self.package_data_directory = package_data_directory
 
     def reset(self, fastboot=False):  # pylint: disable=arguments-differ
         try:
             self.execute('reboot {}'.format(fastboot and 'fastboot' or ''),
-                         as_root=self.is_rooted, timeout=2)
+                         as_root=self.needs_su, timeout=2)
         except (TargetError, TimeoutError, subprocess.CalledProcessError):
             # on some targets "reboot" doesn't return gracefully
             pass
+        self._connected_as_root = None
 
     def connect(self, timeout=10, check_boot_completed=True):  # pylint: disable=arguments-differ
         start = time.time()
@@ -608,11 +856,6 @@ class AndroidTarget(Target):
             # always disconnect first.
             adb_disconnect(device)
         super(AndroidTarget, self).connect(timeout=timeout)
-        if self.working_directory is None:
-            self.working_directory = '/data/local/tmp/devlib-target'
-        self._file_transfer_cache = self.path.join(self.working_directory, '.file-cache')
-        if self.executables_directory is None:
-            self.executables_directory = self.path.join(self.working_directory, 'bin')
 
         if check_boot_completed:
             boot_completed = boolean(self.getprop('sys.boot_completed'))
@@ -626,27 +869,37 @@ class AndroidTarget(Target):
         super(AndroidTarget, self).setup(executables)
         self.execute('mkdir -p {}'.format(self._file_transfer_cache))
 
-    def kick_off(self, command, as_root=False):
+    def kick_off(self, command, as_root=None):
         """
         Like execute but closes adb session and returns immediately, leaving the command running on the
         device (this is different from execute(background=True) which keeps adb connection open and returns
         a subprocess object).
-
-        .. note:: This relies on busybox's nohup applet and so won't work on unrooted devices.
-
         """
-        if not self.is_rooted:
-            raise TargetError('kick_off uses busybox\'s nohup applet and so can only be run a rooted device.')
+        if as_root is None:
+            as_root = self.needs_su
         try:
-            command = 'cd {} && {} nohup {}'.format(self.working_directory, self.bin('busybox'), command)
+            command = 'cd {} && {} nohup {} &'.format(self.working_directory, self.busybox, command)
             output = self.execute(command, timeout=1, as_root=as_root)
         except TimeoutError:
             pass
-        else:
-            raise ValueError('Background command exited before timeout; got "{}"'.format(output))
+
+    def __setup_list_directory(self):
+        # In at least Linaro Android 16.09 (which was their first Android 7 release) and maybe
+        # AOSP 7.0 as well, the ls command was changed.
+        # Previous versions default to a single column listing, which is nice and easy to parse.
+        # Newer versions default to a multi-column listing, which is not, but it does support
+        # a '-1' option to get into single column mode. Older versions do not support this option
+        # so we try the new version, and if it fails we use the old version.
+        self.ls_command = 'ls -1'
+        try:
+            self.execute('ls -1 {}'.format(self.working_directory), as_root=False)
+        except TargetError:
+            self.ls_command = 'ls'
 
     def list_directory(self, path, as_root=False):
-        contents = self.execute('ls {}'.format(path), as_root=as_root)
+        if self.ls_command == '':
+            self.__setup_list_directory()
+        contents = self.execute('{} {}'.format(self.ls_command, path), as_root=as_root)
         return [x.strip() for x in contents.split('\n') if x.strip()]
 
     def install(self, filepath, timeout=None, with_name=None):  # pylint: disable=W0221
@@ -674,7 +927,7 @@ class AndroidTarget(Target):
         lines.next()  # header
         result = []
         for line in lines:
-            parts = line.split()
+            parts = line.split(None, 8)
             if parts:
                 result.append(PsEntry(*(parts[0:1] + map(int, parts[1:5]) + parts[5:])))
         if not kwargs:
@@ -697,28 +950,36 @@ class AndroidTarget(Target):
             self.conn.push(source, dest, timeout=timeout)
         else:
             device_tempfile = self.path.join(self._file_transfer_cache, source.lstrip(self.path.sep))
-            self.execute('mkdir -p {}'.format(self.path.dirname(device_tempfile)))
+            self.execute("mkdir -p '{}'".format(self.path.dirname(device_tempfile)))
             self.conn.push(source, device_tempfile, timeout=timeout)
-            self.execute('cp {} {}'.format(device_tempfile, dest), as_root=True)
+            self.execute("cp '{}' '{}'".format(device_tempfile, dest), as_root=True)
 
     def pull(self, source, dest, as_root=False, timeout=None):  # pylint: disable=arguments-differ
         if not as_root:
             self.conn.pull(source, dest, timeout=timeout)
         else:
             device_tempfile = self.path.join(self._file_transfer_cache, source.lstrip(self.path.sep))
-            self.execute('mkdir -p {}'.format(self.path.dirname(device_tempfile)))
-            self.execute('cp {} {}'.format(source, device_tempfile), as_root=True)
+            self.execute("mkdir -p '{}'".format(self.path.dirname(device_tempfile)))
+            self.execute("cp '{}' '{}'".format(source, device_tempfile), as_root=True)
+            self.execute("chmod 0644 '{}'".format(device_tempfile), as_root=True)
             self.conn.pull(device_tempfile, dest, timeout=timeout)
 
     # Android-specific
 
-    def swipe_to_unlock(self):
+    def swipe_to_unlock(self, direction="horizontal"):
         width, height = self.screen_resolution
-        swipe_heigh = height * 2 // 3
-        start = 100
-        stop = width - start
         command = 'input swipe {} {} {} {}'
-        self.execute(command.format(start, swipe_heigh, stop, swipe_heigh))
+        if direction == "horizontal":
+            swipe_heigh = height * 2 // 3
+            start = 100
+            stop = width - start
+            self.execute(command.format(start, swipe_heigh, stop, swipe_heigh))
+        if direction == "vertical":
+            swipe_middle = height / 2
+            swipe_heigh = height * 2 // 3
+            self.execute(command.format(swipe_middle, swipe_heigh, swipe_middle, 0))
+        else:
+            raise DeviceError("Invalid swipe direction: {}".format(self.swipe_to_unlock))
 
     def getprop(self, prop=None):
         props = AndroidProperties(self.execute('getprop'))
@@ -747,7 +1008,7 @@ class AndroidTarget(Target):
     def install_apk(self, filepath, timeout=None):  # pylint: disable=W0221
         ext = os.path.splitext(filepath)[1].lower()
         if ext == '.apk':
-            return adb_command(self.adb_name, "install {}".format(filepath), timeout=timeout)
+            return adb_command(self.adb_name, "install '{}'".format(filepath), timeout=timeout)
         else:
             raise TargetError('Can\'t install {}: unsupported format.'.format(filepath))
 
@@ -758,9 +1019,9 @@ class AndroidTarget(Target):
         on_device_executable = self.path.join(self.executables_directory, executable_name)
         self.push(filepath, on_device_file)
         if on_device_file != on_device_executable:
-            self.execute('cp {} {}'.format(on_device_file, on_device_executable), as_root=self.is_rooted)
-            self.remove(on_device_file, as_root=self.is_rooted)
-        self.execute('chmod 0777 {}'.format(on_device_executable), as_root=self.is_rooted)
+            self.execute('cp {} {}'.format(on_device_file, on_device_executable), as_root=self.needs_su)
+            self.remove(on_device_file, as_root=self.needs_su)
+        self.execute("chmod 0777 '{}'".format(on_device_executable), as_root=self.needs_su)
         self._installed_binaries[executable_name] = on_device_executable
         return on_device_executable
 
@@ -770,10 +1031,10 @@ class AndroidTarget(Target):
     def uninstall_executable(self, executable_name):
         on_device_executable = self.path.join(self.executables_directory, executable_name)
         self._ensure_executables_directory_is_writable()
-        self.remove(on_device_executable, as_root=self.is_rooted)
+        self.remove(on_device_executable, as_root=self.needs_su)
 
     def dump_logcat(self, filepath, filter=None, append=False, timeout=30):  # pylint: disable=redefined-builtin
-        op = '>>' if append == True else '>'
+        op = '>>' if append else '>'
         filtstr = ' -s {}'.format(filter) if filter else ''
         command = 'logcat -d{} {} {}'.format(filtstr, op, filepath)
         adb_command(self.adb_name, command, timeout=timeout)
@@ -781,6 +1042,19 @@ class AndroidTarget(Target):
     def clear_logcat(self):
         adb_command(self.adb_name, 'logcat -c', timeout=30)
 
+    def adb_reboot_bootloader(self, timeout=30):
+        adb_command(self.adb_name, 'reboot-bootloader', timeout)
+
+    def adb_root(self, enable=True, force=False):
+        if enable:
+            if self._connected_as_root and not force:
+                return
+            adb_command(self.adb_name, 'root', timeout=30)
+            self._connected_as_root = True
+            return
+        adb_command(self.adb_name, 'unroot', timeout=30)
+        self._connected_as_root = False
+
     def is_screen_on(self):
         output = self.execute('dumpsys power')
         match = ANDROID_SCREEN_STATE_REGEX.search(output)
@@ -793,6 +1067,13 @@ class AndroidTarget(Target):
         if not self.is_screen_on():
             self.execute('input keyevent 26')
 
+    def _resolve_paths(self):
+        if self.working_directory is None:
+            self.working_directory = '/data/local/tmp/devlib-target'
+        self._file_transfer_cache = self.path.join(self.working_directory, '.file-cache')
+        if self.executables_directory is None:
+            self.executables_directory = '/data/local/tmp/bin'
+
     def _ensure_executables_directory_is_writable(self):
         matched = []
         for entry in self.list_file_systems():
@@ -808,9 +1089,35 @@ class AndroidTarget(Target):
             message = 'Could not find mount point for executables directory {}'
             raise TargetError(message.format(self.executables_directory))
 
+    _charging_enabled_path = '/sys/class/power_supply/battery/charging_enabled'
+
+    @property
+    def charging_enabled(self):
+        """
+        Whether drawing power to charge the battery is enabled
+
+        Not all devices have the ability to enable/disable battery charging
+        (e.g. because they don't have a battery). In that case,
+        ``charging_enabled`` is None.
+        """
+        if not self.file_exists(self._charging_enabled_path):
+            return None
+        return self.read_bool(self._charging_enabled_path)
+
+    @charging_enabled.setter
+    def charging_enabled(self, enabled):
+        """
+        Enable/disable drawing power to charge the battery
+
+        Not all devices have this facility. In that case, do nothing.
+        """
+        if not self.file_exists(self._charging_enabled_path):
+            return
+        self.write_value(self._charging_enabled_path, int(bool(enabled)))
 
 FstabEntry = namedtuple('FstabEntry', ['device', 'mount_point', 'fs_type', 'options', 'dump_freq', 'pass_num'])
 PsEntry = namedtuple('PsEntry', 'user pid ppid vsize rss wchan pc state name')
+LsmodEntry = namedtuple('LsmodEntry', ['name', 'size', 'use_count', 'used_by'])
 
 
 class Cpuinfo(object):
@@ -855,8 +1162,12 @@ class Cpuinfo(object):
                     continue
                 if 'Features' in section:
                     return section.get('Features').split()
+                elif 'flags' in section:
+                    return section.get('flags').split()
             elif 'Features' in section:
                 global_features = section.get('Features').split()
+            elif 'flags' in section:
+                global_features = section.get('flags').split()
         return global_features
 
     def parse(self, text):
@@ -880,7 +1191,33 @@ class Cpuinfo(object):
 
 
 class KernelVersion(object):
+    """
+    Class representing the version of a target kernel
 
+    Not expected to work for very old (pre-3.0) kernel version numbers.
+
+    :ivar release: Version number/revision string. Typical output of
+                   ``uname -r``
+    :type release: str
+    :ivar version: Extra version info (aside from ``release``) reported by
+                   ``uname``
+    :type version: str
+    :ivar version_number: Main version number (e.g. 3 for Linux 3.18)
+    :type version_number: int
+    :ivar major: Major version number (e.g. 18 for Linux 3.18)
+    :type major: int
+    :ivar minor: Minor version number for stable kernels (e.g. 9 for 4.9.9). May
+                 be None
+    :type minor: int
+    :ivar rc: Release candidate number (e.g. 3 for Linux 4.9-rc3). May be None.
+    :type rc: int
+    :ivar sha1: Kernel git revision hash, if available (otherwise None)
+    :type sha1: str
+
+    :ivar parts: Tuple of version number components. Can be used for
+                 lexicographically comparing kernel versions.
+    :type parts: tuple(int)
+    """
     def __init__(self, version_string):
         if ' #' in version_string:
             release, version = version_string.split(' #')
@@ -893,6 +1230,25 @@ class KernelVersion(object):
             self.release = version_string
             self.version = ''
 
+        self.version_number = None
+        self.major = None
+        self.minor = None
+        self.sha1 = None
+        self.rc = None
+        match = KVERSION_REGEX.match(version_string)
+        if match:
+            groups = match.groupdict()
+            self.version_number = int(groups['version'])
+            self.major = int(groups['major'])
+            if groups['minor'] is not None:
+                self.minor = int(groups['minor'])
+            if groups['rc'] is not None:
+                self.rc = int(groups['rc'])
+            if groups['sha1'] is not None:
+                self.sha1 = match.group('sha1')
+
+        self.parts = (self.version_number, self.major, self.minor)
+
     def __str__(self):
         return '{} {}'.format(self.release, self.version)
 
@@ -952,14 +1308,32 @@ class KernelConfig(object):
 
 class LocalLinuxTarget(LinuxTarget):
 
-    conn_cls = LocalConnection
+    def __init__(self,
+                 connection_settings=None,
+                 platform=None,
+                 working_directory=None,
+                 executables_directory=None,
+                 connect=True,
+                 modules=None,
+                 load_default_modules=True,
+                 shell_prompt=DEFAULT_SHELL_PROMPT,
+                 conn_cls=LocalConnection,
+                 ):
+        super(LocalLinuxTarget, self).__init__(connection_settings=connection_settings,
+                                               platform=platform,
+                                               working_directory=working_directory,
+                                               executables_directory=executables_directory,
+                                               connect=connect,
+                                               modules=modules,
+                                               load_default_modules=load_default_modules,
+                                               shell_prompt=shell_prompt,
+                                               conn_cls=conn_cls)
 
-    def connect(self, timeout=None):
+    def _resolve_paths(self):
         if self.working_directory is None:
             self.working_directory = '/tmp'
         if self.executables_directory is None:
             self.executables_directory = '/tmp'
-        super(LocalLinuxTarget, self).connect(timeout)
 
 
 def _get_model_name(section):
@@ -977,4 +1351,3 @@ def _get_part_name(section):
     if name is None:
         name = '{}/{}/{}'.format(implementer, part, variant)
     return name
-

devlib/trace/ftrace.py

@@ -15,7 +15,9 @@
 
 from __future__ import division
 import os
+import json
 import time
+import re
 import subprocess
 
 from devlib.trace import TraceCollector
@@ -27,6 +29,7 @@ from devlib.utils.misc import check_output, which
 TRACE_MARKER_START = 'TRACE_MARKER_START'
 TRACE_MARKER_STOP = 'TRACE_MARKER_STOP'
 OUTPUT_TRACE_FILE = 'trace.dat'
+OUTPUT_PROFILE_FILE = 'trace_stat.dat'
 DEFAULT_EVENTS = [
     'cpu_frequency',
     'cpu_idle',
@@ -40,26 +43,30 @@ DEFAULT_EVENTS = [
 ]
 TIMEOUT = 180
 
+# Regexps for parsing of function profiling data
+CPU_RE = re.compile(r'  Function \(CPU([0-9]+)\)')
+STATS_RE = re.compile(r'([^ ]*) +([0-9]+) +([0-9.]+) us +([0-9.]+) us +([0-9.]+) us')
 
 class FtraceCollector(TraceCollector):
 
     def __init__(self, target,
                  events=None,
+                 functions=None,
                  buffer_size=None,
                  buffer_size_step=1000,
-                 buffer_size_file='/sys/kernel/debug/tracing/buffer_size_kb',
-                 marker_file='/sys/kernel/debug/tracing/trace_marker',
+                 tracing_path='/sys/kernel/debug/tracing',
                  automark=True,
                  autoreport=True,
                  autoview=False,
                  no_install=False,
+                 strict=False,
                  ):
         super(FtraceCollector, self).__init__(target)
         self.events = events if events is not None else DEFAULT_EVENTS
+        self.functions = functions
         self.buffer_size = buffer_size
         self.buffer_size_step = buffer_size_step
-        self.buffer_size_file = buffer_size_file
-        self.marker_file = marker_file
+        self.tracing_path = tracing_path
         self.automark = automark
         self.autoreport = autoreport
         self.autoview = autoview
@@ -68,9 +75,19 @@ class FtraceCollector(TraceCollector):
         self.host_binary = None
         self.start_time = None
         self.stop_time = None
-        self.event_string = _build_trace_events(self.events)
+        self.event_string = None
+        self.function_string = None
         self._reset_needed = True
 
+        # Setup tracing paths
+        self.available_events_file    = self.target.path.join(self.tracing_path, 'available_events')
+        self.available_functions_file = self.target.path.join(self.tracing_path, 'available_filter_functions')
+        self.buffer_size_file         = self.target.path.join(self.tracing_path, 'buffer_size_kb')
+        self.current_tracer_file      = self.target.path.join(self.tracing_path, 'current_tracer')
+        self.function_profile_file    = self.target.path.join(self.tracing_path, 'function_profile_enabled')
+        self.marker_file              = self.target.path.join(self.tracing_path, 'trace_marker')
+        self.ftrace_filter_file       = self.target.path.join(self.tracing_path, 'set_ftrace_filter')
+
         self.host_binary = which('trace-cmd')
         self.kernelshark = which('kernelshark')
 
@@ -88,25 +105,99 @@ class FtraceCollector(TraceCollector):
                 raise TargetError('No trace-cmd found on device and no_install=True is specified.')
             self.target_binary = 'trace-cmd'
 
+        # Validate required events to be traced
+        available_events = self.target.execute(
+                'cat {}'.format(self.available_events_file),
+                as_root=True).splitlines()
+        selected_events = []
+        for event in self.events:
+            # Convert globs supported by FTrace into valid regexp globs
+            _event = event
+            if event[0] != '*':
+                _event = '*' + event
+            event_re = re.compile(_event.replace('*', '.*'))
+            # Select events matching the required ones
+            if len(filter(event_re.match, available_events)) == 0:
+                message = 'Event [{}] not available for tracing'.format(event)
+                if strict:
+                    raise TargetError(message)
+                self.target.logger.warning(message)
+            else:
+                selected_events.append(event)
+        # If function profiling is enabled we always need at least one event.
+        # Thus, if not other events have been specified, try to add at least
+        # a tracepoint which is always available and possibly triggered few
+        # times.
+        if self.functions and len(selected_events) == 0:
+            selected_events = ['sched_wakeup_new']
+        self.event_string = _build_trace_events(selected_events)
+
+        # Check for function tracing support
+        if self.functions:
+            if not self.target.file_exists(self.function_profile_file):
+                raise TargetError('Function profiling not supported. '\
+                        'A kernel build with CONFIG_FUNCTION_PROFILER enable is required')
+            # Validate required functions to be traced
+            available_functions = self.target.execute(
+                    'cat {}'.format(self.available_functions_file),
+                    as_root=True).splitlines()
+            selected_functions = []
+            for function in self.functions:
+                if function not in available_functions:
+                    message = 'Function [{}] not available for profiling'.format(function)
+                    if strict:
+                        raise TargetError(message)
+                    self.target.logger.warning(message)
+                else:
+                    selected_functions.append(function)
+            self.function_string = _build_trace_functions(selected_functions)
+
     def reset(self):
         if self.buffer_size:
             self._set_buffer_size()
-        self.target.execute('{} reset'.format(self.target_binary), as_root=True, timeout=TIMEOUT)
+        self.target.execute('{} reset'.format(self.target_binary),
+                            as_root=True, timeout=TIMEOUT)
         self._reset_needed = False
 
     def start(self):
         self.start_time = time.time()
         if self._reset_needed:
             self.reset()
+        self.target.execute('{} start {}'.format(self.target_binary, self.event_string),
+                            as_root=True)
         if self.automark:
             self.mark_start()
-        self.target.execute('{} start {}'.format(self.target_binary, self.event_string), as_root=True)
+        if 'cpufreq' in self.target.modules:
+            self.logger.debug('Trace CPUFreq frequencies')
+            self.target.cpufreq.trace_frequencies()
+        if 'cpuidle' in self.target.modules:
+            self.logger.debug('Trace CPUIdle states')
+            self.target.cpuidle.perturb_cpus()
+        # Enable kernel function profiling
+        if self.functions:
+            self.target.execute('echo nop > {}'.format(self.current_tracer_file),
+                                as_root=True)
+            self.target.execute('echo 0 > {}'.format(self.function_profile_file),
+                                as_root=True)
+            self.target.execute('echo {} > {}'.format(self.function_string, self.ftrace_filter_file),
+                                as_root=True)
+            self.target.execute('echo 1 > {}'.format(self.function_profile_file),
+                                as_root=True)
+
 
     def stop(self):
+        # Disable kernel function profiling
+        if self.functions:
+            self.target.execute('echo 1 > {}'.format(self.function_profile_file),
+                                as_root=True)
+        if 'cpufreq' in self.target.modules:
+            self.logger.debug('Trace CPUFreq frequencies')
+            self.target.cpufreq.trace_frequencies()
         self.stop_time = time.time()
         if self.automark:
             self.mark_stop()
-        self.target.execute('{} stop'.format(self.target_binary), timeout=TIMEOUT, as_root=True)
+        self.target.execute('{} stop'.format(self.target_binary),
+                            timeout=TIMEOUT, as_root=True)
         self._reset_needed = True
 
     def get_trace(self, outfile):
@@ -118,7 +209,7 @@ class FtraceCollector(TraceCollector):
         # The size of trace.dat will depend on how long trace-cmd was running.
         # Therefore timout for the pull command must also be adjusted
         # accordingly.
-        pull_timeout = self.stop_time - self.start_time
+        pull_timeout = 5 * (self.stop_time - self.start_time)
         self.target.pull(self.target_output_file, outfile, timeout=pull_timeout)
         if not os.path.isfile(outfile):
             self.logger.warning('Binary trace not pulled from device.')
@@ -129,6 +220,44 @@ class FtraceCollector(TraceCollector):
             if self.autoview:
                 self.view(outfile)
 
+    def get_stats(self, outfile):
+        if not self.functions:
+            return
+
+        if os.path.isdir(outfile):
+            outfile = os.path.join(outfile, OUTPUT_PROFILE_FILE)
+        output = self.target._execute_util('ftrace_get_function_stats',
+                                            as_root=True)
+
+        function_stats = {}
+        for line in output.splitlines():
+            # Match a new CPU dataset
+            match = CPU_RE.search(line)
+            if match:
+                cpu_id = int(match.group(1))
+                function_stats[cpu_id] = {}
+                self.logger.debug("Processing stats for CPU%d...", cpu_id)
+                continue
+            # Match a new function dataset
+            match = STATS_RE.search(line)
+            if match:
+                fname = match.group(1)
+                function_stats[cpu_id][fname] = {
+                        'hits' : int(match.group(2)),
+                        'time' : float(match.group(3)),
+                        'avg'  : float(match.group(4)),
+                        's_2'  : float(match.group(5)),
+                    }
+                self.logger.debug(" %s: %s",
+                             fname, function_stats[cpu_id][fname])
+
+        self.logger.debug("FTrace stats output [%s]...", outfile)
+        with open(outfile, 'w') as fh:
+           json.dump(function_stats, fh, indent=4)
+        self.logger.debug("FTrace function stats save in [%s]", outfile)
+
+        return function_stats
+
     def report(self, binfile, destfile):
         # To get the output of trace.dat, trace-cmd must be installed
         # This is done host-side because the generated file is very large
@@ -197,3 +326,6 @@ def _build_trace_events(events):
     event_string = ' '.join(['-e {}'.format(e) for e in events])
     return event_string
 
+def _build_trace_functions(functions):
+    function_string = " ".join(functions)
+    return function_string

devlib/utils/android.py

@@ -26,8 +26,8 @@ import logging
 import re
 from collections import defaultdict
 
-from devlib.exception import TargetError, HostError
-from devlib.utils.misc import check_output, which
+from devlib.exception import TargetError, HostError, DevlibError
+from devlib.utils.misc import check_output, which, memoized
 from devlib.utils.misc import escape_single_quotes, escape_double_quotes
 
 
@@ -39,6 +39,7 @@ AM_START_ERROR = re.compile(r"Error: Activity class {[\w|.|/]*} does not exist")
 # See:
 # http://developer.android.com/guide/topics/manifest/uses-sdk-element.html#ApiLevels
 ANDROID_VERSION_MAP = {
+    23: 'MARSHMALLOW',
     22: 'LOLLYPOP_MR1',
     21: 'LOLLYPOP',
     20: 'KITKAT_WATCH',
@@ -151,33 +152,81 @@ class AdbConnection(object):
     # maintains the count of parallel active connections to a device, so that
     # adb disconnect is not invoked untill all connections are closed
     active_connections = defaultdict(int)
+    default_timeout = 10
+    ls_command = 'ls'
 
     @property
     def name(self):
         return self.device
 
-    def __init__(self, device=None, timeout=10):
-        self.timeout = timeout
+    @property
+    @memoized
+    def newline_separator(self):
+        output = adb_command(self.device,
+                             "shell '({}); echo \"\n$?\"'".format(self.ls_command))
+        if output.endswith('\r\n'):
+            return '\r\n'
+        elif output.endswith('\n'):
+            return '\n'
+        else:
+            raise DevlibError("Unknown line ending")
+
+    # Again, we need to handle boards where the default output format from ls is
+    # single column *and* boards where the default output is multi-column.
+    # We need to do this purely because the '-1' option causes errors on older
+    # versions of the ls tool in Android pre-v7.
+    def _setup_ls(self):
+        command = "shell '(ls -1); echo \"\n$?\"'"
+        try:
+            output = adb_command(self.device, command, timeout=self.timeout)
+        except subprocess.CalledProcessError as e:
+            raise HostError(
+                'Failed to set up ls command on Android device. Output:\n'
+                + e.output)
+        lines = output.splitlines()
+        retval = lines[-1].strip()
+        if int(retval) == 0:
+            self.ls_command = 'ls -1'
+        else:
+            self.ls_command = 'ls'
+        logger.info("ls command is set to {}".format(self.ls_command))
+
+    def __init__(self, device=None, timeout=None, platform=None):
+        self.timeout = timeout if timeout is not None else self.default_timeout
         if device is None:
             device = adb_get_device(timeout=timeout)
         self.device = device
         adb_connect(self.device)
         AdbConnection.active_connections[self.device] += 1
+        self._setup_ls()
 
     def push(self, source, dest, timeout=None):
         if timeout is None:
             timeout = self.timeout
-        command = 'push {} {}'.format(source, dest)
+        command = "push '{}' '{}'".format(source, dest)
+        if not os.path.exists(source):
+            raise HostError('No such file "{}"'.format(source))
         return adb_command(self.device, command, timeout=timeout)
 
     def pull(self, source, dest, timeout=None):
         if timeout is None:
             timeout = self.timeout
-        command = 'pull {} {}'.format(source, dest)
+        # Pull all files matching a wildcard expression
+        if os.path.isdir(dest) and \
+           ('*' in source or '?' in source):
+            command = 'shell {} {}'.format(self.ls_command, source)
+            output = adb_command(self.device, command, timeout=timeout)
+            for line in output.splitlines():
+                command = "pull '{}' '{}'".format(line.strip(), dest)
+                adb_command(self.device, command, timeout=timeout)
+            return
+        command = "pull '{}' '{}'".format(source, dest)
         return adb_command(self.device, command, timeout=timeout)
 
-    def execute(self, command, timeout=None, check_exit_code=False, as_root=False):
-        return adb_shell(self.device, command, timeout, check_exit_code, as_root)
+    def execute(self, command, timeout=None, check_exit_code=False,
+                as_root=False, strip_colors=True):
+        return adb_shell(self.device, command, timeout, check_exit_code,
+                         as_root, self.newline_separator)
 
     def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
         return adb_background_shell(self.device, command, stdout, stderr, as_root)
@@ -195,9 +244,10 @@ class AdbConnection(object):
         pass
 
 
-def fastboot_command(command, timeout=None):
+def fastboot_command(command, timeout=None, device=None):
     _check_env()
-    full_command = "fastboot {}".format(command)
+    target = '-s {}'.format(device) if device else ''
+    full_command = 'fastboot {} {}'.format(target, command)
     logger.debug(full_command)
     output, _ = check_output(full_command, timeout, shell=True)
     return output
@@ -232,7 +282,7 @@ def adb_get_device(timeout=None):
             return output[1].split('\t')[0]
         elif output_length > 3:
             message = '{} Android devices found; either explicitly specify ' +\
-                    'the device you want, or make sure only one is connected.'
+                      'the device you want, or make sure only one is connected.'
             raise HostError(message.format(output_length - 2))
         else:
             if timeout < time.time() - start:
@@ -242,6 +292,10 @@ def adb_get_device(timeout=None):
 
 def adb_connect(device, timeout=None, attempts=MAX_ATTEMPTS):
     _check_env()
+    # Connect is required only for ADB-over-IP
+    if "." not in device:
+        logger.debug('Device connected via USB, connect not required')
+        return
     tries = 0
     output = None
     while tries <= attempts:
@@ -264,7 +318,7 @@ def adb_disconnect(device):
     _check_env()
     if not device:
         return
-    if ":" in device:
+    if ":" in device and device in adb_list_devices():
         command = "adb disconnect " + device
         logger.debug(command)
         retval = subprocess.call(command, stdout=open(os.devnull, 'wb'), shell=True)
@@ -284,33 +338,39 @@ def _ping(device):
         return False
 
 
-def adb_shell(device, command, timeout=None, check_exit_code=False, as_root=False):  # NOQA
+def adb_shell(device, command, timeout=None, check_exit_code=False,
+              as_root=False, newline_separator='\r\n'):  # NOQA
     _check_env()
     if as_root:
-        command = 'echo "{}" | su'.format(escape_double_quotes(command))
-    device_string = ' -s {}'.format(device) if device else ''
-    full_command = 'adb{} shell "{}"'.format(device_string,
-                                             escape_double_quotes(command))
-    logger.debug(full_command)
-    if check_exit_code:
-        actual_command = "adb{} shell '({}); echo $?'".format(device_string,
-                                                              escape_single_quotes(command))
-        raw_output, error = check_output(actual_command, timeout, shell=True)
-        if raw_output:
-            try:
-                output, exit_code, _ = raw_output.rsplit('\n', 2)
-            except ValueError:
-                exit_code, _ = raw_output.rsplit('\n', 1)
-                output = ''
-        else:  # raw_output is empty
-            exit_code = '969696'  # just because
-            output = ''
+        command = 'echo \'{}\' | su'.format(escape_single_quotes(command))
+    device_part = ['-s', device] if device else []
 
+    # On older combinations of ADB/Android versions, the adb host command always
+    # exits with 0 if it was able to run the command on the target, even if the
+    # command failed (https://code.google.com/p/android/issues/detail?id=3254).
+    # Homogenise this behaviour by running the command then echoing the exit
+    # code.
+    adb_shell_command = '({}); echo \"\n$?\"'.format(command)
+    actual_command = ['adb'] + device_part + ['shell', adb_shell_command]
+    logger.debug('adb {} shell {}'.format(' '.join(device_part), command))
+    raw_output, error = check_output(actual_command, timeout, shell=False)
+    if raw_output:
+        try:
+            output, exit_code, _ = raw_output.rsplit(newline_separator, 2)
+        except ValueError:
+            exit_code, _ = raw_output.rsplit(newline_separator, 1)
+            output = ''
+    else:  # raw_output is empty
+        exit_code = '969696'  # just because
+        output = ''
+
+    if check_exit_code:
         exit_code = exit_code.strip()
         if exit_code.isdigit():
             if int(exit_code):
-                message = 'Got exit code {}\nfrom: {}\nSTDOUT: {}\nSTDERR: {}'
-                raise TargetError(message.format(exit_code, full_command, output, error))
+                message = ('Got exit code {}\nfrom target command: {}\n'
+                           'STDOUT: {}\nSTDERR: {}')
+                raise TargetError(message.format(exit_code, command, output, error))
             elif AM_START_ERROR.findall(output):
                 message = 'Could not start activity; got the following:'
                 message += '\n{}'.format(AM_START_ERROR.findall(output)[0])
@@ -323,8 +383,7 @@ def adb_shell(device, command, timeout=None, check_exit_code=False, as_root=Fals
                 message = 'adb has returned early; did not get an exit code. '\
                           'Was kill-server invoked?'
                 raise TargetError(message)
-    else:  # do not check exit code
-        output, _ = check_output(full_command, timeout, shell=True)
+
     return output
 
 
@@ -377,19 +436,20 @@ def _initialize_with_android_home(env):
     logger.debug('Using ANDROID_HOME from the environment.')
     env.android_home = android_home
     env.platform_tools = os.path.join(android_home, 'platform-tools')
-    os.environ['PATH'] += os.pathsep + env.platform_tools
+    os.environ['PATH'] = env.platform_tools + os.pathsep + os.environ['PATH']
     _init_common(env)
     return env
 
 
 def _initialize_without_android_home(env):
-    if which('adb'):
+    adb_full_path = which('adb')
+    if adb_full_path:
         env.adb = 'adb'
     else:
         raise HostError('ANDROID_HOME is not set and adb is not in PATH. '
                         'Have you installed Android SDK?')
     logger.debug('Discovering ANDROID_HOME from adb path.')
-    env.platform_tools = os.path.dirname(env.adb)
+    env.platform_tools = os.path.dirname(adb_full_path)
     env.android_home = os.path.dirname(env.platform_tools)
     _init_common(env)
     return env

devlib/utils/misc.py

@@ -29,10 +29,15 @@ import subprocess
 import pkgutil
 import logging
 import random
+import ctypes
 from operator import itemgetter
 from itertools import groupby
 from functools import partial
 
+import wrapt
+
+from devlib.exception import HostError, TimeoutError
+
 
 # ABI --> architectures list
 ABI_MAP = {
@@ -55,17 +60,24 @@ CPU_PART_MAP = {
         0xc07: {None: 'A7'},
         0xc08: {None: 'A8'},
         0xc09: {None: 'A9'},
+        0xc0e: {None: 'A17'},
         0xc0f: {None: 'A15'},
         0xc14: {None: 'R4'},
         0xc15: {None: 'R5'},
+        0xc17: {None: 'R7'},
+        0xc18: {None: 'R8'},
         0xc20: {None: 'M0'},
+        0xc60: {None: 'M0+'},
         0xc21: {None: 'M1'},
         0xc23: {None: 'M3'},
         0xc24: {None: 'M4'},
         0xc27: {None: 'M7'},
+        0xd01: {None: 'A32'},
         0xd03: {None: 'A53'},
+        0xd04: {None: 'A35'},
         0xd07: {None: 'A57'},
         0xd08: {None: 'A72'},
+        0xd09: {None: 'A73'},
     },
     0x4e: {  # Nvidia
         0x0: {None: 'Denver'},
@@ -77,6 +89,8 @@ CPU_PART_MAP = {
             0x2: 'Krait400',
             0x3: 'Krait450',
         },
+        0x205: {0x1: 'KryoSilver'},
+        0x211: {0x1: 'KryoGold'},
     },
     0x56: {  # Marvell
         0x131: {
@@ -109,22 +123,6 @@ def preexec_function():
 check_output_logger = logging.getLogger('check_output')
 
 
-# Defined here rather than in devlib.exceptions due to module load dependencies
-class TimeoutError(Exception):
-    """Raised when a subprocess command times out. This is basically a ``WAError``-derived version
-    of ``subprocess.CalledProcessError``, the thinking being that while a timeout could be due to
-    programming error (e.g. not setting long enough timers), it is often due to some failure in the
-    environment, and there fore should be classed as a "user error"."""
-
-    def __init__(self, command, output):
-        super(TimeoutError, self).__init__('Timed out: {}'.format(command))
-        self.command = command
-        self.output = output
-
-    def __str__(self):
-        return '\n'.join([self.message, 'OUTPUT:', self.output or ''])
-
-
 def check_output(command, timeout=None, ignore=None, inputtext=None, **kwargs):
     """This is a version of subprocess.check_output that adds a timeout parameter to kill
     the subprocess if it does not return within the specified time."""
@@ -174,15 +172,35 @@ def walk_modules(path):
     Given package name, return a list of all modules (including submodules, etc)
     in that package.
 
+    :raises HostError: if an exception is raised while trying to import one of the
+                       modules under ``path``. The exception will have addtional
+                       attributes set: ``module`` will be set to the qualified name
+                       of the originating module, and ``orig_exc`` will contain
+                       the original exception.
+
     """
-    root_mod = __import__(path, {}, {}, [''])
+
+    def __try_import(path):
+        try:
+            return __import__(path, {}, {}, [''])
+        except Exception as e:
+            he = HostError('Could not load {}: {}'.format(path, str(e)))
+            he.module = path
+            he.exc_info = sys.exc_info()
+            he.orig_exc = e
+            raise he
+
+    root_mod = __try_import(path)
     mods = [root_mod]
+    if not hasattr(root_mod, '__path__'):
+        # root is a module not a package -- nothing to walk
+        return mods
     for _, name, ispkg in pkgutil.iter_modules(root_mod.__path__):
         submod_path = '.'.join([path, name])
         if ispkg:
             mods.extend(walk_modules(submod_path))
         else:
-            submod = __import__(submod_path, {}, {}, [''])
+            submod = __try_import(submod_path)
             mods.append(submod)
     return mods
 
@@ -536,17 +554,48 @@ def mask_to_list(mask):
 __memo_cache = {}
 
 
-def memoized(func):
+def reset_memo_cache():
+    __memo_cache.clear()
+
+
+def __get_memo_id(obj):
+    """
+    An object's id() may be re-used after an object is freed, so it's not
+    sufficiently unique to identify params for the memo cache (two different
+    params may end up with the same id). this attempts to generate a more unique
+    ID string.
+    """
+    obj_id = id(obj)
+    try:
+        return '{}/{}'.format(obj_id, hash(obj))
+    except TypeError:  # obj is not hashable
+        obj_pyobj = ctypes.cast(obj_id, ctypes.py_object)
+        # TODO: Note: there is still a possibility of a clash here. If Two
+        # different objects get assigned the same ID, an are large and are
+        # identical in the first thirty two bytes. This shouldn't be much of an
+        # issue in the current application of memoizing Target calls, as it's very
+        # unlikely that a target will get passed large params; but may cause
+        # problems in other applications, e.g. when memoizing results of operations
+        # on large arrays. I can't really think of a good way around that apart
+        # form, e.g., md5 hashing the entire raw object, which will have an
+        # undesirable impact on performance.
+        num_bytes = min(ctypes.sizeof(obj_pyobj), 32)
+        obj_bytes = ctypes.string_at(ctypes.addressof(obj_pyobj), num_bytes)
+        return '{}/{}'.format(obj_id, obj_bytes)
+
+
+@wrapt.decorator
+def memoized(wrapped, instance, args, kwargs):
     """A decorator for memoizing functions and methods."""
-    func_id = repr(func)
+    func_id = repr(wrapped)
 
     def memoize_wrapper(*args, **kwargs):
-        id_string = func_id + ','.join([str(id(a)) for a in  args])
+        id_string = func_id + ','.join([__get_memo_id(a) for a in  args])
         id_string += ','.join('{}={}'.format(k, v)
                               for k, v in kwargs.iteritems())
         if id_string not in __memo_cache:
-            __memo_cache[id_string] = func(*args, **kwargs)
+            __memo_cache[id_string] = wrapped(*args, **kwargs)
         return __memo_cache[id_string]
 
-    return memoize_wrapper
+    return memoize_wrapper(*args, **kwargs)
 

devlib/utils/ssh.py

@@ -22,6 +22,8 @@ import re
 import threading
 import tempfile
 import shutil
+import socket
+import time
 
 import pexpect
 from distutils.version import StrictVersion as V
@@ -33,30 +35,41 @@ from pexpect import EOF, TIMEOUT, spawn
 
 from devlib.exception import HostError, TargetError, TimeoutError
 from devlib.utils.misc import which, strip_bash_colors, escape_single_quotes, check_output
+from devlib.utils.types import boolean
 
 
 ssh = None
 scp = None
 sshpass = None
 
+
 logger = logging.getLogger('ssh')
+gem5_logger = logging.getLogger('gem5-connection')
 
-
-def ssh_get_shell(host, username, password=None, keyfile=None, port=None, timeout=10, telnet=False):
+def ssh_get_shell(host, username, password=None, keyfile=None, port=None, timeout=10, telnet=False, original_prompt=None):
     _check_env()
-    if telnet:
-        if keyfile:
-            raise ValueError('keyfile may not be used with a telnet connection.')
-        conn = TelnetConnection()
-    else:  # ssh
-        conn = pxssh.pxssh()
-    try:
-        if keyfile:
-            conn.login(host, username, ssh_key=keyfile, port=port, login_timeout=timeout)
-        else:
-            conn.login(host, username, password, port=port, login_timeout=timeout)
-    except EOF:
-        raise TargetError('Could not connect to {}; is the host name correct?'.format(host))
+    start_time = time.time()
+    while True:
+        if telnet:
+            if keyfile:
+                raise ValueError('keyfile may not be used with a telnet connection.')
+            conn = TelnetPxssh(original_prompt=original_prompt)
+        else:  # ssh
+            conn = pxssh.pxssh()
+
+        try:
+            if keyfile:
+                conn.login(host, username, ssh_key=keyfile, port=port, login_timeout=timeout)
+            else:
+                conn.login(host, username, password, port=port, login_timeout=timeout)
+            break
+        except EOF:
+            timeout -= time.time() - start_time
+            if timeout <= 0:
+                message = 'Could not connect to {}; is the host name correct?'
+                raise TargetError(message.format(host))
+            time.sleep(5)
+
     conn.setwinsize(500,200)
     conn.sendline('')
     conn.prompt()
@@ -64,23 +77,37 @@ def ssh_get_shell(host, username, password=None, keyfile=None, port=None, timeou
     return conn
 
 
-class TelnetConnection(pxssh.pxssh):
+class TelnetPxssh(pxssh.pxssh):
     # pylint: disable=arguments-differ
 
-    def login(self, server, username, password='', original_prompt=r'[#$]', login_timeout=10,
-              auto_prompt_reset=True, sync_multiplier=1):
-        cmd = 'telnet -l {} {}'.format(username, server)
+    def __init__(self, original_prompt):
+        super(TelnetPxssh, self).__init__()
+        self.original_prompt = original_prompt or r'[#$]'
+
+    def login(self, server, username, password='', login_timeout=10,
+              auto_prompt_reset=True, sync_multiplier=1, port=23):
+        args = ['telnet']
+        if username is not None:
+            args += ['-l', username]
+        args += [server, str(port)]
+        cmd = ' '.join(args)
 
         spawn._spawn(self, cmd)  # pylint: disable=protected-access
-        i = self.expect('(?i)(?:password)', timeout=login_timeout)
-        if i == 0:
-            self.sendline(password)
-            i = self.expect([original_prompt, 'Login incorrect'], timeout=login_timeout)
-        else:
-            raise pxssh.ExceptionPxssh('could not log in: did not see a password prompt')
 
-        if i:
-            raise pxssh.ExceptionPxssh('could not log in: password was incorrect')
+        try:
+            i = self.expect('(?i)(?:password)', timeout=login_timeout)
+            if i == 0:
+                self.sendline(password)
+                i = self.expect([self.original_prompt, 'Login incorrect'], timeout=login_timeout)
+            if i:
+                raise pxssh.ExceptionPxssh('could not log in: password was incorrect')
+        except TIMEOUT:
+            if not password:
+                # No password promt before TIMEOUT & no password provided
+                # so assume everything is okay
+                pass
+            else:
+                raise pxssh.ExceptionPxssh('could not log in: did not see a password prompt')
 
         if not self.sync_original_prompt(sync_multiplier):
             self.close()
@@ -117,6 +144,7 @@ class SshConnection(object):
 
     default_password_prompt = '[sudo] password'
     max_cancel_attempts = 5
+    default_timeout=10
 
     @property
     def name(self):
@@ -128,9 +156,11 @@ class SshConnection(object):
                  password=None,
                  keyfile=None,
                  port=None,
-                 timeout=10,
+                 timeout=None,
                  telnet=False,
                  password_prompt=None,
+                 original_prompt=None,
+                 platform=None
                  ):
         self.host = host
         self.username = username
@@ -140,7 +170,8 @@ class SshConnection(object):
         self.lock = threading.Lock()
         self.password_prompt = password_prompt if password_prompt is not None else self.default_password_prompt
         logger.debug('Logging in {}@{}'.format(username, host))
-        self.conn = ssh_get_shell(host, username, password, self.keyfile, port, timeout, telnet)
+        timeout = timeout if timeout is not None else self.default_timeout
+        self.conn = ssh_get_shell(host, username, password, self.keyfile, port, timeout, False, None)
 
     def push(self, source, dest, timeout=30):
         dest = '{}@{}:{}'.format(self.username, self.host, dest)
@@ -150,28 +181,45 @@ class SshConnection(object):
         source = '{}@{}:{}'.format(self.username, self.host, source)
         return self._scp(source, dest, timeout)
 
-    def execute(self, command, timeout=None, check_exit_code=True, as_root=False, strip_colors=True):
-        with self.lock:
-            output = self._execute_and_wait_for_prompt(command, timeout, as_root, strip_colors)
-            if check_exit_code:
-                exit_code_text = self._execute_and_wait_for_prompt('echo $?', strip_colors=strip_colors, log=False)
-                try:
-                    exit_code = int(exit_code_text.split()[0])
-                    if exit_code:
-                        message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}'
-                        raise TargetError(message.format(exit_code, command, output))
-                except (ValueError, IndexError):
-                    logger.warning('Could not get exit code for "{}",\ngot: "{}"'.format(command, exit_code_text))
-            return output
+    def execute(self, command, timeout=None, check_exit_code=True,
+                as_root=False, strip_colors=True): #pylint: disable=unused-argument
+        if command == '':
+            # Empty command is valid but the __devlib_ec stuff below will
+            # produce a syntax error with bash. Treat as a special case.
+            return ''
+        try:
+            with self.lock:
+                _command = '({}); __devlib_ec=$?; echo; echo $__devlib_ec'.format(command)
+                raw_output = self._execute_and_wait_for_prompt(
+                    _command, timeout, as_root, strip_colors)
+                output, exit_code_text, _ = raw_output.rsplit('\r\n', 2)
+                if check_exit_code:
+                    try:
+                        exit_code = int(exit_code_text)
+                        if exit_code:
+                            message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}'
+                            raise TargetError(message.format(exit_code, command, output))
+                    except (ValueError, IndexError):
+                        logger.warning(
+                            'Could not get exit code for "{}",\ngot: "{}"'\
+                            .format(command, exit_code_text))
+                return output
+        except EOF:
+            raise TargetError('Connection lost.')
 
-    def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE):
-        port_string = '-p {}'.format(self.port) if self.port else ''
-        keyfile_string = '-i {}'.format(self.keyfile) if self.keyfile else ''
-        command = '{} {} {} {}@{} {}'.format(ssh, keyfile_string, port_string, self.username, self.host, command)
-        logger.debug(command)
-        if self.password:
-            command = _give_password(self.password, command)
-        return subprocess.Popen(command, stdout=stdout, stderr=stderr, shell=True)
+    def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False):
+        try:
+            port_string = '-p {}'.format(self.port) if self.port else ''
+            keyfile_string = '-i {}'.format(self.keyfile) if self.keyfile else ''
+            if as_root:
+                command = "sudo -- sh -c '{}'".format(command)
+            command = '{} {} {} {}@{} {}'.format(ssh, keyfile_string, port_string, self.username, self.host, command)
+            logger.debug(command)
+            if self.password:
+                command = _give_password(self.password, command)
+            return subprocess.Popen(command, stdout=stdout, stderr=stderr, shell=True)
+        except EOF:
+            raise TargetError('Connection lost.')
 
     def close(self):
         logger.debug('Logging out {}@{}'.format(self.username, self.host))
@@ -188,6 +236,9 @@ class SshConnection(object):
 
     def _execute_and_wait_for_prompt(self, command, timeout=None, as_root=False, strip_colors=True, log=True):
         self.conn.prompt(0.1)  # clear an existing prompt if there is one.
+        if self.username == 'root':
+            # As we're already root, there is no need to use sudo.
+            as_root = False
         if as_root:
             command = "sudo -- sh -c '{}'".format(escape_single_quotes(command))
             if log:
@@ -243,6 +294,526 @@ class SshConnection(object):
             raise TimeoutError(e.command.replace(pass_string, ''), e.output)
 
 
+class TelnetConnection(SshConnection):
+
+    def __init__(self,
+                 host,
+                 username,
+                 password=None,
+                 port=None,
+                 timeout=None,
+                 password_prompt=None,
+                 original_prompt=None,
+                 platform=None):
+        self.host = host
+        self.username = username
+        self.password = password
+        self.port = port
+        self.keyfile = None
+        self.lock = threading.Lock()
+        self.password_prompt = password_prompt if password_prompt is not None else self.default_password_prompt
+        logger.debug('Logging in {}@{}'.format(username, host))
+        timeout = timeout if timeout is not None else self.default_timeout
+        self.conn = ssh_get_shell(host, username, password, None, port, timeout, True, original_prompt)
+
+
+class Gem5Connection(TelnetConnection):
+
+    def __init__(self,
+                 platform,
+                 host=None,
+                 username=None,
+                 password=None,
+                 port=None,
+                 timeout=None,
+                 password_prompt=None,
+                 original_prompt=None,
+                 ):
+        if host is not None:
+            host_system = socket.gethostname()
+            if host_system != host:
+                raise TargetError("Gem5Connection can only connect to gem5 "
+                                   "simulations on your current host, which "
+                                   "differs from the one given {}!"
+                                   .format(host_system, host))
+        if username is not None and username != 'root':
+            raise ValueError('User should be root in gem5!')
+        if password is not None and password != '':
+            raise ValueError('No password needed in gem5!')
+        self.username = 'root'
+        self.is_rooted = True
+        self.password = None
+        self.port = None
+        # Long timeouts to account for gem5 being slow
+        # Can be overriden if the given timeout is longer
+        self.default_timeout = 3600
+        if timeout is not None:
+            if timeout > self.default_timeout:
+                logger.info('Overwriting the default timeout of gem5 ({})'
+                                 ' to {}'.format(self.default_timeout, timeout))
+                self.default_timeout = timeout
+            else:
+                logger.info('Ignoring the given timeout --> gem5 needs longer timeouts')
+        self.ready_timeout = self.default_timeout * 3
+        # Counterpart in gem5_interact_dir
+        self.gem5_input_dir = '/mnt/host/'
+        # Location of m5 binary in the gem5 simulated system
+        self.m5_path = None
+        # Actual telnet connection to gem5 simulation
+        self.conn = None
+        # Flag to indicate the gem5 device is ready to interact with the
+        # outer world
+        self.ready = False
+        # Lock file to prevent multiple connections to same gem5 simulation
+        # (gem5 does not allow this)
+        self.lock_directory = '/tmp/'
+        self.lock_file_name = None # Will be set once connected to gem5
+
+        # These parameters will be set by either the method to connect to the
+        # gem5 platform or directly to the gem5 simulation
+        # Intermediate directory to push things to gem5 using VirtIO
+        self.gem5_interact_dir = None
+        # Directory to store output  from gem5 on the host
+        self.gem5_out_dir = None
+        # Actual gem5 simulation
+        self.gem5simulation = None
+
+        # Connect to gem5
+        if platform:
+            self._connect_gem5_platform(platform)
+
+        # Wait for boot
+        self._wait_for_boot()
+
+        # Mount the virtIO to transfer files in/out gem5 system
+        self._mount_virtio()
+
+    def set_hostinteractdir(self, indir):
+        logger.info('Setting hostinteractdir  from {} to {}'
+                    .format(self.gem5_input_dir, indir))
+        self.gem5_input_dir = indir
+
+    def push(self, source, dest, timeout=None):
+        """
+        Push a file to the gem5 device using VirtIO
+
+        The file to push to the device is copied to the temporary directory on
+        the host, before being copied within the simulation to the destination.
+        Checks, in the form of 'ls' with error code checking, are performed to
+        ensure that the file is copied to the destination.
+        """
+        # First check if the connection is set up to interact with gem5
+        self._check_ready()
+
+        filename = os.path.basename(source)
+        logger.debug("Pushing {} to device.".format(source))
+        logger.debug("gem5interactdir: {}".format(self.gem5_interact_dir))
+        logger.debug("dest: {}".format(dest))
+        logger.debug("filename: {}".format(filename))
+
+        # We need to copy the file to copy to the temporary directory
+        self._move_to_temp_dir(source)
+
+        # Dest in gem5 world is a file rather than directory
+        if os.path.basename(dest) != filename:
+            dest = os.path.join(dest, filename)
+        # Back to the gem5 world
+        self._gem5_shell("ls -al {}{}".format(self.gem5_input_dir, filename))
+        self._gem5_shell("cat '{}''{}' > '{}'".format(self.gem5_input_dir,
+                                                     filename,
+                                                     dest))
+        self._gem5_shell("sync")
+        self._gem5_shell("ls -al {}".format(dest))
+        self._gem5_shell("ls -al {}".format(self.gem5_input_dir))
+        logger.debug("Push complete.")
+
+    def pull(self, source, dest, timeout=0): #pylint: disable=unused-argument
+        """
+        Pull a file from the gem5 device using m5 writefile
+
+        The file is copied to the local directory within the guest as the m5
+        writefile command assumes that the file is local. The file is then
+        written out to the host system using writefile, prior to being moved to
+        the destination on the host.
+        """
+        # First check if the connection is set up to interact with gem5
+        self._check_ready()
+
+        filename = os.path.basename(source)
+
+        logger.debug("pull_file {} {}".format(source, filename))
+        # We don't check the exit code here because it is non-zero if the source
+        # and destination are the same. The ls below will cause an error if the
+        # file was not where we expected it to be.
+        if os.path.dirname(source) != os.getcwd():
+            self._gem5_shell("cat '{}' > '{}'".format(source, filename))
+        self._gem5_shell("sync")
+        self._gem5_shell("ls -la {}".format(filename))
+        logger.debug('Finished the copy in the simulator')
+        self._gem5_util("writefile {}".format(filename))
+
+        if 'cpu' not in filename:
+            while not os.path.exists(os.path.join(self.gem5_out_dir, filename)):
+                time.sleep(1)
+
+        # Perform the local move
+        shutil.move(os.path.join(self.gem5_out_dir, filename), dest)
+        logger.debug("Pull complete.")
+
+    def execute(self, command, timeout=1000, check_exit_code=True,
+                as_root=False, strip_colors=True):
+        """
+        Execute a command on the gem5 platform
+        """
+        # First check if the connection is set up to interact with gem5
+        self._check_ready()
+
+        output = self._gem5_shell(command, as_root=as_root)
+        if strip_colors:
+            output = strip_bash_colors(output)
+        return output
+
+    def background(self, command, stdout=subprocess.PIPE,
+                   stderr=subprocess.PIPE, as_root=False):
+        # First check if the connection is set up to interact with gem5
+        self._check_ready()
+
+        # Create the logfile for stderr/stdout redirection
+        command_name = command.split(' ')[0].split('/')[-1]
+        redirection_file = 'BACKGROUND_{}.log'.format(command_name)
+        trial = 0
+        while os.path.isfile(redirection_file):
+            # Log file already exists so add to name
+           redirection_file = 'BACKGROUND_{}{}.log'.format(command_name, trial)
+           trial += 1
+
+        # Create the command to pass on to gem5 shell
+        complete_command = '{} >> {} 2>&1 &'.format(command, redirection_file)
+        output = self._gem5_shell(complete_command, as_root=as_root)
+        output = strip_bash_colors(output)
+        gem5_logger.info('STDERR/STDOUT of background command will be '
+                         'redirected to {}. Use target.pull() to '
+                         'get this file'.format(redirection_file))
+        return output
+
+    def close(self):
+        """
+        Close and disconnect from the gem5 simulation. Additionally, we remove
+        the temporary directory used to pass files into the simulation.
+        """
+        gem5_logger.info("Gracefully terminating the gem5 simulation.")
+        try:
+            self._gem5_util("exit")
+            self.gem5simulation.wait()
+        except EOF:
+            pass
+        gem5_logger.info("Removing the temporary directory")
+        try:
+            shutil.rmtree(self.gem5_interact_dir)
+        except OSError:
+            gem5_logger.warn("Failed to remove the temporary directory!")
+
+        # Delete the lock file
+        os.remove(self.lock_file_name)
+
+    # Functions only to be called by the Gem5 connection itself
+    def _connect_gem5_platform(self, platform):
+        port = platform.gem5_port
+        gem5_simulation = platform.gem5
+        gem5_interact_dir = platform.gem5_interact_dir
+        gem5_out_dir = platform.gem5_out_dir
+
+        self.connect_gem5(port, gem5_simulation, gem5_interact_dir, gem5_out_dir)
+
+    # This function connects to the gem5 simulation
+    def connect_gem5(self, port, gem5_simulation, gem5_interact_dir,
+                      gem5_out_dir):
+        """
+        Connect to the telnet port of the gem5 simulation.
+
+        We connect, and wait for the prompt to be found. We do not use a timeout
+        for this, and wait for the prompt in a while loop as the gem5 simulation
+        can take many hours to reach a prompt when booting the system. We also
+        inject some newlines periodically to try and force gem5 to show a
+        prompt. Once the prompt has been found, we replace it with a unique
+        prompt to ensure that we are able to match it properly. We also disable
+        the echo as this simplifies parsing the output when executing commands
+        on the device.
+        """
+        host = socket.gethostname()
+        gem5_logger.info("Connecting to the gem5 simulation on port {}".format(port))
+
+        # Check if there is no on-going connection yet
+        lock_file_name = '{}{}_{}.LOCK'.format(self.lock_directory, host, port)
+        if os.path.isfile(lock_file_name):
+            # There is already a connection to this gem5 simulation
+            raise TargetError('There is already a connection to the gem5 '
+                              'simulation using port {} on {}!'
+                              .format(port, host))
+
+        # Connect to the gem5 telnet port. Use a short timeout here.
+        attempts = 0
+        while attempts < 10:
+            attempts += 1
+            try:
+                self.conn = TelnetPxssh(original_prompt=None)
+                self.conn.login(host, self.username, port=port,
+                                login_timeout=10, auto_prompt_reset=False)
+                break
+            except pxssh.ExceptionPxssh:
+                pass
+        else:
+            gem5_simulation.kill()
+            raise TargetError("Failed to connect to the gem5 telnet session.")
+
+        gem5_logger.info("Connected! Waiting for prompt...")
+
+        # Create the lock file
+        self.lock_file_name = lock_file_name
+        open(self.lock_file_name, 'w').close() # Similar to touch
+        gem5_logger.info("Created lock file {} to prevent reconnecting to "
+                         "same simulation".format(self.lock_file_name))
+
+        # We need to find the prompt. It might be different if we are resuming
+        # from a checkpoint. Therefore, we test multiple options here.
+        prompt_found = False
+        while not prompt_found:
+            try:
+                self._login_to_device()
+            except TIMEOUT:
+                pass
+            try:
+                # Try and force a prompt to be shown
+                self.conn.send('\n')
+                self.conn.expect([r'# ', self.conn.UNIQUE_PROMPT, r'\[PEXPECT\][\\\$\#]+ '], timeout=60)
+                prompt_found = True
+            except TIMEOUT:
+                pass
+
+        gem5_logger.info("Successfully logged in")
+        gem5_logger.info("Setting unique prompt...")
+
+        self.conn.set_unique_prompt()
+        self.conn.prompt()
+        gem5_logger.info("Prompt found and replaced with a unique string")
+
+        # We check that the prompt is what we think it should be. If not, we
+        # need to update the regex we use to match.
+        self._find_prompt()
+
+        self.conn.setecho(False)
+        self._sync_gem5_shell()
+
+        # Fully connected to gem5 simulation
+        self.gem5_interact_dir = gem5_interact_dir
+        self.gem5_out_dir = gem5_out_dir
+        self.gem5simulation = gem5_simulation
+
+        # Ready for interaction now
+        self.ready = True
+
+    def _login_to_device(self):
+        """
+        Login to device, will be overwritten if there is an actual login
+        """
+        pass
+
+    def _find_prompt(self):
+        prompt = r'\[PEXPECT\][\\\$\#]+ '
+        synced = False
+        while not synced:
+            self.conn.send('\n')
+            i = self.conn.expect([prompt, self.conn.UNIQUE_PROMPT, r'[\$\#] '], timeout=self.default_timeout)
+            if i == 0:
+                synced = True
+            elif i == 1:
+                prompt = self.conn.UNIQUE_PROMPT
+                synced = True
+            else:
+                prompt = re.sub(r'\$', r'\\\$', self.conn.before.strip() + self.conn.after.strip())
+                prompt = re.sub(r'\#', r'\\\#', prompt)
+                prompt = re.sub(r'\[', r'\[', prompt)
+                prompt = re.sub(r'\]', r'\]', prompt)
+
+        self.conn.PROMPT = prompt
+
+    def _sync_gem5_shell(self):
+        """
+        Synchronise with the gem5 shell.
+
+        Write some unique text to the gem5 device to allow us to synchronise
+        with the shell output. We actually get two prompts so we need to match
+        both of these.
+        """
+        gem5_logger.debug("Sending Sync")
+        self.conn.send("echo \*\*sync\*\*\n")
+        self.conn.expect(r"\*\*sync\*\*", timeout=self.default_timeout)
+        self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)
+        self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)
+
+    def _gem5_util(self, command):
+        """ Execute a gem5 utility command using the m5 binary on the device """
+        if self.m5_path is None:
+            raise TargetError('Path to m5 binary on simulated system  is not set!')
+        self._gem5_shell('{} {}'.format(self.m5_path, command))
+
+    def _gem5_shell(self, command, as_root=False, timeout=None, check_exit_code=True, sync=True):  # pylint: disable=R0912
+        """
+        Execute a command in the gem5 shell
+
+        This wraps the telnet connection to gem5 and processes the raw output.
+
+        This method waits for the shell to return, and then will try and
+        separate the output from the command from the command itself. If this
+        fails, warn, but continue with the potentially wrong output.
+
+        The exit code is also checked by default, and non-zero exit codes will
+        raise a TargetError.
+        """
+        if sync:
+            self._sync_gem5_shell()
+
+        gem5_logger.debug("gem5_shell command: {}".format(command))
+
+        # Send the actual command
+        self.conn.send("{}\n".format(command))
+
+        # Wait for the response. We just sit here and wait for the prompt to
+        # appear, as gem5 might take a long time to provide the output. This
+        # avoids timeout issues.
+        command_index = -1
+        while command_index == -1:
+            if self.conn.prompt():
+                output = re.sub(r' \r([^\n])', r'\1', self.conn.before)
+                output = re.sub(r'[\b]', r'', output)
+                # Deal with line wrapping
+                output = re.sub(r'[\r].+?<', r'', output)
+                command_index = output.find(command)
+
+                # If we have -1, then we cannot match the command, but the
+                # prompt has returned. Hence, we have a bit of an issue. We
+                # warn, and return the whole output.
+                if command_index == -1:
+                    gem5_logger.warn("gem5_shell: Unable to match command in "
+                                     "command output. Expect parsing errors!")
+                    command_index = 0
+
+        output = output[command_index + len(command):].strip()
+
+        # It is possible that gem5 will echo the command. Therefore, we need to
+        # remove that too!
+        command_index = output.find(command)
+        if command_index != -1:
+            output = output[command_index + len(command):].strip()
+
+        gem5_logger.debug("gem5_shell output: {}".format(output))
+
+        # We get a second prompt. Hence, we need to eat one to make sure that we
+        # stay in sync. If we do not do this, we risk getting out of sync for
+        # slower simulations.
+        self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)
+
+        if check_exit_code:
+            exit_code_text = self._gem5_shell('echo $?', as_root=as_root,
+                                             timeout=timeout, check_exit_code=False,
+                                             sync=False)
+            try:
+                exit_code = int(exit_code_text.split()[0])
+                if exit_code:
+                    message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}'
+                    raise TargetError(message.format(exit_code, command, output))
+            except (ValueError, IndexError):
+                gem5_logger.warning('Could not get exit code for "{}",\ngot: "{}"'.format(command, exit_code_text))
+
+        return output
+
+    def _mount_virtio(self):
+        """
+        Mount the VirtIO device in the simulated system.
+        """
+        gem5_logger.info("Mounting VirtIO device in simulated system")
+
+        self._gem5_shell('su -c "mkdir -p {}" root'.format(self.gem5_input_dir))
+        mount_command = "mount -t 9p -o trans=virtio,version=9p2000.L,aname={} gem5 {}".format(self.gem5_interact_dir, self.gem5_input_dir)
+        self._gem5_shell(mount_command)
+
+    def _move_to_temp_dir(self, source):
+        """
+        Move a file to the temporary directory on the host for copying to the
+        gem5 device
+        """
+        command = "cp {} {}".format(source, self.gem5_interact_dir)
+        gem5_logger.debug("Local copy command: {}".format(command))
+        subprocess.call(command.split())
+        subprocess.call("sync".split())
+
+    def _check_ready(self):
+        """
+        Check if the gem5 platform is ready
+        """
+        if not self.ready:
+            raise TargetError('Gem5 is not ready to interact yet')
+
+    def _wait_for_boot(self):
+        pass
+
+    def _probe_file(self, filepath):
+        """
+        Internal method to check if the target has a certain file
+        """
+        command = 'if [ -e \'{}\' ]; then echo 1; else echo 0; fi'
+        output = self.execute(command.format(filepath), as_root=self.is_rooted)
+        return boolean(output.strip())
+
+
+class LinuxGem5Connection(Gem5Connection):
+
+    def _login_to_device(self):
+        gem5_logger.info("Trying to log in to gem5 device")
+        login_prompt = ['login:', 'AEL login:', 'username:', 'aarch64-gem5 login:']
+        login_password_prompt = ['password:']
+        # Wait for the login prompt
+        prompt = login_prompt + [self.conn.UNIQUE_PROMPT]
+        i = self.conn.expect(prompt, timeout=10)
+        # Check if we are already at a prompt, or if we need to log in.
+        if i < len(prompt) - 1:
+            self.conn.sendline("{}".format(self.username))
+            password_prompt = login_password_prompt + [r'# ', self.conn.UNIQUE_PROMPT]
+            j = self.conn.expect(password_prompt, timeout=self.default_timeout)
+            if j < len(password_prompt) - 2:
+                self.conn.sendline("{}".format(self.password))
+                self.conn.expect([r'# ', self.conn.UNIQUE_PROMPT], timeout=self.default_timeout)
+
+
+
+class AndroidGem5Connection(Gem5Connection):
+
+    def _wait_for_boot(self):
+        """
+        Wait for the system to boot
+
+        We monitor the sys.boot_completed and service.bootanim.exit system
+        properties to determine when the system has finished booting. In the
+        event that we cannot coerce the result of service.bootanim.exit to an
+        integer, we assume that the boot animation was disabled and do not wait
+        for it to finish.
+
+        """
+        gem5_logger.info("Waiting for Android to boot...")
+        while True:
+            booted = False
+            anim_finished = True  # Assume boot animation was disabled on except
+            try:
+                booted = (int('0' + self._gem5_shell('getprop sys.boot_completed', check_exit_code=False).strip()) == 1)
+                anim_finished = (int(self._gem5_shell('getprop service.bootanim.exit', check_exit_code=False).strip()) == 1)
+            except ValueError:
+                pass
+            if booted and anim_finished:
+                break
+            time.sleep(60)
+
+        gem5_logger.info("Android booted")
+
 def _give_password(password, command):
     if not sshpass:
         raise HostError('Must have sshpass installed on the host in order to use password-based auth.')

doc/connection.rst

@@ -0,0 +1,240 @@
+Connection
+==========
+
+A :class:`Connection` abstracts an actual physical connection to a device. The
+first connection is created when :func:`Target.connect` method is called. If a
+:class:`Target` is used in a multi-threaded environment, it will maintain a
+connection for each thread in which it is invoked. This allows the same target
+object to be used in parallel in multiple threads.
+
+:class:`Connection`\ s will be automatically created and managed by
+:class:`Target`\ s, so there is usually no reason to create one manually.
+Instead, configuration for a :class:`Connection` is passed as
+`connection_settings` parameter when creating a :class:`Target`. The connection
+to be used target is also specified on instantiation by `conn_cls` parameter,
+though all concrete :class:`Target` implementations will set an appropriate
+default, so there is typically no need to specify this explicitly.
+
+:class:`Connection` classes are not a part of an inheritance hierarchy, i.e.
+they do not derive from a common base. Instead, a :class:`Connection` is any
+class that implements the following methods.
+
+
+.. method:: push(self, source, dest, timeout=None)
+
+   Transfer a file from the host machine to the connected device.
+
+   :param source: path of to the file on the host
+   :param dest: path of to the file on the connected device.
+   :param timeout: timeout (in seconds) for the transfer; if the transfer does
+       not  complete within this period, an exception will be raised.
+
+.. method:: pull(self, source, dest, timeout=None)
+
+   Transfer a file, or files matching a glob pattern, from the connected device
+   to the host machine.
+
+   :param source: path of to the file on the connected device. If ``dest`` is a
+       directory, may be a glob pattern.
+   :param dest: path of to the file on the host
+   :param timeout: timeout (in seconds) for the transfer; if the transfer does
+       not  complete within this period, an exception will be raised.
+
+.. method:: execute(self, command, timeout=None, check_exit_code=False, as_root=False)
+
+   Execute the specified command on the connected device and return its output.
+
+   :param command: The command to be executed.
+   :param timeout: Timeout (in seconds) for the execution of the command. If
+       specified, an exception will be raised if execution does not complete
+       with the specified period.
+   :param check_exit_code: If ``True`` the exit code (on connected device)
+       from execution of the command will be checked, and an exception will be
+       raised if it is not ``0``.
+   :param as_root: The command will be executed as root. This will fail on
+       unrooted connected devices.
+
+.. method:: background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, as_root=False)
+
+   Execute the command on the connected device, invoking it via subprocess on the host.
+   This will return :class:`subprocess.Popen` instance for the command.
+
+   :param command: The command to be executed.
+   :param stdout: By default, standard output will be piped from the subprocess;
+      this may be used to redirect it to an alternative file handle.
+   :param stderr: By default, standard error will be piped from the subprocess;
+      this may be used to redirect it to an alternative file handle.
+   :param as_root: The command will be executed as root. This will fail on
+       unrooted connected devices.
+
+   .. note:: This **will block the connection** until the command completes.
+
+.. note:: The above methods are directly wrapped by :class:`Target` methods,
+          however note that some of the defaults are different.
+
+.. method:: cancel_running_command(self)
+
+   Cancel a running command (previously started with :func:`background`) and free up the connection.
+   It is valid to call this if the command has already terminated (or if no
+   command was issued), in which case this is a no-op.
+
+.. method:: close(self)
+
+   Close the connection to the device. The :class:`Connection` object should not
+   be used after this method is called. There is no way to reopen a previously
+   closed connection, a new connection object should be created instead.
+
+.. note:: There is no :func:`open` method, as the connection is assumed to be
+          opened on instantiation.
+
+
+.. _connection-types:
+
+Connection Types
+----------------
+
+.. class:: AdbConnection(device=None, timeout=None)
+
+    A connection to an android device via ``adb`` (Android Debug Bridge).
+    ``adb`` is part of the Android SDK (though stand-alone versions are also
+    available).
+
+    :param device: The name of the adb divice. This is usually a unique hex
+                   string for USB-connected devices, or an ip address/port
+                   combination. To see connected devices, you can run ``adb
+                   devices`` on the host.
+    :param timeout: Connection timeout in seconds. If a connection to the device
+                    is not esblished within this period, :class:`HostError`
+                    is raised.
+
+
+.. class:: SshConnection(host, username, password=None, keyfile=None, port=None,\
+                         timeout=None, password_prompt=None)
+
+    A connectioned to a device on the network over SSH.
+
+    :param host: SSH host to which to connect
+    :param username: username for SSH login
+    :param password: password for the SSH connection
+
+                     .. note:: In order to user password-based authentication,
+                               ``sshpass`` utility must be installed on the
+                               system.
+
+    :param keyfile: Path to the SSH private key to be used for the connection.
+
+                    .. note:: ``keyfile`` and ``password`` can't be specified
+                              at the same time.
+
+    :param port: TCP port on which SSH server is litening on the remoted device.
+                 Omit to use the default port.
+    :param timeout: Timeout for the connection in seconds. If a connection
+                    cannot be established within this time, an error will be
+                    raised.
+    :param password_prompt: A string with the password prompt used by
+                            ``sshpass``. Set this if your version of ``sshpass``
+                            uses somethin other than ``"[sudo] password"``.
+
+
+.. class:: TelnetConnection(host, username, password=None, port=None,\
+                            timeout=None, password_prompt=None,\
+                            original_prompt=None)
+
+    A connectioned to a device on the network over Telenet.
+
+    .. note:: Since Telenet protocol is does not support file transfer, scp is
+              used for that purpose.
+
+    :param host: SSH host to which to connect
+    :param username: username for SSH login
+    :param password: password for the SSH connection
+
+                     .. note:: In order to user password-based authentication,
+                               ``sshpass`` utility must be installed on the
+                               system.
+
+    :param port: TCP port on which SSH server is litening on the remoted device.
+                 Omit to use the default port.
+    :param timeout: Timeout for the connection in seconds. If a connection
+                    cannot be established within this time, an error will be
+                    raised.
+    :param password_prompt: A string with the password prompt used by
+                            ``sshpass``. Set this if your version of ``sshpass``
+                            uses somethin other than ``"[sudo] password"``.
+    :param original_prompt: A regex for the shell prompted presented in the Telenet
+                            connection (the prompt will be reset to a
+                            randomly-generated pattern for the duration of the
+                            connection to reduce the possibility of clashes).
+                            This paramer is ignored for SSH connections.
+
+
+.. class:: LocalConnection(keep_password=True, unrooted=False, password=None)
+
+    A connection to the local host allowing it to be treated as a Target.
+
+
+    :param keep_password: If this is ``True`` (the default) user's password will
+                          be cached in memory after it is first requested.
+    :param unrooted: If set to ``True``, the platform will be assumed to be
+                     unrooted without testing for root. This is useful to avoid
+                     blocking on password request in scripts.
+    :param password: Specify password on connection creation rather than
+                     prompting for it.
+
+
+.. class:: Gem5Connection(platform, host=None, username=None, password=None,\
+                          timeout=None, password_prompt=None,\
+                          original_prompt=None)
+
+    A connection to a gem5 simulation using a local Telnet connection.
+
+    .. note:: Some of the following input parameters are optional and will be ignored during
+              initialisation. They were kept to keep the anology with a :class:`TelnetConnection`
+              (i.e. ``host``, `username``, ``password``, ``port``,
+              ``password_prompt`` and ``original_promp``)
+
+
+    :param host: Host on which the gem5 simulation is running
+
+                     .. note:: Even thought the input parameter for the ``host``
+                               will be ignored, the gem5 simulation needs to on
+                               the same host as the user as the user is
+                               currently on, so if the host given as input
+                               parameter is not the same as the actual host, a
+                               ``TargetError`` will be raised to prevent
+                               confusion.
+
+    :param username: Username in the simulated system
+    :param password: No password required in gem5 so does not need to be set
+    :param port: Telnet port to connect to gem5. This does not need to be set
+                 at initialisation as this will either be determined by the
+                 :class:`Gem5SimulationPlatform` or can be set using the
+                 :func:`connect_gem5` method
+    :param timeout: Timeout for the connection in seconds. Gem5 has high
+                    latencies so unless the timeout given by the user via
+                    this input parameter is higher than the default one
+                    (3600 seconds), this input parameter will be ignored.
+    :param password_prompt: A string with password prompt
+    :param original_prompt: A regex for the shell prompt
+
+There are two classes that inherit from :class:`Gem5Connection`:
+:class:`AndroidGem5Connection` and :class:`LinuxGem5Connection`.
+They inherit *almost* all methods from the parent class, without altering them.
+The only methods discussed belows are those that will be overwritten by the
+:class:`LinuxGem5Connection` and :class:`AndroidGem5Connection` respectively.
+
+.. class:: LinuxGem5Connection
+
+    A connection to a gem5 simulation that emulates a Linux system.
+
+.. method:: _login_to_device(self)
+
+    Login to the gem5 simulated system.
+
+.. class:: AndroidGem5Connection
+
+    A connection to a gem5 simulation that emulates an Android system.
+
+.. method:: _wait_for_boot(self)
+
+    Wait for the gem5 simulated system to have booted and finished the booting animation.

doc/index.rst

@@ -19,8 +19,8 @@ Contents:
    target
    modules
    instrumentation
-
-
+   platform
+   connection
 
 Indices and tables
 ==================

doc/instrumentation.rst

@@ -28,7 +28,7 @@ Android target.
     # a no-op, but is included here for completeness.
     In [4]: i.setup()
 
-    # Find out what the instrument is capable collecting from the 
+    # Find out what the instrument is capable collecting from the
     # target.
     In [5]: i.list_channels()
     Out[5]:
@@ -40,7 +40,7 @@ Android target.
     In [6]: i.reset(sites=['exynos-therm'])
 
     # HWMON instrument supports INSTANTANEOUS collection, so invoking
-    # take_measurement() will return a list of measurements take from 
+    # take_measurement() will return a list of measurements take from
     # each of the channels configured during reset()
     In [7]: i.take_measurement()
     Out[7]: [exynos-therm_temperature: 36.0 degrees]
@@ -68,7 +68,7 @@ Instrument
                 period of time via ``start()``, ``stop()``, and
                 ``get_data()`` methods.
 
-   .. note:: It's possible for one instrument to support more than a single 
+   .. note:: It's possible for one instrument to support more than a single
              mode.
 
 .. attribute:: Instrument.active_channels
@@ -133,9 +133,9 @@ Instrument
 
 .. method:: Instrument.get_data(outfile)
 
-   Write collected data into ``outfile``. Must be called after :func:`stop()`. 
+   Write collected data into ``outfile``. Must be called after :func:`stop()`.
    Data will be written in CSV format with a column for each channel and a row
-   for each sample. Column heading will be channel, labels in the form 
+   for each sample. Column heading will be channel, labels in the form
    ``<site>_<kind>`` (see :class:`InstrumentChannel`). The order of the coluns
    will be the same as the order of channels in ``Instrument.active_channels``.
 
@@ -146,6 +146,12 @@ Instrument
    .. note:: This method is only implemented by :class:`Instrument`\ s that
              support ``CONTINUOUS`` measurment.
 
+.. attribute:: Instrument.sample_rate_hz
+
+   Sample rate of the instrument in Hz. Assumed to be the same for all channels.
+
+   .. note:: This attribute is only provided by :class:`Instrument`\ s that
+             support ``CONTINUOUS`` measurment.
 
 Instrument Channel
 ~~~~~~~~~~~~~~~~~~

doc/modules.rst

@@ -1,3 +1,5 @@
+.. _modules:
+
 Modules
 =======
 
@@ -9,7 +11,7 @@ hotplug
 -------
 
 Kernel ``hotplug`` subsystem allows offlining ("removing") cores from the
-system, and onlining them back int. The ``devlib`` module exposes a simple
+system, and onlining them back in. The ``devlib`` module exposes a simple
 interface to this subsystem
 
 .. code:: python
@@ -35,10 +37,10 @@ policies (governors). The ``devlib`` module exposes the following interface
 
 .. note:: On ARM big.LITTLE systems, all cores on a cluster (usually all cores
           of the same type) are in the same frequency domain, so setting
-          ``cpufreq`` state on one core on a cluter will affect all cores on
+          ``cpufreq`` state on one core on a cluster will affect all cores on
           that cluster. Because of this, some devices only expose cpufreq sysfs
           interface (which is what is used by the ``devlib`` module) on the
-          first cpu in a cluster. So to keep your scripts proable, always use
+          first cpu in a cluster. So to keep your scripts portable, always use
           the fist (online) CPU in a cluster to set ``cpufreq`` state.
 
 .. method:: target.cpufreq.list_governors(cpu)
@@ -64,26 +66,26 @@ policies (governors). The ``devlib`` module exposes the following interface
    :param cpu: The cpu; could be a numeric or the corresponding string (e.g.
                ``1`` or ``"cpu1"``).
 
-.. method:: target.cpufreq.set_governor(cpu, governor, **kwargs)
+.. method:: target.cpufreq.set_governor(cpu, governor, \*\*kwargs)
 
    Sets the governor for the specified cpu.
 
    :param cpu: The cpu; could be a numeric or the corresponding string (e.g.
         ``1`` or ``"cpu1"``).
    :param governor: The name of the governor. This must be one of the governors 
-                supported by the CPU (as retrunted by ``list_governors()``.
+                supported by the CPU (as returned by ``list_governors()``.
 
    Keyword arguments may be used to specify governor tunable values.
 
 
 .. method:: target.cpufreq.get_governor_tunables(cpu)
 
-   Return a dict with the values of the specfied CPU's current governor.
+   Return a dict with the values of the specified CPU's current governor.
 
    :param cpu: The cpu; could be a numeric or the corresponding string (e.g.
        ``1`` or ``"cpu1"``).
 
-.. method:: target.cpufreq.set_governor_tunables(cpu, **kwargs)
+.. method:: target.cpufreq.set_governor_tunables(cpu, \*\*kwargs)
 
    Set the tunables for the current governor on the specified CPU.
 
@@ -92,7 +94,7 @@ policies (governors). The ``devlib`` module exposes the following interface
 
    Keyword arguments should be used to specify tunable values.
 
-.. method:: target.cpufreq.list_frequencie(cpu)
+.. method:: target.cpufreq.list_frequencies(cpu)
 
    List DVFS frequencies supported by the specified CPU. Returns a list of ints.
 
@@ -104,8 +106,8 @@ policies (governors). The ``devlib`` module exposes the following interface
             target.cpufreq.set_min_frequency(cpu, frequency[, exact=True])
             target.cpufreq.set_max_frequency(cpu, frequency[, exact=True])
 
-   Get and set min and max frequencies on the specfied CPU. "set" functions are
-   avialable with all governors other than ``userspace``.
+   Get and set min and max frequencies on the specified CPU. "set" functions are
+   available with all governors other than ``userspace``.
 
    :param cpu: The cpu; could be a numeric or the corresponding string (e.g.
        ``1`` or ``"cpu1"``).
@@ -126,11 +128,11 @@ cpuidle
 
 ``cpufreq`` is the kernel subsystem for managing CPU low power (idle) states.
 
-.. method:: taget.cpuidle.get_driver()
+.. method:: target.cpuidle.get_driver()
 
    Return the name current cpuidle driver.
 
-.. method:: taget.cpuidle.get_governor()
+.. method:: target.cpuidle.get_governor()
 
    Return the name current cpuidle governor (policy).
 
@@ -169,4 +171,192 @@ TODO
 API
 ---
 
-TODO
+Generic Module API Description
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Modules implement discrete, optional pieces of functionality ("optional" in the
+sense that the functionality may or may not be present on the target device, or
+that it may or may not be necessary for a particular application).
+
+Every module (ultimately) derives from :class:`Module` class.  A module must
+define the following class attributes:
+
+:name: A unique name for the module. This cannot clash with any of the existing
+       names and must be a valid Python identifier, but is otherwise free-from.
+:kind: This identifies the type of functionality a module implements, which in
+       turn determines the interface implemented by the module (all modules of
+       the same kind must expose a consistent interface). This must be a valid
+       Python identifier, but is otherwise free-form, though, where possible,
+       one should try to stick to an already-defined kind/interface, lest we end
+       up with a bunch of modules implementing similar functionality but
+       exposing slightly different interfaces.
+
+       .. note:: It is possible to omit ``kind`` when defining a module, in
+                 which case the module's ``name`` will be treated as its
+                 ``kind`` as well.
+
+:stage: This defines when the module will be installed into a :class:`Target`.
+        Currently, the following values are allowed:
+
+        :connected: The module is installed after a connection to the target has
+                    been established. This is the default.
+        :early: The module will be installed when a :class:`Target` is first
+                created. This should be used for modules that do not rely on a
+                live connection to the target.
+
+Additionally, a module must implement a static (or class) method :func:`probe`:
+
+.. method:: Module.probe(target)
+
+    This method takes a :class:`Target` instance and returns ``True`` if this
+    module is supported by that target, or ``False`` otherwise.
+
+    .. note:: If the module ``stage`` is ``"early"``, this method cannot assume
+              that a connection has been established (i.e. it can only access
+              attributes of the Target that do not rely on a connection).
+
+Installation and invocation
+***************************
+
+The default installation method will create an instance of a module (the
+:class:`Target` instance being the sole argument) and assign it to the target
+instance attribute named after the module's ``kind`` (or ``name`` if ``kind`` is
+``None``).
+
+It is possible to change the installation procedure for a module by overriding
+the default :func:`install` method. The method must have the following
+signature:
+
+.. method:: Module.install(cls, target, **kwargs)
+
+    Install the module into the target instance.
+
+
+Implementation and Usage Patterns
+*********************************
+
+There are two common ways to implement the above API, corresponding to the two
+common uses for modules:
+
+- If a module provides an interface to a particular set of functionality (e.g.
+  an OS subsystem), that  module would typically derive directly form
+  :class:`Module` and  would leave ``kind`` unassigned, so that it is accessed
+  by it name. Its instance's methods and attributes provide the interface for
+  interacting with its functionality. For examples of this type of module, see
+  the subsystem modules listed above (e.g. ``cpufreq``).
+- If a module provides a platform- or infrastructure-specific implementation of
+  a common function, the module would derive from one of :class:`Module`
+  subclasses that define the interface for that function. In that case the
+  module would be accessible via the common ``kind`` defined its super. The
+  module would typically implement :func:`__call__` and be invoked directly. For
+  examples of this type of module, see common function interface definitions
+  below.
+
+
+Common Function Interfaces
+~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+This section documents :class:`Module` classes defining interface for common
+functions. Classes derived from them provide concrete implementations for
+specific platforms.
+
+
+HardResetModule
+***************
+
+.. attribute:: HardResetModule.kind
+
+    "hard_reset"
+
+.. method:: HardResetModule.__call__()
+
+    Must be implemented by derived classes.
+    
+    Implements hard reset for a target devices. The equivalent of physically
+    power cycling the device.  This may be used by client code in situations
+    where the target becomes unresponsive and/or a regular reboot is not
+    possible.
+
+
+BootModule
+**********
+
+.. attribute:: BootModule.kind
+
+    "hard_reset"
+
+.. method:: BootModule.__call__()
+
+    Must be implemented by derived classes.
+
+    Implements a boot procedure. This takes the device from (hard or soft)
+    reset to a booted state where the device is ready to accept connections. For
+    a lot of commercial devices the process is entirely automatic, however some
+    devices (e.g. development boards), my require additional steps, such as
+    interactions with the bootloader, in order to boot into the OS.
+
+.. method:: Bootmodule.update(\*\*kwargs)
+
+    Update the boot settings. Some boot sequences allow specifying settings
+    that will be utilized during boot (e.g. linux kernel boot command line). The
+    default implementation will set each setting in ``kwargs`` as an attribute of
+    the boot module (or update the existing attribute).
+
+
+FlashModule
+***********
+
+.. attribute:: FlashModule.kind
+
+    "flash"
+
+.. method:: __call__(image_bundle=None, images=None, boot_config=None)
+
+    Must be implemented by derived classes.
+
+    Flash the target platform with the specified images.
+
+    :param image_bundle: A compressed bundle of image files with any associated
+                         metadata. The format of the bundle is specific to a
+                         particular implementation.
+    :param images: A dict mapping image names/identifiers to the path on the
+                   host file system of the corresponding image file. If both
+                   this and ``image_bundle`` are specified, individual images
+                   will override those in the bundle.
+    :param boot_config: Some platforms require specifying boot arguments at the
+                        time of flashing the images, rather than during each
+                        reboot. For other platforms, this will be ignored.
+
+
+Module Registration
+~~~~~~~~~~~~~~~~~~~
+
+Modules are specified on :class:`Target` or :class:`Platform` creation by name.
+In order to find the class associated with the name, the module needs to be
+registered with ``devlib``. This is accomplished by passing the module class
+into :func:`register_module` method once it is defined.
+
+.. note:: If you're wiring a module to be included as part of ``devlib`` code
+          base, you can place the file with the module class under
+          ``devlib/modules/`` in the source and it will be automatically
+          enumerated. There is no need to explicitly register it in that case.
+
+The code snippet below illustrates an implementation of a hard reset function
+for an "Acme" device.
+
+.. code:: python
+
+    import os
+    from devlib import HardResetModule, register_module
+
+
+    class AcmeHardReset(HardResetModule):
+
+        name = 'acme_hard_reset'
+
+        def __call__(self):
+            # Assuming Acme board comes with a "reset-acme-board" utility 
+            os.system('reset-acme-board {}'.format(self.target.name))
+
+    register_module(AcmeHardReset)
+

doc/overview.rst

@@ -32,7 +32,7 @@ instantiating each of the three target types.
    # For a Linux device, you will need to provide the normal SSH credentials.
    # Both password-based, and key-based authentication is supported (password
    # authentication requires sshpass to be installed on your host machine).'
-   t2 = LinuxTarget(connetion_settings={'host': '192.168.0.5',
+   t2 = LinuxTarget(connection_settings={'host': '192.168.0.5',
                                         'username': 'root',
                                         'password': 'sekrit',
                                         # or
@@ -57,7 +57,7 @@ Target Interface
 ----------------
 
 This is a quick overview of the basic interface to the device. See
-:class:`Targeet` API documentation for the full list of supported methods and
+:class:`Target` API documentation for the full list of supported methods and
 more detailed documentation.
 
 One-time Setup

doc/platform.rst

@@ -0,0 +1,171 @@
+.. _platform:
+
+Platform
+========
+
+:class:`Platform`\ s describe the system underlying the OS. They encapsulate
+hardware- and firmware-specific details. In most cases, the generic
+:class:`Platform` class, which gets used if a platform is not explicitly
+specified on :class:`Target` creation, will be sufficient. It will automatically
+query as much platform information (such CPU topology, hardware model, etc) if
+it was not specified explicitly by the user.
+
+
+.. class:: Platform(name=None, core_names=None, core_clusters=None,\
+                    big_core=None, model=None, modules=None)
+
+    :param name: A user-friendly identifier for the platform.
+    :param core_names: A list of CPU core names in the order they appear
+                       registered with the OS. If they are not specified,
+                       they will be queried at run time.
+    :param core_clusters: Alist with cluster ids of each core (starting with
+                          0). If this is not specified, clusters will be
+                          inferred from core names (cores with the same name are
+                          assumed to be in a cluster).
+    :param big_core: The name of the big core in a big.LITTLE system. If this is
+                     not specified it will be inferred (on systems with exactly 
+                     two clasters).
+    :param model: Model name of the hardware system. If this is not specified it
+                  will be queried at run time.
+    :param modules: Modules with additional functionality supported by the
+                    platfrom (e.g. for handling flashing, rebooting, etc). These
+                    would be added to the Target's modules. (See :ref:`modules`\ ).
+
+
+Versatile Express
+-----------------
+
+The generic platform may be extended to support hardware- or
+infrastructure-specific functionality. Platforms exist for ARM
+VersatileExpress-based :class:`Juno` and :class:`TC2` development boards. In
+addition to the standard :class:`Platform` parameters above, these platfroms
+support additional configuration:
+
+
+.. class:: VersatileExpressPlatform
+
+    Normally, this would be instatiated via one of its derived classes
+    (:class:`Juno` or :class:`TC2`) that set appropriate defaults for some of
+    the parameters.
+
+    :param serial_port: Identifies the serial port (usual a /dev node) on which the
+                        device is connected.
+    :param baudrate: Baud rate for the serial connection. This defaults to
+                     ``115200`` for :class:`Juno` and ``38400`` for
+                     :class:`TC2`.
+    :param vemsd_mount: Mount point for the VEMSD (Versatile Express MicroSD card
+                        that is used for board configuration files and firmware
+                        images). This defaults to ``"/media/JUNO"`` for
+                        :class:`Juno` and ``"/media/VEMSD"`` for :class:`TC2`,
+                        though you would most likely need to change this for
+                        your setup as it would depend both on the file system
+                        label on the MicroSD card, and on how the card was
+                        mounted on the host system.
+    :param hard_reset_method: Specifies the method for hard-resetting the devices
+                            (e.g. if it becomes unresponsive and normal reboot
+                            method doesn not work). Currently supported methods
+                            are:
+
+                            :dtr: reboot by toggling DTR line on the serial
+                                  connection (this is enabled via a DIP switch
+                                  on the board).
+                            :reboottxt: reboot by writing a filed called
+                                        ``reboot.txt`` to the root of the VEMSD
+                                        mount (this is enabled via board
+                                        configuration file).
+
+                            This defaults to ``dtr`` for :class:`Juno` and
+                            ``reboottxt`` for :class:`TC2`.
+    :param bootloader: Specifies the bootloader configuration used by the board.
+                      The following values are currently supported:
+
+                       :uefi: Boot via UEFI menu, by selecting the entry
+                              specified by ``uefi_entry`` paramter. If this
+                              entry does not exist, it will be automatically
+                              created based on values provided for ``image``,
+                              ``initrd``, ``fdt``, and ``bootargs`` parameters.
+                       :uefi-shell: Boot by going via the UEFI shell.
+                       :u-boot: Boot using Das U-Boot.
+                       :bootmon: Boot directly via Versatile Express Bootmon
+                                 using the values provided for ``image``, 
+                                 ``initrd``, ``fdt``, and ``bootargs`` 
+                                 parameters.
+
+                      This defaults to ``u-boot`` for :class:`Juno` and
+                      ``bootmon`` for :class:`TC2`.
+    :param flash_method: Specifies how the device is flashed. Currently, only
+                        ``"vemsd"`` method is supported, which flashes by
+                        writing firmware images to an appropriate location on
+                        the VEMSD.
+    :param image: Specfies the kernel image name for ``uefi``  or ``bootmon`` boot.
+    :param fdt: Specifies the device tree blob for  ``uefi``  or ``bootmon`` boot.
+    :param initrd: Specifies the ramdisk image for  ``uefi`` or ``bootmon`` boot.
+    :param bootargs: Specifies the boot arguments that will be pass to the
+                     kernel by the bootloader.
+    :param uefi_entry: Then name of the UEFI entry to be used/created by
+                       ``uefi`` bootloader.
+    :param ready_timeout: Timeout, in seconds, for the time it takes the
+                          platform to become ready to accept connections. Note:
+                          this does not mean that the system is fully booted;
+                          just that the services needed to establish a
+                          connection (e.g. sshd or adbd) are up.
+
+
+.. _gem5-platform:
+
+Gem5 Simulation Platform
+------------------------
+
+By initialising a Gem5SimulationPlatform, devlib will start a gem5 simulation (based upon the
+arguments the user provided) and then connect to it using :class:`Gem5Connection`.
+Using the methods discussed above, some methods of the :class:`Target` will be altered
+slightly to better suit gem5.
+
+.. class:: Gem5SimulationPlatform(name, host_output_dir, gem5_bin, gem5_args, gem5_virtio, gem5_telnet_port=None)
+
+    During initialisation the gem5 simulation will be kicked off (based upon the arguments
+    provided by the user) and the telnet port used by the gem5 simulation will be intercepted
+    and stored for use by the :class:`Gem5Connection`.
+
+    :param name: Platform name
+
+    :param host_output_dir: Path on the host where the gem5 outputs will be placed (e.g. stats file)
+
+    :param gem5_bin: gem5 binary
+
+    :param gem5_args: Arguments to be passed onto gem5 such as config file etc.
+
+    :param gem5_virtio: Arguments to be passed onto gem5 in terms of the virtIO device used
+                        to transfer files between the host and the gem5 simulated system.
+
+    :param gem5_telnet_port: Not yet in use as it would be used in future implementations
+                             of devlib in which the user could use the platform to pick
+                             up an existing and running simulation.
+
+
+.. method:: Gem5SimulationPlatform.init_target_connection([target])
+
+    Based upon the OS defined in the :class:`Target`, the type of :class:`Gem5Connection`
+    will be set (:class:`AndroidGem5Connection` or :class:`AndroidGem5Connection`).
+
+.. method:: Gem5SimulationPlatform.update_from_target([target])
+
+    This method provides specific setup procedures for a gem5 simulation. First of all, the m5
+    binary will be installed on the guest (if it is not present). Secondly, three methods
+    in the :class:`Target` will be monkey-patched:
+
+            - **reboot**: this is not supported in gem5
+            - **reset**: this is not supported in gem5
+            - **capture_screen**: gem5 might already have screencaps so the
+              monkey-patched method will first try to
+              transfer the existing screencaps.
+              In case that does not work, it will fall back
+              to the original :class:`Target` implementation
+              of :func:`capture_screen`.
+
+    Finally, it will call the parent implementation of :func:`update_from_target`.
+
+.. method:: Gem5SimulationPlatform.setup([target])
+
+    The m5 binary be installed, if not yet installed on the gem5 simulated system.
+    It will also resize the gem5 shell, to avoid line wrapping issues.

doc/target.rst

@@ -10,15 +10,15 @@ Target
     :class:`Instrument`).
 
     :param connection_settings: A ``dict`` that specifies how to connect to the remote 
-       device. Its contents depend on the specific :class:`Target` type used (e.g.
-       :class:`AndroidTarget` expects the adb ``device`` name).
+       device. Its contents depend on the specific :class:`Target` type (used see
+       :ref:`connection-types`\ ).
 
     :param platform: A :class:`Target` defines interactions at Operating System level. A 
         :class:`Platform` describes the underlying hardware (such as CPUs
         available). If a :class:`Platform` instance is not specified on
         :class:`Target` creation, one will be created automatically and it will
         dynamically probe the device to discover as much about the underlying
-        hardware as it can.
+        hardware as it can. See also :ref:`platform`\ .
 
     :param working_directory: This is primary location for on-target file system
         interactions performed by ``devlib``. This location *must* be readable and
@@ -53,7 +53,7 @@ Target
     :param modules: a list of additional modules to be installed. Some modules will
         try to install by default (if supported by the underlying target).
         Current default modules are ``hotplug``, ``cpufreq``, ``cpuidle``, 
-        ``cgroups``, and ``hwmon``.
+        ``cgroups``, and ``hwmon`` (See :ref:`modules`\ ).
 
         See modules documentation for more detail.
 
@@ -420,3 +420,15 @@ Target
    Returns ``True`` if an executable with the specified name is installed on the
    target and ``False`` other wise.
 
+.. method:: Target.extract(path, dest=None)
+
+   Extracts the specified archive/file and returns the path to the extrated
+   contents. The extraction method is determined based on the file extension.
+   ``zip``, ``tar``, ``gzip``, and ``bzip2`` are supported.
+
+   :param dest: Specified an on-target destination directory (which must exist)
+                for the extrated contents.
+
+    Returns the path to the extracted contents. In case of files (gzip and
+    bzip2), the path to the decompressed file is returned; for archives, the
+    path to the directory with the archive's contents is returned.

setup.py

@@ -59,7 +59,7 @@ for root, dirs, files in os.walk(devlib_dir):
 params = dict(
     name='devlib',
     description='A framework for automating workload execution and measurment collection on ARM devices.',
-    version='0.0.1',
+    version='0.0.4',
     packages=packages,
     package_data=data_files,
     url='N/A',
@@ -69,6 +69,7 @@ params = dict(
         'python-dateutil',  # converting between UTC and local time.
         'pexpect>=3.3',  # Send/recieve to/from device
         'pyserial',  # Serial port interface
+        'wrapt',  # Basic for construction of decorator functions
     ],
     extras_require={
         'daq': ['daqpower'],

src/readenergy/Makefile

@@ -4,7 +4,7 @@
 #
 CROSS_COMPILE?=aarch64-linux-gnu-
 CC=$(CROSS_COMPILE)gcc
-CFLAGS='-Wl,-static -Wl,-lc'
+CFLAGS=-static -lc
 
 readenergy: readenergy.c
 	$(CC) $(CFLAGS) readenergy.c -o readenergy

src/readenergy/readenergy.c

@@ -89,6 +89,9 @@
 // Default counter poll period (in milliseconds).
 #define DEFAULT_PERIOD 100
 
+// Default duration for the instrument execution (in seconds); 0 means 'forever'
+#define DEFAULT_DURATION 0
+
 // A single reading from the energy meter. The values are the proper readings converted
 // to appropriate units (e.g. Watts for power); they are *not* raw counter values.
 struct reading
@@ -141,12 +144,17 @@ int nsleep(const struct timespec *req, struct timespec *rem)
  
 void print_help()
 {
-	fprintf(stderr, "Usage: readenergy [-t PERIOD] -o OUTFILE\n\n"
+	fprintf(stderr, "Usage: readenergy [-t PERIOD] [-o OUTFILE]\n\n"
 			"Read Juno energy counters every PERIOD milliseconds, writing them\n"
-			"to OUTFILE in CSV format until SIGTERM is received.\n\n"
+			"to OUTFILE in CSV format either until SIGTERM is received OR\n"
+			"till the specified duration elapsed.\n"
+			"If OUTFILE is not specified, stdout will be used.\n\n"
 			"Parameters:\n"
 			"	PERIOD is the counter poll period in milliseconds.\n"
 			"	       (Defaults to 100 milliseconds.)\n"
+			"	DURATION is the duration before execution terminates.\n"
+			"		(Defaults to 0 seconds, meaning run till user\n"
+			"		terminates execution.\n"
 			"	OUTFILE is the output file path\n");
 }
 
@@ -163,6 +171,7 @@ struct config
 {
 	struct timespec period;
 	char *output_file;
+	long duration_in_sec;
 };
 
 void config_init_period_from_millis(struct config *this, long millis)
@@ -175,9 +184,10 @@ void config_init(struct config *this, int argc, char *argv[])
 {
 	this->output_file = NULL;
 	config_init_period_from_millis(this, DEFAULT_PERIOD);
+	this->duration_in_sec = DEFAULT_DURATION;
 
 	int opt;
-	while ((opt = getopt(argc, argv, "ht:o:")) != -1)
+	while ((opt = getopt(argc, argv, "ht:o:d:")) != -1)
 	{
 		switch(opt)
 		{
@@ -187,6 +197,9 @@ void config_init(struct config *this, int argc, char *argv[])
 			case 'o':
 				this->output_file = optarg;
 				break;
+			case 'd':
+				this->duration_in_sec = atol(optarg);
+				break;
 			case 'h':
 				print_help();
 				exit(EXIT_SUCCESS);
@@ -197,13 +210,6 @@ void config_init(struct config *this, int argc, char *argv[])
 				exit(EXIT_FAILURE);
 		}
 	}
-
-	if (this->output_file == NULL)
-	{
-		fprintf(stderr, "ERROR: Mandatory -o option not specified.\n\n");
-		print_help();
-		exit(EXIT_FAILURE);
-	}
 }
 
 // -------------------------------------- /config ---------------------------------------------------
@@ -219,13 +225,17 @@ struct emeter
 
 void emeter_init(struct emeter *this, char *outfile)
 {
-	this->out = fopen(outfile, "w");
-	if (this->out == NULL)
+	if(outfile)
 	{
-		fprintf(stderr, "ERROR: Could not open output file %s; got %s\n", outfile, strerror(errno));
-		exit(EXIT_FAILURE);
+		this->out = fopen(outfile, "w");
+		if (this->out == NULL)
+		{
+			fprintf(stderr, "ERROR: Could not open output file %s; got %s\n", outfile, strerror(errno));
+			exit(EXIT_FAILURE);
+		}
+	} else {
+		this->out = stdout;
 	}
-
         this->fd = open("/dev/mem", O_RDONLY);
         if(this->fd < 0)
         {
@@ -243,10 +253,12 @@ void emeter_init(struct emeter *this, char *outfile)
 		exit(EXIT_FAILURE);
 	}
 
-	fprintf(this->out, "sys_curr,a57_curr,a53_curr,gpu_curr,"
- 			   "sys_volt,a57_volt,a53_volt,gpu_volt,"
-			   "sys_pow,a57_pow,a53_pow,gpu_pow,"
-			   "sys_cenr,a57_cenr,a53_cenr,gpu_cenr\n");
+	if(this->out) {
+		fprintf(this->out, "sys_curr,a57_curr,a53_curr,gpu_curr,"
+				   "sys_volt,a57_volt,a53_volt,gpu_volt,"
+				   "sys_pow,a57_pow,a53_pow,gpu_pow,"
+				   "sys_cenr,a57_cenr,a53_cenr,gpu_cenr\n");
+	}
 }
 
 void emeter_read_measurements(struct emeter *this, struct reading *reading)
@@ -314,13 +326,19 @@ void emeter_finalize(struct emeter *this)
 
 // -------------------------------------- /emeter ----------------------------------------------------
 
-int done = 0;
+volatile int done = 0;
 
 void term_handler(int signum)
 {
 	done = 1;
 }
 
+void sigalrm_handler(int signum)
+{
+	done = 1;
+}
+
+
 int main(int argc, char *argv[])
 {
 	struct sigaction action;
@@ -333,11 +351,27 @@ int main(int argc, char *argv[])
 	config_init(&config, argc, argv);
 	emeter_init(&emeter, config.output_file);
 
-	struct timespec remaining;
-	while (!done) 
+	if (0 != config.duration_in_sec)
 	{
+		/*Set the alarm with the duration from use only if a non-zero value is specified
+		  else it will run forever until SIGTERM signal received from user*/
+		/*Set the signal handler first*/
+		signal(SIGALRM, sigalrm_handler);
+		/*Now set the alarm for the duration specified by the user*/
+		alarm(config.duration_in_sec);
+
+	}
+
+	if(config.output_file)
+	{
+		struct timespec remaining;
+		while (!done)
+		{
+			emeter_take_reading(&emeter);
+			nsleep(&config.period, &remaining);
+		}
+	} else 	{
 		emeter_take_reading(&emeter);
-		nsleep(&config.period, &remaining);
 	}
 
 	emeter_finalize(&emeter);