Add a new Arm Energy Probe instrument

Add new energy instrument that is based on arm-probe tool to manage AEP
Main advantages of this tool are:
- uses a config file for describing channels and shunt resistors value
- manages power topology description in the config file. This topology
is then used when computing power figures
- can create virtual power channel and aggregate channels
- support multiple AEP
- support auto-zero of AEP's channel

Signed-off-by: Vincent Guittot <vincent.guittot@linaro.org>

devlib/__init__.py

@@ -13,6 +13,7 @@ from devlib.instrument import Instrument, InstrumentChannel, Measurement, Measur
 from devlib.instrument import MEASUREMENT_TYPES, INSTANTANEOUS, CONTINUOUS
 from devlib.instrument.daq import DaqInstrument
 from devlib.instrument.energy_probe import EnergyProbeInstrument
+from devlib.instrument.arm_energy_probe import ArmEnergyProbeInstrument
 from devlib.instrument.frames import GfxInfoFramesInstrument, SurfaceFlingerFramesInstrument
 from devlib.instrument.hwmon import HwmonInstrument
 from devlib.instrument.monsoon import MonsoonInstrument

devlib/instrument/arm_energy_probe.py

@@ -0,0 +1,132 @@
+#    Copyright 2018 Linaro Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+# pylint: disable=W0613,E1101,access-member-before-definition,attribute-defined-outside-init
+from __future__ import division
+import os
+import csv
+import subprocess
+import signal
+import struct
+import sys
+
+import tempfile
+import shutil
+
+from devlib.instrument import Instrument, CONTINUOUS, MeasurementsCsv
+from devlib.exception import HostError
+from devlib.utils.misc import which
+
+from devlib.utils.parse_aep import AepParser
+
+class ArmEnergyProbeInstrument(Instrument):
+    """
+    Collects power traces using the ARM Energy Probe.
+
+    This instrument requires ``arm-probe`` utility to be installed on the host and be in the PATH.
+    arm-probe is available here:
+    ``https://git.linaro.org/tools/arm-probe.git``.
+
+    Details about how to build and use it is available here:
+    ``https://git.linaro.org/tools/arm-probe.git/tree/README``
+
+    ARM energy probe (AEP) device can simultaneously collect power from up to 3 power rails and
+    arm-probe utility can record data from several AEP devices simultaneously.
+
+    To connect the energy probe on a rail, connect the white wire to the pin that is closer to the
+    Voltage source and the black wire to the pin that is closer to the load (the SoC or the device
+    you are probing). Between the pins there should be a shunt resistor of known resistance in the
+    range of 5 to 500 mOhm but the voltage on the shunt resistor must stay smaller than 165mV.
+    The resistance of the shunt resistors is a mandatory parameter to be set in the ``config`` file.
+    """
+
+    mode = CONTINUOUS
+
+    MAX_CHANNELS = 12 # 4 Arm Energy Probes
+
+    def __init__(self, target, config_file='./config-aep', ):
+        super(ArmEnergyProbeInstrument, self).__init__(target)
+        self.arm_probe = which('arm-probe')
+        if self.arm_probe is None:
+            raise HostError('arm-probe must be installed on the host')
+        #todo detect is config file exist
+        self.attributes = ['power', 'voltage', 'current']
+        self.sample_rate_hz = 10000
+        self.config_file = config_file
+
+        self.parser = AepParser()
+        #TODO make it generic
+        topo = self.parser.topology_from_config(self.config_file)
+        for item in topo:
+            if item == 'time':
+                self.add_channel('timestamp', 'time')
+            else:
+                self.add_channel(item, 'power')
+
+    def reset(self, sites=None, kinds=None, channels=None):
+        super(ArmEnergyProbeInstrument, self).reset(sites, kinds, channels)
+        self.output_directory = tempfile.mkdtemp(prefix='energy_probe')
+        self.output_file_raw = os.path.join(self.output_directory, 'data_raw')
+        self.output_file = os.path.join(self.output_directory, 'data')
+        self.output_file_figure = os.path.join(self.output_directory, 'summary.txt')
+        self.output_file_error = os.path.join(self.output_directory, 'error.log')
+        self.output_fd_error = open(self.output_file_error, 'w')
+        self.command = 'arm-probe --config {} > {}'.format(self.config_file, self.output_file_raw)
+
+    def start(self):
+        self.logger.debug(self.command)
+        self.armprobe = subprocess.Popen(self.command,
+                                       stderr=self.output_fd_error,
+                                       preexec_fn=os.setpgrp,
+                                       shell=True)
+
+    def stop(self):
+       self.logger.debug("kill running arm-probe")
+       os.killpg(self.armprobe.pid, signal.SIGTERM)
+
+    def get_data(self, outfile):  # pylint: disable=R0914
+        self.logger.debug("Parse data and compute consumed energy")
+        self.parser.prepare(self.output_file_raw, self.output_file, self.output_file_figure)
+        self.parser.parse_aep()
+        self.parser.unprepare()
+        skip_header = 1
+
+        all_channels = [c.label for c in self.list_channels()]
+        active_channels = [c.label for c in self.active_channels]
+        active_indexes = [all_channels.index(ac) for ac in active_channels]
+
+        with open(self.output_file, 'rb') as ifile:
+            reader = csv.reader(ifile, delimiter=' ')
+            with open(outfile, 'wb') as wfh:
+                writer = csv.writer(wfh)
+                for row in reader:
+                    if skip_header == 1:
+                        writer.writerow(active_channels)
+                        skip_header = 0
+                        continue
+                    if len(row) < len(active_channels):
+                        continue
+                    # all data are in micro (seconds/watt)
+                    new = [ float(row[i])/1000000 for i in active_indexes ]
+                    writer.writerow(new)
+
+        self.output_fd_error.close()
+        shutil.rmtree(self.output_directory)
+
+        return MeasurementsCsv(outfile, self.active_channels, self.sample_rate_hz)
+
+    def get_raw(self):
+        return [self.output_file_raw]

devlib/utils/parse_aep.py

@@ -0,0 +1,521 @@
+#!/usr/bin/env python
+#    Copyright 2018 Linaro Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import sys
+import getopt
+import subprocess
+import logging
+import signal
+import serial
+import time
+import math
+
+logger = logging.getLogger('aep-parser')
+
+class AepParser(object):
+    prepared = False
+
+    @staticmethod
+    def topology_from_data(array, topo):
+    # Extract topology information for the data file
+    # The header of a data file looks like this ('#' included):
+    # configuration: <file path>
+    # config_name: <file name>
+    # trigger: 0.400000V (hyst 0.200000V) 0.000000W (hyst 0.200000W) 400us
+    # date: Fri, 10 Jun 2016 11:25:07 +0200
+    # host: <host name>
+    #
+    # CHN_0	    Pretty_name_0   PARENT_0    Color0  Class0
+    # CHN_1	    Pretty_name_1   PARENT_1    Color1  Class1
+    # CHN_2	    Pretty_name_2   PARENT_2    Color2  Class2
+    # CHN_3	    Pretty_name_3   PARENT_3    Color3  Class3
+    # ..
+    # CHN_N	    Pretty_name_N   PARENT_N    ColorN  ClassN
+    #
+
+        info = {}
+
+        if len(array) == 6:
+            info['name'] = array[1]
+            info['parent'] = array[3]
+            info['pretty'] = array[2]
+            # add an entry for both name and pretty name in order to not parse
+            # the whole dict when looking for a parent and the parent of parent
+            topo[array[1]] = info
+            topo[array[2]] = info
+        return topo
+
+    @staticmethod
+    def create_virtual(topo, label, hide, duplicate):
+    # Create a list of virtual power domain that are the sum of others
+    # A virtual domain is the parent of several channels but is not sampled by a
+    # channel
+    # This can be useful if a power domain is supplied by 2 power rails
+        virtual = {}
+
+        # Create an entry for each virtual parent
+        for supply in topo.iterkeys():
+            index = topo[supply]['index']
+            # Don't care of hidden columns
+            if hide[index]:
+                continue
+
+            # Parent is in the topology
+            parent = topo[supply]['parent']
+            if parent in topo:
+                continue
+
+            if parent not in virtual:
+                virtual[parent] = { supply : index }
+
+            virtual[parent][supply] = index
+
+        # Remove parent with 1 child as they don't give more information than their
+        # child
+        for supply in virtual.keys():
+            if len(virtual[supply]) == 1:
+                del virtual[supply];
+
+        for supply in virtual.keys():
+            # Add label, hide and duplicate columns for virtual domains
+            hide.append(0)
+            duplicate.append(1)
+            label.append(supply)
+
+        return virtual
+
+    @staticmethod
+    def get_label(array):
+    # Get the label of each column
+    # Remove unit '(X)' from the end of the label
+        label = [""]*len(array)
+        unit = [""]*len(array)
+
+        label[0] = array[0]
+        unit[0] = "(S)"
+        for i in range(1,len(array)):
+            label[i] = array[i][:-3]
+            unit[i] = array[i][-3:]
+
+        return label, unit
+
+    @staticmethod
+    def filter_column(label, unit, topo):
+    # Filter columns
+    # We don't parse Volt and Amper columns: put in hide list
+    # We don't add in Total a column that is the child of another one: put in duplicate list
+
+        # By default we hide all columns
+        hide = [1] * len(label)
+        # By default we assume that there is no child
+        duplicate = [0] * len(label)
+
+        for i in range(len(label)):
+            # We only care about time and Watt
+            if label[i] == 'time':
+                hide[i] = 0
+                continue
+
+            if '(W)' not in unit[i]:
+                continue
+
+            hide[i] = 0
+
+            #label is pretty name
+            pretty = label[i]
+
+            # We don't add a power domain that is already accounted by its parent
+            if topo[pretty]['parent'] in topo:
+                duplicate[i] = 1
+
+            # Set index, that will be used by virtual domain
+            topo[topo[pretty]['name']]['index'] = i
+
+            # remove pretty element that is useless now
+            del topo[pretty]
+
+        return hide, duplicate
+
+    @staticmethod
+    def parse_text(array, hide):
+        data = [0]*len(array)
+        for i in range(len(array)):
+            if hide[i]:
+                continue
+
+            try:
+                data[i] = int(float(array[i])*1000000)
+            except ValueError:
+                continue
+
+        return data
+
+    @staticmethod
+    def add_virtual_data(data, virtual):
+        # write virtual domain
+        for parent in virtual.iterkeys():
+            power = 0
+            for child in virtual[parent].values():
+                try:
+                    power += data[child]
+                except IndexError:
+                    continue
+            data.append(power)
+
+        return data
+
+    @staticmethod
+    def delta_nrj(array, delta, min, max, hide):
+    # Compute the energy consumed in this time slice and add it
+    # delta[0] is used to save the last time stamp
+
+        if (delta[0] < 0):
+            delta[0] = array[0]
+
+        time = array[0] - delta[0]
+        if (time <= 0):
+            return delta
+
+        for i in range(len(array)):
+            if hide[i]:
+                continue
+
+            try:
+                data = array[i]
+            except ValueError:
+                continue
+
+            if (data < min[i]):
+                min[i] = data
+            if (data > max[i]):
+                max[i] = data
+            delta[i] += time * data
+
+        # save last time stamp
+        delta[0] = array[0]
+
+        return delta
+
+    def output_label(self, label, hide):
+        self.fo.write(label[0]+"(uS)")
+        for i in range(1, len(label)):
+            if hide[i]:
+                continue
+            self.fo.write(" "+label[i]+"(uW)")
+
+        self.fo.write("\n")
+
+    def output_power(self, array, hide):
+        #skip partial line. Most probably the last one
+        if len(array) < len(hide):
+            return
+
+        # write not hidden colums
+        self.fo.write(str(array[0]))
+        for i in range(1, len(array)):
+            if hide[i]:
+                continue
+
+            self.fo.write(" "+str(array[i]))
+
+        self.fo.write("\n")
+
+    def prepare(self, infile, outfile, summaryfile):
+
+        try:
+            self.fi = open(infile, "r")
+        except IOError:
+            logger.warn('Unable to open input file {}'.format(infile))
+            logger.warn('Usage: parse_arp.py -i <inputfile> [-o <outputfile>]')
+            sys.exit(2)
+
+        self.parse = True
+        if len(outfile) > 0:
+            try:
+                self.fo = open(outfile, "w")
+            except IOError:
+                logger.warn('Unable to create {}'.format(outfile))
+                self.parse = False
+        else:
+                self.parse = False
+
+        self.summary = True
+        if len(summaryfile) > 0:
+            try:
+                self.fs = open(summaryfile, "w")
+            except IOError:
+                logger.warn('Unable to create {}'.format(summaryfile))
+                self.fs = sys.stdout
+        else:
+                self.fs = sys.stdout
+
+        self.prepared = True
+
+    def unprepare(self):
+        if not self.prepared:
+            # nothing has been prepared
+            return
+
+        self.fi.close()
+
+        if self.parse:
+            self.fo.close()
+
+        self.prepared = False
+
+    def parse_aep(self, start=0, lenght=-1):
+    # Parse aep data and calculate the energy consumed
+        begin = 0
+
+        label_line = 1
+
+        topo = {}
+
+        lines = self.fi.readlines()
+
+        for myline in lines:
+            array  = myline.split()
+
+            if "#" in myline:
+                # update power topology
+                topo = self.topology_from_data(array, topo)
+                continue
+
+            if label_line:
+                label_line = 0
+                # 1st line not starting with # gives label of each column
+                label, unit = self.get_label(array)
+                # hide useless columns and detect channels that are children
+                # of other channels
+                hide, duplicate = self.filter_column(label, unit, topo)
+
+                # Create virtual power domains
+                virtual = self.create_virtual(topo, label, hide, duplicate)
+                if self.parse:
+                    self.output_label(label, hide)
+
+                logger.debug('Topology : {}'.format(topo))
+                logger.debug('Virtual power domain : {}'.format(virtual))
+                logger.debug('Duplicated power domain : : {}'.format(duplicate))
+                logger.debug('Name of columns : {}'.format(label))
+                logger.debug('Hidden columns : {}'.format(hide))
+                logger.debug('Unit of columns : {}'.format(unit))
+
+                # Init arrays
+                nrj = [0]*len(label)
+                min = [100000000]*len(label)
+                max = [0]*len(label)
+                offset = [0]*len(label)
+
+                continue
+
+            # convert text to int and unit to micro-unit
+            data = self.parse_text(array, hide)
+
+            # get 1st time stamp
+            if begin <= 0:
+                being = data[0]
+
+            # skip data before start
+            if (data[0]-begin) < start:
+                continue
+
+            # stop after lenght
+            if lenght >= 0 and (data[0]-begin) > (start + lenght):
+                continue
+
+            # add virtual domains
+            data = self.add_virtual_data(data, virtual)
+
+            # extract power figures
+            self.delta_nrj(data, nrj, min, max, hide)
+
+            # write data into new file
+            if self.parse:
+                self.output_power(data, hide)
+
+        # if there is no data just return
+        if label_line or len(nrj) == 1:
+            raise ValueError('No data found in the data file. Please check the Arm Energy Probe')
+            return
+
+        # display energy consumption of each channel and total energy consumption
+        total = 0
+        results_table = {}
+        for i in range(1, len(nrj)):
+            if hide[i]:
+                continue
+
+            nrj[i] -= offset[i] * nrj[0]
+
+            total_nrj = nrj[i]/1000000000000.0
+            duration = (max[0]-min[0])/1000000.0
+            channel_name = label[i]
+            average_power = total_nrj/duration
+
+            self.fs.write("Total nrj: %8.3f J for %s -- duration %8.3f sec -- min %8.3f W -- max %8.3f W\n" % (nrj[i]/1000000000000.0, label[i], (max[0]-min[0])/1000000.0, min[i]/1000000.0, max[i]/1000000.0))
+
+            # store each AEP channel info  except Platform in the results table
+            results_table[channel_name] = total_nrj, average_power
+
+            if (min[i] < offset[i]):
+                self.fs.write ("!!! Min below offset\n")
+
+            if duplicate[i]:
+                continue
+
+            total += nrj[i]
+
+        self.fs.write ("Total nrj: %8.3f J for %s -- duration %8.3f sec\n" % (total/1000000000000.0, "Platform ", (max[0]-min[0])/1000000.0))
+
+        total_nrj = total/1000000000000.0
+        duration = (max[0]-min[0])/1000000.0
+        average_power = total_nrj/duration
+
+        # store AEP Platform channel info in the results table
+        results_table["Platform"] = total_nrj, average_power
+
+        return results_table
+
+    def topology_from_config(self, topofile):
+        try:
+            ft = open(topofile, "r")
+        except IOError:
+            logger.warn('Unable to open config file {}'.format(topofile))
+            return
+        lines = ft.readlines()
+
+        topo = {}
+        virtual = {}
+        name = ""
+        offset = 0
+        index = 0
+        #parse config file
+        for myline in lines:
+            if myline.startswith("#"):
+                # skip comment
+                continue
+
+            if myline == "\n":
+                # skip empty line
+                continue
+
+            if name == "":
+                # 1st valid line is the config's name
+                name = myline
+                continue
+
+            if not myline.startswith((' ', '\t')):
+                # new device path
+                offset = index
+                continue
+
+            # Get parameters of channel configuration
+            items = myline.split()
+
+            info = {}
+            info['name'] = items[0]
+            info['parent'] = items[9]
+            info['pretty'] = items[8]
+            info['index'] = int(items[2])+offset
+
+            # Add channel
+            topo[items[0]] = info
+
+            # Increase index
+            index +=1
+
+
+        # Create an entry for each virtual parent
+        for supply in topo.iterkeys():
+            # Parent is in the topology
+            parent = topo[supply]['parent']
+            if parent in topo:
+                continue
+
+            if parent not in virtual:
+                virtual[parent] = { supply : topo[supply]['index'] }
+
+            virtual[parent][supply] = topo[supply]['index']
+
+
+        # Remove parent with 1 child as they don't give more information than their
+        # child
+        for supply in virtual.keys():
+            if len(virtual[supply]) == 1:
+                del virtual[supply];
+
+        topo_list = ['']*(1+len(topo)+len(virtual))
+        topo_list[0] = 'time'
+        for chnl in topo.iterkeys():
+            topo_list[topo[chnl]['index']] = chnl
+        for chnl in virtual.iterkeys():
+            index +=1
+            topo_list[index] = chnl
+
+        ft.close()
+
+        return topo_list
+
+    def __del__(self):
+        self.unprepare()
+
+if __name__ == '__main__':
+
+    def handleSigTERM(signum, frame):
+        sys.exit(2)
+
+    signal.signal(signal.SIGTERM, handleSigTERM)
+    signal.signal(signal.SIGINT, handleSigTERM)
+
+    logger.setLevel(logging.WARN)
+    ch = logging.StreamHandler()
+    ch.setLevel(logging.DEBUG)
+    logger.addHandler(ch)
+
+    infile = ""
+    outfile = ""
+    figurefile = ""
+    start = 0
+    lenght = -1
+
+    try:
+        opts, args = getopt.getopt(sys.argv[1:], "i:vo:s:l:t:")
+    except getopt.GetoptError as err:
+        print str(err) # will print something like "option -a not recognized"
+        sys.exit(2)
+
+    for o, a in opts:
+        if o == "-i":
+            infile = a
+        if o == "-v":
+            logger.setLevel(logging.DEBUG)
+        if o == "-o":
+            parse = True
+            outfile = a
+        if o == "-s":
+            start = int(float(a)*1000000)
+        if o == "-l":
+            lenght = int(float(a)*1000000)
+        if o == "-t":
+            topofile = a
+            parser = AepParser()
+            print parser.topology_from_config(topofile)
+            exit(0)
+
+    parser = AepParser()
+    parser.prepare(infile, outfile, figurefile)
+    parser.parse_aep(start, lenght)