Merge pull request #181 from setrofim/master

Module initialization optimizations.

devlib/bin/scripts/shutils.in

@@ -195,6 +195,35 @@ cgroups_freezer_set_state() {
     exit 1
 }
 
+################################################################################
+# Hotplug
+################################################################################
+
+hotplug_online_all() {
+    PATHS=(/sys/devices/system/cpu/cpu[0-9]*)
+    for path in "${PATHS[@]}"; do
+        if [ $(cat $path/online) -eq 0 ]; then
+            echo 1 > $path/online
+        fi
+    done
+}
+
+################################################################################
+# Misc
+################################################################################
+
+read_tree_values() {
+    PATH=$1
+    MAXDEPTH=$2
+
+    PATHS=$($BUSYBOX find $PATH -follow -maxdepth $MAXDEPTH)
+    if [ ${#PATHS[@]} -eq 0 ]; then
+        echo "ERROR: '$1' does not exist"
+    else
+        $BUSYBOX grep -s '' $PATHS
+    fi
+}
+
 ################################################################################
 # Main Function Dispatcher
 ################################################################################
@@ -236,6 +265,12 @@ cgroups_freezer_set_state)
 ftrace_get_function_stats)
     ftrace_get_function_stats
     ;;
+hotplug_online_all)
+	hotplug_online_all
+    ;;
+read_tree_values)
+	read_tree_values $*
+    ;;
 *)
     echo "Command [$CMD] not supported"
     exit -1

devlib/module/__init__.py

@@ -56,7 +56,7 @@ class Module(object):
 
     def __init__(self, target):
         self.target = target
-        self.logger = logging.getLogger(self.__class__.__name__)
+        self.logger = logging.getLogger(self.name)
 
 
 class HardRestModule(Module):  # pylint: disable=R0921

devlib/module/cpuidle.py

@@ -41,51 +41,17 @@ class CpuidleState(object):
                 raise ValueError('invalid idle state name: "{}"'.format(self.id))
         return int(self.id[i:])
 
-    def __init__(self, target, index, path):
+    def __init__(self, target, index, path, name, desc, power, latency, residency):
         self.target = target
         self.index = index
         self.path = path
+        self.name = name
+        self.desc = desc
+        self.power = power
+        self.latency = latency
         self.id = self.target.path.basename(self.path)
         self.cpu = self.target.path.basename(self.target.path.dirname(path))
 
-    @property
-    @memoized
-    def desc(self):
-        return self.get('desc')
-
-    @property
-    @memoized
-    def name(self):
-        return self.get('name')
-
-    @property
-    @memoized
-    def latency(self):
-        """Exit latency in uS"""
-        return self.get('latency')
-
-    @property
-    @memoized
-    def power(self):
-        """Power usage in mW
-
-        ..note::
-
-            This value is not always populated by the kernel and may be garbage.
-        """
-        return self.get('power')
-
-    @property
-    @memoized
-    def target_residency(self):
-        """Target residency in uS
-
-        This is the amount of time in the state required to 'break even' on
-        power - the system should avoid entering the state for less time than
-        this.
-        """
-        return self.get('residency')
-
     def enable(self):
         self.set('disable', 0)
 
@@ -126,23 +92,47 @@ class Cpuidle(Module):
     def probe(target):
         return target.file_exists(Cpuidle.root_path)
 
-    def get_driver(self):
+    def __init__(self, target):
-        return self.target.read_value(self.target.path.join(self.root_path, 'current_driver'))
+        super(Cpuidle, self).__init__(target)
+        self._states = {}
 
-    def get_governor(self):
+        basepath = '/sys/devices/system/cpu/'
-        return self.target.read_value(self.target.path.join(self.root_path, 'current_governor_ro'))
+        values_tree = self.target.read_tree_values(basepath, depth=4, check_exit_code=False)
+        i = 0
+        cpu_id = 'cpu{}'.format(i)
+        while cpu_id in values_tree:
+            cpu_node = values_tree[cpu_id]
+
+            if 'cpuidle' in cpu_node:
+                idle_node = cpu_node['cpuidle']
+                self._states[cpu_id] = []
+                j = 0
+                state_id = 'state{}'.format(j)
+                while state_id in idle_node:
+                    state_node = idle_node[state_id]
+                    state = CpuidleState(
+                        self.target,
+                        index=j,
+                        path=self.target.path.join(basepath, cpu_id, 'cpuidle', state_id),
+                        name=state_node['name'],
+                        desc=state_node['desc'],
+                        power=int(state_node['power']),
+                        latency=int(state_node['latency']),
+                        residency=int(state_node['residency']) if 'residency' in state_node else None,
+                    )
+                    msg = 'Adding {} state {}: {} {}'
+                    self.logger.debug(msg.format(cpu_id, j, state.name, state.desc))
+                    self._states[cpu_id].append(state)
+                    j += 1
+                    state_id = 'state{}'.format(j)
+
+            i += 1
+            cpu_id = 'cpu{}'.format(i)
 
-    @memoized
     def get_states(self, cpu=0):
         if isinstance(cpu, int):
             cpu = 'cpu{}'.format(cpu)
-        states_dir = self.target.path.join(self.target.path.dirname(self.root_path), cpu, 'cpuidle')
+        return self._states.get(cpu)
-        idle_states = []
-        for state in self.target.list_directory(states_dir):
-            if state.startswith('state'):
-                index = int(state[5:])
-                idle_states.append(CpuidleState(self.target, index, self.target.path.join(states_dir, state)))
-        return idle_states
 
     def get_state(self, state, cpu=0):
         if isinstance(state, int):
@@ -176,3 +166,9 @@ class Cpuidle(Module):
         """
         output = self.target._execute_util('cpuidle_wake_all_cpus')
         print(output)
+
+    def get_driver(self):
+        return self.target.read_value(self.target.path.join(self.root_path, 'current_driver'))
+
+    def get_governor(self):
+        return self.target.read_value(self.target.path.join(self.root_path, 'current_governor_ro'))

devlib/module/hotplug.py

@@ -21,7 +21,8 @@ class HotplugModule(Module):
         return target.path.join(cls.base_path, cpu, 'online')
 
     def online_all(self):
-        self.online(*range(self.target.number_of_cpus))
+        self.target._execute_util('hotplug_online_all',
+                                  as_root=self.target.is_rooted)
 
     def online(self, *args):
         for cpu in args:

devlib/module/hwmon.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import os
 import re
 from collections import defaultdict
 
@@ -78,16 +79,15 @@ class HwmonDevice(object):
             all_sensors.extend(sensors_of_kind.values())
         return all_sensors
 
-    def __init__(self, target, path):
+    def __init__(self, target, path, name, fields):
         self.target = target
         self.path = path
-        self.name = self.target.read_value(self.target.path.join(self.path, 'name'))
+        self.name = name
         self._sensors = defaultdict(dict)
         path = self.path
         if not path.endswith(self.target.path.sep):
             path += self.target.path.sep
-        for entry in self.target.list_directory(path,
+        for entry in fields:
-                                                as_root=self.target.is_rooted):
             match = HWMON_FILE_REGEX.search(entry)
             if match:
                 kind = match.group('kind')
@@ -117,14 +117,11 @@ class HwmonModule(Module):
 
     @staticmethod
     def probe(target):
-        if not target.file_exists(HWMON_ROOT):
-            return False
         try:
             target.list_directory(HWMON_ROOT, as_root=target.is_rooted)
         except TargetError:
-            # Probably no permissions
+            # Doesn't exist or no permissions
             return False
-
         return True
 
     @property
@@ -141,11 +138,13 @@ class HwmonModule(Module):
         self.scan()
 
     def scan(self):
-        for entry in self.target.list_directory(self.root,
+        values_tree = self.target.read_tree_values(self.root, depth=3)
-                                                as_root=self.target.is_rooted):
+        for entry_id, fields in values_tree.iteritems():
-            if entry.startswith('hwmon'):
+            path = self.target.path.join(self.root, entry_id)
-                entry_path = self.target.path.join(self.root, entry)
+            name = fields.pop('name', None)
-                if self.target.file_exists(self.target.path.join(entry_path, 'name')):
+            if name is None:
-                    device = HwmonDevice(self.target, entry_path)
+                continue
-                    self.devices.append(device)
+            self.logger.debug('Adding device {}'.format(name))
+            device = HwmonDevice(self.target, path, name, fields)
+            self.devices.append(device)
 

devlib/target.py

@@ -149,6 +149,12 @@ class Target(object):
         else:
             return None
 
+    @property
+    def shutils(self):
+        if self._shutils is None:
+            self._setup_shutils()
+        return self._shutils
+
     def __init__(self,
                  connection_settings=None,
                  platform=None,
@@ -189,6 +195,7 @@ class Target(object):
         self._installed_modules = {}
         self._cache = {}
         self._connections = {}
+        self._shutils = None
         self.busybox = None
 
         if load_default_modules:
@@ -229,20 +236,7 @@ class Target(object):
         self.execute('mkdir -p {}'.format(self.executables_directory))
         self.busybox = self.install(os.path.join(PACKAGE_BIN_DIRECTORY, self.abi, 'busybox'))
 
-        # Setup shutils script for the target
+        self._setup_shutils()
-        shutils_ifile = os.path.join(PACKAGE_BIN_DIRECTORY, 'scripts', 'shutils.in')
-        shutils_ofile = os.path.join(PACKAGE_BIN_DIRECTORY, 'scripts', 'shutils')
-        shell_path = '/bin/sh'
-        if self.os == 'android':
-            shell_path = '/system/bin/sh'
-        with open(shutils_ifile) as fh:
-            lines = fh.readlines()
-        with open(shutils_ofile, 'w') as ofile:
-            for line in lines:
-                line = line.replace("__DEVLIB_SHELL__", shell_path)
-                line = line.replace("__DEVLIB_BUSYBOX__", self.busybox)
-                ofile.write(line)
-        self.shutils = self.install(os.path.join(PACKAGE_BIN_DIRECTORY, 'scripts', 'shutils'))
 
         for host_exe in (executables or []):  # pylint: disable=superfluous-parens
             self.install(host_exe)
@@ -620,8 +614,37 @@ class Target(object):
         timeout = duration + 10
         self.execute('sleep {}'.format(duration), timeout=timeout)
 
+    def read_tree_values_flat(self, path, depth=1, check_exit_code=True):
+        command = 'read_tree_values {} {}'.format(path, depth)
+        output = self._execute_util(command, as_root=self.is_rooted,
+                                    check_exit_code=check_exit_code)
+        result = {}
+        for entry in output.strip().split('\n'):
+            path, value = entry.strip().split(':', 1)
+            result[path] = value
+        return result
+
+    def read_tree_values(self, path, depth=1, dictcls=dict, check_exit_code=True):
+	value_map = self.read_tree_values_flat(path, depth, check_exit_code)
+	return _build_path_tree(value_map, path, self.path.sep, dictcls)
+
     # internal methods
 
+    def _setup_shutils(self):
+        shutils_ifile = os.path.join(PACKAGE_BIN_DIRECTORY, 'scripts', 'shutils.in')
+        shutils_ofile = os.path.join(PACKAGE_BIN_DIRECTORY, 'scripts', 'shutils')
+        shell_path = '/bin/sh'
+        if self.os == 'android':
+            shell_path = '/system/bin/sh'
+        with open(shutils_ifile) as fh:
+            lines = fh.readlines()
+        with open(shutils_ofile, 'w') as ofile:
+            for line in lines:
+                line = line.replace("__DEVLIB_SHELL__", shell_path)
+                line = line.replace("__DEVLIB_BUSYBOX__", self.busybox)
+                ofile.write(line)
+        self._shutils = self.install(os.path.join(PACKAGE_BIN_DIRECTORY, 'scripts', 'shutils'))
+
     def _execute_util(self, command, timeout=None, check_exit_code=True, as_root=False):
         command = '{} {}'.format(self.shutils, command)
         return self.conn.execute(command, timeout, check_exit_code, as_root)
@@ -1549,3 +1572,32 @@ def _get_part_name(section):
     if name is None:
         name = '{}/{}/{}'.format(implementer, part, variant)
     return name
+
+
+def _build_path_tree(path_map, basepath, sep=os.path.sep, dictcls=dict):
+    """
+    Convert a flat mapping of paths to values into a nested structure of
+    dict-line object (``dict``'s by default), mirroring the directory hierarchy
+    represented by the paths relative to ``basepath``.
+
+    """
+    def process_node(node, path, value):
+        parts = path.split(sep, 1)
+        if len(parts) == 1:   # leaf
+            node[parts[0]] = value
+        else:  # branch
+            if parts[0] not in node:
+                node[parts[0]] = dictcls()
+            process_node(node[parts[0]], parts[1], value)
+
+    relpath_map = {os.path.relpath(p, basepath): v
+                   for p, v in path_map.iteritems()}
+
+    if len(relpath_map) == 1 and relpath_map.keys()[0] == '.':
+        result = relpath_map.values()[0]
+    else:
+        result = dictcls()
+        for path, value in relpath_map.iteritems():
+            process_node(result, path, value)
+
+    return result

doc/target.rst

@@ -327,6 +327,32 @@ Target
        some sysfs entries silently failing to set the written value without
        returning an error code.
 
+.. method:: Target.read_tree_values(path, depth=1, dictcls=dict):
+
+   Read values of all sysfs (or similar) file nodes under ``path``, traversing
+   up to the maximum depth ``depth``.
+
+   Returns a nested structure of dict-like objects (``dict``\ s by default) that
+   follows the structure of the scanned sub-directory tree. The top-level entry
+   has a single item who's key is ``path``. If ``path`` points to a single file,
+   the value of the entry is the value ready from that file node. Otherwise, the
+   value is a dict-line object  with a key for every entry under ``path``
+   mapping onto its value or further dict-like objects as appropriate.
+
+   :param path: sysfs path to scan
+   :param depth: maximum depth to descend
+   :param dictcls: a dict-like type to be used for each level of the hierarchy.
+
+.. method:: Target.read_tree_values_flat(path, depth=1):
+
+   Read values of all sysfs (or similar) file nodes under ``path``, traversing
+   up to the maximum depth ``depth``.
+
+   Returns a dict mapping paths of file nodes to corresponding values.
+
+   :param path: sysfs path to scan
+   :param depth: maximum depth to descend
+
 .. method:: Target.reset()
 
    Soft reset the target. Typically, this means executing ``reboot`` on the