BaseLinuxDevice: Tidied up the way binaries are handled

Added:
get_binary_path: Checks binary_directory for the wanted binary, if
                 if its not there, it will use which to find a
                 system one. returns the full path

install_if_needed: will install a binary only if it is not present.

Changes:
 - Busybox is now deployed to non-rooted devices
 - is_installed has now been removed as the new functions supersede it
 - binaries will now always be installed to `binaries_directory` and
   not system folders.
 - updated workloads to use these new functions
   - rt-app and sysbench might still need work

wlauto/common/android/device.py

@@ -51,7 +51,7 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
                   description='The format  of matching the shell prompt in Android.'),
         Parameter('working_directory', default='/sdcard/wa-working',
                   description='Directory that will be used WA on the device for output files etc.'),
-        Parameter('binaries_directory', default='/system/bin',
+        Parameter('binaries_directory', default='/data/local/tmp', override=True,
                   description='Location of binaries on the device.'),
         Parameter('package_data_directory', default='/data/data',
                   description='Location of of data for an installed package (APK).'),
@@ -78,6 +78,8 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
                   If set a swipe of the specified direction will be performed.
                   This should unlock the screen.
                   """),
+        Parameter('binaries_directory', default="/data/local/tmp", override=True,
+                  description='Location of executable binaries on this device (must be in PATH).'),
     ]
 
     default_timeout = 30
@@ -193,9 +195,7 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
         self._is_ready = True
 
     def initialize(self, context):
-        self.execute('mkdir -p {}'.format(self.working_directory))
         if self.is_rooted:
-            self.busybox = self.deploy_busybox(context)
             self.disable_screen_lock()
             self.disable_selinux()
         if self.enable_screen_check:
@@ -281,11 +281,24 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
         """
         return package_name in self.list_packages()
 
-    def executable_is_installed(self, executable_name):
-        return executable_name in self.listdir(self.binaries_directory)
+    def executable_is_installed(self, executable_name):  # pylint: disable=unused-argument,no-self-use
+        raise AttributeError("""Instead of using is_installed, please use
+            ``get_binary_path`` or ``install_if_needed`` instead. You should
+            use the path returned by these functions to then invoke the binary
+
+            please see: https://pythonhosted.org/wlauto/writing_extensions.html""")
 
     def is_installed(self, name):
-        return self.executable_is_installed(name) or self.package_is_installed(name)
+        if self.package_is_installed(name):
+            return True
+        elif "." in name:  # assumes android packages have a . in their name and binaries documentation
+            return False
+        else:
+            raise AttributeError("""Instead of using is_installed, please use
+                ``get_binary_path`` or ``install_if_needed`` instead. You should
+                use the path returned by these functions to then invoke the binary
+
+                please see: https://pythonhosted.org/wlauto/writing_extensions.html""")
 
     def listdir(self, path, as_root=False, **kwargs):
         contents = self.execute('ls {}'.format(path), as_root=as_root)
@@ -352,7 +365,7 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
 
     def install_executable(self, filepath, with_name=None):
         """
-        Installs a binary executable on device. Requires root access. Returns
+        Installs a binary executable on device. Returns
         the path to the installed binary, or ``None`` if the installation has failed.
         Optionally, ``with_name`` parameter may be used to specify a different name under
         which the executable will be installed.
@@ -376,12 +389,13 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
 
     def uninstall_executable(self, executable_name):
         """
-        Requires root access.
 
         Added in version 2.1.3.
 
         """
-        on_device_executable = self.path.join(self.binaries_directory, executable_name)
+        on_device_executable = self.get_binary_path(executable_name, search_system_binaries=False)
+        if not on_device_executable:
+            raise DeviceError("Could not uninstall {}, binary not found".format(on_device_executable))
         self._ensure_binaries_directory_is_writable()
         self.delete_file(on_device_executable, as_root=self.is_rooted)
 
@@ -405,7 +419,7 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
 
                             Added in version 2.1.3
 
-                            .. note:: The device must be rooted to be able to use busybox.
+                            .. note:: The device must be rooted to be able to use some busybox features.
 
             :param as_root: If ``True``, will attempt to execute command in privileged mode. The device
                             must be rooted, otherwise an error will be raised. Defaults to ``False``.
@@ -424,9 +438,6 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
         if as_root and not self.is_rooted:
             raise DeviceError('Attempting to execute "{}" as root on unrooted device.'.format(command))
         if busybox:
-            if not self.is_rooted:
-                DeviceError('Attempting to execute "{}" with busybox. '.format(command) +
-                            'Busybox can only be deployed to rooted devices.')
             command = ' '.join([self.busybox, command])
         if background:
             return adb_background_shell(self.adb_name, command, as_root=as_root)

wlauto/common/linux/device.py

@@ -89,6 +89,8 @@ class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
                   These paths do not have to exist and will be ignored if the path is not present on a
                   particular device.
                   '''),
+        Parameter('binaries_directory',
+                  description='Location of executable binaries on this device (must be in PATH).'),
 
     ]
 
@@ -183,11 +185,14 @@ class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
 
     def initialize(self, context):
         self.execute('mkdir -p {}'.format(self.working_directory))
-        if self.is_rooted:
-            if not self.is_installed('busybox'):
-                self.busybox = self.deploy_busybox(context)
-            else:
-                self.busybox = 'busybox'
+        if not self.binaries_directory:
+            self._set_binaries_dir()
+        self.execute('mkdir -p {}'.format(self.binaries_directory))
+        self.busybox = self.deploy_busybox(context)
+
+    def _set_binaries_dir(self):
+        # pylint: disable=attribute-defined-outside-init
+        self.binaries_directory = self.path.join(self.working_directory, "bin")
 
     def is_file(self, filepath):
         output = self.execute('if [ -f \'{}\' ]; then echo 1; else echo 0; fi'.format(filepath))
@@ -281,7 +286,6 @@ class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
         Deploys the busybox binary to the specified device and returns
         the path to the binary on the device.
 
-        :param device: device to deploy the binary to.
         :param context: an instance of ExecutionContext
         :param force: by default, if the binary is already present on the
                     device, it will not be deployed again. Setting force
@@ -291,11 +295,61 @@ class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
         :returns: The on-device path to the busybox binary.
 
         """
-        on_device_executable = self.path.join(self.binaries_directory, 'busybox')
-        if not force and self.file_exists(on_device_executable):
-            return on_device_executable
-        host_file = context.resolver.get(Executable(NO_ONE, self.abi, 'busybox'))
-        return self.install(host_file)
+        on_device_executable = self.get_binary_path("busybox", search_system_binaries=False)
+        if force or not on_device_executable:
+            host_file = context.resolver.get(Executable(NO_ONE, self.abi, 'busybox'))
+            return self.install(host_file)
+        return on_device_executable
+
+    def is_installed(self, name):  # pylint: disable=unused-argument,no-self-use
+        raise AttributeError("""Instead of using is_installed, please use
+            ``get_binary_path`` or ``install_if_needed`` instead. You should
+            use the path returned by these functions to then invoke the binary
+
+            please see: https://pythonhosted.org/wlauto/writing_extensions.html""")
+
+    def get_binary_path(self, name, search_system_binaries=True):
+        """
+        Searches the devices ``binary_directory`` for the given binary,
+        if it cant find it there it tries using which to find it.
+
+        :param name: The name of the binary
+        :param search_system_binaries: By default this function will try using
+                                       which to find the binary if it isn't in
+                                       ``binary_directory``. When this is set
+                                       to ``False`` it will not try this.
+
+        :returns: The on-device path to the binary.
+
+        """
+        wa_binary = self.path.join(self.binaries_directory, name)
+        if self.file_exists(wa_binary):
+            return wa_binary
+        if search_system_binaries:
+            try:
+                return self.execute('{} which {}'.format(self.busybox, name)).strip()
+            except DeviceError:
+                pass
+        return None
+
+    def install_if_needed(self, host_path, search_system_binaries=True):
+        """
+        Similar to get_binary_path but will install the binary if not found.
+
+        :param host_path: The path to the binary on the host
+        :param search_system_binaries: By default this function will try using
+                                       which to find the binary if it isn't in
+                                       ``binary_directory``. When this is set
+                                       to ``False`` it will not try this.
+
+        :returns: The on-device path to the binary.
+
+        """
+        binary_path = self.get_binary_path(os.path.split(host_path)[1],
+                                           search_system_binaries=search_system_binaries)
+        if not binary_path:
+            binary_path = self.install(host_path)
+        return binary_path
 
     def list_file_systems(self):
         output = self.execute('mount')
@@ -527,8 +581,6 @@ class LinuxDevice(BaseLinuxDevice):
                   has write permissions. This will default to /home/<username>/wa (or to /root/wa, if
                   username is 'root').
                   '''),
-        Parameter('binaries_directory', default='/usr/local/bin',
-                  description='Location of executable binaries on this device (must be in PATH).'),
     ]
 
     @property
@@ -554,7 +606,6 @@ class LinuxDevice(BaseLinuxDevice):
     def __init__(self, *args, **kwargs):
         super(LinuxDevice, self).__init__(*args, **kwargs)
         self.shell = None
-        self.local_binaries_directory = None
         self._is_rooted = None
 
     def validate(self):
@@ -563,12 +614,9 @@ class LinuxDevice(BaseLinuxDevice):
                 self.working_directory = '/root/wa'  # pylint: disable=attribute-defined-outside-init
             else:
                 self.working_directory = '/home/{}/wa'.format(self.username)  # pylint: disable=attribute-defined-outside-init
-        self.local_binaries_directory = self.path.join(self.working_directory, 'bin')
 
     def initialize(self, context, *args, **kwargs):
-        self.execute('mkdir -p {}'.format(self.local_binaries_directory))
         self.execute('mkdir -p {}'.format(self.binaries_directory))
-        self.execute('export PATH={}:$PATH'.format(self.local_binaries_directory))
         self.execute('export PATH={}:$PATH'.format(self.binaries_directory))
         super(LinuxDevice, self).initialize(context, *args, **kwargs)
 
@@ -747,37 +795,22 @@ class LinuxDevice(BaseLinuxDevice):
         return [x.strip() for x in contents.split('\n')]  # pylint: disable=maybe-no-member
 
     def install(self, filepath, timeout=default_timeout, with_name=None):  # pylint: disable=W0221
-        if self.is_rooted:
-            destpath = self.path.join(self.binaries_directory,
-                                      with_name and with_name or self.path.basename(filepath))
-            self.push_file(filepath, destpath, as_root=True)
-            self.execute('chmod a+x {}'.format(destpath), timeout=timeout, as_root=True)
-        else:
-            destpath = self.path.join(self.local_binaries_directory,
-                                      with_name and with_name or self.path.basename(filepath))
-            self.push_file(filepath, destpath)
-            self.execute('chmod a+x {}'.format(destpath), timeout=timeout)
+        destpath = self.path.join(self.binaries_directory,
+                                  with_name or self.path.basename(filepath))
+        self.push_file(filepath, destpath, as_root=True)
+        self.execute('chmod a+x {}'.format(destpath), timeout=timeout, as_root=True)
         return destpath
 
     install_executable = install  # compatibility
 
-    def uninstall(self, name):
-        if self.is_rooted:
-            path = self.path.join(self.binaries_directory, name)
-            self.delete_file(path, as_root=True)
-        else:
-            path = self.path.join(self.local_binaries_directory, name)
-            self.delete_file(path)
+    def uninstall(self, executable_name):
+        on_device_executable = self.get_binary_path(executable_name, search_system_binaries=False)
+        if not on_device_executable:
+            raise DeviceError("Could not uninstall {}, binary not found".format(on_device_executable))
+        self.delete_file(on_device_executable, as_root=self.is_rooted)
 
     uninstall_executable = uninstall  # compatibility
 
-    def is_installed(self, name):
-        try:
-            self.execute('which {}'.format(name))
-            return True
-        except DeviceError:
-            return False
-
     # misc
 
     def ping(self):
@@ -788,7 +821,7 @@ class LinuxDevice(BaseLinuxDevice):
             raise DeviceNotRespondingError(self.host)
 
     def capture_screen(self, filepath):
-        if not self.is_installed('scrot'):
+        if not self.get_binary_path('scrot'):
             self.logger.debug('Could not take screenshot as scrot is not installed.')
             return
         try:

wlauto/instrumentation/perf/__init__.py

@@ -91,11 +91,11 @@ class PerfInstrument(Instrument):
     ]
 
     def on_run_init(self, context):
-        if not self.device.is_installed('perf') or self.force_install:
-            binary = context.resolver.get(Executable(self, self.device.abi, 'perf'))
+        binary = context.resolver.get(Executable(self, self.device.abi, 'perf'))
+        if self.force_install:
             self.binary = self.device.install(binary)
         else:
-            self.binary = 'perf'
+            self.binary = self.device.install_if_needed(binary)
         self.commands = self._build_commands()
 
     def setup(self, context):

wlauto/instrumentation/trace_cmd/__init__.py

@@ -164,11 +164,12 @@ class TraceCmdInstrument(Instrument):
             raise InstrumentError('trace-cmd instrument cannot be used on an unrooted device.')
         if not self.no_install:
             host_file = context.resolver.get(Executable(self, self.device.abi, 'trace-cmd'))
-            self.trace_cmd = self.device.install_executable(host_file)
+            self.trace_cmd = self.device.install(host_file)
         else:
-            if not self.device.is_installed('trace-cmd'):
+            self.trace_cmd = self.device.get_binary_path("trace-cmd")
+            if not self.trace_cmd:
                 raise ConfigError('No trace-cmd found on device and no_install=True is specified.')
-            self.trace_cmd = 'trace-cmd'
+
         # Register ourselves as absolute last event before and
         #   first after so we can mark the trace at the right time
         signal.connect(self.insert_start_mark, signal.BEFORE_WORKLOAD_EXECUTION, priority=11)

wlauto/workloads/cyclictest/__init__.py

@@ -79,11 +79,8 @@ class Cyclictest(Workload):
         if not self.device.is_rooted:
             raise WorkloadError("This workload requires a device with root premissions to run")
 
-        if not self.device.is_installed('cyclictest'):
-            host_binary = context.resolver.get(Executable(self, self.device.abi, 'cyclictest'))
-            self.device_binary = self.device.install(host_binary)
-        else:
-            self.device_binary = 'cyclictest'
+        host_binary = context.resolver.get(Executable(self, self.device.abi, 'cyclictest'))
+        self.device_binary = self.device.install(host_binary)
 
         self.cyclictest_command = self.cyclictest_command.format(self.device_binary,
                                                                  0 if self.clock == 'monotonic' else 1,

wlauto/workloads/ebizzy/__init__.py

@@ -57,11 +57,8 @@ class Ebizzy(Workload):
         self.run_timeout = self.seconds + timeout_buf
 
         self.binary_name = 'ebizzy'
-        if not self.device.is_installed(self.binary_name):
-            host_binary = context.resolver.get(Executable(self, self.device.abi, self.binary_name))
-            self.device_binary = self.device.install(host_binary)
-        else:
-            self.device_binary = self.binary_name
+        host_binary = context.resolver.get(Executable(self, self.device.abi, self.binary_name))
+        self.device_binary = self.device.install_if_needed(host_binary)
 
         self.command = self.command.format(self.device_binary, self.threads, self.seconds,
                                            self.chunks, self.extra_params, self.ebizzy_results)

wlauto/workloads/hackbench/__init__.py

@@ -61,11 +61,8 @@ class Hackbench(Workload):
         self.run_timeout = self.duration + timeout_buf
 
         self.binary_name = 'hackbench'
-        if not self.device.is_installed(self.binary_name):
-            host_binary = context.resolver.get(Executable(self, self.device.abi, self.binary_name))
-            self.device_binary = self.device.install(host_binary)
-        else:
-            self.device_binary = self.binary_name
+        host_binary = context.resolver.get(Executable(self, self.device.abi, self.binary_name))
+        self.device_binary = self.device.install(host_binary)
 
         self.command = self.command.format(self.device_binary, self.datasize, self.groups,
                                            self.loops, self.extra_params, self.hackbench_result)

wlauto/workloads/memcpy/__init__.py

@@ -18,7 +18,7 @@
 import os
 import re
 
-from wlauto import Workload, Parameter
+from wlauto import Workload, Parameter, Executable
 
 
 THIS_DIR = os.path.dirname(__file__)
@@ -54,11 +54,10 @@ class MemcpyTest(Workload):
     ]
 
     def setup(self, context):
-        self.host_binary = os.path.join(THIS_DIR, 'memcpy')
-        if not self.device.is_installed('memcpy'):
-            self.device_binary = self.device.install(self.host_binary)
-        else:
-            self.device_binary = 'memcpy'
+        self.binary_name = 'memcpy'
+        host_binary = context.resolver.get(Executable(self, self.device.abi, self.binary_name))
+        self.device_binary = self.device.install_if_needed(host_binary)
+
         self.command = '{} -i {} -s {}'.format(self.device_binary, self.iterations, self.buffer_size)
         if self.cpus:
             for c in self.cpus:

wlauto/workloads/rt_app/__init__.py

@@ -216,14 +216,13 @@ class RtApp(Workload):
 
     def _deploy_rt_app_binary_if_necessary(self):
         # called from initialize() so gets invoked once per run
-        if self.force_install or not self.device.is_installed(BINARY_NAME):
+        RtApp.device_binary = self.device.get_binary_path("rt-app")
+        if self.force_install or not RtApp.device_binary:
             if not self.host_binary:
                 message = '''rt-app is not installed on the device and could not be
                              found in workload resources'''
                 raise ResourceError(message)
             RtApp.device_binary = self.device.install(self.host_binary)
-        else:
-            RtApp.device_binary = BINARY_NAME
 
     def _load_json_config(self, context):
         user_config_file = self._get_raw_json_config(context.resolver)
@@ -280,4 +279,3 @@ class RtApp(Workload):
             tf.extractall(context.output_directory)
         os.remove(host_path)
         self.device.execute('rm -rf {}/*'.format(self.device_working_directory))
-

wlauto/workloads/sysbench/__init__.py

@@ -132,13 +132,13 @@ class Sysbench(Workload):
         self.device.delete_file(self.results_file)
 
     def _check_executable(self):
-        self.on_device_binary = self.device.path.join(self.device.binaries_directory, 'sysbench')
-        if self.device.is_installed('sysbench') and not self.force_install:
-            self.logger.debug('sysbench found on device')
-            return
-        if not self.on_host_binary:
+        self.on_device_binary = self.device.get_binary_path("sysbench")
+        if not self.on_device_binary and not self.on_host_binary:
             raise WorkloadError('sysbench binary is not installed on the device, and it is not found on the host.')
-        self.device.install(self.on_host_binary)
+        if self.force_install:
+            self.device.install(self.on_host_binary)
+        else:
+            self.device.install_if_needed(self.on_host_binary)
 
     def _build_command(self, **parameters):
         param_strings = ['--{}={}'.format(k.replace('_', '-'), v)