sascha/workload-automation
1
0
Fork 0
mirror of https://github.com/ARM-software/workload-automation.git synced 2025-09-02 11:22:41 +01:00
Code Issues Releases Wiki Activity

Merge pull request #78 from ep1cman/binary_install

Browse Source 
BaseLinuxDevice: Tidied up the way binaries are handled
This commit is contained in:
setrofim
2016-01-19 10:52:54 +00:00
parent c11cc7d0d2 f47ba6fea6
commit 120f0ff94f
15 changed files with 218 additions and 130 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
doc/source/resources.rst
View File

@@ -1,3 +1,5 @@
.. _resources:
Dynamic Resource Resolution
===========================

53
doc/source/writing_extensions.rst
View File

@@ -132,12 +132,63 @@ device, the ``os.path`` modules should *not* be used for on-device path
manipulation. Instead device has an equipment module exposed through
``device.path`` attribute. This has all the same attributes and behaves the
same way as ``os.path``, but is guaranteed to produce valid paths for the device,
irrespective of the host's path notation.
irrespective of the host's path notation. For example:
.. code:: python
result_file = self.device.path.join(self.device.working_directory, "result.txt")
self.command = "{} -a -b -c {}".format(target_binary, result_file)
.. note:: result processors, unlike workloads and instruments, do not have their
own device attribute; however they can access the device through the
context.
Deploying executables to a device
---------------------------------
Some devices may have certain restrictions on where executable binaries may be
placed and how they should be invoked. To ensure your extension works with as
wide a range of devices as possible, you should use WA APIs for deploying and
invoking executables on a device, as outlined below.
As with other resources (see :ref:`resources`) , host-side paths to the exectuable
binary to be deployed should be obtained via the resource resolver. A special
resource type, ``Executable`` is used to identify a binary to be deployed.
This is simiar to the regular ``File`` resource, however it takes an additional
parameter that specifies the ABI for which executable was compiled.
In order for the binary to be obtained in this way, it must be stored in one of
the locations scanned by the resource resolver in a directry structure
``<root>/bin/<abi>/<binary>`` (where ``root`` is the base resource location to
be searched, e.g. ``~/.workload_automation/depencencies/<extension name>``, and
``<abi>`` is the ABI for which the exectuable has been compiled, as returned by
``self.device.abi``).
Once the path to the host-side binary has been obtained, it may be deployed using
one of two methods of a ``Device`` instace -- ``install`` or ``install_if_needed``.
The latter will check a version of that binary has been perviously deployed by
WA and will not try to re-install.
.. code:: python
from wlauto import Executable
host_binary = context.resolver.get(Executable(self, self.device.abi, 'some_binary'))
target_binary = self.device.install_if_needed(host_binary)
.. note:: Please also note that the check is done based solely on the binary name.
For more information please see: :func:`wlauto.common.linux.BaseLinuxDevice.install_if_needed`
Both of the above methods will return the path to the installed binary on the
device. The executable should be invoked *only* via that path; do **not** assume
that it will be in ``PATH`` on the target (or that the executable with the same
name in ``PATH`` is the version deployed by WA.
.. code:: python
self.command = "{} -a -b -c".format(target_binary)
self.device.execute(self.command)
Parameters
----------

37
wlauto/common/android/device.py
View File

@@ -51,7 +51,7 @@ class AndroidDevice(BaseLinuxDevice): # pylint: disable=W0223
description='The format of matching the shell prompt in Android.'),
Parameter('working_directory', default='/sdcard/wa-working',
description='Directory that will be used WA on the device for output files etc.'),
Parameter('binaries_directory', default='/system/bin',
Parameter('binaries_directory', default='/data/local/tmp', override=True,
description='Location of binaries on the device.'),
Parameter('package_data_directory', default='/data/data',
description='Location of of data for an installed package (APK).'),
@@ -78,6 +78,8 @@ class AndroidDevice(BaseLinuxDevice): # pylint: disable=W0223
If set a swipe of the specified direction will be performed.
This should unlock the screen.
"""),
Parameter('binaries_directory', default="/data/local/tmp", override=True,
description='Location of executable binaries on this device (must be in PATH).'),
]
default_timeout = 30
@@ -193,9 +195,7 @@ class AndroidDevice(BaseLinuxDevice): # pylint: disable=W0223
self._is_ready = True
def initialize(self, context):
self.execute('mkdir -p {}'.format(self.working_directory))
if self.is_rooted:
self.busybox = self.deploy_busybox(context)
self.disable_screen_lock()
self.disable_selinux()
if self.enable_screen_check:
@@ -281,11 +281,24 @@ class AndroidDevice(BaseLinuxDevice): # pylint: disable=W0223
"""
return package_name in self.list_packages()
def executable_is_installed(self, executable_name):
return executable_name in self.listdir(self.binaries_directory)
def executable_is_installed(self, executable_name): # pylint: disable=unused-argument,no-self-use
raise AttributeError("""Instead of using is_installed, please use
``get_binary_path`` or ``install_if_needed`` instead. You should
use the path returned by these functions to then invoke the binary
please see: https://pythonhosted.org/wlauto/writing_extensions.html""")
def is_installed(self, name):
return self.executable_is_installed(name) or self.package_is_installed(name)
if self.package_is_installed(name):
return True
elif "." in name: # assumes android packages have a . in their name and binaries documentation
return False
else:
raise AttributeError("""Instead of using is_installed, please use
``get_binary_path`` or ``install_if_needed`` instead. You should
use the path returned by these functions to then invoke the binary
please see: https://pythonhosted.org/wlauto/writing_extensions.html""")
def listdir(self, path, as_root=False, **kwargs):
contents = self.execute('ls {}'.format(path), as_root=as_root)
@@ -352,7 +365,7 @@ class AndroidDevice(BaseLinuxDevice): # pylint: disable=W0223
def install_executable(self, filepath, with_name=None):
"""
Installs a binary executable on device. Requires root access. Returns
Installs a binary executable on device. Returns
the path to the installed binary, or ``None`` if the installation has failed.
Optionally, ``with_name`` parameter may be used to specify a different name under
which the executable will be installed.
@@ -376,12 +389,13 @@ class AndroidDevice(BaseLinuxDevice): # pylint: disable=W0223
def uninstall_executable(self, executable_name):
"""
Requires root access.
Added in version 2.1.3.
"""
on_device_executable = self.path.join(self.binaries_directory, executable_name)
on_device_executable = self.get_binary_path(executable_name, search_system_binaries=False)
if not on_device_executable:
raise DeviceError("Could not uninstall {}, binary not found".format(on_device_executable))
self._ensure_binaries_directory_is_writable()
self.delete_file(on_device_executable, as_root=self.is_rooted)
@@ -405,7 +419,7 @@ class AndroidDevice(BaseLinuxDevice): # pylint: disable=W0223
Added in version 2.1.3
.. note:: The device must be rooted to be able to use busybox.
.. note:: The device must be rooted to be able to use some busybox features.
:param as_root: If ``True``, will attempt to execute command in privileged mode. The device
must be rooted, otherwise an error will be raised. Defaults to ``False``.
@@ -424,9 +438,6 @@ class AndroidDevice(BaseLinuxDevice): # pylint: disable=W0223
if as_root and not self.is_rooted:
raise DeviceError('Attempting to execute "{}" as root on unrooted device.'.format(command))
if busybox:
if not self.is_rooted:
DeviceError('Attempting to execute "{}" with busybox. '.format(command) +
'Busybox can only be deployed to rooted devices.')
command = ' '.join([self.busybox, command])
if background:
return adb_background_shell(self.adb_name, command, as_root=as_root)

117
wlauto/common/linux/device.py
View File

@@ -89,6 +89,8 @@ class BaseLinuxDevice(Device): # pylint: disable=abstract-method
These paths do not have to exist and will be ignored if the path is not present on a
particular device.
'''),
Parameter('binaries_directory',
description='Location of executable binaries on this device (must be in PATH).'),
]
@@ -183,11 +185,14 @@ class BaseLinuxDevice(Device): # pylint: disable=abstract-method
def initialize(self, context):
self.execute('mkdir -p {}'.format(self.working_directory))
if self.is_rooted:
if not self.is_installed('busybox'):
self.busybox = self.deploy_busybox(context)
else:
self.busybox = 'busybox'
if not self.binaries_directory:
self._set_binaries_dir()
self.execute('mkdir -p {}'.format(self.binaries_directory))
self.busybox = self.deploy_busybox(context)
def _set_binaries_dir(self):
# pylint: disable=attribute-defined-outside-init
self.binaries_directory = self.path.join(self.working_directory, "bin")
def is_file(self, filepath):
output = self.execute('if [ -f \'{}\' ]; then echo 1; else echo 0; fi'.format(filepath))
@@ -281,7 +286,6 @@ class BaseLinuxDevice(Device): # pylint: disable=abstract-method
Deploys the busybox binary to the specified device and returns
the path to the binary on the device.
:param device: device to deploy the binary to.
:param context: an instance of ExecutionContext
:param force: by default, if the binary is already present on the
device, it will not be deployed again. Setting force
@@ -291,11 +295,61 @@ class BaseLinuxDevice(Device): # pylint: disable=abstract-method
:returns: The on-device path to the busybox binary.
"""
on_device_executable = self.path.join(self.binaries_directory, 'busybox')
if not force and self.file_exists(on_device_executable):
return on_device_executable
host_file = context.resolver.get(Executable(NO_ONE, self.abi, 'busybox'))
return self.install(host_file)
on_device_executable = self.get_binary_path("busybox", search_system_binaries=False)
if force or not on_device_executable:
host_file = context.resolver.get(Executable(NO_ONE, self.abi, 'busybox'))
return self.install(host_file)
return on_device_executable
def is_installed(self, name): # pylint: disable=unused-argument,no-self-use
raise AttributeError("""Instead of using is_installed, please use
``get_binary_path`` or ``install_if_needed`` instead. You should
use the path returned by these functions to then invoke the binary
please see: https://pythonhosted.org/wlauto/writing_extensions.html""")
def get_binary_path(self, name, search_system_binaries=True):
"""
Searches the devices ``binary_directory`` for the given binary,
if it cant find it there it tries using which to find it.
:param name: The name of the binary
:param search_system_binaries: By default this function will try using
which to find the binary if it isn't in
``binary_directory``. When this is set
to ``False`` it will not try this.
:returns: The on-device path to the binary.
"""
wa_binary = self.path.join(self.binaries_directory, name)
if self.file_exists(wa_binary):
return wa_binary
if search_system_binaries:
try:
return self.execute('{} which {}'.format(self.busybox, name)).strip()
except DeviceError:
pass
return None
def install_if_needed(self, host_path, search_system_binaries=True):
"""
Similar to get_binary_path but will install the binary if not found.
:param host_path: The path to the binary on the host
:param search_system_binaries: By default this function will try using
which to find the binary if it isn't in
``binary_directory``. When this is set
to ``False`` it will not try this.
:returns: The on-device path to the binary.
"""
binary_path = self.get_binary_path(os.path.split(host_path)[1],
search_system_binaries=search_system_binaries)
if not binary_path:
binary_path = self.install(host_path)
return binary_path
def list_file_systems(self):
output = self.execute('mount')
@@ -527,8 +581,6 @@ class LinuxDevice(BaseLinuxDevice):
has write permissions. This will default to /home/<username>/wa (or to /root/wa, if
username is 'root').
'''),
Parameter('binaries_directory', default='/usr/local/bin',
description='Location of executable binaries on this device (must be in PATH).'),
]
@property
@@ -554,7 +606,6 @@ class LinuxDevice(BaseLinuxDevice):
def __init__(self, *args, **kwargs):
super(LinuxDevice, self).__init__(*args, **kwargs)
self.shell = None
self.local_binaries_directory = None
self._is_rooted = None
def validate(self):
@@ -563,12 +614,9 @@ class LinuxDevice(BaseLinuxDevice):
self.working_directory = '/root/wa' # pylint: disable=attribute-defined-outside-init
else:
self.working_directory = '/home/{}/wa'.format(self.username) # pylint: disable=attribute-defined-outside-init
self.local_binaries_directory = self.path.join(self.working_directory, 'bin')
def initialize(self, context, *args, **kwargs):
self.execute('mkdir -p {}'.format(self.local_binaries_directory))
self.execute('mkdir -p {}'.format(self.binaries_directory))
self.execute('export PATH={}:$PATH'.format(self.local_binaries_directory))
self.execute('export PATH={}:$PATH'.format(self.binaries_directory))
super(LinuxDevice, self).initialize(context, *args, **kwargs)
@@ -747,37 +795,22 @@ class LinuxDevice(BaseLinuxDevice):
return [x.strip() for x in contents.split('\n')] # pylint: disable=maybe-no-member
def install(self, filepath, timeout=default_timeout, with_name=None): # pylint: disable=W0221
if self.is_rooted:
destpath = self.path.join(self.binaries_directory,
with_name and with_name or self.path.basename(filepath))
self.push_file(filepath, destpath, as_root=True)
self.execute('chmod a+x {}'.format(destpath), timeout=timeout, as_root=True)
else:
destpath = self.path.join(self.local_binaries_directory,
with_name and with_name or self.path.basename(filepath))
self.push_file(filepath, destpath)
self.execute('chmod a+x {}'.format(destpath), timeout=timeout)
destpath = self.path.join(self.binaries_directory,
with_name or self.path.basename(filepath))
self.push_file(filepath, destpath, as_root=True)
self.execute('chmod a+x {}'.format(destpath), timeout=timeout, as_root=True)
return destpath
install_executable = install # compatibility
def uninstall(self, name):
if self.is_rooted:
path = self.path.join(self.binaries_directory, name)
self.delete_file(path, as_root=True)
else:
path = self.path.join(self.local_binaries_directory, name)
self.delete_file(path)
def uninstall(self, executable_name):
on_device_executable = self.get_binary_path(executable_name, search_system_binaries=False)
if not on_device_executable:
raise DeviceError("Could not uninstall {}, binary not found".format(on_device_executable))
self.delete_file(on_device_executable, as_root=self.is_rooted)
uninstall_executable = uninstall # compatibility
def is_installed(self, name):
try:
self.execute('which {}'.format(name))
return True
except DeviceError:
return False
# misc
def ping(self):
@@ -788,7 +821,7 @@ class LinuxDevice(BaseLinuxDevice):
raise DeviceNotRespondingError(self.host)
def capture_screen(self, filepath):
if not self.is_installed('scrot'):
if not self.get_binary_path('scrot'):
self.logger.debug('Could not take screenshot as scrot is not installed.')
return
try:

6
wlauto/instrumentation/perf/__init__.py
View File

@@ -91,11 +91,11 @@ class PerfInstrument(Instrument):
]
def on_run_init(self, context):
if not self.device.is_installed('perf') or self.force_install:
binary = context.resolver.get(Executable(self, self.device.abi, 'perf'))
binary = context.resolver.get(Executable(self, self.device.abi, 'perf'))
if self.force_install:
self.binary = self.device.install(binary)
else:
self.binary = 'perf'
self.binary = self.device.install_if_needed(binary)
self.commands = self._build_commands()
def setup(self, context):

7
wlauto/instrumentation/trace_cmd/__init__.py
View File

@@ -164,11 +164,12 @@ class TraceCmdInstrument(Instrument):
raise InstrumentError('trace-cmd instrument cannot be used on an unrooted device.')
if not self.no_install:
host_file = context.resolver.get(Executable(self, self.device.abi, 'trace-cmd'))
self.trace_cmd = self.device.install_executable(host_file)
self.trace_cmd = self.device.install(host_file)
else:
if not self.device.is_installed('trace-cmd'):
self.trace_cmd = self.device.get_binary_path("trace-cmd")
if not self.trace_cmd:
raise ConfigError('No trace-cmd found on device and no_install=True is specified.')
self.trace_cmd = 'trace-cmd'
# Register ourselves as absolute last event before and
# first after so we can mark the trace at the right time
signal.connect(self.insert_start_mark, signal.BEFORE_WORKLOAD_EXECUTION, priority=11)

2
wlauto/workloads/antutu/__init__.py
View File

@@ -80,6 +80,8 @@ class Antutu(AndroidUiAutoBenchmark):
info = ApkInfo(antutu_3d)
if not context.device.is_installed(info.package):
self.device.install_apk(antutu_3d, timeout=120)
# Antutu doesnt seem to list this as one of its permissions, but it asks for it.
self.device.execute("pm grant com.antutu.ABenchMark android.permission.ACCESS_FINE_LOCATION")
super(Antutu, self).setup(context)
def update_result(self, context):

7
wlauto/workloads/cyclictest/__init__.py
View File

@@ -79,11 +79,8 @@ class Cyclictest(Workload):
if not self.device.is_rooted:
raise WorkloadError("This workload requires a device with root premissions to run")
if not self.device.is_installed('cyclictest'):
host_binary = context.resolver.get(Executable(self, self.device.abi, 'cyclictest'))
self.device_binary = self.device.install(host_binary)
else:
self.device_binary = 'cyclictest'
host_binary = context.resolver.get(Executable(self, self.device.abi, 'cyclictest'))
self.device_binary = self.device.install(host_binary)
self.cyclictest_command = self.cyclictest_command.format(self.device_binary,
0 if self.clock == 'monotonic' else 1,

9
wlauto/workloads/ebizzy/__init__.py
View File

@@ -52,16 +52,13 @@ class Ebizzy(Workload):
def setup(self, context):
timeout_buf = 10
self.command = '{} -t {} -S {} -n {} {} > {}'
self.ebizzy_results = os.path.join(self.device.working_directory, results_txt)
self.ebizzy_results = self.device.path.join(self.device.working_directory, results_txt)
self.device_binary = None
self.run_timeout = self.seconds + timeout_buf
self.binary_name = 'ebizzy'
if not self.device.is_installed(self.binary_name):
host_binary = context.resolver.get(Executable(self, self.device.abi, self.binary_name))
self.device_binary = self.device.install(host_binary)
else:
self.device_binary = self.binary_name
host_binary = context.resolver.get(Executable(self, self.device.abi, self.binary_name))
self.device_binary = self.device.install_if_needed(host_binary)
self.command = self.command.format(self.device_binary, self.threads, self.seconds,
self.chunks, self.extra_params, self.ebizzy_results)

7
wlauto/workloads/hackbench/__init__.py
View File

@@ -61,11 +61,8 @@ class Hackbench(Workload):
self.run_timeout = self.duration + timeout_buf
self.binary_name = 'hackbench'
if not self.device.is_installed(self.binary_name):
host_binary = context.resolver.get(Executable(self, self.device.abi, self.binary_name))
self.device_binary = self.device.install(host_binary)
else:
self.device_binary = self.binary_name
host_binary = context.resolver.get(Executable(self, self.device.abi, self.binary_name))
self.device_binary = self.device.install(host_binary)
self.command = self.command.format(self.device_binary, self.datasize, self.groups,
self.loops, self.extra_params, self.hackbench_result)

11
wlauto/workloads/memcpy/__init__.py
View File

@@ -18,7 +18,7 @@
import os
import re
from wlauto import Workload, Parameter
from wlauto import Workload, Parameter, Executable
THIS_DIR = os.path.dirname(__file__)
@@ -54,11 +54,10 @@ class MemcpyTest(Workload):
]
def setup(self, context):
self.host_binary = os.path.join(THIS_DIR, 'memcpy')
if not self.device.is_installed('memcpy'):
self.device_binary = self.device.install(self.host_binary)
else:
self.device_binary = 'memcpy'
self.binary_name = 'memcpy'
host_binary = context.resolver.get(Executable(self, self.device.abi, self.binary_name))
self.device_binary = self.device.install_if_needed(host_binary)
self.command = '{} -i {} -s {}'.format(self.device_binary, self.iterations, self.buffer_size)
if self.cpus:
for c in self.cpus:

BIN
wlauto/workloads/memcpy/bin/arm64/memcpy Executable file
View File

Binary file not shown.

0
wlauto/workloads/memcpy/memcpy → wlauto/workloads/memcpy/bin/armeabi/memcpy
View File

6
wlauto/workloads/rt_app/__init__.py
View File

@@ -216,14 +216,13 @@ class RtApp(Workload):
def _deploy_rt_app_binary_if_necessary(self):
# called from initialize() so gets invoked once per run
if self.force_install or not self.device.is_installed(BINARY_NAME):
RtApp.device_binary = self.device.get_binary_path("rt-app")
if self.force_install or not RtApp.device_binary:
if not self.host_binary:
message = '''rt-app is not installed on the device and could not be
found in workload resources'''
raise ResourceError(message)
RtApp.device_binary = self.device.install(self.host_binary)
else:
RtApp.device_binary = BINARY_NAME
def _load_json_config(self, context):
user_config_file = self._get_raw_json_config(context.resolver)
@@ -280,4 +279,3 @@ class RtApp(Workload):
tf.extractall(context.output_directory)
os.remove(host_path)
self.device.execute('rm -rf {}/*'.format(self.device_working_directory))

12
wlauto/workloads/sysbench/__init__.py
View File

@@ -132,13 +132,13 @@ class Sysbench(Workload):
self.device.delete_file(self.results_file)
def _check_executable(self):
self.on_device_binary = self.device.path.join(self.device.binaries_directory, 'sysbench')
if self.device.is_installed('sysbench') and not self.force_install:
self.logger.debug('sysbench found on device')
return
if not self.on_host_binary:
self.on_device_binary = self.device.get_binary_path("sysbench")
if not self.on_device_binary and not self.on_host_binary:
raise WorkloadError('sysbench binary is not installed on the device, and it is not found on the host.')
self.device.install(self.on_host_binary)
if self.force_install:
self.device.install(self.on_host_binary)
else:
self.device.install_if_needed(self.on_host_binary)
def _build_command(self, **parameters):
param_strings = ['--{}={}'.format(k.replace('_', '-'), v)