Version bump

Documentation and get-assets Update

.gitignore

@@ -14,9 +14,9 @@ wa_output/
 doc/source/api/
 doc/source/extensions/
 MANIFEST
-wlauto/external/uiautomator/bin/
-wlauto/external/uiautomator/*.properties
-wlauto/external/uiautomator/build.xml
+wlauto/external/uiauto/bin/
+wlauto/external/uiauto/*.properties
+wlauto/external/uiauto/build.xml
 *.orig
 local.properties
 wlauto/external/revent/libs/

dev_scripts/README

@@ -6,6 +6,11 @@ distributed as part of WA releases.
 Scripts
 -------
 
+:check_apk_versions: Compares WA workload versions with the versions listed in APK
+                     if there are any incistency it will highlight these. This 
+                     requires all APK files to be present for workloads with 
+                     versions.
+
 :clean_install: Performs a clean install of WA from source. This will remove any
                 existing WA install (regardless of whether it was made from
                 source or through a tarball with pip).

dev_scripts/check_apk_versions

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+import os
+from distutils.version import StrictVersion
+
+from wlauto.core.extension_loader import ExtensionLoader
+from wlauto.common.android.workload import ApkWorkload
+from wlauto.utils.android import ApkInfo
+
+el = ExtensionLoader()
+
+
+class fake_config(object):
+    def __init__(self, ext_loader):
+        self.ext_loader = ext_loader
+        self.get_extension = ext_loader.get_extension
+
+
+class fake_device(object):
+    platform = "android"
+
+config = fake_config(el)
+device = fake_device()
+
+if "WA_USER_DIRECTORY" in os.environ:
+    base_path = os.environ["WA_USER_DIRECTORY"]
+else:
+    base_path = "~/.workload_automation/dependencies/"
+
+apk_workloads = [e for e in el.list_workloads()
+                 if issubclass(el.get_extension_class(e.name), ApkWorkload)]
+
+for wl in apk_workloads:
+    # Get versions from workloads
+    workload_versions = []
+    for p in wl.parameters:
+        if p.name == "version" and p.allowed_values:
+            workload_versions = p.allowed_values
+            break
+    else:
+        continue
+
+    dep_path = os.path.join(os.path.expanduser(base_path), wl.name)
+    apks = [apk for apk in os.listdir(dep_path) if apk.endswith(".apk")]
+
+    # Get versions from APK files
+    apk_versions = []
+    for apk in apks:
+        # skip antutu 3d benchmark apk
+        if apk == "com.antutu.benchmark.full-1.apk":
+            continue
+        apk_versions.append(ApkInfo(os.path.join(dep_path, apk)).version_name)
+
+    # Output workload info
+    print "Workload:          {}".format(wl.name)
+    print "Workload Versions: {}".format(sorted(workload_versions, key=StrictVersion))
+    print "APK versions:      {}".format(sorted(apk_versions, key=StrictVersion))
+
+    # Check for bad/missing versions
+    error = False
+    for v in apk_versions:
+        if v not in workload_versions:
+            msg = "APK version '{}' not present in workload list of versions"
+            print msg.format(v)
+            error = True
+    if not error:
+        print "OK"

doc/source/apk_workloads.rst

@@ -0,0 +1,57 @@
+.. _apk_workload_settings:
+
+APK Workloads
+=============
+
+APK resolution
+--------------
+
+WA has various resource getters that can be configured to locate APK files but for most people APK files
+should be kept in the ``$WA_HOME/dependencies/SOME_WORKLOAD/`` directory. (by default 
+``~/.workload_automation/dependencies/SOME_WORKLOAD/``). The ``WA_HOME`` enviroment variable can be used
+to chnage the location of this folder. The APK files need to be put into the corresponding directories
+for the workload they belong to. The name of the file can be anything but as explained below may need
+to contain certain peices of information.
+
+All ApkWorkloads have parameters that affect the way in which APK files are resolved, ``exact_abi``,
+``force_install`` and ``check_apk``. Their exact behaviours are outlined below.
+
+.. confval:: exact_abi
+
+   If this setting is enabled WA's resource resolvers will look for the devices ABI with any native 
+   code present in the apk. By default this setting is disabled since most apks will work across all
+   devices. You may wish to enable this feature when working with devices that support multiple ABI's (like 
+   64-bit devices that can run 32-bit APK files) and are specifically trying to test one or the other.
+
+.. confval:: force_install
+
+   If this setting is enabled WA will *always* use the APK file on the host, and re-install it on every
+   iteration. If there is no APK on the host that is a suitable version and/or ABI for the workload WA
+   will error when ``force_install`` is enabled.
+
+.. confval:: check_apk
+
+   This parameter is used to specify a preference over host or target versions of the app. When set to
+   ``True`` WA will prefer the host side version of the APK. It will check if the host has the APK and
+   if the host APK meets the version requirements of the workload. If does and the target already has
+   same version nothing will be done, other wise it will overwrite the targets app with the host version.
+   If the hosts is missing the APK or it does not meet version requirements WA will fall back to the app
+   on the target if it has the app and it is of a suitable version. When this parameter is set to 
+   ``false`` WA will prefer to use the version already on the target if it meets the workloads version
+   requirements. If it does not it will fall back to search the host for the correct version. In both modes
+   if neither the host nor target have a suitable version, WA will error and not run the workload.
+
+Some workloads will also feature the follow parameters which will alter the way their APK files are resolved.
+
+.. confval:: version
+
+   This parameter is used to specify which version of uiautomation for the workload is used. In some workloads
+   e.g. ``geekbench`` multiple versions with drastically different UI's are supported. When a workload uses a
+   version it is required for the APK file to contain the uiautomation version in the file name. In the case
+   of antutu the file names could be: ``geekbench_2.apk`` or ``geekbench_3.apk``.
+
+.. confval:: variant_name
+
+   Some workloads use variants of APK files, this is usually the case with web browser APK files, these work
+   in exactly the same way as the version,  the variant of the apk 
+

doc/source/changes.rst

@@ -1,6 +1,454 @@
 =================================
 What's New in Workload Automation
 =================================
+-------------
+Version 2.6.0
+-------------
+
+ .. note:: Users who are currently using the GitHub master version of WA should
+           uninstall the existing version before upgrading to avoid potential issues.
+
+Additions:
+##########
+
+Workloads
+~~~~~~~~~
+- ``AdobeReader``: A workload that carries out following typical productivity
+  tasks. These include opening a file, performing various gestures and
+  zooms on screen and searching for a predefined set of strings.
+- ``octaned8``: A workload to run the binary (non-browser) version of the JS
+  benchmark Octane.
+- ``GooglePlayBooks``: A workload to perform standard productivity tasks with
+  Google Play Books. This workload performs various tasks, such as searching
+  for a book title online, browsing through a book, adding and removing notes,
+  word searching, and querying information about the book.
+- ``GooglePhotos``: A workload to perform standard productivity tasks with
+  Google Photos. Carries out various tasks, such as browsing images,
+  performing zooms, and post-processing the image.
+- ``GoogleSlides``: Carries out various tasks, such as creating a new
+  presentation, adding text, images, and shapes, as well as basic editing and
+  playing a slideshow.
+- ``Youtube``: The workload plays a video, determined by the ``video_source``
+  parameter. While the video is playing, some common actions such as video
+  seeking, pausing playback and navigating the comments section are performed.
+- ``Skype``: Replacement for the ``skypevideo`` workload. Logs into Skype
+  and initiates a voice or video call with a contact.
+
+Framework
+~~~~~~~~~
+- ``AndroidUxPerfWorkload``: Added a new workload class to encapsulate
+  functionality common to all uxperf workloads.
+- ``UxPerfUiAutomation``: Added class which contains methods specific to
+  UX performance
+  testing.
+- ``get-assets``: Added new script and command to retrieve external assets
+  for workloads
+
+Results Processors
+~~~~~~~~~~~~~~~~~~~
+- ``uxperf``: Parses device logcat for `UX_PERF` markers to produce performance
+  metrics for workload actions using specified instrumentation.
+
+Other
+~~~~~
+- ``State Detection``: Added feature to use visual state detection to
+  verify the state of a workload after setup and run.
+
+
+Fixes/Improvements:
+###################
+
+Documentation
+~~~~~~~~~~~~~~
+- ``Revent``: Added file structure to the documentation.
+- Clarified documentation regarding binary dependencies.
+- Updated documentation with ``create`` and ``get-assets`` commands.
+
+Instruments
+~~~~~~~~~~~~
+- ``sysfs_extractor``: Fixed error when `tar.gz` file already existed on device,
+  now overwrites.
+- ``cpufreq``: Fixed error when `tar.gz` file already existed on device, now
+  overwrites.
+- ``file-poller``:
+    - Improved csv output.
+    - Added error checking and reporting.
+    - Changed ``files`` to be a mandatory parameter.
+- ``fps``:
+    - Added a new parameter to fps instrument to specify the time period between
+      calls to ``dumpsys SurfaceFlinger --latency`` when collecting frame data.
+    - Added gfxinfo methods to obtain fps stats. Auto detects and uses appropriate
+      method via android version of device.
+    - Fixed issue with regex.
+    - Now handles empty frames correctly.
+- ``energy_model``: Ensures that the ``ui`` runtime parameter is only set for
+  ChromeOS devices.
+- ``ftrace``: Added support to handle traces collected by both WA and devlib.
+- ``Perf``: Updated 32bit binary file for little endian devices.
+
+Resource Getters
+~~~~~~~~~~~~~~~~
+- ``http_getter``: Now used to try and find executables files from a
+  provided ``remove_assets_url``.
+
+Result Processors
+~~~~~~~~~~~~~~~~~
+- ``cpu_states``: Fixes using stand-alone script with timeline option.
+
+Workloads
+~~~~~~~~~
+- ``antutu``: Fixed setting permissions of ``FINE_LOCATION`` on some devices.
+- ``bbench`` Fixed handling of missing results.
+- ``camerarecord``:
+    - Added frame stats collection through dumpsys gfxinfo.
+    - Added possibility to select slow_motion recording mode.
+- ``Geekbench``:
+    - Fixed output file listing causing pull failure.
+    - Added support for Geekbench 4.
+- ``recentfling``:
+    - Fixed issue when binaries were not uninstalled correctly.
+    - Scripts are now deployed via ``install()`` to ensure they are executable.
+    - Fixed handling of when a PID file is deleted before reaching processing
+      results stage.
+    - Added parameter to not start any apps before flinging.
+- ``rt-app``: Added camera recorder simulation.
+- ``sysbench``: Added arm64 binary.
+- ``Vellamo``: Fixed capitalization in part of UIAutomation to prevent
+  potential issues.
+- ``Spec2000``: Now uses WA deployed version of busybox.
+- ``NetStat``: Updated to support new default logcat format in Android 6.
+- ``Dex2oat``: Now uses root if available.
+
+Framework
+~~~~~~~~~
+- ``adb_shell``:
+    - Fixed issue when using single quoted command with ``adb_shell``.
+    - Correctly forward stderror to the caller for newer version of adb.
+- ``revent``
+    - Added ``-S`` argument to "record" command to automatically record a
+      screen capture after a recording is completed.
+    - Fixed issue with multiple iterations of a revent workload.
+    - Added ``-s`` option to executable to allow waiting on stdin.
+    - Removed timeout in command as ``-s`` is specified.
+    - Revent recordings can now be parsed and used within WA.
+    - Fixed issue when some recordings wouldn't be retrieved correctly.
+    - Timeout is now based on recording duration.
+    - Added `magic` and file version to revent files. Revent files should now
+      start with ``REVENT`` followed by the file format version.
+    - Added support for gamepad recording. This type of recording contains
+      only the events from a gamepad device (which is automatically
+      identified).
+    - A ``mode`` field has been added to the recording format to help
+      distinguish between the normal and gamepad recording types.
+    - Added ``-g`` option to ``record`` command to expose the gamepad recording
+      mode.
+    - The structure of revent code has undergone a major overhaul to improve
+      maintainability and robustness.
+    - More detailed ``info`` command output.
+    - Updated Makefile to support debug/production builds.
+- ``Android API``: Upgraded Android API level from 17 to 18.
+- ``uiautomator``: The window hierarchy is now dumped to a file when WA fails
+  on android devices.
+- ``AndroidDevice``:
+    - Added support for downgrading when installing an APK.
+    - Added a ``broadcast_media_mounted`` method to force a re-index of the
+      mediaserver cache for a specified directory.
+    - Now correctly handles ``None`` output for ``get_pids_of()`` when there are no
+      running processes with the specified name.
+    - Renamed the capture method from ``capture_view_hierachy`` to
+      ``capture_ui_hierarchy``.
+    - Changed the file extension of the capture file to ``.uix``
+    - Added ``-rf`` to delete_files to be consistent with ``LinuxDevice``.
+- ``LinuxDevice``: Now ensures output from both stdout and etderr is propagated in
+  the event of a DeviceError.
+- ``APKWorkload``:
+    - Now ensure APKs are replaced properly when reinstalling.
+    - Now checks APK version and ABI when installing.
+    - Fixed error on some devices when trying to grant permissions that were
+      already granted.
+    - Fixed some permissions not being granted.
+    - Now allows disabling the main activity launch in setup (required for some
+      apps).
+    - Added parameter to clear data on reset (default behaviour unchanged).
+    - Ignores exception for non-fatal permission grant failure.
+    - Fixed issue of multiple versions of the same workload failing to find their APK.
+    - Added method to ensure a valid apk version is used within a workload.
+    - Updated how APK resolution is performed to maximise likelihood of
+      a workload running.
+    - When ``check_apk`` is ``True`` will prefer host APK and if no suitable APK
+      is found, will use target APK if the correct version is present. When ``False``
+      will prefer target apk if it is a valid version otherwise will fallback to
+      host APK.
+- ``RunConfiguration``: Fixed disabling of instruments in workload specs.
+- ``Devices``:
+    - Added network connectivity check for devices.
+    - Subclasses can now set ``requires_network`` to ``True`` and network
+      connectivity check will be performed during ``setup()``.
+- ``Workloads``:
+    - Added network check methods.
+    - Fixed versions to be backwards compatible.
+    - Updated workload versions to match APK files.
+    - Fixed issues with calling super.
+- ``Assets``: Added script to retrieve external assets for workloads.
+- ``Execution``: Added a ``clean_up`` global config option to delete WA files from
+  devices.
+- ``Runner``: No longer takes a screenshot or dump of UI hierarchy for some errors when
+  unnecessary, e.g. host errors.
+- ``core``: Constraints and allowed values are now checked when set instead of
+  when validating.
+- ``FpsProcessor``:
+    - Added requirement on ``filtered_vsyncs_to_compose`` for ``total_vsync metric``.
+    - Removed misleading comment in class description.
+- ``BaseUiAutomation``: Added new Marker API so workloads generate start and end
+  markers with a string name.
+- ``AndroidUiAutoBenchmark``: Automatically checks for known package versions
+  that don't work well with AndroidUiAutoBenchmark workloads.
+
+Other
+~~~~~
+- Updated setup.py url to be a valid URI.
+- Fixed workload name in big.Little sample agenda.
+
+Incompatible changes
+####################
+
+Framework
+~~~~~~~~~
+- ``check_abi``: Now renamed to ``exact_abi``, is used to ensure that if enabled,
+  only an apk containing no native code or code designed for the devices primary
+  abi is use.
+- ``AndroidDevice``: Renamed ``supported_eabis`` property to ``supported_abis``
+  to be consistent with linux devices.
+
+Workloads
+~~~~~~~~~~
+- ``skypevideo``: Workload removed and replaced with ``skype`` workload.
+
+-------------
+Version 2.5.0
+-------------
+
+Additions:
+##########
+
+Instruments
+~~~~~~~~~~~
+- ``servo_power``: Added support for chromebook servo boards.
+- ``file_poller``: polls files and outputs a CSV of their values over time.
+- ``systrace``: The Systrace tool helps analyze the performance of your
+  application by capturing and displaying execution times of your applications
+  processes and other Android system processes.
+
+Workloads
+~~~~~~~~~
+- ``blogbench``: Blogbench is a portable filesystem benchmark that tries to
+  reproduce the load of a real-world busy file server.
+- ``stress-ng``: Designed to exercise various physical subsystems of a computer
+  as well as the various operating system kernel interfaces.
+- ``hwuitest``: Uses hwuitest from AOSP to test rendering latency on Android
+  devices.
+- ``recentfling``: Tests UI jank on android devices.
+- ``apklaunch``: installs and runs an arbitrary apk file.
+- ``googlemap``: Launches Google Maps and replays previously recorded
+  interactions.
+
+Framework
+~~~~~~~~~
+- ``wlauto.utils.misc``: Added ``memoised`` function decorator that allows
+  caching of previous function/method call results.
+- Added new ``Device`` APIs:
+   - ``lsmod``: lists kernel modules
+   - ``insmod``: inserts a kernel module from a ``.ko`` file on the host.
+   - ``get_binary_path``: Checks ``binary_directory`` for the wanted binary,
+     if it is not found there it will try to use ``which``
+   - ``install_if_needed``: Will only install a binary if it is not already
+     on the target.
+   - ``get_device_model``: Gets the model of the device.
+- ``wlauto.core.execution.ExecutionContext``:
+   - ``add_classfiers``: Allows adding a classfier to all metrics for the
+     current result.
+
+Other
+~~~~~
+- Commands:
+   - ``record``: Simplifies recording revent files.
+   - ``replay``: Plays back revent files.
+
+Fixes/Improvements:
+###################
+
+Devices
+~~~~~~~
+- ``juno``:
+   - Fixed ``bootargs`` parameter not being passed  to ``_boot_via_uboot``.
+   - Removed default ``bootargs``
+- ``gem5_linux``:
+   - Added ``login_prompt`` and ``login_password_prompt`` parameters.
+- ``generic_linux``: ABI is now read from the target device.
+
+Instruments
+~~~~~~~~~~~
+- ``trace-cmd``:
+   - Added the ability to report the binary trace on the target device,
+     removing the need for ``trace-cmd`` binary to be present on the host.
+   - Updated to handle messages that the trace for a CPU is empty.
+   - Made timeout for pulling trace 1 minute at minimum.
+- ``perf``: per-cpu statistics now get added as metrics to the results (with a
+   classifier used to identify the cpu).
+- ``daq``:
+   - Fixed bug where an exception would be raised if ``merge_channels=False``
+   - No longer allows duplicate channel labels
+- ``juno_energy``:
+   - Summary metrics are now calculated from the contents of ``energy.csv`` and
+     added to the overall results.
+   - Added a ``strict`` parameter. When this is set to ``False`` the device
+     check during validation is omitted.
+- ``sysfs_extractor``: tar and gzip are now performed separately to solve
+  permission issues.
+- ``fps``:
+   - Now only checks for crashed content if ``crash_check`` is ``True``.
+   - Can now process multiple ``view`` attributes.
+- ``hwmon``: Sensor naming fixed, they are also now added as result classifiers
+
+Resource Getters
+~~~~~~~~~~~~~~~~
+- ``extension_asset``: Now picks up the path to the mounted filer from the
+  ``remote_assets_path`` global setting.
+
+Result Processors
+~~~~~~~~~~~~~~~~~
+- ``cpustates``:
+   - Added the ability to configure how a missing ``START`` marker in the trace
+     is handled.
+   - Now raises a warning when there is a ``START`` marker in the trace but no
+     ``STOP`` marker.
+   - Exceptions in PowerStateProcessor no longer stop the processing of the
+     rest of the trace.
+   - Now ensures a known initial state by nudging each CPU to bring it out of
+     idle and writing starting CPU frequencies to the trace.
+   - Added the ability to create a CPU utilisation timeline.
+   - Fixed issues with getting frequencies of hotplugged CPUs
+- ``csv``: Zero-value classifieres are no longer converted to an empty entry.
+- ``ipynb_exporter``: Default template no longer shows a blank plot for
+  workloads without ``summary_metrics``
+
+Workloads
+~~~~~~~~~
+- ``vellamo``:
+   - Added support for v3.2.4.
+   - Fixed getting values from logcat.
+- ``cameracapture``: Updated to work with Android M+.
+- ``camerarecord``: Updated to work with Android M+.
+- ``lmbench``:
+   - Added the output file as an artifact.
+   - Added taskset support
+- ``antutu`` - Added support for v6.0.1
+- ``ebizzy``: Fixed use of ``os.path`` to ``self.device.path``.
+- ``bbench``: Fixed browser crashes & permissions issues on android M+.
+- ``geekbench``:
+   - Added check whether device is rooted.
+- ``manual``: Now only uses logcat on Android devices.
+- ``applaunch``:
+   - Fixed ``cleanup`` not getting forwarded to script.
+   - Added the ability to stress IO during app launch.
+- ``dhrystone``: Now uses WA's resource resolution to find it's binary so it
+  uses the correct ABI.
+- ``glbench``: Updated for new logcat formatting.
+
+Framework
+~~~~~~~~~
+- ``ReventWorkload``:
+   - Now kills all revent instances on teardown.
+   - Device model name is now used when searching for revent files, falling back
+     to WA device name.
+- ``BaseLinuxDevice``:
+   - ``killall`` will now run as root by default if the device
+     is rooted.
+   - ``list_file_systems`` now handles blank lines.
+   - All binaries are now installed into ``binaries_directory`` this allows..
+   - Busybox is now deployed on non-root devices.
+   - gzipped property files are no zcat'ed
+- ``LinuxDevice``:
+   - ``kick_off`` no longer requires root.
+   - ``kick_off`` will now run as root by default if the device is rooted.
+   - No longer raises an exception if a connection was dropped during a reboot.
+   - Added a delay before polling for a connection to avoid re-connecting to a
+     device that is still in the process of rebooting.
+- ``wlauto.utils.types``: ``list_or_string`` now ensures that elements of a list
+  are strings.
+- ``AndroidDevice``:
+   - ``kick_off`` no longer requires root.
+   - Build props are now gathered via ``getprop`` rather than trying to parse
+     build.prop directly.
+   - WA now pushes its own ``sqlite3`` binary.
+   - Now uses ``content`` instead of ``settings`` to get ``ANDROID_ID``
+   - ``swipe_to_unlock`` parameter is now actually used. It has been changed to
+      take a direction to accomodate various devices.
+   - ``ensure_screen_is_on`` will now also unlock the screen if swipe_to_unlock
+     is set.
+   - Fixed use of variables in as_root=True commands.
+   - ``get_pids_of`` now used ``busybox grep`` since as of Android M+ ps cannot
+     filter by process name anymore.
+   - Fixed installing APK files with whitespace in their path/name.
+- ``adb_shell``:
+   - Fixed handling of line breaks at the end of command output.
+   - Newline separator is now detected from the target.
+   - As of ADB v1.0.35, ADB returns the return code of the command run. WA now
+     handles this correctly.
+- ``ApkWorkload``:
+   - Now attempts to grant all runtime permissions for devices on Android M+.
+   - Can now launch packages that don't have a launch activity defined.
+   - Package version is now added to results as a classifier.
+   - Now clears app data if an uninstall failed to ensure it starts from a known
+     state.
+- ``wlauto.utils.ipython``: Updated to work with ipython v5.
+- ``Gem5Device``:
+   - Added support for deploying the ``m5`` binary.
+   - No longer waits for the boot animation to finish if it has been disabled.
+   - Fixed runtime error caused by lack of kwargs.
+   - No longer depends on ``busybox``.
+   - Split out commands to resize shell to ``resize_shell``.
+   - Now tries to connect to the shell up to 10 times.
+   - No longer renames gzipped files.
+- Agendas:
+  - Now errors when an agenda key is empty.
+- ``wlauto.core.execution.RunInfo``: ``run_name`` will now default to
+  ``{output_folder}_{date}_{time}``.
+- Extensions:
+   - Two different parameters can now have the same global alias as long as they
+     their types match.
+   - You can no longer ``override`` parameters that are defined at the same
+     level.
+- ``wlauto.core.entry_point``: Now gives a better error when a config file
+  doesn't exist.
+- ``wlauto.utils.misc``: Added ``aarch64`` to list for arm64 ABI.
+- ``wlauto.core.resolver``: Now shows what version was being search for when a
+  resource is not found.
+- Will no longer start instruments ect. if a run has no workload specs.
+- ``wlauto.utils.uboot``: Now detects uboot version to use correct line endings.
+- ``wlauto.utils.trace_cmd``: Added a parser for sched_switch events.
+
+Other
+~~~~~
+- Updated to pylint v1.5.1
+- Rebuilt ``busybox`` binaries to prefer built-in applets over system binaries.
+- ``BaseUiAutomation``: Added functions for checking version strings.
+
+Incompatible changes
+####################
+
+Instruments
+~~~~~~~~~~~
+- ``apk_version``: Removed, use result classifiers instead.
+
+Framework
+~~~~~~~~~
+- ``BaseLinuxDevice``: Removed ``is_installed`` use ``install_if_needed`` and
+  ``get_binary_path`` instead.
+- ``LinuxDevice``: Removed ``has_root`` method, use ``is_rooted`` instead.
+- ``AndroidDevice``: ``swipe_to_unlock`` method replaced with
+  ``perform_unlock_swipe``.
+
 -------------
 Version 2.4.0
 -------------

doc/source/index.rst

@@ -118,6 +118,7 @@ and detailed descriptions of how WA functions under the hood.
    additional_topics
    daq_device_setup
    revent
+   apk_workloads
    contributing
 
 API Reference

doc/source/installation.rst

@@ -59,6 +59,11 @@ usually the best bet.
 Optionally (but recommended), you should also set ``ANDROID_HOME`` to point to
 the install location of the SDK (i.e. ``<path_to_android_sdk>/sdk``).
 
+.. note:: You may need to install 32-bit compatibility libararies for the SDK
+          to work properly. On Ubuntu you need to run::
+
+                  sudo apt-get install lib32stdc++6 lib32z1
+
 
 Python
 ------
@@ -87,7 +92,7 @@ similar distributions, this may be done with APT::
           If you do run  into this issue after already installing some packages,
           you can resolve it by running ::
 
-                  sudo chmod -R a+r /usr/local/lib/python2.7/dist-packagessudo 
+                  sudo chmod -R a+r /usr/local/lib/python2.7/dist-packagessudo
                   find /usr/local/lib/python2.7/dist-packages -type d -exec chmod a+x {} \;
 
           (The paths above will work for Ubuntu; they may need to be adjusted
@@ -187,10 +192,28 @@ version $version".
 Some WA extensions have additional dependencies that need to be
 statisfied before they can be used. Not all of these can be provided with WA and
 so will need to be supplied by the user. They should be placed into
-``~/.workload_uatomation/dependencies/<extenion name>`` so that WA can find
+``~/.workload_automation/dependencies/<extenion name>`` so that WA can find
 them (you may need to create the directory if it doesn't already exist). You
 only need to provide the dependencies for workloads you want to use.
 
+Binary Files
+------------
+
+Some workloads require native binaries to work. Different binaries will be required
+for different ABIs. WA may not include the required binary for a workload due to 
+licensing/distribution issues, or may not have a binary compiled for your device's 
+ABI. In such cases, you will have to supply the missing binaries. 
+
+Executable binaries for a workload should be placed inside 
+``~/.workload_automation/dependencies/<extension name>/bin/<ABI>`` directory.
+This directory may not already exist, in which case you would have to create it.
+
+Binaries placed in that location will take precidence over any already inclueded with
+WA. For example, if you have your own ``drystone`` binary compiled for ``arm64``, 
+and you want WA to pick it up, you can do the following on WA host machine ::
+
+    mkdir -p ~/.workload_automation/dependencies/dhrystone/bin/arm64/
+    cp /path/to/your/dhrystone ~/.workload_automation/dependencies/dhrystone/bin/arm64/
 
 APK Files
 ---------
@@ -307,7 +330,7 @@ that location.
 
 If you have installed Workload Automation via ``pip`` and wish to remove it, run this command to
 uninstall it::
-    
+
     sudo -H pip uninstall wlauto
 
 .. Note:: This will *not* remove any user configuration (e.g. the ~/.workload_automation directory)
@@ -317,5 +340,5 @@ uninstall it::
 ====================
 
 To upgrade Workload Automation to the latest version via ``pip``, run::
-    
+
     sudo -H pip install --upgrade --no-deps wlauto

doc/source/invocation.rst

@@ -1,11 +1,12 @@
 .. _invocation:
+.. highlight:: none
 
 ========
 Commands
 ========
 
 Installing the wlauto package will add ``wa`` command to your system,
-which you can run from anywhere. This has a number of sub-commands, which can 
+which you can run from anywhere. This has a number of sub-commands, which can
 be viewed by executing ::
 
         wa -h
@@ -15,7 +16,7 @@ Individual sub-commands are discussed in detail below.
 run
 ---
 
-The most common sub-command you will use is ``run``. This will run specfied
+The most common sub-command you will use is ``run``. This will run specified
 workload(s) and process resulting output. This takes a single mandatory
 argument that specifies what you want WA to run. This could be either a
 workload name, or a path  to an "agenda" file that allows to specify multiple
@@ -24,7 +25,7 @@ section for details). Executing ::
 
         wa run -h
 
-Will display help for this subcommand that will look somehtign like this::
+Will display help for this subcommand that will look something like this::
 
         usage: run [-d DIR] [-f] AGENDA
 
@@ -47,13 +48,13 @@ Will display help for this subcommand that will look somehtign like this::
           --debug               Enable debug mode. Note: this implies --verbose.
           -d DIR, --output-directory DIR
                                 Specify a directory where the output will be
-                                generated. If the directoryalready exists, the script
+                                generated. If the directory already exists, the script
                                 will abort unless -f option (see below) is used,in
                                 which case the contents of the directory will be
-                                overwritten. If this optionis not specified, then
+                                overwritten. If this option is not specified, then
                                 wa_output will be used instead.
           -f, --force           Overwrite output directory if it exists. By default,
-                                the script will abort in thissituation to prevent
+                                the script will abort in this situation to prevent
                                 accidental data loss.
           -i ID, --id ID        Specify a workload spec ID from an agenda to run. If
                                 this is specified, only that particular spec will be
@@ -81,10 +82,74 @@ agenda file used to run the workloads along with any other device-specific
 configuration files used during execution.
 
 
+create
+------
+
+This can be used to create various WA-related objects, currently workloads, packages and agendas.
+The full set of options for this command are::
+
+    usage: wa create [-h] [-c CONFIG] [-v] [--debug] [--version]
+                     {workload,package,agenda} ...
+
+    positional arguments:
+      {workload,package,agenda}
+        workload            Create a new workload. By default, a basic workload
+                            template will be used but you can use options to
+                            specify a different template.
+        package             Create a new empty Python package for WA extensions.
+                            On installation, this package will "advertise" itself
+                            to WA so that Extensions with in it will be loaded by
+                            WA when it runs.
+        agenda              Create an agenda whit the specified extensions
+                            enabled. And parameters set to their default values.
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      -c CONFIG, --config CONFIG
+                            specify an additional config.py
+      -v, --verbose         The scripts will produce verbose output.
+      --debug               Enable debug mode. Note: this implies --verbose.
+      --version             show program's version number and exit
+
+Use "wa create <object> -h" to see all the object-specific arguments. For example::
+
+        wa create agenda -h
+
+will display the relevant options that can be used to create an agenda. 
+
+get-assets
+----------
+
+This command can download external extension dependencies used by Workload Automation.
+It can be used to download assets for all available extensions or those specificity listed. 
+The full set of options for this command are::
+
+    usage: wa get-assets [-h] [-c CONFIG] [-v] [--debug] [--version] [-f]
+                         [--url URL] (-a | -e EXT [EXT ...])
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      -c CONFIG, --config CONFIG
+                            specify an additional config.py
+      -v, --verbose         The scripts will produce verbose output.
+      --debug               Enable debug mode. Note: this implies --verbose.
+      --version             show program's version number and exit
+      -f, --force           Always fetch the assets, even if matching versions
+                            exist in local cache.
+      --url URL             The location from which to download the files. If not
+                            provided, config setting ``remote_assets_url`` will be
+                            used if available, else uses the default
+                            REMOTE_ASSETS_URL parameter in the script.
+      -a, --all             Download assets for all extensions found in the index.
+                            Cannot be used with -e.
+      -e EXT [EXT ...]      One or more extensions whose assets to download.
+                            Cannot be used with --all.
+
+
 list
 ----
 
-This lists all extensions of a particular type. For example ::
+This lists all extensions of a particular type. For example::
 
         wa list workloads
 
@@ -97,11 +162,11 @@ show
 
 This will show detailed information about an extension, including more in-depth
 description and any parameters/configuration that are available.  For example
-executing ::
+executing::
 
         wa show andebench
 
-will produce something like ::
+will produce something like::
 
 
         andebench
@@ -131,5 +196,64 @@ will produce something like ::
         - Results displayed in Iterations per second
         - Detailed log file for comprehensive engineering analysis
 
+.. _record-command:
 
+record
+------
 
+This command simplifies the process of recording an revent file. It
+will automatically deploy revent and even has the option of automatically
+opening apps. WA uses two parts to the names of revent recordings in the
+format, {device_name}.{suffix}.revent. - device_name can either be specified
+manually with the ``-d`` argument or it can be automatically determined. On
+Android device it will be obtained from ``build.prop``, on Linux devices it is
+obtained from ``/proc/device-tree/model``. - suffix is used by WA to determine
+which part of the app execution the recording is for, currently these are
+either ``setup`` or ``run``. This should be specified with the ``-s``
+argument. The full set of options for this command are::
+
+    usage: wa record [-h] [-c CONFIG] [-v] [--debug] [--version] [-d DEVICE]
+                 [-s SUFFIX] [-o OUTPUT] [-p PACKAGE] [-g] [-C]
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      -c CONFIG, --config CONFIG
+                            specify an additional config.py
+      -v, --verbose         The scripts will produce verbose output.
+      --debug               Enable debug mode. Note: this implies --verbose.
+      --version             show program's version number and exit
+      -d DEVICE, --device DEVICE
+                            The name of the device
+      -s SUFFIX, --suffix SUFFIX
+                            The suffix of the revent file, e.g. ``setup``
+      -o OUTPUT, --output OUTPUT
+                            Directory to save the recording in
+      -p PACKAGE, --package PACKAGE
+                            Package to launch before recording
+      -g, --gamepad         Record from a gamepad rather than all devices.
+      -C, --clear           Clear app cache before launching it
+
+.. _replay-command:
+
+replay
+------
+
+Along side ``record`` wa also has a command to playback recorded revent files.
+It behaves very similar to the ``record`` command taking many of the same options::
+
+    usage: wa replay [-h] [-c CONFIG] [-v] [--debug] [--version] [-p PACKAGE] [-C]
+                 revent
+
+    positional arguments:
+      revent                The name of the file to replay
+
+    optional arguments:
+      -h, --help            show this help message and exit
+      -c CONFIG, --config CONFIG
+                            specify an additional config.py
+      -v, --verbose         The scripts will produce verbose output.
+      --debug               Enable debug mode. Note: this implies --verbose.
+      --version             show program's version number and exit
+      -p PACKAGE, --package PACKAGE
+                            Package to launch before recording
+      -C, --clear           Clear app cache before launching it

doc/source/revent.rst

@@ -1,7 +1,10 @@
 .. _revent_files_creation:
 
 revent
-======
+++++++
+
+Overview and Usage
+==================
 
 revent utility can be used to record and later play back a sequence of user
 input events, such as key presses and touch screen taps. This is an alternative
@@ -17,36 +20,47 @@ to Android UI Automator for providing automation for workloads. ::
                         info:shows info about each event char device
                         any additional parameters make it verbose
 
-
-.. note:: There are now also WA commands that perform the below steps.
-          Please see ``wa show record/replay`` and ``wa record/replay --help``
-          for details.
-
 Recording
 ---------
 
-To record, transfer the revent binary to the device, then invoke ``revent
-record``, giving it the time (in seconds) you want to record for, and the
-file you want to record to (WA expects these files to have .revent
-extension)::
+WA features a ``record`` command that will automatically deploy and start
+revent on the target device::
 
-        host$  adb push revent /data/local/revent
-        host$  adb shell
-        device#  cd /data/local
-        device#  ./revent record 1000 my_recording.revent
+    wa record
+    INFO     Connecting to device...
+    INFO     Press Enter when you are ready to record...
+    [Pressed Enter]
+    INFO     Press Enter when you have finished recording...
+    [Pressed Enter]
+    INFO     Pulling files from device
+
+Once started, you will need to get the target device ready to record (e.g.
+unlock screen, navigate menus and launch an app) then press ``ENTER``.
+The recording has now started and button presses, taps, etc you perform on
+the device will go into the .revent file. To stop the recording simply press
+``ENTER`` again.
+
+Once you have finished recording the revent file will be pulled from the device
+to the current directory. It will be named ``{device_model}.revent``. When
+recording revent files for a ``GameWorkload`` you can use the ``-s`` option to
+add ``run`` or ``setup`` suffixes.
+
+From version 2.6 of WA onwards, a "gamepad" recording mode is also supported.
+This mode requires a gamepad to be connected to the device when recoridng, but
+the recordings produced in this mode should be portable across devices.
+
+For more information run please read :ref:`record-command`
 
-The recording has now started and button presses, taps, etc you perform on the
-device will go into the .revent file. The recording will stop after the
-specified time period, and you can also stop it by hitting return in the adb
-shell.
 
 Replaying
 ---------
 
-To replay a recorded file, run ``revent replay`` on the device, giving it the
-file you want to replay::
+To replay a recorded file, run ``wa replay``, giving it the file you want to
+replay::
 
-        device#  ./revent replay my_recording.revent
+        wa replay my_recording.revent
+
+For more information run please read :ref:`replay-command`
 
 
 Using revent With Workloads
@@ -100,3 +114,359 @@ where as UI Automator only works for Android UI elements (such as text boxes or
 radio buttons), which makes the latter useless for things like games. Recording
 revent sequence is also faster than writing automation code (on the other hand,
 one would need maintain a different revent log for each screen resolution).
+
+
+Using state detection with revent
+=================================
+
+State detection can be used to verify that a workload is executing as expected.
+This utility, if enabled, and if state definitions are available for the
+particular workload, takes a screenshot after the setup and the run revent
+sequence, matches the screenshot to a state and compares with the expected
+state. A WorkloadError is raised if an unexpected state is encountered.
+
+To enable state detection, make sure a valid state definition file and
+templates exist for your workload and set the check_states parameter to True.
+
+State definition directory
+--------------------------
+
+State and phase definitions should be placed in a directory of the following
+structure inside the dependencies directory of each workload (along with
+revent files etc):
+
+::
+
+   dependencies/
+      <workload_name>/
+         state_definitions/
+            definition.yaml
+            templates/
+               <oneTemplate>.png
+               <anotherTemplate>.png
+               ...
+
+definition.yaml file
+--------------------
+
+This defines each state of the workload and lists which templates are expected
+to be found and how many are required to be detected for a conclusive match. It
+also defines the expected state in each workload phase where a state detection
+is run (currently those are setup_complete and run_complete).
+
+Templates are picture elements to be matched in a screenshot. Each template
+mentioned in the definition file should be placed as a file with the same name
+and a .png extension inside the templates folder. Creating template png files
+is as simple as taking a screenshot of the workload in a given state, cropping
+out the relevant templates (eg. a button, label or other unique element that is
+present in that state) and storing them in PNG format.
+
+Please see the definition file for Angry Birds below as an example to
+understand the format. Note that more than just two states (for the afterSetup
+and afterRun phase) can be defined and this helps track the cause of errors in
+case an unexpected state is encountered.
+
+.. code-block:: yaml
+
+    workload_name: angrybirds
+
+    workload_states:
+      - state_name: titleScreen
+        templates:
+          - play_button
+          - logo
+        matches: 2
+      - state_name: worldSelection
+        templates:
+          - first_world_thumb
+          - second_world_thumb
+          - third_world_thumb
+          - fourth_world_thumb
+        matches: 3
+      - state_name: level_selection
+        templates:
+          - locked_level
+          - first_level
+        matches: 2
+      - state_name: gameplay
+        templates:
+          - pause_button
+          - score_label_text
+        matches: 2
+      - state_name: pause_screen
+        templates:
+          - replay_button
+          - menu_button
+          - resume_button
+          - help_button
+        matches: 4
+      - state_name: level_cleared_screen
+        templates:
+          - level_cleared_text
+          - menu_button
+          - replay_button
+          - fast_forward_button
+        matches: 4
+
+    workload_phases:
+      - phase_name: setup_complete
+        expected_state: gameplay
+      - phase_name: run_complete
+        expected_state: level_cleared_screen
+
+
+File format of revent recordings
+================================
+
+You do not need to understand recording format in order to use revent. This
+section is intended for those looking to extend revent in some way, or to
+utilize revent recordings for other purposes.
+
+Format Overview
+---------------
+
+Recordings are stored in a binary format. A recording consists of three
+sections::
+
+    +-+-+-+-+-+-+-+-+-+-+-+
+    |       Header        |
+    +-+-+-+-+-+-+-+-+-+-+-+
+    |                     |
+    |  Device Description |
+    |                     |
+    +-+-+-+-+-+-+-+-+-+-+-+
+    |                     |
+    |                     |
+    |     Event Stream    |
+    |                     |
+    |                     |
+    +-+-+-+-+-+-+-+-+-+-+-+
+
+The header contains metadata describing the recording. The device description
+contains information about input devices involved in this recording. Finally,
+the event stream contains the recorded input events.
+
+All fields are either fixed size or prefixed with their length or the number of
+(fixed-sized) elements.
+
+.. note:: All values below are little endian
+
+
+Recording Header
+----------------
+
+An revent recoding header has the following structure
+
+ * It starts with the "magic" string ``REVENT`` to indicate that this is an
+   revent recording.
+ * The magic is followed by a 16 bit version number. This indicates the format
+   version of the recording that follows. Current version is ``2``.
+ * The next 16 bits indicate the type of the recording. This dictates the
+   structure of the Device Description section. Valid values are:
+
+        ``0``
+                This is a general input event recording. The device description
+                contains a list of paths from which the events where recorded.
+        ``1``
+                This a gamepad recording. The device description contains the
+                description of the gamepad used to create the recording.
+
+ * The header is zero-padded to 128 bits.
+
+::
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |      'R'      |      'E'      |      'V'      |      'E'      |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |      'N'      |      'T'      |            Version            |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |             Mode              |            PADDING            |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                            PADDING                            |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+Device Description
+------------------
+
+This section describes the input devices used in the recording. Its structure is
+determined by the value of ``Mode`` field in the header.
+
+general recording
+~~~~~~~~~~~~~~~~~
+
+.. note:: This is the only format supported prior to version ``2``.
+
+The recording has been made from all available input devices. This section
+contains the list of ``/dev/input`` paths for the devices, prefixed with total
+number of the devices recorded.
+
+::
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                       Number of devices                       |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                                               |
+    |             Device paths              +-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                       |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+Similarly, each device path is a length-prefixed string. Unlike C strings, the
+path is *not* NULL-terminated.
+
+::
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                     Length of device path                     |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                                               |
+    |                          Device path                          |
+    |                                                               |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+gamepad recording
+~~~~~~~~~~~~~~~~~
+
+The recording has been made from a specific gamepad. All events in the stream
+will be for that device only. The section describes the device properties that
+will be used to create a virtual input device using ``/dev/uinput``. Please
+see ``linux/input.h`` header in the Linux kernel source for more information
+about the fields in this section.
+
+::
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |            bustype            |             vendor            |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |            product            |            version            |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                         name_length                           |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                                               |
+    |                             name                              |
+    |                                                               |
+    |                                                               |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                            ev_bits                            |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                                               |
+    |                                                               |
+    |                       key_bits (96 bytes)                     |
+    |                                                               |
+    |                                                               |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                                               |
+    |                                                               |
+    |                       rel_bits (96 bytes)                     |
+    |                                                               |
+    |                                                               |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                                               |
+    |                                                               |
+    |                       abs_bits (96 bytes)                     |
+    |                                                               |
+    |                                                               |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                          num_absinfo                          |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                                                               |
+    |                                                               |
+    |                                                               |
+    |                                                               |
+    |                        absinfo entries                        |
+    |                                                               |
+    |                                                               |
+    |                                                               |
+    |                                                               |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+Each ``absinfo`` entry consists of six 32 bit values. The number of entries is
+determined by the ``abs_bits`` field.
+
+
+::
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                            value                              |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                           minimum                             |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                           maximum                             |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                             fuzz                              |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                             flat                              |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                          resolution                           |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+Event structure
+---------------
+
+The majority of an revent recording will be made up of the input events that were
+recorded. The event stream is prefixed with the number of events in the stream.
+
+Each event entry structured as follows:
+
+ * An unsigned integer representing which device from the list of device paths
+   this event is for (zero indexed). E.g. Device ID = 3 would be the 4th
+   device in the list of device paths.
+ * A signed integer representing the number of seconds since "epoch" when the
+   event was recorded.
+ * A signed integer representing the microseconds part of the timestamp.
+ * An unsigned integer representing the event type
+ * An unsigned integer representing the event code
+ * An unsigned integer representing the event value
+
+For more information about the event type, code and value please read:
+https://www.kernel.org/doc/Documentation/input/event-codes.txt
+
+::
+
+     0                   1                   2                   3
+     0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |           Device ID           |        Timestamp Seconds      |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |                       Timestamp Seconds (cont.)               |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |   Timestamp Seconds (cont.)   |        stamp Micoseconds      |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |              Timestamp Micoseconds (cont.)                    |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    | Timestamp Micoseconds (cont.) |          Event Type           |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |          Event Code           |          Event Value          |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+    |       Event Value (cont.)     |
+    +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+Parser
+------
+
+WA has a parser for revent recordings. This can be used to work with revent
+recordings in scripts. Here is an example:
+
+.. code:: python
+
+    from wlauto.utils.revent import ReventRecording
+
+    with ReventRecording('/path/to/recording.revent') as recording:
+        print "Recording: {}".format(recording.filepath)
+        print "There are {} input events".format(recording.num_events)
+        print "Over a total of {} seconds".format(recording.duration)

doc/source/writing_extensions.rst

@@ -161,8 +161,8 @@ In order for the binary to be obtained in this way, it must be stored in one of
 the locations scanned by the resource resolver in a directry structure
 ``<root>/bin/<abi>/<binary>`` (where ``root`` is the base resource location to
 be searched, e.g. ``~/.workload_automation/depencencies/<extension name>``, and
- ``<abi>`` is the ABI for which the exectuable has been compiled, as returned by
-  ``self.device.abi``).
+``<abi>`` is the ABI for which the exectuable has been compiled, as returned by
+``self.device.abi``).
 
 Once the path to the host-side binary has been obtained, it may be deployed using
 one of two methods of a ``Device`` instace -- ``install`` or ``install_if_needed``.
@@ -182,8 +182,8 @@ WA and will not try to re-install.
 
 Both of the above methods will return the path to the installed binary on the
 device. The executable should be invoked *only* via that path; do **not** assume
- that it will be in ``PATH`` on the target (or that the executable with the same
-  name in ``PATH`` is the version deployed by WA.
+that it will be in ``PATH`` on the target (or that the executable with the same
+name in ``PATH`` is the version deployed by WA.
 
 .. code:: python
 

setup.py

@@ -66,7 +66,7 @@ params = dict(
     packages=packages,
     package_data=data_files,
     scripts=scripts,
-    url='N/A',
+    url='http://github.com/arm-sowftware/workload-automation',
     license='Apache v2',
     maintainer='ARM Architecture & Technology Device Lab',
     maintainer_email='workload-automation@arm.com',
@@ -80,6 +80,7 @@ params = dict(
     ],
     extras_require={
         'other': ['jinja2', 'pandas>=0.13.1'],
+        'statedetect': ['numpy', 'imutils', 'opencv-python'],
         'test': ['nose'],
         'mongodb': ['pymongo'],
         'notify': ['notify2'],

wlauto/__init__.py

@@ -29,7 +29,7 @@ from wlauto.common.linux.device import LinuxDevice  # NOQA
 from wlauto.common.android.device import AndroidDevice, BigLittleDevice   # NOQA
 from wlauto.common.android.resources import ApkFile, JarFile
 from wlauto.common.android.workload import (UiAutomatorWorkload, ApkWorkload, AndroidBenchmark,  # NOQA
-                                    AndroidUiAutoBenchmark, GameWorkload)  # NOQA
+                                    AndroidUiAutoBenchmark, AndroidUxPerfWorkload, GameWorkload)  # NOQA
 
 from wlauto.core.version import get_wa_version
 

wlauto/agenda-example-biglittle.yaml

@@ -1,6 +1,6 @@
 # This agenda specifies configuration that may be used for regression runs 
-# on big.LITTLE systems. This agenda will with a TC2 device configured as 
-# described in the documentation.
+# on big.LITTLE systems. This agenda will work with a TC2 device configured 
+# as described in the documentation.
 config:
         device: tc2
         run_name: big.LITTLE_regression
@@ -69,7 +69,7 @@ workloads:
         - id: b10
           name: smartbench
         - id: b11
-          name: sqlite
+          name: sqlitebm
         - id: b12
           name: vellamo
 

wlauto/commands/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-

wlauto/commands/get_assets.py

@@ -0,0 +1,122 @@
+#    Copyright 2014-2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import sys
+import argparse
+
+from requests import ConnectionError, RequestException
+
+from wlauto import File, ExtensionLoader, Command, settings
+from wlauto.core.extension import Extension
+
+
+REMOTE_ASSETS_URL = 'https://github.com/ARM-software/workload-automation-assets/raw/master/dependencies'
+
+
+class GetAssetsCommand(Command):
+    name = 'get-assets'
+    description = '''
+    This command downloads external extension dependencies used by Workload Automation.
+    Works by first downloading a directory index of the assets, then iterating through
+    it to get assets for the specified extensions.
+    '''
+
+    # Uses config setting if available otherwise defaults to ARM-software repo
+    # Can be overriden with the --url argument
+    assets_url = settings.remote_assets_url or REMOTE_ASSETS_URL
+
+    def initialize(self, context):
+        self.parser.add_argument('-f', '--force', action='store_true',
+                                 help='Always fetch the assets, even if matching versions exist in local cache.')
+        self.parser.add_argument('--url', metavar='URL', type=not_empty, default=self.assets_url,
+                                 help='''The location from which to download the files. If not provided,
+                                 config setting ``remote_assets_url`` will be used if available, else
+                                 uses the default REMOTE_ASSETS_URL parameter in the script.''')
+        group = self.parser.add_mutually_exclusive_group(required=True)
+        group.add_argument('-a', '--all', action='store_true',
+                           help='Download assets for all extensions found in the index. Cannot be used with -e.')
+        group.add_argument('-e', dest='exts', metavar='EXT', nargs='+', type=not_empty,
+                           help='One or more extensions whose assets to download. Cannot be used with --all.')
+
+    def execute(self, args):
+        self.logger.debug('Program arguments: {}'.format(vars(args)))
+        if args.force:
+            self.logger.info('Force-download of assets requested')
+        if not args.url:
+            self.logger.debug('URL not provided, falling back to default setting in config')
+        self.logger.info('Downloading external assets from {}'.format(args.url))
+
+        # Get file index of assets
+        ext_loader = ExtensionLoader(packages=settings.extension_packages, paths=settings.extension_paths)
+        getter = ext_loader.get_resource_getter('http_assets', None, url=args.url, always_fetch=args.force)
+        try:
+            getter.index = getter.fetch_index()
+        except (ConnectionError, RequestException) as e:
+            self.exit_with_error(str(e))
+        all_assets = dict()
+        for k, v in getter.index.iteritems():
+            all_assets[str(k)] = [str(asset['path']) for asset in v]
+
+        # Here we get a list of all extensions present in the current WA installation,
+        # and cross-check that against the list of extensions whose assets are requested.
+        # The aim is to avoid downloading assets for extensions that do not exist, since
+        # WA extensions and asset index can be updated independently and go out of sync.
+        all_extensions = [ext.name for ext in ext_loader.list_extensions()]
+        assets_to_get = set(all_assets).intersection(all_extensions)
+        if args.exts:
+            assets_to_get = assets_to_get.intersection(args.exts)
+        # Check list is not empty
+        if not assets_to_get:
+            if args.all:
+                self.exit_with_error('Could not find extensions: {}'.format(', '.join(all_assets.keys())))
+            else:  # args.exts
+                self.exit_with_error('Asset index has no entries for: {}'.format(', '.join(args.exts)))
+
+        # Check out of sync extensions i.e. do not exist in both WA and assets index
+        missing = set(all_assets).difference(all_extensions) | set(args.exts or []).difference(all_assets)
+        if missing:
+            self.logger.warning('Not getting assets for missing extensions: {}'.format(', '.join(missing)))
+
+        # Ideally the extension loader would be used to instantiate, but it does full
+        # validation of the extension, like checking connected devices or supported
+        # platform(s). This info might be unavailable and is not required to download
+        # assets, since they are classified by extension name alone. So instead we use
+        # a simple subclass of ``Extension`` providing a valid ``name`` attribute.
+        for ext_name in assets_to_get:
+            owner = _instantiate(NamedExtension, ext_name)
+            self.logger.info('Getting assets for: {}'.format(ext_name))
+            for asset in all_assets[ext_name]:
+                getter.get(File(owner, asset))  # Download the files
+
+    def exit_with_error(self, message, code=1):
+        self.logger.error(message)
+        sys.exit(code)
+
+
+class NamedExtension(Extension):
+    def __init__(self, name, **kwargs):
+        super(NamedExtension, self).__init__(**kwargs)
+        self.name = name
+
+
+def not_empty(val):
+    if val:
+        return val
+    else:
+        raise argparse.ArgumentTypeError('Extension name cannot be blank')
+
+
+def _instantiate(cls, *args, **kwargs):
+    return cls(*args, **kwargs)

wlauto/commands/record.py

@@ -15,6 +15,8 @@
 
 import os
 import sys
+import signal
+from math import ceil
 
 from wlauto import ExtensionLoader, Command, settings
 from wlauto.common.resources import Executable
@@ -22,40 +24,10 @@ from wlauto.core.resource import NO_ONE
 from wlauto.core.resolver import ResourceResolver
 from wlauto.core.configuration import RunConfiguration
 from wlauto.core.agenda import Agenda
+from wlauto.utils.revent import ReventRecording, GAMEPAD_MODE
 
 
-class RecordCommand(Command):
-
-    name = 'record'
-    description = '''Performs a revent recording
-
-    This command helps making revent recordings. It will automatically
-    deploy revent and even has the option of automatically opening apps.
-
-    Revent allows you to record raw inputs such as screen swipes or button presses.
-    This can be useful for recording inputs for workloads such as games that don't
-    have XML UI layouts that can be used with UIAutomator. As a drawback from this,
-    revent recordings are specific to the device type they were recorded on.
-
-    WA uses two parts to the names of revent recordings in the format,
-    {device_name}.{suffix}.revent.
-
-     - device_name can either be specified manually with the ``-d`` argument or
-       it can be automatically determined. On Android device it will be obtained
-       from ``build.prop``, on Linux devices it is obtained from ``/proc/device-tree/model``.
-     - suffix is used by WA to determine which part of the app execution the
-       recording is for, currently these are either ``setup`` or ``run``. This
-       should be specified with the ``-s`` argument.
-    '''
-
-    def initialize(self, context):
-        self.context = context
-        self.parser.add_argument('-d', '--device', help='The name of the device')
-        self.parser.add_argument('-s', '--suffix', help='The suffix of the revent file, e.g. ``setup``')
-        self.parser.add_argument('-o', '--output', help='Directory to save the recording in')
-        self.parser.add_argument('-p', '--package', help='Package to launch before recording')
-        self.parser.add_argument('-C', '--clear', help='Clear app cache before launching it',
-                                 action="store_true")
+class ReventCommand(Command):
 
     # Validate command options
     def validate_args(self, args):
@@ -89,21 +61,83 @@ class RecordCommand(Command):
         self.device.initialize(context)
 
         host_binary = context.resolver.get(Executable(NO_ONE, self.device.abi, 'revent'))
-        self.target_binary = self.device.install_if_needed(host_binary)
+        self.target_binary = self.device.install_executable(host_binary)
 
         self.run(args)
 
+    def run(self, args):
+        raise NotImplementedError()
+
+
+class RecordCommand(ReventCommand):
+
+    name = 'record'
+    description = '''Performs a revent recording
+
+    This command helps making revent recordings. It will automatically
+    deploy revent and even has the option of automatically opening apps.
+
+    Revent allows you to record raw inputs such as screen swipes or button presses.
+    This can be useful for recording inputs for workloads such as games that don't
+    have XML UI layouts that can be used with UIAutomator. As a drawback from this,
+    revent recordings are specific to the device type they were recorded on.
+
+    WA uses two parts to the names of revent recordings in the format,
+    {device_name}.{suffix}.revent.
+
+     - device_name can either be specified manually with the ``-d`` argument or
+       it can be automatically determined. On Android device it will be obtained
+       from ``build.prop``, on Linux devices it is obtained from ``/proc/device-tree/model``.
+     - suffix is used by WA to determine which part of the app execution the
+       recording is for, currently these are either ``setup`` or ``run``. This
+       should be specified with the ``-s`` argument.
+
+
+    **gamepad recording**
+
+    revent supports an alternative recording mode, where it will record events
+    from a single gamepad device. In this mode, revent will store the
+    description of this device as a part of the recording. When replaying such
+    a recording, revent will first create a virtual gamepad using the
+    description, and will replay the events into it, so a physical controller
+    does not need to be connected on replay. Unlike standard revent recordings,
+    recordings generated in this mode should be (to an extent) portable across
+    different devices.
+
+    note:
+
+      - The device on which a recording is being made in gamepad mode, must have
+        exactly one gamepad connected to it.
+      - The device on which a gamepad recording is being replayed must have
+        /dev/uinput enabled in the kernel (this interface is necessary to create
+        virtual gamepad).
+
+    '''
+
+    def initialize(self, context):
+        self.context = context
+        self.parser.add_argument('-d', '--device', help='The name of the device')
+        self.parser.add_argument('-s', '--suffix', help='The suffix of the revent file, e.g. ``setup``')
+        self.parser.add_argument('-o', '--output', help='Directory to save the recording in')
+        self.parser.add_argument('-p', '--package', help='Package to launch before recording')
+        self.parser.add_argument('-g', '--gamepad', help='Record from a gamepad rather than all devices.',
+                                 action="store_true")
+        self.parser.add_argument('-C', '--clear', help='Clear app cache before launching it',
+                                 action="store_true")
+        self.parser.add_argument('-S', '--capture-screen', help='Record a screen capture after recording',
+                                 action="store_true")
+
     def run(self, args):
         if args.device:
-            self.device_name = args.device
+            device_name = args.device
         else:
-            self.device_name = self.device.get_device_model()
+            device_name = self.device.get_device_model()
 
         if args.suffix:
             args.suffix += "."
 
         revent_file = self.device.path.join(self.device.working_directory,
-                                            '{}.{}revent'.format(self.device_name, args.suffix or ""))
+                                            '{}.{}revent'.format(device_name, args.suffix or ""))
 
         if args.clear:
             self.device.execute("pm clear {}".format(args.package))
@@ -114,18 +148,24 @@ class RecordCommand(Command):
 
         self.logger.info("Press Enter when you are ready to record...")
         raw_input("")
-        command = "{} record -t 100000 -s {}".format(self.target_binary, revent_file)
+        gamepad_flag = '-g ' if args.gamepad else ''
+        command = "{} record {}-s {}".format(self.target_binary, gamepad_flag, revent_file)
         self.device.kick_off(command)
 
         self.logger.info("Press Enter when you have finished recording...")
         raw_input("")
-        self.device.killall("revent")
-
+        if args.capture_screen:
+            self.logger.info("Recording screen capture")
+            self.device.capture_screen(args.output or os.getcwdu())
+        self.device.killall("revent", signal.SIGINT)
+        self.logger.info("Waiting for revent to finish")
+        while self.device.get_pids_of("revent"):
+            pass
         self.logger.info("Pulling files from device")
         self.device.pull_file(revent_file, args.output or os.getcwdu())
 
 
-class ReplayCommand(RecordCommand):
+class ReplayCommand(ReventCommand):
 
     name = 'replay'
     description = '''Replay a revent recording
@@ -154,8 +194,13 @@ class ReplayCommand(RecordCommand):
             self.logger.info("Starting {}".format(args.package))
             self.device.execute('monkey -p {} -c android.intent.category.LAUNCHER 1'.format(args.package))
 
+        self.logger.info("Replaying recording")
         command = "{} replay {}".format(self.target_binary, revent_file)
-        self.device.execute(command)
+        recording = ReventRecording(args.revent)
+        timeout = ceil(recording.duration) + 30
+        recording.close()
+        self.device.execute(command, timeout=timeout,
+                            as_root=(recording.mode == GAMEPAD_MODE))
         self.logger.info("Finished replay")
 
 

wlauto/commands/run.py

@@ -20,6 +20,7 @@ import shutil
 
 import wlauto
 from wlauto import Command, settings
+from wlauto.exceptions import ConfigError
 from wlauto.core.agenda import Agenda
 from wlauto.core.execution import Executor
 from wlauto.utils.log import add_log_file
@@ -76,6 +77,11 @@ class RunCommand(Command):
             agenda = Agenda(args.agenda)
             settings.agenda = args.agenda
             shutil.copy(args.agenda, settings.meta_directory)
+
+            if len(agenda.workloads) == 0:
+                raise ConfigError("No workloads specified")
+        elif '.' in args.agenda or os.sep in args.agenda:
+            raise ConfigError('Agenda "{}" does not exist.'.format(args.agenda))
         else:
             self.logger.debug('{} is not a file; assuming workload name.'.format(args.agenda))
             agenda = Agenda()

wlauto/commands/show.py

@@ -111,4 +111,3 @@ def format_extension_parameters(extension, out, width, shift=4):
         param_texts.append(indent(param_text, shift))
 
     out.write(format_column('\n'.join(param_texts), width))
-

wlauto/commands/templates/android_benchmark

@@ -14,7 +14,7 @@ class ${class_name}(AndroidBenchmark):
 
     parameters = [
         # Workload parameters go here e.g.
-        Parameter('Example parameter', kind=int, allowed_values=[1,2,3], default=1, override=True, mandatory=False,
+        Parameter('example_parameter', kind=int, allowed_values=[1,2,3], default=1, override=True, mandatory=False,
                   description='This is an example parameter')
     ]
 

wlauto/commands/templates/android_uiauto_benchmark

@@ -14,7 +14,7 @@ class ${class_name}(AndroidUiAutoBenchmark):
 
     parameters = [
         # Workload parameters go here e.g.
-        Parameter('Example parameter', kind=int, allowed_values=[1,2,3], default=1, override=True, mandatory=False,
+        Parameter('example_parameter', kind=int, allowed_values=[1,2,3], default=1, override=True, mandatory=False,
                   description='This is an example parameter')
     ]
 

wlauto/commands/templates/basic_workload

@@ -8,7 +8,7 @@ class ${class_name}(Workload):
 
     parameters = [
         # Workload parameters go here e.g.
-        Parameter('Example parameter', kind=int, allowed_values=[1,2,3], default=1, override=True, mandatory=False,
+        Parameter('example_parameter', kind=int, allowed_values=[1,2,3], default=1, override=True, mandatory=False,
                   description='This is an example parameter')
     ]
 

wlauto/commands/templates/uiauto_workload

@@ -8,7 +8,7 @@ class  ${class_name}(UiAutomatorWorkload):
 
     parameters = [
         # Workload parameters go here e.g.
-        Parameter('Example parameter', kind=int, allowed_values=[1,2,3], default=1, override=True, mandatory=False,
+        Parameter('example_parameter', kind=int, allowed_values=[1,2,3], default=1, override=True, mandatory=False,
                   description='This is an example parameter')
     ]
 

wlauto/common/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-

wlauto/common/android/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-

wlauto/common/android/device.py

@@ -21,12 +21,16 @@ import time
 import tempfile
 import shutil
 import threading
+import json
+import xml.dom.minidom
 from subprocess import CalledProcessError
 
 from wlauto.core.extension import Parameter
+from wlauto.common.resources import Executable
+from wlauto.core.resource import NO_ONE
 from wlauto.common.linux.device import BaseLinuxDevice, PsEntry
 from wlauto.exceptions import DeviceError, WorkerThreadError, TimeoutError, DeviceNotRespondingError
-from wlauto.utils.misc import convert_new_lines
+from wlauto.utils.misc import convert_new_lines, ABI_MAP
 from wlauto.utils.types import boolean, regex
 from wlauto.utils.android import (adb_shell, adb_background_shell, adb_list_devices,
                                   adb_command, AndroidProperties, ANDROID_VERSION_MAP)
@@ -49,9 +53,8 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
                   description='The unique ID of the device as output by "adb devices".'),
         Parameter('android_prompt', kind=regex, default=re.compile('^.*(shell|root)@.*:/\S* [#$] ', re.MULTILINE),
                   description='The format  of matching the shell prompt in Android.'),
-        Parameter('working_directory', default='/sdcard/wa-working',
-                  description='Directory that will be used WA on the device for output files etc.'),
-        Parameter('binaries_directory', default='/data/local/tmp', override=True,
+        Parameter('working_directory', default='/sdcard/wa-working', override=True),
+        Parameter('binaries_directory', default='/data/local/tmp/wa-bin', override=True,
                   description='Location of binaries on the device.'),
         Parameter('package_data_directory', default='/data/data',
                   description='Location of of data for an installed package (APK).'),
@@ -105,19 +108,34 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
 
     @property
     def abi(self):
-        return self.getprop()['ro.product.cpu.abi'].split('-')[0]
+        val = self.getprop()['ro.product.cpu.abi'].split('-')[0]
+        for abi, architectures in ABI_MAP.iteritems():
+            if val in architectures:
+                return abi
+        return val
 
     @property
-    def supported_eabi(self):
+    def supported_abi(self):
         props = self.getprop()
         result = [props['ro.product.cpu.abi']]
         if 'ro.product.cpu.abi2' in props:
             result.append(props['ro.product.cpu.abi2'])
         if 'ro.product.cpu.abilist' in props:
-            for eabi in props['ro.product.cpu.abilist'].split(','):
-                if eabi not in result:
-                    result.append(eabi)
-        return result
+            for abi in props['ro.product.cpu.abilist'].split(','):
+                if abi not in result:
+                    result.append(abi)
+
+        mapped_result = []
+        for supported_abi in result:
+            for abi, architectures in ABI_MAP.iteritems():
+                found = False
+                if supported_abi in architectures and abi not in mapped_result:
+                    mapped_result.append(abi)
+                    found = True
+                    break
+            if not found and supported_abi not in mapped_result:
+                mapped_result.append(supported_abi)
+        return mapped_result
 
     def __init__(self, **kwargs):
         super(AndroidDevice, self).__init__(**kwargs)
@@ -193,6 +211,7 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
         self._is_ready = True
 
     def initialize(self, context):
+        self.sqlite = self.deploy_sqlite3(context)  # pylint: disable=attribute-defined-outside-init
         if self.is_rooted:
             self.disable_screen_lock()
             self.disable_selinux()
@@ -258,6 +277,24 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
                 return line.split('=', 1)[1]
         return None
 
+    def get_installed_package_abi(self, package):
+        """
+        Returns the primary abi of the specified package if it is installed
+        on the device, or ``None`` otherwise.
+        """
+        output = self.execute('dumpsys package {}'.format(package))
+        val = None
+        for line in convert_new_lines(output).split('\n'):
+            if 'primaryCpuAbi' in line:
+                val = line.split('=', 1)[1]
+                break
+        if val == 'null':
+            return None
+        for abi, architectures in ABI_MAP.iteritems():
+            if val in architectures:
+                return abi
+        return val
+
     def list_packages(self):
         """
         List packages installed on the device.
@@ -338,7 +375,7 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
 
     def delete_file(self, filepath, as_root=False):  # pylint: disable=W0221
         self._check_ready()
-        adb_shell(self.adb_name, "rm '{}'".format(filepath), as_root=as_root, timeout=self.default_timeout)
+        adb_shell(self.adb_name, "rm -rf '{}'".format(filepath), as_root=as_root, timeout=self.default_timeout)
 
     def file_exists(self, filepath):
         self._check_ready()
@@ -346,18 +383,26 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
                            timeout=self.default_timeout)
         return bool(int(output))
 
-    def install(self, filepath, timeout=default_timeout, with_name=None):  # pylint: disable=W0221
+    def install(self, filepath, timeout=default_timeout, with_name=None, replace=False):  # pylint: disable=W0221
         ext = os.path.splitext(filepath)[1].lower()
         if ext == '.apk':
-            return self.install_apk(filepath, timeout)
+            return self.install_apk(filepath, timeout, replace)
         else:
             return self.install_executable(filepath, with_name)
 
-    def install_apk(self, filepath, timeout=default_timeout):  # pylint: disable=W0221
+    def install_apk(self, filepath, timeout=default_timeout, replace=False, allow_downgrade=False):  # pylint: disable=W0221
         self._check_ready()
         ext = os.path.splitext(filepath)[1].lower()
         if ext == '.apk':
-            return adb_command(self.adb_name, "install {}".format(filepath), timeout=timeout)
+            flags = []
+            if replace:
+                flags.append('-r')  # Replace existing APK
+            if allow_downgrade:
+                flags.append('-d')  # Install the APK even if a newer version is already installed
+            if self.get_sdk_version() >= 23:
+                flags.append('-g')  # Grant all runtime permissions
+            self.logger.debug("Replace APK = {}, ADB flags = '{}'".format(replace, ' '.join(flags)))
+            return adb_command(self.adb_name, "install {} '{}'".format(' '.join(flags), filepath), timeout=timeout)
         else:
             raise DeviceError('Can\'t install {}: unsupported format.'.format(filepath))
 
@@ -442,22 +487,20 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
         else:
             return adb_shell(self.adb_name, command, timeout, check_exit_code, as_root)
 
-    def kick_off(self, command):
+    def kick_off(self, command, as_root=None):
         """
         Like execute but closes adb session and returns immediately, leaving the command running on the
         device (this is different from execute(background=True) which keeps adb connection open and returns
         a subprocess object).
 
-        .. note:: This relies on busybox's nohup applet and so won't work on unrooted devices.
-
         Added in version 2.1.4
 
         """
-        if not self.is_rooted:
-            raise DeviceError('kick_off uses busybox\'s nohup applet and so can only be run a rooted device.')
+        if as_root is None:
+            as_root = self.is_rooted
         try:
-            command = 'cd {} && busybox nohup {}'.format(self.working_directory, command)
-            output = self.execute(command, timeout=1, as_root=True)
+            command = 'cd {} && {} nohup {}'.format(self.working_directory, self.busybox, command)
+            output = self.execute(command, timeout=1, as_root=as_root)
         except TimeoutError:
             pass
         else:
@@ -465,8 +508,8 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
 
     def get_pids_of(self, process_name):
         """Returns a list of PIDs of all processes with the specified name."""
-        result = self.execute('ps | {} grep {}'.format(self.busybox, process_name),
-                              check_exit_code=False).strip()
+        result = (self.execute('ps | {} grep {}'.format(self.busybox, process_name),
+                               check_exit_code=False) or '').strip()
         if result and 'not found' not in result:
             return [int(x.split()[1]) for x in result.split('\n')]
         else:
@@ -505,17 +548,18 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
     def _get_android_properties(self, context):
         props = {}
         props['android_id'] = self.get_android_id()
-        buildprop_file = os.path.join(context.host_working_directory, 'build.prop')
-        if not os.path.isfile(buildprop_file):
-            self.pull_file('/system/build.prop', context.host_working_directory)
-        self._update_build_properties(buildprop_file, props)
-        context.add_run_artifact('build_properties', buildprop_file, 'export')
+        self._update_build_properties(props)
 
         dumpsys_target_file = self.path.join(self.working_directory, 'window.dumpsys')
         dumpsys_host_file = os.path.join(context.host_working_directory, 'window.dumpsys')
         self.execute('{} > {}'.format('dumpsys window', dumpsys_target_file))
         self.pull_file(dumpsys_target_file, dumpsys_host_file)
         context.add_run_artifact('dumpsys_window', dumpsys_host_file, 'meta')
+
+        prop_file = os.path.join(context.host_working_directory, 'android-props.json')
+        with open(prop_file, 'w') as wfh:
+            json.dump(props, wfh)
+        context.add_run_artifact('android_properties', prop_file, 'export')
         return props
 
     def getprop(self, prop=None):
@@ -529,6 +573,11 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
             return props[prop]
         return props
 
+    def deploy_sqlite3(self, context):
+        host_file = context.resolver.get(Executable(NO_ONE, self.abi, 'sqlite3'))
+        target_file = self.install_if_needed(host_file)
+        return target_file
+
     # Android-specific methods. These either rely on specifics of adb or other
     # Android-only concepts in their interface and/or implementation.
 
@@ -603,6 +652,17 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
         self.pull_file(on_device_file, filepath)
         self.delete_file(on_device_file)
 
+    def capture_ui_hierarchy(self, filepath):
+        """Captures the current view hierarchy into the specified file in a XML format."""
+        on_device_file = self.path.join(self.working_directory, 'screen_capture.xml')
+        self.execute('uiautomator dump {}'.format(on_device_file))
+        self.pull_file(on_device_file, filepath)
+        self.delete_file(on_device_file)
+
+        parsed_xml = xml.dom.minidom.parse(filepath)
+        with open(filepath, 'w') as f:
+            f.write(parsed_xml.toprettyxml())
+
     def is_screen_on(self):
         """Returns ``True`` if the device screen is currently on, ``False`` otherwise."""
         output = self.execute('dumpsys power')
@@ -629,7 +689,15 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
         """
         lockdb = '/data/system/locksettings.db'
         sqlcommand = "update locksettings set value='0' where name='screenlock.disabled';"
-        self.execute('sqlite3 {} "{}"'.format(lockdb, sqlcommand), as_root=True)
+        f = tempfile.NamedTemporaryFile()
+        try:
+            f.write('{} {} "{}"'.format(self.sqlite, lockdb, sqlcommand))
+            f.flush()
+            on_device_executable = self.install_executable(f.name,
+                                                           with_name="disable_screen_lock")
+        finally:
+            f.close()
+        self.execute(on_device_executable, as_root=True)
 
     def disable_selinux(self):
         # This may be invoked from intialize() so we can't use execute() or the
@@ -649,17 +717,24 @@ class AndroidDevice(BaseLinuxDevice):  # pylint: disable=W0223
         except KeyError:
             return None
 
+    def broadcast_media_mounted(self, dirpath):
+        """
+        Force a re-index of the mediaserver cache for the specified directory.
+        """
+        command = 'am broadcast -a android.intent.action.MEDIA_MOUNTED -d file://'
+        self.execute(command + dirpath)
+
     # Internal methods: do not use outside of the class.
 
-    def _update_build_properties(self, filepath, props):
+    def _update_build_properties(self, props):
         try:
-            with open(filepath) as fh:
-                for line in fh:
-                    line = re.sub(r'#.*', '', line).strip()
-                    if not line:
-                        continue
-                    key, value = line.split('=', 1)
-                    props[key] = value
+            def strip(somestring):
+                return somestring.strip().replace('[', '').replace(']', '')
+            for line in self.execute("getprop").splitlines():
+                key, value = line.split(':', 1)
+                key = strip(key)
+                value = strip(value)
+                props[key] = value
         except ValueError:
             self.logger.warning('Could not parse build.prop.')
 

wlauto/common/android/resources.py

@@ -34,3 +34,10 @@ class JarFile(FileResource):
 class ApkFile(FileResource):
 
     name = 'apk'
+
+    def __init__(self, owner, platform=None):
+        super(ApkFile, self).__init__(owner)
+        self.platform = platform
+
+    def __str__(self):
+        return '<{}\'s {} APK>'.format(self.owner, self.platform)

wlauto/common/android/workload.py

@@ -16,19 +16,30 @@
 import os
 import sys
 import time
+from math import ceil
 
-from wlauto.core.extension import Parameter
+from distutils.version import LooseVersion
+
+from wlauto.core.extension import Parameter, ExtensionMeta, ListCollection
 from wlauto.core.workload import Workload
 from wlauto.core.resource import NO_ONE
-from wlauto.common.resources import ExtensionAsset, Executable
-from wlauto.exceptions import WorkloadError, ResourceError, ConfigError
-from wlauto.utils.android import ApkInfo, ANDROID_NORMAL_PERMISSIONS
+from wlauto.common.android.resources import ApkFile, ReventFile
+from wlauto.common.resources import ExtensionAsset, Executable, File
+from wlauto.exceptions import WorkloadError, ResourceError, DeviceError
+from wlauto.utils.android import ApkInfo, ANDROID_NORMAL_PERMISSIONS, UNSUPPORTED_PACKAGES
 from wlauto.utils.types import boolean
+from wlauto.utils.revent import ReventRecording
+import wlauto.utils.statedetect as state_detector
 import wlauto.common.android.resources
 
 
 DELAY = 5
 
+# Due to the way `super` works you have to call it at every level but WA executes some
+# methods conditionally and so has to call them directly via the class, this breaks super
+# and causes it to run things mutiple times ect. As a work around for this untill workloads
+# are reworked everything that subclasses workload calls parent methods explicitly
+
 
 class UiAutomatorWorkload(Workload):
     """
@@ -66,7 +77,7 @@ class UiAutomatorWorkload(Workload):
 
     def __init__(self, device, _call_super=True, **kwargs):  # pylint: disable=W0613
         if _call_super:
-            super(UiAutomatorWorkload, self).__init__(device, **kwargs)
+            Workload.__init__(self, device, **kwargs)
         self.uiauto_file = None
         self.device_uiauto_file = None
         self.command = None
@@ -82,12 +93,13 @@ class UiAutomatorWorkload(Workload):
             self.uiauto_package = os.path.splitext(os.path.basename(self.uiauto_file))[0]
 
     def setup(self, context):
+        Workload.setup(self, context)
         method_string = '{}.{}#{}'.format(self.uiauto_package, self.uiauto_class, self.uiauto_method)
         params_dict = self.uiauto_params
         params_dict['workdir'] = self.device.working_directory
         params = ''
         for k, v in self.uiauto_params.iteritems():
-            params += ' -e {} {}'.format(k, v)
+            params += ' -e {} "{}"'.format(k, v)
         self.command = 'uiautomator runtest {}{} -c {}'.format(self.device_uiauto_file, params, method_string)
         self.device.push_file(self.uiauto_file, self.device_uiauto_file)
         self.device.killall('uiautomator')
@@ -122,10 +134,16 @@ class ApkWorkload(Workload):
     :package: The package name of the app. This is usually a Java-style name of the form
               ``com.companyname.appname``.
     :activity: This is the initial activity of the app. This will be used to launch the
-               app during the setup.
+               app during the setup.  Many applications do not specify a launch activity so
+               this may be left blank if necessary.
     :view: The class of the main view pane of the app. This needs to be defined in order
            to collect SurfaceFlinger-derived statistics (such as FPS) for the app, but
            may otherwise be left as ``None``.
+    :launch_main: If ``False``, the default activity will not be launched (during setup),
+                  allowing workloads to start the app with an intent of their choice in
+                  the run step. This is useful for apps without a launchable default/main
+                  activity or those where it cannot be launched without intent data (which
+                  is provided at the run phase).
     :install_timeout: Timeout for the installation of the APK. This may vary wildly based on
                       the size and nature of a specific APK, and so should be defined on
                       per-workload basis.
@@ -135,6 +153,9 @@ class ApkWorkload(Workload):
                                 so, as with all timeouts, so leeway must be included in
                                 the specified value.
 
+    :min_apk_version: The minimum supported apk version for this workload. May be ``None``.
+    :max_apk_version: The maximum supported apk version for this workload. May be ``None``.
+
     .. note:: Both package and activity for a workload may be obtained from the APK using
               the ``aapt`` tool that comes with the ADT  (Android Developemnt Tools) bundle.
 
@@ -142,91 +163,223 @@ class ApkWorkload(Workload):
     package = None
     activity = None
     view = None
+    min_apk_version = None
+    max_apk_version = None
     supported_platforms = ['android']
+    launch_main = True
 
     parameters = [
         Parameter('install_timeout', kind=int, default=300,
                   description='Timeout for the installation of the apk.'),
         Parameter('check_apk', kind=boolean, default=True,
                   description='''
-                  Discover the APK for this workload on the host, and check that
-                  the version matches the one on device (if already installed).
+                  When set to True the APK file on the host will be prefered if
+                  it is a valid version and ABI, if not it will fall back to the
+                  version on the targer. When set to False the target version is
+                  prefered.
                   '''),
         Parameter('force_install', kind=boolean, default=False,
                   description='''
-                  Always re-install the APK, even if matching version is found
-                  on already installed on the device.
+                  Always re-install the APK, even if matching version is found already installed
+                  on the device. Runs ``adb install -r`` to ensure existing APK is replaced. When
+                  this is set, check_apk is ignored.
                   '''),
         Parameter('uninstall_apk', kind=boolean, default=False,
                   description='If ``True``, will uninstall workload\'s APK as part of teardown.'),
+        Parameter('exact_abi', kind=bool, default=False,
+                  description='''
+                  If ``True``, workload will check that the APK matches the target
+                  device ABI, otherwise any APK found will be used.
+                  '''),
     ]
 
     def __init__(self, device, _call_super=True, **kwargs):
         if _call_super:
-            super(ApkWorkload, self).__init__(device, **kwargs)
+            Workload.__init__(self, device, **kwargs)
         self.apk_file = None
         self.apk_version = None
         self.logcat_log = None
+        self.exact_apk_version = None
+        self.exact_abi = kwargs.get('exact_abi')
 
-    def init_resources(self, context):
-        self.apk_file = context.resolver.get(wlauto.common.android.resources.ApkFile(self),
+    def setup(self, context):  # pylint: disable=too-many-branches
+        Workload.setup(self, context)
+
+        # Get target version
+        target_version = self.device.get_installed_package_version(self.package)
+        if target_version:
+            target_version = LooseVersion(target_version)
+            self.logger.debug("Found version '{}' on target device".format(target_version))
+
+        # Get host version
+        self.apk_file = context.resolver.get(ApkFile(self, self.device.abi),
                                              version=getattr(self, 'version', None),
-                                             strict=self.check_apk)
+                                             variant_name=getattr(self, 'variant_name', None),
+                                             strict=False)
 
-    def validate(self):
-        if self.check_apk:
-            if not self.apk_file:
-                raise WorkloadError('No APK file found for workload {}.'.format(self.name))
+        # Get target abi
+        target_abi = self.device.get_installed_package_abi(self.package)
+        if target_abi:
+            self.logger.debug("Found apk with primary abi '{}' on target device".format(target_abi))
+
+        # Get host version, primary abi is first, and then try to find supported.
+        for abi in self.device.supported_abi:
+            self.apk_file = context.resolver.get(ApkFile(self, abi),
+                                                 version=getattr(self, 'version', None),
+                                                 variant_name=getattr(self, 'variant_name', None),
+                                                 strict=False)
+
+            # Stop if apk found, or if exact_abi is set only look for primary abi.
+            if self.apk_file or self.exact_abi:
+                break
+
+        host_version = None
+        if self.apk_file is not None:
+            host_version = ApkInfo(self.apk_file).version_name
+            if host_version:
+                host_version = LooseVersion(host_version)
+            self.logger.debug("Found version '{}' on host".format(host_version))
+
+        # Error if apk was not found anywhere
+        if target_version is None and host_version is None:
+            msg = "Could not find APK for '{}' on the host or target device"
+            raise ResourceError(msg.format(self.name))
+
+        if self.exact_apk_version is not None:
+            if self.exact_apk_version != target_version and self.exact_apk_version != host_version:
+                msg = "APK version '{}' not found on the host '{}' or target '{}'"
+                raise ResourceError(msg.format(self.exact_apk_version, host_version, target_version))
+
+        # Error if exact_abi and suitable apk not found on host and incorrect version on device
+        if self.exact_abi and host_version is None:
+            if target_abi != self.device.abi:
+                msg = "APK abi '{}' not found on the host and target is '{}'"
+                raise ResourceError(msg.format(self.device.abi, target_abi))
+
+        # Ensure the apk is setup on the device
+        if self.force_install:
+            self.force_install_apk(context, host_version)
+        elif self.check_apk:
+            self.prefer_host_apk(context, host_version, target_version)
         else:
-            if self.force_install:
-                raise ConfigError('force_install cannot be "True" when check_apk is set to "False".')
+            self.prefer_target_apk(context, host_version, target_version)
 
-    def setup(self, context):
-        self.initialize_package(context)
-        self.start_activity()
+        self.reset(context)
+        self.apk_version = self.device.get_installed_package_version(self.package)
+        context.add_classifiers(apk_version=self.apk_version)
+
+        if self.launch_main:
+            self.launch_package()  # launch default activity without intent data
         self.device.execute('am kill-all')  # kill all *background* activities
         self.device.clear_logcat()
 
-    def initialize_package(self, context):
-        installed_version = self.device.get_installed_package_version(self.package)
-        if self.check_apk:
-            self.initialize_with_host_apk(context, installed_version)
-        else:
-            if not installed_version:
-                message = '''{} not found found on the device and check_apk is set to "False"
-                             so host version was not checked.'''
-                raise WorkloadError(message.format(self.package))
-            message = 'Version {} installed on device; skipping host APK check.'
-            self.logger.debug(message.format(installed_version))
-            self.reset(context)
-            self.apk_version = installed_version
+    def force_install_apk(self, context, host_version):
+        if host_version is None:
+            raise ResourceError("force_install is 'True' but could not find APK on the host")
+        try:
+            self.validate_version(host_version)
+        except ResourceError as e:
+            msg = "force_install is 'True' but the host version is invalid:\n\t{}"
+            raise ResourceError(msg.format(str(e)))
+        self.install_apk(context, replace=True)
 
-    def initialize_with_host_apk(self, context, installed_version):
-        host_version = ApkInfo(self.apk_file).version_name
-        if installed_version != host_version:
-            if installed_version:
-                message = '{} host version: {}, device version: {}; re-installing...'
-                self.logger.debug(message.format(os.path.basename(self.apk_file),
-                                                 host_version, installed_version))
+    def prefer_host_apk(self, context, host_version, target_version):
+        msg = "check_apk is 'True' "
+        if host_version is None:
+            try:
+                self.validate_version(target_version)
+            except ResourceError as e:
+                msg += "but the APK was not found on the host and the target version is invalid:\n\t{}"
+                raise ResourceError(msg.format(str(e)))
             else:
-                message = '{} host version: {}, not found on device; installing...'
-                self.logger.debug(message.format(os.path.basename(self.apk_file),
-                                                 host_version))
-            self.force_install = True  # pylint: disable=attribute-defined-outside-init
-        else:
-            message = '{} version {} found on both device and host.'
-            self.logger.debug(message.format(os.path.basename(self.apk_file),
-                                             host_version))
-        if self.force_install:
-            if installed_version:
-                self.device.uninstall(self.package)
-            self.install_apk(context)
-        else:
-            self.reset(context)
-        self.apk_version = host_version
+                msg += "but the APK was not found on the host, using target version"
+                self.logger.debug(msg)
+                return
+        try:
+            self.validate_version(host_version)
+        except ResourceError as e1:
+            msg += "but the host APK version is invalid:\n\t{}\n"
+            if target_version is None:
+                msg += "The target does not have the app either"
+                raise ResourceError(msg.format(str(e1)))
+            try:
+                self.validate_version(target_version)
+            except ResourceError as e2:
+                msg += "The target version is also invalid:\n\t{}"
+                raise ResourceError(msg.format(str(e1), str(e2)))
+            else:
+                msg += "using the target version instead"
+                self.logger.debug(msg.format(str(e1)))
+        else:  # Host version is valid
+            if target_version is not None and target_version == host_version:
+                msg += " and a matching version is alread on the device, doing nothing"
+                self.logger.debug(msg)
+                return
+            msg += " and the host version is not on the target, installing APK"
+            self.logger.debug(msg)
+            self.install_apk(context, replace=True)
 
-    def start_activity(self):
-        output = self.device.execute('am start -W -n {}/{}'.format(self.package, self.activity))
+    def prefer_target_apk(self, context, host_version, target_version):
+        msg = "check_apk is 'False' "
+        if target_version is None:
+            try:
+                self.validate_version(host_version)
+            except ResourceError as e:
+                msg += "but the app was not found on the target and the host version is invalid:\n\t{}"
+                raise ResourceError(msg.format(str(e)))
+            else:
+                msg += "and the app was not found on the target, using host version"
+                self.logger.debug(msg)
+                self.install_apk(context)
+                return
+        try:
+            self.validate_version(target_version)
+        except ResourceError as e1:
+            msg += "but the target app version is invalid:\n\t{}\n"
+            if host_version is None:
+                msg += "The host does not have the APK either"
+                raise ResourceError(msg.format(str(e1)))
+            try:
+                self.validate_version(host_version)
+            except ResourceError as e2:
+                msg += "The host version is also invalid:\n\t{}"
+                raise ResourceError(msg.format(str(e1), str(e2)))
+            else:
+                msg += "Using the host APK instead"
+                self.logger.debug(msg.format(str(e1)))
+                self.install_apk(context, replace=True)
+        else:
+            msg += "and a valid version of the app is already on the target, using target app"
+            self.logger.debug(msg)
+
+    def validate_version(self, version):
+        min_apk_version = getattr(self, 'min_apk_version', None)
+        max_apk_version = getattr(self, 'max_apk_version', None)
+
+        if min_apk_version is not None and max_apk_version is not None:
+            if version < LooseVersion(min_apk_version) or \
+                    version > LooseVersion(max_apk_version):
+                msg = "version '{}' not supported. " \
+                      "Minimum version required: '{}', Maximum version known to work: '{}'"
+                raise ResourceError(msg.format(version, min_apk_version, max_apk_version))
+
+        elif min_apk_version is not None:
+            if version < LooseVersion(min_apk_version):
+                msg = "version '{}' not supported. " \
+                      "Minimum version required: '{}'"
+                raise ResourceError(msg.format(version, min_apk_version))
+
+        elif max_apk_version is not None:
+            if version > LooseVersion(max_apk_version):
+                msg = "version '{}' not supported. " \
+                      "Maximum version known to work: '{}'"
+                raise ResourceError(msg.format(version, max_apk_version))
+
+    def launch_package(self):
+        if not self.activity:
+            output = self.device.execute('am start -W {}'.format(self.package))
+        else:
+            output = self.device.execute('am start -W -n {}/{}'.format(self.package, self.activity))
         if 'Error:' in output:
             self.device.execute('am force-stop {}'.format(self.package))  # this will dismiss any erro dialogs
             raise WorkloadError(output)
@@ -238,19 +391,27 @@ class ApkWorkload(Workload):
 
         # As of android API level 23, apps can request permissions at runtime,
         # this will grant all of them so requests do not pop up when running the app
+        # This can also be done less "manually" during adb install using the -g flag
         if self.device.get_sdk_version() >= 23:
             self._grant_requested_permissions()
 
-    def install_apk(self, context):
-        output = self.device.install(self.apk_file, self.install_timeout)
+    def install_apk(self, context, replace=False):
+        success = False
+        if replace:
+            self.device.uninstall(self.package)
+        output = self.device.install_apk(self.apk_file, timeout=self.install_timeout,
+                                         replace=replace, allow_downgrade=True)
         if 'Failure' in output:
             if 'ALREADY_EXISTS' in output:
                 self.logger.warn('Using already installed APK (did not unistall properly?)')
+                self.reset(context)
             else:
                 raise WorkloadError(output)
         else:
             self.logger.debug(output)
+            success = True
         self.do_post_install(context)
+        return success
 
     def _grant_requested_permissions(self):
         dumpsys_output = self.device.execute(command="dumpsys package {}".format(self.package))
@@ -263,17 +424,27 @@ class ApkWorkload(Workload):
         for line in lines:
             if "android.permission." in line:
                 permissions.append(line.split(":")[0].strip())
-            else:
+            # Matching either of these means the end of requested permissions section
+            elif "install permissions:" in line or "runtime permissions:" in line:
                 break
 
-        for permission in permissions:
+        for permission in set(permissions):
             # "Normal" Permisions are automatically granted and cannot be changed
             permission_name = permission.rsplit('.', 1)[1]
             if permission_name not in ANDROID_NORMAL_PERMISSIONS:
-                self.device.execute("pm grant {} {}".format(self.package, permission))
+                # On some API 23+ devices, this may fail with a SecurityException
+                # on previously granted permissions. In that case, just skip as it
+                # is not fatal to the workload execution
+                try:
+                    self.device.execute("pm grant {} {}".format(self.package, permission))
+                except DeviceError as e:
+                    if "changeable permission" in e.message or "Unknown permission" in e.message:
+                        self.logger.debug(e)
+                    else:
+                        raise e
 
     def do_post_install(self, context):
-        """ May be overwritten by dervied classes."""
+        """ May be overwritten by derived classes."""
         pass
 
     def run(self, context):
@@ -297,28 +468,39 @@ AndroidBenchmark = ApkWorkload  # backward compatibility
 
 
 class ReventWorkload(Workload):
-
-    default_setup_timeout = 5 * 60  # in seconds
-    default_run_timeout = 10 * 60  # in seconds
+    # pylint: disable=attribute-defined-outside-init
 
     def __init__(self, device, _call_super=True, **kwargs):
         if _call_super:
-            super(ReventWorkload, self).__init__(device, **kwargs)
+            Workload.__init__(self, device, **kwargs)
         devpath = self.device.path
         self.on_device_revent_binary = devpath.join(self.device.binaries_directory, 'revent')
-        self.on_device_setup_revent = devpath.join(self.device.working_directory, '{}.setup.revent'.format(self.device.name))
-        self.on_device_run_revent = devpath.join(self.device.working_directory, '{}.run.revent'.format(self.device.name))
-        self.setup_timeout = kwargs.get('setup_timeout', self.default_setup_timeout)
-        self.run_timeout = kwargs.get('run_timeout', self.default_run_timeout)
+        self.setup_timeout = kwargs.get('setup_timeout', None)
+        self.run_timeout = kwargs.get('run_timeout', None)
         self.revent_setup_file = None
         self.revent_run_file = None
+        self.on_device_setup_revent = None
+        self.on_device_run_revent = None
+        self.statedefs_dir = None
 
-    def init_resources(self, context):
-        self.revent_setup_file = context.resolver.get(wlauto.common.android.resources.ReventFile(self, 'setup'))
-        self.revent_run_file = context.resolver.get(wlauto.common.android.resources.ReventFile(self, 'run'))
+        if self.check_states:
+            state_detector.check_match_state_dependencies()
 
     def setup(self, context):
+        self.revent_setup_file = context.resolver.get(ReventFile(self, 'setup'))
+        self.revent_run_file = context.resolver.get(ReventFile(self, 'run'))
+        devpath = self.device.path
+        self.on_device_setup_revent = devpath.join(self.device.working_directory,
+                                                   os.path.split(self.revent_setup_file)[-1])
+        self.on_device_run_revent = devpath.join(self.device.working_directory,
+                                                 os.path.split(self.revent_run_file)[-1])
         self._check_revent_files(context)
+        default_setup_timeout = ceil(ReventRecording(self.revent_setup_file).duration) + 30
+        default_run_timeout = ceil(ReventRecording(self.revent_run_file).duration) + 30
+        self.setup_timeout = self.setup_timeout or default_setup_timeout
+        self.run_timeout = self.run_timeout or default_run_timeout
+
+        Workload.setup(self, context)
         self.device.killall('revent')
         command = '{} replay {}'.format(self.on_device_revent_binary, self.on_device_setup_revent)
         self.device.execute(command, timeout=self.setup_timeout)
@@ -333,6 +515,7 @@ class ReventWorkload(Workload):
         pass
 
     def teardown(self, context):
+        self.device.killall('revent')
         self.device.delete_file(self.on_device_setup_revent)
         self.device.delete_file(self.on_device_run_revent)
 
@@ -356,6 +539,25 @@ class ReventWorkload(Workload):
         self.device.push_file(self.revent_run_file, self.on_device_run_revent)
         self.device.push_file(self.revent_setup_file, self.on_device_setup_revent)
 
+    def _check_statedetection_files(self, context):
+        try:
+            self.statedefs_dir = context.resolver.get(File(self, 'state_definitions'))
+        except ResourceError:
+            self.logger.warning("State definitions directory not found. Disabling state detection.")
+            self.check_states = False
+
+    def check_state(self, context, phase):
+        try:
+            self.logger.info("\tChecking workload state...")
+            screenshotPath = os.path.join(context.output_directory, "screen.png")
+            self.device.capture_screen(screenshotPath)
+            stateCheck = state_detector.verify_state(screenshotPath, self.statedefs_dir, phase)
+            if not stateCheck:
+                raise WorkloadError("Unexpected state after setup")
+        except state_detector.StateDefinitionError as e:
+            msg = "State definitions or template files missing or invalid ({}). Skipping state detection."
+            self.logger.warning(msg.format(e.message))
+
 
 class AndroidUiAutoBenchmark(UiAutomatorWorkload, AndroidBenchmark):
 
@@ -365,6 +567,11 @@ class AndroidUiAutoBenchmark(UiAutomatorWorkload, AndroidBenchmark):
         UiAutomatorWorkload.__init__(self, device, **kwargs)
         AndroidBenchmark.__init__(self, device, _call_super=False, **kwargs)
 
+    def initialize(self, context):
+        UiAutomatorWorkload.initialize(self, context)
+        AndroidBenchmark.initialize(self, context)
+        self._check_unsupported_packages()
+
     def init_resources(self, context):
         UiAutomatorWorkload.init_resources(self, context)
         AndroidBenchmark.init_resources(self, context)
@@ -381,6 +588,88 @@ class AndroidUiAutoBenchmark(UiAutomatorWorkload, AndroidBenchmark):
         UiAutomatorWorkload.teardown(self, context)
         AndroidBenchmark.teardown(self, context)
 
+    def _check_unsupported_packages(self):
+        """
+        Check for any unsupported package versions and raise an
+        exception if detected.
+
+        """
+        for package in UNSUPPORTED_PACKAGES:
+            version = self.device.get_installed_package_version(package)
+            if version is None:
+                continue
+
+            if '-' in version:
+                version = version.split('-')[0]  # ignore abi version
+
+            if version in UNSUPPORTED_PACKAGES[package]:
+                message = 'This workload does not support version "{}" of package "{}"'
+                raise WorkloadError(message.format(version, package))
+
+
+class AndroidUxPerfWorkloadMeta(ExtensionMeta):
+    to_propagate = ExtensionMeta.to_propagate + [('deployable_assets', str, ListCollection)]
+
+
+class AndroidUxPerfWorkload(AndroidUiAutoBenchmark):
+    __metaclass__ = AndroidUxPerfWorkloadMeta
+
+    deployable_assets = []
+    parameters = [
+        Parameter('markers_enabled', kind=bool, default=False,
+                  description="""
+                  If ``True``, UX_PERF action markers will be emitted to logcat during
+                  the test run.
+                  """),
+        Parameter('clean_assets', kind=bool, default=False,
+                  description="""
+                  If ``True`` pushed assets will be deleted at the end of each iteration
+                  """),
+        Parameter('force_push_assets', kind=bool, default=False,
+                  description="""
+                  If ``True`` always push assets on each iteration, even if the
+                  assets already exists in the device path
+                  """),
+    ]
+
+    def _path_on_device(self, fpath, dirname=None):
+        if dirname is None:
+            dirname = self.device.working_directory
+        fname = os.path.basename(fpath)
+        return self.device.path.join(dirname, fname)
+
+    def push_assets(self, context):
+        for f in self.deployable_assets:
+            fpath = context.resolver.get(File(self, f))
+            device_path = self._path_on_device(fpath)
+            if self.force_push_assets or not self.device.file_exists(device_path):
+                self.device.push_file(fpath, device_path, timeout=300)
+        self.device.broadcast_media_mounted(self.device.working_directory)
+
+    def delete_assets(self):
+        for f in self.deployable_assets:
+            self.device.delete_file(self._path_on_device(f))
+        self.device.broadcast_media_mounted(self.device.working_directory)
+
+    def __init__(self, device, **kwargs):
+        super(AndroidUxPerfWorkload, self).__init__(device, **kwargs)
+        # Turn class attribute into instance attribute
+        self.deployable_assets = list(self.deployable_assets)
+
+    def validate(self):
+        super(AndroidUxPerfWorkload, self).validate()
+        self.uiauto_params['package'] = self.package
+        self.uiauto_params['markers_enabled'] = self.markers_enabled
+
+    def setup(self, context):
+        super(AndroidUxPerfWorkload, self).setup(context)
+        self.push_assets(context)
+
+    def teardown(self, context):
+        super(AndroidUxPerfWorkload, self).teardown(context)
+        if self.clean_assets:
+            self.delete_assets()
+
 
 class GameWorkload(ApkWorkload, ReventWorkload):
     """
@@ -417,6 +706,9 @@ class GameWorkload(ApkWorkload, ReventWorkload):
 
     parameters = [
         Parameter('install_timeout', default=500, override=True),
+        Parameter('check_states', kind=bool, default=False, global_alias='check_game_states',
+                  description="""Use visual state detection to verify the state of the workload
+                  after setup and run"""),
         Parameter('assets_push_timeout', kind=int, default=500,
                   description='Timeout used during deployment of the assets package (if there is one).'),
         Parameter('clear_data_on_reset', kind=bool, default=True,
@@ -436,6 +728,8 @@ class GameWorkload(ApkWorkload, ReventWorkload):
     def init_resources(self, context):
         ApkWorkload.init_resources(self, context)
         ReventWorkload.init_resources(self, context)
+        if self.check_states:
+            self._check_statedetection_files(context)
 
     def setup(self, context):
         ApkWorkload.setup(self, context)
@@ -443,6 +737,10 @@ class GameWorkload(ApkWorkload, ReventWorkload):
         time.sleep(self.loading_time)
         ReventWorkload.setup(self, context)
 
+        # state detection check if it's enabled in the config
+        if self.check_states:
+            self.check_state(context, "setup_complete")
+
     def do_post_install(self, context):
         ApkWorkload.do_post_install(self, context)
         self._deploy_assets(context, self.assets_push_timeout)
@@ -462,6 +760,10 @@ class GameWorkload(ApkWorkload, ReventWorkload):
         ReventWorkload.run(self, context)
 
     def teardown(self, context):
+        # state detection check if it's enabled in the config
+        if self.check_states:
+            self.check_state(context, "run_complete")
+
         if not self.saved_state_file:
             ApkWorkload.teardown(self, context)
         else:

wlauto/common/gem5/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-

wlauto/common/linux/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-

wlauto/common/linux/device.py

@@ -39,6 +39,8 @@ FstabEntry = namedtuple('FstabEntry', ['device', 'mount_point', 'fs_type', 'opti
 PsEntry = namedtuple('PsEntry', 'user pid ppid vsize rss wchan pc state name')
 LsmodEntry = namedtuple('LsmodEntry', ['name', 'size', 'use_count', 'used_by'])
 
+GOOGLE_DNS_SERVER_ADDRESS = '8.8.8.8'
+
 
 class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
 
@@ -92,6 +94,12 @@ class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
                   '''),
         Parameter('binaries_directory',
                   description='Location of executable binaries on this device (must be in PATH).'),
+        Parameter('working_directory',
+                  description='''
+                  Working directory to be used by WA. This must be in a location where the specified user
+                  has write permissions. This will default to /home/<username>/wa (or to /root/wa, if
+                  username is 'root').
+                  '''),
 
     ]
 
@@ -128,6 +136,10 @@ class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
                 self._abi = val
         return self._abi
 
+    @property
+    def supported_abi(self):
+        return [self.abi]
+
     @property
     def online_cpus(self):
         val = self.get_sysfile_value('/sys/devices/system/cpu/online')
@@ -214,7 +226,10 @@ class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
                 outfile = os.path.join(context.host_working_directory, normname)
                 if self.is_file(propfile):
                     with open(outfile, 'w') as wfh:
-                        wfh.write(self.execute('cat {}'.format(propfile)))
+                        if propfile.endswith(".gz"):
+                            wfh.write(self.execute('{} zcat {}'.format(self.busybox, propfile)))
+                        else:
+                            wfh.write(self.execute('cat {}'.format(propfile)))
                 elif self.is_directory(propfile):
                     self.pull_file(propfile, outfile)
                 else:
@@ -309,6 +324,29 @@ class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
 
             please see: https://pythonhosted.org/wlauto/writing_extensions.html""")
 
+    def is_network_connected(self):
+        """
+        Checks for internet connectivity on the device by pinging IP address provided.
+
+        :param ip_address: IP address to ping. Default is Google's public DNS server (8.8.8.8)
+
+        :returns: ``True`` if internet is available, ``False`` otherwise.
+
+        """
+        self.logger.debug('Checking for internet connectivity...')
+        return self._ping_server(GOOGLE_DNS_SERVER_ADDRESS)
+
+    def _ping_server(self, ip_address, timeout=1, packet_count=1):
+        output = self.execute('ping -q -c {} -w {} {}'.format(packet_count, timeout, ip_address),
+                              check_exit_code=False)
+
+        if 'network is unreachable' in output.lower():
+            self.logger.debug('Cannot find IP address {}'.format(ip_address))
+            return False
+        else:
+            self.logger.debug('Found IP address {}'.format(ip_address))
+            return True
+
     def get_binary_path(self, name, search_system_binaries=True):
         """
         Searches the devices ``binary_directory`` for the given binary,
@@ -390,7 +428,7 @@ class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
         signal_string = '-s {}'.format(signal) if signal else ''
         self.execute('kill {} {}'.format(signal_string, pid), as_root=as_root)
 
-    def killall(self, process_name, signal=None, as_root=False):  # pylint: disable=W0221
+    def killall(self, process_name, signal=None, as_root=None):  # pylint: disable=W0221
         """
         Kill all processes with the specified name.
 
@@ -401,6 +439,8 @@ class BaseLinuxDevice(Device):  # pylint: disable=abstract-method
         Modified in version 2.1.5: added ``as_root`` parameter.
 
         """
+        if as_root is None:
+            as_root = self.is_rooted
         for pid in self.get_pids_of(process_name):
             self.kill(pid, signal=signal, as_root=as_root)
 
@@ -583,17 +623,11 @@ class LinuxDevice(BaseLinuxDevice):
                   description='Optionally, telnet may be used instead of ssh, though this is discouraged.'),
         Parameter('boot_timeout', kind=int, default=120,
                   description='How long to try to connect to the device after a reboot.'),
-
-        Parameter('working_directory', default=None,
-                  description='''
-                  Working directory to be used by WA. This must be in a location where the specified user
-                  has write permissions. This will default to /home/<username>/wa (or to /root/wa, if
-                  username is 'root').
-                  '''),
     ]
 
     @property
     def is_rooted(self):
+        self._check_ready()
         if self._is_rooted is None:
             # First check if the user is root
             try:
@@ -632,7 +666,11 @@ class LinuxDevice(BaseLinuxDevice):
     # Power control
 
     def reset(self):
-        self.execute('reboot', as_root=True)
+        try:
+            self.execute('reboot', as_root=True)
+        except DeviceError as e:
+            if 'Connection dropped' not in e.message:
+                raise e
         self._is_ready = False
 
     def hard_reset(self):
@@ -644,8 +682,15 @@ class LinuxDevice(BaseLinuxDevice):
         else:
             self.reset()
         self.logger.debug('Waiting for device...')
+        # Wait a fixed delay before starting polling to give the device time to
+        # shut down, otherwise, might create the connection while it's still shutting
+        # down resulting in subsequenct connection failing.
+        initial_delay = 20
+        time.sleep(initial_delay)
+        boot_timeout = max(self.boot_timeout - initial_delay, 10)
+
         start_time = time.time()
-        while (time.time() - start_time) < self.boot_timeout:
+        while (time.time() - start_time) < boot_timeout:
             try:
                 s = socket.create_connection((self.host, self.port), timeout=5)
                 s.close()
@@ -669,15 +714,6 @@ class LinuxDevice(BaseLinuxDevice):
 
     # Execution
 
-    def has_root(self):
-        try:
-            self.execute('ls /', as_root=True)
-            return True
-        except DeviceError as e:
-            if 'not in the sudoers file' not in e.message:
-                raise e
-            return False
-
     def execute(self, command, timeout=default_timeout, check_exit_code=True, background=False,
                 as_root=False, strip_colors=True, **kwargs):
         """
@@ -720,13 +756,15 @@ class LinuxDevice(BaseLinuxDevice):
         except CalledProcessError as e:
             raise DeviceError(e)
 
-    def kick_off(self, command, as_root=False):
+    def kick_off(self, command, as_root=None):
         """
-        Like execute but closes adb session and returns immediately, leaving the command running on the
-        device (this is different from execute(background=True) which keeps adb connection open and returns
+        Like execute but closes ssh session and returns immediately, leaving the command running on the
+        device (this is different from execute(background=True) which keeps ssh connection open and returns
         a subprocess object).
 
         """
+        if as_root is None:
+            as_root = self.is_rooted
         self._check_ready()
         command = 'sh -c "{}" 1>/dev/null 2>/dev/null &'.format(escape_double_quotes(command))
         return self.shell.execute(command, as_root=as_root)

wlauto/config_example.py

@@ -54,6 +54,9 @@ retry_on_status = ['FAILED', 'PARTIAL']
 # How many times a job will be re-run before giving up
 max_retries = 3
 
+# If WA should delete its files from the device after the run is completed
+clean_up = False
+
 ####################################################################################################
 ######################################### Device Settings ##########################################
 ####################################################################################################
@@ -189,7 +192,7 @@ logging = {
 ####################################################################################################
 #################################### Instruments Configuration #####################################
 ####################################################################################################
-# Instrumention Configuration is related to specific insturment's settings. Some of the            #
+# Instrumention Configuration is related to specific instrument's settings. Some of the            #
 # instrumentations require specific settings in order for them to work. These settings are         #
 # specified here.                                                                                  #
 # Note that these settings only take effect if the corresponding instrument is
@@ -222,10 +225,10 @@ logging = {
 ####################################################################################################
 ######################################### DAQ configuration ########################################
 
-# The host address of the machine that runs the daq Server which the insturment communicates with
+# The host address of the machine that runs the daq Server which the instrument communicates with
 #daq_server_host = '10.1.17.56'
 
-# The port number for daq Server in which daq insturment communicates with
+# The port number for daq Server in which daq instrument communicates with
 #daq_server_port = 56788
 
 # The values of resistors 1 and 2 (in Ohms) across which the voltages are measured

wlauto/core/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-

wlauto/core/bootstrap.py

@@ -114,8 +114,8 @@ class ConfigLoader(object):
                 new_config = load_struct_from_yaml(source)
             else:
                 raise ConfigError('Unknown config format: {}'.format(source))
-        except LoadSyntaxError as e:
-            raise ConfigError(e)
+        except (LoadSyntaxError, ValueError) as e:
+            raise ConfigError('Invalid config "{}":\n\t{}'.format(source, e))
 
         self._config = merge_dicts(self._config, new_config,
                                    list_duplicates='first',
@@ -211,4 +211,3 @@ if os.path.isfile(_packages_file):
 
 for config in _env_configs:
     settings.update(config)
-

wlauto/core/configuration.py

@@ -481,6 +481,7 @@ class RunConfiguration(object):
         RunConfigurationItem('flashing_config', 'dict', 'replace'),
         RunConfigurationItem('retry_on_status', 'list', 'replace'),
         RunConfigurationItem('max_retries', 'scalar', 'replace'),
+        RunConfigurationItem('clean_up', 'scalar', 'replace'),
     ]
 
     # Configuration specified for each workload spec. "workload_parameters"
@@ -757,7 +758,7 @@ class RunConfiguration(object):
         if spec.match_selectors(selectors):
             instrumentation_config = self._raw_config['instrumentation']
             for instname in spec.instrumentation:
-                if instname not in instrumentation_config:
+                if instname not in instrumentation_config and not instname.startswith('~'):
                     instrumentation_config.append(instname)
             self.workload_specs.append(spec)
 

wlauto/core/device.py

@@ -426,6 +426,13 @@ class Device(Extension):
         """
         pass
 
+    def is_network_connected(self):
+        """
+        Checks if the device is connected to the internet
+
+        """
+        raise NotImplementedError()
+
     def __str__(self):
         return 'Device<{}>'.format(self.name)
 

wlauto/core/execution.py

@@ -56,7 +56,8 @@ from wlauto.core.extension_loader import ExtensionLoader
 from wlauto.core.resolver import ResourceResolver
 from wlauto.core.result import ResultManager, IterationResult, RunResult
 from wlauto.exceptions import (WAError, ConfigError, TimeoutError, InstrumentError,
-                               DeviceError, DeviceNotRespondingError)
+                               DeviceError, DeviceNotRespondingError, ResourceError,
+                               HostError)
 from wlauto.utils.misc import ensure_directory_exists as _d, get_traceback, merge_dicts, format_duration
 
 
@@ -204,6 +205,9 @@ class ExecutionContext(object):
     def add_metric(self, *args, **kwargs):
         self.result.add_metric(*args, **kwargs)
 
+    def add_classifiers(self, **kwargs):
+        self.result.classifiers.update(kwargs)
+
     def add_artifact(self, name, path, kind, *args, **kwargs):
         if self.current_job is None:
             self.add_run_artifact(name, path, kind, *args, **kwargs)
@@ -341,6 +345,11 @@ class Executor(object):
         runner = self._get_runner(result_manager)
         runner.init_queue(self.config.workload_specs)
         runner.run()
+
+        if getattr(self.config, "clean_up", False):
+            self.logger.info('Clearing WA files from device')
+            self.device.delete_file(self.device.binaries_directory)
+            self.device.delete_file(self.device.working_directory)
         self.execute_postamble()
 
     def execute_postamble(self):
@@ -728,6 +737,13 @@ class Runner(object):
             filepath = os.path.join(settings.output_directory, filename)
         self.device.capture_screen(filepath)
 
+    def _take_uiautomator_dump(self, filename):
+        if self.context.output_directory:
+            filepath = os.path.join(self.context.output_directory, filename)
+        else:
+            filepath = os.path.join(settings.output_directory, filename)
+        self.device.capture_ui_hierarchy(filepath)
+
     @contextmanager
     def _handle_errors(self, action, on_error_status=IterationResult.FAILED):
         try:
@@ -741,15 +757,21 @@ class Runner(object):
             if self.current_job:
                 self.current_job.result.status = on_error_status
                 self.current_job.result.add_event(str(we))
-            try:
-                self._take_screenshot('error.png')
-            except Exception, e:  # pylint: disable=W0703
-                # We're already in error state, so the fact that taking a
-                # screenshot failed is not surprising...
-                pass
+
+            # There is no point in taking a screenshot ect if the issue is not
+            # with the device but with the host or a missing resource
+            if not (isinstance(we, ResourceError) or isinstance(we, HostError)):
+                try:
+                    self._take_screenshot('error.png')
+                    if self.device.platform == 'android':
+                        self._take_uiautomator_dump('error.uix')
+                except Exception, e:  # pylint: disable=W0703
+                    # We're already in error state, so the fact that taking a
+                    # screenshot failed is not surprising...
+                    pass
             if action:
                 action = action[0].lower() + action[1:]
-            self.logger.error('Error while {}:\n\t{}'.format(action, we))
+            self.logger.error('Error while {}:\n\t{}'.format(action, str(we).replace("\n", "\n\t")))
         except Exception, e:  # pylint: disable=W0703
             error_text = '{}("{}")'.format(e.__class__.__name__, e)
             if self.current_job:

wlauto/core/extension.py

@@ -224,18 +224,11 @@ class Param(object):
         else:
             new_value = current_value + [value]
             setattr(obj, self.name, new_value)
-
-    def validate(self, obj):
-        value = getattr(obj, self.name, None)
         if value is not None:
             if self.allowed_values:
                 self._validate_allowed_values(obj, value)
             if self.constraint:
                 self._validate_constraint(obj, value)
-        else:
-            if self.mandatory:
-                msg = 'No value specified for mandatory parameter {} in {}.'
-                raise ConfigError(msg.format(self.name, obj.name))
 
     def get_type_name(self):
         typename = str(self.kind)
@@ -567,7 +560,9 @@ class Extension(object):
         if self.name is None:
             raise ValidationError('Name not set for {}'.format(self._classname))
         for param in self.parameters:
-            param.validate(self)
+            if param.mandatory and getattr(self, param.name, None) is None:
+                msg = 'No value specified for mandatory parameter {} in {}.'
+                raise ConfigError(msg.format(param.name, self.name))
 
     def initialize(self, context):
         pass

wlauto/core/exttype.py

@@ -32,4 +32,3 @@ def get_extension_type(ext):
         if isinstance(ext, cls):
             return name
     raise ValueError('Unknown extension type: {}'.format(ext.__class__.__name__))
-

wlauto/core/instrumentation.py

@@ -241,7 +241,7 @@ class ManagedCallback(object):
             except (KeyboardInterrupt, DeviceNotRespondingError, TimeoutError):  # pylint: disable=W0703
                 raise
             except Exception as e:  # pylint: disable=W0703
-                logger.error('Error in insturment {}'.format(self.instrument.name))
+                logger.error('Error in instrument {}'.format(self.instrument.name))
                 global failures_detected  # pylint: disable=W0603
                 failures_detected = True
                 if isinstance(e, WAError):
@@ -396,4 +396,3 @@ class Instrument(Extension):
 
     def __repr__(self):
         return 'Instrument({})'.format(self.name)
-

wlauto/core/resolver.py

@@ -69,7 +69,11 @@ class ResourceResolver(object):
                 self.logger.debug('\t{}'.format(result))
                 return result
         if strict:
-            raise ResourceError('{} could not be found'.format(resource))
+            if kwargs:
+                criteria = ', '.join(['{}:{}'.format(k, v) for k, v in kwargs.iteritems()])
+                raise ResourceError('{} ({}) could not be found'.format(resource, criteria))
+            else:
+                raise ResourceError('{} could not be found'.format(resource))
         self.logger.debug('Resource {} not found.'.format(resource))
         return None
 

wlauto/core/result.py

@@ -327,4 +327,3 @@ class Metric(object):
         return '<{}>'.format(result)
 
     __repr__ = __str__
-

wlauto/core/signal.py

@@ -186,4 +186,3 @@ def send(signal, sender, *args, **kwargs):
 
     """
     dispatcher.send(signal, sender, *args, **kwargs)
-

wlauto/core/version.py

@@ -18,7 +18,7 @@ from collections import namedtuple
 
 VersionTuple = namedtuple('Version', ['major', 'minor', 'revision'])
 
-version = VersionTuple(2, 4, 0)
+version = VersionTuple(2, 6, 0)
 
 
 def get_wa_version():

wlauto/core/workload.py

@@ -37,6 +37,7 @@ class Workload(Extension):
     supported_devices = []
     supported_platforms = []
     summary_metrics = []
+    requires_network = False
 
     def __init__(self, device, **kwargs):
         """
@@ -69,7 +70,7 @@ class Workload(Extension):
         """
         pass
 
-    def setup(self, context):
+    def setup(self, context):  # pylint: disable=unused-argument
         """
         Perform the setup necessary to run the workload, such as copying the necessary files
         to the device, configuring the environments, etc.
@@ -78,7 +79,8 @@ class Workload(Extension):
         the workload.
 
         """
-        pass
+        if self.requires_network:
+            self.check_network_connected()
 
     def run(self, context):
         """Execute the workload. This is the method that performs the actual "work" of the"""
@@ -99,6 +101,10 @@ class Workload(Extension):
     def finalize(self, context):
         pass
 
+    def check_network_connected(self):
+        if not self.device.is_network_connected():
+            message = 'Workload "{}" requires internet. Device "{}" does not appear to be connected to the internet.'
+            raise WorkloadError(message.format(self.name, self.device.name))
+
     def __str__(self):
         return '<Workload {}>'.format(self.name)
-

wlauto/devices/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-

wlauto/devices/android/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-

wlauto/devices/android/gem5/__init__.py

@@ -199,16 +199,6 @@ class Gem5AndroidDevice(BaseGem5Device, AndroidDevice):
         props = self._get_android_properties(context)
         return props
 
-    def disable_screen_lock(self):
-        """
-        Attempts to disable he screen lock on the device.
-
-        Overridden here as otherwise we have issues with too many backslashes.
-        """
-        lockdb = '/data/system/locksettings.db'
-        sqlcommand = "update locksettings set value=\'0\' where name=\'screenlock.disabled\';"
-        self.execute('sqlite3 {} "{}"'.format(lockdb, sqlcommand), as_root=True)
-
     def capture_screen(self, filepath):
         if BaseGem5Device.capture_screen(self, filepath):
             return

wlauto/devices/android/juno/__init__.py

@@ -81,9 +81,7 @@ class Juno(BigLittleDevice):
                       'fdt_support': True,
                   }
                   ),
-        Parameter('bootargs', default='console=ttyAMA0,115200 earlyprintk=pl011,0x7ff80000 '
-                                      'verbose debug init=/init root=/dev/sda1 rw ip=dhcp '
-                                      'rootwait video=DVI-D-1:1920x1080R@60',
+        Parameter('bootargs',
                   description='''Default boot arguments to use when boot_arguments were not.'''),
     ]
 
@@ -158,7 +156,7 @@ class Juno(BigLittleDevice):
                         target.sendline('ip addr list eth0')
                         time.sleep(1)
                         try:
-                            target.expect('inet ([1-9]\d*.\d+.\d+.\d+)', timeout=10)
+                            target.expect(r'inet ([1-9]\d*.\d+.\d+.\d+)', timeout=10)
                             self.adb_name = target.match.group(1) + ':5555'  # pylint: disable=W0201
                             break
                         except pexpect.TIMEOUT:
@@ -220,4 +218,3 @@ class Juno(BigLittleDevice):
     def get_android_id(self):
         # Android ID currenlty not set properly in Juno Android builds.
         return 'abad1deadeadbeef'
-

wlauto/devices/android/odroidxu3/__init__.py

@@ -35,4 +35,3 @@ class OdroidXU3(AndroidDevice):
                   description='Serial port on which the device is connected'),
         Parameter('baudrate', default=115200, kind=int, description='Serial connection baud rate'),
     ]
-

wlauto/devices/android/tc2/__init__.py

@@ -847,4 +847,3 @@ def _slow_sendline(target, line):
         target.send(c)
         time.sleep(0.1)
     target.sendline('')
-

wlauto/devices/linux/XE503C12/__init__.py

@@ -33,4 +33,3 @@ class Xe503c12Chormebook(LinuxDevice):
     ]
 
     abi = 'armeabi'
-

wlauto/devices/linux/__init__.py

@@ -12,5 +12,3 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
-
-

wlauto/devices/linux/chromeos_test_image/__init__.py

@@ -97,4 +97,3 @@ class ChromeOsDevice(LinuxDevice):
         else:
             pass
         self.ui_status = None
-

wlauto/devices/linux/odroidxu3_linux/__init__.py

@@ -32,4 +32,3 @@ class OdroidXU3LinuxDevice(LinuxDevice):
     ]
 
     abi = 'armeabi'
-

wlauto/external/daq_server/src/daqpower/log.py

@@ -55,4 +55,3 @@ logObserver.start()
 
 def start_logging(level, fmt='%(asctime)s %(levelname)-8s: %(message)s'):
     logging.basicConfig(level=getattr(logging, level), format=fmt)
-

wlauto/external/revent/Makefile

@@ -1,7 +1,12 @@
 # CROSS_COMPILE=aarch64-linux-gnu- make
 #
 CC=gcc
-CFLAGS=-static -lc
+
+ifdef DEBUG
+	CFLAGS=-static -lc -g
+else
+	CFLAGS=-static -lc -O2
+endif
 
 revent: revent.c
 	$(CROSS_COMPILE)$(CC) $(CFLAGS) revent.c -o revent

wlauto/external/sqlite/README

@@ -0,0 +1,11 @@
+For WA we use a slightly modified version of sqlite3 so that it can
+be built statically. We used the amalgamated sqlite3 version 3.12.2.
+which is under the public domain.
+
+https://www.sqlite.org/download.html
+
+Build command:
+gcc shell.c sqlite3.c -lpthread -ldl -static -O2 -fPIC -DPIC  -DSQLITE_OMIT_LOAD_EXTENSION
+
+You will need to apply the diff in static.patch
+

wlauto/external/sqlite/static.patch

@@ -0,0 +1,20 @@
+--- shell.c	2016-05-09 15:35:26.952309563 +0100
++++ shell.c.bak	2016-05-09 15:33:41.991259588 +0100
+@@ -4503,7 +4503,7 @@
+   static char *home_dir = NULL;
+   if( home_dir ) return home_dir;
+ 
++/*#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \
+-#if !defined(_WIN32) && !defined(WIN32) && !defined(_WIN32_WCE) \
+      && !defined(__RTP__) && !defined(_WRS_KERNEL)
+   {
+     struct passwd *pwent;
+@@ -4512,7 +4512,7 @@
+       home_dir = pwent->pw_dir;
+     }
+   }
++#endif*/
+-#endif
+ 
+ #if defined(_WIN32_WCE)
+   /* Windows CE (arm-wince-mingw32ce-gcc) does not provide getenv()

wlauto/external/uiauto/build.sh

@@ -16,6 +16,12 @@
 
 
 
+# Build and return appropriate exit code if failed
 ant build
+exit_code=$?
+if [ $exit_code -ne 0 ]; then
+    echo "ERROR: 'ant build' exited with code $exit_code"
+    exit $exit_code
+fi
 
-cp bin/classes/com/arm/wlauto/uiauto/BaseUiAutomation.class ../../common
+cp bin/classes/com/arm/wlauto/uiauto/*.class ../../common/android

wlauto/external/uiauto/project.properties

@@ -11,4 +11,4 @@
 #proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt
 
 # Project target.
-target=android-17
+target=android-18

wlauto/external/uiauto/src/com/arm/wlauto/uiauto/BaseUiAutomation.java

@@ -20,19 +20,85 @@ import java.io.File;
 import java.io.BufferedReader;
 import java.io.InputStreamReader;
 import java.util.concurrent.TimeoutException;
+import java.util.concurrent.TimeUnit;
+import java.util.Arrays;
+import java.util.ArrayList;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
 
 import android.app.Activity;
 import android.os.Bundle;
+import android.os.SystemClock;
+import android.graphics.Point;
+import android.graphics.Rect;
+import android.util.Log;
 
 // Import the uiautomator libraries
 import com.android.uiautomator.core.UiObject;
 import com.android.uiautomator.core.UiObjectNotFoundException;
 import com.android.uiautomator.core.UiScrollable;
 import com.android.uiautomator.core.UiSelector;
+import com.android.uiautomator.core.UiDevice;
+import com.android.uiautomator.core.UiWatcher;
 import com.android.uiautomator.testrunner.UiAutomatorTestCase;
 
-public class BaseUiAutomation extends UiAutomatorTestCase {   
+public class BaseUiAutomation extends UiAutomatorTestCase {
 
+    public long uiAutoTimeout = TimeUnit.SECONDS.toMillis(4);
+
+    public enum ScreenOrientation { RIGHT, NATURAL, LEFT };
+    public enum Direction { UP, DOWN, LEFT, RIGHT, NULL };
+    public enum PinchType { IN, OUT, NULL };
+
+    public static final int CLICK_REPEAT_INTERVAL_MINIMUM = 5;
+    public static final int CLICK_REPEAT_INTERVAL_DEFAULT = 50;
+
+    /*
+     * Used by clickUiObject() methods in order to provide a consistent API
+     */
+    public enum FindByCriteria { BY_ID, BY_TEXT, BY_DESC; }
+
+    /**
+     * Basic marker API for workloads to generate start and end markers for
+     * deliminating and timing actions. Markers are output to logcat with debug
+     * priority. Actions represent a series of UI interactions to time.
+     *
+     * The marker API provides a way for instruments and result processors to hook into
+     * per-action timings by parsing logcat logs produced per workload iteration.
+     *
+     * The marker output consists of a logcat tag 'UX_PERF' and a message. The
+     * message consists of a name for the action and a timestamp. The timestamp
+     * is separated by a single space from the name of the action.
+     *
+     * Typical usage:
+     *
+     * ActionLogger logger = ActionLogger("testTag", parameters);
+     * logger.start();
+     * // actions to be recorded
+     * logger.stop();
+     */
+    public class ActionLogger {
+
+        private String testTag;
+        private boolean enabled;
+
+        public ActionLogger(String testTag, Bundle parameters) {
+            this.testTag = testTag;
+            this.enabled = Boolean.parseBoolean(parameters.getString("markers_enabled"));
+        }
+
+        public void start() {
+            if (enabled) {
+                Log.d("UX_PERF", testTag + "_start " + System.nanoTime());
+            }
+        }
+
+        public void stop() throws Exception {
+            if (enabled) {
+                Log.d("UX_PERF", testTag + "_end " + System.nanoTime());
+            }
+        }
+    }
 
     public void sleep(int second) {
         super.sleep(second * 1000);
@@ -40,11 +106,11 @@ public class BaseUiAutomation extends UiAutomatorTestCase {
 
     public boolean takeScreenshot(String name) {
         Bundle params = getParams();
-	String png_dir = params.getString("workdir");
+        String pngDir = params.getString("workdir");
 
         try {
-            return getUiDevice().takeScreenshot(new File(png_dir, name + ".png"));
-        } catch(NoSuchMethodError e) {
+            return getUiDevice().takeScreenshot(new File(pngDir, name + ".png"));
+        } catch (NoSuchMethodError e) {
             return true;
         }
     }
@@ -55,9 +121,9 @@ public class BaseUiAutomation extends UiAutomatorTestCase {
 
     public void waitText(String text, int second) throws UiObjectNotFoundException {
         UiSelector selector = new UiSelector();
-        UiObject text_obj = new UiObject(selector.text(text)
+        UiObject textObj = new UiObject(selector.text(text)
                                        .className("android.widget.TextView"));
-        waitObject(text_obj, second);
+        waitObject(textObj, second);
     }
 
     public void waitObject(UiObject obj) throws UiObjectNotFoundException {
@@ -65,7 +131,7 @@ public class BaseUiAutomation extends UiAutomatorTestCase {
     }
 
     public void waitObject(UiObject obj, int second) throws UiObjectNotFoundException {
-        if (! obj.waitForExists(second * 1000)){
+        if (!obj.waitForExists(second * 1000)) {
             throw new UiObjectNotFoundException("UiObject is not found: "
                     + obj.getSelector().toString());
         }
@@ -87,10 +153,10 @@ public class BaseUiAutomation extends UiAutomatorTestCase {
 
         long currentTime = System.currentTimeMillis();
         boolean found = false;
-        while ((currentTime - startTime) < timeout){ 
+        while ((currentTime - startTime) < timeout) {
             sleep(2);  // poll every two seconds
 
-            while((line=reader.readLine())!=null) {
+            while ((line = reader.readLine()) != null) {
                 if (line.contains(searchText)) {
                     found = true;
                     break;
@@ -106,8 +172,447 @@ public class BaseUiAutomation extends UiAutomatorTestCase {
         process.destroy();
 
         if ((currentTime - startTime) >= timeout) {
-            throw new TimeoutException("Timed out waiting for Logcat text \"%s\"".format(searchText));
+            throw new TimeoutException(String.format("Timed out waiting for Logcat text \"%s\"",
+                                                     searchText));
+        }
+    }
+
+    public Integer[] splitVersion(String versionString) {
+        String pattern = "(\\d+).(\\d+).(\\d+)";
+        Pattern r = Pattern.compile(pattern);
+        ArrayList<Integer> result = new ArrayList<Integer>();
+
+        Matcher m = r.matcher(versionString);
+        if (m.find() && m.groupCount() > 0) {
+            for(int i=1; i<=m.groupCount(); i++) {
+                result.add(Integer.parseInt(m.group(i)));
+            }
+        } else {
+            throw new IllegalArgumentException(versionString + " - unknown format");
+        }
+        return result.toArray(new Integer[result.size()]);
+    }
+
+    //Return values:
+    // -1 = a lower than b
+    //  0 = a and b equal
+    //  1 = a greater than b
+    public int compareVersions(Integer[] a, Integer[] b) {
+        if (a.length != b.length) {
+            String msg = "Versions do not match format:\n %1$s\n %1$s";
+            msg = String.format(msg, Arrays.toString(a), Arrays.toString(b));
+            throw new IllegalArgumentException(msg);
+        }
+        for(int i=0; i<a.length; i++) {
+            if(a[i] > b[i])
+                return 1;
+            else if(a[i] < b[i])
+                return -1;
+        }
+        return 0;
+    }
+
+    public void registerWatcher(String name, UiWatcher watcher) {
+        UiDevice.getInstance().registerWatcher(name, watcher);
+    }
+
+    public void runWatchers() {
+        UiDevice.getInstance().runWatchers();
+    }
+
+    public void removeWatcher(String name) {
+        UiDevice.getInstance().removeWatcher(name);
+    }
+
+    public void pressEnter() {
+        UiDevice.getInstance().pressEnter();
+    }
+
+    public void pressBack() {
+        UiDevice.getInstance().pressBack();
+    }
+
+    public void pressDPadUp() {
+        UiDevice.getInstance().pressDPadUp();
+    }
+
+    public void pressDPadDown() {
+        UiDevice.getInstance().pressDPadDown();
+    }
+
+    public void pressDPadLeft() {
+        UiDevice.getInstance().pressDPadLeft();
+    }
+
+    public void pressDPadRight() {
+        UiDevice.getInstance().pressDPadRight();
+    }
+
+    public int getDisplayHeight() {
+        return UiDevice.getInstance().getDisplayHeight();
+    }
+
+    public int getDisplayWidth() {
+        return UiDevice.getInstance().getDisplayWidth();
+    }
+
+    public int getDisplayCentreWidth() {
+        return getDisplayWidth() / 2;
+    }
+
+    public int getDisplayCentreHeight() {
+        return getDisplayHeight() / 2;
+    }
+
+    public void tapDisplayCentre() {
+        tapDisplay(getDisplayCentreWidth(),  getDisplayCentreHeight());
+    }
+
+    public void tapDisplay(int x, int y) {
+        UiDevice.getInstance().click(x, y);
+    }
+
+    public void uiDeviceSwipeUp(int steps) {
+        UiDevice.getInstance().swipe(
+            getDisplayCentreWidth(),
+            (getDisplayCentreHeight() + (getDisplayCentreHeight() / 2)),
+            getDisplayCentreWidth(),
+            (getDisplayCentreHeight() / 2),
+            steps);
+    }
+
+    public void uiDeviceSwipeDown(int steps) {
+        UiDevice.getInstance().swipe(
+            getDisplayCentreWidth(),
+            (getDisplayCentreHeight() / 2),
+            getDisplayCentreWidth(),
+            (getDisplayCentreHeight() + (getDisplayCentreHeight() / 2)),
+            steps);
+    }
+
+    public void uiDeviceSwipeLeft(int steps) {
+        UiDevice.getInstance().swipe(
+            (getDisplayCentreWidth() + (getDisplayCentreWidth() / 2)),
+            getDisplayCentreHeight(),
+            (getDisplayCentreWidth() / 2),
+            getDisplayCentreHeight(),
+            steps);
+    }
+
+    public void uiDeviceSwipeRight(int steps) {
+        UiDevice.getInstance().swipe(
+            (getDisplayCentreWidth() / 2),
+            getDisplayCentreHeight(),
+            (getDisplayCentreWidth() + (getDisplayCentreWidth() / 2)),
+            getDisplayCentreHeight(),
+            steps);
+    }
+
+    public void uiDeviceSwipe(Direction direction, int steps) throws Exception {
+        switch (direction) {
+            case UP:
+                uiDeviceSwipeUp(steps);
+                break;
+            case DOWN:
+                uiDeviceSwipeDown(steps);
+                break;
+            case LEFT:
+                uiDeviceSwipeLeft(steps);
+                break;
+            case RIGHT:
+                uiDeviceSwipeRight(steps);
+                break;
+            case NULL:
+                throw new Exception("No direction specified");
+            default:
+                break;
+        }
+    }
+
+    public void uiObjectSwipe(UiObject view, Direction direction, int steps) throws Exception {
+        switch (direction) {
+            case UP:
+                view.swipeUp(steps);
+                break;
+            case DOWN:
+                view.swipeDown(steps);
+                break;
+            case LEFT:
+                view.swipeLeft(steps);
+                break;
+            case RIGHT:
+                view.swipeRight(steps);
+                break;
+            case NULL:
+                throw new Exception("No direction specified");
+            default:
+                break;
+        }
+    }
+
+    public void uiObjectVertPinchIn(UiObject view, int steps, int percent) throws Exception {
+        final int FINGER_TOUCH_HALF_WIDTH = 20;
+
+        // Make value between 1 and 100
+        int nPercent = (percent < 0) ? 1 : (percent > 100) ? 100 : percent;
+        float percentage = nPercent / 100f;
+
+        Rect rect = view.getVisibleBounds();
+
+        if (rect.width() <= FINGER_TOUCH_HALF_WIDTH * 2) {
+            throw new IllegalStateException("Object width is too small for operation");
+        }
+
+        // Start at the top-center and bottom-center of the control
+        Point startPoint1 = new Point(rect.centerX(), rect.centerY()
+                          + (int) ((rect.height() / 2) * percentage));
+        Point startPoint2 = new Point(rect.centerX(), rect.centerY()
+                          - (int) ((rect.height() / 2) * percentage));
+
+        // End at the same point at the center of the control
+        Point endPoint1 = new Point(rect.centerX(), rect.centerY() + FINGER_TOUCH_HALF_WIDTH);
+        Point endPoint2 = new Point(rect.centerX(), rect.centerY() - FINGER_TOUCH_HALF_WIDTH);
+
+        view.performTwoPointerGesture(startPoint1, startPoint2, endPoint1, endPoint2, steps);
+    }
+
+    public void uiObjectVertPinchOut(UiObject view, int steps, int percent) throws Exception {
+        final int FINGER_TOUCH_HALF_WIDTH = 20;
+
+        // Make value between 1 and 100
+        int nPercent = (percent < 0) ? 1 : (percent > 100) ? 100 : percent;
+        float percentage = nPercent / 100f;
+
+        Rect rect = view.getVisibleBounds();
+
+        if (rect.width() <= FINGER_TOUCH_HALF_WIDTH * 2) {
+            throw new IllegalStateException("Object width is too small for operation");
+        }
+
+        // Start from the same point at the center of the control
+        Point startPoint1 = new Point(rect.centerX(), rect.centerY() + FINGER_TOUCH_HALF_WIDTH);
+        Point startPoint2 = new Point(rect.centerX(), rect.centerY() - FINGER_TOUCH_HALF_WIDTH);
+
+        // End at the top-center and bottom-center of the control
+        Point endPoint1 = new Point(rect.centerX(), rect.centerY()
+                        + (int) ((rect.height() / 2) * percentage));
+        Point endPoint2 = new Point(rect.centerX(), rect.centerY()
+                        - (int) ((rect.height() / 2) * percentage));
+
+        view.performTwoPointerGesture(startPoint1, startPoint2, endPoint1, endPoint2, steps);
+    }
+
+    public void setScreenOrientation(ScreenOrientation orientation) throws Exception {
+        switch (orientation) {
+            case RIGHT:
+                getUiDevice().setOrientationRight();
+                break;
+            case NATURAL:
+                getUiDevice().setOrientationNatural();
+                break;
+            case LEFT:
+                getUiDevice().setOrientationLeft();
+                break;
+            default:
+                throw new Exception("No orientation specified");
+        }
+    }
+
+    public void unsetScreenOrientation() throws Exception {
+        getUiDevice().unfreezeRotation();
+    }
+
+   public void uiObjectPerformLongClick(UiObject view, int steps) throws Exception {
+        Rect rect = view.getBounds();
+        UiDevice.getInstance().swipe(rect.centerX(), rect.centerY(),
+                                     rect.centerX(), rect.centerY(), steps);
+    }
+
+    public void uiDeviceSwipeVertical(int startY, int endY, int xCoordinate, int steps) {
+        getUiDevice().swipe(startY, xCoordinate, endY, xCoordinate, steps);
+    }
+
+    public void uiDeviceSwipeHorizontal(int startX, int endX, int yCoordinate, int steps) {
+        getUiDevice().swipe(startX, yCoordinate, endX, yCoordinate, steps);
+    }
+
+    public void uiObjectPinch(UiObject view, PinchType direction, int steps,
+                                  int percent) throws Exception {
+        if (direction.equals(PinchType.IN)) {
+            view.pinchIn(percent, steps);
+        } else if (direction.equals(PinchType.OUT)) {
+            view.pinchOut(percent, steps);
+        }
+    }
+
+    public void uiObjectVertPinch(UiObject view, PinchType direction,
+                                       int steps, int percent) throws Exception {
+        if (direction.equals(PinchType.IN)) {
+            uiObjectVertPinchIn(view, steps, percent);
+        } else if (direction.equals(PinchType.OUT)) {
+            uiObjectVertPinchOut(view, steps, percent);
+        }
+    }
+
+    public void repeatClickUiObject(UiObject view, int repeatCount, int intervalInMillis) throws Exception {
+        int repeatInterval = intervalInMillis > CLICK_REPEAT_INTERVAL_MINIMUM
+                           ? intervalInMillis : CLICK_REPEAT_INTERVAL_DEFAULT;
+        if (repeatCount < 1 || !view.isClickable()) {
+            return;
+        }
+
+        for (int i = 0; i < repeatCount; ++i) {
+            view.click();
+            SystemClock.sleep(repeatInterval); // in order to register as separate click
+        }
+    }
+
+    public UiObject clickUiObject(FindByCriteria criteria, String matching) throws Exception {
+        return clickUiObject(criteria, matching, null, false);
+    }
+
+    public UiObject clickUiObject(FindByCriteria criteria, String matching, boolean wait) throws Exception {
+        return clickUiObject(criteria, matching, null, wait);
+    }
+
+    public UiObject clickUiObject(FindByCriteria criteria, String matching, String clazz) throws Exception {
+        return clickUiObject(criteria, matching, clazz, false);
+    }
+
+    public UiObject clickUiObject(FindByCriteria criteria, String matching, String clazz, boolean wait) throws Exception {
+        UiObject view;
+
+        switch (criteria) {
+            case BY_ID:
+                view = (clazz == null)
+                     ? getUiObjectByResourceId(matching) : getUiObjectByResourceId(matching, clazz);
+                break;
+            case BY_DESC:
+                view = (clazz == null)
+                     ? getUiObjectByDescription(matching) : getUiObjectByDescription(matching, clazz);
+                break;
+            case BY_TEXT:
+            default:
+                view = (clazz == null)
+                     ? getUiObjectByText(matching) : getUiObjectByText(matching, clazz);
+                break;
+        }
+
+        if (wait) {
+            view.clickAndWaitForNewWindow();
+        } else {
+            view.click();
+        }
+        return view;
+    }
+
+    public UiObject getUiObjectByResourceId(String resourceId, String className) throws Exception {
+        return getUiObjectByResourceId(resourceId, className, uiAutoTimeout);
+    }
+
+    public UiObject getUiObjectByResourceId(String resourceId, String className, long timeout) throws Exception {
+        UiObject object = new UiObject(new UiSelector().resourceId(resourceId)
+                                                       .className(className));
+        if (!object.waitForExists(timeout)) {
+           throw new UiObjectNotFoundException(String.format("Could not find \"%s\" \"%s\"",
+                                                              resourceId, className));
+        }
+        return object;
+    }
+
+    public UiObject getUiObjectByResourceId(String id) throws Exception {
+        UiObject object = new UiObject(new UiSelector().resourceId(id));
+
+        if (!object.waitForExists(uiAutoTimeout)) {
+           throw new UiObjectNotFoundException("Could not find view with resource ID: " + id);
+        }
+        return object;
+    }
+
+    public UiObject getUiObjectByDescription(String description, String className) throws Exception {
+        return getUiObjectByDescription(description, className, uiAutoTimeout);
+    }
+
+    public UiObject getUiObjectByDescription(String description, String className, long timeout) throws Exception {
+        UiObject object = new UiObject(new UiSelector().descriptionContains(description)
+                                                       .className(className));
+        if (!object.waitForExists(timeout)) {
+            throw new UiObjectNotFoundException(String.format("Could not find \"%s\" \"%s\"",
+                                                              description, className));
+        }
+        return object;
+    }
+
+    public UiObject getUiObjectByDescription(String desc) throws Exception {
+        UiObject object = new UiObject(new UiSelector().descriptionContains(desc));
+
+        if (!object.waitForExists(uiAutoTimeout)) {
+           throw new UiObjectNotFoundException("Could not find view with description: " + desc);
+        }
+        return object;
+    }
+
+    public UiObject getUiObjectByText(String text, String className) throws Exception {
+        return getUiObjectByText(text, className, uiAutoTimeout);
+    }
+
+    public UiObject getUiObjectByText(String text, String className, long timeout) throws Exception {
+        UiObject object = new UiObject(new UiSelector().textContains(text)
+                                                       .className(className));
+        if (!object.waitForExists(timeout)) {
+            throw new UiObjectNotFoundException(String.format("Could not find \"%s\" \"%s\"",
+                                                              text, className));
+        }
+        return object;
+    }
+
+    public UiObject getUiObjectByText(String text) throws Exception {
+        UiObject object = new UiObject(new UiSelector().textContains(text));
+
+        if (!object.waitForExists(uiAutoTimeout)) {
+           throw new UiObjectNotFoundException("Could not find view with text: " + text);
+        }
+        return object;
+    }
+
+    // Helper to select a folder in the gallery
+    public void selectGalleryFolder(String directory) throws Exception {
+        UiObject workdir =
+            new UiObject(new UiSelector().text(directory)
+                                         .className("android.widget.TextView"));
+        UiScrollable scrollView =
+            new UiScrollable(new UiSelector().scrollable(true));
+
+        // If the folder is not present wait for a short time for
+        // the media server to refresh its index.
+        boolean discovered = workdir.waitForExists(TimeUnit.SECONDS.toMillis(10));
+        if (!discovered && scrollView.exists()) {
+            // First check if the directory is visible on the first
+            // screen and if not scroll to the bottom of the screen to look for it.
+            discovered = scrollView.scrollIntoView(workdir);
+
+            // If still not discovered scroll back to the top of the screen and
+            // wait for a longer amount of time for the media server to refresh
+            // its index.
+            if (!discovered) {
+                // scrollView.scrollToBeggining() doesn't work for this
+                // particular scrollable view so use device method instead
+                for (int i = 0; i < 10; i++) {
+                    uiDeviceSwipeUp(20);
+                }
+                discovered = workdir.waitForExists(TimeUnit.SECONDS.toMillis(60));
+
+                // Scroll to the bottom of the screen one last time
+                if (!discovered) {
+                    discovered = scrollView.scrollIntoView(workdir);
+                }
+            }
+        }
+
+        if (discovered) {
+            workdir.clickAndWaitForNewWindow();
+        } else {
+            throw new UiObjectNotFoundException("Could not find folder : " + directory);
         }
     }
 }
-

wlauto/external/uiauto/src/com/arm/wlauto/uiauto/UxPerfUiAutomation.java

@@ -0,0 +1,49 @@
+/*    Copyright 2013-2016 ARM Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package com.arm.wlauto.uiauto;
+
+import java.util.logging.Logger;
+
+public class UxPerfUiAutomation extends BaseUiAutomation {
+
+    private Logger logger = Logger.getLogger(UxPerfUiAutomation.class.getName());
+
+    public enum GestureType { UIDEVICE_SWIPE, UIOBJECT_SWIPE, PINCH };
+
+    public static class GestureTestParams {
+        public GestureType gestureType;
+        public Direction gestureDirection;
+        public PinchType pinchType;
+        public int percent;
+        public int steps;
+
+        public GestureTestParams(GestureType gesture, Direction direction, int steps) {
+            this.gestureType = gesture;
+            this.gestureDirection = direction;
+            this.pinchType = PinchType.NULL;
+            this.steps = steps;
+            this.percent = 0;
+        }
+
+        public GestureTestParams(GestureType gesture, PinchType pinchType, int steps, int percent) {
+            this.gestureType = gesture;
+            this.gestureDirection = Direction.NULL;
+            this.pinchType = pinchType;
+            this.steps = steps;
+            this.percent = percent;
+        }
+    }
+}

wlauto/instrumentation/__init__.py

@@ -18,14 +18,14 @@ from wlauto.core import instrumentation
 
 def instrument_is_installed(instrument):
     """Returns ``True`` if the specified instrument is installed, and ``False``
-    other wise. The insturment maybe specified either as a name or a subclass (or
+    other wise. The instrument maybe specified either as a name or a subclass (or
     instance of subclass) of :class:`wlauto.core.Instrument`."""
     return instrumentation.is_installed(instrument)
 
 
 def instrument_is_enabled(instrument):
     """Returns ``True`` if the specified instrument is installed and is currently
-    enabled, and ``False`` other wise. The insturment maybe specified either
+    enabled, and ``False`` other wise. The instrument maybe specified either
     as a name or a subclass (or instance of subclass) of
     :class:`wlauto.core.Instrument`."""
     return instrumentation.is_enabled(instrument)

wlauto/instrumentation/daq/__init__.py

@@ -112,10 +112,10 @@ class Daq(Instrument):
         Parameter('server_host', kind=str, default='localhost',
                   global_alias='daq_server_host',
                   description='The host address of the machine that runs the daq Server which the '
-                              'insturment communicates with.'),
+                              'instrument communicates with.'),
         Parameter('server_port', kind=int, default=45677,
                   global_alias='daq_server_port',
-                  description='The port number for daq Server in which daq insturment communicates '
+                  description='The port number for daq Server in which daq instrument communicates '
                               'with.'),
         Parameter('device_id', kind=str, default='Dev1',
                   global_alias='daq_device_id',
@@ -286,6 +286,10 @@ class Daq(Instrument):
         if self.labels:
             if len(self.labels) != len(self.resistor_values):
                 raise ConfigError('Number of DAQ port labels does not match the number of resistor values.')
+
+            duplicates = set([x for x in self.labels if self.labels.count(x) > 1])
+            if len(duplicates) > 0:
+                raise ConfigError('Duplicate labels: {}'.format(', '.join(duplicates)))
         else:
             self.labels = ['PORT_{}'.format(i) for i, _ in enumerate(self.resistor_values)]
         self.server_config = ServerConfiguration(host=self.server_host,
@@ -306,7 +310,10 @@ class Daq(Instrument):
         if isinstance(self.merge_channels, bool):
             if self.merge_channels:
                 # Create a dict of potential prefixes and a list of their suffixes
-                grouped_suffixes = {label[:-1]: label for label in sorted(self.labels) if len(label) > 1}
+                grouped_suffixes = defaultdict(list)
+                for label in sorted(self.labels):
+                    if len(label) > 1:
+                        grouped_suffixes[label[:-1]].append(label)
                 # Only merge channels if more than one channel has the same prefix and the prefixes
                 # are consecutive letters starting with 'a'.
                 self.label_map = {}

wlauto/instrumentation/delay/__init__.py

@@ -196,4 +196,3 @@ class DelayInstrument(Instrument):
         if self.active_cooling and not self.device.has('active_cooling'):
             message = 'Your device does not support active cooling. Did you configure it with an approprite module?'
             raise InstrumentError(message)
-

wlauto/instrumentation/dmesg/__init__.py

@@ -58,5 +58,3 @@ class DmesgInstrument(Instrument):
     def teardown(self, context):  # pylint: disable=unused-argument
         if self.loglevel:
             self.device.set_sysfile_value(self.loglevel_file, self.old_loglevel, verify=False)
-
-

wlauto/instrumentation/energy_model/__init__.py

@@ -348,7 +348,7 @@ class EnergyModelInstrument(Instrument):
     desicription = """
     Generates a power mode for the device based on specified workload.
 
-    This insturment will execute the workload specified by the agenda (currently, only ``sysbench`` is
+    This instrument will execute the workload specified by the agenda (currently, only ``sysbench`` is
     supported) and will use the resulting performance and power measurments to generate a power mode for
     the device.
 
@@ -725,7 +725,8 @@ class EnergyModelInstrument(Instrument):
                 if not self.no_hotplug:
                     spec.runtime_parameters['{}_cores'.format(core)] = num_cpus
                 spec.runtime_parameters['{}_frequency'.format(core)] = min_frequency
-                spec.runtime_parameters['ui'] = 'off'
+                if self.device.platform == 'chromeos':
+                    spec.runtime_parameters['ui'] = 'off'
                 spec.cluster = cluster
                 spec.num_cpus = num_cpus
                 spec.id = '{}_idle_{}_{}'.format(cluster, state.id, num_cpus)
@@ -744,7 +745,8 @@ class EnergyModelInstrument(Instrument):
                     spec.runtime_parameters['{}_frequency'.format(core)] = freq
                     if not self.no_hotplug:
                         spec.runtime_parameters['{}_cores'.format(core)] = num_cpus
-                    spec.runtime_parameters['ui'] = 'off'
+                    if self.device.platform == 'chromeos':
+                        spec.runtime_parameters['ui'] = 'off'
                     spec.id = '{}_{}_{}'.format(cluster, num_cpus, freq)
                     spec.label = 'freq_{}_{}'.format(cluster, spec.label)
                     spec.workload_parameters['taskset_mask'] = list_to_mask(self.get_cpus(cluster))

wlauto/instrumentation/energy_probe/__init__.py

@@ -144,4 +144,3 @@ class EnergyProbe(Instrument):
         writer = csv.DictWriter(f, self.attributes)
         writer.writeheader()
         writer.writerows(new_data)
-

wlauto/instrumentation/fps/__init__.py

@@ -24,6 +24,8 @@ import shutil
 import threading
 import errno
 import tempfile
+import collections
+import re
 
 from distutils.version import LooseVersion
 
@@ -37,6 +39,7 @@ from wlauto.instrumentation import instrument_is_installed
 from wlauto.exceptions import (InstrumentError, WorkerThreadError, ConfigError,
                                DeviceNotRespondingError, TimeoutError)
 from wlauto.utils.types import boolean, numeric
+from wlauto.utils.fps import FpsProcessor, SurfaceFlingerFrame, GfxInfoFrame, GFXINFO_EXEMPT
 
 
 VSYNC_INTERVAL = 16666667
@@ -48,20 +51,28 @@ class FpsInstrument(Instrument):
 
     name = 'fps'
     description = """
-    Measures Frames Per Second (FPS) and associated metrics for a workload's main View.
+    Measures Frames Per Second (FPS) and associated metrics for a workload.
 
     .. note:: This instrument depends on pandas Python library (which is not part of standard
               WA dependencies), so you will need to install that first, before you can use it.
 
+    Android L and below use SurfaceFlinger to calculate the FPS data.
+    Android M and above use gfxinfo to calculate the FPS data.
+
+    SurfaceFlinger:
     The view is specified by the workload as ``view`` attribute. This defaults
     to ``'SurfaceView'`` for game workloads, and ``None`` for non-game
     workloads (as for them FPS mesurement usually doesn't make sense).
     Individual workloads may override this.
 
+    gfxinfo:
+    The view is specified by the workload as ``package`` attribute.
+    This is because gfxinfo already processes for all views in a package.
+
     This instrument adds four metrics to the results:
 
         :FPS: Frames Per Second. This is the frame rate of the workload.
-        :frames: The total number of frames rendered during the execution of
+        :frame_count: The total number of frames rendered during the execution of
                  the workload.
         :janks: The number of "janks" that occured during execution of the
                 workload. Janks are sudden shifts in frame rate. They result
@@ -110,9 +121,19 @@ class FpsInstrument(Instrument):
                   a content crash. E.g. a value of ``0.75`` means the number of actual frames counted is a
                   quarter lower than expected, it will treated as a content crash.
                   """),
-    ]
+        Parameter('dumpsys_period', kind=float, default=2, constraint=lambda x: x > 0,
+                  description="""
+                  Specifies the time period between calls to ``dumpsys SurfaceFlinger --latency`` in
+                  seconds when collecting frame data. Using a lower value improves the granularity
+                  of timings when recording actions that take a short time to complete. Note, this
+                  will produce duplicate frame data in the raw dumpsys output, however, this is
+                  filtered out in frames.csv. It may also affect the overall load on the system.
 
-    clear_command = 'dumpsys SurfaceFlinger --latency-clear '
+                  The default value of 2 seconds corresponds with the NUM_FRAME_RECORDS in
+                  android/services/surfaceflinger/FrameTracker.h (as of the time of writing
+                  currently 128) and a frame rate of 60 fps that is applicable to most devices.
+                  """),
+    ]
 
     def __init__(self, device, **kwargs):
         super(FpsInstrument, self).__init__(device, **kwargs)
@@ -120,6 +141,7 @@ class FpsInstrument(Instrument):
         self.outfile = None
         self.fps_outfile = None
         self.is_enabled = True
+        self.fps_method = ''
 
     def validate(self):
         if not pd or LooseVersion(pd.__version__) < LooseVersion('0.13.1'):
@@ -134,39 +156,76 @@ class FpsInstrument(Instrument):
         if hasattr(workload, 'view'):
             self.fps_outfile = os.path.join(context.output_directory, 'fps.csv')
             self.outfile = os.path.join(context.output_directory, 'frames.csv')
-            self.collector = LatencyCollector(self.outfile, self.device, workload.view or '', self.keep_raw, self.logger)
-            self.device.execute(self.clear_command)
+            # Android M brings a new method of collecting FPS data
+            if self.device.get_sdk_version() >= 23:
+                # gfxinfo takes in the package name rather than a single view/activity
+                # so there is no 'list_command' to run and compare against a list of
+                # views/activities. Additionally, clearing the stats requires the package
+                # so we need to clear for every package in the workload.
+                # Usually there is only one package, but some workloads may run multiple
+                # packages so each one must be reset before continuing
+                self.fps_method = 'gfxinfo'
+                runcmd = 'dumpsys gfxinfo {} framestats'
+                lstcmd = None
+                params = workload.package
+                params = [params] if isinstance(params, basestring) else params
+                for pkg in params:
+                    self.device.execute('dumpsys gfxinfo {} reset'.format(pkg))
+            else:
+                self.fps_method = 'surfaceflinger'
+                runcmd = 'dumpsys SurfaceFlinger --latency {}'
+                lstcmd = 'dumpsys SurfaceFlinger --list'
+                params = workload.view
+                self.device.execute('dumpsys SurfaceFlinger --latency-clear ')
+
+            self.collector = LatencyCollector(self.outfile, self.device, params or '',
+                                              self.keep_raw, self.logger, self.dumpsys_period,
+                                              runcmd, lstcmd, self.fps_method)
         else:
             self.logger.debug('Workload does not contain a view; disabling...')
             self.is_enabled = False
 
     def start(self, context):
         if self.is_enabled:
-            self.logger.debug('Starting SurfaceFlinger collection...')
+            self.logger.debug('Starting Frame Statistics collection...')
             self.collector.start()
 
     def stop(self, context):
         if self.is_enabled and self.collector.is_alive():
-            self.logger.debug('Stopping SurfaceFlinger collection...')
+            self.logger.debug('Stopping Frame Statistics collection...')
             self.collector.stop()
 
     def update_result(self, context):
         if self.is_enabled:
+            fps, frame_count, janks, not_at_vsync = float('nan'), 0, 0, 0
+            p90, p95, p99 = [float('nan')] * 3
             data = pd.read_csv(self.outfile)
             if not data.empty:  # pylint: disable=maybe-no-member
-                per_frame_fps = self._update_stats(context, data)
+                # gfxinfo method has an additional file generated that contains statistics
+                stats_file = None
+                if self.fps_method == 'gfxinfo':
+                    stats_file = os.path.join(os.path.dirname(self.outfile), 'gfxinfo.csv')
+                fp = FpsProcessor(data, extra_data=stats_file)
+                per_frame_fps, metrics = fp.process(self.collector.refresh_period, self.drop_threshold)
+                fps, frame_count, janks, not_at_vsync = metrics
+
                 if self.generate_csv:
                     per_frame_fps.to_csv(self.fps_outfile, index=False, header=True)
                     context.add_artifact('fps', path='fps.csv', kind='data')
-            else:
-                context.result.add_metric('FPS', float('nan'))
-                context.result.add_metric('frame_count', 0)
-                context.result.add_metric('janks', 0)
-                context.result.add_metric('not_at_vsync', 0)
+
+                p90, p95, p99 = fp.percentiles()
+
+            context.result.add_metric('FPS', fps)
+            context.result.add_metric('frame_count', frame_count)
+            context.result.add_metric('janks', janks, lower_is_better=True)
+            context.result.add_metric('not_at_vsync', not_at_vsync, lower_is_better=True)
+            context.result.add_metric('frame_time_90percentile', p90, 'ms', lower_is_better=True)
+            context.result.add_metric('frame_time_95percentile', p95, 'ms', lower_is_better=True)
+            context.result.add_metric('frame_time_99percentile', p99, 'ms', lower_is_better=True)
 
     def slow_update_result(self, context):
         result = context.result
-        if result.has_metric('execution_time'):
+        if self.crash_check and result.has_metric('execution_time'):
             self.logger.debug('Checking for crashed content.')
             exec_time = result['execution_time'].value
             fps = result['FPS'].value
@@ -180,63 +239,23 @@ class FpsInstrument(Instrument):
                     result.status = IterationResult.FAILED
                     result.add_event('Content crash detected (actual/expected frames: {:.2}).'.format(ratio))
 
-    def _update_stats(self, context, data):  # pylint: disable=too-many-locals
-        vsync_interval = self.collector.refresh_period
-        # fiter out bogus frames.
-        actual_present_times = data.actual_present_time[data.actual_present_time != 0x7fffffffffffffff]
-        actual_present_time_deltas = (actual_present_times - actual_present_times.shift()).drop(0)  # pylint: disable=E1103
-        vsyncs_to_compose = (actual_present_time_deltas / vsync_interval).apply(lambda x: int(round(x, 0)))
-        # drop values lower than drop_threshold FPS as real in-game frame
-        # rate is unlikely to drop below that (except on loading screens
-        # etc, which should not be factored in frame rate calculation).
-        per_frame_fps = (1.0 / (vsyncs_to_compose * (vsync_interval / 1e9)))
-        keep_filter = per_frame_fps > self.drop_threshold
-        filtered_vsyncs_to_compose = vsyncs_to_compose[keep_filter]
-        if not filtered_vsyncs_to_compose.empty:
-            total_vsyncs = filtered_vsyncs_to_compose.sum()
-            if total_vsyncs:
-                frame_count = filtered_vsyncs_to_compose.size
-                fps = 1e9 * frame_count / (vsync_interval * total_vsyncs)
-                context.result.add_metric('FPS', fps)
-                context.result.add_metric('frame_count', frame_count)
-            else:
-                context.result.add_metric('FPS', float('nan'))
-                context.result.add_metric('frame_count', 0)
-
-            vtc_deltas = filtered_vsyncs_to_compose - filtered_vsyncs_to_compose.shift()
-            vtc_deltas.index = range(0, vtc_deltas.size)
-            vtc_deltas = vtc_deltas.drop(0).abs()
-            janks = vtc_deltas.apply(lambda x: (PAUSE_LATENCY > x > 1.5) and 1 or 0).sum()
-            not_at_vsync = vsyncs_to_compose.apply(lambda x: (abs(x - 1.0) > EPSYLON) and 1 or 0).sum()
-            context.result.add_metric('janks', janks)
-            context.result.add_metric('not_at_vsync', not_at_vsync)
-        else:  # no filtered_vsyncs_to_compose
-            context.result.add_metric('FPS', float('nan'))
-            context.result.add_metric('frame_count', 0)
-            context.result.add_metric('janks', 0)
-            context.result.add_metric('not_at_vsync', 0)
-        per_frame_fps.name = 'fps'
-        return per_frame_fps
-
 
 class LatencyCollector(threading.Thread):
-
     # Note: the size of the frames buffer for a particular surface is defined
     #       by NUM_FRAME_RECORDS inside android/services/surfaceflinger/FrameTracker.h.
     #       At the time of writing, this was hard-coded to 128. So at 60 fps
     #       (and there is no reason to go above that, as it matches vsync rate
     #       on pretty much all phones), there is just over 2 seconds' worth of
-    #       frames in there. Hence the sleep time of 2 seconds between dumps.
-    #command_template = 'while (true); do dumpsys SurfaceFlinger --latency {}; sleep 2; done'
-    command_template = 'dumpsys SurfaceFlinger --latency {}'
+    #       frames in there. Hence the default sleep time of 2 seconds between dumps.
 
-    def __init__(self, outfile, device, activity, keep_raw, logger):
+    def __init__(self, outfile, device, activities, keep_raw, logger, dumpsys_period,
+                 run_command, list_command, fps_method):
         super(LatencyCollector, self).__init__()
         self.outfile = outfile
         self.device = device
-        self.command = self.command_template.format(activity)
         self.keep_raw = keep_raw
         self.logger = logger
+        self.dumpsys_period = dumpsys_period
         self.stop_signal = threading.Event()
         self.frames = []
         self.last_ready_time = 0
@@ -244,18 +263,54 @@ class LatencyCollector(threading.Thread):
         self.drop_threshold = self.refresh_period * 1000
         self.exc = None
         self.unresponsive_count = 0
+        if isinstance(activities, basestring):
+            activities = [activities]
+        self.activities = activities
+        self.command_template = run_command
+        self.list_command = list_command
+        self.fps_method = fps_method
+        # Based on the fps_method, setup the header for the csv,
+        # and set the process_trace_line function accordingly
+        if fps_method == 'surfaceflinger':
+            self.header = SurfaceFlingerFrame._fields
+            self.process_trace_line = self._process_surfaceflinger_line
+        else:
+            self.header = GfxInfoFrame._fields
+            self.process_trace_line = self._process_gfxinfo_line
+            self.re_frame = re.compile('([0-9]+,)+')
+            self.re_stats = re.compile('.*(percentile|frames|Number).*')
+            # Create a template summary text block that matches what gfxinfo gives after a reset
+            # - 133 is the default ms value for percentiles after reset
+            self.summary = collections.OrderedDict((('Total frames rendered', 0),
+                                                    ('Janky frames', 0),
+                                                    ('90th percentile', 133),
+                                                    ('95th percentile', 133),
+                                                    ('99th percentile', 133),
+                                                    ('Number Missed Vsync', 0),
+                                                    ('Number High input latency', 0),
+                                                    ('Number Slow UI thread', 0),
+                                                    ('Number Slow bitmap uploads', 0),
+                                                    ('Number Slow issue draw commands', 0)))
 
     def run(self):
         try:
-            self.logger.debug('SurfaceFlinger collection started.')
+            self.logger.debug('Frame Statistics collection started. Method: ' + self.fps_method)
             self.stop_signal.clear()
             fd, temp_file = tempfile.mkstemp()
             self.logger.debug('temp file: {}'.format(temp_file))
             wfh = os.fdopen(fd, 'wb')
             try:
+                view_list = self.activities
                 while not self.stop_signal.is_set():
-                    wfh.write(self.device.execute(self.command))
-                    time.sleep(2)
+                    # If a list_command is provided, set the view_list to be its output
+                    # Then check for each activity in this list and if there is a match,
+                    # process the output. If no command is provided, then always process.
+                    if self.list_command:
+                        view_list = self.device.execute(self.list_command).split()
+                    for activity in self.activities:
+                        if activity in view_list:
+                            wfh.write(self.device.execute(self.command_template.format(activity)))
+                    time.sleep(self.dumpsys_period)
             finally:
                 wfh.close()
             # TODO: this can happen after the run during results processing
@@ -264,9 +319,9 @@ class LatencyCollector(threading.Thread):
                 for line in text.split('\n'):
                     line = line.strip()
                     if line:
-                        self._process_trace_line(line)
+                        self.process_trace_line(line)
             if self.keep_raw:
-                raw_file = os.path.join(os.path.dirname(self.outfile), 'surfaceflinger.raw')
+                raw_file = os.path.join(os.path.dirname(self.outfile), self.fps_method + '.raw')
                 shutil.copy(temp_file, raw_file)
             os.unlink(temp_file)
         except (DeviceNotRespondingError, TimeoutError):  # pylint: disable=W0703
@@ -274,42 +329,81 @@ class LatencyCollector(threading.Thread):
         except Exception, e:  # pylint: disable=W0703
             self.logger.warning('Exception on collector thread: {}({})'.format(e.__class__.__name__, e))
             self.exc = WorkerThreadError(self.name, sys.exc_info())
-        self.logger.debug('SurfaceFlinger collection stopped.')
+        self.logger.debug('Frame Statistics collection stopped.')
 
         with open(self.outfile, 'w') as wfh:
             writer = csv.writer(wfh)
-            writer.writerow(['desired_present_time', 'actual_present_time', 'frame_ready_time'])
+            writer.writerow(self.header)
             writer.writerows(self.frames)
         self.logger.debug('Frames data written.')
 
+        # gfxinfo outputs its own summary statistics for the run.
+        # No point calculating those from the raw data, so store in its own file for later use.
+        if self.fps_method == 'gfxinfo':
+            stats_file = os.path.join(os.path.dirname(self.outfile), 'gfxinfo.csv')
+            with open(stats_file, 'w') as wfh:
+                writer = csv.writer(wfh)
+                writer.writerows(zip(self.summary.keys(), self.summary.values()))
+            self.logger.debug('Gfxinfo summary data written.')
+
     def stop(self):
         self.stop_signal.set()
         self.join()
         if self.unresponsive_count:
-            message = 'SurfaceFlinger was unrepsonsive {} times.'.format(self.unresponsive_count)
+            message = 'LatencyCollector was unrepsonsive {} times.'.format(self.unresponsive_count)
             if self.unresponsive_count > 10:
                 self.logger.warning(message)
             else:
                 self.logger.debug(message)
         if self.exc:
             raise self.exc  # pylint: disable=E0702
-        self.logger.debug('FSP collection complete.')
+        self.logger.debug('Frame Statistics complete.')
 
-    def _process_trace_line(self, line):
+    def _process_surfaceflinger_line(self, line):
         parts = line.split()
         if len(parts) == 3:
-            desired_present_time, actual_present_time, frame_ready_time = map(int, parts)
-            if frame_ready_time <= self.last_ready_time:
+            frame = SurfaceFlingerFrame(*map(int, parts))
+            if frame.frame_ready_time <= self.last_ready_time:
                 return  # duplicate frame
-            if (frame_ready_time - desired_present_time) > self.drop_threshold:
+            if (frame.frame_ready_time - frame.desired_present_time) > self.drop_threshold:
                 self.logger.debug('Dropping bogus frame {}.'.format(line))
                 return  # bogus data
-            self.last_ready_time = frame_ready_time
-            self.frames.append((desired_present_time, actual_present_time, frame_ready_time))
+            self.last_ready_time = frame.frame_ready_time
+            self.frames.append(frame)
         elif len(parts) == 1:
             self.refresh_period = int(parts[0])
-            self.drop_threshold = self.refresh_period * 10
+            self.drop_threshold = self.refresh_period * 1000
         elif 'SurfaceFlinger appears to be unresponsive, dumping anyways' in line:
             self.unresponsive_count += 1
         else:
             self.logger.warning('Unexpected SurfaceFlinger dump output: {}'.format(line))
+
+    def _process_gfxinfo_line(self, line):
+        if 'No process found for' in line:
+            self.unresponsive_count += 1
+            return
+        # Process lines related to the frame data
+        match = self.re_frame.match(line)
+        if match:
+            data = match.group(0)[:-1]
+            data = map(int, data.split(','))
+            frame = GfxInfoFrame(*data)
+            if frame not in self.frames:
+                if frame.Flags & GFXINFO_EXEMPT:
+                    self.logger.debug('Dropping exempt frame {}.'.format(line))
+                else:
+                    self.frames.append(frame)
+            return
+        # Process lines related to the summary statistics
+        match = self.re_stats.match(line)
+        if match:
+            data = match.group(0)
+            title, value = data.split(':', 1)
+            title = title.strip()
+            value = value.strip()
+            if title in self.summary:
+                if 'ms' in value:
+                    value = value.strip('ms')
+                if '%' in value:
+                    value = value.split()[0]
+                self.summary[title] = int(value)

wlauto/instrumentation/freqsweep/__init__.py

@@ -33,6 +33,9 @@ class FreqSweep(Instrument):
         - Setting the runner to 'by_spec' increases the chance of successfully
           completing an agenda without encountering hotplug issues
         - If possible disable dynamic hotplug on the target device
+        - This instrument does not automatically pin workloads to the cores
+          being swept since it is not aware of what the workloads do.
+          To achieve this use the workload's taskset parameter (if it has one).
     """
 
     parameters = [
@@ -44,24 +47,24 @@ class FreqSweep(Instrument):
                   can do so by specifying this parameter.
 
                   Sweeps should be a lists of dictionaries that can contain:
-                    - Cluster (mandatory): The name of the cluster this sweep will be
-                                           performed on. E.g A7
-                    - Frequencies: A list of frequencies (in KHz) to use. If this is
-                                   not provided all frequencies available for this
-                                   cluster will be used.
-                                   E.g: [800000, 900000, 100000]
-                    - label: Workload specs will be named '{spec id}_{label}_{frequency}'.
-                             If a label is not provided it will be named 'sweep{sweep No.}'
+                   - Cluster (mandatory): The name of the cluster this sweep
+                     will be performed on. E.g `A7`
+                   - Frequencies: A list of frequencies (in KHz) to use. If
+                     this is not provided all frequencies available for this
+                     cluster will be used. E.g: `[800000, 900000, 100000]`
+                   - label: Workload specs will be named
+                     `{spec id}_{label}_{frequency}`. If a label is not
+                     provided it will be named `sweep{sweep No.}`
 
-                 Example sweep specification:
+                  Example sweep specification: ::
 
-                     freq_sweep:
-                         sweeps:
-                             - cluster: A53
-                               label: littles
-                               frequencies: [800000, 900000, 100000]
-                             - cluster: A57
-                               label: bigs
+                      freq_sweep:
+                          sweeps:
+                              - cluster: A53
+                                label: littles
+                                frequencies: [800000, 900000, 100000]
+                              - cluster: A57
+                                label: bigs
                   """),
     ]
 

wlauto/instrumentation/hwmon/__init__.py

@@ -64,6 +64,7 @@ class HwmonInstrument(Instrument):
     parameters = [
         Parameter('sensors', kind=list_of_strs, default=['energy', 'temp'],
                   global_alias='hwmon_sensors',
+                  allowed_values=HWMON_SENSORS.keys(),
                   description='The kinds of sensors hwmon instrument will look for')
     ]
 
@@ -73,11 +74,7 @@ class HwmonInstrument(Instrument):
         if self.sensors:
             self.sensor_kinds = {}
             for kind in self.sensors:
-                if kind in HWMON_SENSORS:
-                    self.sensor_kinds[kind] = HWMON_SENSORS[kind]
-                else:
-                    message = 'Unexpected sensor type: {}; must be in {}'.format(kind, HWMON_SENSORS.keys())
-                    raise ConfigError(message)
+                self.sensor_kinds[kind] = HWMON_SENSORS[kind]
         else:
             self.sensor_kinds = HWMON_SENSORS
 
@@ -110,16 +107,19 @@ class HwmonInstrument(Instrument):
                 if report_type == 'diff':
                     before, after = sensor.readings
                     diff = conversion(after - before)
-                    context.result.add_metric(sensor.label, diff, units)
+                    context.result.add_metric(sensor.label, diff, units,
+                                              classifiers={"hwmon_device": sensor.device_name})
                 elif report_type == 'before/after':
                     before, after = sensor.readings
                     mean = conversion((after + before) / 2)
-                    context.result.add_metric(sensor.label, mean, units)
-                    context.result.add_metric(sensor.label + ' before', conversion(before), units)
-                    context.result.add_metric(sensor.label + ' after', conversion(after), units)
+                    context.result.add_metric(sensor.label, mean, units,
+                                              classifiers={"hwmon_device": sensor.device_name})
+                    context.result.add_metric(sensor.label + ' before', conversion(before), units,
+                                              classifiers={"hwmon_device": sensor.device_name})
+                    context.result.add_metric(sensor.label + ' after', conversion(after), units,
+                                              classifiers={"hwmon_device": sensor.device_name})
                 else:
                     raise InstrumentError('Unexpected report_type: {}'.format(report_type))
             except ValueError, e:
                 self.logger.error('Could not collect all {} readings for {}'.format(sensor.kind, sensor.label))
                 self.logger.error('Got: {}'.format(e))
-

wlauto/instrumentation/juno_energy/__init__.py

@@ -106,4 +106,3 @@ class JunoEnergy(Instrument):
             if self.device.name.lower() != 'juno':
                 message = 'juno_energy instrument is only supported on juno devices; found {}'
                 raise InstrumentError(message.format(self.device.name))
-

wlauto/instrumentation/misc/__init__.py

@@ -57,7 +57,7 @@ class SysfsExtractor(Instrument):
 
     mount_command = 'mount -t tmpfs -o size={} tmpfs {}'
     extract_timeout = 30
-    tarname = 'sysfs.tar.gz'
+    tarname = 'sysfs.tar'
     DEVICE_PATH = 0
     BEFORE_PATH = 1
     AFTER_PATH = 2
@@ -151,16 +151,18 @@ class SysfsExtractor(Instrument):
     def update_result(self, context):
         if self.use_tmpfs:
             on_device_tarball = self.device.path.join(self.device.working_directory, self.tarname)
-            on_host_tarball = self.device.path.join(context.output_directory, self.tarname)
-            self.device.execute('{} tar czf {} -C {} .'.format(self.device.busybox,
-                                                               on_device_tarball,
-                                                               self.tmpfs_mount_point),
+            on_host_tarball = self.device.path.join(context.output_directory, self.tarname + ".gz")
+            self.device.execute('{} tar cf {} -C {} .'.format(self.device.busybox,
+                                                              on_device_tarball,
+                                                              self.tmpfs_mount_point),
                                 as_root=True)
             self.device.execute('chmod 0777 {}'.format(on_device_tarball), as_root=True)
-            self.device.pull_file(on_device_tarball, on_host_tarball)
+            self.device.execute('{} gzip -f {}'.format(self.device.busybox,
+                                                       on_device_tarball))
+            self.device.pull_file(on_device_tarball + ".gz", on_host_tarball)
             with tarfile.open(on_host_tarball, 'r:gz') as tf:
                 tf.extractall(context.output_directory)
-            self.device.delete_file(on_device_tarball)
+            self.device.delete_file(on_device_tarball + ".gz")
             os.remove(on_host_tarball)
 
         for paths in self.device_and_host_paths:
@@ -225,29 +227,6 @@ class ExecutionTimeInstrument(Instrument):
         context.result.add_metric('execution_time', execution_time, 'seconds')
 
 
-class ApkVersion(Instrument):
-
-    name = 'apk_version'
-    description = """
-    Extracts APK versions for workloads that have them.
-
-    """
-
-    def __init__(self, device, **kwargs):
-        super(ApkVersion, self).__init__(device, **kwargs)
-        self.apk_info = None
-
-    def setup(self, context):
-        if hasattr(context.workload, 'apk_file'):
-            self.apk_info = ApkInfo(context.workload.apk_file)
-        else:
-            self.apk_info = None
-
-    def update_result(self, context):
-        if self.apk_info:
-            context.result.add_metric(self.name, self.apk_info.version_name)
-
-
 class InterruptStatsInstrument(Instrument):
 
     name = 'interrupts'
@@ -290,7 +269,7 @@ class DynamicFrequencyInstrument(SysfsExtractor):
 
     """
 
-    tarname = 'cpufreq.tar.gz'
+    tarname = 'cpufreq.tar'
 
     parameters = [
         Parameter('paths', mandatory=False, override=True),
@@ -386,4 +365,3 @@ def _diff_sysfs_dirs(before, after, result):  # pylint: disable=R0914
                     else:
                         dchunks = [diff_tokens(b, a) for b, a in zip(bchunks, achunks)]
                     dfh.write(''.join(dchunks))
-

wlauto/instrumentation/netstats/__init__.py

@@ -9,14 +9,14 @@ from itertools import izip_longest
 
 from wlauto import Instrument, Parameter
 from wlauto import ApkFile
-from wlauto.exceptions import DeviceError, HostError
+from wlauto.exceptions import InstrumentError, HostError
 from wlauto.utils.android import ApkInfo
 from wlauto.utils.types import list_of_strings
 
 
 THIS_DIR = os.path.dirname(__file__)
 
-NETSTAT_REGEX = re.compile(r'I/(?P<tag>netstats-\d+)\(\s*\d*\): (?P<ts>\d+) '
+NETSTAT_REGEX = re.compile(r'I[\/ ](?P<tag>netstats-\d+).*?: (?P<ts>\d+) '
                            r'"(?P<package>[^"]+)" TX: (?P<tx>\S+) RX: (?P<rx>\S+)')
 
 
@@ -160,7 +160,7 @@ class NetstatsInstrument(Instrument):
 
     def initialize(self, context):
         if self.device.platform != 'android':
-            raise DeviceError('nestats instrument only supports on Android devices.')
+            raise InstrumentError('nestats instrument is only supported on Android devices.')
         apk = context.resolver.get(ApkFile(self))
         self.collector = NetstatsCollector(self.device, apk)  # pylint: disable=attribute-defined-outside-init
         self.collector.setup(force=self.force_reinstall)
@@ -189,4 +189,3 @@ class NetstatsInstrument(Instrument):
     def finalize(self, context):
         if self.uninstall_on_completion:
             self.collector.teardown()
-

wlauto/instrumentation/poller/__init__.py

@@ -0,0 +1,122 @@
+#    Copyright 2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# pylint: disable=access-member-before-definition,attribute-defined-outside-init,unused-argument
+import os
+
+from wlauto import Instrument, Parameter, Executable
+from wlauto.exceptions import ConfigError, InstrumentError
+from wlauto.utils.types import list_or_string
+
+
+class FilePoller(Instrument):
+    name = 'file_poller'
+    description = """
+    Polls the given files at a set sample interval. The values are output in CSV format.
+
+    This instrument places a file called poller.csv in each iterations result directory.
+    This file will contain a timestamp column which will be in uS, the rest of the columns
+    will be the contents of the polled files at that time.
+
+    This instrument will strip any commas or new lines for the files' values
+    before writing them.
+    """
+
+    parameters = [
+        Parameter('sample_interval', kind=int, default=1000,
+                  description="""The interval between samples in mS."""),
+        Parameter('files', kind=list_or_string, mandatory=True,
+                  description="""A list of paths to the files to be polled"""),
+        Parameter('labels', kind=list_or_string,
+                  description="""A list of lables to be used in the CSV output for
+                                 the corresponding files. This cannot be used if
+                                 a `*` wildcard is used in a path."""),
+        Parameter('as_root', kind=bool, default=False,
+                  description="""
+                  Whether or not the poller will be run as root. This should be
+                  used when the file you need to poll can only be accessed by root.
+                  """),
+    ]
+
+    def validate(self):
+        if not self.files:
+            raise ConfigError('You must specify atleast one file to poll')
+        if self.labels and any(['*' in f for f in self.files]):
+            raise ConfigError('You cannot used manual labels with `*` wildcards')
+
+    def initialize(self, context):
+        if not self.device.is_rooted and self.as_root:
+            raise ConfigError('The device is not rooted, cannot run poller as root.')
+        host_poller = context.resolver.get(Executable(self, self.device.abi,
+                                                      "poller"))
+        target_poller = self.device.install(host_poller)
+
+        expanded_paths = []
+        for path in self.files:
+            if "*" in path:
+                for p in self.device.listdir(path):
+                    expanded_paths.append(p)
+            else:
+                expanded_paths.append(path)
+        self.files = expanded_paths
+        if not self.labels:
+            self.labels = self._generate_labels()
+
+        self.target_output_path = self.device.path.join(self.device.working_directory, 'poller.csv')
+        self.target_log_path = self.device.path.join(self.device.working_directory, 'poller.log')
+        self.command = '{} -t {} -l {} {} > {} 2>{}'.format(target_poller,
+                                                            self.sample_interval * 1000,
+                                                            ','.join(self.labels),
+                                                            ' '.join(self.files),
+                                                            self.target_output_path,
+                                                            self.target_log_path)
+
+    def start(self, context):
+        self.device.kick_off(self.command, as_root=self.as_root)
+
+    def stop(self, context):
+        self.device.killall('poller', signal='TERM', as_root=self.as_root)
+
+    def update_result(self, context):
+        host_output_file = os.path.join(context.output_directory, 'poller.csv')
+        self.device.pull_file(self.target_output_path, host_output_file)
+        context.add_artifact('poller_output', host_output_file, kind='data')
+        host_log_file = os.path.join(context.output_directory, 'poller.log')
+        self.device.pull_file(self.target_log_path, host_log_file)
+        context.add_artifact('poller_log', host_log_file, kind='log')
+
+        with open(host_log_file) as fh:
+            for line in fh:
+                if 'ERROR' in line:
+                    raise InstrumentError(line.strip())
+                if 'WARNING' in line:
+                    self.logger.warning(line.strip())
+
+    def teardown(self, context):
+        self.device.delete_file(self.target_output_path)
+        self.device.delete_file(self.target_log_path)
+
+    def _generate_labels(self):
+        # Split paths into their parts
+        path_parts = [f.split(self.device.path.sep) for f in self.files]
+        # Identify which parts differ between at least two of the paths
+        differ_map = [len(set(x)) > 1 for x in zip(*path_parts)]
+
+        # compose labels from path parts that differ
+        labels = []
+        for pp in path_parts:
+            label_parts = [p for i, p in enumerate(pp[:-1])
+                           if i >= len(differ_map) or differ_map[i]]
+            label_parts.append(pp[-1])  # always use file name even if same for all
+            labels.append('-'.join(label_parts))
+        return labels