From 23b3b165d5bdce935ff48be8f2206da490c3225c Mon Sep 17 00:00:00 2001 From: Sebastian Goscik Date: Fri, 10 Jun 2016 10:33:47 +0100 Subject: [PATCH] docs: Change log & updates --- doc/source/changes.rst | 226 +++++++++++++++++++++++ doc/source/installation.rst | 11 +- doc/source/invocation.rst | 60 +++++- doc/source/revent.rst | 46 +++-- wlauto/workloads/facebook/__init__.py | 7 +- wlauto/workloads/recentfling/__init__.py | 2 +- 6 files changed, 323 insertions(+), 29 deletions(-) diff --git a/doc/source/changes.rst b/doc/source/changes.rst index 6bc3fba2..3a68647f 100644 --- a/doc/source/changes.rst +++ b/doc/source/changes.rst @@ -1,6 +1,232 @@ ================================= What's New in Workload Automation ================================= + +------------- +Version 2.5.0 +------------- + +Additions: +########## + +Instruments +~~~~~~~~~~~ +- ``servo_power``: Added support for chromebook servo boards. +- ``file_poller``: polls files and outputs a CSV of their values over time. +- ``systrace``: The Systrace tool helps analyze the performance of your + application by capturing and displaying execution times of your applications + processes and other Android system processes. + +Workloads +~~~~~~~~~ +- ``blogbench``: Blogbench is a portable filesystem benchmark that tries to + reproduce the load of a real-world busy file server. +- ``stress-ng``: Designed to exercise various physical subsystems of a computer + as well as the various operating system kernel interfaces. +- ``hwuitest``: Uses hwuitest from AOSP to test rendering latency on Android + devices. +- ``recentfling``: Tests UI jank on android devices. +- ``apklaunch``: installs and runs an arbitrary apk file. +- ``googlemap``: Launches Google Maps and replays previously recorded + interactions. + +Framework +~~~~~~~~~ +- ``wlauto.utils.misc``: Added ``memoised`` function decorator that allows + caching of previous function/method call results. +- Added new ``Device`` APIs: + - ``lsmod``: lists kernel modules + - ``insmod``: inserts a kernel module from a ``.ko`` file on the host. + - ``get_binary_path``: Checks ``binary_directory`` for the wanted binary, + if it is not found there it will try to use ``which`` + - ``install_if_needed``: Will only install a binary if it is not already + on the target. + - ``get_device_model``: Gets the model of the device. +- ``wlauto.core.execution.ExecutionContext``: + - ``add_classfiers``: Allows adding a classfier to all metrics for the + current result. + +Other +~~~~~ +- Commands: + - ``record``: Simplifies recording revent files. + - ``replay``: Plays back revent files. + +Fixes/Improvements: +################### + +Devices +~~~~~~~ +- ``juno``: + - Fixed ``bootargs`` parameter not being passed to ``_boot_via_uboot``. + - Removed default ``bootargs`` +- ``gem5_linux``: + - Added ``login_prompt`` and ``login_password_prompt`` parameters. +- ``generic_linux``: ABI is now read from the target device. + +Instruments +~~~~~~~~~~~ +- ``trace-cmd``: + - Added the ability to report the binary trace on the target device, + removing the need for ``trace-cmd`` binary to be present on the host. + - Updated to handle messages that the trace for a CPU is empty. + - Made timeout for pulling trace 1 minute at minimum. +- ``perf``: per-cpu statistics now get added as metrics to the results (with a + classifier used to identify the cpu). +- ``daq``: + - Fixed bug where an exception would be raised if ``merge_channels=False`` + - No longer allows duplicate channel labels +- ``juno_energy``: + - Summary metrics are now calculated from the contents of ``energy.csv`` and + added to the overall results. + - Added a ``strict`` parameter. When this is set to ``False`` the device + check during validation is omitted. +- ``sysfs_extractor``: tar and gzip are now performed separately to solve + permission issues. +- ``fps``: + - Now only checks for crashed content if ``crash_check`` is ``True``. + - Can now process multiple ``view`` attributes. +- ``hwmon``: Sensor naming fixed, they are also now added as result classifiers + +Resource Getters +~~~~~~~~~~~~~~~~ +- ``extension_asset``: Now picks up the path to the mounted filer from the + ``remote_assets_path`` global setting. + +Result Processors +~~~~~~~~~~~~~~~~~ +- ``cpustates``: + - Added the ability to configure how a missing ``START`` marker in the trace + is handled. + - Now raises a warning when there is a ``START`` marker in the trace but no + ``STOP`` marker. + - Exceptions in PowerStateProcessor no longer stop the processing of the + rest of the trace. + - Now ensures a known initial state by nudging each CPU to bring it out of + idle and writing starting CPU frequencies to the trace. + - Added the ability to create a CPU utilisation timeline. + - Fixed issues with getting frequencies of hotplugged CPUs +- ``csv``: Zero-value classifieres are no longer converted to an empty entry. +- ``ipynb_exporter``: Default template no longer shows a blank plot for + workloads without ``summary_metrics`` + +Workloads +~~~~~~~~~ +- ``vellamo``: + - Added support for v3.2.4. + - Fixed getting values from logcat. +- ``cameracapture``: Updated to work with Android M+. +- ``camerarecord``: Updated to work with Android M+. +- ``lmbench``: + - Added the output file as an artifact. + - Added taskset support +- ``antutu`` - Added support for v6.0.1 +- ``ebizzy``: Fixed use of ``os.path`` to ``self.device.path``. +- ``bbench``: Fixed browser crashes & permissions issues on android M+. +- ``geekbench``: + - Added check whether device is rooted. +- ``manual``: Now only uses logcat on Android devices. +- ``applaunch``: + - Fixed ``cleanup`` not getting forwarded to script. + - Added the ability to stress IO during app launch. +- ``dhrystone``: Now uses WA's resource resolution to find it's binary so it + uses the correct ABI. +- ``glbench``: Updated for new logcat formatting. + +Framework +~~~~~~~~~ +- ``ReventWorkload``: + - Now kills all revent instances on teardown. + - Device model name is now used when searching for revent files, falling back + to WA device name. +- ``BaseLinuxDevice``: + - ``killall`` will now run as root by default if the device + is rooted. + - ``list_file_systems`` now handles blank lines. + - All binaries are now installed into ``binaries_directory`` this allows.. + - Busybox is now deployed on non-root devices. + - gzipped property files are no zcat'ed +- ``LinuxDevice``: + - ``kick_off`` no longer requires root. + - ``kick_off`` will now run as root by default if the device is rooted. + - No longer raises an exception if a connection was dropped during a reboot. + - Added a delay before polling for a connection to avoid re-connecting to a + device that is still in the process of rebooting. +- ``wlauto.utils.types``: ``list_or_string`` now ensures that elements of a list + are strings. +- ``AndroidDevice``: + - ``kick_off`` no longer requires root. + - Build props are now gathered via ``getprop`` rather than trying to parse + build.prop directly. + - WA now pushes its own ``sqlite3`` binary. + - Now uses ``content`` instead of ``settings`` to get ``ANDROID_ID`` + - ``swipe_to_unlock`` parameter is now actually used. It has been changed to + take a direction to accomodate various devices. + - ``ensure_screen_is_on`` will now also unlock the screen if swipe_to_unlock + is set. + - Fixed use of variables in as_root=True commands. + - ``get_pids_of`` now used ``busybox grep`` since as of Android M+ ps cannot + filter by process name anymore. + - Fixed installing APK files with whitespace in their path/name. +- ``adb_shell``: + - Fixed handling of line breaks at the end of command output. + - Newline separator is now detected from the target. + - As of ADB v1.0.35, ADB returns the return code of the command run. WA now + handles this correctly. +- ``ApkWorkload``: + - Now attempts to grant all runtime permissions for devices on Android M+. + - Can now launch packages that don't have a launch activity defined. + - Package version is now added to results as a classifier. + - Now clears app data if an uninstall failed to ensure it starts from a known + state. +- ``wlauto.utils.ipython``: Updated to work with ipython v5. +- ``Gem5Device``: + - Added support for deploying the ``m5`` binary. + - No longer waits for the boot animation to finish if it has been disabled. + - Fixed runtime error caused by lack of kwargs. + - No longer depends on ``busybox``. + - Split out commands to resize shell to ``resize_shell``. + - Now tries to connect to the shell up to 10 times. + - No longer renames gzipped files. +- Agendas: + - Now errors when an agenda key is empty. +- ``wlauto.core.execution.RunInfo``: ``run_name`` will now default to + ``{output_folder}_{date}_{time}``. +- Extensions: + - Two different parameters can now have the same global alias as long as they + their types match. + - You can no longer ``override`` parameters that are defined at the same + level. +- ``wlauto.core.entry_point``: Now gives a better error when a config file + doesn't exist. +- ``wlauto.utils.misc``: Added ``aarch64`` to list for arm64 ABI. +- ``wlauto.core.resolver``: Now shows what version was being search for when a + resource is not found. +- Will no longer start instruments ect. if a run has no workload specs. +- ``wlauto.utils.uboot``: Now detects uboot version to use correct line endings. +- ``wlauto.utils.trace_cmd``: Added a parser for sched_switch events. + +Other +~~~~~ +- Updated to pylint v1.5.1 +- Rebuilt ``busybox`` binaries to prefer built-in applets over system binaries. +- ``BaseUiAutomation``: Added functions for checking version strings. + +Incompatible changes +#################### + +Instruments +~~~~~~~~~~~ +- ``apk_version``: Removed, use result classifiers instead. + +Framework +~~~~~~~~~ +- ``BaseLinuxDevice``: Removed ``is_installed`` use ``install_if_needed`` and + ``get_binary_path`` instead. +- ``LinuxDevice``: Removed ``has_root`` method, use ``is_rooted`` instead. +- ``AndroidDevice``: ``swipe_to_unlock`` method replaced with + ``perform_unlock_swipe``. + ------------- Version 2.4.0 ------------- diff --git a/doc/source/installation.rst b/doc/source/installation.rst index 7c2fbf09..e24dfb12 100644 --- a/doc/source/installation.rst +++ b/doc/source/installation.rst @@ -59,6 +59,11 @@ usually the best bet. Optionally (but recommended), you should also set ``ANDROID_HOME`` to point to the install location of the SDK (i.e. ``/sdk``). +.. note:: You may need to install 32-bit compatibility libararies for the SDK + to work properly. On Ubuntu you need to run:: + + sudo apt-get install lib32stdc++6 lib32z1 + Python ------ @@ -87,7 +92,7 @@ similar distributions, this may be done with APT:: If you do run into this issue after already installing some packages, you can resolve it by running :: - sudo chmod -R a+r /usr/local/lib/python2.7/dist-packagessudo + sudo chmod -R a+r /usr/local/lib/python2.7/dist-packagessudo find /usr/local/lib/python2.7/dist-packages -type d -exec chmod a+x {} \; (The paths above will work for Ubuntu; they may need to be adjusted @@ -307,7 +312,7 @@ that location. If you have installed Workload Automation via ``pip`` and wish to remove it, run this command to uninstall it:: - + sudo -H pip uninstall wlauto .. Note:: This will *not* remove any user configuration (e.g. the ~/.workload_automation directory) @@ -317,5 +322,5 @@ uninstall it:: ==================== To upgrade Workload Automation to the latest version via ``pip``, run:: - + sudo -H pip install --upgrade --no-deps wlauto diff --git a/doc/source/invocation.rst b/doc/source/invocation.rst index 5c8ead92..a322b324 100644 --- a/doc/source/invocation.rst +++ b/doc/source/invocation.rst @@ -5,7 +5,7 @@ Commands ======== Installing the wlauto package will add ``wa`` command to your system, -which you can run from anywhere. This has a number of sub-commands, which can +which you can run from anywhere. This has a number of sub-commands, which can be viewed by executing :: wa -h @@ -131,5 +131,63 @@ will produce something like :: - Results displayed in Iterations per second - Detailed log file for comprehensive engineering analysis +.. _record-command: +record +------ +This command simplifies the process of recording an revent file. It +will automatically deploy revent and even has the option of automatically +opening apps. WA uses two parts to the names of revent recordings in the +format, {device_name}.{suffix}.revent. - device_name can either be specified +manually with the ``-d`` argument or it can be automatically determined. On +Android device it will be obtained from ``build.prop``, on Linux devices it is +obtained from ``/proc/device-tree/model``. - suffix is used by WA to determine +which part of the app execution the recording is for, currently these are +either ``setup`` or ``run``. This should be specified with the ``-s`` +argument. The full set of options for this command are:: + + usage: wa record [-h] [-c CONFIG] [-v] [--debug] [--version] [-d DEVICE] + [-s SUFFIX] [-o OUTPUT] [-p PACKAGE] [-C] + + optional arguments: + -h, --help show this help message and exit + -c CONFIG, --config CONFIG + specify an additional config.py + -v, --verbose The scripts will produce verbose output. + --debug Enable debug mode. Note: this implies --verbose. + --version show program's version number and exit + -d DEVICE, --device DEVICE + The name of the device + -s SUFFIX, --suffix SUFFIX + The suffix of the revent file, e.g. ``setup`` + -o OUTPUT, --output OUTPUT + Directory to save the recording in + -p PACKAGE, --package PACKAGE + Package to launch before recording + -C, --clear Clear app cache before launching it + +.. _replay-command: + +replay +------ + +Along side ``record`` wa also has a command to playback recorded revent files. +It behaves very similar to the ``record`` command taking many of the same options:: + + usage: wa replay [-h] [-c CONFIG] [-v] [--debug] [--version] [-p PACKAGE] [-C] + revent + + positional arguments: + revent The name of the file to replay + + optional arguments: + -h, --help show this help message and exit + -c CONFIG, --config CONFIG + specify an additional config.py + -v, --verbose The scripts will produce verbose output. + --debug Enable debug mode. Note: this implies --verbose. + --version show program's version number and exit + -p PACKAGE, --package PACKAGE + Package to launch before recording + -C, --clear Clear app cache before launching it diff --git a/doc/source/revent.rst b/doc/source/revent.rst index d5811683..b3e6e6cf 100644 --- a/doc/source/revent.rst +++ b/doc/source/revent.rst @@ -17,37 +17,43 @@ to Android UI Automator for providing automation for workloads. :: info:shows info about each event char device any additional parameters make it verbose - -.. note:: There are now also WA commands that perform the below steps. - Please see ``wa show record/replay`` and ``wa record/replay --help`` - for details. - Recording --------- -To record, transfer the revent binary to the device, then invoke ``revent -record``, giving it the time (in seconds) you want to record for, and the -file you want to record to (WA expects these files to have .revent -extension):: +WA features a ``record`` command that will automatically deploy and start +revent on the target device:: - host$ adb push revent /data/local/revent - host$ adb shell - device# cd /data/local - device# ./revent record 1000 my_recording.revent + wa record + INFO Connecting to device... + INFO Press Enter when you are ready to record... + [Pressed Enter] + INFO Press Enter when you have finished recording... + [Pressed Enter] + INFO Pulling files from device + +Once started, you will need to get the target device ready to record (e.g. +unlock screen, navigate menus and launch an app) then press ``ENTER``. +The recording has now started and button presses, taps, etc you perform on +the device will go into the .revent file. To stop the recording simply press +``ENTER`` again. + +Once you have finished recording the revent file will be pulled from the device +to the current directory. It will be named ``{device_model}.revent``. When +recording revent files for a ``GameWorkload`` you can use the ``-s`` option to +add ``run`` or ``setup`` suffixes. + +For more information run please read :ref:`record-command` -The recording has now started and button presses, taps, etc you perform on the -device will go into the .revent file. The recording will stop after the -specified time period, and you can also stop it by hitting return in the adb -shell. Replaying --------- -To replay a recorded file, run ``revent replay`` on the device, giving it the -file you want to replay:: +To replay a recorded file, run ``wa replay``, giving it the file you want to +replay:: - device# ./revent replay my_recording.revent + wa replay my_recording.revent +For more information run please read :ref:`replay-command` Using revent With Workloads --------------------------- diff --git a/wlauto/workloads/facebook/__init__.py b/wlauto/workloads/facebook/__init__.py index cbc9a7c8..69b2af1a 100644 --- a/wlauto/workloads/facebook/__init__.py +++ b/wlauto/workloads/facebook/__init__.py @@ -37,9 +37,9 @@ class Facebook(AndroidUiAutoBenchmark): Find friends. Update the facebook status - [NOTE: This workload starts disableUpdate workload as a part of setup to - disable online updates, which helps to tackle problem of uncertain - behavier during facebook workload run.] + .. note:: This workload starts disableUpdate workload as a part of setup to + disable online updates, which helps to tackle problem of uncertain + behavier during facebook workload run.] """ package = 'com.facebook.katana' @@ -79,4 +79,3 @@ class Facebook(AndroidUiAutoBenchmark): def teardown(self, context): pass - diff --git a/wlauto/workloads/recentfling/__init__.py b/wlauto/workloads/recentfling/__init__.py index 829afebc..a254b378 100644 --- a/wlauto/workloads/recentfling/__init__.py +++ b/wlauto/workloads/recentfling/__init__.py @@ -32,7 +32,7 @@ class Recentfling(Workload): For this workload to work, ``recentfling.sh`` and ``defs.sh`` must be placed in ``~/.workload_automation/dependencies/recentfling/``. These can be found - in the [AOSP Git repository](https://android.googlesource.com/platform/system/extras/+/master/tests/). + in the `AOSP Git repository `_. To change the apps that are opened at the start of the workload you will need to modify the ``defs.sh`` file. You will need to add your app to ``dfltAppList``