Initial commit of open source Workload Automation.

wlauto/workloads/antutu/__init__.py

@@ -0,0 +1,136 @@
+#    Copyright 2013-2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import os
+from collections import defaultdict, OrderedDict
+
+from wlauto import AndroidUiAutoBenchmark, Parameter
+
+
+class Antutu(AndroidUiAutoBenchmark):
+
+    name = 'antutu'
+    description = """
+    AnTuTu Benchmark is an benchmarking tool for Android Mobile Phone/Pad. It
+    can run a full test of a key project, through the "Memory Performance","CPU
+    Integer Performance","CPU Floating point Performance","2D 3D Graphics
+    Performance","SD card reading/writing speed","Database IO" performance
+    testing, and gives accurate analysis for Andriod smart phones.
+
+    http://www.antutulabs.com/AnTuTu-Benchmark
+
+    From the website:
+
+    AnTuTu Benchmark can support the latest quad-core cpu. In reaching the
+    overall and individual scores of the hardware, AnTuTu Benchmark could judge
+    your phone by the scores of the performance of the hardware. By uploading
+    the scores, Benchmark can view your device in the world rankings, allowing
+    points to let you know the level of hardware performance equipment.
+
+    """
+    #pylint: disable=E1101
+
+    package = "com.antutu.ABenchMark"
+    activity = ".ABenchMarkStart"
+    summary_metrics = ['score', 'Overall_Score']
+
+    valid_versions = ['3.3.2', '4.0.3', '5.3.0']
+
+    device_prefs_directory = '/data/data/com.antutu.ABenchMark/shared_prefs'
+    device_prefs_file = '/'.join([device_prefs_directory, 'com.antutu.ABenchMark_preferences.xml'])
+    local_prefs_directory = os.path.join(os.path.dirname(__file__), 'shared_prefs')
+
+    parameters = [
+        Parameter('version', allowed_values=valid_versions, default=sorted(valid_versions, reverse=True)[0],
+                  description=('Specify the version of AnTuTu to be run. If not specified, the latest available '
+                               'version will be used.')),
+        Parameter('times', kind=int, default=1,
+                  description=('The number of times the benchmark will be executed in a row (i.e. '
+                               'without going through the full setup/teardown process). Note: this does '
+                               'not work with versions prior to 4.0.3.')),
+        Parameter('enable_sd_tests', kind=bool, default=False,
+                  description=('If ``True`` enables SD card tests in pre version 4 AnTuTu. These tests '
+                               'were know to cause problems on platforms without an SD card. This parameter '
+                               'will be ignored on AnTuTu version 4 and higher.')),
+    ]
+
+    def __init__(self, device, **kwargs):  # pylint: disable=W0613
+        super(Antutu, self).__init__(device, **kwargs)
+        self.run_timeout = 6 * 60 * self.times
+        self.uiauto_params['version'] = self.version
+        self.uiauto_params['times'] = self.times
+        self.uiauto_params['enable_sd_tests'] = self.enable_sd_tests
+
+    def update_result(self, context):
+        super(Antutu, self).update_result(context)
+        with open(self.logcat_log) as fh:
+            if self.version == '4.0.3':
+                metrics = extract_version4_metrics(fh)
+            else:
+                metrics = extract_older_version_metrics(fh)
+        for key, value in metrics.iteritems():
+            key = key.replace(' ', '_')
+            context.result.add_metric(key, value)
+
+
+# Utility functions
+
+def extract_version4_metrics(fh):
+    metrics = OrderedDict()
+    metric_counts = defaultdict(int)
+    for line in fh:
+        if 'ANTUTU RESULT:' in line:
+            parts = line.split(':')
+            metric = parts[2].strip()
+            # If times prameter > 1 the same metric will appear
+            # multiple times in logcat -- we want to collet all of
+            # them as they're from different iterations.
+            metric_counts[metric] += 1
+            if metric_counts[metric] > 1:
+                metric += '_' + str(metric_counts[metric])
+
+            value_string = parts[3].strip()
+            # Grahics results report resolution in square brackets
+            # as part of value string.
+            if ']' in value_string:
+                value = int(value_string.split(']')[1].strip())
+            else:
+                value = int(value_string)
+
+            metrics[metric] = value
+    return metrics
+
+
+def extract_older_version_metrics(fh):
+    metrics = {}
+    metric_counts = defaultdict(int)
+    for line in fh:
+        if 'i/antutu' in line.lower():
+            parts = line.split(':')
+            if not len(parts) == 3:
+                continue
+            metric = parts[1].strip()
+            value = int(parts[2].strip())
+
+            # If times prameter > 1 the same metric will appear
+            # multiple times in logcat -- we want to collet all of
+            # them as they're from different iterations.
+            metric_counts[metric] += 1
+            if metric_counts[metric] > 1:
+                metric += ' ' + str(metric_counts[metric])
+
+            metrics[metric] = value
+    return metrics
+

wlauto/workloads/antutu/uiauto/build.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#    Copyright 2013-2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+
+class_dir=bin/classes/com/arm/wlauto/uiauto
+base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"`
+mkdir -p $class_dir
+cp $base_class $class_dir
+
+ant build
+
+if [[ -f bin/com.arm.wlauto.uiauto.antutu.jar ]]; then
+    cp bin/com.arm.wlauto.uiauto.antutu.jar ..
+fi

wlauto/workloads/antutu/uiauto/build.xml

@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project name="com.arm.wlauto.uiauto.antutu" default="help">
+
+    <!-- The local.properties file is created and updated by the 'android' tool.
+         It contains the path to the SDK. It should *NOT* be checked into
+         Version Control Systems. -->
+    <property file="local.properties" />
+
+    <!-- The ant.properties file can be created by you. It is only edited by the
+         'android' tool to add properties to it.
+         This is the place to change some Ant specific build properties.
+         Here are some properties you may want to change/update:
+
+         source.dir
+             The name of the source directory. Default is 'src'.
+         out.dir
+             The name of the output directory. Default is 'bin'.
+
+         For other overridable properties, look at the beginning of the rules
+         files in the SDK, at tools/ant/build.xml
+
+         Properties related to the SDK location or the project target should
+         be updated using the 'android' tool with the 'update' action.
+
+         This file is an integral part of the build system for your
+         application and should be checked into Version Control Systems.
+
+         -->
+    <property file="ant.properties" />
+
+    <!-- if sdk.dir was not set from one of the property file, then
+         get it from the ANDROID_HOME env var.
+         This must be done before we load project.properties since
+         the proguard config can use sdk.dir -->
+    <property environment="env" />
+    <condition property="sdk.dir" value="${env.ANDROID_HOME}">
+        <isset property="env.ANDROID_HOME" />
+    </condition>
+
+    <!-- The project.properties file is created and updated by the 'android'
+         tool, as well as ADT.
+
+         This contains project specific properties such as project target, and library
+         dependencies. Lower level build properties are stored in ant.properties
+         (or in .classpath for Eclipse projects).
+
+         This file is an integral part of the build system for your
+         application and should be checked into Version Control Systems. -->
+    <loadproperties srcFile="project.properties" />
+
+    <!-- quick check on sdk.dir -->
+    <fail
+            message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable."
+            unless="sdk.dir"
+    />
+
+    <!--
+        Import per project custom build rules if present at the root of the project.
+        This is the place to put custom intermediary targets such as:
+            -pre-build
+            -pre-compile
+            -post-compile (This is typically used for code obfuscation.
+                           Compiled code location: ${out.classes.absolute.dir}
+                           If this is not done in place, override ${out.dex.input.absolute.dir})
+            -post-package
+            -post-build
+            -pre-clean
+    -->
+    <import file="custom_rules.xml" optional="true" />
+
+    <!-- Import the actual build file.
+
+         To customize existing targets, there are two options:
+         - Customize only one target:
+             - copy/paste the target into this file, *before* the
+               <import> task.
+             - customize it to your needs.
+         - Customize the whole content of build.xml
+             - copy/paste the content of the rules files (minus the top node)
+               into this file, replacing the <import> task.
+             - customize to your needs.
+
+         ***********************
+         ****** IMPORTANT ******
+         ***********************
+         In all cases you must update the value of version-tag below to read 'custom' instead of an integer,
+         in order to avoid having your file be overridden by tools such as "android update project"
+    -->
+    <!-- version-tag: VERSION_TAG -->
+    <import file="${sdk.dir}/tools/ant/uibuild.xml" />
+
+</project>

wlauto/workloads/antutu/uiauto/project.properties

@@ -0,0 +1,14 @@
+# This file is automatically generated by Android Tools.
+# Do not modify this file -- YOUR CHANGES WILL BE ERASED!
+#
+# This file must be checked in Version Control Systems.
+#
+# To customize properties used by the Ant build system edit
+# "ant.properties", and override values to adapt the script to your
+# project structure.
+#
+# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home):
+#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt
+
+# Project target.
+target=android-19

wlauto/workloads/antutu/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java

@@ -0,0 +1,295 @@
+/*    Copyright 2013-2015 ARM Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package com.arm.wlauto.uiauto.antutu;
+
+import java.util.Set;
+import java.util.HashSet;
+import java.util.concurrent.TimeUnit;
+
+import android.app.Activity;
+import android.os.Bundle;
+import android.util.Log;
+
+// Import the uiautomator libraries
+import com.android.uiautomator.core.UiObject;
+import com.android.uiautomator.core.UiObjectNotFoundException;
+import com.android.uiautomator.core.UiScrollable;
+import com.android.uiautomator.core.UiSelector;
+import com.android.uiautomator.core.UiCollection;
+import com.android.uiautomator.testrunner.UiAutomatorTestCase;
+
+import com.arm.wlauto.uiauto.BaseUiAutomation;
+
+public class UiAutomation extends BaseUiAutomation {   
+
+    public static String TAG = "antutu";
+
+    private static int initialTimeoutSeconds = 20;
+
+    public void runUiAutomation() throws Exception{
+        Bundle parameters = getParams();
+
+        String version = parameters.getString("version");
+        boolean enableSdTests = Boolean.parseBoolean(parameters.getString("enable_sd_tests")); 
+
+        int times = Integer.parseInt(parameters.getString("times"));
+        if (times < 1) {
+                times = 1;
+        }
+
+        if (version.equals("4.0.3") || version.equals("5.3.0")){
+            int iteration = 0;
+	    dismissNewVersionNotificationIfNecessary();
+            hitTestButton();
+            while (true) {
+                    if (version.equals("5.3.0"))
+                        hitTestButtonVersion5();
+                    else
+                        hitTestButton();
+
+                    waitForVersion4Results();
+                    viewDetails();
+                    extractResults();
+                    iteration++;
+                    if (iteration >= times) {
+                        break;
+                    }
+                    returnToTestScreen();
+            }
+        } else { // version earlier than 4.0.3
+            dismissReleaseNotesDialogIfNecessary();
+            if(!enableSdTests){
+               disableSdCardTests();
+            }
+            hitStart();
+            waitForAndViewResults();
+        }
+
+        Bundle status = new Bundle();
+        getAutomationSupport().sendStatus(Activity.RESULT_OK, status);
+    }
+
+    public boolean dismissNewVersionNotificationIfNecessary() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject closeButton = new UiObject(selector.text("Cancel"));
+        if (closeButton.waitForExists(TimeUnit.SECONDS.toMillis(initialTimeoutSeconds))) {
+            closeButton.click();
+            sleep(1); // diaglog dismissal
+            return true;
+        } else {
+            return false;
+        }
+    }
+    
+    public boolean dismissReleaseNotesDialogIfNecessary() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject closeButton = new UiObject(selector.text("Close"));
+        if (closeButton.waitForExists(TimeUnit.SECONDS.toMillis(initialTimeoutSeconds))) {
+            closeButton.click();
+            sleep(1); // diaglog dismissal
+            return true;
+        } else {
+            return false;
+        }
+    }
+
+    public void hitTestButton() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject test = new UiObject(selector.text("Test")
+                                             .className("android.widget.Button"));
+        test.waitForExists(initialTimeoutSeconds);
+        test.click();
+        sleep(1); // possible tab transtion
+    }
+
+   /* In version 5 of antutu, the test has been changed from a button widget to a textview */
+
+   public void hitTestButtonVersion5() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject test = new UiObject(selector.resourceId("com.antutu.ABenchMark:id/start_test_region")
+                                             .className("android.widget.TextView"));
+        test.waitForExists(initialTimeoutSeconds);
+        test.click();
+        sleep(1); // possible tab transtion
+    }
+
+
+
+    public void hitTest() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject test = new UiObject(selector.text("Test"));
+        test.click();
+        sleep(1); // possible tab transtion
+    }
+
+    public void disableSdCardTests() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject custom = new UiObject(selector.textContains("Custom"));
+        custom.click();
+        sleep(1); // tab transition
+
+        UiObject sdCardButton = new UiObject(selector.text("SD card IO"));
+        sdCardButton.click();
+    }
+
+    public void hitStart() throws Exception {
+        UiSelector selector = new UiSelector();
+        Log.v(TAG, "Start the test");
+        UiObject startButton = new UiObject(selector.text("Start Test")
+                                                    .className("android.widget.Button"));
+        startButton.click();
+    }
+
+    public void waitForVersion4Results() throws Exception {
+        // The observed behaviour seems to vary between devices. On some platforms,
+        // the benchmark terminates in the barchart screen; on others, it terminates in
+        // details screen. So we have to wait for either and then act appropriatesl (on the barchart
+        // screen a back button press is required to get to the details screen.
+        UiSelector selector = new UiSelector();
+        UiObject barChart = new UiObject(new UiSelector().className("android.widget.TextView")
+                                                         .text("Bar Chart"));
+        UiObject detailsButton = new UiObject(new UiSelector().className("android.widget.Button")
+                                                              .text("Details"));
+        for (int i = 0; i < 60; i++) {
+            if (detailsButton.exists() || barChart.exists()) {
+                break;
+            }
+            sleep(5);
+        }
+
+        if (barChart.exists()) {
+            getUiDevice().pressBack();
+        }
+    }
+
+    public void viewDetails() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject detailsButton = new UiObject(new UiSelector().className("android.widget.Button")
+                                                              .text("Details"));
+        detailsButton.clickAndWaitForNewWindow();
+    }
+
+    public void extractResults() throws Exception {
+        extractOverallResult();
+        extractSectionResults();
+    }
+
+    public void extractOverallResult() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiSelector resultTextSelector = selector.className("android.widget.TextView").index(0);
+        UiSelector relativeLayoutSelector = selector.className("android.widget.RelativeLayout").index(1);
+        UiObject result = new UiObject(selector.className("android.widget.LinearLayout")
+                                               .childSelector(relativeLayoutSelector)
+                                               .childSelector(resultTextSelector));
+        if (result.exists()) {
+            Log.v(TAG, String.format("ANTUTU RESULT: Overall Score: %s", result.getText()));
+        }
+    }
+
+    public void extractSectionResults() throws Exception {
+        UiSelector selector = new UiSelector();
+	Set<String> processedMetrics = new HashSet<String>();
+
+        actuallyExtractSectionResults(processedMetrics);
+        UiScrollable resultsList = new UiScrollable(selector.className("android.widget.ScrollView"));
+        // Note: there is an assumption here that the entire results list fits on at most
+        //       two screens on the device. Given then number of entries in the current
+        //       antutu verion and the devices we're dealing with, this is a reasonable
+        //       assumption. But if this changes, this will need to be adapted to scroll more
+        //       slowly.
+        resultsList.scrollToEnd(10);
+        actuallyExtractSectionResults(processedMetrics);
+    }
+
+    public void actuallyExtractSectionResults(Set<String> processedMetrics) throws Exception {
+        UiSelector selector = new UiSelector();
+
+        for (int i = 1; i < 8; i += 2) {
+            UiObject table = new UiObject(selector.className("android.widget.TableLayout").index(i));
+            for (int j = 0; j < 3; j += 2) {
+                UiObject row = table.getChild(selector.className("android.widget.TableRow").index(j));
+                UiObject metric =  row.getChild(selector.className("android.widget.TextView").index(0));
+                UiObject value =  row.getChild(selector.className("android.widget.TextView").index(1));
+
+                if (metric.exists() && value.exists()) {
+                    String metricText = metric.getText();
+                    if (!processedMetrics.contains(metricText)) {
+                        Log.v(TAG, String.format("ANTUTU RESULT: %s %s", metric.getText(), value.getText()));
+                        processedMetrics.add(metricText);
+                    }
+                }
+            }
+        }
+    }
+
+    public void returnToTestScreen() throws Exception {
+        getUiDevice().pressBack();
+        UiSelector selector = new UiSelector();
+        UiObject retestButton = new UiObject(selector.text("Test Again")
+                                                     .className("android.widget.Button"));
+        retestButton.clickAndWaitForNewWindow();
+    }
+
+    public void waitForAndViewResults() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject submitTextView = new UiObject(selector.text("Submit Scores")
+                                                       .className("android.widget.TextView"));
+        UiObject detailTextView = new UiObject(selector.text("Detailed Scores")
+                                                       .className("android.widget.TextView"));
+        UiObject commentTextView = new UiObject(selector.text("User comment")
+                                                        .className("android.widget.TextView"));
+        boolean foundResults = false;
+        for (int i = 0; i < 60; i++) {
+            if (detailTextView.exists() || submitTextView.exists() || commentTextView.exists()) {
+                foundResults = true;
+                break;
+            }
+            sleep(5);
+        }
+
+        if (!foundResults) {
+                throw new UiObjectNotFoundException("Did not see AnTuTu results screen.");
+        }
+
+        if (commentTextView.exists()) {
+            getUiDevice().pressBack();
+        }
+        // Yes, sometimes, it needs to be done twice...
+        if (commentTextView.exists()) {
+            getUiDevice().pressBack();
+        }
+
+        if (detailTextView.exists()) {
+            detailTextView.click();
+            sleep(1); // tab transition
+
+            UiObject testTextView = new UiObject(selector.text("Test")
+                                                    .className("android.widget.TextView"));
+            if (testTextView.exists()) {
+            testTextView.click();
+            sleep(1); // tab transition
+            }
+
+            UiObject scoresTextView = new UiObject(selector.text("Scores")
+                                                    .className("android.widget.TextView"));
+            if (scoresTextView.exists()) {
+            scoresTextView.click();
+            sleep(1); // tab transition
+            }
+        }
+    }
+}