Initial commit of open source Workload Automation.

wlauto/workloads/quadrant/__init__.py

@@ -0,0 +1,112 @@
+#    Copyright 2013-2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+import re
+from collections import defaultdict
+
+from wlauto import AndroidUiAutoBenchmark
+
+
+TEST_TYPES = {
+    'benchmark_cpu_branching_logic': 'time',
+    'benchmark_cpu_matrix_int': 'time',
+    'benchmark_cpu_matrix_long': 'time',
+    'benchmark_cpu_matrix_short': 'time',
+    'benchmark_cpu_matrix_byte': 'time',
+    'benchmark_cpu_matrix_float': 'time',
+    'benchmark_cpu_matrix_double': 'time',
+    'benchmark_cpu_checksum': 'time',
+    'benchmark_cpu': 'aggregate',
+    'benchmark_memory_transfer': 'time',
+    'benchmark_memory': 'aggregate',
+    'benchmark_io_fs_write': 'time',
+    'benchmark_io_fs_read': 'time',
+    'benchmark_io_db_write': 'time',
+    'benchmark_io_db_read': 'time',
+    'benchmark_io': 'aggregate',
+    'benchmark_g2d_fractal': 'rate',
+    'benchmark_g2d': 'aggregate',
+    'benchmark_g3d_corridor': 'rate',
+    'benchmark_g3d_planet': 'rate',
+    'benchmark_g3d_dna': 'rate',
+    'benchmark_g3d': 'aggregate',
+    'benchmark': 'aggregate',
+}
+
+TYPE_TESTS = defaultdict(list)
+for k, v in TEST_TYPES.iteritems():
+    TYPE_TESTS[v].append(k)
+
+TYPE_UNITS = {
+    'time': 'ms',
+    'rate': 'Hz',
+}
+
+REGEX_TEMPLATES = {
+    'aggregate': r'(?P<metric>{}) aggregate score is (?P<score>\d+)',
+    'time': r'(?P<metric>{}) executed in (?P<time>\d+) ms, '
+            r'reference time: (?P<reference>\d+) ms, '
+            r'score: (?P<score>\d+)',
+    'rate': r'(?P<metric>{}) executed with a rate of (?P<rate>[0-9.]+)/sec, '
+            r'reference rate: (?P<reference>[0-9.]+)/sec, '
+            r'score: (?P<score>\d+)',
+}
+
+TEST_REGEXES = {}
+for test_, type_ in TEST_TYPES.items():
+    TEST_REGEXES[test_] = re.compile(REGEX_TEMPLATES[type_].format(test_))
+
+
+class Quadrant(AndroidUiAutoBenchmark):
+
+    name = 'quadrant'
+    description = """
+    Quadrant is a benchmark for mobile devices, capable of measuring CPU, memory,
+    I/O and 3D graphics performance.
+
+    http://www.aurorasoftworks.com/products/quadrant
+
+    From the website:
+    Quadrant outputs a score for the following categories: 2D, 3D, Mem, I/O, CPU
+    , Total.
+    """
+    package = 'com.aurorasoftworks.quadrant.ui.professional'
+    activity = '.QuadrantProfessionalLauncherActivity'
+    summary_metrics = ['benchmark_score']
+
+    run_timeout = 10 * 60
+
+    def __init__(self, device, **kwargs):
+        super(Quadrant, self).__init__(device, **kwargs)
+        self.uiauto_params['has_gpu'] = self.device.has_gpu
+        self.regex = {}
+
+    def update_result(self, context):
+        super(Quadrant, self).update_result(context)
+        with open(self.logcat_log) as fh:
+            for line in fh:
+                for test, regex in TEST_REGEXES.items():
+                    match = regex.search(line)
+                    if match:
+                        test_type = TEST_TYPES[test]
+                        data = match.groupdict()
+                        if test_type != 'aggregate':
+                            context.result.add_metric(data['metric'] + '_' + test_type,
+                                                      data[test_type],
+                                                      TYPE_UNITS[test_type])
+                        context.result.add_metric(data['metric'] + '_score', data['score'])
+                        break
+

wlauto/workloads/quadrant/uiauto/build.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+#    Copyright 2013-2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+
+class_dir=bin/classes/com/arm/wlauto/uiauto
+base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"`
+mkdir -p $class_dir
+cp $base_class $class_dir
+
+ant build
+
+if [[ -f bin/com.arm.wlauto.uiauto.quadrant.jar ]]; then
+    cp bin/com.arm.wlauto.uiauto.quadrant.jar ..
+fi

wlauto/workloads/quadrant/uiauto/build.xml

@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project name="com.arm.wlauto.uiauto.quadrant" default="help">
+
+    <!-- The local.properties file is created and updated by the 'android' tool.
+         It contains the path to the SDK. It should *NOT* be checked into
+         Version Control Systems. -->
+    <property file="local.properties" />
+
+    <!-- The ant.properties file can be created by you. It is only edited by the
+         'android' tool to add properties to it.
+         This is the place to change some Ant specific build properties.
+         Here are some properties you may want to change/update:
+
+         source.dir
+             The name of the source directory. Default is 'src'.
+         out.dir
+             The name of the output directory. Default is 'bin'.
+
+         For other overridable properties, look at the beginning of the rules
+         files in the SDK, at tools/ant/build.xml
+
+         Properties related to the SDK location or the project target should
+         be updated using the 'android' tool with the 'update' action.
+
+         This file is an integral part of the build system for your
+         application and should be checked into Version Control Systems.
+
+         -->
+    <property file="ant.properties" />
+
+    <!-- if sdk.dir was not set from one of the property file, then
+         get it from the ANDROID_HOME env var.
+         This must be done before we load project.properties since
+         the proguard config can use sdk.dir -->
+    <property environment="env" />
+    <condition property="sdk.dir" value="${env.ANDROID_HOME}">
+        <isset property="env.ANDROID_HOME" />
+    </condition>
+
+    <!-- The project.properties file is created and updated by the 'android'
+         tool, as well as ADT.
+
+         This contains project specific properties such as project target, and library
+         dependencies. Lower level build properties are stored in ant.properties
+         (or in .classpath for Eclipse projects).
+
+         This file is an integral part of the build system for your
+         application and should be checked into Version Control Systems. -->
+    <loadproperties srcFile="project.properties" />
+
+    <!-- quick check on sdk.dir -->
+    <fail
+            message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable."
+            unless="sdk.dir"
+    />
+
+    <!--
+        Import per project custom build rules if present at the root of the project.
+        This is the place to put custom intermediary targets such as:
+            -pre-build
+            -pre-compile
+            -post-compile (This is typically used for code obfuscation.
+                           Compiled code location: ${out.classes.absolute.dir}
+                           If this is not done in place, override ${out.dex.input.absolute.dir})
+            -post-package
+            -post-build
+            -pre-clean
+    -->
+    <import file="custom_rules.xml" optional="true" />
+
+    <!-- Import the actual build file.
+
+         To customize existing targets, there are two options:
+         - Customize only one target:
+             - copy/paste the target into this file, *before* the
+               <import> task.
+             - customize it to your needs.
+         - Customize the whole content of build.xml
+             - copy/paste the content of the rules files (minus the top node)
+               into this file, replacing the <import> task.
+             - customize to your needs.
+
+         ***********************
+         ****** IMPORTANT ******
+         ***********************
+         In all cases you must update the value of version-tag below to read 'custom' instead of an integer,
+         in order to avoid having your file be overridden by tools such as "android update project"
+    -->
+    <!-- version-tag: VERSION_TAG -->
+    <import file="${sdk.dir}/tools/ant/uibuild.xml" />
+
+</project>

wlauto/workloads/quadrant/uiauto/project.properties

@@ -0,0 +1,14 @@
+# This file is automatically generated by Android Tools.
+# Do not modify this file -- YOUR CHANGES WILL BE ERASED!
+#
+# This file must be checked in Version Control Systems.
+#
+# To customize properties used by the Ant build system edit
+# "ant.properties", and override values to adapt the script to your
+# project structure.
+#
+# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home):
+#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt
+
+# Project target.
+target=android-17

wlauto/workloads/quadrant/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java

@@ -0,0 +1,120 @@
+/*    Copyright 2013-2015 ARM Limited
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+
+package com.arm.wlauto.uiauto.quadrant;
+
+import java.util.concurrent.TimeUnit;
+
+import android.app.Activity;
+import android.os.Bundle;
+import android.util.Log;
+
+// Import the uiautomator libraries
+import com.android.uiautomator.core.UiObject;
+import com.android.uiautomator.core.UiObjectNotFoundException;
+import com.android.uiautomator.core.UiScrollable;
+import com.android.uiautomator.core.UiSelector;
+import com.android.uiautomator.testrunner.UiAutomatorTestCase;
+
+import com.arm.wlauto.uiauto.BaseUiAutomation;
+
+public class UiAutomation extends BaseUiAutomation {   
+
+    public static String TAG = "quadrant";
+
+    public void runUiAutomation() throws Exception {
+        Bundle status = new Bundle();
+        Bundle params = getParams();
+        boolean hasGpu = Boolean.parseBoolean(params.getString("has_gpu").toLowerCase());
+
+        clearLogcat();
+        handleFtuInfoDialogIfNecessary();
+        goToRunCustomBenchmark();
+        selectTestsToRun(hasGpu);
+        hitStart();
+        handleWarningIfNecessary();
+        waitForResults();
+
+        getAutomationSupport().sendStatus(Activity.RESULT_OK, status);
+    }
+
+    public void handleFtuInfoDialogIfNecessary() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject infoText = new UiObject(selector.text("Information"));
+        if (infoText.waitForExists(TimeUnit.SECONDS.toMillis(10)))
+        {
+            UiObject okButton = new UiObject(selector.text("OK")
+                                                     .className("android.widget.Button"));
+            okButton.click();
+        }
+        else
+        {
+            // FTU dialog didn't come up.
+        }
+    }
+
+    public void goToRunCustomBenchmark() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject runCustom = new UiObject(selector.text("Run custom benchmark")
+                                                  .className("android.widget.TextView"));
+        runCustom.clickAndWaitForNewWindow();
+    }
+
+    // By default, all tests are selected. However, if our device does not have a GPU, then
+    // running graphics tests may cause a crash, so we disable those.
+    public void selectTestsToRun(boolean hasGpu) throws Exception {
+        if(!hasGpu) {
+            UiSelector selector = new UiSelector();
+            UiObject gfx2d = new UiObject(selector.text("2D graphics")
+                                                  .className("android.widget.CheckBox"));
+            gfx2d.click();
+
+            UiObject gfx3d = new UiObject(selector.text("3D graphics")
+                                                  .className("android.widget.CheckBox"));
+            gfx3d.click();
+        }       
+    }
+
+    public void hitStart() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject startButton = new UiObject(selector.text("Start")
+                                                    .className("android.widget.Button")
+                                                    .packageName("com.aurorasoftworks.quadrant.ui.professional"));
+        startButton.click();
+    }
+
+    // Even if graphics tests aren't selected, Quadrant will still show a warning about running
+    // with software rendering.
+    public void handleWarningIfNecessary() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject  warning = new UiObject(selector.text("Warning"));
+        if (warning.waitForExists(TimeUnit.SECONDS.toMillis(2))) {
+            UiObject closeButton = new UiObject(selector.text("Close")
+                                                        .className("android.widget.Button"));
+            if (closeButton.exists()) {
+                closeButton.click();
+            }
+        }
+        else
+        {
+            // Warning dialog didn't come up.
+        }
+    }
+
+    public void waitForResults() throws Exception {
+        waitForLogcatText("benchmark aggregate score is", TimeUnit.SECONDS.toMillis(200));
+    }
+}