diff --git a/wa/workloads/aitutu/__init__.py b/wa/workloads/aitutu/__init__.py new file mode 100755 index 00000000..e2794475 --- /dev/null +++ b/wa/workloads/aitutu/__init__.py @@ -0,0 +1,67 @@ +# Copyright 2014-2018 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import re + +from wa import ApkUiautoWorkload +from wa.framework.exception import WorkloadError + + +class Aitutu(ApkUiautoWorkload): + + name = 'aitutu' + package_names = ['com.antutu.aibenchmark'] + regex_matches = [re.compile(r'Overall Score ([\d.]+)'), + re.compile(r'Image Total Score ([\d.]+) ([\w]+) ([\w]+)'), + re.compile(r'Image Speed Score ([\d.]+) ([\w]+) ([\w]+)'), + re.compile(r'Image Accuracy Score ([\d.]+) ([\w]+) ([\w]+)'), + re.compile(r'Object Total Score ([\d.]+) ([\w]+) ([\w]+)'), + re.compile(r'Object Speed Score ([\d.]+) ([\w]+) ([\w]+)'), + re.compile(r'Object Accuracy Score ([\d.]+) ([\w]+) ([\w]+)')] + description = ''' + Executes Aitutu Image Speed/Accuracy and Object Speed/Accuracy tests + + The Aitutu workflow carries out the following tasks. + 1. Open Aitutu application + 2. Download the resources for the test + 3. Execute the tests + + Known working APK version: 1.0.3 + ''' + + def __init__(self, target, **kwargs): + super(Aitutu, self).__init__(target, **kwargs) + self.gui.timeout = 1200000 + + def update_output(self, context): + super(Aitutu, self).update_output(context) + expected_results = len(self.regex_matches) + logcat_file = context.get_artifact_path('logcat') + with open(logcat_file) as fh: + for line in fh: + for regex in self.regex_matches: + match = regex.search(line) + if match: + classifiers = {} + result = match.group(1) + if (len(match.groups())) > 1: + entry = regex.pattern.rsplit(None, 3)[0] + classifiers = {'model': match.group(3)} + else: + entry = regex.pattern.rsplit(None, 1)[0] + context.add_metric(entry, result, '', lower_is_better=False, classifiers=classifiers) + expected_results -= 1 + if expected_results > 0: + msg = "The Aitutu workload has failed. Expected {} scores, Detected {} scores." + raise WorkloadError(msg.format(len(self.regex_matches), expected_results)) diff --git a/wa/workloads/aitutu/com.arm.wa.uiauto.aitutu.apk b/wa/workloads/aitutu/com.arm.wa.uiauto.aitutu.apk new file mode 100644 index 00000000..2ee3cacc Binary files /dev/null and b/wa/workloads/aitutu/com.arm.wa.uiauto.aitutu.apk differ diff --git a/wa/workloads/aitutu/uiauto/app/build.gradle b/wa/workloads/aitutu/uiauto/app/build.gradle new file mode 100644 index 00000000..26d7f184 --- /dev/null +++ b/wa/workloads/aitutu/uiauto/app/build.gradle @@ -0,0 +1,35 @@ +apply plugin: 'com.android.application' + +def packageName = "com.arm.wa.uiauto.aitutu" + +android { + compileSdkVersion 25 + buildToolsVersion '25.0.0' + defaultConfig { + applicationId "${packageName}" + minSdkVersion 18 + targetSdkVersion 25 + testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + } + buildTypes { + applicationVariants.all { variant -> + variant.outputs.each { output -> + output.outputFile = file("$project.buildDir/apk/${packageName}.apk") + } + } + } +} + +dependencies { + compile fileTree(include: ['*.jar'], dir: 'libs') + compile 'com.android.support.test:runner:0.5' + compile 'com.android.support.test:rules:0.5' + compile 'com.android.support.test.uiautomator:uiautomator-v18:2.1.2' + compile(name: 'uiauto', ext: 'aar') +} + +repositories { + flatDir { + dirs 'libs' + } +} diff --git a/wa/workloads/aitutu/uiauto/app/src/main/AndroidManifest.xml b/wa/workloads/aitutu/uiauto/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000..b75ae97e --- /dev/null +++ b/wa/workloads/aitutu/uiauto/app/src/main/AndroidManifest.xml @@ -0,0 +1,12 @@ + + + + + + + diff --git a/wa/workloads/aitutu/uiauto/app/src/main/java/com/arm/wa/uiauto/aitutu/UiAutomation.java b/wa/workloads/aitutu/uiauto/app/src/main/java/com/arm/wa/uiauto/aitutu/UiAutomation.java new file mode 100755 index 00000000..ef19b736 --- /dev/null +++ b/wa/workloads/aitutu/uiauto/app/src/main/java/com/arm/wa/uiauto/aitutu/UiAutomation.java @@ -0,0 +1,143 @@ +/* Copyright 2013-2018 ARM Limited + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. +*/ + + +package com.arm.wa.uiauto.aitutu; + +import android.app.Activity; +import android.os.Bundle; +import android.graphics.Rect; +import android.support.test.runner.AndroidJUnit4; +import android.support.test.uiautomator.UiObject; +import android.support.test.uiautomator.UiObjectNotFoundException; +import android.support.test.uiautomator.UiSelector; +import android.support.test.uiautomator.UiScrollable; +import android.view.KeyEvent; +import android.util.Log; + +import com.arm.wa.uiauto.BaseUiAutomation; + +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import java.util.concurrent.TimeUnit; + + +@RunWith(AndroidJUnit4.class) +public class UiAutomation extends BaseUiAutomation { + + public static String TAG = "UXPERF"; + + @Test + public void setup() throws Exception { + clearPopups(); + downloadAssets(); + } + + @Test + public void runWorkload() throws Exception { + runBenchmark(); + } + + @Test + public void extractResults() throws Exception { + getScores(); + } + + public void clearPopups() throws Exception { + UiSelector selector = new UiSelector(); + + UiObject cancel = mDevice.findObject(selector.textContains("CANCEL") + .className("android.widget.Button")); + cancel.waitForExists(60000); + if (cancel.exists()){ + cancel.click(); + } + //waitObject(cancel); + //cancel.click(); + } + + public void downloadAssets() throws Exception { + UiSelector selector = new UiSelector(); + //Start the tests + UiObject start = mDevice.findObject(selector.textContains("Start Testing") + .className("android.widget.TextView")); + waitObject(start); + start.click(); + + UiObject check = mDevice.findObject(selector.textContains("classification") + .className("android.widget.TextView")); + waitObject(check); + } + + public void runBenchmark() throws Exception { + UiSelector selector = new UiSelector(); + + //Wait for the tests to complete + UiObject complete = + mDevice.findObject(selector.text("TEST AGAIN") + .className("android.widget.Button")); + complete.waitForExists(1200000); + + } + + public void getScores() throws Exception { + UiSelector selector = new UiSelector(); + //Declare the models used + UiObject imageMod = + mDevice.findObject(selector.resourceId("com.antutu.aibenchmark:id/recyclerView")) + .getChild(selector.index(1)) + .getChild(selector.resourceId("com.antutu.aibenchmark:id/textViewAIModelName")); + UiObject objectMod = + mDevice.findObject(selector.resourceId("com.antutu.aibenchmark:id/recyclerView")) + .getChild(selector.index(4)) + .getChild(selector.resourceId("com.antutu.aibenchmark:id/textViewAIModelName")); + //Log the scores and models + UiObject totalScore = + mDevice.findObject(selector.resourceId("com.antutu.aibenchmark:id/textViewTotalScore")); + Log.d(TAG, "Overall Score " + totalScore.getText()); + UiObject imageTotal = + mDevice.findObject(selector.resourceId("com.antutu.aibenchmark:id/recyclerView")) + .getChild(selector.index(1)) + .getChild(selector.resourceId("com.antutu.aibenchmark:id/textViewSIDScore")); + Log.d(TAG, "Image Total Score " + imageTotal.getText() + " Model " + imageMod.getText()); + UiObject imageSpeed = + mDevice.findObject(selector.resourceId("com.antutu.aibenchmark:id/recyclerView")) + .getChild(selector.index(2)) + .getChild(selector.resourceId("com.antutu.aibenchmark:id/textViewBIDScore")); + Log.d(TAG, "Image Speed Score " + imageSpeed.getText() + " Model " + imageMod.getText()); + UiObject imageAcc = + mDevice.findObject(selector.resourceId("com.antutu.aibenchmark:id/recyclerView")) + .getChild(selector.index(3)) + .getChild(selector.resourceId("com.antutu.aibenchmark:id/textViewBIDScore")); + Log.d(TAG, "Image Accuracy Score " + imageAcc.getText() + " Model " + imageMod.getText()); + UiObject objectTotal = + mDevice.findObject(selector.resourceId("com.antutu.aibenchmark:id/recyclerView")) + .getChild(selector.index(4)) + .getChild(selector.resourceId("com.antutu.aibenchmark:id/textViewSIDScore")); + Log.d(TAG, "Object Total Score " + objectTotal.getText() + " Model " + objectMod.getText()); + UiObject objectSpeed = + mDevice.findObject(selector.resourceId("com.antutu.aibenchmark:id/recyclerView")) + .getChild(selector.index(5)) + .getChild(selector.resourceId("com.antutu.aibenchmark:id/textViewBIDScore")); + Log.d(TAG, "Object Speed Score " + objectSpeed.getText() + " Model " + objectMod.getText()); + UiObject objectAcc = + mDevice.findObject(selector.resourceId("com.antutu.aibenchmark:id/recyclerView")) + .getChild(selector.index(6)) + .getChild(selector.resourceId("com.antutu.aibenchmark:id/textViewBIDScore")); + Log.d(TAG, "Object Accuracy Score " + objectAcc.getText() + " Model " + objectMod.getText()); + } +} diff --git a/wa/workloads/aitutu/uiauto/build.gradle b/wa/workloads/aitutu/uiauto/build.gradle new file mode 100644 index 00000000..b78a0b86 --- /dev/null +++ b/wa/workloads/aitutu/uiauto/build.gradle @@ -0,0 +1,23 @@ +// Top-level build file where you can add configuration options common to all sub-projects/modules. + +buildscript { + repositories { + jcenter() + } + dependencies { + classpath 'com.android.tools.build:gradle:2.3.1' + + // NOTE: Do not place your application dependencies here; they belong + // in the individual module build.gradle files + } +} + +allprojects { + repositories { + jcenter() + } +} + +task clean(type: Delete) { + delete rootProject.buildDir +} diff --git a/wa/workloads/aitutu/uiauto/build.sh b/wa/workloads/aitutu/uiauto/build.sh new file mode 100755 index 00000000..75f3a090 --- /dev/null +++ b/wa/workloads/aitutu/uiauto/build.sh @@ -0,0 +1,55 @@ +#!/bin/bash +# Copyright 2013-2018 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +set -e + +# CD into build dir if possible - allows building from any directory +script_path='.' +if `readlink -f $0 &>/dev/null`; then + script_path=`readlink -f $0 2>/dev/null` +fi +script_dir=`dirname $script_path` +cd $script_dir + +# Ensure gradelw exists before starting +if [[ ! -f gradlew ]]; then + echo 'gradlew file not found! Check that you are in the right directory.' + exit 9 +fi + +# Copy base class library from wa dist +libs_dir=app/libs +base_class=`python -c "import os, wa; print os.path.join(os.path.dirname(wa.__file__), 'framework', 'uiauto', 'uiauto.aar')"` +mkdir -p $libs_dir +cp $base_class $libs_dir + +# Build and return appropriate exit code if failed +# gradle build +./gradlew clean :app:assembleDebug +exit_code=$? +if [[ $exit_code -ne 0 ]]; then + echo "ERROR: 'gradle build' exited with code $exit_code" + exit $exit_code +fi + +# If successful move APK file to workload folder (overwrite previous) +package=com.arm.wa.uiauto.aitutu +rm -f ../$package +if [[ -f app/build/apk/$package.apk ]]; then + cp app/build/apk/$package.apk ../$package.apk +else + echo 'ERROR: UiAutomator apk could not be found!' + exit 9 +fi diff --git a/wa/workloads/aitutu/uiauto/gradle/wrapper/gradle-wrapper.jar b/wa/workloads/aitutu/uiauto/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 00000000..13372aef Binary files /dev/null and b/wa/workloads/aitutu/uiauto/gradle/wrapper/gradle-wrapper.jar differ diff --git a/wa/workloads/aitutu/uiauto/gradle/wrapper/gradle-wrapper.properties b/wa/workloads/aitutu/uiauto/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 00000000..760fcce7 --- /dev/null +++ b/wa/workloads/aitutu/uiauto/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,6 @@ +#Wed May 03 15:42:44 BST 2017 +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip diff --git a/wa/workloads/aitutu/uiauto/gradlew b/wa/workloads/aitutu/uiauto/gradlew new file mode 100755 index 00000000..9d82f789 --- /dev/null +++ b/wa/workloads/aitutu/uiauto/gradlew @@ -0,0 +1,160 @@ +#!/usr/bin/env bash + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn ( ) { + echo "$*" +} + +die ( ) { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; +esac + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules +function splitJvmOpts() { + JVM_OPTS=("$@") +} +eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS +JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" + +exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" diff --git a/wa/workloads/aitutu/uiauto/gradlew.bat b/wa/workloads/aitutu/uiauto/gradlew.bat new file mode 100644 index 00000000..aec99730 --- /dev/null +++ b/wa/workloads/aitutu/uiauto/gradlew.bat @@ -0,0 +1,90 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windowz variants + +if not "%OS%" == "Windows_NT" goto win9xME_args +if "%@eval[2+2]" == "4" goto 4NT_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* +goto execute + +:4NT_args +@rem Get arguments from the 4NT Shell from JP Software +set CMD_LINE_ARGS=%$ + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/wa/workloads/aitutu/uiauto/settings.gradle b/wa/workloads/aitutu/uiauto/settings.gradle new file mode 100644 index 00000000..e7b4def4 --- /dev/null +++ b/wa/workloads/aitutu/uiauto/settings.gradle @@ -0,0 +1 @@ +include ':app'