1
0
mirror of https://github.com/ARM-software/workload-automation.git synced 2025-01-31 02:01:16 +00:00

workloads/glbench: Port workload to WA3

This commit is contained in:
Marc Bonnici 2018-11-16 14:50:55 +00:00 committed by setrofim
parent 88f708abf5
commit d3d5ca9154
12 changed files with 679 additions and 0 deletions

View File

@ -0,0 +1,144 @@
# Copyright 2013-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=E1101,E0203
import re
import os
from wa import ApkUiautoWorkload, Parameter, Alias
from wa.framework.exception import ConfigError
# These maps provide use-friendly aliases for the most common options.
USE_CASE_MAP = {
'egypt': 'GLBenchmark 2.5 Egypt HD',
'egypt-classic': 'GLBenchmark 2.1 Egypt Classic',
't-rex': 'GLBenchmark 2.7 T-Rex HD',
}
TYPE_MAP = {
'onscreen': 'C24Z16 Onscreen Auto',
'offscreen': 'C24Z16 Offscreen Auto',
}
class Glb(ApkUiautoWorkload):
name = 'glbenchmark'
description = """
Measures the graphics performance of Android devices by testing
the underlying OpenGL (ES) implementation.
http://gfxbench.com/about-gfxbench.jsp
From the website:
The benchmark includes console-quality high-level 3D animations
(T-Rex HD and Egypt HD) and low-level graphics measurements.
With high vertex count and complex effects such as motion blur, parallax
mapping and particle systems, the engine of GFXBench stresses GPUs in order
provide users a realistic feedback on their device.
"""
activity = 'com.glbenchmark.activities.GLBenchmarkDownloaderActivity'
view = 'com.glbenchmark.glbenchmark27/com.glbenchmark.activities.GLBRender'
package_names = ['com.glbenchmark.glbenchmark27', 'com.glbenchmark.glbenchmark25']
packages = {
'2.7': 'com.glbenchmark.glbenchmark27',
'2.5': 'com.glbenchmark.glbenchmark25',
}
# If usecase is not specified the default usecase is the first supported usecase alias
# for the specified version.
supported_usecase_aliases = {
'2.7': ['t-rex', 'egypt'],
'2.5': ['egypt-classic', 'egypt'],
}
default_iterations = 1
install_timeout = 500
run_timeout = 4 * 60
regex = re.compile(r'GLBenchmark (metric|FPS): (.*)')
parameters = [
Parameter('version', default='2.7', allowed_values=['2.7', '2.5'], override=True,
description=('Specifies which version of the benchmark to run (different versions '
'support different use cases).')),
Parameter('use_case', default=None,
description="""Specifies which usecase to run, as listed in the benchmark menu; e.g.
``'GLBenchmark 2.5 Egypt HD'``. For convenience, two aliases are provided
for the most common use cases: ``'egypt'`` and ``'t-rex'``. These could
be use instead of the full use case title. For version ``'2.7'`` it defaults
to ``'t-rex'``, for version ``'2.5'`` it defaults to ``'egypt-classic'``.
"""),
Parameter('type', default='onscreen',
description="""Specifies which type of the use case to run, as listed in the benchmarks
menu (small text underneath the use case name); e.g. ``'C24Z16 Onscreen Auto'``.
For convenience, two aliases are provided for the most common types:
``'onscreen'`` and ``'offscreen'``. These may be used instead of full type
names.
"""),
Parameter('timeout', kind=int, default=200,
description="""Specifies how long, in seconds, UI automation will wait for results screen to
appear before assuming something went wrong.
"""),
]
aliases = [
Alias('glbench'),
Alias('egypt', use_case='egypt'),
Alias('t-rex', use_case='t-rex'),
Alias('egypt_onscreen', use_case='egypt', type='onscreen'),
Alias('t-rex_onscreen', use_case='t-rex', type='onscreen'),
Alias('egypt_offscreen', use_case='egypt', type='offscreen'),
Alias('t-rex_offscreen', use_case='t-rex', type='offscreen'),
]
def __init__(self, target, **kwargs):
super(Glb, self).__init__(target, **kwargs)
self.gui.uiauto_params['version'] = self.version
if self.use_case is None:
self.use_case = self.supported_usecase_aliases[self.version][0]
if self.use_case.lower() in USE_CASE_MAP:
if self.use_case not in self.supported_usecase_aliases[self.version]:
raise ConfigError('usecases {} is not supported in version {}'.format(self.use_case, self.version))
self.use_case = USE_CASE_MAP[self.use_case.lower()]
self.gui.uiauto_params['use_case'] = self.use_case.replace(' ', '_')
if self.type.lower() in TYPE_MAP:
self.type = TYPE_MAP[self.type.lower()]
self.gui.uiauto_params['usecase_type'] = self.type.replace(' ', '_')
self.gui.uiauto_params['timeout'] = self.run_timeout
self.package_names = [self.packages[self.version]]
def update_output(self, context):
super(Glb, self).update_output(context)
match_count = 0
with open(context.get_artifact_path('logcat')) as fh:
for line in fh:
match = self.regex.search(line)
if match:
metric = match.group(1)
value, units = match.group(2).split()
value = value.replace('*', '')
if metric == 'metric':
metric = 'Frames'
units = 'frames'
metric = metric + '_' + str(match_count // 2)
context.add_metric(metric, value, units)
match_count += 1

View File

@ -0,0 +1,35 @@
apply plugin: 'com.android.application'
def packageName = "com.arm.wa.uiauto.glbenchmark"
android {
compileSdkVersion 25
buildToolsVersion "25.0.3"
defaultConfig {
applicationId "${packageName}"
minSdkVersion 18
targetSdkVersion 25
testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
}
buildTypes {
applicationVariants.all { variant ->
variant.outputs.each { output ->
output.outputFile = file("$project.buildDir/apk/${packageName}.apk")
}
}
}
}
dependencies {
compile fileTree(dir: 'libs', include: ['*.jar'])
compile 'com.android.support.test:runner:0.5'
compile 'com.android.support.test:rules:0.5'
compile 'com.android.support.test.uiautomator:uiautomator-v18:2.1.2'
compile(name: 'uiauto', ext:'aar')
}
repositories {
flatDir {
dirs 'libs'
}
}

View File

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest xmlns:android="http://schemas.android.com/apk/res/android"
package="com.arm.wa.uiauto.glbenchmark"
android:versionCode="1"
android:versionName="1.0">
<instrumentation
android:name="android.support.test.runner.AndroidJUnitRunner"
android:targetPackage="${applicationId}"/>
</manifest>

View File

@ -0,0 +1,167 @@
/* Copyright 2013-2015 ARM Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.arm.wa.uiauto.glbenchmark;
import android.app.Activity;
import android.os.Bundle;
import android.support.test.runner.AndroidJUnit4;
import android.support.test.uiautomator.UiObject;
import android.support.test.uiautomator.UiObjectNotFoundException;
import android.support.test.uiautomator.UiScrollable;
import android.support.test.uiautomator.UiSelector;
import android.util.Log;
import com.arm.wa.uiauto.BaseUiAutomation;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import java.util.concurrent.TimeUnit;
@RunWith(AndroidJUnit4.class)
public class UiAutomation extends BaseUiAutomation {
public static String TAG = "glb";
public static int maxScrolls = 15;
private Bundle parameters;
private String version;
private String useCase;
private String type;
private int testTimeoutSeconds;
@Before
public void initialize() {
parameters = getParams();
version = parameters.getString("version");
useCase = parameters.getString("use_case").replace('_', ' ');
type = parameters.getString("usecase_type").replace('_', ' ');
testTimeoutSeconds = parameters.getInt("timeout");
}
@Test
public void setup() throws Exception {
dismissAndroidVersionPopup();
goToPreformanceTestsMenu();
selectUseCase(version, useCase, type);
}
@Test
public void runWorkload() throws Exception {
hitStart();
waitForResults(version, useCase, testTimeoutSeconds);
}
@Test
public void extractResults() throws Exception {
extractBenchmarkResults();
}
public void goToPreformanceTestsMenu() throws Exception {
UiSelector selector = new UiSelector();
UiObject choosePerfTest = mDevice.findObject(selector.text("Performance Tests")
.className("android.widget.TextView"));
choosePerfTest.clickAndWaitForNewWindow();
}
public void selectUseCase(String version, String useCase, String type) throws Exception {
UiSelector selector = new UiSelector();
UiScrollable testList = new UiScrollable(selector.className("android.widget.ListView"));
UiObject useCaseText = mDevice.findObject(selector.className("android.widget.TextView")
.text(useCase));
if (version.equals("2.7")){
UiObject typeText = useCaseText.getFromParent(selector.className("android.widget.TextView")
.text(type));
int scrolls = 0;
while(!typeText.exists()) {
testList.scrollForward();
scrolls += 1;
if (scrolls >= maxScrolls) {
break;
}
}
typeText.click();
}
else if (version.equals("2.5")){
int scrolls = 0;
while(!useCaseText.exists()) {
testList.scrollForward();
scrolls += 1;
if (scrolls >= maxScrolls) {
break;
}
}
useCaseText.click();
UiObject modeDisableModeButton = null;
if (type.contains("Onscreen")){
modeDisableModeButton = mDevice.findObject(selector.text("Offscreen"));
}
else {
modeDisableModeButton = mDevice.findObject(selector.text("Onscreen"));
}
modeDisableModeButton.click();
}
}
public void hitStart() throws Exception {
UiSelector selector = new UiSelector();
UiObject startButton = mDevice.findObject(selector.text("Start"));
startButton.clickAndWaitForNewWindow();
}
public void waitForResults(String version, String useCase, int timeout) throws Exception {
UiSelector selector = new UiSelector();
UiObject results = null;
if (version.equals("2.7"))
results = mDevice.findObject(selector.text("Results").className("android.widget.TextView"));
else
results = mDevice.findObject(selector.text(useCase).className("android.widget.TextView"));
Log.v(TAG, "Waiting for results screen.");
// On some devices, the results screen sometimes gets "backgrounded" (or
// rather, doesn't seem to come to foreground to begin with). This code
// attemps to deal with that by explicitly bringing glbench to the
// foreground if results screen doesn't appear within testTimeoutSeconds seconds of
// starting GLB.
if (!results.waitForExists(TimeUnit.SECONDS.toMillis(timeout))) {
Log.v(TAG, "Results screen not found. Attempting to bring to foreground.");
String[] commandLine = {"am", "start",
"-a", "android.intent.action.MAIN",
"-c", "android.intent.category.LAUNCHER",
"-n", "com.glbenchmark.glbenchmark27/com.glbenchmark.activities.GLBenchmarkDownloaderActivity"};
Process proc = Runtime.getRuntime().exec(commandLine);
proc.waitFor();
Log.v(TAG, String.format("am start exit value: %d", proc.exitValue()));
if (!results.exists()) {
throw new UiObjectNotFoundException("Could not find results screen.");
}
}
Log.v(TAG, "Results screen found.");
}
public void extractBenchmarkResults() throws Exception {
Log.v(TAG, "Extracting results.");
sleep(2); // wait for the results screen to fully load.
UiSelector selector = new UiSelector();
UiObject fpsText = mDevice.findObject(selector.className("android.widget.TextView")
.textContains("fps"));
UiObject otherText = fpsText.getFromParent(selector.className("android.widget.TextView").index(0));
Log.v(TAG, String.format("GLBenchmark metric: %s", otherText.getText().replace('\n', ' ')));
Log.v(TAG, String.format("GLBenchmark FPS: %s", fpsText.getText().replace('\n', ' ')));
}
}

View File

@ -0,0 +1,23 @@
// Top-level build file where you can add configuration options common to all sub-projects/modules.
buildscript {
repositories {
jcenter()
}
dependencies {
classpath 'com.android.tools.build:gradle:2.3.2'
// NOTE: Do not place your application dependencies here; they belong
// in the individual module build.gradle files
}
}
allprojects {
repositories {
jcenter()
}
}
task clean(type: Delete) {
delete rootProject.buildDir
}

View File

@ -0,0 +1,40 @@
#!/bin/bash
# CD into build dir if possible - allows building from any directory
script_path='.'
if `readlink -f $0 &>/dev/null`; then
script_path=`readlink -f $0 2>/dev/null`
fi
script_dir=`dirname $script_path`
cd $script_dir
# Ensure gradelw exists before starting
if [[ ! -f gradlew ]]; then
echo 'gradlew file not found! Check that you are in the right directory.'
exit 9
fi
# Copy base class library from wa dist
libs_dir=app/libs
base_class=`python -c "import os, wa; print os.path.join(os.path.dirname(wa.__file__), 'framework', 'uiauto', 'uiauto.aar')"`
mkdir -p $libs_dir
cp $base_class $libs_dir
# Build and return appropriate exit code if failed
# gradle build
./gradlew clean :app:assembleDebug
exit_code=$?
if [[ $exit_code -ne 0 ]]; then
echo "ERROR: 'gradle build' exited with code $exit_code"
exit $exit_code
fi
# If successful move APK file to workload folder (overwrite previous)
package=com.arm.wa.uiauto.glbenchmark
rm -f ../$package
if [[ -f app/build/apk/$package.apk ]]; then
cp app/build/apk/$package.apk ../$package.apk
else
echo 'ERROR: UiAutomator apk could not be found!'
exit 9
fi

Binary file not shown.

View File

@ -0,0 +1,6 @@
#Wed May 03 15:42:44 BST 2017
distributionBase=GRADLE_USER_HOME
distributionPath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
zipStorePath=wrapper/dists
distributionUrl=https\://services.gradle.org/distributions/gradle-3.3-all.zip

160
wa/workloads/glbenchmark/uiauto/gradlew vendored Executable file
View File

@ -0,0 +1,160 @@
#!/usr/bin/env bash
##############################################################################
##
## Gradle start up script for UN*X
##
##############################################################################
# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
DEFAULT_JVM_OPTS=""
APP_NAME="Gradle"
APP_BASE_NAME=`basename "$0"`
# Use the maximum available, or set MAX_FD != -1 to use that value.
MAX_FD="maximum"
warn ( ) {
echo "$*"
}
die ( ) {
echo
echo "$*"
echo
exit 1
}
# OS specific support (must be 'true' or 'false').
cygwin=false
msys=false
darwin=false
case "`uname`" in
CYGWIN* )
cygwin=true
;;
Darwin* )
darwin=true
;;
MINGW* )
msys=true
;;
esac
# Attempt to set APP_HOME
# Resolve links: $0 may be a link
PRG="$0"
# Need this for relative symlinks.
while [ -h "$PRG" ] ; do
ls=`ls -ld "$PRG"`
link=`expr "$ls" : '.*-> \(.*\)$'`
if expr "$link" : '/.*' > /dev/null; then
PRG="$link"
else
PRG=`dirname "$PRG"`"/$link"
fi
done
SAVED="`pwd`"
cd "`dirname \"$PRG\"`/" >/dev/null
APP_HOME="`pwd -P`"
cd "$SAVED" >/dev/null
CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar
# Determine the Java command to use to start the JVM.
if [ -n "$JAVA_HOME" ] ; then
if [ -x "$JAVA_HOME/jre/sh/java" ] ; then
# IBM's JDK on AIX uses strange locations for the executables
JAVACMD="$JAVA_HOME/jre/sh/java"
else
JAVACMD="$JAVA_HOME/bin/java"
fi
if [ ! -x "$JAVACMD" ] ; then
die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
else
JAVACMD="java"
which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
Please set the JAVA_HOME variable in your environment to match the
location of your Java installation."
fi
# Increase the maximum file descriptors if we can.
if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then
MAX_FD_LIMIT=`ulimit -H -n`
if [ $? -eq 0 ] ; then
if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then
MAX_FD="$MAX_FD_LIMIT"
fi
ulimit -n $MAX_FD
if [ $? -ne 0 ] ; then
warn "Could not set maximum file descriptor limit: $MAX_FD"
fi
else
warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT"
fi
fi
# For Darwin, add options to specify how the application appears in the dock
if $darwin; then
GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\""
fi
# For Cygwin, switch paths to Windows format before running java
if $cygwin ; then
APP_HOME=`cygpath --path --mixed "$APP_HOME"`
CLASSPATH=`cygpath --path --mixed "$CLASSPATH"`
JAVACMD=`cygpath --unix "$JAVACMD"`
# We build the pattern for arguments to be converted via cygpath
ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null`
SEP=""
for dir in $ROOTDIRSRAW ; do
ROOTDIRS="$ROOTDIRS$SEP$dir"
SEP="|"
done
OURCYGPATTERN="(^($ROOTDIRS))"
# Add a user-defined pattern to the cygpath arguments
if [ "$GRADLE_CYGPATTERN" != "" ] ; then
OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)"
fi
# Now convert the arguments - kludge to limit ourselves to /bin/sh
i=0
for arg in "$@" ; do
CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -`
CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option
if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition
eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"`
else
eval `echo args$i`="\"$arg\""
fi
i=$((i+1))
done
case $i in
(0) set -- ;;
(1) set -- "$args0" ;;
(2) set -- "$args0" "$args1" ;;
(3) set -- "$args0" "$args1" "$args2" ;;
(4) set -- "$args0" "$args1" "$args2" "$args3" ;;
(5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;;
(6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;;
(7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;;
(8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;;
(9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;;
esac
fi
# Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules
function splitJvmOpts() {
JVM_OPTS=("$@")
}
eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS
JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME"
exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@"

View File

@ -0,0 +1,90 @@
@if "%DEBUG%" == "" @echo off
@rem ##########################################################################
@rem
@rem Gradle startup script for Windows
@rem
@rem ##########################################################################
@rem Set local scope for the variables with windows NT shell
if "%OS%"=="Windows_NT" setlocal
@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script.
set DEFAULT_JVM_OPTS=
set DIRNAME=%~dp0
if "%DIRNAME%" == "" set DIRNAME=.
set APP_BASE_NAME=%~n0
set APP_HOME=%DIRNAME%
@rem Find java.exe
if defined JAVA_HOME goto findJavaFromJavaHome
set JAVA_EXE=java.exe
%JAVA_EXE% -version >NUL 2>&1
if "%ERRORLEVEL%" == "0" goto init
echo.
echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH.
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:findJavaFromJavaHome
set JAVA_HOME=%JAVA_HOME:"=%
set JAVA_EXE=%JAVA_HOME%/bin/java.exe
if exist "%JAVA_EXE%" goto init
echo.
echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME%
echo.
echo Please set the JAVA_HOME variable in your environment to match the
echo location of your Java installation.
goto fail
:init
@rem Get command-line arguments, handling Windowz variants
if not "%OS%" == "Windows_NT" goto win9xME_args
if "%@eval[2+2]" == "4" goto 4NT_args
:win9xME_args
@rem Slurp the command line arguments.
set CMD_LINE_ARGS=
set _SKIP=2
:win9xME_args_slurp
if "x%~1" == "x" goto execute
set CMD_LINE_ARGS=%*
goto execute
:4NT_args
@rem Get arguments from the 4NT Shell from JP Software
set CMD_LINE_ARGS=%$
:execute
@rem Setup the command line
set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar
@rem Execute Gradle
"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS%
:end
@rem End local scope for the variables with windows NT shell
if "%ERRORLEVEL%"=="0" goto mainEnd
:fail
rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of
rem the _cmd.exe /c_ return code!
if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1
exit /b 1
:mainEnd
if "%OS%"=="Windows_NT" endlocal
:omega

View File

@ -0,0 +1 @@
include ':app'