sascha/workload-automation
1
0
Fork 0
mirror of https://github.com/ARM-software/workload-automation.git synced 2025-10-29 22:24:51 +00:00
Code Issues Releases Wiki Activity

Initial commit of open source Workload Automation.

Browse Source
This commit is contained in:
Sergei Trofimov
2015-03-10 13:09:31 +00:00
commit a747ec7e4c
412 changed files with 41401 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

158
wlauto/workloads/glbenchmark/__init__.py Normal file
View File

@@ -0,0 +1,158 @@
# Copyright 2013-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# pylint: disable=E1101,E0203
import re
import os
from wlauto import AndroidUiAutoBenchmark, Parameter, Alias
from wlauto.exceptions import ConfigError
import wlauto.common.android.resources
# These maps provide use-friendly aliases for the most common options.
USE_CASE_MAP = {
'egypt': 'GLBenchmark 2.5 Egypt HD',
'egypt-classic': 'GLBenchmark 2.1 Egypt Classic',
't-rex': 'GLBenchmark 2.7 T-Rex HD',
}
VARIANT_MAP = {
'onscreen': 'C24Z16 Onscreen Auto',
'offscreen': 'C24Z16 Offscreen Auto',
}
class Glb(AndroidUiAutoBenchmark):
name = 'glbenchmark'
description = """
Measures the graphics performance of Android devices by testing
the underlying OpenGL (ES) implementation.
http://gfxbench.com/about-gfxbench.jsp
From the website:
The benchmark includes console-quality high-level 3D animations
(T-Rex HD and Egypt HD) and low-level graphics measurements.
With high vertex count and complex effects such as motion blur, parallax
mapping and particle systems, the engine of GFXBench stresses GPUs in order
provide users a realistic feedback on their device.
"""
activity = 'com.glbenchmark.activities.GLBenchmarkDownloaderActivity'
view = 'com.glbenchmark.glbenchmark27/com.glbenchmark.activities.GLBRender'
packages = {
'2.7.0': 'com.glbenchmark.glbenchmark27',
'2.5.1': 'com.glbenchmark.glbenchmark25',
}
# If usecase is not specified the default usecase is the first supported usecase alias
# for the specified version.
supported_usecase_aliases = {
'2.7.0': ['t-rex', 'egypt'],
'2.5.1': ['egypt-classic', 'egypt'],
}
default_iterations = 1
install_timeout = 500
regex = re.compile(r'GLBenchmark (metric|FPS): (.*)')
parameters = [
Parameter('version', default='2.7.0', allowed_values=['2.7.0', '2.5.1'],
description=('Specifies which version of the benchmark to run (different versions '
'support different use cases).')),
Parameter('use_case', default=None,
description="""Specifies which usecase to run, as listed in the benchmark menu; e.g.
``'GLBenchmark 2.5 Egypt HD'``. For convenience, two aliases are provided
for the most common use cases: ``'egypt'`` and ``'t-rex'``. These could
be use instead of the full use case title. For version ``'2.7.0'`` it defaults
to ``'t-rex'``, for version ``'2.5.1'`` it defaults to ``'egypt-classic'``.
"""),
Parameter('variant', default='onscreen',
description="""Specifies which variant of the use case to run, as listed in the benchmarks
menu (small text underneath the use case name); e.g. ``'C24Z16 Onscreen Auto'``.
For convenience, two aliases are provided for the most common variants:
``'onscreen'`` and ``'offscreen'``. These may be used instead of full variant
names.
"""),
Parameter('times', kind=int, default=1,
description=('Specfies the number of times the benchmark will be run in a "tight '
'loop", i.e. without performaing setup/teardown inbetween.')),
Parameter('timeout', kind=int, default=200,
description="""Specifies how long, in seconds, UI automation will wait for results screen to
appear before assuming something went wrong.
"""),
]
aliases = [
Alias('glbench'),
Alias('egypt', use_case='egypt'),
Alias('t-rex', use_case='t-rex'),
Alias('egypt_onscreen', use_case='egypt', variant='onscreen'),
Alias('t-rex_onscreen', use_case='t-rex', variant='onscreen'),
Alias('egypt_offscreen', use_case='egypt', variant='offscreen'),
Alias('t-rex_offscreen', use_case='t-rex', variant='offscreen'),
]
def __init__(self, device, **kwargs):
super(Glb, self).__init__(device, **kwargs)
self.uiauto_params['version'] = self.version
if self.use_case is None:
self.use_case = self.supported_usecase_aliases[self.version][0]
if self.use_case.lower() in USE_CASE_MAP:
if self.use_case not in self.supported_usecase_aliases[self.version]:
raise ConfigError('usecases {} is not supported in version {}'.format(self.use_case, self.version))
self.use_case = USE_CASE_MAP[self.use_case.lower()]
self.uiauto_params['use_case'] = self.use_case.replace(' ', '_')
if self.variant.lower() in VARIANT_MAP:
self.variant = VARIANT_MAP[self.variant.lower()]
self.uiauto_params['variant'] = self.variant.replace(' ', '_')
self.uiauto_params['iterations'] = self.times
self.run_timeout = 4 * 60 * self.times
self.uiauto_params['timeout'] = self.timeout
self.package = self.packages[self.version]
def init_resources(self, context):
self.apk_file = context.resolver.get(wlauto.common.android.resources.ApkFile(self), version=self.version)
self.uiauto_file = context.resolver.get(wlauto.common.android.resources.JarFile(self))
self.device_uiauto_file = self.device.path.join(self.device.working_directory,
os.path.basename(self.uiauto_file))
if not self.uiauto_package:
self.uiauto_package = os.path.splitext(os.path.basename(self.uiauto_file))[0]
def update_result(self, context):
super(Glb, self).update_result(context)
match_count = 0
with open(self.logcat_log) as fh:
for line in fh:
match = self.regex.search(line)
if match:
metric = match.group(1)
value, units = match.group(2).split()
value = value.replace('*', '')
if metric == 'metric':
metric = 'Frames'
units = 'frames'
metric = metric + '_' + str(match_count // 2)
context.result.add_metric(metric, value, units)
match_count += 1

BIN
wlauto/workloads/glbenchmark/com.arm.wlauto.uiauto.glb.jar Normal file
View File

Binary file not shown.

28
wlauto/workloads/glbenchmark/uiauto/build.sh Executable file
View File

@@ -0,0 +1,28 @@
#!/bin/bash
# Copyright 2013-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class_dir=bin/classes/com/arm/wlauto/uiauto
base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"`
mkdir -p $class_dir
cp $base_class $class_dir
ant build
if [[ -f bin/com.arm.wlauto.uiauto.glb.jar ]]; then
cp bin/com.arm.wlauto.uiauto.glb.jar ..
fi

92
wlauto/workloads/glbenchmark/uiauto/build.xml Normal file
View File

@@ -0,0 +1,92 @@
<?xml version="1.0" encoding="UTF-8"?>
<project name="com.arm.wlauto.uiauto.glb" default="help">
<!-- The local.properties file is created and updated by the 'android' tool.
It contains the path to the SDK. It should *NOT* be checked into
Version Control Systems. -->
<property file="local.properties" />
<!-- The ant.properties file can be created by you. It is only edited by the
'android' tool to add properties to it.
This is the place to change some Ant specific build properties.
Here are some properties you may want to change/update:
source.dir
The name of the source directory. Default is 'src'.
out.dir
The name of the output directory. Default is 'bin'.
For other overridable properties, look at the beginning of the rules
files in the SDK, at tools/ant/build.xml
Properties related to the SDK location or the project target should
be updated using the 'android' tool with the 'update' action.
This file is an integral part of the build system for your
application and should be checked into Version Control Systems.
-->
<property file="ant.properties" />
<!-- if sdk.dir was not set from one of the property file, then
get it from the ANDROID_HOME env var.
This must be done before we load project.properties since
the proguard config can use sdk.dir -->
<property environment="env" />
<condition property="sdk.dir" value="${env.ANDROID_HOME}">
<isset property="env.ANDROID_HOME" />
</condition>
<!-- The project.properties file is created and updated by the 'android'
tool, as well as ADT.
This contains project specific properties such as project target, and library
dependencies. Lower level build properties are stored in ant.properties
(or in .classpath for Eclipse projects).
This file is an integral part of the build system for your
application and should be checked into Version Control Systems. -->
<loadproperties srcFile="project.properties" />
<!-- quick check on sdk.dir -->
<fail
message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable."
unless="sdk.dir"
/>
<!--
Import per project custom build rules if present at the root of the project.
This is the place to put custom intermediary targets such as:
-pre-build
-pre-compile
-post-compile (This is typically used for code obfuscation.
Compiled code location: ${out.classes.absolute.dir}
If this is not done in place, override ${out.dex.input.absolute.dir})
-post-package
-post-build
-pre-clean
-->
<import file="custom_rules.xml" optional="true" />
<!-- Import the actual build file.
To customize existing targets, there are two options:
- Customize only one target:
- copy/paste the target into this file, *before* the
<import> task.
- customize it to your needs.
- Customize the whole content of build.xml
- copy/paste the content of the rules files (minus the top node)
into this file, replacing the <import> task.
- customize to your needs.
***********************
****** IMPORTANT ******
***********************
In all cases you must update the value of version-tag below to read 'custom' instead of an integer,
in order to avoid having your file be overridden by tools such as "android update project"
-->
<!-- version-tag: VERSION_TAG -->
<import file="${sdk.dir}/tools/ant/uibuild.xml" />
</project>

14
wlauto/workloads/glbenchmark/uiauto/project.properties Normal file
View File

@@ -0,0 +1,14 @@
# This file is automatically generated by Android Tools.
# Do not modify this file -- YOUR CHANGES WILL BE ERASED!
#
# This file must be checked in Version Control Systems.
#
# To customize properties used by the Ant build system edit
# "ant.properties", and override values to adapt the script to your
# project structure.
#
# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home):
#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt
# Project target.
target=android-17

164
wlauto/workloads/glbenchmark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java Normal file
View File

@@ -0,0 +1,164 @@
/* Copyright 2013-2015 ARM Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.arm.wlauto.uiauto.glb;
import java.lang.Runtime;
import java.lang.Process;
import java.util.concurrent.TimeUnit;
import android.app.Activity;
import android.os.Bundle;
import android.util.Log;
import android.view.KeyEvent;
import com.android.uiautomator.core.UiObject;
import com.android.uiautomator.core.UiObjectNotFoundException;
import com.android.uiautomator.core.UiScrollable;
import com.android.uiautomator.core.UiSelector;
import com.android.uiautomator.testrunner.UiAutomatorTestCase;
import com.arm.wlauto.uiauto.BaseUiAutomation;
public class UiAutomation extends BaseUiAutomation {
public static String TAG = "glb";
public static int maxScrolls = 15;
public void runUiAutomation() throws Exception {
Bundle parameters = getParams();
String version = parameters.getString("version");
String useCase = parameters.getString("use_case").replace('_', ' ');
String variant = parameters.getString("variant").replace('_', ' ');
int iterations = Integer.parseInt(parameters.getString("iterations"));
int testTimeoutSeconds = Integer.parseInt(parameters.getString("timeout"));
if (iterations < 1)
iterations = 1;
goToPreformanceTestsMenu();
selectUseCase(version, useCase, variant);
hitStart();
waitForResults(version, useCase, testTimeoutSeconds);
extractResults();
iterations -= 1;
while (iterations > 0) {
getUiDevice().pressBack();
goToPreformanceTestsMenu();
hitStart();
waitForResults(version, useCase, testTimeoutSeconds);
extractResults();
iterations -= 1;
}
Bundle status = new Bundle();
getAutomationSupport().sendStatus(Activity.RESULT_OK, status);
}
public void goToPreformanceTestsMenu() throws Exception {
UiSelector selector = new UiSelector();
UiObject choosePerfTest = new UiObject(selector.text("Performance Tests")
.className("android.widget.TextView"));
choosePerfTest.clickAndWaitForNewWindow();
}
public void selectUseCase(String version, String useCase, String variant) throws Exception {
UiSelector selector = new UiSelector();
UiScrollable testList = new UiScrollable(selector.className("android.widget.ListView"));
UiObject useCaseText = new UiObject(selector.className("android.widget.TextView")
.text(useCase)
);
if (version.equals("2.7.0")){
UiObject variantText = useCaseText.getFromParent(selector.className("android.widget.TextView")
.text(variant));
int scrolls = 0;
while(!variantText.exists()) {
testList.scrollForward();
scrolls += 1;
if (scrolls >= maxScrolls) {
break;
}
}
variantText.click();
}
else if (version.equals("2.5.1")){
int scrolls = 0;
while(!useCaseText.exists()) {
testList.scrollForward();
scrolls += 1;
if (scrolls >= maxScrolls) {
break;
}
}
useCaseText.click();
//UiSelector selector = new UiSelector();
UiObject modeDisableModeButton = null;
if (variant.contains("Onscreen"))
modeDisableModeButton = new UiObject(selector.text("Offscreen"));
else
modeDisableModeButton = new UiObject(selector.text("Onscreen"));
modeDisableModeButton.click();
}
}
public void hitStart() throws Exception {
UiSelector selector = new UiSelector();
UiObject startButton = new UiObject(selector.text("Start"));
startButton.clickAndWaitForNewWindow();
}
public void waitForResults(String version, String useCase, int timeout) throws Exception {
UiSelector selector = new UiSelector();
UiObject results = null;
if (version.equals("2.7.0"))
results = new UiObject(selector.text("Results").className("android.widget.TextView"));
else
results = new UiObject(selector.text(useCase).className("android.widget.TextView"));
Log.v(TAG, "Waiting for results screen.");
// On some devices, the results screen sometimes gets "backgrounded" (or
// rather, doesn't seem to come to foreground to begin with). This code
// attemps to deal with that by explicitly bringing glbench to the
// foreground if results screen doesn't appear within testTimeoutSeconds seconds of
// starting GLB.
if (!results.waitForExists(TimeUnit.SECONDS.toMillis(timeout))) {
Log.v(TAG, "Results screen not found. Attempting to bring to foreground.");
String[] commandLine = {"am", "start",
"-a", "android.intent.action.MAIN",
"-c", "android.intent.category.LAUNCHER",
"-n", "com.glbenchmark.glbenchmark27/com.glbenchmark.activities.GLBenchmarkDownloaderActivity"};
Process proc = Runtime.getRuntime().exec(commandLine);
proc.waitFor();
Log.v(TAG, String.format("am start exit value: %d", proc.exitValue()));
if (!results.exists()) {
throw new UiObjectNotFoundException("Could not find results screen.");
}
}
Log.v(TAG, "Results screen found.");
}
public void extractResults() throws Exception {
Log.v(TAG, "Extracting results.");
sleep(2); // wait for the results screen to fully load.
UiSelector selector = new UiSelector();
UiObject fpsText = new UiObject(selector.className("android.widget.TextView")
.textContains("fps")
);
UiObject otherText = fpsText.getFromParent(selector.className("android.widget.TextView").index(0));
Log.v(TAG, String.format("GLBenchmark metric: %s", otherText.getText().replace('\n', ' ')));
Log.v(TAG, String.format("GLBenchmark FPS: %s", fpsText.getText().replace('\n', ' ')));
}
}