1
0
mirror of https://github.com/ARM-software/workload-automation.git synced 2025-10-30 06:34:13 +00:00

Initial commit of open source Workload Automation.

This commit is contained in:
Sergei Trofimov
2015-03-10 13:09:31 +00:00
commit a747ec7e4c
412 changed files with 41401 additions and 0 deletions

View File

@@ -0,0 +1,215 @@
# Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
import os
import logging
from HTMLParser import HTMLParser
from collections import defaultdict, OrderedDict
from wlauto import AndroidUiAutoBenchmark, Parameter
from wlauto.utils.types import list_of_strs, numeric
from wlauto.exceptions import WorkloadError
#pylint: disable=no-member
class Vellamo(AndroidUiAutoBenchmark):
name = 'vellamo'
description = """
Android benchmark designed by Qualcomm.
Vellamo began as a mobile web benchmarking tool that today has expanded
to include three primary chapters. The Browser Chapter evaluates mobile
web browser performance, the Multicore chapter measures the synergy of
multiple CPU cores, and the Metal Chapter measures the CPU subsystem
performance of mobile processors. Through click-and-go test suites,
organized by chapter, Vellamo is designed to evaluate: UX, 3D graphics,
and memory read/write and peak bandwidth performance, and much more!
Note: Vellamo v3.0 fails to run on Juno
"""
package = 'com.quicinc.vellamo'
run_timeout = 15 * 60
benchmark_types = {
'2.0.3': ['html5', 'metal'],
'3.0': ['Browser', 'Metal', 'Multi'],
}
valid_versions = benchmark_types.keys()
summary_metrics = None
parameters = [
Parameter('version', kind=str, allowed_values=valid_versions, default=sorted(benchmark_types, reverse=True)[0],
description=('Specify the version of Vellamo to be run. '
'If not specified, the latest available version will be used.')),
Parameter('benchmarks', kind=list_of_strs, allowed_values=benchmark_types['3.0'], default=benchmark_types['3.0'],
description=('Specify which benchmark sections of Vellamo to be run. Only valid on version 3.0 and newer.'
'\nNOTE: Browser benchmark can be problematic and seem to hang,'
'just wait and it will progress after ~5 minutes')),
Parameter('browser', kind=int, default=1,
description=('Specify which of the installed browsers will be used for the tests. The number refers to '
'the order in which browsers are listed by Vellamo. E.g. ``1`` will select the first browser '
'listed, ``2`` -- the second, etc. Only valid for version ``3.0``.'))
]
def __init__(self, device, **kwargs):
super(Vellamo, self).__init__(device, **kwargs)
if self.version == '2.0.3':
self.activity = 'com.quicinc.vellamo.VellamoActivity'
if self.version == '3.0':
self.activity = 'com.quicinc.vellamo.main.MainActivity'
self.summary_metrics = self.benchmark_types[self.version]
def setup(self, context):
self.uiauto_params['version'] = self.version
self.uiauto_params['browserToUse'] = self.browser
self.uiauto_params['metal'] = 'Metal' in self.benchmarks
self.uiauto_params['browser'] = 'Browser' in self.benchmarks
self.uiauto_params['multicore'] = 'Multi' in self.benchmarks
super(Vellamo, self).setup(context)
def validate(self):
super(Vellamo, self).validate()
if self.version == '2.0.3' or not self.benchmarks or self.benchmarks == []: # pylint: disable=access-member-before-definition
self.benchmarks = self.benchmark_types[self.version] # pylint: disable=attribute-defined-outside-init
else:
for benchmark in self.benchmarks:
if benchmark not in self.benchmark_types[self.version]:
raise WorkloadError('Version {} does not support {} benchmarks'.format(self.version, benchmark))
def update_result(self, context):
super(Vellamo, self).update_result(context)
# Get total scores from logcat
self.non_root_update_result(context)
if not self.device.is_rooted:
return
for test in self.benchmarks: # Get all scores from HTML files
filename = None
if test == "Browser":
result_folder = self.device.path.join(self.device.package_data_directory, self.package, 'files')
for result_file in self.device.listdir(result_folder, as_root=True):
if result_file.startswith("Browser"):
filename = result_file
else:
filename = '{}_results.html'.format(test)
device_file = self.device.path.join(self.device.package_data_directory, self.package, 'files', filename)
host_file = os.path.join(context.output_directory, filename)
self.device.pull_file(device_file, host_file, as_root=True)
with open(host_file) as fh:
parser = VellamoResultParser()
parser.feed(fh.read())
for benchmark in parser.benchmarks:
benchmark.name = benchmark.name.replace(' ', '_')
context.result.add_metric('{}_Total'.format(benchmark.name), benchmark.score)
for name, score in benchmark.metrics.items():
name = name.replace(' ', '_')
context.result.add_metric('{}_{}'.format(benchmark.name, name), score)
context.add_iteration_artifact('vellamo_output', kind='raw', path=filename)
def non_root_update_result(self, context):
failed = []
with open(self.logcat_log) as logcat:
metrics = OrderedDict()
for line in logcat:
if 'VELLAMO RESULT:' in line:
info = line.split(':')
parts = info[2].split(" ")
metric = parts[1].strip()
value = int(parts[2].strip())
metrics[metric] = value
if 'VELLAMO ERROR:' in line:
self.logger.warning("Browser crashed during benchmark, results may not be accurate")
for key, value in metrics.iteritems():
key = key.replace(' ', '_')
context.result.add_metric(key, value)
if value == 0:
failed.append(key)
if failed:
raise WorkloadError("The following benchmark groups failed: {}".format(", ".join(failed)))
class VellamoResult(object):
def __init__(self, name):
self.name = name
self.score = None
self.metrics = {}
def add_metric(self, data):
split_data = data.split(":")
name = split_data[0].strip()
score = split_data[1].strip()
if name in self.metrics:
raise KeyError("A metric of that name is already present")
self.metrics[name] = float(score)
class VellamoResultParser(HTMLParser):
class StopParsingException(Exception):
pass
def __init__(self):
HTMLParser.__init__(self)
self.inside_div = False
self.inside_span = 0
self.inside_li = False
self.got_data = False
self.failed = False
self.benchmarks = []
def feed(self, text):
try:
HTMLParser.feed(self, text)
except self.StopParsingException:
pass
def handle_starttag(self, tag, attrs):
if tag == 'div':
self.inside_div = True
if tag == 'span':
self.inside_span += 1
if tag == 'li':
self.inside_li = True
def handle_endtag(self, tag):
if tag == 'div':
self.inside_div = False
self.inside_span = 0
self.got_data = False
self.failed = False
if tag == 'li':
self.inside_li = False
def handle_data(self, data):
if self.inside_div and not self.failed:
if "Problem" in data:
self.failed = True
elif self.inside_span == 1:
self.benchmarks.append(VellamoResult(data))
elif self.inside_span == 3 and not self.got_data:
self.benchmarks[-1].score = int(data)
self.got_data = True
elif self.inside_li and self.got_data:
if 'failed' not in data:
self.benchmarks[-1].add_metric(data)
else:
self.failed = True

View File

@@ -0,0 +1,28 @@
#!/bin/bash
# Copyright 2014-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
class_dir=bin/classes/com/arm/wlauto/uiauto
base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"`
mkdir -p $class_dir
cp $base_class $class_dir
ant build
if [[ -f bin/com.arm.wlauto.uiauto.vellamo.jar ]]; then
cp bin/com.arm.wlauto.uiauto.vellamo.jar ..
fi

View File

@@ -0,0 +1,92 @@
<?xml version="1.0" encoding="UTF-8"?>
<project name="com.arm.wlauto.uiauto.vellamo" default="help">
<!-- The local.properties file is created and updated by the 'android' tool.
It contains the path to the SDK. It should *NOT* be checked into
Version Control Systems. -->
<property file="local.properties" />
<!-- The ant.properties file can be created by you. It is only edited by the
'android' tool to add properties to it.
This is the place to change some Ant specific build properties.
Here are some properties you may want to change/update:
source.dir
The name of the source directory. Default is 'src'.
out.dir
The name of the output directory. Default is 'bin'.
For other overridable properties, look at the beginning of the rules
files in the SDK, at tools/ant/build.xml
Properties related to the SDK location or the project target should
be updated using the 'android' tool with the 'update' action.
This file is an integral part of the build system for your
application and should be checked into Version Control Systems.
-->
<property file="ant.properties" />
<!-- if sdk.dir was not set from one of the property file, then
get it from the ANDROID_HOME env var.
This must be done before we load project.properties since
the proguard config can use sdk.dir -->
<property environment="env" />
<condition property="sdk.dir" value="${env.ANDROID_HOME}">
<isset property="env.ANDROID_HOME" />
</condition>
<!-- The project.properties file is created and updated by the 'android'
tool, as well as ADT.
This contains project specific properties such as project target, and library
dependencies. Lower level build properties are stored in ant.properties
(or in .classpath for Eclipse projects).
This file is an integral part of the build system for your
application and should be checked into Version Control Systems. -->
<loadproperties srcFile="project.properties" />
<!-- quick check on sdk.dir -->
<fail
message="sdk.dir is missing. Make sure to generate local.properties using 'android update project' or to inject it through the ANDROID_HOME environment variable."
unless="sdk.dir"
/>
<!--
Import per project custom build rules if present at the root of the project.
This is the place to put custom intermediary targets such as:
-pre-build
-pre-compile
-post-compile (This is typically used for code obfuscation.
Compiled code location: ${out.classes.absolute.dir}
If this is not done in place, override ${out.dex.input.absolute.dir})
-post-package
-post-build
-pre-clean
-->
<import file="custom_rules.xml" optional="true" />
<!-- Import the actual build file.
To customize existing targets, there are two options:
- Customize only one target:
- copy/paste the target into this file, *before* the
<import> task.
- customize it to your needs.
- Customize the whole content of build.xml
- copy/paste the content of the rules files (minus the top node)
into this file, replacing the <import> task.
- customize to your needs.
***********************
****** IMPORTANT ******
***********************
In all cases you must update the value of version-tag below to read 'custom' instead of an integer,
in order to avoid having your file be overridden by tools such as "android update project"
-->
<!-- version-tag: VERSION_TAG -->
<import file="${sdk.dir}/tools/ant/uibuild.xml" />
</project>

View File

@@ -0,0 +1,14 @@
# This file is automatically generated by Android Tools.
# Do not modify this file -- YOUR CHANGES WILL BE ERASED!
#
# This file must be checked in Version Control Systems.
#
# To customize properties used by the Ant build system edit
# "ant.properties", and override values to adapt the script to your
# project structure.
#
# To enable ProGuard to shrink and obfuscate your code, uncomment this (available properties: sdk.dir, user.home):
#proguard.config=${sdk.dir}/tools/proguard/proguard-android.txt:proguard-project.txt
# Project target.
target=android-18

View File

@@ -0,0 +1,260 @@
/* Copyright 2014-2015 ARM Limited
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.arm.wlauto.uiauto.vellamo;
import android.app.Activity;
import android.os.Bundle;
import android.util.Log;
import android.view.KeyEvent;
import java.util.concurrent.TimeUnit;
import java.util.ArrayList;
// Import the uiautomator libraries
import com.android.uiautomator.core.UiObject;
import com.android.uiautomator.core.UiObjectNotFoundException;
import com.android.uiautomator.core.UiScrollable;
import com.android.uiautomator.core.UiSelector;
import com.android.uiautomator.core.UiDevice;
import com.android.uiautomator.core.UiWatcher;
import com.android.uiautomator.testrunner.UiAutomatorTestCase;
import com.arm.wlauto.uiauto.BaseUiAutomation;
public class UiAutomation extends BaseUiAutomation {
public static String TAG = "vellamo";
public static ArrayList<String> scores = new ArrayList();
public static Boolean wasError = false;
public void runUiAutomation() throws Exception {
Bundle parameters = getParams();
String version = parameters.getString("version");
Boolean browser = Boolean.parseBoolean(parameters.getString("browser"));
Boolean metal = Boolean.parseBoolean(parameters.getString("metal"));
Boolean multicore = Boolean.parseBoolean(parameters.getString("multicore"));
Integer browserToUse = Integer.parseInt(parameters.getString("browserToUse")) - 1;
dismissEULA();
if (version.equals("2.0.3")) {
dissmissWelcomebanner();
startTest();
dismissNetworkConnectionDialogIfNecessary();
dismissExplanationDialogIfNecessary();
waitForTestCompletion(15 * 60, "com.quicinc.vellamo:id/act_ba_results_btn_no");
getScore("html5", "com.quicinc.vellamo:id/act_ba_results_img_0");
getScore("metal", "com.quicinc.vellamo:id/act_ba_results_img_1");
}
else {
dismissLetsRoll();
if (browser) {
startBrowserTest(browserToUse);
proccessTest("Browser");
}
if (multicore) {
startTestV3(1);
proccessTest("Multicore");
}
if (metal) {
startTestV3(2);
proccessTest("Metal");
}
}
for(String result : scores){
Log.v(TAG, String.format("VELLAMO RESULT: %s", result));
}
if (wasError) Log.v("vellamoWatcher", "VELLAMO ERROR: Something crashed while running browser benchmark");
}
public void startTest() throws Exception {
UiSelector selector = new UiSelector();
UiObject runButton = new UiObject(selector.textContains("Run All Chapters"));
if (!runButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
UiObject pager = new UiObject(selector.className("android.support.v4.view.ViewPager"));
pager.swipeLeft(2);
if (!runButton.exists()) {
throw new UiObjectNotFoundException("Could not find \"Run All Chapters\" button.");
}
}
runButton.click();
}
public void startBrowserTest(int browserToUse) throws Exception {
//Ensure chrome is selected as "browser" fails to run the benchmark
UiSelector selector = new UiSelector();
UiObject browserToUseButton = new UiObject(selector.className("android.widget.ImageButton")
.longClickable(true).instance(browserToUse));
UiObject browserButton = new UiObject(selector.className("android.widget.ImageButton")
.longClickable(true).selected(true));
//Disable browsers
while(browserButton.exists()) browserButton.click();
if (browserToUseButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
if (browserToUseButton.exists()) {
browserToUseButton.click();
}
}
//enable a watcher to dismiss browser dialogs
UiWatcher stoppedWorkingDialogWatcher = new UiWatcher() {
@Override
public boolean checkForCondition() {
UiObject stoppedWorkingDialog = new UiObject(new UiSelector().textStartsWith("Unfortunately"));
if(stoppedWorkingDialog.exists()){
wasError = true;
UiObject okButton = new UiObject(new UiSelector().className("android.widget.Button").text("OK"));
try {
okButton.click();
} catch (UiObjectNotFoundException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
return (stoppedWorkingDialog.waitUntilGone(25000));
}
return false;
}
};
// Register watcher
UiDevice.getInstance().registerWatcher("stoppedWorkingDialogWatcher", stoppedWorkingDialogWatcher);
// Run watcher
UiDevice.getInstance().runWatchers();
startTestV3(0);
}
public void startTestV3(int run) throws Exception {
UiSelector selector = new UiSelector();
UiObject thirdRunButton = new UiObject(selector.resourceId("com.quicinc.vellamo:id/card_launcher_run_button").instance(run));
if (!thirdRunButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
if (!thirdRunButton.exists()) {
throw new UiObjectNotFoundException("Could not find three \"Run\" buttons.");
}
}
//Run benchmarks
UiObject runButton = new UiObject(selector.resourceId("com.quicinc.vellamo:id/card_launcher_run_button").instance(run));
if (!runButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
if (!runButton.exists()) {
throw new UiObjectNotFoundException("Could not find correct \"Run\" button.");
}
}
runButton.click();
//Skip tutorial screens
UiObject swipeScreen = new UiObject(selector.textContains("Swipe left to continue"));
if (!swipeScreen.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
if (!swipeScreen.exists()) {
throw new UiObjectNotFoundException("Could not find \"Swipe screen\".");
}
}
sleep(1);
swipeScreen.swipeLeft(2);
sleep(1);
swipeScreen.swipeLeft(2);
}
public void proccessTest(String metric) throws Exception{
waitForTestCompletion(15 * 60, "com.quicinc.vellamo:id/button_no");
//Remove watcher
UiDevice.getInstance().removeWatcher("stoppedWorkingDialogWatcher");
getScore(metric, "com.quicinc.vellamo:id/card_score_score");
getUiDevice().pressBack();
getUiDevice().pressBack();
getUiDevice().pressBack();
}
public void getScore(String metric, String resourceID) throws Exception {
UiSelector selector = new UiSelector();
UiObject score = new UiObject(selector.resourceId(resourceID));
if (!score.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
if (!score.exists()) {
throw new UiObjectNotFoundException("Could not find score on screen.");
}
}
scores.add(metric + " " + score.getText().trim());
}
public void waitForTestCompletion(int timeout, String resourceID) throws Exception {
UiSelector selector = new UiSelector();
UiObject resultsNoButton = new UiObject(selector.resourceId(resourceID));
if (!resultsNoButton.waitForExists(TimeUnit.SECONDS.toMillis(timeout))) {
throw new UiObjectNotFoundException("Did not see results screen.");
}
}
public void dismissEULA() throws Exception {
UiSelector selector = new UiSelector();
waitText("Vellamo EULA");
UiObject acceptButton = new UiObject(selector.text("Accept")
.className("android.widget.Button"));
if (acceptButton.exists()) {
acceptButton.click();
}
}
public void dissmissWelcomebanner() throws Exception {
UiSelector selector = new UiSelector();
UiObject welcomeBanner = new UiObject(selector.textContains("WELCOME"));
if (welcomeBanner.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
UiObject pager = new UiObject(selector.className("android.support.v4.view.ViewPager"));
pager.swipeLeft(2);
pager.swipeLeft(2);
}
}
public void dismissLetsRoll() throws Exception {
UiSelector selector = new UiSelector();
UiObject letsRollButton = new UiObject(selector.className("android.widget.Button")
.textContains("Let's Roll"));
if (!letsRollButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
if (!letsRollButton.exists()) {
throw new UiObjectNotFoundException("Could not find \"Let's Roll\" button.");
}
}
letsRollButton.click();
}
public void dismissNetworkConnectionDialogIfNecessary() throws Exception {
UiSelector selector = new UiSelector();
UiObject dialog = new UiObject(selector.className("android.widget.TextView")
.textContains("No Network Connection"));
if (dialog.exists()) {
UiObject yesButton = new UiObject(selector.className("android.widget.Button")
.text("Yes"));
yesButton.click();
}
}
public void dismissExplanationDialogIfNecessary() throws Exception {
UiSelector selector = new UiSelector();
UiObject dialog = new UiObject(selector.className("android.widget.TextView")
.textContains("Benchmarks Explanation"));
if (dialog.exists()) {
UiObject noButton = new UiObject(selector.className("android.widget.Button")
.text("No"));
noButton.click();
}
}
}