diff --git a/wlauto/common/android/workload.py b/wlauto/common/android/workload.py index a1298e4f..8c1b0067 100644 --- a/wlauto/common/android/workload.py +++ b/wlauto/common/android/workload.py @@ -199,6 +199,7 @@ class ApkWorkload(Workload): self.apk_file = None self.apk_version = None self.logcat_log = None + self.exact_apk_version = None def setup(self, context): Workload.setup(self, context) @@ -227,6 +228,11 @@ class ApkWorkload(Workload): msg = "Could not find APK for '{}' on the host or target device" raise ResourceError(msg.format(self.name)) + if self.exact_apk_version is not None: + if self.exact_apk_version != target_version and self.version != host_version: + msg = "APK version '{}' not found on the host '{}' or target '{}'" + raise ResourceError(msg.format(self.version, host_version, target_version)) + # Ensure the apk is setup on the device if self.force_install: self.force_install_apk(context, host_version) diff --git a/wlauto/workloads/geekbench/__init__.py b/wlauto/workloads/geekbench/__init__.py index 8d484cc2..7f8cc182 100644 --- a/wlauto/workloads/geekbench/__init__.py +++ b/wlauto/workloads/geekbench/__init__.py @@ -59,6 +59,15 @@ class Geekbench(AndroidUiAutoBenchmark): """ summary_metrics = ['score', 'multicore_score'] versions = { + '4.0.1': { + 'package': 'com.primatelabs.geekbench', + 'activity': '.HomeActivity', + }, + # Version 3.4.1 was the final version 3 variant + '3.4.1': { + 'package': 'com.primatelabs.geekbench', + 'activity': '.HomeActivity', + }, '3': { 'package': 'com.primatelabs.geekbench3', 'activity': '.HomeActivity', @@ -92,23 +101,13 @@ class Geekbench(AndroidUiAutoBenchmark): super(Geekbench, self).__init__(device, **kwargs) self.uiauto_params['version'] = self.version self.uiauto_params['times'] = self.times - self.run_timeout = 5 * 60 * self.times - - def initialize(self, context): - if self.version == '3' and not self.device.is_rooted: - raise WorkloadError('Geekbench workload only works on rooted devices.') - - def init_resources(self, context): - self.apk_file = context.resolver.get(wlauto.common.android.resources.ApkFile(self), version=self.version) - self.uiauto_file = context.resolver.get(wlauto.common.android.resources.JarFile(self)) - self.device_uiauto_file = self.device.path.join(self.device.working_directory, - os.path.basename(self.uiauto_file)) - if not self.uiauto_package: - self.uiauto_package = os.path.splitext(os.path.basename(self.uiauto_file))[0] + self.run_timeout = 10 * 60 * self.times + self.exact_apk_version = self.version def update_result(self, context): super(Geekbench, self).update_result(context) - update_method = getattr(self, 'update_result_{}'.format(self.version)) + major_version = versiontuple(self.version)[0] + update_method = getattr(self, 'update_result_{}'.format(major_version)) update_method(context) def validate(self): @@ -143,6 +142,30 @@ class Geekbench(AndroidUiAutoBenchmark): context.result.add_metric(namemify(section['name'] + '_multicore_score', i), section['multicore_score']) + def update_result_4(self, context): + outfile_glob = self.device.path.join(self.device.package_data_directory, self.package, 'files', '*gb4') + on_device_output_files = [f.strip() for f in self.device.execute('ls {}'.format(outfile_glob), + as_root=True).split('\n') if f] + for i, on_device_output_file in enumerate(on_device_output_files): + host_temp_file = tempfile.mktemp() + self.device.pull_file(on_device_output_file, host_temp_file) + host_output_file = os.path.join(context.output_directory, os.path.basename(on_device_output_file)) + with open(host_temp_file) as fh: + data = json.load(fh) + os.remove(host_temp_file) + with open(host_output_file, 'w') as wfh: + json.dump(data, wfh, indent=4) + context.iteration_artifacts.append(Artifact('geekout', path=os.path.basename(on_device_output_file), + kind='data', + description='Geekbench 4 output from device.')) + context.result.add_metric(namemify('score', i), data['score']) + context.result.add_metric(namemify('multicore_score', i), data['multicore_score']) + for section in data['sections']: + context.result.add_metric(namemify(section['name'] + '_score', i), section['score']) + for workloads in section['workloads']: + workload_name = workloads['name'].replace(" ", "-") + context.result.add_metric(namemify(section['name'] + '_' + workload_name + '_score', i), + workloads['score']) class GBWorkload(object): """ @@ -353,3 +376,6 @@ class GBScoreCalculator(object): def namemify(basename, i): return basename + (' {}'.format(i) if i else '') + +def versiontuple(v): + return tuple(map(int, (v.split(".")))) diff --git a/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar b/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar index 5359cc30..69955503 100644 Binary files a/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar and b/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar differ diff --git a/wlauto/workloads/geekbench/uiauto/build.sh b/wlauto/workloads/geekbench/uiauto/build.sh index 7da9f5fe..d0f1ccc7 100755 --- a/wlauto/workloads/geekbench/uiauto/build.sh +++ b/wlauto/workloads/geekbench/uiauto/build.sh @@ -17,9 +17,9 @@ class_dir=bin/classes/com/arm/wlauto/uiauto -base_class=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', 'BaseUiAutomation.class')"` +base_classes=`python -c "import os, wlauto; print os.path.join(os.path.dirname(wlauto.__file__), 'common', 'android', '*.class')"` mkdir -p $class_dir -cp $base_class $class_dir +cp $base_classes $class_dir ant build diff --git a/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java index 968d2abc..8f1bc0b2 100644 --- a/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java +++ b/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -16,8 +16,6 @@ package com.arm.wlauto.uiauto.geekbench; -import java.util.concurrent.TimeUnit; - import android.app.Activity; import android.os.Bundle; import android.util.Log; @@ -30,52 +28,96 @@ import com.android.uiautomator.core.UiScrollable; import com.android.uiautomator.core.UiSelector; import com.android.uiautomator.testrunner.UiAutomatorTestCase; -import com.arm.wlauto.uiauto.BaseUiAutomation; +import com.arm.wlauto.uiauto.UxPerfUiAutomation; -public class UiAutomation extends BaseUiAutomation { +import java.util.concurrent.TimeUnit; + +public class UiAutomation extends UxPerfUiAutomation { public static String TAG = "geekbench"; + public static final int WAIT_TIMEOUT_1SEC = 1000; + public static final long WAIT_TIMEOUT_5MIN = TimeUnit.SECONDS.toMillis(5 * 60); + public static final long WAIT_TIMEOUT_10MIN = TimeUnit.SECONDS.toMillis(10 * 60); public void runUiAutomation() throws Exception { Bundle params = getParams(); - int version = Integer.parseInt(params.getString("version")); + String[] version = params.getString("version").split("\\."); + int majorVersion = Integer.parseInt(version[0]); + int minorVersion = Integer.parseInt(version[1]); int times = Integer.parseInt(params.getString("times")); + dismissEula(); + for (int i = 0; i < times; i++) { - runBenchmarks(); - switch(version) { - case 2: - // In version 2, we scroll through the results WebView to make sure - // all results appear on the screen, which causes them to be dumped into - // logcat by the Linaro hacks. - waitForResultsv2(); - scrollThroughResults(); - break; - case 3: + switch (majorVersion) { + case 2: + // In version 2, we scroll through the results WebView to make sure + // all results appear on the screen, which causes them to be dumped into + // logcat by the Linaro hacks. + runBenchmarks(); + waitForResultsv2(); + scrollThroughResults(); + break; + case 3: + runBenchmarks(); + waitForResultsv3onwards(); + if (minorVersion < 4) { // Attempting to share the results will generate the .gb3 file with // results that can then be pulled from the device. This is not possible // in verison 2 of Geekbench (Share option was added later). - waitForResultsv3(); + // Sharing is not necessary from 3.4.1 onwards as the .gb3 files are always + // created. shareResults(); - break; - } + } + break; + case 4: + runCpuBenchmarks(); + waitForResultsv3onwards(); + break; + default : + throw new RuntimeException("Invalid version of Geekbench requested"); + } - if (i < (times - 1)) { - getUiDevice().pressBack(); - getUiDevice().pressBack(); // twice - } + if (i < (times - 1)) { + getUiDevice().pressBack(); + getUiDevice().pressBack(); // twice + } } Bundle status = new Bundle(); getAutomationSupport().sendStatus(Activity.RESULT_OK, status); } + public void dismissEula() throws Exception { + UiObject acceptButton = + // new UiObject(new UiSelector().textContains("Accept") + new UiObject(new UiSelector().resourceId("android:id/button1") + .className("android.widget.Button")); + if (!acceptButton.waitForExists(WAIT_TIMEOUT_1SEC)) { + throw new UiObjectNotFoundException("Could not find Accept button"); + } + acceptButton.click(); + } + public void runBenchmarks() throws Exception { - UiSelector selector = new UiSelector(); - UiObject runButton = new UiObject(selector.text("Run Benchmarks") - .className("android.widget.Button")); - if (!runButton.exists()) { - getUiDevice().pressBack(); + UiObject runButton = + new UiObject(new UiSelector().textContains("Run Benchmarks") + .className("android.widget.Button")); + if (!runButton.waitForExists(WAIT_TIMEOUT_1SEC)) { + throw new UiObjectNotFoundException("Could not find Run button"); + } + runButton.click(); + } + + public void runCpuBenchmarks() throws Exception { + // The run button is at the bottom of the view and may be off the screen so swipe to be sure + uiDeviceSwipe(Direction.DOWN, 50); + + UiObject runButton = + new UiObject(new UiSelector().resourceId("com.primatelabs.geekbench:id/runCpuBenchmarks") + .className("android.widget.Button")); + if (!runButton.waitForExists(WAIT_TIMEOUT_1SEC)) { + throw new UiObjectNotFoundException("Could not find Run button"); } runButton.click(); } @@ -83,18 +125,21 @@ public class UiAutomation extends BaseUiAutomation { public void waitForResultsv2() throws Exception { UiSelector selector = new UiSelector(); UiObject resultsWebview = new UiObject(selector.className("android.webkit.WebView")); - if (!resultsWebview.waitForExists(TimeUnit.SECONDS.toMillis(200))) { - throw new UiObjectNotFoundException("Did not see Geekbench results screen."); + if (!resultsWebview.waitForExists(WAIT_TIMEOUT_5MIN)) { + throw new UiObjectNotFoundException("Did not see Geekbench results screen."); } } - public void waitForResultsv3() throws Exception { + public void waitForResultsv3onwards() throws Exception { UiSelector selector = new UiSelector(); UiObject runningTextView = new UiObject(selector.text("Running Benchmarks...") .className("android.widget.TextView")); - runningTextView.waitForExists(TimeUnit.SECONDS.toMillis(2)); - if (!runningTextView.waitUntilGone(TimeUnit.SECONDS.toMillis(200))) { - throw new UiObjectNotFoundException("Did not get to Geekbench results screen."); + + if (!runningTextView.waitForExists(WAIT_TIMEOUT_1SEC)) { + throw new UiObjectNotFoundException("Did not get to Running Benchmarks... screen."); + } + if (!runningTextView.waitUntilGone(WAIT_TIMEOUT_10MIN)) { + throw new UiObjectNotFoundException("Did not get to Geekbench results screen."); } } @@ -110,12 +155,12 @@ public class UiAutomation extends BaseUiAutomation { } public void shareResults() throws Exception { - sleep(2); // transition + sleep(2); // transition UiSelector selector = new UiSelector(); getUiDevice().pressMenu(); - UiObject runButton = new UiObject(selector.text("Share") - .className("android.widget.TextView")); - runButton.waitForExists(500); - runButton.click(); + UiObject shareButton = new UiObject(selector.text("Share") + .className("android.widget.TextView")); + shareButton.waitForExists(WAIT_TIMEOUT_1SEC); + shareButton.click(); } }