diff --git a/dev_scripts/README b/dev_scripts/README index 6ea0e95c..5332b33f 100644 --- a/dev_scripts/README +++ b/dev_scripts/README @@ -6,6 +6,11 @@ distributed as part of WA releases. Scripts ------- +:check_apk_versions: Compares WA workload versions with the versions listed in APK + if there are any incistency it will highlight these. This + requires all APK files to be present for workloads with + versions. + :clean_install: Performs a clean install of WA from source. This will remove any existing WA install (regardless of whether it was made from source or through a tarball with pip). diff --git a/dev_scripts/check_apk_versions b/dev_scripts/check_apk_versions new file mode 100644 index 00000000..4636caa2 --- /dev/null +++ b/dev_scripts/check_apk_versions @@ -0,0 +1,66 @@ +#!/usr/bin/env python +import os +from distutils.version import StrictVersion + +from wlauto.core.extension_loader import ExtensionLoader +from wlauto.common.android.workload import ApkWorkload +from wlauto.utils.android import ApkInfo + +el = ExtensionLoader() + + +class fake_config(object): + def __init__(self, ext_loader): + self.ext_loader = ext_loader + self.get_extension = ext_loader.get_extension + + +class fake_device(object): + platform = "android" + +config = fake_config(el) +device = fake_device() + +if "WA_USER_DIRECTORY" in os.environ: + base_path = os.environ["WA_USER_DIRECTORY"] +else: + base_path = "~/.workload_automation/dependencies/" + +apk_workloads = [e for e in el.list_workloads() + if issubclass(el.get_extension_class(e.name), ApkWorkload)] + +for wl in apk_workloads: + # Get versions from workloads + workload_versions = [] + for p in wl.parameters: + if p.name == "version" and p.allowed_values: + workload_versions = p.allowed_values + break + else: + continue + + dep_path = os.path.join(os.path.expanduser(base_path), wl.name) + apks = [apk for apk in os.listdir(dep_path) if apk.endswith(".apk")] + + # Get versions from APK files + apk_versions = [] + for apk in apks: + # skip antutu 3d benchmark apk + if apk == "com.antutu.benchmark.full-1.apk": + continue + apk_versions.append(ApkInfo(os.path.join(dep_path, apk)).version_name) + + # Output workload info + print "Workload: {}".format(wl.name) + print "Workload Versions: {}".format(sorted(workload_versions, key=StrictVersion)) + print "APK versions: {}".format(sorted(apk_versions, key=StrictVersion)) + + # Check for bad/missing versions + error = False + for v in apk_versions: + if v not in workload_versions: + msg = "APK version '{}' not present in workload list of versions" + print msg.format(v) + error = True + if not error: + print "OK" diff --git a/wlauto/common/android/workload.py b/wlauto/common/android/workload.py index 21ce3d56..6be34ace 100644 --- a/wlauto/common/android/workload.py +++ b/wlauto/common/android/workload.py @@ -179,7 +179,7 @@ class ApkWorkload(Workload): self.apk_version = None self.logcat_log = None - def initialize(self, context): + def setup(self, context): # Get APK for the correct version and device ABI self.apk_file = context.resolver.get(ApkFile(self, self.device.abi), version=getattr(self, 'version', None), @@ -194,7 +194,6 @@ class ApkWorkload(Workload): if self.force_install: raise ConfigError('force_install cannot be "True" when check_apk is set to "False".') - def setup(self, context): self.initialize_package(context) self.launch_package() self.device.execute('am kill-all') # kill all *background* activities @@ -525,7 +524,7 @@ class GameWorkload(ApkWorkload, ReventWorkload): def init_resources(self, context): ApkWorkload.init_resources(self, context) ReventWorkload.init_resources(self, context) - if self.check_states: + if self.check_states: self._check_statedetection_files(self, context) def setup(self, context): diff --git a/wlauto/workloads/antutu/__init__.py b/wlauto/workloads/antutu/__init__.py index f863148f..13cffa73 100644 --- a/wlauto/workloads/antutu/__init__.py +++ b/wlauto/workloads/antutu/__init__.py @@ -17,6 +17,7 @@ import os from collections import defaultdict, OrderedDict from wlauto import AndroidUiAutoBenchmark, Parameter, File +from wlauto.exceptions import DeviceError from wlauto.utils.android import ApkInfo @@ -47,7 +48,7 @@ class Antutu(AndroidUiAutoBenchmark): activity = ".ABenchMarkStart" summary_metrics = ['score', 'Overall_Score'] - valid_versions = ['3.3.2', '4.0.3', '5.3.0', '6.0.1'] + valid_versions = ['3.3.2', '4.0.3', '5.3', '6.0.1'] device_prefs_directory = '/data/data/com.antutu.ABenchMark/shared_prefs' device_prefs_file = '/'.join([device_prefs_directory, 'com.antutu.ABenchMark_preferences.xml']) @@ -80,8 +81,12 @@ class Antutu(AndroidUiAutoBenchmark): info = ApkInfo(antutu_3d) if not context.device.is_installed(info.package): self.device.install_apk(antutu_3d, timeout=120) - # Antutu doesnt seem to list this as one of its permissions, but it asks for it. - self.device.execute("pm grant com.antutu.ABenchMark android.permission.ACCESS_FINE_LOCATION") + if self.device.get_sdk_version() >= 23: + # Antutu doesnt seem to list this as one of its permissions, but on some devices it asks for it. + try: + self.device.execute("pm grant com.antutu.ABenchMark android.permission.ACCESS_FINE_LOCATION") + except DeviceError: + self.logger.debug("failed to grant ACCESS_FINE_LOCATION, continuing") super(Antutu, self).setup(context) def update_result(self, context): diff --git a/wlauto/workloads/antutu/com.arm.wlauto.uiauto.antutu.jar b/wlauto/workloads/antutu/com.arm.wlauto.uiauto.antutu.jar index 4aecde1d..7c83ec3b 100644 Binary files a/wlauto/workloads/antutu/com.arm.wlauto.uiauto.antutu.jar and b/wlauto/workloads/antutu/com.arm.wlauto.uiauto.antutu.jar differ diff --git a/wlauto/workloads/antutu/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/antutu/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java index 4d7a08d1..fc44e9a3 100644 --- a/wlauto/workloads/antutu/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java +++ b/wlauto/workloads/antutu/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -66,7 +66,7 @@ public class UiAutomation extends BaseUiAutomation { while (true) { if(version.equals("6.0.1")) hitTestButtonVersion5(TestButton6); - else if (version.equals("5.3.0")) { + else if (version.equals("5.3")) { hitTestButton(); hitTestButtonVersion5(TestButton5); } @@ -308,7 +308,7 @@ public class UiAutomation extends BaseUiAutomation { public void returnToTestScreen(String version) throws Exception { getUiDevice().pressBack(); - if (version.equals("5.3.0")) + if (version.equals("5.3")) { UiSelector selector = new UiSelector(); UiObject detailsButton = new UiObject(new UiSelector().className("android.widget.Button") diff --git a/wlauto/workloads/geekbench/__init__.py b/wlauto/workloads/geekbench/__init__.py index 8d484cc2..b7dc7a9f 100644 --- a/wlauto/workloads/geekbench/__init__.py +++ b/wlauto/workloads/geekbench/__init__.py @@ -59,11 +59,11 @@ class Geekbench(AndroidUiAutoBenchmark): """ summary_metrics = ['score', 'multicore_score'] versions = { - '3': { + '3.0.0': { 'package': 'com.primatelabs.geekbench3', 'activity': '.HomeActivity', }, - '2': { + '2.2.7': { 'package': 'ca.primatelabs.geekbench2', 'activity': '.HomeActivity', }, @@ -95,7 +95,7 @@ class Geekbench(AndroidUiAutoBenchmark): self.run_timeout = 5 * 60 * self.times def initialize(self, context): - if self.version == '3' and not self.device.is_rooted: + if self.version == '3.0.0' and not self.device.is_rooted: raise WorkloadError('Geekbench workload only works on rooted devices.') def init_resources(self, context): @@ -108,12 +108,14 @@ class Geekbench(AndroidUiAutoBenchmark): def update_result(self, context): super(Geekbench, self).update_result(context) - update_method = getattr(self, 'update_result_{}'.format(self.version)) - update_method(context) + if self.version == "2.2.7": + self.update_result_2(context) + else: + self.update_result_3(context) def validate(self): - if (self.times > 1) and (self.version == '2'): - raise ConfigError('times parameter is not supported for version 2 of Geekbench.') + if (self.times > 1) and (self.version == '2.2.7'): + raise ConfigError('times parameter is not supported for version 2.2.7 of Geekbench.') def update_result_2(self, context): score_calculator = GBScoreCalculator() diff --git a/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar b/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar index 5359cc30..deec40bd 100644 Binary files a/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar and b/wlauto/workloads/geekbench/com.arm.wlauto.uiauto.geekbench.jar differ diff --git a/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java index 968d2abc..2ef4b9ea 100644 --- a/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java +++ b/wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -32,26 +32,26 @@ import com.android.uiautomator.testrunner.UiAutomatorTestCase; import com.arm.wlauto.uiauto.BaseUiAutomation; -public class UiAutomation extends BaseUiAutomation { +public class UiAutomation extends BaseUiAutomation { public static String TAG = "geekbench"; public void runUiAutomation() throws Exception { Bundle params = getParams(); - int version = Integer.parseInt(params.getString("version")); + String version = params.getString("version"); int times = Integer.parseInt(params.getString("times")); for (int i = 0; i < times; i++) { runBenchmarks(); - switch(version) { - case 2: + if(version.equals("2.2.7")) { // In version 2, we scroll through the results WebView to make sure // all results appear on the screen, which causes them to be dumped into // logcat by the Linaro hacks. waitForResultsv2(); scrollThroughResults(); break; - case 3: + } + else if(version.equals("3.0.0")) { // Attempting to share the results will generate the .gb3 file with // results that can then be pulled from the device. This is not possible // in verison 2 of Geekbench (Share option was added later). diff --git a/wlauto/workloads/glbenchmark/__init__.py b/wlauto/workloads/glbenchmark/__init__.py index d8206cc2..0e54e149 100644 --- a/wlauto/workloads/glbenchmark/__init__.py +++ b/wlauto/workloads/glbenchmark/__init__.py @@ -57,14 +57,14 @@ class Glb(AndroidUiAutoBenchmark): view = 'com.glbenchmark.glbenchmark27/com.glbenchmark.activities.GLBRender' packages = { - '2.7.0': 'com.glbenchmark.glbenchmark27', - '2.5.1': 'com.glbenchmark.glbenchmark25', + '2.7': 'com.glbenchmark.glbenchmark27', + '2.5': 'com.glbenchmark.glbenchmark25', } # If usecase is not specified the default usecase is the first supported usecase alias # for the specified version. supported_usecase_aliases = { - '2.7.0': ['t-rex', 'egypt'], - '2.5.1': ['egypt-classic', 'egypt'], + '2.7': ['t-rex', 'egypt'], + '2.5': ['egypt-classic', 'egypt'], } default_iterations = 1 @@ -73,15 +73,15 @@ class Glb(AndroidUiAutoBenchmark): regex = re.compile(r'GLBenchmark (metric|FPS): (.*)') parameters = [ - Parameter('version', default='2.7.0', allowed_values=['2.7.0', '2.5.1'], + Parameter('version', default='2.7', allowed_values=['2.7', '2.5'], description=('Specifies which version of the benchmark to run (different versions ' 'support different use cases).')), Parameter('use_case', default=None, description="""Specifies which usecase to run, as listed in the benchmark menu; e.g. ``'GLBenchmark 2.5 Egypt HD'``. For convenience, two aliases are provided for the most common use cases: ``'egypt'`` and ``'t-rex'``. These could - be use instead of the full use case title. For version ``'2.7.0'`` it defaults - to ``'t-rex'``, for version ``'2.5.1'`` it defaults to ``'egypt-classic'``. + be use instead of the full use case title. For version ``'2.7'`` it defaults + to ``'t-rex'``, for version ``'2.5'`` it defaults to ``'egypt-classic'``. """), Parameter('variant', default='onscreen', description="""Specifies which variant of the use case to run, as listed in the benchmarks diff --git a/wlauto/workloads/glbenchmark/com.arm.wlauto.uiauto.glb.jar b/wlauto/workloads/glbenchmark/com.arm.wlauto.uiauto.glb.jar index 57d0fb1e..71773c9b 100644 Binary files a/wlauto/workloads/glbenchmark/com.arm.wlauto.uiauto.glb.jar and b/wlauto/workloads/glbenchmark/com.arm.wlauto.uiauto.glb.jar differ diff --git a/wlauto/workloads/glbenchmark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java b/wlauto/workloads/glbenchmark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java index 2c244d64..22d0c0f0 100644 --- a/wlauto/workloads/glbenchmark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java +++ b/wlauto/workloads/glbenchmark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java @@ -33,7 +33,7 @@ import com.android.uiautomator.testrunner.UiAutomatorTestCase; import com.arm.wlauto.uiauto.BaseUiAutomation; -public class UiAutomation extends BaseUiAutomation { +public class UiAutomation extends BaseUiAutomation { public static String TAG = "glb"; public static int maxScrolls = 15; @@ -63,7 +63,7 @@ public class UiAutomation extends BaseUiAutomation { extractResults(); iterations -= 1; } - + Bundle status = new Bundle(); getAutomationSupport().sendStatus(Activity.RESULT_OK, status); } @@ -81,9 +81,9 @@ public class UiAutomation extends BaseUiAutomation { UiObject useCaseText = new UiObject(selector.className("android.widget.TextView") .text(useCase) ); - if (version.equals("2.7.0")){ + if (version.equals("2.7")){ UiObject variantText = useCaseText.getFromParent(selector.className("android.widget.TextView") - .text(variant)); + .text(variant)); int scrolls = 0; while(!variantText.exists()) { testList.scrollForward(); @@ -94,7 +94,7 @@ public class UiAutomation extends BaseUiAutomation { } variantText.click(); } - else if (version.equals("2.5.1")){ + else if (version.equals("2.5")){ int scrolls = 0; while(!useCaseText.exists()) { testList.scrollForward(); @@ -123,7 +123,7 @@ public class UiAutomation extends BaseUiAutomation { public void waitForResults(String version, String useCase, int timeout) throws Exception { UiSelector selector = new UiSelector(); UiObject results = null; - if (version.equals("2.7.0")) + if (version.equals("2.7")) results = new UiObject(selector.text("Results").className("android.widget.TextView")); else results = new UiObject(selector.text(useCase).className("android.widget.TextView")); @@ -135,7 +135,7 @@ public class UiAutomation extends BaseUiAutomation { // starting GLB. if (!results.waitForExists(TimeUnit.SECONDS.toMillis(timeout))) { Log.v(TAG, "Results screen not found. Attempting to bring to foreground."); - String[] commandLine = {"am", "start", + String[] commandLine = {"am", "start", "-a", "android.intent.action.MAIN", "-c", "android.intent.category.LAUNCHER", "-n", "com.glbenchmark.glbenchmark27/com.glbenchmark.activities.GLBenchmarkDownloaderActivity"};