Merge pull request #226 from ep1cman/version_check_fixes

Version check fixes

dev_scripts/README

@@ -6,6 +6,11 @@ distributed as part of WA releases.
 Scripts
 -------
 
+:check_apk_versions: Compares WA workload versions with the versions listed in APK
+                     if there are any incistency it will highlight these. This 
+                     requires all APK files to be present for workloads with 
+                     versions.
+
 :clean_install: Performs a clean install of WA from source. This will remove any
                 existing WA install (regardless of whether it was made from
                 source or through a tarball with pip).

dev_scripts/check_apk_versions

@@ -0,0 +1,66 @@
+#!/usr/bin/env python
+import os
+from distutils.version import StrictVersion
+
+from wlauto.core.extension_loader import ExtensionLoader
+from wlauto.common.android.workload import ApkWorkload
+from wlauto.utils.android import ApkInfo
+
+el = ExtensionLoader()
+
+
+class fake_config(object):
+    def __init__(self, ext_loader):
+        self.ext_loader = ext_loader
+        self.get_extension = ext_loader.get_extension
+
+
+class fake_device(object):
+    platform = "android"
+
+config = fake_config(el)
+device = fake_device()
+
+if "WA_USER_DIRECTORY" in os.environ:
+    base_path = os.environ["WA_USER_DIRECTORY"]
+else:
+    base_path = "~/.workload_automation/dependencies/"
+
+apk_workloads = [e for e in el.list_workloads()
+                 if issubclass(el.get_extension_class(e.name), ApkWorkload)]
+
+for wl in apk_workloads:
+    # Get versions from workloads
+    workload_versions = []
+    for p in wl.parameters:
+        if p.name == "version" and p.allowed_values:
+            workload_versions = p.allowed_values
+            break
+    else:
+        continue
+
+    dep_path = os.path.join(os.path.expanduser(base_path), wl.name)
+    apks = [apk for apk in os.listdir(dep_path) if apk.endswith(".apk")]
+
+    # Get versions from APK files
+    apk_versions = []
+    for apk in apks:
+        # skip antutu 3d benchmark apk
+        if apk == "com.antutu.benchmark.full-1.apk":
+            continue
+        apk_versions.append(ApkInfo(os.path.join(dep_path, apk)).version_name)
+
+    # Output workload info
+    print "Workload:          {}".format(wl.name)
+    print "Workload Versions: {}".format(sorted(workload_versions, key=StrictVersion))
+    print "APK versions:      {}".format(sorted(apk_versions, key=StrictVersion))
+
+    # Check for bad/missing versions
+    error = False
+    for v in apk_versions:
+        if v not in workload_versions:
+            msg = "APK version '{}' not present in workload list of versions"
+            print msg.format(v)
+            error = True
+    if not error:
+        print "OK"

wlauto/common/android/workload.py

@@ -179,7 +179,7 @@ class ApkWorkload(Workload):
         self.apk_version = None
         self.logcat_log = None
 
-    def initialize(self, context):
+    def setup(self, context):
         # Get APK for the correct version and device ABI
         self.apk_file = context.resolver.get(ApkFile(self, self.device.abi),
                                              version=getattr(self, 'version', None),
@@ -194,7 +194,6 @@ class ApkWorkload(Workload):
             if self.force_install:
                 raise ConfigError('force_install cannot be "True" when check_apk is set to "False".')
 
-    def setup(self, context):
         self.initialize_package(context)
         self.launch_package()
         self.device.execute('am kill-all')  # kill all *background* activities
@@ -525,7 +524,7 @@ class GameWorkload(ApkWorkload, ReventWorkload):
     def init_resources(self, context):
         ApkWorkload.init_resources(self, context)
         ReventWorkload.init_resources(self, context)
-        if self.check_states:        
+        if self.check_states:
             self._check_statedetection_files(self, context)
 
     def setup(self, context):

wlauto/workloads/antutu/__init__.py

@@ -17,6 +17,7 @@ import os
 from collections import defaultdict, OrderedDict
 
 from wlauto import AndroidUiAutoBenchmark, Parameter, File
+from wlauto.exceptions import DeviceError
 from wlauto.utils.android import ApkInfo
 
 
@@ -47,7 +48,7 @@ class Antutu(AndroidUiAutoBenchmark):
     activity = ".ABenchMarkStart"
     summary_metrics = ['score', 'Overall_Score']
 
-    valid_versions = ['3.3.2', '4.0.3', '5.3.0', '6.0.1']
+    valid_versions = ['3.3.2', '4.0.3', '5.3', '6.0.1']
 
     device_prefs_directory = '/data/data/com.antutu.ABenchMark/shared_prefs'
     device_prefs_file = '/'.join([device_prefs_directory, 'com.antutu.ABenchMark_preferences.xml'])
@@ -80,8 +81,12 @@ class Antutu(AndroidUiAutoBenchmark):
             info = ApkInfo(antutu_3d)
             if not context.device.is_installed(info.package):
                 self.device.install_apk(antutu_3d, timeout=120)
-            # Antutu doesnt seem to list this as one of its permissions, but it asks for it.
-            self.device.execute("pm grant com.antutu.ABenchMark android.permission.ACCESS_FINE_LOCATION")
+            if self.device.get_sdk_version() >= 23:
+                # Antutu doesnt seem to list this as one of its permissions, but on some devices it asks for it.
+                try:
+                    self.device.execute("pm grant com.antutu.ABenchMark android.permission.ACCESS_FINE_LOCATION")
+                except DeviceError:
+                    self.logger.debug("failed to grant ACCESS_FINE_LOCATION, continuing")
         super(Antutu, self).setup(context)
 
     def update_result(self, context):

wlauto/workloads/antutu/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java

@@ -66,7 +66,7 @@ public class UiAutomation extends BaseUiAutomation {
             while (true) {
                     if(version.equals("6.0.1"))
                         hitTestButtonVersion5(TestButton6);
-                    else if (version.equals("5.3.0")) {
+                    else if (version.equals("5.3")) {
                         hitTestButton();
                         hitTestButtonVersion5(TestButton5);
                     }
@@ -308,7 +308,7 @@ public class UiAutomation extends BaseUiAutomation {
 
     public void returnToTestScreen(String version) throws Exception {
         getUiDevice().pressBack();
-        if (version.equals("5.3.0"))
+        if (version.equals("5.3"))
         {
             UiSelector selector = new UiSelector();
             UiObject detailsButton = new UiObject(new UiSelector().className("android.widget.Button")

wlauto/workloads/geekbench/__init__.py

@@ -59,11 +59,11 @@ class Geekbench(AndroidUiAutoBenchmark):
     """
     summary_metrics = ['score', 'multicore_score']
     versions = {
-        '3': {
+        '3.0.0': {
             'package': 'com.primatelabs.geekbench3',
             'activity': '.HomeActivity',
         },
-        '2': {
+        '2.2.7': {
             'package': 'ca.primatelabs.geekbench2',
             'activity': '.HomeActivity',
         },
@@ -95,7 +95,7 @@ class Geekbench(AndroidUiAutoBenchmark):
         self.run_timeout = 5 * 60 * self.times
 
     def initialize(self, context):
-        if self.version == '3' and not self.device.is_rooted:
+        if self.version == '3.0.0' and not self.device.is_rooted:
             raise WorkloadError('Geekbench workload only works on rooted devices.')
 
     def init_resources(self, context):
@@ -108,12 +108,14 @@ class Geekbench(AndroidUiAutoBenchmark):
 
     def update_result(self, context):
         super(Geekbench, self).update_result(context)
-        update_method = getattr(self, 'update_result_{}'.format(self.version))
-        update_method(context)
+        if self.version == "2.2.7":
+            self.update_result_2(context)
+        else:
+            self.update_result_3(context)
 
     def validate(self):
-        if (self.times > 1) and (self.version == '2'):
-            raise ConfigError('times parameter is not supported for version 2 of Geekbench.')
+        if (self.times > 1) and (self.version == '2.2.7'):
+            raise ConfigError('times parameter is not supported for version 2.2.7 of Geekbench.')
 
     def update_result_2(self, context):
         score_calculator = GBScoreCalculator()

wlauto/workloads/geekbench/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java

@@ -32,26 +32,26 @@ import com.android.uiautomator.testrunner.UiAutomatorTestCase;
 
 import com.arm.wlauto.uiauto.BaseUiAutomation;
 
-public class UiAutomation extends BaseUiAutomation {   
+public class UiAutomation extends BaseUiAutomation {
 
     public static String TAG = "geekbench";
 
     public void runUiAutomation() throws Exception {
         Bundle params = getParams();
-        int version = Integer.parseInt(params.getString("version"));
+        String version = params.getString("version");
         int times = Integer.parseInt(params.getString("times"));
 
         for (int i = 0; i < times; i++) {
                 runBenchmarks();
-                switch(version) {
-                case 2: 
+                if(version.equals("2.2.7")) {
                         // In version 2, we scroll through the results WebView to make sure
                         // all results appear on the screen, which causes them to be dumped into
                         // logcat by the Linaro hacks.
                         waitForResultsv2();
                         scrollThroughResults();
                         break;
-                case 3: 
+                }
+                else if(version.equals("3.0.0")) {
                         // Attempting to share the results will generate the .gb3 file with
                         // results that can then be pulled from the device. This is not possible
                         // in verison 2 of Geekbench (Share option was added later).

wlauto/workloads/glbenchmark/__init__.py

@@ -57,14 +57,14 @@ class Glb(AndroidUiAutoBenchmark):
     view = 'com.glbenchmark.glbenchmark27/com.glbenchmark.activities.GLBRender'
 
     packages = {
-        '2.7.0': 'com.glbenchmark.glbenchmark27',
-        '2.5.1': 'com.glbenchmark.glbenchmark25',
+        '2.7': 'com.glbenchmark.glbenchmark27',
+        '2.5': 'com.glbenchmark.glbenchmark25',
     }
     # If usecase is not specified the default usecase is the first supported usecase alias
     # for the specified version.
     supported_usecase_aliases = {
-        '2.7.0': ['t-rex', 'egypt'],
-        '2.5.1': ['egypt-classic', 'egypt'],
+        '2.7': ['t-rex', 'egypt'],
+        '2.5': ['egypt-classic', 'egypt'],
     }
 
     default_iterations = 1
@@ -73,15 +73,15 @@ class Glb(AndroidUiAutoBenchmark):
     regex = re.compile(r'GLBenchmark (metric|FPS): (.*)')
 
     parameters = [
-        Parameter('version', default='2.7.0', allowed_values=['2.7.0', '2.5.1'],
+        Parameter('version', default='2.7', allowed_values=['2.7', '2.5'],
                   description=('Specifies which version of the benchmark to run (different versions '
                                'support different use cases).')),
         Parameter('use_case', default=None,
                   description="""Specifies which usecase to run, as listed in the benchmark menu; e.g.
                                  ``'GLBenchmark 2.5 Egypt HD'``. For convenience, two aliases are provided
                                  for the most common use cases: ``'egypt'`` and ``'t-rex'``. These could
-                                 be use instead of the full use case title. For version ``'2.7.0'`` it defaults
-                                 to ``'t-rex'``, for version ``'2.5.1'`` it defaults to ``'egypt-classic'``.
+                                 be use instead of the full use case title. For version ``'2.7'`` it defaults
+                                 to ``'t-rex'``, for version ``'2.5'`` it defaults to ``'egypt-classic'``.
                   """),
         Parameter('variant', default='onscreen',
                   description="""Specifies which variant of the use case to run, as listed in the benchmarks

wlauto/workloads/glbenchmark/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java

@@ -33,7 +33,7 @@ import com.android.uiautomator.testrunner.UiAutomatorTestCase;
 
 import com.arm.wlauto.uiauto.BaseUiAutomation;
 
-public class UiAutomation extends BaseUiAutomation {   
+public class UiAutomation extends BaseUiAutomation {
 
     public static String TAG = "glb";
     public static int maxScrolls = 15;
@@ -63,7 +63,7 @@ public class UiAutomation extends BaseUiAutomation {
                 extractResults();
                 iterations -= 1;
         }
-        
+
         Bundle status = new Bundle();
         getAutomationSupport().sendStatus(Activity.RESULT_OK, status);
     }
@@ -81,9 +81,9 @@ public class UiAutomation extends BaseUiAutomation {
         UiObject useCaseText = new UiObject(selector.className("android.widget.TextView")
                                                     .text(useCase)
                                            );
-        if (version.equals("2.7.0")){
+        if (version.equals("2.7")){
                 UiObject variantText =  useCaseText.getFromParent(selector.className("android.widget.TextView")
-                                                                          .text(variant));    
+                                                                          .text(variant));
                 int scrolls = 0;
                 while(!variantText.exists()) {
                         testList.scrollForward();
@@ -94,7 +94,7 @@ public class UiAutomation extends BaseUiAutomation {
                 }
                 variantText.click();
         }
-        else if (version.equals("2.5.1")){
+        else if (version.equals("2.5")){
                 int scrolls = 0;
                 while(!useCaseText.exists()) {
                         testList.scrollForward();
@@ -123,7 +123,7 @@ public class UiAutomation extends BaseUiAutomation {
     public void waitForResults(String version, String useCase, int timeout) throws Exception {
         UiSelector selector = new UiSelector();
         UiObject results = null;
-        if (version.equals("2.7.0"))
+        if (version.equals("2.7"))
                 results = new UiObject(selector.text("Results").className("android.widget.TextView"));
         else
                 results =  new UiObject(selector.text(useCase).className("android.widget.TextView"));
@@ -135,7 +135,7 @@ public class UiAutomation extends BaseUiAutomation {
 	// starting GLB.
         if (!results.waitForExists(TimeUnit.SECONDS.toMillis(timeout))) {
 		Log.v(TAG, "Results screen not found. Attempting to bring to foreground.");
-		String[] commandLine = {"am", "start", 
+		String[] commandLine = {"am", "start",
 					"-a", "android.intent.action.MAIN",
 					"-c", "android.intent.category.LAUNCHER",
 					"-n", "com.glbenchmark.glbenchmark27/com.glbenchmark.activities.GLBenchmarkDownloaderActivity"};