Merge pull request #168 from ep1cman/vellamo-update

Vellamo update

wlauto/workloads/vellamo/__init__.py

@@ -15,8 +15,12 @@
 
 import os
 import logging
+import json
+import re
+
 from HTMLParser import HTMLParser
 from collections import defaultdict, OrderedDict
+from distutils.version import StrictVersion
 
 from wlauto import AndroidUiAutoBenchmark, Parameter
 from wlauto.utils.types import list_of_strs, numeric
@@ -46,6 +50,7 @@ class Vellamo(AndroidUiAutoBenchmark):
     benchmark_types = {
         '2.0.3': ['html5', 'metal'],
         '3.0': ['Browser', 'Metal', 'Multi'],
+        '3.2.4': ['Browser', 'Metal', 'Multi'],
     }
     valid_versions = benchmark_types.keys()
     summary_metrics = None
@@ -66,11 +71,10 @@ class Vellamo(AndroidUiAutoBenchmark):
 
     def __init__(self, device, **kwargs):
         super(Vellamo, self).__init__(device, **kwargs)
-        if self.version == '2.0.3':
+        if StrictVersion(self.version) >= StrictVersion("3.0.0"):
-            self.activity = 'com.quicinc.vellamo.VellamoActivity'
-        if self.version == '3.0':
             self.activity = 'com.quicinc.vellamo.main.MainActivity'
-        self.summary_metrics = self.benchmark_types[self.version]
+        if StrictVersion(self.version) == StrictVersion('2.0.3'):
+            self.activity = 'com.quicinc.vellamo.VellamoActivity'
 
     def setup(self, context):
         self.uiauto_params['version'] = self.version
@@ -97,7 +101,12 @@ class Vellamo(AndroidUiAutoBenchmark):
 
         if not self.device.is_rooted:
             return
+        elif self.version == '3.0.0':
+            self.update_result_v3(context)
+        elif self.version == '3.2.4':
+            self.update_result_v3_2(context)
 
+    def update_result_v3(self, context):
         for test in self.benchmarks:  # Get all scores from HTML files
             filename = None
             if test == "Browser":
@@ -122,24 +131,36 @@ class Vellamo(AndroidUiAutoBenchmark):
                         context.result.add_metric('{}_{}'.format(benchmark.name, name), score)
             context.add_iteration_artifact('vellamo_output', kind='raw', path=filename)
 
+    def update_result_v3_2(self, context):
+        device_file = self.device.path.join(self.device.package_data_directory,
+                                            self.package,
+                                            'files',
+                                            'chapterscores.json')
+        host_file = os.path.join(context.output_directory, 'vellamo.json')
+        self.device.pull_file(device_file, host_file, as_root=True)
+        context.add_iteration_artifact('vellamo_output', kind='raw', path=host_file)
+        with open(host_file) as results_file:
+            data = json.load(results_file)
+            for chapter in data:
+                for result in chapter['benchmark_results']:
+                    name = result['id']
+                    score = result['score']
+                    context.result.add_metric(name, score)
+
     def non_root_update_result(self, context):
         failed = []
-        with open(self.logcat_log) as logcat:
+        with open(self.logcat_log) as fh:
-            metrics = OrderedDict()
+            iteration_result_regex = re.compile("VELLAMO RESULT: (Browser|Metal|Multicore) (\d+)")
-            for line in logcat:
+            for line in fh:
-                if 'VELLAMO RESULT:' in line:
-                    info = line.split(':')
-                    parts = info[2].split(" ")
-                    metric = parts[1].strip()
-                    value = int(parts[2].strip())
-                    metrics[metric] = value
                 if 'VELLAMO ERROR:' in line:
                     self.logger.warning("Browser crashed during benchmark, results may not be accurate")
-            for key, value in metrics.iteritems():
+                result = iteration_result_regex.findall(line)
-                key = key.replace(' ', '_')
+                if result:
-                context.result.add_metric(key, value)
+                    for (metric, score) in result:
-                if value == 0:
+                        if not score:
-                    failed.append(key)
+                            failed.append(metric)
+                        else:
+                            context.result.add_metric(metric, score)
         if failed:
             raise WorkloadError("The following benchmark groups failed: {}".format(", ".join(failed)))
 

wlauto/workloads/vellamo/uiauto/src/com/arm/wlauto/uiauto/UiAutomation.java

@@ -59,20 +59,22 @@ public class UiAutomation extends BaseUiAutomation {
             getScore("html5", "com.quicinc.vellamo:id/act_ba_results_img_0");
             getScore("metal", "com.quicinc.vellamo:id/act_ba_results_img_1");
         }
-
         else {
             dismissLetsRoll();
+            if (version.equals("3.2.4")) {
+                dismissArrow();
+            }
             if (browser) {
-                startBrowserTest(browserToUse);
+                startBrowserTest(browserToUse, version);
                 proccessTest("Browser");
             }
             if (multicore) {
-                startTestV3(1);
+                startTestV3(1, version);
                 proccessTest("Multicore");
 
             }
             if (metal) {
-                startTestV3(2);
+                startTestV3(2, version);
                 proccessTest("Metal");
             }
         }
@@ -96,7 +98,7 @@ public class UiAutomation extends BaseUiAutomation {
         runButton.click();
     }
 
-    public void startBrowserTest(int browserToUse) throws Exception {
+    public void startBrowserTest(int browserToUse, String version) throws Exception {
         //Ensure chrome is selected as "browser" fails to run the benchmark
         UiSelector selector = new UiSelector();
         UiObject browserToUseButton = new UiObject(selector.className("android.widget.ImageButton")
@@ -136,13 +138,13 @@ public class UiAutomation extends BaseUiAutomation {
         // Run watcher
         UiDevice.getInstance().runWatchers();
 
-        startTestV3(0);
+        startTestV3(0, version);
     }
 
-    public void startTestV3(int run) throws Exception {
+    public void startTestV3(int run, String version) throws Exception {
         UiSelector selector = new UiSelector();
 
-        UiObject thirdRunButton = new UiObject(selector.resourceId("com.quicinc.vellamo:id/card_launcher_run_button").instance(run));
+        UiObject thirdRunButton = new UiObject(selector.resourceId("com.quicinc.vellamo:id/card_launcher_run_button").instance(2));
         if (!thirdRunButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
             if (!thirdRunButton.exists()) {
                 throw new UiObjectNotFoundException("Could not find three \"Run\" buttons.");
@@ -158,17 +160,29 @@ public class UiAutomation extends BaseUiAutomation {
         }
         runButton.click();
 
-        //Skip tutorial screens
+        //Skip tutorial screen
-        UiObject swipeScreen = new UiObject(selector.textContains("Swipe left to continue"));
+        if (version.equals("3.2.4")) {
-        if (!swipeScreen.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
+            UiObject gotItButton = new UiObject(selector.textContains("Got it"));
-            if (!swipeScreen.exists()) {
+            if (!gotItButton.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
-                throw new UiObjectNotFoundException("Could not find \"Swipe screen\".");
+                if (!gotItButton.exists()) {
+                    throw new UiObjectNotFoundException("Could not find correct \"GOT IT\" button.");
+                }
             }
+            gotItButton.click();
+        }
+
+        else {
+            UiObject swipeScreen = new UiObject(selector.textContains("Swipe left to continue"));
+            if (!swipeScreen.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
+                if (!swipeScreen.exists()) {
+                    throw new UiObjectNotFoundException("Could not find \"Swipe screen\".");
+                }
+            }
+            sleep(1);
+            swipeScreen.swipeLeft(2);
+            sleep(1);
+            swipeScreen.swipeLeft(2);
         }
-        sleep(1);
-        swipeScreen.swipeLeft(2);
-        sleep(1);
-        swipeScreen.swipeLeft(2);
 
     }
 
@@ -236,6 +250,17 @@ public class UiAutomation extends BaseUiAutomation {
         letsRollButton.click();
     }
 
+    public void dismissArrow() throws Exception {
+        UiSelector selector = new UiSelector();
+        UiObject cardContainer = new UiObject(selector.resourceId("com.quicinc.vellamo:id/cards_container")) ;
+        if (!cardContainer.waitForExists(TimeUnit.SECONDS.toMillis(5))) {
+            if (!cardContainer.exists()) {
+                throw new UiObjectNotFoundException("Could not find vellamo main screen");
+            }
+        }
+        cardContainer.click();
+    }
+
     public void dismissNetworkConnectionDialogIfNecessary() throws Exception {
         UiSelector selector = new UiSelector();
         UiObject dialog = new UiObject(selector.className("android.widget.TextView")