Merge pull request #210 from drcef/master

Implemented visual state detection functionality for revent workloads

doc/source/revent.rst

@@ -210,3 +210,100 @@ https://www.kernel.org/doc/Documentation/input/event-codes.txt
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
     |                          Event Value                          |
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+Using state detection with revent
+=================================
+
+State detection can be used to verify that a workload is executing as expected.
+This utility, if enabled, and if state definitions are available for the
+particular workload, takes a screenshot after the setup and the run revent
+sequence, matches the screenshot to a state and compares with the expected
+state. A WorkloadError is raised if an unexpected state is encountered.
+
+To enable state detection, make sure a valid state definition file and
+templates exist for your workload and set the check_states parameter to True.
+
+State definition directory
+--------------------------
+
+State and phase definitions should be placed in a directory of the following
+structure inside the dependencies directory of each workload (along with
+revent files etc):
+
+   dependencies/
+      <workload_name>/
+         state_definitions/
+            definition.yaml
+            templates/
+               <oneTemplate>.png
+               <anotherTemplate>.png
+               ...
+
+definition.yaml file
+--------------------
+
+This defines each state of the workload and lists which templates are expected
+to be found and how many are required to be detected for a conclusive match. It
+also defines the expected state in each workload phase where a state detection
+is run (currently those are setupComplete and runComplete).
+
+Templates are picture elements to be matched in a screenshot. Each template
+mentioned in the definition file should be placed as a file with the same name
+and a .png extension inside the templates folder. Creating template png files
+is as simple as taking a screenshot of the workload in a given state, cropping
+out the relevant templates (eg. a button, label or other unique element that is
+present in that state) and storing them in PNG format.
+
+Please see the definition file for Angry Birds below as an example to
+understand the format. Note that more than just two states (for the afterSetup
+and afterRun phase) can be defined and this helps track the cause of errors in
+case an unexpected state is encountered.
+
+.. code-block:: python
+
+    workload_name: angrybirds
+
+    workload_states:
+      - state_name: titleScreen
+        templates:
+          - play_button
+          - logo
+        matches: 2
+      - state_name: worldSelection
+        templates:
+          - first_world_thumb
+          - second_world_thumb
+          - third_world_thumb
+          - fourth_world_thumb
+        matches: 3
+      - state_name: level_selection
+        templates:
+          - locked_level
+          - first_level
+        matches: 2
+      - state_name: gameplay
+        templates:
+          - pause_button
+          - score_label_text
+        matches: 2
+      - state_name: pause_screen
+        templates:
+          - replay_button
+          - menu_button
+          - resume_button
+          - help_button
+        matches: 4
+      - state_name: level_cleared_screen
+        templates:
+          - level_cleared_text
+          - menu_button
+          - replay_button
+          - fast_forward_button
+        matches: 4
+
+    workload_phases:
+      - phase_name: setup_complete
+        expected_state: gameplay
+      - phase_name: run_complete
+        expected_state: level_cleared_screen

setup.py

@@ -80,6 +80,7 @@ params = dict(
     ],
     extras_require={
         'other': ['jinja2', 'pandas>=0.13.1'],
+        'statedetect': ['numpy', 'imutils', 'cv2'],
         'test': ['nose'],
         'mongodb': ['pymongo'],
         'notify': ['notify2'],

wlauto/common/android/workload.py

@@ -27,6 +27,8 @@ from wlauto.exceptions import WorkloadError, ResourceError, ConfigError, DeviceE
 from wlauto.utils.android import ApkInfo, ANDROID_NORMAL_PERMISSIONS
 from wlauto.utils.types import boolean
 from wlauto.utils.revent import ReventParser
+from wlauto import File
+import wlauto.utils.statedetect as state_detector
 import wlauto.common.android.resources
 
 
@@ -399,6 +401,24 @@ class ReventWorkload(Workload):
         self.device.push_file(self.revent_run_file, self.on_device_run_revent)
         self.device.push_file(self.revent_setup_file, self.on_device_setup_revent)
 
+    def _check_statedetection_files(self, context):
+        try:
+            self.statedefs_dir = context.resolver.get(File(self, 'state_definitions'))
+        except ResourceError:
+            self.logger.warning("State definitions directory not found. Disabling state detection.")
+            self.check_states = False
+
+    def check_state(self, context, phase):
+        try:
+            self.logger.info("\tChecking workload state...")
+            screenshotPath = os.path.join(context.output_directory, "screen.png")
+            self.device.capture_screen(screenshotPath)
+            stateCheck = state_detector.verify_state(screenshotPath, self.statedefs_dir, phase)
+            if not stateCheck:
+                raise WorkloadError("Unexpected state after setup")
+        except state_detector.StateDefinitionError as e:
+            msg = "State definitions or template files missing or invalid ({}). Skipping state detection."
+            self.logger.warning(msg.format(e.message))
 
 class AndroidUiAutoBenchmark(UiAutomatorWorkload, AndroidBenchmark):
 
@@ -460,6 +480,9 @@ class GameWorkload(ApkWorkload, ReventWorkload):
 
     parameters = [
         Parameter('install_timeout', default=500, override=True),
+        Parameter('check_states', kind=bool, default=False, global_alias='check_game_states',
+                  description="""Use visual state detection to verify the state of the workload
+                  after setup and run"""),
         Parameter('assets_push_timeout', kind=int, default=500,
                   description='Timeout used during deployment of the assets package (if there is one).'),
         Parameter('clear_data_on_reset', kind=bool, default=True,
@@ -479,6 +502,8 @@ class GameWorkload(ApkWorkload, ReventWorkload):
     def init_resources(self, context):
         ApkWorkload.init_resources(self, context)
         ReventWorkload.init_resources(self, context)
+        if self.check_states:        
+            self._check_statedetection_files(self, context)
 
     def setup(self, context):
         ApkWorkload.setup(self, context)
@@ -486,6 +511,10 @@ class GameWorkload(ApkWorkload, ReventWorkload):
         time.sleep(self.loading_time)
         ReventWorkload.setup(self, context)
 
+        # state detection check if it's enabled in the config
+        if self.check_states:
+            self.check_state(self, context, "setup_complete")
+
     def do_post_install(self, context):
         ApkWorkload.do_post_install(self, context)
         self._deploy_assets(context, self.assets_push_timeout)
@@ -505,6 +534,10 @@ class GameWorkload(ApkWorkload, ReventWorkload):
         ReventWorkload.run(self, context)
 
     def teardown(self, context):
+        # state detection check if it's enabled in the config
+        if self.check_states:
+            self.check_state(self, context, "run_complete")
+
         if not self.saved_state_file:
             ApkWorkload.teardown(self, context)
         else:

wlauto/utils/statedetect.py

@@ -0,0 +1,136 @@
+#    Copyright 2013-2016 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+"""
+State detection functionality for revent workloads. Uses OpenCV to analyse screenshots from the device.
+Requires a 'statedetection' directory in the workload directory that includes the state definition yaml file,
+and the 'templates' folder with PNGs of all templates mentioned in the yaml file.
+
+Requires the following Python libraries:
+numpy, pyyaml (yaml), imutils and opencv (cv2)
+
+"""
+
+import os
+
+import yaml
+try:
+    import numpy as np
+except ImportError:
+    np = None
+try:
+    import cv2
+except ImportError:
+    cv2 = None
+try:
+    import imutils
+except ImportError:
+    imutils = None
+
+
+class StateDefinitionError(RuntimeError):
+    def __init__(self, arg):
+        self.args = arg
+
+
+def auto_canny(image, sigma=0.33):
+    # compute the median of the single channel pixel intensities
+    v = np.median(image)
+
+    # apply automatic Canny edge detection using the computed median
+    lower = int(max(0, (1.0 - sigma) * v))
+    upper = int(min(255, (1.0 + sigma) * v))
+    edged = cv2.Canny(image, lower, upper)
+
+    # return the edged image
+    return edged
+
+
+def match_state(screenshot_file, defpath, state_definitions):
+    # check dependencies
+    if np == None or cv2 == None or imutils == None:
+        raise RuntimeError("State detection requires numpy, opencv (cv2) and imutils.")
+
+    # check if file exists, then load screenshot into opencv and create edge map
+    if not os.path.isfile(screenshot_file):
+        raise StateDefinitionError("Screenshot file not found")
+    img_rgb = cv2.imread(screenshot_file)
+    img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY)
+    img_edge = auto_canny(img_gray)
+
+    # make a list of all templates defined in the state definitions
+    template_list = []
+    for state in state_definitions["workload_states"]:
+        template_list.extend(state["templates"])
+
+    # check all template PNGs exist
+    for template_png in template_list:
+        if not os.path.isfile(os.path.join(defpath, 'templates', template_png+'.png')):
+            raise StateDefinitionError("Missing template PNG file: " + template_png + ".png")
+
+    # try to match each PNG
+    matched_templates = []
+    for template_png in template_list:
+        template = cv2.imread(os.path.join(defpath, 'templates', template_png+'.png'), 0)
+        template_edge = auto_canny(template)
+        w, h = template.shape[::-1]
+
+        res = cv2.matchTemplate(img_edge, template_edge, cv2.TM_CCOEFF_NORMED)
+        threshold = 0.5
+        loc = np.where(res >= threshold)
+        zipped = zip(*loc[::-1])
+
+        if len(zipped) > 0:
+            matched_templates.append(template_png)
+
+    # determine the state according to the matched templates
+    matched_state = "none"
+    for state in state_definitions["workload_states"]:
+        # look in the matched templates list for each template of this state
+        match_count = 0
+        for template in state["templates"]:
+            if template in matched_templates:
+                match_count += 1
+
+        if match_count >= state["matches"]:
+            # we have a match
+            matched_state = state["state_name"]
+            break
+
+    return matched_state
+
+
+def verify_state(screenshot_file, state_defs_path, workload_phase):
+    # load and parse state definition file
+    statedefs_file = os.path.join(state_defs_path, 'definition.yaml')
+    if not os.path.isfile(statedefs_file):
+        raise StateDefinitionError("Missing state definitions yaml file: "+statedefs_file)
+    with open(statedefs_file) as fh:
+        state_definitions = yaml.load(fh)
+
+    # run a match on the screenshot
+    matched_state = match_state(screenshot_file, state_defs_path, state_definitions)
+
+    # find what the expected state is for the given workload phase
+    expected_state = None
+    for phase in state_definitions["workload_phases"]:
+        if phase["phase_name"] == workload_phase:
+            expected_state = phase["expected_state"]
+
+    if expected_state is None:
+        raise StateDefinitionError("Phase not defined")
+
+    return expected_state == matched_state