Implemented visual state detection functionality for revent workloads

- Added statedetect.py in utils which is a standalone module that
    contains all the methods needed for state detection

  - Modified the setup() and run() methods of the GameWorkload class
    in common/android/workload.py to have a parameter that enables
    state checks and run the check after setup and run if requested.

State detection uses the template matching method available in
OpenCV to determine the state of the workload by detecting
predefined unique elements on a screenshot from the device.

doc/source/revent.rst

@@ -210,3 +210,100 @@ https://www.kernel.org/doc/Documentation/input/event-codes.txt
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
     |                          Event Value                          |
     +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
+
+
+Using state detection with revent
+=================================
+
+State detection can be used to verify that a workload is executing as expected.
+This utility, if enabled, and if state definitions are available for the
+particular workload, takes a screenshot after the setup and the run revent
+sequence, matches the screenshot to a state and compares with the expected
+state. A WorkloadError is raised if an unexpected state is encountered.
+
+To enable state detection, make sure a valid state definition file and
+templates exist for your workload and set the check_states parameter to True.
+
+State definition directory
+--------------------------
+
+State and phase definitions should be placed in a directory of the following
+structure inside the dependencies directory of each workload (along with
+revent files etc):
+
+   dependencies/
+      <workload_name>/
+         state_definitions/
+            definition.yaml
+            templates/
+               <oneTemplate>.png
+               <anotherTemplate>.png
+               ...
+
+definition.yaml file
+--------------------
+
+This defines each state of the workload and lists which templates are expected
+to be found and how many are required to be detected for a conclusive match. It
+also defines the expected state in each workload phase where a state detection
+is run (currently those are setupComplete and runComplete).
+
+Templates are picture elements to be matched in a screenshot. Each template
+mentioned in the definition file should be placed as a file with the same name
+and a .png extension inside the templates folder. Creating template png files
+is as simple as taking a screenshot of the workload in a given state, cropping
+out the relevant templates (eg. a button, label or other unique element that is
+present in that state) and storing them in PNG format.
+
+Please see the definition file for Angry Birds below as an example to
+understand the format. Note that more than just two states (for the afterSetup
+and afterRun phase) can be defined and this helps track the cause of errors in
+case an unexpected state is encountered.
+
+.. code-block:: python
+
+    workload_name: angrybirds
+
+    workload_states:
+      - state_name: titleScreen
+        templates:
+          - play_button
+          - logo
+        matches: 2
+      - state_name: worldSelection
+        templates:
+          - first_world_thumb
+          - second_world_thumb
+          - third_world_thumb
+          - fourth_world_thumb
+        matches: 3
+      - state_name: level_selection
+        templates:
+          - locked_level
+          - first_level
+        matches: 2
+      - state_name: gameplay
+        templates:
+          - pause_button
+          - score_label_text
+        matches: 2
+      - state_name: pause_screen
+        templates:
+          - replay_button
+          - menu_button
+          - resume_button
+          - help_button
+        matches: 4
+      - state_name: level_cleared_screen
+        templates:
+          - level_cleared_text
+          - menu_button
+          - replay_button
+          - fast_forward_button
+        matches: 4
+
+    workload_phases:
+      - phase_name: setup_complete
+        expected_state: gameplay
+      - phase_name: run_complete
+        expected_state: level_cleared_screen

setup.py

@@ -80,6 +80,7 @@ params = dict(
     ],
     extras_require={
         'other': ['jinja2', 'pandas>=0.13.1'],
+        'statedetect': ['numpy', 'imutils', 'cv2'],
         'test': ['nose'],
         'mongodb': ['pymongo'],
         'notify': ['notify2'],

wlauto/common/android/workload.py

@@ -27,6 +27,8 @@ from wlauto.exceptions import WorkloadError, ResourceError, ConfigError
 from wlauto.utils.android import ApkInfo, ANDROID_NORMAL_PERMISSIONS
 from wlauto.utils.types import boolean
 from wlauto.utils.revent import ReventParser
+from wlauto import File
+import wlauto.utils.statedetect as state_detector
 import wlauto.common.android.resources
 
 
@@ -389,6 +391,24 @@ class ReventWorkload(Workload):
         self.device.push_file(self.revent_run_file, self.on_device_run_revent)
         self.device.push_file(self.revent_setup_file, self.on_device_setup_revent)
 
+    def _check_statedetection_files(self, context):
+        try:
+            self.statedefs_dir = context.resolver.get(File(self, 'state_definitions'))
+        except ResourceError:
+            self.logger.warning("State definitions directory not found. Disabling state detection.")
+            self.check_states = False
+
+    def check_state(self, context, phase):
+        try:
+            self.logger.info("\tChecking workload state...")
+            screenshotPath = os.path.join(context.output_directory, "screen.png")
+            self.device.capture_screen(screenshotPath)
+            stateCheck = state_detector.verify_state(screenshotPath, self.statedefs_dir, phase)
+            if not stateCheck:
+                raise WorkloadError("Unexpected state after setup")
+        except state_detector.StateDefinitionError as e:
+            msg = "State definitions or template files missing or invalid ({}). Skipping state detection."
+            self.logger.warning(msg.format(e.message))
 
 class AndroidUiAutoBenchmark(UiAutomatorWorkload, AndroidBenchmark):
 
@@ -450,6 +470,9 @@ class GameWorkload(ApkWorkload, ReventWorkload):
 
     parameters = [
         Parameter('install_timeout', default=500, override=True),
+        Parameter('check_states', kind=bool, default=False, global_alias='check_game_states',
+                  description="""Use visual state detection to verify the state of the workload
+                  after setup and run"""),
         Parameter('assets_push_timeout', kind=int, default=500,
                   description='Timeout used during deployment of the assets package (if there is one).'),
         Parameter('clear_data_on_reset', kind=bool, default=True,
@@ -469,6 +492,8 @@ class GameWorkload(ApkWorkload, ReventWorkload):
     def init_resources(self, context):
         ApkWorkload.init_resources(self, context)
         ReventWorkload.init_resources(self, context)
+        if self.check_states:        
+            self._check_statedetection_files(self, context)
 
     def setup(self, context):
         ApkWorkload.setup(self, context)
@@ -476,6 +501,10 @@ class GameWorkload(ApkWorkload, ReventWorkload):
         time.sleep(self.loading_time)
         ReventWorkload.setup(self, context)
 
+        # state detection check if it's enabled in the config
+        if self.check_states:
+            self.check_state(self, context, "setup_complete")
+
     def do_post_install(self, context):
         ApkWorkload.do_post_install(self, context)
         self._deploy_assets(context, self.assets_push_timeout)
@@ -495,6 +524,10 @@ class GameWorkload(ApkWorkload, ReventWorkload):
         ReventWorkload.run(self, context)
 
     def teardown(self, context):
+        # state detection check if it's enabled in the config
+        if self.check_states:
+            self.check_state(self, context, "run_complete")
+
         if not self.saved_state_file:
             ApkWorkload.teardown(self, context)
         else:

wlauto/utils/statedetect.py

@@ -0,0 +1,136 @@
+#    Copyright 2013-2016 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+"""
+State detection functionality for revent workloads. Uses OpenCV to analyse screenshots from the device.
+Requires a 'statedetection' directory in the workload directory that includes the state definition yaml file,
+and the 'templates' folder with PNGs of all templates mentioned in the yaml file.
+
+Requires the following Python libraries:
+numpy, pyyaml (yaml), imutils and opencv (cv2)
+
+"""
+
+import os
+
+import yaml
+try:
+    import numpy as np
+except ImportError:
+    np = None
+try:
+    import cv2
+except ImportError:
+    cv2 = None
+try:
+    import imutils
+except ImportError:
+    imutils = None
+
+
+class StateDefinitionError(RuntimeError):
+    def __init__(self, arg):
+        self.args = arg
+
+
+def auto_canny(image, sigma=0.33):
+    # compute the median of the single channel pixel intensities
+    v = np.median(image)
+
+    # apply automatic Canny edge detection using the computed median
+    lower = int(max(0, (1.0 - sigma) * v))
+    upper = int(min(255, (1.0 + sigma) * v))
+    edged = cv2.Canny(image, lower, upper)
+
+    # return the edged image
+    return edged
+
+
+def match_state(screenshot_file, defpath, state_definitions):
+    # check dependencies
+    if np == None or cv2 == None or imutils == None:
+        raise RuntimeError("State detection requires numpy, opencv (cv2) and imutils.")
+
+    # check if file exists, then load screenshot into opencv and create edge map
+    if not os.path.isfile(screenshot_file):
+        raise StateDefinitionError("Screenshot file not found")
+    img_rgb = cv2.imread(screenshot_file)
+    img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY)
+    img_edge = auto_canny(img_gray)
+
+    # make a list of all templates defined in the state definitions
+    template_list = []
+    for state in state_definitions["workload_states"]:
+        template_list.extend(state["templates"])
+
+    # check all template PNGs exist
+    for template_png in template_list:
+        if not os.path.isfile(os.path.join(defpath, 'templates', template_png+'.png')):
+            raise StateDefinitionError("Missing template PNG file: " + template_png + ".png")
+
+    # try to match each PNG
+    matched_templates = []
+    for template_png in template_list:
+        template = cv2.imread(os.path.join(defpath, 'templates', template_png+'.png'), 0)
+        template_edge = auto_canny(template)
+        w, h = template.shape[::-1]
+
+        res = cv2.matchTemplate(img_edge, template_edge, cv2.TM_CCOEFF_NORMED)
+        threshold = 0.5
+        loc = np.where(res >= threshold)
+        zipped = zip(*loc[::-1])
+
+        if len(zipped) > 0:
+            matched_templates.append(template_png)
+
+    # determine the state according to the matched templates
+    matched_state = "none"
+    for state in state_definitions["workload_states"]:
+        # look in the matched templates list for each template of this state
+        match_count = 0
+        for template in state["templates"]:
+            if template in matched_templates:
+                match_count += 1
+
+        if match_count >= state["matches"]:
+            # we have a match
+            matched_state = state["state_name"]
+            break
+
+    return matched_state
+
+
+def verify_state(screenshot_file, state_defs_path, workload_phase):
+    # load and parse state definition file
+    statedefs_file = os.path.join(state_defs_path, 'definition.yaml')
+    if not os.path.isfile(statedefs_file):
+        raise StateDefinitionError("Missing state definitions yaml file: "+statedefs_file)
+    with open(statedefs_file) as fh:
+        state_definitions = yaml.load(fh)
+
+    # run a match on the screenshot
+    matched_state = match_state(screenshot_file, state_defs_path, state_definitions)
+
+    # find what the expected state is for the given workload phase
+    expected_state = None
+    for phase in state_definitions["workload_phases"]:
+        if phase["phase_name"] == workload_phase:
+            expected_state = phase["expected_state"]
+
+    if expected_state is None:
+        raise StateDefinitionError("Phase not defined")
+
+    return expected_state == matched_state