diff --git a/doc/source/revent.rst b/doc/source/revent.rst index 39d80b3b..8a0d8adc 100644 --- a/doc/source/revent.rst +++ b/doc/source/revent.rst @@ -210,3 +210,100 @@ https://www.kernel.org/doc/Documentation/input/event-codes.txt +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ | Event Value | +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + +Using state detection with revent +================================= + +State detection can be used to verify that a workload is executing as expected. +This utility, if enabled, and if state definitions are available for the +particular workload, takes a screenshot after the setup and the run revent +sequence, matches the screenshot to a state and compares with the expected +state. A WorkloadError is raised if an unexpected state is encountered. + +To enable state detection, make sure a valid state definition file and +templates exist for your workload and set the check_states parameter to True. + +State definition directory +-------------------------- + +State and phase definitions should be placed in a directory of the following +structure inside the dependencies directory of each workload (along with +revent files etc): + + dependencies/ + / + state_definitions/ + definition.yaml + templates/ + .png + .png + ... + +definition.yaml file +-------------------- + +This defines each state of the workload and lists which templates are expected +to be found and how many are required to be detected for a conclusive match. It +also defines the expected state in each workload phase where a state detection +is run (currently those are setupComplete and runComplete). + +Templates are picture elements to be matched in a screenshot. Each template +mentioned in the definition file should be placed as a file with the same name +and a .png extension inside the templates folder. Creating template png files +is as simple as taking a screenshot of the workload in a given state, cropping +out the relevant templates (eg. a button, label or other unique element that is +present in that state) and storing them in PNG format. + +Please see the definition file for Angry Birds below as an example to +understand the format. Note that more than just two states (for the afterSetup +and afterRun phase) can be defined and this helps track the cause of errors in +case an unexpected state is encountered. + +.. code-block:: python + + workload_name: angrybirds + + workload_states: + - state_name: titleScreen + templates: + - play_button + - logo + matches: 2 + - state_name: worldSelection + templates: + - first_world_thumb + - second_world_thumb + - third_world_thumb + - fourth_world_thumb + matches: 3 + - state_name: level_selection + templates: + - locked_level + - first_level + matches: 2 + - state_name: gameplay + templates: + - pause_button + - score_label_text + matches: 2 + - state_name: pause_screen + templates: + - replay_button + - menu_button + - resume_button + - help_button + matches: 4 + - state_name: level_cleared_screen + templates: + - level_cleared_text + - menu_button + - replay_button + - fast_forward_button + matches: 4 + + workload_phases: + - phase_name: setup_complete + expected_state: gameplay + - phase_name: run_complete + expected_state: level_cleared_screen diff --git a/setup.py b/setup.py index d9638b0c..e28ec7d2 100644 --- a/setup.py +++ b/setup.py @@ -80,6 +80,7 @@ params = dict( ], extras_require={ 'other': ['jinja2', 'pandas>=0.13.1'], + 'statedetect': ['numpy', 'imutils', 'cv2'], 'test': ['nose'], 'mongodb': ['pymongo'], 'notify': ['notify2'], diff --git a/wlauto/common/android/workload.py b/wlauto/common/android/workload.py index b1fdfeaa..71502ebf 100644 --- a/wlauto/common/android/workload.py +++ b/wlauto/common/android/workload.py @@ -27,6 +27,8 @@ from wlauto.exceptions import WorkloadError, ResourceError, ConfigError, DeviceE from wlauto.utils.android import ApkInfo, ANDROID_NORMAL_PERMISSIONS from wlauto.utils.types import boolean from wlauto.utils.revent import ReventParser +from wlauto import File +import wlauto.utils.statedetect as state_detector import wlauto.common.android.resources @@ -399,6 +401,24 @@ class ReventWorkload(Workload): self.device.push_file(self.revent_run_file, self.on_device_run_revent) self.device.push_file(self.revent_setup_file, self.on_device_setup_revent) + def _check_statedetection_files(self, context): + try: + self.statedefs_dir = context.resolver.get(File(self, 'state_definitions')) + except ResourceError: + self.logger.warning("State definitions directory not found. Disabling state detection.") + self.check_states = False + + def check_state(self, context, phase): + try: + self.logger.info("\tChecking workload state...") + screenshotPath = os.path.join(context.output_directory, "screen.png") + self.device.capture_screen(screenshotPath) + stateCheck = state_detector.verify_state(screenshotPath, self.statedefs_dir, phase) + if not stateCheck: + raise WorkloadError("Unexpected state after setup") + except state_detector.StateDefinitionError as e: + msg = "State definitions or template files missing or invalid ({}). Skipping state detection." + self.logger.warning(msg.format(e.message)) class AndroidUiAutoBenchmark(UiAutomatorWorkload, AndroidBenchmark): @@ -460,6 +480,9 @@ class GameWorkload(ApkWorkload, ReventWorkload): parameters = [ Parameter('install_timeout', default=500, override=True), + Parameter('check_states', kind=bool, default=False, global_alias='check_game_states', + description="""Use visual state detection to verify the state of the workload + after setup and run"""), Parameter('assets_push_timeout', kind=int, default=500, description='Timeout used during deployment of the assets package (if there is one).'), Parameter('clear_data_on_reset', kind=bool, default=True, @@ -479,6 +502,8 @@ class GameWorkload(ApkWorkload, ReventWorkload): def init_resources(self, context): ApkWorkload.init_resources(self, context) ReventWorkload.init_resources(self, context) + if self.check_states: + self._check_statedetection_files(self, context) def setup(self, context): ApkWorkload.setup(self, context) @@ -486,6 +511,10 @@ class GameWorkload(ApkWorkload, ReventWorkload): time.sleep(self.loading_time) ReventWorkload.setup(self, context) + # state detection check if it's enabled in the config + if self.check_states: + self.check_state(self, context, "setup_complete") + def do_post_install(self, context): ApkWorkload.do_post_install(self, context) self._deploy_assets(context, self.assets_push_timeout) @@ -505,6 +534,10 @@ class GameWorkload(ApkWorkload, ReventWorkload): ReventWorkload.run(self, context) def teardown(self, context): + # state detection check if it's enabled in the config + if self.check_states: + self.check_state(self, context, "run_complete") + if not self.saved_state_file: ApkWorkload.teardown(self, context) else: diff --git a/wlauto/utils/statedetect.py b/wlauto/utils/statedetect.py new file mode 100755 index 00000000..f5f5e8fc --- /dev/null +++ b/wlauto/utils/statedetect.py @@ -0,0 +1,136 @@ +# Copyright 2013-2016 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +""" +State detection functionality for revent workloads. Uses OpenCV to analyse screenshots from the device. +Requires a 'statedetection' directory in the workload directory that includes the state definition yaml file, +and the 'templates' folder with PNGs of all templates mentioned in the yaml file. + +Requires the following Python libraries: +numpy, pyyaml (yaml), imutils and opencv (cv2) + +""" + +import os + +import yaml +try: + import numpy as np +except ImportError: + np = None +try: + import cv2 +except ImportError: + cv2 = None +try: + import imutils +except ImportError: + imutils = None + + +class StateDefinitionError(RuntimeError): + def __init__(self, arg): + self.args = arg + + +def auto_canny(image, sigma=0.33): + # compute the median of the single channel pixel intensities + v = np.median(image) + + # apply automatic Canny edge detection using the computed median + lower = int(max(0, (1.0 - sigma) * v)) + upper = int(min(255, (1.0 + sigma) * v)) + edged = cv2.Canny(image, lower, upper) + + # return the edged image + return edged + + +def match_state(screenshot_file, defpath, state_definitions): + # check dependencies + if np == None or cv2 == None or imutils == None: + raise RuntimeError("State detection requires numpy, opencv (cv2) and imutils.") + + # check if file exists, then load screenshot into opencv and create edge map + if not os.path.isfile(screenshot_file): + raise StateDefinitionError("Screenshot file not found") + img_rgb = cv2.imread(screenshot_file) + img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY) + img_edge = auto_canny(img_gray) + + # make a list of all templates defined in the state definitions + template_list = [] + for state in state_definitions["workload_states"]: + template_list.extend(state["templates"]) + + # check all template PNGs exist + for template_png in template_list: + if not os.path.isfile(os.path.join(defpath, 'templates', template_png+'.png')): + raise StateDefinitionError("Missing template PNG file: " + template_png + ".png") + + # try to match each PNG + matched_templates = [] + for template_png in template_list: + template = cv2.imread(os.path.join(defpath, 'templates', template_png+'.png'), 0) + template_edge = auto_canny(template) + w, h = template.shape[::-1] + + res = cv2.matchTemplate(img_edge, template_edge, cv2.TM_CCOEFF_NORMED) + threshold = 0.5 + loc = np.where(res >= threshold) + zipped = zip(*loc[::-1]) + + if len(zipped) > 0: + matched_templates.append(template_png) + + # determine the state according to the matched templates + matched_state = "none" + for state in state_definitions["workload_states"]: + # look in the matched templates list for each template of this state + match_count = 0 + for template in state["templates"]: + if template in matched_templates: + match_count += 1 + + if match_count >= state["matches"]: + # we have a match + matched_state = state["state_name"] + break + + return matched_state + + +def verify_state(screenshot_file, state_defs_path, workload_phase): + # load and parse state definition file + statedefs_file = os.path.join(state_defs_path, 'definition.yaml') + if not os.path.isfile(statedefs_file): + raise StateDefinitionError("Missing state definitions yaml file: "+statedefs_file) + with open(statedefs_file) as fh: + state_definitions = yaml.load(fh) + + # run a match on the screenshot + matched_state = match_state(screenshot_file, state_defs_path, state_definitions) + + # find what the expected state is for the given workload phase + expected_state = None + for phase in state_definitions["workload_phases"]: + if phase["phase_name"] == workload_phase: + expected_state = phase["expected_state"] + + if expected_state is None: + raise StateDefinitionError("Phase not defined") + + return expected_state == matched_state