From beee17f9ffaf9daff7fc9be98be5be2336dd5951 Mon Sep 17 00:00:00 2001 From: George Psimenos Date: Thu, 21 Jul 2016 15:36:17 +0100 Subject: [PATCH] Implemented visual state detection functionality for revent workloads - Added statedetect.py in utils which is a standalone module that contains all the methods needed for state detection - Modified the setup() and run() methods of the GameWorkload class in common/android/workload.py to have a parameter that enables state checks and run the check after setup and run if requested. - Modified angrybirds workload to enable state detection for testing purposes State detection uses the template matching method available in OpenCV to determine the state of the workload by detecting predefined unique elements on a screenshot from the device. --- wlauto/common/android/workload.py | 45 +++++++-- wlauto/utils/statedetect.py | 120 ++++++++++++++++++++++++ wlauto/workloads/angrybirds/__init__.py | 2 +- 3 files changed, 158 insertions(+), 9 deletions(-) mode change 100644 => 100755 wlauto/common/android/workload.py create mode 100755 wlauto/utils/statedetect.py diff --git a/wlauto/common/android/workload.py b/wlauto/common/android/workload.py old mode 100644 new mode 100755 index f784946a..c16cdccb --- a/wlauto/common/android/workload.py +++ b/wlauto/common/android/workload.py @@ -16,7 +16,6 @@ import os import sys import time -from math import ceil from wlauto.core.extension import Parameter from wlauto.core.workload import Workload @@ -26,8 +25,10 @@ from wlauto.common.resources import ExtensionAsset, Executable from wlauto.exceptions import WorkloadError, ResourceError, ConfigError from wlauto.utils.android import ApkInfo, ANDROID_NORMAL_PERMISSIONS from wlauto.utils.types import boolean -from wlauto.utils.revent import ReventParser import wlauto.common.android.resources +from wlauto import File + +import wlauto.utils.statedetect as stateDetector DELAY = 5 @@ -324,13 +325,16 @@ AndroidBenchmark = ApkWorkload # backward compatibility class ReventWorkload(Workload): + default_setup_timeout = 5 * 60 # in seconds + default_run_timeout = 10 * 60 # in seconds + def __init__(self, device, _call_super=True, **kwargs): if _call_super: super(ReventWorkload, self).__init__(device, **kwargs) devpath = self.device.path self.on_device_revent_binary = devpath.join(self.device.binaries_directory, 'revent') - self.setup_timeout = kwargs.get('setup_timeout', None) - self.run_timeout = kwargs.get('run_timeout', None) + self.setup_timeout = kwargs.get('setup_timeout', self.default_setup_timeout) + self.run_timeout = kwargs.get('run_timeout', self.default_run_timeout) self.revent_setup_file = None self.revent_run_file = None self.on_device_setup_revent = None @@ -345,10 +349,6 @@ class ReventWorkload(Workload): self.on_device_run_revent = devpath.join(self.device.working_directory, os.path.split(self.revent_run_file)[-1]) self._check_revent_files(context) - default_setup_timeout = ceil(ReventParser.get_revent_duration(self.revent_setup_file)) + 30 - default_run_timeout = ceil(ReventParser.get_revent_duration(self.revent_run_file)) + 30 - self.setup_timeout = self.setup_timeout or default_setup_timeout - self.run_timeout = self.run_timeout or default_run_timeout def setup(self, context): self.device.killall('revent') @@ -447,9 +447,12 @@ class GameWorkload(ApkWorkload, ReventWorkload): view = 'SurfaceView' loading_time = 10 supported_platforms = ['android'] + check_game_states = None parameters = [ Parameter('install_timeout', default=500, override=True), + Parameter('check_states', kind=bool, default=False, global_alias='check_game_states', + description='Use visual state detection to verify the state of the workload after setup and run'), Parameter('assets_push_timeout', kind=int, default=500, description='Timeout used during deployment of the assets package (if there is one).'), Parameter('clear_data_on_reset', kind=bool, default=True, @@ -476,6 +479,19 @@ class GameWorkload(ApkWorkload, ReventWorkload): time.sleep(self.loading_time) ReventWorkload.setup(self, context) + # state detection check if it's enabled in the config + if self.check_game_states: + try: + self.logger.info("\tChecking workload state...") + statedefs_dir = context.resolver.get(File(self, 'state_definitions')) + self.device.capture_screen(context.output_directory+"/aftersetup.png") + stateCheck = stateDetector.verify_state(context.output_directory+"/aftersetup.png", statedefs_dir, "setupComplete") + if not stateCheck: raise WorkloadError("Unexpected state after setup") + except ResourceError: + self.logger.warning("State definitions directory not found. Skipping state detection.") + except stateDetector.StateDefinitionError, errorMsg: + self.logger.warning("State definitions or template files missing or invalid (" + errorMsg + "). Skipping state detection.") + def do_post_install(self, context): ApkWorkload.do_post_install(self, context) self._deploy_assets(context, self.assets_push_timeout) @@ -494,6 +510,19 @@ class GameWorkload(ApkWorkload, ReventWorkload): def run(self, context): ReventWorkload.run(self, context) + # state detection check if it's enabled in the config + if self.check_game_states: + try: + self.logger.info("\tChecking workload state...") + statedefs_dir = context.resolver.get(File(self, 'state_definitions')) + self.device.capture_screen(context.output_directory+"/afterrun.png") + stateCheck = stateDetector.verify_state(context.output_directory+"/afterrun.png", statedefs_dir, "runComplete") + if not stateCheck: raise WorkloadError("Unexpected state after run") + except ResourceError: + self.logger.warning("State definitions directory not found. Skipping state detection.") + except stateDetector.StateDefinitionError, errorMsg: + self.logger.warning("State definitions or template files missing or invalid (" + errorMsg + "). Skipping state detection.") + def teardown(self, context): if not self.saved_state_file: ApkWorkload.teardown(self, context) diff --git a/wlauto/utils/statedetect.py b/wlauto/utils/statedetect.py new file mode 100755 index 00000000..7923c717 --- /dev/null +++ b/wlauto/utils/statedetect.py @@ -0,0 +1,120 @@ +# Copyright 2013-2016 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + + +""" +State detection functionality for revent workloads. Uses OpenCV to analyse screenshots from the device. +Requires a 'statedetection' directory in the workload directory that includes the state definition yaml file, +and the 'templates' folder with PNGs of all templates mentioned in the yaml file. + +Requires the following python plugins: +numpy, pyyaml (yaml), imutils and opencv (cv2) + +""" + +import cv2 +import numpy as np +import imutils +import yaml +import os + +class StateDefinitionError(RuntimeError): + def __init__(self, arg): + self.args = arg + +def auto_canny(image, sigma=0.33): + # compute the median of the single channel pixel intensities + v = np.median(image) + + # apply automatic Canny edge detection using the computed median + lower = int(max(0, (1.0 - sigma) * v)) + upper = int(min(255, (1.0 + sigma) * v)) + edged = cv2.Canny(image, lower, upper) + + # return the edged image + return edged + +def match_state(screenshotFile, defpath): + # load and parse state definition file + if not os.path.isfile(defpath+'/definition.yaml'): raise StateDefinitionError("Missing state definitions yaml file") + stateDefinitions = yaml.load(file(defpath+'/definition.yaml', 'r')) + + # check if file exists, then load screenshot into opencv and create edge map + if not os.path.isfile(screenshotFile): raise StateDefinitionError("Screenshot file not found") + img_rgb = cv2.imread(screenshotFile) + img_gray = cv2.cvtColor(img_rgb, cv2.COLOR_BGR2GRAY) + img_edge = auto_canny(img_gray) + + # make a list of all templates defined in the state definitions + templateList = [] + for state in stateDefinitions["workloadStates"]: + templateList.extend(state["templates"]) + + # check all template PNGs exist + missingFiles = 0 + for templatePng in templateList: + if not os.path.isfile(defpath+'/templates/'+templatePng+'.png'): + missingFiles += 1 + + if missingFiles: raise StateDefinitionError("Missing template PNG files") + + # try to match each PNG + matchedTemplates = [] + for templatePng in templateList: + template = cv2.imread(defpath+'/templates/'+templatePng+'.png',0) + template_edge = auto_canny(template) + w, h = template.shape[::-1] + + res = cv2.matchTemplate(img_edge,template_edge,cv2.TM_CCOEFF_NORMED) + threshold = 0.5 + loc = np.where(res >= threshold) + zipped = zip(*loc[::-1]) + + if len(zipped) > 0: matchedTemplates.append(templatePng) + + + # determine the state according to the matched templates + matchedState = "none" + for state in stateDefinitions["workloadStates"]: + # look in the matched templates list for each template of this state + matchCount = 0 + for template in state["templates"]: + if template in matchedTemplates: + matchCount += 1 + + if matchCount >= state["matches"]: + # we have a match + matchedState = state["stateName"] + break + + return matchedState + +def verify_state(screenshotFile, stateDefsPath, workloadPhase): + # run a match on the screenshot + matchedState = match_state(screenshotFile, stateDefsPath) + + # load and parse state definition file + if not os.path.isfile(stateDefsPath+'/definition.yaml'): raise StateDefinitionError("Missing state definitions yaml file") + stateDefinitions = yaml.load(file(stateDefsPath+'/definition.yaml', 'r')) + + # find what the expected state is for the given workload phase + expectedState = None + for phase in stateDefinitions["workloadExpectedStates"]: + if phase["phaseName"] == workloadPhase: + expectedState = phase["expectedState"] + + if expectedState is None: raise StateDefinitionError("Phase not defined") + + return expectedState == matchedState diff --git a/wlauto/workloads/angrybirds/__init__.py b/wlauto/workloads/angrybirds/__init__.py index 92ef6828..bb0b35c9 100644 --- a/wlauto/workloads/angrybirds/__init__.py +++ b/wlauto/workloads/angrybirds/__init__.py @@ -27,4 +27,4 @@ class AngryBirds(GameWorkload): """ package = 'com.rovio.angrybirds' activity = 'com.rovio.ka3d.App' - + check_game_states = True