sascha/workload-automation
1
0
Fork 0
mirror of https://github.com/ARM-software/workload-automation.git synced 2025-01-18 20:11:20 +00:00
Code Issues Releases Wiki Activity

workloads/gfxbench: Rework score detection

Browse Source 
Rework how the result matching is performed. Some tests from
gfxbench provide more than 1 score per test and
some provide their output in a different format to others.
Update the matching to perform more flexible matching as well
as dealing with entries that do not fit on a single results screen.
This commit is contained in:
Marc Bonnici 2020-06-01 16:12:26 +01:00
parent 586d95a4f0
commit 443358f513
3 changed files with 63 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
wa/workloads/gfxbench/__init__.py
View File

@ -23,7 +23,7 @@ class Gfxbench(ApkUiautoWorkload):
name = 'gfxbench-corporate'
package_names = ['net.kishonti.gfxbench.gl.v50000.corporate']
clear_data_on_reset = False
score_regex = re.compile(r'.*?([\d.]+).*')
regex_template = 'name: \((?P<test_name>.*)\).*result: \((?P<result>.*)?\).* sub_result:.*\((?P<sub_result>.*?)?\).*'
description = '''
Execute a subset of graphical performance benchmarks
@ -58,26 +58,43 @@ class Gfxbench(ApkUiautoWorkload):
self.gui.timeout = self.timeout
self.gui.uiauto_params['tests'] = self.tests
# pylint: disable=too-many-locals
def update_output(self, context):
super(Gfxbench, self).update_output(context)
expected_results = len(self.tests)
regex_matches = [re.compile('{} score (.+)'.format(t)) for t in self.tests]
regex_matches = [re.compile(self.regex_template.format(t)) for t in self.tests]
logcat_file = context.get_artifact_path('logcat')
found = []
detected_results = 0
failed = False
with open(logcat_file, errors='replace') as fh:
for line in fh:
for regex in regex_matches:
match = regex.search(line)
# Check if we have matched the score string in logcat
if match:
score_match = self.score_regex.search(match.group(1))
# Check if there is valid number found for the score.
if score_match:
result = float(score_match.group(1))
else:
result = 'NaN'
entry = regex.pattern.rsplit(None, 1)[0]
context.add_metric(entry, result, 'FPS', lower_is_better=False)
expected_results -= 1
if expected_results > 0:
msg = "The GFXBench workload has failed. Expected {} scores, Detected {} scores."
raise WorkloadError(msg.format(len(self.regex_matches), expected_results))
# Check if we have matched the score string in logcat and not already found.
if match and match.group('test_name') not in found:
found.append(match.group('test_name'))
# Set Default values
result = 'NaN'
unit = 'FPS'
# For most tests we usually want the `sub_result`
# as this is our FPS value
try:
result = float(match.group('sub_result').split()[0].replace(',', ''))
except (ValueError, TypeError):
# However for some tests the value is stored in `result`
# and the unit is saved in the `sub_result`.
try:
result = float(match.group('result').replace(',', ''))
if match.group('sub_result'):
unit = match.group('sub_result').upper()
except (ValueError, TypeError):
failed = True
entry = match.group('test_name')
context.add_metric(entry, result, unit, lower_is_better=False)
detected_results += 1
if failed or detected_results < len(regex_matches):
msg = "The workload has failed to process all scores. Expected >={} scores, Detected {} scores."
raise WorkloadError(msg.format(len(regex_matches), detected_results))

BIN
wa/workloads/gfxbench/com.arm.wa.uiauto.gfxbench.apk
View File

Binary file not shown.

55
wa/workloads/gfxbench/uiauto/app/src/main/java/com/arm/wa/uiauto/gfxbench/UiAutomation.java
View File

@ -142,15 +142,36 @@ public class UiAutomation extends BaseUiAutomation {
}
public void getScores() throws Exception {
UiScrollable list = new UiScrollable(new UiSelector().scrollable(true));
UiObject results =
mDevice.findObject(new UiSelector().resourceId("net.kishonti.gfxbench.gl.v50000.corporate:id/results_testList"));
for (String test : testList) {
getTestScore(list, results, test);
// To ensure we print all scores, some will be printed multiple times but these are filtered on the python side.
UiScrollable scrollable = new UiScrollable(new UiSelector().scrollable(true));
// Start at the bottom of the list as this seems more reliable when extracting results.
scrollable.flingToEnd(10);
Boolean top_of_list = false;
while(true) {
UiObject resultsList =
mDevice.findObject(new UiSelector().resourceId("net.kishonti.gfxbench.gl.v50000.corporate:id/results_testList"));
// Find the element in the list that contains our test and pull result and sub_result
for (int i=1; i < resultsList.getChildCount(); i++) {
UiObject testName = resultsList.getChild(new UiSelector().index(i))
.getChild(new UiSelector().resourceId("net.kishonti.gfxbench.gl.v50000.corporate:id/updated_result_item_name"));
UiObject result = resultsList.getChild(new UiSelector()
.index(i))
.getChild(new UiSelector()
.resourceId("net.kishonti.gfxbench.gl.v50000.corporate:id/updated_result_item_result"));
UiObject subResult = resultsList.getChild(new UiSelector()
.index(i))
.getChild(new UiSelector()
.resourceId("net.kishonti.gfxbench.gl.v50000.corporate:id/updated_result_item_subresult"));
if (testName.waitForExists(500) && result.waitForExists(500) && subResult.waitForExists(500)) {
Log.d(TAG, "name: (" + testName.getText() + ") result: (" + result.getText() + ") sub_result: (" + subResult.getText() + ")");
}
}
// Ensure we loop over the first screen an extra time once the top of the list has been reached.
if (top_of_list){
break;
}
top_of_list = !scrollable.scrollBackward(100);
}
}
public void toggleTest(String testname) throws Exception {
@ -163,22 +184,4 @@ public class UiAutomation extends BaseUiAutomation {
}
test.click();
}
public void getTestScore(UiScrollable scrollable, UiObject resultsList, String test) throws Exception {
if (test.equals("Tessellation")) {
scrollable.scrollToEnd(1000);
}
for (int i=1; i < resultsList.getChildCount(); i++) {
UiObject testname = resultsList.getChild(new UiSelector().index(i))
.getChild(new UiSelector().resourceId("net.kishonti.gfxbench.gl.v50000.corporate:id/updated_result_item_name"));
if (testname.exists() && testname.getText().equals(test)) {
UiObject result = resultsList.getChild(new UiSelector()
.index(i))
.getChild(new UiSelector()
.resourceId("net.kishonti.gfxbench.gl.v50000.corporate:id/updated_result_item_subresult"));
Log.d(TAG, test + " score " + result.getText());
return;
}
}
}
}