diff --git a/wa/workloads/jankbench/__init__.py b/wa/workloads/jankbench/__init__.py new file mode 100644 index 00000000..22349b84 --- /dev/null +++ b/wa/workloads/jankbench/__init__.py @@ -0,0 +1,234 @@ +# Copyright 2018 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# pylint: disable=E1101,W0201,E0203 + +from __future__ import division +import os +import re +import select +import json +import threading +import sqlite3 +import subprocess +from copy import copy + +import pandas as pd + +from wa import ApkWorkload, Parameter, WorkloadError, ConfigError +from wa.utils.types import list_or_string, numeric + + +DELAY = 2 + + +class Jankbench(ApkWorkload): + + name = 'jankbench' + description = """ + Internal Google benchmark for evaluating jank on Android. + + """ + package_names = ['com.android.benchmark'] + activity = '.app.RunLocalBenchmarksActivity' + + results_db_file = 'BenchmarkResults' + + iteration_regex = re.compile(r'System.out: iteration: (?P[0-9]+)') + metrics_regex = re.compile( + r'System.out: Mean: (?P[0-9\.]+)\s+JankP: (?P[0-9\.]+)\s+' + r'StdDev: (?P[0-9\.]+)\s+Count Bad: (?P[0-9]+)\s+' + r'Count Jank: (?P[0-9]+)' + ) + + valid_test_ids = [ + # Order matters -- the index of the id must match what is expected by + # the App. + 'list_view', + 'image_list_view', + 'shadow_grid', + 'low_hitrate_text', + 'high_hitrate_text', + 'edit_text', + 'overdraw_test', + ] + + parameters = [ + Parameter('test_ids', kind=list_or_string, + allowed_values=valid_test_ids, + description='ID of the jankbench test to be run.'), + Parameter('reps', kind=int, default=1, constraint=lambda x: x > 0, + description=''' + Specifies the number of times the benchmark will be run in a "tight loop", + i.e. without performaing setup/teardown inbetween. + '''), + Parameter('pull_results_db', kind=bool, + description=''' + Secifies whether an sqlite database with detailed results should be pulled + from benchmark app's data. This requires the device to be rooted. + + This defaults to ``True`` for rooted devices and ``False`` otherwise. + '''), + Parameter('run_timeout', kind=int, default=10 * 60, + description=""" + Time out for workload execution. The workload will be killed if it hasn't completed + within this period. + """), + ] + + def setup(self, context): + super(Jankbench, self).setup(context) + + if self.pull_results_db is None: + self.pull_results_db = self.target.is_rooted + elif self.pull_results_db and not self.target.is_rooted: + raise ConfigError('pull_results_db set for an unrooted device') + + self.target.ensure_screen_is_on() + self.command = self._build_command() + self.monitor = JankbenchRunMonitor(self.target) + self.monitor.start() + + def run(self, context): + result = self.target.execute(self.command, timeout=self.run_timeout) + if 'FAILURE' in result: + raise WorkloadError(result) + else: + self.logger.debug(result) + self.target.sleep(DELAY) + self.monitor.wait_for_run_end(self.run_timeout) + + def extract_results(self, context): + self.monitor.stop() + if self.pull_results_db: + target_file = self.target.path.join(self.target.package_data_directory, + self.package, 'databases', self.results_db_file) + host_file = os.path.join(context.output_directory,self.results_db_file) + self.target.pull(target_file, host_file, as_root=True) + context.add_artifact('jankbench-results', host_file, 'data') + + def update_output(self, context): # NOQA + super(Jankbench, self).update_output(context) + if self.pull_results_db: + self.extract_metrics_from_db(context) + else: + self.extract_metrics_from_logcat(context) + + def extract_metrics_from_db(self, context): + dbfile = context.get_artifact_path('jankbench-results') + with sqlite3.connect(dbfile) as conn: + df = pd.read_sql('select name, iteration, total_duration, jank_frame from ui_results', conn) + g = df.groupby(['name', 'iteration']) + janks = g.jank_frame.sum() + janks_pc = janks / g.jank_frame.count() * 100 + results = pd.concat([ + g.total_duration.mean(), + g.total_duration.std(), + janks, + janks_pc, + ], axis=1) + results.columns = ['mean', 'std_dev', 'count_jank', 'jank_p'] + + for test_name, rep in results.index: + test_results = results.ix[test_name, rep] + for metric, value in test_results.iteritems(): + context.add_metric(metric, value, units=None, lower_is_better=True, + classifiers={'test_name': test_name, 'rep': rep}) + + def exract_metrics_from_logcat(self, context): + metric_names = ['mean', 'junk_p', 'std_dev', 'count_bad', 'count_junk'] + logcat_file = context.get_artifact_path('logcat') + with open(logcat_file) as fh: + run_tests = copy(self.test_ids or self.valid_test_ids) + current_iter = None + current_test = None + for line in fh: + + match = self.iteration_regex.search(line) + if match: + if current_iter is not None: + msg = 'Did not see results for iteration {} of {}' + self.logger.warning(msg.format(current_iter, current_test)) + current_iter = int(match.group('iteration')) + if current_iter == 0: + try: + current_test = run_tests.pop(0) + except IndexError: + self.logger.warning('Encountered an iteration for an unknown test.') + current_test = 'unknown' + continue + + match = self.metrics_regex.search(line) + if match: + if current_iter is None: + self.logger.warning('Encountered unexpected metrics (no iteration)') + continue + + for name in metric_names: + value = numeric(match.group(name)) + context.add_metric(name, value, units=None, lower_is_better=True, + classifiers={'test_id': current_test, 'rep': current_iter}) + + current_iter = None + + def _build_command(self): + command_params = [] + if self.test_ids: + test_idxs = [str(self.valid_test_ids.index(i)) for i in self.test_ids] + command_params.append('--eia com.android.benchmark.EXTRA_ENABLED_BENCHMARK_IDS {}'.format(','.join(test_idxs))) + if self.reps: + command_params.append('--ei com.android.benchmark.EXTRA_RUN_COUNT {}'.format(self.reps)) + return 'am start -W -S -n {}/{} {}'.format(self.package, + self.activity, + ' '.join(command_params)) + + +class JankbenchRunMonitor(threading.Thread): + + regex = re.compile(r'I BENCH\s+:\s+BenchmarkDone!') + + def __init__(self, device): + super(JankbenchRunMonitor, self).__init__() + self.target = device + self.daemon = True + self.run_ended = threading.Event() + self.stop_event = threading.Event() + # Not using clear_logcat() because command collects directly, i.e. will + # ignore poller. + self.target.execute('logcat -c') + if self.target.adb_name: + self.command = ['adb', '-s', self.target.adb_name, 'logcat'] + else: + self.command = ['adb', 'logcat'] + + def run(self): + proc = subprocess.Popen(self.command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + while not self.stop_event.is_set(): + if self.run_ended.is_set(): + self.target.sleep(DELAY) + else: + ready, _, _ = select.select([proc.stdout, proc.stderr], [], [], 2) + if ready: + line = ready[0].readline() + if self.regex.search(line): + self.run_ended.set() + + def stop(self): + self.stop_event.set() + self.join() + + def wait_for_run_end(self, timeout): + self.run_ended.wait(timeout) + self.run_ended.clear()