# pylint: disable=attribute-defined-outside-init import os import re import csv import math import shutil from zipfile import is_zipfile, ZipFile from collections import defaultdict from wlauto import Workload, Parameter from wlauto.exceptions import WorkloadError from wlauto.utils.misc import check_output, get_null, get_meansd from wlauto.utils.types import numeric, identifier RESULT_REGEX = re.compile(r'RESULT ([^:]+): ([^=]+)\s*=\s*' # preamble and test/metric name r'(\[([^\]]+)\]|(\S+))' # value r'\s*(\S+)') # units TRACE_REGEX = re.compile(r'Trace saved as ([^\n]+)') class Telemetry(Workload): name = 'telemetry' description = """ Executes Google's Telemetery benchmarking framework (must be installed). Url: https://www.chromium.org/developers/telemetry From the web site: Telemetry is Chrome's performance testing framework. It allows you to perform arbitrary actions on a set of web pages and report metrics about it. The framework abstracts: - Launching a browser with arbitrary flags on any platform. - Opening a tab and navigating to the page under test. - Fetching data via the Inspector timeline and traces. - Using Web Page Replay to cache real-world websites so they don't change when used in benchmarks. Design Principles - Write one performance test that runs on all platforms - Windows, Mac, Linux, Chrome OS, and Android for both Chrome and ContentShell. - Runs on browser binaries, without a full Chromium checkout, and without having to build the browser yourself. - Use WebPageReplay to get repeatable test results. - Clean architecture for writing benchmarks that keeps measurements and use cases separate. - Run on non-Chrome browsers for comparative studies. This instrument runs telemetry via its ``run_benchmarks`` script (which must be in PATH or specified using ``run_benchmarks_path`` parameter) and parses metrics from the resulting output. **device setup** The device setup will depend on whether you're running a test image (in which case little or no setup should be necessary) """ parameters = [ Parameter('run_benchmark_path', default='run_benchmark', description=""" This is the path to run_benchmark script which runs a Telemetry benchmark. If not specified, the assumption will be that it is in path (i.e. with be invoked as ``run_benchmark``). """), Parameter('test', default='page_cycler.top_10_mobile', description=""" Specifies with of the the telemetry tests is to be run. """), Parameter('run_benchmark_params', default='', description=""" Additional paramters to be passed to ``run_benchmarks``. """), Parameter('run_timeout', kind=int, default=900, description=""" Timeout for execution of the test. """), ] summary_metrics = ['cold_times', 'commit_charge', 'cpu_utilization', 'processes', 'resident_set_size_peak_size_browser', 'resident_set_size_peak_size_gpu', 'vm_final_size_browser', 'vm_final_size_gpu', 'vm_final_size_renderer', 'vm_final_size_total', 'vm_peak_size_browser', 'vm_peak_size_gpu', 'vm_private_dirty_final_browser', 'vm_private_dirty_final_gpu', 'vm_private_dirty_final_renderer', 'vm_private_dirty_final_total', 'vm_resident_set_size_final_size_browser', 'vm_resident_set_size_final_size_gpu', 'vm_resident_set_size_final_size_renderer', 'vm_resident_set_size_final_size_total', 'warm_times'] def validate(self): ret = os.system('{} > {} 2>&1'.format(self.run_benchmark_path, get_null())) if ret == 0xff00: # is it supposed to be 0xff? pass # telemetry found and appears to be installed properly. elif ret == 127: raise WorkloadError('run_benchmarks not found (did you specify correct run_benchmarks_path?)') else: raise WorkloadError('Unexected error from run_benchmarks: {}'.format(ret)) def setup(self, context): self.raw_output = None self.command = self.build_command() def run(self, context): self.logger.debug(self.command) self.raw_output, _ = check_output(self.command, shell=True, timeout=self.run_timeout, ignore=1) def update_result(self, context): # pylint: disable=too-many-locals if not self.raw_output: self.logger.warning('Did not get run_benchmark output.') return raw_outfile = os.path.join(context.output_directory, 'telemetry_raw.out') with open(raw_outfile, 'w') as wfh: wfh.write(self.raw_output) context.add_artifact('telemetry-raw', raw_outfile, kind='raw') results, artifacts = parse_telemetry_results(raw_outfile) csv_outfile = os.path.join(context.output_directory, 'telemetry.csv') averages = defaultdict(list) with open(csv_outfile, 'wb') as wfh: writer = csv.writer(wfh) writer.writerow(['kind', 'url', 'iteration', 'value', 'units']) for result in results: name_template = identifier('{}_{}_{{}}'.format(result.url, result.kind)) averages[result.kind].append(result.average) context.result.add_metric(name_template.format('avg'), result.average, result.units, lower_is_better=True) context.result.add_metric(name_template.format('sd'), result.std, result.units, lower_is_better=True) writer.writerows(result.rows) context.add_artifact('telemetry', csv_outfile, kind='data') for kind, values in averages.iteritems(): context.result.add_metric(kind, special_average(values), lower_is_better=True) for idx, artifact in enumerate(artifacts): wa_path = os.path.join(context.output_directory, os.path.basename(artifact)) shutil.copy(artifact, wa_path) context.add_artifact('telemetry_trace_{}'.format(idx), path=wa_path, kind='data') if is_zipfile(wa_path): zf = ZipFile(wa_path) zf.extractall(context.output_directory) zf.close() def teardown(self, context): pass def build_command(self): if self.device.platform == 'chromeos': device_opts = '--remote={} --browser=cros-chrome'.format(self.device.host) else: raise WorkloadError('Currently, telemetry workload supports only ChromeOS devices.') return '{} {} {} {}'.format(self.run_benchmark_path, self.test, device_opts, self.run_benchmark_params) class TelemetryResult(object): @property def average(self): return get_meansd(self.values)[0] @property def std(self): return get_meansd(self.values)[1] @property def rows(self): for i, v in enumerate(self.values): yield [self.kind, self.url, i, v, self.units] def __init__(self, kind=None, url=None, values=None, units=None): self.kind = kind self.url = url self.values = values or [] self.units = units def __str__(self): return 'TR({kind},{url},{values},{units})'.format(**self.__dict__) __repr__ = __str__ def parse_telemetry_results(filepath): results = [] artifacts = [] with open(filepath) as fh: for line in fh: match = RESULT_REGEX.search(line) if match: result = TelemetryResult() result.kind = match.group(1) result.url = match.group(2) if match.group(4): result.values = map(numeric, match.group(4).split(',')) else: result.values = [numeric(match.group(5))] result.units = match.group(6) results.append(result) match = TRACE_REGEX.search(line) if match: artifacts.append(match.group(1)) return results, artifacts def special_average(values): """Overall score calculation. Tries to accound for large differences between different pages.""" negs = [v < 0 for v in values] abs_logs = [(av and math.log(av, 10) or av) for av in map(abs, values)] signed_logs = [] for lv, n in zip(abs_logs, negs): if n: signed_logs.append(-lv) else: signed_logs.append(lv) return get_meansd(signed_logs)[0] if __name__ == '__main__': import sys from pprint import pprint path = sys.argv[1] pprint(parse_telemetry_results(path))