mirror of
https://github.com/ARM-software/workload-automation.git
synced 2024-10-07 03:11:53 +01:00
1993007d49
It seems valid install may return values other than 0xff00. Relaxing the check to consider anything above 0xff to be valid.
248 lines
9.3 KiB
Python
248 lines
9.3 KiB
Python
# pylint: disable=attribute-defined-outside-init
|
|
import os
|
|
import re
|
|
import csv
|
|
import math
|
|
import shutil
|
|
from zipfile import is_zipfile, ZipFile
|
|
from collections import defaultdict
|
|
|
|
from wlauto import Workload, Parameter
|
|
from wlauto.exceptions import WorkloadError
|
|
from wlauto.utils.misc import check_output, get_null, get_meansd
|
|
from wlauto.utils.types import numeric, identifier
|
|
|
|
|
|
RESULT_REGEX = re.compile(r'RESULT ([^:]+): ([^=]+)\s*=\s*' # preamble and test/metric name
|
|
r'(\[([^\]]+)\]|(\S+))' # value
|
|
r'\s*(\S+)') # units
|
|
TRACE_REGEX = re.compile(r'Trace saved as ([^\n]+)')
|
|
|
|
|
|
class Telemetry(Workload):
|
|
|
|
name = 'telemetry'
|
|
description = """
|
|
Executes Google's Telemetery benchmarking framework (must be installed).
|
|
|
|
Url: https://www.chromium.org/developers/telemetry
|
|
|
|
From the web site:
|
|
|
|
Telemetry is Chrome's performance testing framework. It allows you to
|
|
perform arbitrary actions on a set of web pages and report metrics about
|
|
it. The framework abstracts:
|
|
|
|
- Launching a browser with arbitrary flags on any platform.
|
|
- Opening a tab and navigating to the page under test.
|
|
- Fetching data via the Inspector timeline and traces.
|
|
- Using Web Page Replay to cache real-world websites so they don't
|
|
change when used in benchmarks.
|
|
|
|
Design Principles
|
|
|
|
- Write one performance test that runs on all platforms - Windows, Mac,
|
|
Linux, Chrome OS, and Android for both Chrome and ContentShell.
|
|
- Runs on browser binaries, without a full Chromium checkout, and without
|
|
having to build the browser yourself.
|
|
- Use WebPageReplay to get repeatable test results.
|
|
- Clean architecture for writing benchmarks that keeps measurements and
|
|
use cases separate.
|
|
- Run on non-Chrome browsers for comparative studies.
|
|
|
|
This instrument runs telemetry via its ``run_benchmarks`` script (which
|
|
must be in PATH or specified using ``run_benchmarks_path`` parameter) and
|
|
parses metrics from the resulting output.
|
|
|
|
**device setup**
|
|
|
|
The device setup will depend on whether you're running a test image (in
|
|
which case little or no setup should be necessary)
|
|
|
|
|
|
"""
|
|
|
|
parameters = [
|
|
Parameter('run_benchmark_path', default='run_benchmark',
|
|
description="""
|
|
This is the path to run_benchmark script which runs a
|
|
Telemetry benchmark. If not specified, the assumption will be
|
|
that it is in path (i.e. with be invoked as ``run_benchmark``).
|
|
"""),
|
|
Parameter('test', default='page_cycler.top_10_mobile',
|
|
description="""
|
|
Specifies with of the the telemetry tests is to be run.
|
|
"""),
|
|
Parameter('run_benchmark_params', default='',
|
|
description="""
|
|
Additional paramters to be passed to ``run_benchmarks``.
|
|
"""),
|
|
Parameter('run_timeout', kind=int, default=900,
|
|
description="""
|
|
Timeout for execution of the test.
|
|
"""),
|
|
]
|
|
|
|
summary_metrics = ['cold_times',
|
|
'commit_charge',
|
|
'cpu_utilization',
|
|
'processes',
|
|
'resident_set_size_peak_size_browser',
|
|
'resident_set_size_peak_size_gpu',
|
|
'vm_final_size_browser',
|
|
'vm_final_size_gpu',
|
|
'vm_final_size_renderer',
|
|
'vm_final_size_total',
|
|
'vm_peak_size_browser',
|
|
'vm_peak_size_gpu',
|
|
'vm_private_dirty_final_browser',
|
|
'vm_private_dirty_final_gpu',
|
|
'vm_private_dirty_final_renderer',
|
|
'vm_private_dirty_final_total',
|
|
'vm_resident_set_size_final_size_browser',
|
|
'vm_resident_set_size_final_size_gpu',
|
|
'vm_resident_set_size_final_size_renderer',
|
|
'vm_resident_set_size_final_size_total',
|
|
'warm_times']
|
|
|
|
def validate(self):
|
|
ret = os.system('{} > {} 2>&1'.format(self.run_benchmark_path, get_null()))
|
|
if ret > 255:
|
|
pass # telemetry found and appears to be installed properly.
|
|
elif ret == 127:
|
|
raise WorkloadError('run_benchmarks not found (did you specify correct run_benchmarks_path?)')
|
|
else:
|
|
raise WorkloadError('Unexected error from run_benchmarks: {}'.format(ret))
|
|
|
|
def setup(self, context):
|
|
self.raw_output = None
|
|
self.command = self.build_command()
|
|
|
|
def run(self, context):
|
|
self.logger.debug(self.command)
|
|
self.raw_output, _ = check_output(self.command, shell=True, timeout=self.run_timeout, ignore=1)
|
|
|
|
def update_result(self, context): # pylint: disable=too-many-locals
|
|
if not self.raw_output:
|
|
self.logger.warning('Did not get run_benchmark output.')
|
|
return
|
|
raw_outfile = os.path.join(context.output_directory, 'telemetry_raw.out')
|
|
with open(raw_outfile, 'w') as wfh:
|
|
wfh.write(self.raw_output)
|
|
context.add_artifact('telemetry-raw', raw_outfile, kind='raw')
|
|
|
|
results, artifacts = parse_telemetry_results(raw_outfile)
|
|
csv_outfile = os.path.join(context.output_directory, 'telemetry.csv')
|
|
averages = defaultdict(list)
|
|
with open(csv_outfile, 'wb') as wfh:
|
|
writer = csv.writer(wfh)
|
|
writer.writerow(['kind', 'url', 'iteration', 'value', 'units'])
|
|
for result in results:
|
|
name_template = identifier('{}_{}_{{}}'.format(result.url, result.kind))
|
|
averages[result.kind].append(result.average)
|
|
context.result.add_metric(name_template.format('avg'), result.average,
|
|
result.units, lower_is_better=True)
|
|
context.result.add_metric(name_template.format('sd'), result.std,
|
|
result.units, lower_is_better=True)
|
|
writer.writerows(result.rows)
|
|
context.add_artifact('telemetry', csv_outfile, kind='data')
|
|
|
|
for kind, values in averages.iteritems():
|
|
context.result.add_metric(kind, special_average(values), lower_is_better=True)
|
|
|
|
for idx, artifact in enumerate(artifacts):
|
|
wa_path = os.path.join(context.output_directory,
|
|
os.path.basename(artifact))
|
|
shutil.copy(artifact, wa_path)
|
|
context.add_artifact('telemetry_trace_{}'.format(idx), path=wa_path, kind='data')
|
|
|
|
if is_zipfile(wa_path):
|
|
zf = ZipFile(wa_path)
|
|
zf.extractall(context.output_directory)
|
|
zf.close()
|
|
|
|
def teardown(self, context):
|
|
pass
|
|
|
|
def build_command(self):
|
|
if self.device.platform == 'chromeos':
|
|
device_opts = '--remote={} --browser=cros-chrome'.format(self.device.host)
|
|
else:
|
|
raise WorkloadError('Currently, telemetry workload supports only ChromeOS devices.')
|
|
return '{} {} {} {}'.format(self.run_benchmark_path,
|
|
self.test,
|
|
device_opts,
|
|
self.run_benchmark_params)
|
|
|
|
|
|
class TelemetryResult(object):
|
|
|
|
@property
|
|
def average(self):
|
|
return get_meansd(self.values)[0]
|
|
|
|
@property
|
|
def std(self):
|
|
return get_meansd(self.values)[1]
|
|
|
|
@property
|
|
def rows(self):
|
|
for i, v in enumerate(self.values):
|
|
yield [self.kind, self.url, i, v, self.units]
|
|
|
|
def __init__(self, kind=None, url=None, values=None, units=None):
|
|
self.kind = kind
|
|
self.url = url
|
|
self.values = values or []
|
|
self.units = units
|
|
|
|
def __str__(self):
|
|
return 'TR({kind},{url},{values},{units})'.format(**self.__dict__)
|
|
|
|
__repr__ = __str__
|
|
|
|
|
|
def parse_telemetry_results(filepath):
|
|
results = []
|
|
artifacts = []
|
|
with open(filepath) as fh:
|
|
for line in fh:
|
|
match = RESULT_REGEX.search(line)
|
|
if match:
|
|
result = TelemetryResult()
|
|
result.kind = match.group(1)
|
|
result.url = match.group(2)
|
|
if match.group(4):
|
|
result.values = map(numeric, match.group(4).split(','))
|
|
else:
|
|
result.values = [numeric(match.group(5))]
|
|
result.units = match.group(6)
|
|
results.append(result)
|
|
match = TRACE_REGEX.search(line)
|
|
if match:
|
|
artifacts.append(match.group(1))
|
|
return results, artifacts
|
|
|
|
|
|
def special_average(values):
|
|
"""Overall score calculation. Tries to accound for large differences
|
|
between different pages."""
|
|
negs = [v < 0 for v in values]
|
|
abs_logs = [(av and math.log(av, 10) or av)
|
|
for av in map(abs, values)]
|
|
signed_logs = []
|
|
for lv, n in zip(abs_logs, negs):
|
|
if n:
|
|
signed_logs.append(-lv)
|
|
else:
|
|
signed_logs.append(lv)
|
|
return get_meansd(signed_logs)[0]
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import sys
|
|
from pprint import pprint
|
|
path = sys.argv[1]
|
|
pprint(parse_telemetry_results(path))
|
|
|