mirror of
https://github.com/ARM-software/workload-automation.git
synced 2025-02-07 13:41:24 +00:00
272 lines
10 KiB
Python
272 lines
10 KiB
Python
# Copyright 2015 ARM Limited
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
# pylint: disable=attribute-defined-outside-init
|
|
import os
|
|
import re
|
|
import csv
|
|
import math
|
|
import shutil
|
|
from zipfile import is_zipfile, ZipFile
|
|
from collections import defaultdict
|
|
|
|
from wlauto import Workload, Parameter
|
|
from wlauto.exceptions import WorkloadError
|
|
from wlauto.utils.misc import check_output, get_null, get_meansd
|
|
from wlauto.utils.types import numeric, identifier
|
|
|
|
|
|
RESULT_REGEX = re.compile(r'RESULT ([^:]+): ([^=]+)\s*=\s*' # preamble and test/metric name
|
|
r'(\[([^\]]+)\]|(\S+))' # value
|
|
r'\s*(\S+)') # units
|
|
TRACE_REGEX = re.compile(r'Trace saved as ([^\n]+)')
|
|
|
|
|
|
class Telemetry(Workload):
|
|
|
|
name = 'telemetry'
|
|
description = """
|
|
Executes Google's Telemetery benchmarking framework (must be installed).
|
|
|
|
Url: https://www.chromium.org/developers/telemetry
|
|
|
|
From the web site:
|
|
|
|
Telemetry is Chrome's performance testing framework. It allows you to
|
|
perform arbitrary actions on a set of web pages and report metrics about
|
|
it. The framework abstracts:
|
|
|
|
- Launching a browser with arbitrary flags on any platform.
|
|
- Opening a tab and navigating to the page under test.
|
|
- Fetching data via the Inspector timeline and traces.
|
|
- Using Web Page Replay to cache real-world websites so they don't
|
|
change when used in benchmarks.
|
|
|
|
Design Principles
|
|
|
|
- Write one performance test that runs on all platforms - Windows, Mac,
|
|
Linux, Chrome OS, and Android for both Chrome and ContentShell.
|
|
- Runs on browser binaries, without a full Chromium checkout, and without
|
|
having to build the browser yourself.
|
|
- Use WebPageReplay to get repeatable test results.
|
|
- Clean architecture for writing benchmarks that keeps measurements and
|
|
use cases separate.
|
|
- Run on non-Chrome browsers for comparative studies.
|
|
|
|
This instrument runs telemetry via its ``run_benchmarks`` script (which
|
|
must be in PATH or specified using ``run_benchmarks_path`` parameter) and
|
|
parses metrics from the resulting output.
|
|
|
|
**device setup**
|
|
|
|
The device setup will depend on whether you're running a test image (in
|
|
which case little or no setup should be necessary)
|
|
|
|
|
|
"""
|
|
|
|
parameters = [
|
|
Parameter('run_benchmark_path', default='run_benchmark',
|
|
description="""
|
|
This is the path to run_benchmark script which runs a
|
|
Telemetry benchmark. If not specified, the assumption will be
|
|
that it is in path (i.e. with be invoked as ``run_benchmark``).
|
|
"""),
|
|
Parameter('test', default='page_cycler.top_10_mobile',
|
|
description="""
|
|
Specifies with of the the telemetry tests is to be run.
|
|
"""),
|
|
Parameter('run_benchmark_params', default='',
|
|
description="""
|
|
Additional paramters to be passed to ``run_benchmarks``.
|
|
"""),
|
|
Parameter('run_timeout', kind=int, default=900,
|
|
description="""
|
|
Timeout for execution of the test.
|
|
"""),
|
|
]
|
|
|
|
summary_metrics = ['cold_times',
|
|
'commit_charge',
|
|
'cpu_utilization',
|
|
'processes',
|
|
'resident_set_size_peak_size_browser',
|
|
'resident_set_size_peak_size_gpu',
|
|
'vm_final_size_browser',
|
|
'vm_final_size_gpu',
|
|
'vm_final_size_renderer',
|
|
'vm_final_size_total',
|
|
'vm_peak_size_browser',
|
|
'vm_peak_size_gpu',
|
|
'vm_private_dirty_final_browser',
|
|
'vm_private_dirty_final_gpu',
|
|
'vm_private_dirty_final_renderer',
|
|
'vm_private_dirty_final_total',
|
|
'vm_resident_set_size_final_size_browser',
|
|
'vm_resident_set_size_final_size_gpu',
|
|
'vm_resident_set_size_final_size_renderer',
|
|
'vm_resident_set_size_final_size_total',
|
|
'warm_times']
|
|
|
|
def validate(self):
|
|
ret = os.system('{} > {} 2>&1'.format(self.run_benchmark_path, get_null()))
|
|
if ret > 255:
|
|
pass # telemetry found and appears to be installed properly.
|
|
elif ret == 127:
|
|
raise WorkloadError('run_benchmarks not found (did you specify correct run_benchmarks_path?)')
|
|
else:
|
|
raise WorkloadError('Unexected error from run_benchmarks: {}'.format(ret))
|
|
|
|
def setup(self, context):
|
|
self.raw_output = None
|
|
self.command = self.build_command()
|
|
|
|
def run(self, context):
|
|
self.logger.debug(self.command)
|
|
self.raw_output, _ = check_output(self.command, shell=True, timeout=self.run_timeout, ignore=1)
|
|
|
|
def update_result(self, context): # pylint: disable=too-many-locals
|
|
if not self.raw_output:
|
|
self.logger.warning('Did not get run_benchmark output.')
|
|
return
|
|
raw_outfile = os.path.join(context.output_directory, 'telemetry_raw.out')
|
|
with open(raw_outfile, 'w') as wfh:
|
|
wfh.write(self.raw_output)
|
|
context.add_artifact('telemetry-raw', raw_outfile, kind='raw')
|
|
|
|
results, artifacts = parse_telemetry_results(raw_outfile)
|
|
csv_outfile = os.path.join(context.output_directory, 'telemetry.csv')
|
|
averages = defaultdict(list)
|
|
with open(csv_outfile, 'wb') as wfh:
|
|
writer = csv.writer(wfh)
|
|
writer.writerow(['kind', 'url', 'iteration', 'value', 'units'])
|
|
for result in results:
|
|
name_template = identifier('{}_{}_{{}}'.format(result.url, result.kind))
|
|
averages[result.kind].append(result.average)
|
|
context.result.add_metric(name_template.format('avg'), result.average,
|
|
result.units, lower_is_better=True)
|
|
context.result.add_metric(name_template.format('sd'), result.std,
|
|
result.units, lower_is_better=True)
|
|
writer.writerows(result.rows)
|
|
context.add_artifact('telemetry', csv_outfile, kind='data')
|
|
|
|
for kind, values in averages.iteritems():
|
|
context.result.add_metric(kind, special_average(values), lower_is_better=True)
|
|
|
|
for idx, artifact in enumerate(artifacts):
|
|
wa_path = os.path.join(context.output_directory,
|
|
os.path.basename(artifact))
|
|
shutil.copy(artifact, wa_path)
|
|
context.add_artifact('telemetry_trace_{}'.format(idx), path=wa_path, kind='data')
|
|
|
|
if is_zipfile(wa_path):
|
|
zf = ZipFile(wa_path)
|
|
zf.extractall(context.output_directory)
|
|
zf.close()
|
|
|
|
def teardown(self, context):
|
|
pass
|
|
|
|
def build_command(self):
|
|
device_opts = ''
|
|
if self.device.platform == 'chromeos':
|
|
if '--remote' not in self.run_benchmark_params:
|
|
device_opts += '--remote={} '.format(self.device.host)
|
|
if '--browser' not in self.run_benchmark_params:
|
|
device_opts += '--browser=cros-chrome '
|
|
elif self.device.platform == 'android':
|
|
if '--device' not in self.run_benchmark_params and self.device.adb_name:
|
|
device_opts += '--device={} '.format(self.device.adb_name)
|
|
if '--browser' not in self.run_benchmark_params:
|
|
device_opts += '--browser=android-webview-shell '
|
|
else:
|
|
raise WorkloadError('Currently, telemetry workload supports only ChromeOS or Android devices.')
|
|
return '{} {} {} {}'.format(self.run_benchmark_path,
|
|
self.test,
|
|
device_opts,
|
|
self.run_benchmark_params)
|
|
|
|
|
|
class TelemetryResult(object):
|
|
|
|
@property
|
|
def average(self):
|
|
return get_meansd(self.values)[0]
|
|
|
|
@property
|
|
def std(self):
|
|
return get_meansd(self.values)[1]
|
|
|
|
@property
|
|
def rows(self):
|
|
for i, v in enumerate(self.values):
|
|
yield [self.kind, self.url, i, v, self.units]
|
|
|
|
def __init__(self, kind=None, url=None, values=None, units=None):
|
|
self.kind = kind
|
|
self.url = url
|
|
self.values = values or []
|
|
self.units = units
|
|
|
|
def __str__(self):
|
|
return 'TR({kind},{url},{values},{units})'.format(**self.__dict__)
|
|
|
|
__repr__ = __str__
|
|
|
|
|
|
def parse_telemetry_results(filepath):
|
|
results = []
|
|
artifacts = []
|
|
with open(filepath) as fh:
|
|
for line in fh:
|
|
match = RESULT_REGEX.search(line)
|
|
if match:
|
|
result = TelemetryResult()
|
|
result.kind = match.group(1)
|
|
result.url = match.group(2)
|
|
if match.group(4):
|
|
result.values = map(numeric, match.group(4).split(','))
|
|
else:
|
|
result.values = [numeric(match.group(5))]
|
|
result.units = match.group(6)
|
|
results.append(result)
|
|
match = TRACE_REGEX.search(line)
|
|
if match:
|
|
artifacts.append(match.group(1))
|
|
return results, artifacts
|
|
|
|
|
|
def special_average(values):
|
|
"""Overall score calculation. Tries to accound for large differences
|
|
between different pages."""
|
|
negs = [v < 0 for v in values]
|
|
abs_logs = [(av and math.log(av, 10) or av)
|
|
for av in map(abs, values)]
|
|
signed_logs = []
|
|
for lv, n in zip(abs_logs, negs):
|
|
if n:
|
|
signed_logs.append(-lv)
|
|
else:
|
|
signed_logs.append(lv)
|
|
return get_meansd(signed_logs)[0]
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import sys
|
|
from pprint import pprint
|
|
path = sys.argv[1]
|
|
pprint(parse_telemetry_results(path))
|
|
|