telemetry: added fps extraction

Also fixed text that mistakenly referred to "run_benchmarks" (plural) as
the name of the script.

wlauto/workloads/telemetry/__init__.py

@@ -19,11 +19,17 @@ import re
 import csv
 import math
 import shutil
+import json
 from zipfile import is_zipfile, ZipFile
 from collections import defaultdict
 
+try:
+    import pandas as pd
+except ImportError:
+    pd = None
+
 from wlauto import Workload, Parameter
-from wlauto.exceptions import WorkloadError
+from wlauto.exceptions import WorkloadError, ConfigError
 from wlauto.utils.misc import check_output, get_null, get_meansd
 from wlauto.utils.types import numeric, identifier
 
@@ -33,6 +39,9 @@ RESULT_REGEX = re.compile(r'RESULT ([^:]+): ([^=]+)\s*=\s*'  # preamble and test
                           r'\s*(\S+)')  # units
 TRACE_REGEX = re.compile(r'Trace saved as ([^\n]+)')
 
+# Trace event that signifies rendition of a Frame
+FRAME_EVENT = 'SwapBuffersLatency'
+
 
 class Telemetry(Workload):
 
@@ -65,8 +74,8 @@ class Telemetry(Workload):
         use cases separate.
       - Run on non-Chrome browsers for comparative studies.
 
-    This instrument runs  telemetry via its ``run_benchmarks`` script (which
+    This instrument runs  telemetry via its ``run_benchmark`` script (which
-    must be in PATH or specified using ``run_benchmarks_path`` parameter) and
+    must be in PATH or specified using ``run_benchmark_path`` parameter) and
     parses metrics from the resulting output.
 
     **device setup**
@@ -90,44 +99,28 @@ class Telemetry(Workload):
                   """),
         Parameter('run_benchmark_params', default='',
                   description="""
-                  Additional paramters to be passed to ``run_benchmarks``.
+                  Additional paramters to be passed to ``run_benchmark``.
                   """),
         Parameter('run_timeout', kind=int, default=900,
                   description="""
                   Timeout for execution of the test.
                   """),
+        Parameter('extract_fps', kind=bool, default=False,
+                  description="""
+                  if ``True``, FPS for the run will be computed from the trace (must be enabled).
+                  """),
     ]
 
-    summary_metrics = ['cold_times',
-                       'commit_charge',
-                       'cpu_utilization',
-                       'processes',
-                       'resident_set_size_peak_size_browser',
-                       'resident_set_size_peak_size_gpu',
-                       'vm_final_size_browser',
-                       'vm_final_size_gpu',
-                       'vm_final_size_renderer',
-                       'vm_final_size_total',
-                       'vm_peak_size_browser',
-                       'vm_peak_size_gpu',
-                       'vm_private_dirty_final_browser',
-                       'vm_private_dirty_final_gpu',
-                       'vm_private_dirty_final_renderer',
-                       'vm_private_dirty_final_total',
-                       'vm_resident_set_size_final_size_browser',
-                       'vm_resident_set_size_final_size_gpu',
-                       'vm_resident_set_size_final_size_renderer',
-                       'vm_resident_set_size_final_size_total',
-                       'warm_times']
-
     def validate(self):
         ret = os.system('{} > {} 2>&1'.format(self.run_benchmark_path, get_null()))
         if ret > 255:
             pass  # telemetry found and appears to be installed properly.
         elif ret == 127:
-            raise WorkloadError('run_benchmarks not found (did you specify correct run_benchmarks_path?)')
+            raise WorkloadError('run_benchmark not found (did you specify correct run_benchmark_path?)')
         else:
-            raise WorkloadError('Unexected error from run_benchmarks: {}'.format(ret))
+            raise WorkloadError('Unexected error from run_benchmark: {}'.format(ret))
+        if self.extract_fps and 'trace' not in self.run_benchmark_params:
+            raise ConfigError('"trace" profiler must be enabled in order to extract FPS for Telemetry')
 
     def setup(self, context):
         self.raw_output = None
@@ -166,15 +159,21 @@ class Telemetry(Workload):
             context.result.add_metric(kind, special_average(values), lower_is_better=True)
 
         for idx, artifact in enumerate(artifacts):
+            if is_zipfile(artifact):
+                zf = ZipFile(artifact)
+                for item in zf.infolist():
+                    zf.extract(item, context.output_directory)
+                    zf.close()
+                    context.add_artifact('telemetry_trace_{}'.format(idx), path=item.filename, kind='data')
+            else:  # not a zip archive
                 wa_path = os.path.join(context.output_directory,
                                        os.path.basename(artifact))
                 shutil.copy(artifact, wa_path)
-            context.add_artifact('telemetry_trace_{}'.format(idx), path=wa_path, kind='data')
+                context.add_artifact('telemetry_artifact_{}'.format(idx), path=wa_path, kind='data')
 
-            if is_zipfile(wa_path):
+        if self.extract_fps:
-                zf = ZipFile(wa_path)
+            self.logger.debug('Extracting FPS...')
-                zf.extractall(context.output_directory)
+            _extract_fps(context)
-                zf.close()
 
     def teardown(self, context):
         pass
@@ -199,6 +198,29 @@ class Telemetry(Workload):
                                     self.run_benchmark_params)
 
 
+def _extract_fps(context):
+    trace_files = [a.path for a in context.iteration_artifacts
+                   if a.name.startswith('telemetry_trace_')]
+    for tf in trace_files:
+        name = os.path.splitext(os.path.basename(tf))[0]
+        fps_file = os.path.join(context.output_directory, name + '-fps.csv')
+        with open(tf) as fh:
+            data = json.load(fh)
+            events = pd.Series([e['ts'] for e in data['traceEvents'] if
+                                FRAME_EVENT == e['name']])
+            fps = (1000000 / (events - events.shift(1)))
+            fps.index = events
+            df = fps.dropna().reset_index()
+            df.columns = ['timestamp', 'fps']
+            with open(fps_file, 'w') as wfh:
+                df.to_csv(wfh, index=False)
+            context.add_artifact('{}_fps'.format(name), fps_file, kind='data')
+            context.result.add_metric('{} FPS'.format(name), df.fps.mean(),
+                                      units='fps')
+            context.result.add_metric('{} FPS (std)'.format(name), df.fps.std(),
+                                      units='fps', lower_is_better=True)
+
+
 class TelemetryResult(object):
 
     @property