From 4e6ba647377a03818d780eeccc6f33c52f531a5f Mon Sep 17 00:00:00 2001
From: Sergei Trofimov <sergei.trofimov@arm.com>
Date: Fri, 15 Sep 2017 17:30:02 +0100
Subject: [PATCH] instrumentation: add FPS instrument.
Add an instrument to collect FPS (frames per second) and associated
rendering statics using corresponding devlib functionality.
---
wa/instrumentation/fps.py | 169 ++++++++++++++++++++++++++++++++++++++
1 file changed, 169 insertions(+)
create mode 100644 wa/instrumentation/fps.py
diff --git a/wa/instrumentation/fps.py b/wa/instrumentation/fps.py
new file mode 100644
index 00000000..42bc673f
--- /dev/null
+++ b/wa/instrumentation/fps.py
@@ -0,0 +1,169 @@
+import os
+import shutil
+
+from devlib import SurfaceFlingerFramesInstrument, GfxInfoFramesInstrument
+from devlib import DerivedSurfaceFlingerStats, DerivedGfxInfoStats
+
+from wa import Instrument, Parameter, WorkloadError
+from wa.utils.types import numeric
+
+
+class FpsInstrument(Instrument):
+
+ name = 'fps'
+ description = """
+ Measures Frames Per Second (FPS) and associated metrics for a workload.
+
+ .. note:: This instrument depends on pandas Python library (which is not part of standard
+ WA dependencies), so you will need to install that first, before you can use it.
+
+ Android L and below use SurfaceFlinger to calculate the FPS data.
+ Android M and above use gfxinfo to calculate the FPS data.
+
+ SurfaceFlinger:
+ The view is specified by the workload as ``view`` attribute. This defaults
+ to ``'SurfaceView'`` for game workloads, and ``None`` for non-game
+ workloads (as for them FPS mesurement usually doesn't make sense).
+ Individual workloads may override this.
+
+ gfxinfo:
+ The view is specified by the workload as ``package`` attribute.
+ This is because gfxinfo already processes for all views in a package.
+
+ """
+
+ parameters = [
+ Parameter('drop_threshold', kind=numeric, default=5,
+ description="""
+ Data points below this FPS will be dropped as they do not
+ constitute "real" gameplay. The assumption being that while
+ actually running, the FPS in the game will not drop below X
+ frames per second, except on loading screens, menus, etc,
+ which should not contribute to FPS calculation.
+ """),
+ Parameter('keep_raw', kind=bool, default=False,
+ description="""
+ If set to ``True``, this will keep the raw dumpsys output in
+ the results directory (this is maily used for debugging)
+ Note: frames.csv with collected frames data will always be
+ generated regardless of this setting.
+ """),
+ Parameter('crash_threshold', kind=float, default=0.7,
+ description="""
+ Specifies the threshold used to decided whether a
+ measured/expected frames ration indicates a content crash.
+ E.g. a value of ``0.75`` means the number of actual frames
+ counted is a quarter lower than expected, it will treated as
+ a content crash.
+
+ If set to zero, no crash check will be performed.
+ """),
+ Parameter('period', kind=float, default=2, constraint=lambda x: x > 0,
+ description="""
+ Specifies the time period between polling frame data in
+ seconds when collecting frame data. Using a lower value
+ improves the granularity of timings when recording actions
+ that take a short time to complete. Note, this will produce
+ duplicate frame data in the raw dumpsys output, however, this
+ is filtered out in frames.csv. It may also affect the
+ overall load on the system.
+
+ The default value of 2 seconds corresponds with the
+ NUM_FRAME_RECORDS in
+ android/services/surfaceflinger/FrameTracker.h (as of the
+ time of writing currently 128) and a frame rate of 60 fps
+ that is applicable to most devices.
+ """),
+ Parameter('force_surfaceflinger', kind=bool, default=False,
+ description="""
+ By default, the method to capture fps data is based on
+ Android version. If this is set to true, force the
+ instrument to use the SurfaceFlinger method regardless of its
+ Android version.
+ """),
+ ]
+
+ def __init__(self, target, **kwargs):
+ super(FpsInstrument, self).__init__(target, **kwargs)
+ self.collector = None
+ self.processor = None
+ self._is_enabled = None
+
+ def setup(self, context):
+ use_gfxinfo = self.target.get_sdk_version() >= 23 and not self.force_surfaceflinger
+ if use_gfxinfo:
+ collector_target_attr = 'package'
+ else:
+ collector_target_attr = 'view'
+ collector_target = getattr(context.workload, collector_target_attr, None)
+
+ if not collector_target:
+ self._is_enabled = False
+ msg = 'Workload {} does not define a {}; disabling frame collection and FPS evaluation.'
+ self.logger.info(msg.format(context.workload.name, collector_target_attr))
+ return
+
+ self._is_enabled = True
+ if use_gfxinfo:
+ self.collector = GfxInfoFramesInstrument(self.target, collector_target, self.period)
+ self.processor = DerivedGfxInfoStats(self.drop_threshold, filename='fps.csv')
+ else:
+ self.collector = SurfaceFlingerFramesInstrument(self.target, collector_target, self.period)
+ self.processor = DerivedSurfaceFlingerStats(self.drop_threshold, filename='fps.csv')
+ self.collector.reset()
+
+ def start(self, context):
+ if not self._is_enabled:
+ return
+ self.collector.start()
+
+ def stop(self, context):
+ if not self._is_enabled:
+ return
+ self.collector.stop()
+
+ def update_result(self, context):
+ if not self._is_enabled:
+ return
+ outpath = os.path.join(context.output_directory, 'frames.csv')
+ frames_csv = self.collector.get_data(outpath)
+ raw_output = self.collector.get_raw()
+
+ processed = self.processor.process(frames_csv)
+ processed.extend(self.processor.process_raw(*raw_output))
+ fps, frame_count, fps_csv = processed[:3]
+ rest = processed[3:]
+
+ context.add_metric(fps.name, fps.value, fps.units)
+ context.add_metric(frame_count.name, frame_count.value, frame_count.units)
+ context.add_artifact('frames', frames_csv.path, kind='raw')
+ context.add_artifact('fps', fps_csv.path, kind='data')
+ for metric in rest:
+ context.add_metric(metric.name, metric.value, metric.units, lower_is_better=True)
+
+ if not self.keep_raw:
+ for entry in raw_output:
+ if os.path.isdir(entry):
+ shutil.rmtree(entry)
+ elif os.path.isfile(entry):
+ os.remove(entry)
+
+ if not frame_count.value:
+ context.add_event('Could not frind frames data in gfxinfo output')
+ context.set_status('PARTIAL')
+
+ self.check_for_crash(context, fps.value, frame_count.value,
+ context.current_job.run_time.total_seconds())
+
+ def check_for_crash(self, context, fps, frames, exec_time):
+ if not self.crash_threshold:
+ return
+ self.logger.debug('Checking for crashed content.')
+ if all([exec_time, fps, frames]):
+ expected_frames = fps * exec_time
+ ratio = frames / expected_frames
+ self.logger.debug('actual/expected frames: {:.2}'.format(ratio))
+ if ratio < self.crash_threshold:
+ msg = 'Content for {} appears to have crashed.\n'.format(context.current_job.spec.label)
+ msg += 'Content crash detected (actual/expected frames: {:.2}).'.format(ratio)
+ raise WorkloadError(msg)