From 4e6ba647377a03818d780eeccc6f33c52f531a5f Mon Sep 17 00:00:00 2001 From: Sergei Trofimov Date: Fri, 15 Sep 2017 17:30:02 +0100 Subject: [PATCH] instrumentation: add FPS instrument. Add an instrument to collect FPS (frames per second) and associated rendering statics using corresponding devlib functionality. --- wa/instrumentation/fps.py | 169 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 169 insertions(+) create mode 100644 wa/instrumentation/fps.py diff --git a/wa/instrumentation/fps.py b/wa/instrumentation/fps.py new file mode 100644 index 00000000..42bc673f --- /dev/null +++ b/wa/instrumentation/fps.py @@ -0,0 +1,169 @@ +import os +import shutil + +from devlib import SurfaceFlingerFramesInstrument, GfxInfoFramesInstrument +from devlib import DerivedSurfaceFlingerStats, DerivedGfxInfoStats + +from wa import Instrument, Parameter, WorkloadError +from wa.utils.types import numeric + + +class FpsInstrument(Instrument): + + name = 'fps' + description = """ + Measures Frames Per Second (FPS) and associated metrics for a workload. + + .. note:: This instrument depends on pandas Python library (which is not part of standard + WA dependencies), so you will need to install that first, before you can use it. + + Android L and below use SurfaceFlinger to calculate the FPS data. + Android M and above use gfxinfo to calculate the FPS data. + + SurfaceFlinger: + The view is specified by the workload as ``view`` attribute. This defaults + to ``'SurfaceView'`` for game workloads, and ``None`` for non-game + workloads (as for them FPS mesurement usually doesn't make sense). + Individual workloads may override this. + + gfxinfo: + The view is specified by the workload as ``package`` attribute. + This is because gfxinfo already processes for all views in a package. + + """ + + parameters = [ + Parameter('drop_threshold', kind=numeric, default=5, + description=""" + Data points below this FPS will be dropped as they do not + constitute "real" gameplay. The assumption being that while + actually running, the FPS in the game will not drop below X + frames per second, except on loading screens, menus, etc, + which should not contribute to FPS calculation. + """), + Parameter('keep_raw', kind=bool, default=False, + description=""" + If set to ``True``, this will keep the raw dumpsys output in + the results directory (this is maily used for debugging) + Note: frames.csv with collected frames data will always be + generated regardless of this setting. + """), + Parameter('crash_threshold', kind=float, default=0.7, + description=""" + Specifies the threshold used to decided whether a + measured/expected frames ration indicates a content crash. + E.g. a value of ``0.75`` means the number of actual frames + counted is a quarter lower than expected, it will treated as + a content crash. + + If set to zero, no crash check will be performed. + """), + Parameter('period', kind=float, default=2, constraint=lambda x: x > 0, + description=""" + Specifies the time period between polling frame data in + seconds when collecting frame data. Using a lower value + improves the granularity of timings when recording actions + that take a short time to complete. Note, this will produce + duplicate frame data in the raw dumpsys output, however, this + is filtered out in frames.csv. It may also affect the + overall load on the system. + + The default value of 2 seconds corresponds with the + NUM_FRAME_RECORDS in + android/services/surfaceflinger/FrameTracker.h (as of the + time of writing currently 128) and a frame rate of 60 fps + that is applicable to most devices. + """), + Parameter('force_surfaceflinger', kind=bool, default=False, + description=""" + By default, the method to capture fps data is based on + Android version. If this is set to true, force the + instrument to use the SurfaceFlinger method regardless of its + Android version. + """), + ] + + def __init__(self, target, **kwargs): + super(FpsInstrument, self).__init__(target, **kwargs) + self.collector = None + self.processor = None + self._is_enabled = None + + def setup(self, context): + use_gfxinfo = self.target.get_sdk_version() >= 23 and not self.force_surfaceflinger + if use_gfxinfo: + collector_target_attr = 'package' + else: + collector_target_attr = 'view' + collector_target = getattr(context.workload, collector_target_attr, None) + + if not collector_target: + self._is_enabled = False + msg = 'Workload {} does not define a {}; disabling frame collection and FPS evaluation.' + self.logger.info(msg.format(context.workload.name, collector_target_attr)) + return + + self._is_enabled = True + if use_gfxinfo: + self.collector = GfxInfoFramesInstrument(self.target, collector_target, self.period) + self.processor = DerivedGfxInfoStats(self.drop_threshold, filename='fps.csv') + else: + self.collector = SurfaceFlingerFramesInstrument(self.target, collector_target, self.period) + self.processor = DerivedSurfaceFlingerStats(self.drop_threshold, filename='fps.csv') + self.collector.reset() + + def start(self, context): + if not self._is_enabled: + return + self.collector.start() + + def stop(self, context): + if not self._is_enabled: + return + self.collector.stop() + + def update_result(self, context): + if not self._is_enabled: + return + outpath = os.path.join(context.output_directory, 'frames.csv') + frames_csv = self.collector.get_data(outpath) + raw_output = self.collector.get_raw() + + processed = self.processor.process(frames_csv) + processed.extend(self.processor.process_raw(*raw_output)) + fps, frame_count, fps_csv = processed[:3] + rest = processed[3:] + + context.add_metric(fps.name, fps.value, fps.units) + context.add_metric(frame_count.name, frame_count.value, frame_count.units) + context.add_artifact('frames', frames_csv.path, kind='raw') + context.add_artifact('fps', fps_csv.path, kind='data') + for metric in rest: + context.add_metric(metric.name, metric.value, metric.units, lower_is_better=True) + + if not self.keep_raw: + for entry in raw_output: + if os.path.isdir(entry): + shutil.rmtree(entry) + elif os.path.isfile(entry): + os.remove(entry) + + if not frame_count.value: + context.add_event('Could not frind frames data in gfxinfo output') + context.set_status('PARTIAL') + + self.check_for_crash(context, fps.value, frame_count.value, + context.current_job.run_time.total_seconds()) + + def check_for_crash(self, context, fps, frames, exec_time): + if not self.crash_threshold: + return + self.logger.debug('Checking for crashed content.') + if all([exec_time, fps, frames]): + expected_frames = fps * exec_time + ratio = frames / expected_frames + self.logger.debug('actual/expected frames: {:.2}'.format(ratio)) + if ratio < self.crash_threshold: + msg = 'Content for {} appears to have crashed.\n'.format(context.current_job.spec.label) + msg += 'Content crash detected (actual/expected frames: {:.2}).'.format(ratio) + raise WorkloadError(msg)