From cf8ebf666824f86ecd5495f339127cec7d0b762d Mon Sep 17 00:00:00 2001 From: Marc Bonnici Date: Thu, 5 Dec 2019 13:37:55 +0000 Subject: [PATCH] devlib/collector: Update Collectors to implement collector interface --- devlib/collector/dmesg.py | 14 ++++++++--- devlib/collector/ftrace.py | 29 ++++++++++++++------- devlib/collector/logcat.py | 21 ++++++++++------ devlib/collector/perf.py | 31 ++++++++++++++++------- devlib/collector/serial_trace.py | 43 ++++++++++++++++++-------------- devlib/collector/systrace.py | 33 +++++++++++++----------- 6 files changed, 109 insertions(+), 62 deletions(-) diff --git a/devlib/collector/dmesg.py b/devlib/collector/dmesg.py index 16166e5..8b25f5d 100644 --- a/devlib/collector/dmesg.py +++ b/devlib/collector/dmesg.py @@ -18,7 +18,8 @@ import re from itertools import takewhile from datetime import timedelta -from devlib.collector import CollectorBase +from devlib.collector import (CollectorBase, CollectorOutput, + CollectorOutputEntry) class KernelLogEntry(object): @@ -151,6 +152,7 @@ class DmesgCollector(CollectorBase): def __init__(self, target, level=LOG_LEVELS[-1], facility='kern'): super(DmesgCollector, self).__init__(target) + self.output_path = None if level not in self.LOG_LEVELS: raise ValueError('level needs to be one of: {}'.format( @@ -195,6 +197,12 @@ class DmesgCollector(CollectorBase): self.dmesg_out = self.target.execute(cmd) - def get_trace(self, outfile): - with open(outfile, 'wt') as f: + def set_output(self, output_path): + self.output_path = output_path + + def get_data(self): + if self.output_path is None: + raise RuntimeError("Output path was not set.") + with open(self.output_path, 'wt') as f: f.write(self.dmesg_out + '\n') + return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')]) diff --git a/devlib/collector/ftrace.py b/devlib/collector/ftrace.py index b6e9181..165fc8c 100644 --- a/devlib/collector/ftrace.py +++ b/devlib/collector/ftrace.py @@ -23,7 +23,8 @@ import sys import contextlib from pipes import quote -from devlib.collector import CollectorBase +from devlib.collector import (CollectorBase, CollectorOutput, + CollectorOutputEntry) from devlib.host import PACKAGE_BIN_DIRECTORY from devlib.exception import TargetStableError, HostError from devlib.utils.misc import check_output, which, memoized @@ -86,6 +87,7 @@ class FtraceCollector(CollectorBase): self.target_output_file = target.path.join(self.target.working_directory, OUTPUT_TRACE_FILE) text_file_name = target.path.splitext(OUTPUT_TRACE_FILE)[0] + '.txt' self.target_text_file = target.path.join(self.target.working_directory, text_file_name) + self.output_path = None self.target_binary = None self.host_binary = None self.start_time = None @@ -300,9 +302,14 @@ class FtraceCollector(CollectorBase): timeout=TIMEOUT, as_root=True) self._reset_needed = True - def get_trace(self, outfile): - if os.path.isdir(outfile): - outfile = os.path.join(outfile, os.path.basename(self.target_output_file)) + def set_output(self, output_path): + if os.path.isdir(output_path): + output_path = os.path.join(output_path, os.path.basename(self.target_output_file)) + self.output_path = output_path + + def get_data(self): + if self.output_path is None: + raise RuntimeError("Output path was not set.") self.target.execute('{0} extract -o {1}; chmod 666 {1}'.format(self.target_binary, self.target_output_file), timeout=TIMEOUT, as_root=True) @@ -311,20 +318,24 @@ class FtraceCollector(CollectorBase): # Therefore timout for the pull command must also be adjusted # accordingly. pull_timeout = 10 * (self.stop_time - self.start_time) - self.target.pull(self.target_output_file, outfile, timeout=pull_timeout) - if not os.path.isfile(outfile): + self.target.pull(self.target_output_file, self.output_path, timeout=pull_timeout) + output = CollectorOutput() + if not os.path.isfile(self.output_path): self.logger.warning('Binary trace not pulled from device.') else: + output.append(CollectorOutputEntry(self.output_path, 'file')) if self.autoreport: - textfile = os.path.splitext(outfile)[0] + '.txt' + textfile = os.path.splitext(self.output_path)[0] + '.txt' if self.report_on_target: self.generate_report_on_target() self.target.pull(self.target_text_file, textfile, timeout=pull_timeout) else: - self.report(outfile, textfile) + self.report(self.output_path, textfile) + output.append(CollectorOutputEntry(textfile, 'file')) if self.autoview: - self.view(outfile) + self.view(self.output_path) + return output def get_stats(self, outfile): if not (self.functions and self.tracer is None): diff --git a/devlib/collector/logcat.py b/devlib/collector/logcat.py index ccc9e9a..b3104ed 100644 --- a/devlib/collector/logcat.py +++ b/devlib/collector/logcat.py @@ -16,7 +16,8 @@ import os import shutil -from devlib.collector import CollectorBase +from devlib.collector import (CollectorBase, CollectorOutput, + CollectorOutputEntry) from devlib.utils.android import LogcatMonitor class LogcatCollector(CollectorBase): @@ -24,6 +25,7 @@ class LogcatCollector(CollectorBase): def __init__(self, target, regexps=None): super(LogcatCollector, self).__init__(target) self.regexps = regexps + self.output_path = None self._collecting = False self._prev_log = None self._monitor = None @@ -45,12 +47,14 @@ class LogcatCollector(CollectorBase): """ Start collecting logcat lines """ + if self.output_path is None: + raise RuntimeError("Output path was not set.") self._monitor = LogcatMonitor(self.target, self.regexps) if self._prev_log: # Append new data collection to previous collection self._monitor.start(self._prev_log) else: - self._monitor.start() + self._monitor.start(self.output_path) self._collecting = True @@ -65,9 +69,10 @@ class LogcatCollector(CollectorBase): self._collecting = False self._prev_log = self._monitor.logfile - def get_trace(self, outfile): - """ - Output collected logcat lines to designated file - """ - # copy self._monitor.logfile to outfile - shutil.copy(self._monitor.logfile, outfile) + def set_output(self, output_path): + self.output_path = output_path + + def get_data(self): + if self.output_path is None: + raise RuntimeError("No data collected.") + return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')]) diff --git a/devlib/collector/perf.py b/devlib/collector/perf.py index 0b83f52..aa4c162 100644 --- a/devlib/collector/perf.py +++ b/devlib/collector/perf.py @@ -19,7 +19,8 @@ import time from past.builtins import basestring, zip from devlib.host import PACKAGE_BIN_DIRECTORY -from devlib.collector import CollectorBase +from devlib.collector import (CollectorBase, CollectorOutput, + CollectorOutputEntry) from devlib.utils.misc import ensure_file_directory_exists as _f @@ -95,6 +96,7 @@ class PerfCollector(CollectorBase): self.force_install = force_install self.labels = labels self.report_options = report_options + self.output_path = None # Validate parameters if isinstance(optionstring, list): @@ -148,14 +150,24 @@ class PerfCollector(CollectorBase): self.target.killall('sleep', as_root=self.target.is_rooted) # NB: we hope that no other "important" sleep is on-going - # pylint: disable=arguments-differ - def get_trace(self, outdir): + def set_output(self, output_path): + self.output_path = output_path + + def get_data(self): + if self.output_path is None: + raise RuntimeError("Output path was not set.") + + output = CollectorOutput() + for label in self.labels: if self.command == 'record': - self._wait_for_data_file_write(label, outdir) - self._pull_target_file_to_host(label, 'rpt', outdir) + self._wait_for_data_file_write(label, self.output_path) + path = self._pull_target_file_to_host(label, 'rpt', self.output_path) + output.append(CollectorOutputEntry(path, 'file')) else: - self._pull_target_file_to_host(label, 'out', outdir) + path = self._pull_target_file_to_host(label, 'out', self.output_path) + output.append(CollectorOutputEntry(path, 'file')) + return output def _deploy_perf(self): host_executable = os.path.join(PACKAGE_BIN_DIRECTORY, @@ -198,13 +210,14 @@ class PerfCollector(CollectorBase): outfile=self._get_target_file(label, 'data')) return command - def _pull_target_file_to_host(self, label, extension, outdir): + def _pull_target_file_to_host(self, label, extension, output_path): target_file = self._get_target_file(label, extension) host_relpath = os.path.basename(target_file) - host_file = _f(os.path.join(outdir, host_relpath)) + host_file = _f(os.path.join(output_path, host_relpath)) self.target.pull(target_file, host_file) + return host_file - def _wait_for_data_file_write(self, label, outdir): + def _wait_for_data_file_write(self, label, output_path): data_file_finished_writing = False max_tries = 80 current_tries = 0 diff --git a/devlib/collector/serial_trace.py b/devlib/collector/serial_trace.py index 5a29f6b..7743956 100644 --- a/devlib/collector/serial_trace.py +++ b/devlib/collector/serial_trace.py @@ -17,7 +17,8 @@ import shutil from tempfile import NamedTemporaryFile from pexpect.exceptions import TIMEOUT -from devlib.collector import CollectorBase +from devlib.collector import (CollectorBase, CollectorOutput, + CollectorOutputEntry) from devlib.utils.serial_port import get_connection @@ -32,33 +33,35 @@ class SerialTraceCollector(CollectorBase): self.serial_port = serial_port self.baudrate = baudrate self.timeout = timeout + self.output_path - None self._serial_target = None self._conn = None - self._tmpfile = None + self._outfile_fh = None self._collecting = False def reset(self): if self._collecting: raise RuntimeError("reset was called whilst collecting") - if self._tmpfile: - self._tmpfile.close() - self._tmpfile = None + if self._outfile_fh: + self._outfile_fh.close() + self._outfile_fh = None def start(self): if self._collecting: raise RuntimeError("start was called whilst collecting") + if self.output_path is None: + raise RuntimeError("Output path was not set.") - - self._tmpfile = NamedTemporaryFile() + self._outfile_fh = open(self.output_path, 'w') start_marker = "-------- Starting serial logging --------\n" - self._tmpfile.write(start_marker.encode('utf-8')) + self._outfile_fh.write(start_marker.encode('utf-8')) self._serial_target, self._conn = get_connection(port=self.serial_port, baudrate=self.baudrate, timeout=self.timeout, - logfile=self._tmpfile, + logfile=self._outfile_fh, init_dtr=0) self._collecting = True @@ -78,17 +81,19 @@ class SerialTraceCollector(CollectorBase): del self._conn stop_marker = "-------- Stopping serial logging --------\n" - self._tmpfile.write(stop_marker.encode('utf-8')) + self._outfile_fh.write(stop_marker.encode('utf-8')) + self._outfile_fh.flush() + self._outfile_fh.close() + self._outfile_fh = None self._collecting = False - def get_trace(self, outfile): + def set_output(self, output_path): + self.output_path = output_path + + def get_data(self): if self._collecting: - raise RuntimeError("get_trace was called whilst collecting") - - self._tmpfile.flush() - - shutil.copy(self._tmpfile.name, outfile) - - self._tmpfile.close() - self._tmpfile = None + raise RuntimeError("get_data was called whilst collecting") + if self.output_path is None: + raise RuntimeError("No data collected.") + return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')]) diff --git a/devlib/collector/systrace.py b/devlib/collector/systrace.py index bdea7c6..538542e 100644 --- a/devlib/collector/systrace.py +++ b/devlib/collector/systrace.py @@ -19,8 +19,9 @@ import subprocess from shutil import copyfile from tempfile import NamedTemporaryFile +from devlib.collector import (CollectorBase, CollectorOutput, + CollectorOutputEntry) from devlib.exception import TargetStableError, HostError -from devlib.collector import CollectorBase import devlib.utils.android from devlib.utils.misc import memoized @@ -74,9 +75,10 @@ class SystraceCollector(CollectorBase): self.categories = categories or DEFAULT_CATEGORIES self.buffer_size = buffer_size + self.output_path = None self._systrace_process = None - self._tmpfile = None + self._outfile_fh = None # Try to find a systrace binary self.systrace_binary = None @@ -104,12 +106,12 @@ class SystraceCollector(CollectorBase): self.reset() def _build_cmd(self): - self._tmpfile = NamedTemporaryFile() + self._outfile_fh = open(self.output_path, 'w') # pylint: disable=attribute-defined-outside-init self.systrace_cmd = 'python2 -u {} -o {} -e {}'.format( self.systrace_binary, - self._tmpfile.name, + self._outfile_fh.name, self.target.adb_name ) @@ -122,13 +124,11 @@ class SystraceCollector(CollectorBase): if self._systrace_process: self.stop() - if self._tmpfile: - self._tmpfile.close() - self._tmpfile = None - def start(self): if self._systrace_process: raise RuntimeError("Tracing is already underway, call stop() first") + if self.output_path is None: + raise RuntimeError("Output path was not set.") self.reset() @@ -151,11 +151,16 @@ class SystraceCollector(CollectorBase): self._systrace_process.communicate('\n') self._systrace_process = None - def get_trace(self, outfile): + if self._outfile_fh: + self._outfile_fh.close() + self._outfile_fh = None + + def set_output(self, output_path): + self.output_path = output_path + + def get_data(self): if self._systrace_process: raise RuntimeError("Tracing is underway, call stop() first") - - if not self._tmpfile: - raise RuntimeError("No tracing data available") - - copyfile(self._tmpfile.name, outfile) + if self.output_path is None: + raise RuntimeError("No data collected.") + return CollectorOutput([CollectorOutputEntry(self.output_path, 'file')])