revent: Updated WA to use the new revent

- Updated the revent parser to handle the new revent format.
- Updated the revent parser to expose device info and recording
  metatdata.
- The parser can now be used in streaming and non-streaming mode (stream
  events from the file as they are being accessed, or read them all in
  at once).
- -g option added to "record" command to expose the gamepad recording
  mode.

wlauto/commands/record.py

@@ -24,7 +24,7 @@ from wlauto.core.resource import NO_ONE
 from wlauto.core.resolver import ResourceResolver
 from wlauto.core.configuration import RunConfiguration
 from wlauto.core.agenda import Agenda
-from wlauto.utils.revent import ReventParser
+from wlauto.utils.revent import ReventRecording, GAMEPAD_MODE
 
 
 class ReventCommand(Command):
@@ -91,6 +91,27 @@ class RecordCommand(ReventCommand):
      - suffix is used by WA to determine which part of the app execution the
        recording is for, currently these are either ``setup`` or ``run``. This
        should be specified with the ``-s`` argument.
+
+
+    **gamepad recording**
+
+    revent supports an alternative recording mode, where it will record events
+    from a single gamepad device. In this mode, revent will store the
+    description of this device as a part of the recording. When replaying such
+    a recording, revent will first create a virtual gamepad using the
+    description, and will replay the events into it, so a physical controller
+    does not need to be connected on replay. Unlike standard revent recordings,
+    recordings generated in this mode should be (to an extent) portable across
+    different devices.
+
+    note:
+
+      - The device on which a recording is being made in gamepad mode, must have
+        exactly one gamepad connected to it.
+      - The device on which a gamepad recording is being replayed must have
+        /dev/uinput enabled in the kernel (this interface is necessary to create
+        virtual gamepad).
+
     '''
 
     def initialize(self, context):
@@ -99,6 +120,8 @@ class RecordCommand(ReventCommand):
         self.parser.add_argument('-s', '--suffix', help='The suffix of the revent file, e.g. ``setup``')
         self.parser.add_argument('-o', '--output', help='Directory to save the recording in')
         self.parser.add_argument('-p', '--package', help='Package to launch before recording')
+        self.parser.add_argument('-g', '--gamepad', help='Record from a gamepad rather than all devices.',
+                                 action="store_true")
         self.parser.add_argument('-C', '--clear', help='Clear app cache before launching it',
                                  action="store_true")
         self.parser.add_argument('-S', '--capture-screen', help='Record a screen capture after recording',
@@ -125,7 +148,8 @@ class RecordCommand(ReventCommand):
 
         self.logger.info("Press Enter when you are ready to record...")
         raw_input("")
-        command = "{} record -s {}".format(self.target_binary, revent_file)
+        gamepad_flag = '-g ' if args.gamepad else ''
+        command = "{} record {}-s {}".format(self.target_binary, gamepad_flag, revent_file)
         self.device.kick_off(command)
 
         self.logger.info("Press Enter when you have finished recording...")
@@ -172,8 +196,11 @@ class ReplayCommand(ReventCommand):
 
         self.logger.info("Replaying recording")
         command = "{} replay {}".format(self.target_binary, revent_file)
-        timeout = ceil(ReventParser.get_revent_duration(args.revent)) + 30
-        self.device.execute(command, timeout=timeout)
+        recording = ReventRecording(args.revent)
+        timeout = ceil(recording.duration) + 30
+        recording.close()
+        self.device.execute(command, timeout=timeout,
+                            as_root=(recording.mode == GAMEPAD_MODE))
         self.logger.info("Finished replay")
 
 

wlauto/common/android/workload.py

@@ -28,7 +28,7 @@ from wlauto.common.resources import ExtensionAsset, Executable, File
 from wlauto.exceptions import WorkloadError, ResourceError, DeviceError
 from wlauto.utils.android import ApkInfo, ANDROID_NORMAL_PERMISSIONS, UNSUPPORTED_PACKAGES
 from wlauto.utils.types import boolean
-from wlauto.utils.revent import ReventParser
+from wlauto.utils.revent import ReventRecording
 import wlauto.utils.statedetect as state_detector
 import wlauto.common.android.resources
 
@@ -466,8 +466,8 @@ class ReventWorkload(Workload):
         self.on_device_run_revent = devpath.join(self.device.working_directory,
                                                  os.path.split(self.revent_run_file)[-1])
         self._check_revent_files(context)
-        default_setup_timeout = ceil(ReventParser.get_revent_duration(self.revent_setup_file)) + 30
-        default_run_timeout = ceil(ReventParser.get_revent_duration(self.revent_run_file)) + 30
+        default_setup_timeout = ceil(ReventRecording(self.revent_setup_file).duration) + 30
+        default_run_timeout = ceil(ReventRecording(self.revent_run_file).duration) + 30
         self.setup_timeout = self.setup_timeout or default_setup_timeout
         self.run_timeout = self.run_timeout or default_run_timeout
 

wlauto/resource_getters/standard.py

@@ -33,7 +33,7 @@ from wlauto.exceptions import ResourceError
 from wlauto.utils.android import ApkInfo
 from wlauto.utils.misc import ensure_directory_exists as _d, ensure_file_directory_exists as _f, sha256, urljoin
 from wlauto.utils.types import boolean
-from wlauto.utils.revent import ReventParser
+from wlauto.utils.revent import ReventRecording
 
 
 logging.getLogger("requests").setLevel(logging.WARNING)
@@ -101,7 +101,7 @@ class ReventGetter(ResourceGetter):
                 if candidate.lower() == filename.lower():
                     path = os.path.join(location, candidate)
                     try:
-                        ReventParser.check_revent_file(path)
+                        ReventRecording(path).close()  # Check valid recording
                         return path
                     except ValueError as e:
                         self.logger.warning(e.message)
@@ -437,7 +437,7 @@ class HttpGetter(ResourceGetter):
                     pathname = os.path.basename(asset['path']).lower()
                     if pathname == filename:
                         try:
-                            ReventParser.check_revent_file(asset['path'])
+                            ReventRecording(asset['path']).close()  # Check valid recording
                             return asset
                         except ValueError as e:
                             self.logger.warning(e.message)
@@ -535,7 +535,7 @@ class RemoteFilerGetter(ResourceGetter):
                             path = os.path.join(location, candidate)
                 if path:
                     try:
-                        ReventParser.check_revent_file(path)
+                        ReventRecording(path).close()  # Check valid recording
                         return path
                     except ValueError as e:
                         self.logger.warning(e.message)

wlauto/utils/revent.py

@@ -13,66 +13,235 @@
 # limitations under the License.
 #
 
-import struct
-import datetime
 import os
+import struct
+from datetime import datetime
+from collections import namedtuple
 
 
-class ReventParser(object):
-    """
-    Parses revent binary recording files so they can be easily read within python.
-    """
-
-    int32_struct = struct.Struct("<i")
-    header_struct = struct.Struct("<6sH")
-    event_struct = struct.Struct("<i4xqqHHi")
-
-    def __init__(self):
-        self.path = None
-        self.device_paths = []
-
-    def parse(self, path):
-        ReventParser.check_revent_file(path)
-
-        with open(path, "rb") as f:
-            _read_struct(f, ReventParser.header_struct)
-            path_count, = _read_struct(f, self.int32_struct)
-            for _ in xrange(path_count):
-                path_length, = _read_struct(f, self.int32_struct)
-                if path_length >= 30:
-                    raise ValueError("path length too long. corrupt file")
-                self.device_paths.append(f.read(path_length))
-
-            while f.tell() < os.path.getsize(path):
-                device_id, sec, usec, typ, code, value = _read_struct(f, self.event_struct)
-                yield (device_id, datetime.datetime.fromtimestamp(sec + float(usec) / 1000000),
-                       typ, code, value)
-
-    @staticmethod
-    def check_revent_file(path):
-        """
-        Checks whether a file starts with "REVENT"
-        """
-        with open(path, "rb") as f:
-            magic, file_version = _read_struct(f, ReventParser.header_struct)
-
-            if magic != "REVENT":
-                msg = "'{}' isn't an revent file, are you using an old recording?"
-                raise ValueError(msg.format(path))
-            return file_version
-
-    @staticmethod
-    def get_revent_duration(path):
-        """
-        Takes an ReventParser and returns the duration of the revent recording in seconds.
-        """
-        revent_parser = ReventParser().parse(path)
-        first = last = next(revent_parser)
-        for last in revent_parser:
-            pass
-        return (last[1] - first[1]).total_seconds()
+GENERAL_MODE = 0
+GAMEPAD_MODE = 1
 
 
-def _read_struct(f, struct_spec):
-    data = f.read(struct_spec.size)
+u16_struct = struct.Struct('<H')
+u32_struct = struct.Struct('<I')
+u64_struct = struct.Struct('<Q')
+
+# See revent section in WA documentation for the detailed description of
+# the recording format.
+header_one_struct = struct.Struct('<6sH')
+header_two_struct = struct.Struct('<H6x')  # version 2 onwards
+
+devid_struct = struct.Struct('<4H')
+devinfo_struct = struct.Struct('<4s96s96s96sI')
+absinfo_struct = struct.Struct('<7i')
+
+event_struct = struct.Struct('<HqqHHi')
+old_event_struct = struct.Struct("<i4xqqHHi")  # prior to version 2
+
+
+def read_struct(fh, struct_spec):
+    data = fh.read(struct_spec.size)
     return struct_spec.unpack(data)
+
+
+def read_string(fh):
+    length, = read_struct(fh, u32_struct)
+    str_struct = struct.Struct('<{}s'.format(length))
+    return read_struct(fh, str_struct)[0]
+
+
+def count_bits(bitarr):
+    return sum(bin(b).count('1') for b in bitarr)
+
+
+def is_set(bitarr, bit):
+    byte = bit // 8
+    bytebit = bit % 8
+    return bitarr[byte] & bytebit
+
+
+absinfo = namedtuple('absinfo', 'ev_code value min max fuzz flat resolution')
+
+
+class UinputDeviceInfo(object):
+
+    def __init__(self, fh):
+        parts = read_struct(fh, devid_struct)
+        self.bustype = parts[0]
+        self.vendor = parts[1]
+        self.product = parts[2]
+        self.version = parts[3]
+
+        self.name = read_string(fh)
+
+        parts = read_struct(fh, devinfo_struct)
+        self.ev_bits = bytearray(parts[0])
+        self.key_bits = bytearray(parts[1])
+        self.rel_bits = bytearray(parts[2])
+        self.abs_bits = bytearray(parts[3])
+        self.num_absinfo = parts[4]
+        self.absinfo = [absinfo(*read_struct(fh, absinfo_struct))
+                        for _ in xrange(self.num_absinfo)]
+
+    def __str__(self):
+        return 'UInputInfo({})'.format(self.__dict__)
+
+
+class ReventEvent(object):
+
+    def __init__(self, fh, legacy=False):
+        if not legacy:
+            dev_id, ts_sec, ts_usec, type_, code, value = read_struct(fh, event_struct)
+        else:
+            dev_id, ts_sec, ts_usec, type_, code, value = read_struct(fh, old_event_struct)
+        self.device_id = dev_id
+        self.time = datetime.fromtimestamp(ts_sec + float(ts_usec) / 1000000)
+        self.type = type_
+        self.code = code
+        self.value = value
+
+    def __str__(self):
+        return 'InputEvent({})'.format(self.__dict__)
+
+
+class ReventRecording(object):
+    """
+    Represents a parsed revent recording. This contains input events and device
+    descriptions recorded by revent. Two parsing modes are supported. By
+    default, the recording will be parsed in the "streaming" mode. In this
+    mode, initial headers and device descritions are parsed on creation and an
+    open file handle to the recording is saved. Events will be read from the
+    file as they are being iterated over. In this mode, the entire recording is
+    never loaded into memory at once. The underlying file may be "released" by
+    calling ``close`` on the recroding, after which further iteration over the
+    events will not be possible (but would still be possible to access the file
+    description and header information).
+
+    The alternative is to load the entire recording on creation (in which case
+    the file handle will be closed once the recroding is loaded). This can be
+    enabled by specifying ``streaming=False``. This will make it faster to
+    subsequently iterate over the events, and also will not "hold" the file
+    open.
+
+    .. note:: When starting a new iteration over the events in streaming mode,
+              the postion in the open file will be automatically reset to the
+              beginning of the event stream. This means it's possible to iterate
+              over the events multiple times without having to re-open the
+              recording, however it is not possible to do so in parallel. If
+              parallel iteration is required, streaming should be disabled.
+
+    """
+
+    @property
+    def duration(self):
+        if self._duration is None:
+            if self.stream:
+                events = self._iter_events()
+                try:
+                    first = last = events.next()
+                except StopIteration:
+                    self._duration = 0
+                for last in events:
+                    pass
+                self._duration = (last.time - first.time).total_seconds()
+            else:  # not streaming
+                if not self._events:
+                    self._duration = 0
+                self._duration = (self._events[-1].time -
+                                  self._events[0].time).total_seconds()
+        return self._duration
+
+    @property
+    def events(self):
+        if self.stream:
+            return self._iter_events()
+        else:
+            return self._events
+
+    def __init__(self, f, stream=True):
+        self.device_paths = []
+        self.gamepad_device = None
+        self.num_events = None
+        self.stream = stream
+        self._events = None
+        self._close_when_done = False
+        self._events_start = None
+        self._duration = None
+
+        if hasattr(f, 'name'):  # file-like object
+            self.filepath = f.name
+            self.fh = f
+        else:  # path to file
+            self.filepath = f
+            self.fh = open(self.filepath, 'rb')
+            if not self.stream:
+                self._close_when_done = True
+        try:
+            self._parse_header_and_devices(self.fh)
+            self._events_start = self.fh.tell()
+            if not self.stream:
+                self._events = [e for e in self._iter_events()]
+        finally:
+            if self._close_when_done:
+                self.close()
+
+    def close(self):
+        if self.fh is not None:
+            self.fh.close()
+            self.fh = None
+            self._events_start = None
+
+    def _parse_header_and_devices(self, fh):
+        magic, version = read_struct(fh, header_one_struct)
+        if magic != 'REVENT':
+            raise ValueError('{} does not appear to be an revent recording'.format(self.filepath))
+        self.version = version
+
+        if self.version == 2:
+            self.mode, = read_struct(fh, header_two_struct)
+            if self.mode == GENERAL_MODE:
+                self._read_devices(fh)
+            elif self.mode == GAMEPAD_MODE:
+                self._read_gamepad_info(fh)
+            else:
+                raise ValueError('Unexpected recording mode: {}'.format(self.mode))
+            self.num_events, = read_struct(fh, u64_struct)
+        elif 2 > self.version >= 0:
+            self._read_devices(fh)
+        else:
+            raise ValueError('Invalid recording version: {}'.format(self.version))
+
+    def _read_devices(self, fh):
+        num_devices, = read_struct(fh, u32_struct)
+        for _ in xrange(num_devices):
+            self.device_paths.append(read_string(fh))
+
+    def _read_gamepad_info(self, fh):
+        self.gamepad_device = UinputDeviceInfo(fh)
+        self.device_paths.append('[GAMEPAD]')
+
+    def _iter_events(self):
+        if self.fh is None:
+            raise RuntimeError('Attempting to iterate over events of a closed recording')
+        self.fh.seek(self._events_start)
+        if self.version >= 2:
+            for _ in xrange(self.num_events):
+                yield ReventEvent(self.fh)
+        else:
+            file_size = os.path.getsize(self.filepath)
+            while self.fh.tell() < file_size:
+                yield ReventEvent(self.fh, legacy=True)
+
+    def __iter__(self):
+        for event in self.events:
+            yield event
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, *args):
+        self.close()
+
+    def __del__(self):
+        self.close()