Gem5Device: Refactor into BaseGem5Device, Gem5LinuxDevice and Gem5AndroidDevice

- Refactored the Gem5Device to avoid duplicated code. There is now a
  BaseGem5Device which includes all of the shared functionality. The
  Gem5LinuxDevice and the Gem5AndroidDevice both inherit from
  BaseGem5Device, and LinuxDevice or AndroidDevice, respectively.

wlauto/common/gem5/__init__.py

@@ -0,0 +1,16 @@
+#    Copyright 2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+

wlauto/common/gem5/device.py

@@ -0,0 +1,648 @@
+#    Copyright 2014-2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Original implementation by Rene de Jong. Updated by Sascha Bischoff.
+
+# pylint: disable=E1101
+
+import logging
+import os
+import re
+import shutil
+import socket
+import subprocess
+import sys
+import tarfile
+import time
+from pexpect import EOF, TIMEOUT
+
+from wlauto import settings, Parameter
+from wlauto.core import signal as sig
+from wlauto.exceptions import DeviceError
+from wlauto.utils import ssh, types
+
+
+class BaseGem5Device(object):
+    """
+    Base implementation for a gem5-based device
+
+    This class is used as the base class for OS-specific devices such as the
+    G3m5LinuxDevice and the Gem5AndroidDevice. The majority of the gem5-specific
+    functionality is included here.
+
+    Note: When inheriting from this class, make sure to inherit from this class
+    prior to inheriting from the OS-specific class, i.e. LinuxDevice, to ensure
+    that the methods are correctly overridden.
+    """
+    # gem5 can be very slow. Hence, we use some very long timeouts!
+    delay = 3600
+    long_delay = 3 * delay
+    ready_timeout = long_delay
+    default_timeout = delay
+
+    platform = None
+    path_module = 'posixpath'
+
+    parameters = [
+        Parameter('gem5_binary', kind=str, default='./build/ARM/gem5.fast',
+                  mandatory=False, description="Command used to execute gem5. "
+                  "Adjust according to needs."),
+        Parameter('gem5_args', kind=types.arguments, mandatory=True,
+                  description="Command line passed to the gem5 simulation. This"
+                  " command line is used to set up the simulated system, and "
+                  "should be the same as used for a standard gem5 simulation "
+                  "without workload automation. Note that this is simulation "
+                  "script specific and will hence need to be tailored to each "
+                  "particular use case."),
+        Parameter('gem5_vio_args', kind=types.arguments, mandatory=True,
+                  constraint=lambda x: "{}" in str(x),
+                  description="gem5 VirtIO command line used to enable the "
+                  "VirtIO device in the simulated system. At the very least, "
+                  "the root parameter of the VirtIO9PDiod device must be "
+                  "exposed on the command line. Please set this root mount to "
+                  "{}, as it will be replaced with the directory used by "
+                  "Workload Automation at runtime."),
+        Parameter('temp_dir', kind=str, default='/tmp',
+                  description="Temporary directory used to pass files into the "
+                  "gem5 simulation. Workload Automation will automatically "
+                  "create a directory in this folder, and will remove it again "
+                  "once the simulation completes."),
+        Parameter('checkpoint', kind=bool, default=False,
+                  mandatory=False, description="This parameter "
+                  "tells Workload Automation to create a checkpoint of the "
+                  "simulated system once the guest system has finished booting."
+                  " This checkpoint can then be used at a later stage by other "
+                  "WA runs to avoid booting the guest system a second time. Set"
+                  " to True to take a checkpoint of the simulated system post "
+                  "boot."),
+        Parameter('run_delay', kind=int, default=0, mandatory=False,
+                  constraint=lambda x: x >= 0,
+                  description="This sets the time that the "
+                  "system should sleep in the simulated system prior to "
+                  "running and workloads or taking checkpoints. This allows "
+                  "the system to quieten down prior to running the workloads. "
+                  "When this is combined with the checkpoint_post_boot"
+                  " option, it allows the checkpoint to be created post-sleep,"
+                  " and therefore the set of workloads resuming from this "
+                  "checkpoint will not be required to sleep.")
+    ]
+
+    @property
+    def is_rooted(self):  # pylint: disable=R0201
+        # gem5 is always rooted
+        return True
+
+    # pylint: disable=E0203
+    def __init__(self, _):
+        self.logger = logging.getLogger('gem5Device')
+
+        # The gem5 subprocess
+        self.gem5 = None
+        self.gem5_port = -1
+        self.gem5outdir = os.path.join(settings.output_directory, "gem5")
+        self.stdout_file = None
+        self.stderr_file = None
+        self.stderr_filename = None
+        self.sckt = None
+
+        # Find the first one that does not exist. Ensures that we do not re-use
+        # the directory used by someone else.
+        for i in xrange(sys.maxint):
+            directory = os.path.join(self.temp_dir, "wa_{}".format(i))
+            try:
+                os.stat(directory)
+                continue
+            except OSError:
+                break
+        self.temp_dir = directory
+        self.logger.debug("Using {} as the temporary directory.".format(self.temp_dir))
+
+        # Start the gem5 simulation when WA starts a run using a signal.
+        sig.connect(self.init_gem5, sig.RUN_START)
+
+    def validate(self):
+        # Assemble the virtio args
+        self.gem5_vio_args = str(self.gem5_vio_args).format(self.temp_dir)  # pylint: disable=W0201
+        self.logger.debug("gem5 VirtIO command: {}".format(self.gem5_vio_args))
+
+    def init_gem5(self, _):
+        """
+        Start gem5, find out the telnet port and connect to the simulation.
+
+        We first create the temporary directory used by VirtIO to pass files
+        into the simulation, as well as the gem5 output directory.We then create
+        files for the standard output and error for the gem5 process. The gem5
+        process then is started.
+        """
+        self.logger.info("Creating temporary directory: {}".format(self.temp_dir))
+        os.mkdir(self.temp_dir)
+        os.mkdir(self.gem5outdir)
+
+        # We need to redirect the standard output and standard error for the
+        # gem5 process to a file so that we can debug when things go wrong.
+        f = os.path.join(self.gem5outdir, 'stdout')
+        self.stdout_file = open(f, 'w')
+        f = os.path.join(self.gem5outdir, 'stderr')
+        self.stderr_file = open(f, 'w')
+        # We need to keep this so we can check which port to use for the telnet
+        # connection.
+        self.stderr_filename = f
+
+        self.start_gem5()
+
+    def start_gem5(self):
+        """
+        Starts the gem5 simulator, and parses the output to get the telnet port.
+        """
+        self.logger.info("Starting the gem5 simulator")
+
+        command_line = "{} --outdir={}/gem5 {} {}".format(self.gem5_binary,
+                                                          settings.output_directory,
+                                                          self.gem5_args,
+                                                          self.gem5_vio_args)
+        self.logger.debug("gem5 command line: {}".format(command_line))
+        self.gem5 = subprocess.Popen(command_line.split(),
+                                     stdout=self.stdout_file,
+                                     stderr=self.stderr_file)
+
+        while self.gem5_port == -1:
+            # Check that gem5 is running!
+            if self.gem5.poll():
+                raise DeviceError("The gem5 process has crashed with error code {}!".format(self.gem5.poll()))
+
+            # Open the stderr file
+            f = open(self.stderr_filename, 'r')
+            for line in f:
+                m = re.search(r"Listening\ for\ system\ connection\ on\ port\ (?P<port>\d+)", line)
+                if m:
+                    port = int(m.group('port'))
+                    if port >= 3456 and port < 5900:
+                        self.gem5_port = port
+                        f.close()
+                        break
+            else:
+                time.sleep(1)
+            f.close()
+
+    def connect(self):  # pylint: disable=R0912,W0201
+        """
+        Connect to the gem5 simulation and wait for Android to boot. Then,
+        create checkpoints, and mount the VirtIO device.
+        """
+        self.connect_gem5()
+
+        self.wait_for_boot()
+
+        if self.run_delay:
+            self.logger.info("Sleeping for {} seconds in the guest".format(self.run_delay))
+            self.gem5_shell("sleep {}".format(self.run_delay))
+
+        if self.checkpoint:
+            self.checkpoint_gem5()
+
+        self.mount_virtio()
+        self.logger.info("Creating the working directory in the simulated system")
+        self.gem5_shell('mkdir -p {}'.format(self.working_directory))
+        self._is_ready = True  # pylint: disable=W0201
+
+    def wait_for_boot(self):
+        pass
+
+    def connect_gem5(self):  # pylint: disable=R0912
+        """
+        Connect to the telnet port of the gem5 simulation.
+
+        We connect, and wait for the prompt to be found. We do not use a timeout
+        for this, and wait for the prompt in a while loop as the gem5 simulation
+        can take many hours to reach a prompt when booting the system. We also
+        inject some newlines periodically to try and force gem5 to show a
+        prompt. Once the prompt has been found, we replace it with a unique
+        prompt to ensure that we are able to match it properly. We also disable
+        the echo as this simplifies parsing the output when executing commands
+        on the device.
+        """
+        self.logger.info("Connecting to the gem5 simulation on port {}".format(self.gem5_port))
+        host = socket.gethostname()
+        port = self.gem5_port
+
+        # Connect to the gem5 telnet port. Use a short timeout here.
+        self.sckt = ssh.TelnetConnection()
+        self.sckt.login(host, 'None', port=port, auto_prompt_reset=False,
+                        login_timeout=10)
+
+        self.logger.info("Connected! Waiting for prompt...")
+
+        # We need to find the prompt. It might be different if we are resuming
+        # from a checkpoint. Therefore, we test multiple options here.
+        prompt_found = False
+        while not prompt_found:
+            try:
+                self.login_to_device()
+            except TIMEOUT:
+                pass
+            try:
+                # Try and force a prompt to be shown
+                self.sckt.send('\n')
+                self.sckt.expect([r'# ', self.sckt.UNIQUE_PROMPT, r'\[PEXPECT\][\\\$\#]+ '], timeout=60)
+                prompt_found = True
+            except TIMEOUT:
+                pass
+
+        self.logger.info("Setting unique prompt...")
+
+        self.sckt.set_unique_prompt()
+        self.sckt.prompt()
+        self.logger.info("Prompt found and replaced with a unique string")
+
+        # We check that the prompt is what we think it should be. If not, we
+        # need to update the regex we use to match.
+        self.find_prompt()
+
+        self.sckt.setecho(False)
+        self.sync_gem5_shell()
+
+        # Try and avoid line wrapping as much as possible. Don't check the error
+        # codes from these command because some of them WILL fail.
+        self.gem5_shell('stty columns 1024', check_exit_code=False)
+        self.gem5_shell('busybox stty columns 1024', check_exit_code=False)
+        self.gem5_shell('stty cols 1024', check_exit_code=False)
+        self.gem5_shell('busybox stty cols 1024', check_exit_code=False)
+        self.gem5_shell('reset', check_exit_code=False)
+
+    def get_properties(self, context):  # pylint: disable=R0801
+        """ Get the property files from the device """
+        for propfile in self.property_files:
+            try:
+                normname = propfile.lstrip(self.path.sep).replace(self.path.sep, '.')
+                outfile = os.path.join(context.host_working_directory, normname)
+                if self.is_file(propfile):
+                    self.execute('cat {} > {}'.format(propfile, normname))
+                    self.pull_file(normname, outfile)
+                elif self.is_directory(propfile):
+                    self.get_directory(context, propfile)
+                    continue
+                else:
+                    continue
+            except DeviceError:
+                # We pull these files "opportunistically", so if a pull fails
+                # (e.g. we don't have permissions to read the file), just note
+                # it quietly (not as an error/warning) and move on.
+                self.logger.debug('Could not pull property file "{}"'.format(propfile))
+        return {}
+
+    def get_directory(self, context, directory):
+        """ Pull a directory from the device """
+        normname = directory.lstrip(self.path.sep).replace(self.path.sep, '.')
+        outdir = os.path.join(context.host_working_directory, normname)
+        temp_file = os.path.join(context.host_working_directory, "{}.tar".format(normname))
+        # Check that the folder exists
+        self.gem5_shell("ls -la {}".format(directory))
+        # Compress the folder
+        try:
+            self.gem5_shell("{} tar -cvf {}.tar {}".format(self.busybox, normname, directory))
+        except DeviceError:
+            self.logger.debug("Failed to run tar command on device! Not pulling {}".format(directory))
+            return
+        self.pull_file(normname, temp_file)
+        f = tarfile.open(temp_file, 'r')
+        os.mkdir(outdir)
+        f.extractall(outdir)
+        os.remove(temp_file)
+
+    def get_pids_of(self, process_name):
+        """ Returns a list of PIDs of all processes with the specified name. """
+        result = self.gem5_shell('ps | busybox grep {}'.format(process_name),
+                                 check_exit_code=False).strip()
+        if result and 'not found' not in result and len(result.split('\n')) > 2:
+            return [int(x.split()[1]) for x in result.split('\n')]
+        else:
+            return []
+
+    def find_prompt(self):
+        prompt = r'\[PEXPECT\][\\\$\#]+ '
+        synced = False
+        while not synced:
+            self.sckt.send('\n')
+            i = self.sckt.expect([prompt, self.sckt.UNIQUE_PROMPT, r'[\$\#] '], timeout=self.delay)
+            if i == 0:
+                synced = True
+            elif i == 1:
+                prompt = self.sckt.UNIQUE_PROMPT
+                synced = True
+            else:
+                prompt = re.sub(r'\$', r'\\\$', self.sckt.before.strip() + self.sckt.after.strip())
+                prompt = re.sub(r'\#', r'\\\#', prompt)
+                prompt = re.sub(r'\[', r'\[', prompt)
+                prompt = re.sub(r'\]', r'\]', prompt)
+
+        self.sckt.PROMPT = prompt
+
+    def login(self):
+        pass
+
+    def close(self):
+        if self._logcat_poller:
+            self._logcat_poller.stop()
+
+    def reset(self):
+        self.logger.warn("Attempt to restart the gem5 device. This is not "
+                         "supported!")
+
+    def init(self):
+        pass
+
+    def push_file(self, source, dest, _):
+        """
+        Push a file to the gem5 device using VirtIO
+
+        The file to push to the device is copied to the temporary directory on
+        the host, before being copied within the simulation to the destination.
+        Checks, in the form of 'ls' with error code checking, are performed to
+        ensure that the file is copied to the destination.
+        """
+        filename = os.path.basename(source)
+        self.logger.debug("Pushing {} to device.".format(source))
+        self.logger.debug("temp_dir: {}".format(self.temp_dir))
+        self.logger.debug("dest: {}".format(dest))
+        self.logger.debug("filename: {}".format(filename))
+
+        # We need to copy the file to copy to the temporary directory
+        self.move_to_temp_dir(source)
+
+        # Back to the gem5 world
+        self.gem5_shell("ls -al /mnt/obb/{}".format(filename))
+        self.gem5_shell("busybox cp /mnt/obb/{} {}".format(filename, dest))
+        self.gem5_shell("busybox sync")
+        self.gem5_shell("ls -al {}".format(dest))
+        self.gem5_shell("ls -al /mnt/obb/")
+        self.logger.debug("Push complete.")
+
+    def pull_file(self, source, dest, _):
+        """
+        Pull a file from the gem5 device using m5 writefile
+
+        First, we check the extension of the file to be copied. If the file ends
+        in .gz, then gem5 wrongly assumes that it should create a gzipped output
+        stream, which results in a gem5 error. Therefore, we rename the file on
+        the local device prior to the writefile command when required. Next, the
+        file is copied to the local directory within the guest as the m5
+        writefile command assumes that the file is local. The file is then
+        written out to the host system using writefile, prior to being moved to
+        the destination on the host.
+        """
+        filename = os.path.basename(source)
+        self.logger.debug("Pulling {} from device.".format(filename))
+
+        # gem5 assumes that files ending in .gz are gzip-compressed. We need to
+        # work around this, else gem5 panics on us. Rename the file for use in
+        # gem5
+        if filename[-3:] == '.gz':
+            filename += '.fugem5'
+
+        self.logger.debug("pull_file {} {}".format(source, filename))
+        # We don't check the exit code here because it is non-zero if the source
+        # and destination are the same. The ls below will cause an error if the
+        # file was not where we expected it to be.
+        self.gem5_shell("busybox cp {} {}".format(source, filename), check_exit_code=False)
+        self.gem5_shell("busybox sync")
+        self.gem5_shell("ls -la {}".format(filename))
+        self.logger.debug('Finished the copy in the simulator')
+        self.gem5_util("writefile {}".format(filename))
+
+        if 'cpu' not in filename:
+            while not os.path.exists(os.path.join(self.gem5outdir, filename)):
+                time.sleep(1)
+
+        # Perform the local move
+        shutil.move(os.path.join(self.gem5outdir, filename), dest)
+        self.logger.debug("Pull complete.")
+
+    def delete_file(self, filepath, _):
+        """ Delete a file on the device """
+        self._check_ready()
+        self.gem5_shell("rm '{}'".format(filepath))
+
+    def file_exists(self, filepath):
+        """ Check if a file exists """
+        self._check_ready()
+        output = self.gem5_shell('if [ -e \'{}\' ]; then echo 1; else echo 0; fi'.format(filepath))
+        try:
+            if int(output):
+                return True
+            else:
+                return False
+        except ValueError:
+            # If we cannot process the output, assume that there is no file
+            return False
+
+    def disconnect(self):
+        """
+        Close and disconnect from the gem5 simulation. Additionally, we remove
+        the temporary directory used to pass files into the simulation.
+        """
+        self.logger.info("Gracefully terminating the gem5 simulation.")
+        try:
+            self.gem5_util("exit")
+            self.gem5.wait()
+        except EOF:
+            pass
+        self.logger.info("Removing the temporary directory")
+        try:
+            shutil.rmtree(self.temp_dir)
+        except OSError:
+            self.logger.warn("Failed to remove the temporary directory!")
+
+    # gem5 might be slow. Hence, we need to make the ping timeout very long.
+    def ping(self):
+        self.logger.debug("Pinging gem5 to see if it is still alive")
+        self.gem5_shell('ls /', timeout=self.longdelay)
+
+    # Additional Android-specific methods.
+    def forward_port(self, _):  # pylint: disable=R0201
+        raise DeviceError('we do not need forwarding')
+
+    # gem5 should dump out a framebuffer. We can use this if it exists. Failing
+    # that, fall back to the parent class implementation.
+    def capture_screen(self, filepath):
+        file_list = os.listdir(self.gem5outdir)
+        screen_caps = []
+        for f in file_list:
+            if '.bmp' in f:
+                screen_caps.append(f)
+
+        if len(screen_caps) == 1:
+            # Bail out if we do not have image, and resort to the slower, built
+            # in method.
+            try:
+                import Image
+                gem5_image = os.path.join(self.gem5outdir, screen_caps[0])
+                temp_image = os.path.join(self.gem5outdir, "file.png")
+                im = Image.open(gem5_image)
+                im.save(temp_image, "PNG")
+                shutil.copy(temp_image, filepath)
+                os.remove(temp_image)
+                self.logger.debug("capture_screen: using gem5 screencap")
+                return True
+            except (shutil.Error, ImportError, IOError):
+                pass
+        return False
+
+    # pylint: disable=W0613
+    def execute(self, command, timeout=1000, check_exit_code=True, background=False,
+                as_root=False, busybox=False, **kwargs):
+        self._check_ready()
+        if as_root and not self.is_rooted:
+            raise DeviceError('Attempting to execute "{}" as root on unrooted device.'.format(command))
+        if busybox:
+            if not self.is_rooted:
+                raise DeviceError('Attempting to execute "{}" with busybox. '.format(command) +
+                                  'Busybox can only be deployed to rooted devices.')
+            command = ' '.join([self.busybox, command])
+        if background:
+            self.logger.debug("Attempt to execute in background. Not supported "
+                              "in gem5, hence ignored.")
+        return self.gem5_shell(command, as_root=as_root)
+
+    # Internal methods: do not use outside of the class.
+
+    def _check_ready(self):
+        """
+        Check if the device is ready.
+
+        As this is gem5, we just assume that the device is ready once we have
+        connected to the gem5 simulation, and updated the prompt.
+        """
+        if not self._is_ready:
+            raise DeviceError('Device not ready.')
+
+    def gem5_shell(self, command, as_root=False, timeout=None, check_exit_code=True, sync=True):  # pylint: disable=R0912
+        """
+        Execute a command in the gem5 shell
+
+        This wraps the telnet connection to gem5 and processes the raw output.
+
+        This method waits for the shell to return, and then will try and
+        separate the output from the command from the command itself. If this
+        fails, warn, but continue with the potentially wrong output.
+
+        The exit code is also checked by default, and non-zero exit codes will
+        raise a DeviceError.
+        """
+        conn = self.sckt
+        if sync:
+            self.sync_gem5_shell()
+
+        self.logger.debug("gem5_shell command: {}".format(command))
+
+        # Send the actual command
+        conn.send("{}\n".format(command))
+
+        # Wait for the response. We just sit here and wait for the prompt to
+        # appear, as gem5 might take a long time to provide the output. This
+        # avoids timeout issues.
+        command_index = -1
+        while command_index == -1:
+            if conn.prompt():
+                output = re.sub(r' \r([^\n])', r'\1', conn.before)
+                output = re.sub(r'[\b]', r'', output)
+                # Deal with line wrapping
+                output = re.sub(r'[\r].+?<', r'', output)
+                command_index = output.find(command)
+
+                # If we have -1, then we cannot match the command, but the
+                # prompt has returned. Hence, we have a bit of an issue. We
+                # warn, and return the whole output.
+                if command_index == -1:
+                    self.logger.warn("gem5_shell: Unable to match command in "
+                                     "command output. Expect parsing errors!")
+                    command_index = 0
+
+        output = output[command_index + len(command):].strip()
+
+        # It is possible that gem5 will echo the command. Therefore, we need to
+        # remove that too!
+        command_index = output.find(command)
+        if command_index != -1:
+            output = output[command_index + len(command):].strip()
+
+        self.logger.debug("gem5_shell output: {}".format(output))
+
+        # We get a second prompt. Hence, we need to eat one to make sure that we
+        # stay in sync. If we do not do this, we risk getting out of sync for
+        # slower simulations.
+        self.sckt.expect([self.sckt.UNIQUE_PROMPT, self.sckt.PROMPT], timeout=self.delay)
+
+        if check_exit_code:
+            exit_code_text = self.gem5_shell('echo $?', as_root=as_root,
+                                             timeout=timeout, check_exit_code=False,
+                                             sync=False)
+            try:
+                exit_code = int(exit_code_text.split()[0])
+                if exit_code:
+                    message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}'
+                    raise DeviceError(message.format(exit_code, command, output))
+            except (ValueError, IndexError):
+                self.logger.warning('Could not get exit code for "{}",\ngot: "{}"'.format(command, exit_code_text))
+
+        return output
+
+    def gem5_util(self, command):
+        """ Execute a gem5 utility command using the m5 binary on the device """
+        self.gem5_shell('/sbin/m5 ' + command)
+
+    def sync_gem5_shell(self):
+        """
+        Synchronise with the gem5 shell.
+
+        Write some unique text to the gem5 device to allow us to synchronise
+        with the shell output. We actually get two prompts so we need to match
+        both of these.
+        """
+        self.logger.debug("Sending Sync")
+        self.sckt.send("echo \*\*sync\*\*\n")
+        self.sckt.expect(r"\*\*sync\*\*", timeout=self.delay)
+        self.sckt.expect([self.sckt.UNIQUE_PROMPT, self.sckt.PROMPT], timeout=self.delay)
+        self.sckt.expect([self.sckt.UNIQUE_PROMPT, self.sckt.PROMPT], timeout=self.delay)
+
+    def move_to_temp_dir(self, source):
+        """
+        Move a file to the temporary directory on the host for copying to the
+        gem5 device
+        """
+        command = "cp {} {}".format(source, self.temp_dir)
+        self.logger.debug("Local copy command: {}".format(command))
+        subprocess.call(command.split())
+        subprocess.call("sync".split())
+
+    def checkpoint_gem5(self, end_simulation=False):
+        """ Checkpoint the gem5 simulation, storing all system state """
+        self.logger.info("Taking a post-boot checkpoint")
+        self.gem5_util("checkpoint")
+        if end_simulation:
+            self.disconnect()
+
+    def mount_virtio(self):
+        """
+        Mount the VirtIO device in the simulated system.
+        """
+        self.logger.info("Mounting VirtIO device in simulated system")
+
+        self.gem5_shell('busybox mkdir -p /mnt/obb')
+
+        mount_command = "mount -t 9p -o trans=virtio,version=9p2000.L,aname={} gem5 /mnt/obb".format(self.temp_dir)
+        if self.platform == 'linux':
+            self.gem5_shell(mount_command)
+        else:
+            self.gem5_shell('busybox {}'.format(mount_command))

wlauto/devices/android/gem5/__init__.py

@@ -17,22 +17,14 @@
 
 import logging
 import os
-import re
-import shutil
-import socket
-import subprocess
-import sys
-import tarfile
 import time
-from pexpect import EOF, TIMEOUT
 
-from wlauto import AndroidDevice, settings, Parameter
+from wlauto import AndroidDevice, Parameter
-from wlauto.core import signal as sig
+from wlauto.common.gem5.device import BaseGem5Device
-from wlauto.utils import ssh, types
+from wlauto.exceptions import DeviceError
-from wlauto.exceptions import DeviceError, ConfigError
 
 
-class Gem5Device(AndroidDevice):
+class Gem5AndroidDevice(BaseGem5Device, AndroidDevice):
     """
     Implements gem5 Android device.
 
@@ -84,176 +76,31 @@ class Gem5Device(AndroidDevice):
     """
 
     name = 'gem5_android'
-    has_gpu = False
     platform = 'android'
-    path_module = 'posixpath'
-
-    # gem5 can be very slow. Hence, we use some very long timeouts!
-    delay = 3600
-    long_delay = 3 * delay
-    ready_timeout = long_delay
-    default_timeout = delay
 
     parameters = [
         Parameter('core_names', default=[], override=True),
         Parameter('core_clusters', default=[], override=True),
-        Parameter('gem5_binary', kind=str, default='./build/ARM/gem5.fast',
-                  mandatory=False, description="Command used to execute gem5. "
-                  "Adjust according to needs."),
-        Parameter('gem5_args', kind=types.arguments, mandatory=True,
-                  description="Command line passed to the gem5 simulation. This"
-                  " command line is used to set up the simulated system, and "
-                  "should be the same as used for a standard gem5 simulation "
-                  "without workload automation. Note that this is simulation "
-                  "script specific and will hence need to be tailored to each "
-                  "particular use case."),
-        Parameter('gem5_vio_args', kind=types.arguments, mandatory=True,
-                  constraint=lambda x: "{}" in str(x),
-                  description="gem5 VirtIO command line used to enable the "
-                  "VirtIO device in the simulated system. At the very least, "
-                  "the root parameter of the VirtIO9PDiod device must be "
-                  "exposed on the command line. Please set this root mount to "
-                  "{}, as it will be replaced with the directory used by "
-                  "Workload Automation at runtime."),
-        Parameter('temp_dir', kind=str, default='/tmp',
-                  description="Temporary directory used to pass files into the "
-                  "gem5 simulation. Workload Automation will automatically "
-                  "create a directory in this folder, and will remove it again "
-                  "once the simulation completes."),
-        Parameter('checkpoint', kind=bool, default=False,
-                  mandatory=False, description="This parameter "
-                  "tells Workload Automation to create a checkpoint of the "
-                  "simulated system once the guest system has finished booting."
-                  " This checkpoint can then be used at a later stage by other "
-                  "WA runs to avoid booting the guest system a second time. Set"
-                  " to True to take a checkpoint of the simulated system post "
-                  "boot."),
-        Parameter('run_delay', kind=int, default=0, mandatory=False,
-                  constraint=lambda x: x >= 0,
-                  description="This sets the time that the "
-                  "system should sleep in the simulated system prior to "
-                  "running and workloads or taking checkpoints. This allows "
-                  "the system to quieten down prior to running the workloads. "
-                  "When this is combined with the checkpoint_post_boot"
-                  " option, it allows the checkpoint to be created post-sleep,"
-                  " and therefore the set of workloads resuming from this "
-                  "checkpoint will not be required to sleep.")
     ]
 
     # Overwritten from Device. For documentation, see corresponding method in
     # Device.
 
-    @property
-    def is_rooted(self):
-        # gem5 is always rooted
-        return True
-
     def __init__(self, **kwargs):
-        self.logger = logging.getLogger('gem5Device')
+        self.logger = logging.getLogger('Gem5AndroidDevice')
+        AndroidDevice.__init__(self, **kwargs)
+        BaseGem5Device.__init__(self, **kwargs)
 
-        super(Gem5Device, self).__init__(**kwargs)
+    def login_to_device(self):
-
+        pass
-        # The gem5 subprocess
-        self.gem5 = None
-        self.gem5_port = -1
-        self.gem5outdir = os.path.join(settings.output_directory, "gem5")
-        self.stdout_file = None
-        self.stderr_file = None
-        self.stderr_filename = None
-        self.checkpoint = False
-        self.sckt = None
-
-        # Find the first one that does not exist. Ensures that we do not re-use
-        # the directory used by someone else.
-        for i in xrange(sys.maxint):
-            directory = os.path.join(self.temp_dir, "wa_{}".format(i))
-            try:
-                os.stat(directory)
-                continue
-            except OSError:
-                break
-        self.temp_dir = directory
-        self.logger.debug("Using {} as the temporary directory.".format(self.temp_dir))
-
-        # Start the gem5 simulation when WA starts a run using a signal.
-        sig.connect(self.init_gem5, sig.RUN_START)
-
-    def validate(self):
-        # Assemble the virtio args
-        self.gem5_vio_args = str(self.gem5_vio_args).format(self.temp_dir)
-        self.logger.debug("gem5 VirtIO command: {}".format(self.gem5_vio_args))
-
-    def init_gem5(self, _):
-        """
-        Start gem5, find out the telnet port and connect to the simulation.
-
-        We first create the temporary directory used by VirtIO to pass files
-        into the simulation, as well as the gem5 output directory.We then create
-        files for the standard output and error for the gem5 process. The gem5
-        process then is started.
-        """
-        self.logger.info("Creating temporary directory: {}".format(self.temp_dir))
-        os.mkdir(self.temp_dir)
-        os.mkdir(self.gem5outdir)
-
-        # We need to redirect the standard output and standard error for the
-        # gem5 process to a file so that we can debug when things go wrong.
-        f = os.path.join(self.gem5outdir, 'stdout')
-        self.stdout_file = open(f, 'w')
-        f = os.path.join(self.gem5outdir, 'stderr')
-        self.stderr_file = open(f, 'w')
-        # We need to keep this so we can check which port to use for the telnet
-        # connection.
-        self.stderr_filename = f
-
-        self.start_gem5()
-
-    def start_gem5(self):
-        """
-        Starts the gem5 simulator, and parses the output to get the telnet port.
-        """
-        self.logger.info("Starting the gem5 simulator")
-
-        command_line = "{} --outdir={}/gem5 {} {}".format(self.gem5_binary,
-                                                          settings.output_directory,
-                                                          self.gem5_args,
-                                                          self.gem5_vio_args)
-        self.logger.debug("gem5 command line: {}".format(command_line))
-        self.gem5 = subprocess.Popen(command_line.split(),
-                                     stdout=self.stdout_file,
-                                     stderr=self.stderr_file)
-
-        while self.gem5_port == -1:
-            # Check that gem5 is running!
-            if self.gem5.poll():
-                raise DeviceError("The gem5 process has crashed with error code {}!".format(self.gem5.poll()))
-
-            # Open the stderr file
-            f = open(self.stderr_filename, 'r')
-            for line in f:
-                m = re.search(r"Listening\ for\ system\ connection\ on\ port\ (?P<port>\d+)", line)
-                if m:
-                    port = int(m.group('port'))
-                    if port >= 3456 and port < 5900:
-                        self.gem5_port = port
-                        f.close()
-                        break
-            else:
-                time.sleep(1)
-            f.close()
-
-    def connect(self):  # pylint: disable=R0912
-        """
-        Connect to the gem5 simulation and wait for Android to boot. Then,
-        create checkpoints, and mount the VirtIO device.
-        """
-        self.connect_gem5()
 
+    def wait_for_boot(self):
         self.logger.info("Waiting for Android to boot...")
         while True:
             try:
-                available = (1 == int('0' + self.gem5_shell('getprop sys.boot_completed', check_exit_code=False)))
+                booted = (1 == int('0' + self.gem5_shell('getprop sys.boot_completed', check_exit_code=False)))
-                if available:
+                anim_finished = (1 == int('0' + self.gem5_shell('getprop service.bootanim.exit', check_exit_code=False)))
+                if booted and anim_finished:
                     break
             except (DeviceError, ValueError):
                 pass
@@ -261,176 +108,7 @@ class Gem5Device(AndroidDevice):
 
         self.logger.info("Android booted")
 
-        if self.run_delay:
+    def install(self, filepath, timeout=3 * 3600):  # pylint: disable=W0221
-            self.logger.info("Sleeping for {} seconds in the guest".format(self.run_delay))
-            self.gem5_shell("sleep {}".format(self.run_delay))
-
-        if self.checkpoint:
-            self.checkpoint_gem5()
-
-        self.mount_virtio()
-        self.logger.info("Creating the working directory in the simulated system")
-        self.gem5_shell('mkdir -p {}'.format(self.working_directory))
-        self._is_ready = True
-
-    def connect_gem5(self):  # pylint: disable=R0912
-        """
-        Connect to the telnet port of the gem5 simulation.
-
-        We connect, and wait for the prompt to be found. We do not use a timeout
-        for this, and wait for the prompt in a while loop as the gem5 simulation
-        can take many hours to reach a prompt when booting the system. We also
-        inject some newlines periodically to try and force gem5 to show a
-        prompt. Once the prompt has been found, we replace it with a unique
-        prompt to ensure that we are able to match it properly. We also disable
-        the echo as this simplifies parsing the output when executing commands
-        on the device.
-        """
-        self.logger.info("Connecting to the gem5 simulation on port {}".format(self.gem5_port))
-        host = socket.gethostname()
-        port = self.gem5_port
-
-        # Connect to the gem5 telnet port. Use a short timeout here.
-        self.sckt = ssh.TelnetConnection()
-        self.sckt.login(host, 'None', port=port, auto_prompt_reset=False,
-                        login_timeout=10)
-
-        self.logger.info("Connected! Waiting for prompt...")
-
-        # We need to find the prompt. It might be different if we are resuming
-        # from a checkpoint. Therefore, we test multiple options here.
-        prompt_found = False
-        while not prompt_found:
-            try:
-                # Try and force a prompt to be shown
-                self.sckt.send('\n')
-                self.sckt.expect([r'# ', self.sckt.UNIQUE_PROMPT], timeout=self.delay)
-                prompt_found = True
-            except TIMEOUT:
-                pass
-
-        self.logger.info("Setting unique prompt...")
-
-        self.sckt.set_unique_prompt()
-        self.sckt.prompt()
-        self.logger.info("Prompt found and replaced with a unique string")
-
-        self.sckt.setecho(False)
-        self.sync_gem5_shell()
-
-    def disconnect(self):
-        """
-        Close and disconnect from the gem5 simulation. Additionally, we remove
-        the temporary directory used to pass files into the simulation.
-        """
-        self.logger.info("Gracefully terminating the gem5 simulation.")
-        try:
-            self.gem5_util("exit")
-            self.gem5.wait()
-        except EOF:
-            pass
-        self.logger.info("Removing the temporary directory")
-        try:
-            shutil.rmtree(self.temp_dir)
-        except OSError:
-            self.logger.warn("Failed to remove the temporary directory!")
-
-    def close(self):
-        if self._logcat_poller:
-            self._logcat_poller.stop()
-
-    def reset(self):
-        self.logger.warn("Attempt to restart the gem5 device. This is not "
-                         "supported!")
-
-    def init(self):
-        pass
-
-    def push_file(self, source, dest, as_root=False):  # pylint: disable=W0221
-        """
-        Push a file to the gem5 device using VirtIO
-
-        The file to push to the device is copied to the temporary directory on
-        the host, before being copied within the simulation to the destination.
-        Checks, in the form of 'ls' with error code checking, are performed to
-        ensure that the file is copied to the destination.
-        """
-        filename = os.path.basename(source)
-        self.logger.debug("Pushing {} to device.".format(source))
-        self.logger.debug("temp_dir: {}".format(self.temp_dir))
-        self.logger.debug("dest: {}".format(dest))
-        self.logger.debug("filename: {}".format(filename))
-
-        # We need to copy the file to copy to the temporary directory
-        self.move_to_temp_dir(source)
-
-        # Back to the gem5 world
-        self.gem5_shell("ls -al /mnt/obb/{}".format(filename))
-        self.gem5_shell("busybox cp /mnt/obb/{} {}".format(filename, dest))
-        self.gem5_shell("busybox sync")
-        self.gem5_shell("ls -al {}".format(dest))
-        self.gem5_shell("ls -al /mnt/obb/")
-        self.logger.debug("Push complete.")
-
-    def pull_file(self, source, dest, as_root=False):  # pylint: disable=W0221
-        """
-        Pull a file from the gem5 device using m5 writefile
-
-        First, we check the extension of the file to be copied. If the file ends
-        in .gz, then gem5 wrongly assumes that it should create a gzipped output
-        stream, which results in a gem5 error. Therefore, we rename the file on
-        the local device prior to the writefile command when required. Next, the
-        file is copied to the local directory within the guest as the m5
-        writefile command assumes that the file is local. The file is then
-        written out to the host system using writefile, prior to being moved to
-        the destination on the host.
-        """
-        filename = source.split('/')[-1]
-        self.logger.debug("Pulling {} from device.".format(filename))
-
-        # gem5 assumes that files ending in .gz are gzip-compressed. We need to
-        # work around this, else gem5 panics on us. Rename the file for use in
-        # gem5
-        if filename[-3:] == '.gz':
-            filename += '.fugem5'
-
-        self.logger.debug("pull_file {} {}".format(source, filename))
-        # We don't check the exit code here because it is non-zero if the source
-        # and destination are the same. The ls below will cause an error if the
-        # file was not where we expected it to be.
-        self.gem5_shell("busybox cp {} {}".format(source, filename), check_exit_code=False)
-        self.gem5_shell("busybox sync")
-        self.gem5_shell("ls -la {}".format(filename))
-        self.logger.debug('Finished the copy in the simulator')
-        self.gem5_util("writefile {}".format(filename))
-
-        if 'cpu' not in filename:
-            while not os.path.exists(os.path.join(self.gem5outdir, filename)):
-                time.sleep(1)
-
-        # Perform the local move
-        shutil.move(os.path.join(self.gem5outdir, filename), dest)
-        self.logger.debug("Pull complete.")
-
-    def delete_file(self, filepath, as_root=False):  # pylint: disable=W0221
-        """ Delete a file on the device """
-        self._check_ready()
-        self.gem5_shell("rm '{}'".format(filepath))
-
-    def file_exists(self, filepath):
-        """ Check if a file exists """
-        self._check_ready()
-        output = self.gem5_shell('if [ -e \'{}\' ]; then echo 1; else echo 0; fi'.format(filepath))
-        try:
-            if int(output):
-                return True
-            else:
-                return False
-        except ValueError:
-            # If we cannot process the output, assume that there is no file
-            return False
-
-    def install(self, filepath, timeout=default_timeout):  # pylint: disable=W0221
         """ Install an APK or a normal executable """
         ext = os.path.splitext(filepath)[1].lower()
         if ext == '.apk':
@@ -438,7 +116,7 @@ class Gem5Device(AndroidDevice):
         else:
             return self.install_executable(filepath)
 
-    def install_apk(self, filepath, timeout=default_timeout):  # pylint: disable=W0221
+    def install_apk(self, filepath, timeout=3 * 3600):  # pylint: disable=W0221
         """
         Install an APK on the gem5 device
 
@@ -476,21 +154,6 @@ class Gem5Device(AndroidDevice):
         self._check_ready()
         self.gem5_shell("pm uninstall {}".format(package))
 
-    def execute(self, command, timeout=default_timeout, check_exit_code=True, background=False,
-                as_root=False, busybox=False, **kwargs):
-        self._check_ready()
-        if as_root and not self.is_rooted:
-            raise DeviceError('Attempting to execute "{}" as root on unrooted device.'.format(command))
-        if busybox:
-            if not self.is_rooted:
-                raise DeviceError('Attempting to execute "{}" with busybox. '.format(command) +
-                                  'Busybox can only be deployed to rooted devices.')
-            command = ' '.join([self.busybox, command])
-        if background:
-            self.logger.debug("Attempt to execute in background. Not supported "
-                              "in gem5, hence ignored.")
-        return self.gem5_shell(command, as_root=as_root)
-
     def dump_logcat(self, outfile, filter_spec=None):
         """ Extract logcat from simulation """
         self.logger.info("Extracting logcat from the simulated system")
@@ -519,55 +182,9 @@ class Gem5Device(AndroidDevice):
 
     def get_properties(self, context):  # pylint: disable=R0801
         """ Get the property files from the device """
-        for propfile in self.property_files:
+        BaseGem5Device.get_properties(self, context)
-            try:
-                normname = propfile.lstrip(self.path.sep).replace(self.path.sep, '.')
-                outfile = os.path.join(context.host_working_directory, normname)
-                if self.is_file(propfile):
-                    self.execute('cat {} > {}'.format(propfile, normname))
-                    self.pull_file(normname, outfile)
-                elif self.is_directory(propfile):
-                    self.get_directory(context, propfile)
-                    continue
-                else:
-                    continue
-            except DeviceError:
-                # We pull these files "opportunistically", so if a pull fails
-                # (e.g. we don't have permissions to read the file), just note
-                # it quietly (not as an error/warning) and move on.
-                self.logger.debug('Could not pull property file "{}"'.format(propfile))
-
-        # As we do not call the super class, we explicitly need to
-        # process the Android properties.
         props = self._get_android_properties(context)
-
+        return props
-    def get_directory(self, context, directory):
-        """ Pull a directory from the device """
-        normname = directory.lstrip(self.path.sep).replace(self.path.sep, '.')
-        outdir = os.path.join(context.host_working_directory, normname)
-        temp_file = os.path.join(context.host_working_directory, "{}.tar".format(normname))
-        # Check that the folder exists
-        self.gem5_shell("ls -la {}".format(directory))
-        # Compress the folder
-        try:
-            self.gem5_shell("{} tar -cvf {}.tar {}".format(self.busybox, normname, directory))
-        except DeviceError:
-            self.logger.debug("Failed to run tar command on device! Not pulling {}".format(directory))
-            return
-        self.pull_file(normname, temp_file)
-        f = tarfile.open(temp_file, 'r')
-        os.mkdir(outdir)
-        f.extractall(outdir)
-        os.remove(temp_file)
-
-    def get_pids_of(self, process_name):
-        """ Returns a list of PIDs of all processes with the specified name. """
-        result = self.gem5_shell('ps | busybox grep {}'.format(process_name),
-                                 check_exit_code=False).strip()
-        if result and 'not found' not in result and len(result.split('\n')) > 2:
-            return [int(x.split()[1]) for x in result.split('\n')]
-        else:
-            return []
 
     def disable_screen_lock(self):
         """
@@ -579,166 +196,10 @@ class Gem5Device(AndroidDevice):
         sqlcommand = "update locksettings set value=\'0\' where name=\'screenlock.disabled\';"
         self.execute('sqlite3 {} "{}"'.format(lockdb, sqlcommand), as_root=True)
 
-    # gem5 should dump out a framebuffer. We can use this if it exists. Failing
-    # that, fall back to the parent class implementation.
     def capture_screen(self, filepath):
-        file_list = os.listdir(self.gem5outdir)
+        if BaseGem5Device.capture_screen(self, filepath):
-        screen_caps = []
-        for f in file_list:
-            if '.bmp' in f:
-                screen_caps.append(f)
-
-        if len(screen_caps) == 1:
-            # Bail out if we do not have image, and resort to the slower, built
-            # in method.
-            try:
-                import Image
-                gem5_image = os.path.join(self.gem5outdir, screen_caps[0])
-                temp_image = os.path.join(self.gem5outdir, "file.png")
-                im = Image.open(gem5_image)
-                im.save(temp_image, "PNG")
-                shutil.copy(temp_image, filepath)
-                os.remove(temp_image)
-                self.logger.debug("capture_screen: using gem5 screencap")
             return
-            except (shutil.Error, ImportError, IOError):
-                pass
 
         # If we didn't manage to do the above, call the parent class.
         self.logger.warning("capture_screen: falling back to parent class implementation")
-        super(Gem5Device, self).capture_screen(filepath)
+        AndroidDevice.capture_screen(self, filepath)
-
-    # gem5 might be slow. Hence, we need to make the ping timeout very long.
-    def ping(self):
-        self.logger.debug("Pinging gem5 to see if it is still alive")
-        self.gem5_shell('ls /', timeout=self.longdelay)
-
-    # Additional Android-specific methods.
-    def forward_port(self, from_port, to_port):
-        self.logger.warning('Tried to forward port. We do not need any forwarding.')
-
-    # Internal methods: do not use outside of the class.
-
-    def _check_ready(self):
-        """
-        Check if the device is ready.
-
-        As this is gem5, we just assume that the device is ready once we have
-        connected to the gem5 simulation, and updated the prompt.
-        """
-        if not self._is_ready:
-            raise DeviceError('Device not ready.')
-
-    def gem5_shell(self, command, as_root=False, timeout=None, check_exit_code=True, sync=True):  # pylint: disable=R0912
-        """
-        Execute a command in the gem5 shell
-
-        This wraps the telnet connection to gem5 and processes the raw output.
-
-        This method waits for the shell to return, and then will try and
-        separate the output from the command from the command itself. If this
-        fails, warn, but continue with the potentially wrong output.
-
-        The exit code is also checked by default, and non-zero exit codes will
-        raise a DeviceError.
-        """
-        conn = self.sckt
-        if sync:
-            self.sync_gem5_shell()
-
-        self.logger.debug("gem5_shell command: {}".format(command))
-
-        # Send the actual command
-        conn.send("{}\n".format(command))
-
-        # Wait for the response. We just sit here and wait for the prompt to
-        # appear, as gem5 might take a long time to provide the output. This
-        # avoids timeout issues.
-        command_index = -1
-        while command_index == -1:
-            if conn.prompt():
-                output = re.sub(r' \r([^\n])', r'\1', conn.before)
-                output = re.sub(r'[\b]', r'', output)
-                # Deal with line wrapping
-                output = re.sub(r'[\r].+?<', r'', output)
-                command_index = output.find(command)
-
-                # If we have -1, then we cannot match the command, but the
-                # prompt has returned. Hence, we have a bit of an issue. We
-                # warn, and return the whole output.
-                if command_index == -1:
-                    self.logger.warn("gem5_shell: Unable to match command in "
-                                     "command output. Expect parsing errors!")
-                    command_index = 0
-
-        output = output[command_index + len(command):].strip()
-
-        # It is possible that gem5 will echo the command. Therefore, we need to
-        # remove that too!
-        command_index = output.find(command)
-        if command_index != -1:
-            output = output[command_index + len(command):].strip()
-
-        self.logger.debug("gem5_shell output: {}".format(output))
-
-        # We get a second prompt. Hence, we need to eat one to make sure that we
-        # stay in sync. If we do not do this, we risk getting out of sync for
-        # slower simulations.
-        self.sckt.expect(self.sckt.UNIQUE_PROMPT, timeout=10000)
-
-        if check_exit_code:
-            exit_code_text = self.gem5_shell('echo $?', as_root=as_root,
-                                             timeout=timeout, check_exit_code=False,
-                                             sync=False)
-            try:
-                exit_code = int(exit_code_text.split()[0])
-                if exit_code:
-                    message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}'
-                    raise DeviceError(message.format(exit_code, command, output))
-            except (ValueError, IndexError):
-                self.logger.warning('Could not get exit code for "{}",\ngot: "{}"'.format(command, exit_code_text))
-
-        return output
-
-    def gem5_util(self, command):
-        """ Execute a gem5 utility command using the m5 binary on the device """
-        self.gem5_shell('/sbin/m5 ' + command)
-
-    def sync_gem5_shell(self):
-        """
-        Synchronise with the gem5 shell.
-
-        Write some unique text to the gem5 device to allow us to synchronise
-        with the shell output. We actually get two prompts so we need to match
-        both of these.
-        """
-        self.sckt.send("echo \*\*sync\*\*\n")
-        self.sckt.expect(r"\*\*sync\*\*", timeout=self.delay)
-        self.sckt.expect(self.sckt.UNIQUE_PROMPT, timeout=self.delay)
-        self.sckt.expect(self.sckt.UNIQUE_PROMPT, timeout=self.delay)
-
-    def move_to_temp_dir(self, source):
-        """
-        Move a file to the temporary directory on the host for copying to the
-        gem5 device
-        """
-        command = "cp {} {}".format(source, self.temp_dir)
-        self.logger.debug("Local copy command: {}".format(command))
-        subprocess.call(command.split())
-        subprocess.call("sync".split())
-
-    def checkpoint_gem5(self, end_simulation=False):
-        """ Checkpoint the gem5 simulation, storing all system state """
-        self.logger.info("Taking a post-boot checkpoint")
-        self.gem5_util("checkpoint")
-        if end_simulation:
-            self.disconnect()
-
-    def mount_virtio(self):
-        """
-        Mount the VirtIO device in the simulated system.
-        """
-        self.logger.info("Mounting VirtIO device in simulated system")
-
-        mount_command = "mount -t 9p -o trans=virtio,version=9p2000.L,aname={} gem5 /mnt/obb".format(self.temp_dir)
-        self.gem5_shell('busybox {}'.format(mount_command))

wlauto/devices/linux/gem5/__init__.py

@@ -0,0 +1,112 @@
+#    Copyright 2014-2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Original implementation by Rene de Jong. Updated by Sascha Bischoff.
+
+import logging
+
+from wlauto import LinuxDevice, Parameter
+from wlauto.common.gem5.device import BaseGem5Device
+
+
+class Gem5LinuxDevice(BaseGem5Device, LinuxDevice):
+    """
+    Implements gem5 Linux device.
+
+    This class allows a user to connect WA to a simulation using gem5. The
+    connection to the device is made using the telnet connection of the
+    simulator, and is used for all commands. The simulator does not have ADB
+    support, and therefore we need to fall back to using standard shell
+    commands.
+
+    Files are copied into the simulation using a VirtIO 9P device in gem5. Files
+    are copied out of the simulated environment using the m5 writefile command
+    within the simulated system.
+
+    When starting the workload run, the simulator is automatically started by
+    Workload Automation, and a connection to the simulator is established. WA
+    will then wait for Android to boot on the simulated system (which can take
+    hours), prior to executing any other commands on the device. It is also
+    possible to resume from a checkpoint when starting the simulation. To do
+    this, please append the relevant checkpoint commands from the gem5
+    simulation script to the gem5_discription argument in the agenda.
+
+    Host system requirements:
+        * VirtIO support. We rely on diod on the host system. This can be
+          installed on ubuntu using the following command:
+
+                sudo apt-get install diod
+
+    Guest requirements:
+        * VirtIO support. We rely on VirtIO to move files into the simulation.
+          Please make sure that the following are set in the kernel
+          configuration:
+
+                CONFIG_NET_9P=y
+
+                CONFIG_NET_9P_VIRTIO=y
+
+                CONFIG_9P_FS=y
+
+                CONFIG_9P_FS_POSIX_ACL=y
+
+                CONFIG_9P_FS_SECURITY=y
+
+                CONFIG_VIRTIO_BLK=y
+
+        * m5 binary. Please make sure that the m5 binary is on the device and
+          can by found in the path.
+        * Busybox. Due to restrictions, we assume that busybox is installed in
+          the guest system, and can be found in the path.
+    """
+
+    name = 'gem5_linux'
+    platform = 'linux'
+
+    parameters = [
+        Parameter('core_names', default=[], override=True),
+        Parameter('core_clusters', default=[], override=True),
+        Parameter('host', default='localhost', override=True,
+                  description='Host name or IP address for the device.'),
+    ]
+
+    # Overwritten from Device. For documentation, see corresponding method in
+    # Device.
+
+    def __init__(self, **kwargs):
+        self.logger = logging.getLogger('Gem5LinuxDevice')
+        LinuxDevice.__init__(self, **kwargs)
+        BaseGem5Device.__init__(self, **kwargs)
+
+    def login_to_device(self):
+        # Wait for the login prompt
+        i = self.sckt.expect([r'login:', r'username:', self.sckt.UNIQUE_PROMPT],
+                             timeout=10)
+        # Check if we are already at a prompt, or if we need to log in.
+        if i < 2:
+            self.sckt.sendline("{}".format(self.username))
+            j = self.sckt.expect([r'password:', r'# ', self.sckt.UNIQUE_PROMPT],
+                                 timeout=self.delay)
+            if j == 2:
+                self.sckt.sendline("{}".format(self.password))
+                self.sckt.expect([r'# ', self.sckt.UNIQUE_PROMPT], timeout=self.delay)
+
+    def capture_screen(self, filepath):
+        if BaseGem5Device.capture_screen(self, filepath):
+            return
+
+        # If we didn't manage to do the above, call the parent class.
+        self.logger.warning("capture_screen: falling back to parent class implementation")
+        LinuxDevice.capture_screen(self, filepath)