Merge pull request #83 from avanlaer/gem5_integration

Gem5 integration

devlib/__init__.py

@@ -7,6 +7,7 @@ from devlib.module import get_module, register_module
 
 from devlib.platform import Platform
 from devlib.platform.arm import TC2, Juno, JunoEnergyInstrument
+from devlib.platform.gem5 import Gem5SimulationPlatform
 
 from devlib.instrument import Instrument, InstrumentChannel, Measurement, MeasurementsCsv
 from devlib.instrument import MEASUREMENT_TYPES, INSTANTANEOUS, CONTINUOUS
@@ -19,4 +20,4 @@ from devlib.trace.ftrace import FtraceCollector
 
 from devlib.host import LocalConnection
 from devlib.utils.android import AdbConnection
-from devlib.utils.ssh import SshConnection, TelnetConnection
+from devlib.utils.ssh import SshConnection, TelnetConnection, Gem5Connection

devlib/host.py

@@ -49,7 +49,8 @@ class LocalConnection(object):
         else:
             shutil.copy(source, dest)
 
-    def execute(self, command, timeout=None, check_exit_code=True, as_root=False):
+    def execute(self, command, timeout=None, check_exit_code=True,
+                as_root=False, strip_colors=True):
         self.logger.debug(command)
         if as_root:
             if self.unrooted:

devlib/platform/__init__.py

@@ -48,6 +48,11 @@ class Platform(object):
             self.name = self.model
         self._validate()
 
+    def setup(self, target):
+        # May be overwritten by subclasses to provide platform-specific
+        # setup procedures.
+        pass
+
     def _set_core_clusters_from_core_names(self):
         self.core_clusters = []
         clusters = []

devlib/platform/gem5.py

@@ -0,0 +1,282 @@
+#    Copyright 2016 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import re
+import subprocess
+import sys
+import shutil
+import time
+import types
+
+from devlib.exception import TargetError
+from devlib.host import PACKAGE_BIN_DIRECTORY
+from devlib.platform import Platform
+from devlib.utils.ssh import AndroidGem5Connection, LinuxGem5Connection
+
+class Gem5SimulationPlatform(Platform):
+
+    def __init__(self, name, host_output_dir, gem5_bin, gem5_args, gem5_virtio,
+                 gem5_telnet_port=None):
+
+        # First call the parent class
+        super(Gem5SimulationPlatform, self).__init__(name=name)
+
+        # Start setting up the gem5 parameters/directories
+        # The gem5 subprocess
+        self.gem5 = None
+        self.gem5_port = gem5_telnet_port or None
+        self.stats_directory = host_output_dir
+        self.gem5_out_dir = os.path.join(self.stats_directory, "gem5")
+        self.gem5_interact_dir = '/tmp' # Host directory
+        self.executable_dir = None # Device directory
+        self.working_dir = None # Device directory
+        self.stdout_file = None
+        self.stderr_file = None
+        self.stderr_filename = None
+        if self.gem5_port is None:
+            # Allows devlib to pick up already running simulations
+            self.start_gem5_simulation = True
+        else:
+            self.start_gem5_simulation = False
+
+        # Find the first one that does not exist. Ensures that we do not re-use
+        # the directory used by someone else.
+        for i in xrange(sys.maxint):
+            directory = os.path.join(self.gem5_interact_dir, "wa_{}".format(i))
+            try:
+                os.stat(directory)
+                continue
+            except OSError:
+                break
+        self.gem5_interact_dir = directory
+        self.logger.debug("Using {} as the temporary directory."
+                          .format(self.gem5_interact_dir))
+
+        # Parameters passed onto gem5
+        self.gem5args_binary = gem5_bin
+        self.gem5args_args = gem5_args
+        self.gem5args_virtio = gem5_virtio
+        self._check_gem5_command()
+
+        # Start the interaction with gem5
+        self._start_interaction_gem5()
+
+    def _check_gem5_command(self):
+        """
+        Check if the command to start gem5 makes sense
+        """
+        if self.gem5args_binary is None:
+            raise TargetError('Please specify a gem5 binary.')
+        if self.gem5args_args is None:
+            raise TargetError('Please specify the arguments passed on to gem5.')
+        self.gem5args_virtio = str(self.gem5args_virtio).format(self.gem5_interact_dir)
+        if self.gem5args_virtio is None:
+            raise TargetError('Please specify arguments needed for virtIO.')
+
+    def _start_interaction_gem5(self):
+        """
+        Starts the interaction of devlib with gem5.
+        """
+
+        # First create the input and output directories for gem5
+        if self.start_gem5_simulation:
+            # Create the directory to send data to/from gem5 system
+            self.logger.info("Creating temporary directory for interaction "
+                             " with gem5 via virtIO: {}"
+                             .format(self.gem5_interact_dir))
+            os.mkdir(self.gem5_interact_dir)
+
+            # Create the directory for gem5 output (stats files etc)
+            if not os.path.exists(self.stats_directory):
+                os.mkdir(self.stats_directory)
+            if os.path.exists(self.gem5_out_dir):
+                raise TargetError("The gem5 stats directory {} already "
+                                  "exists.".format(self.gem5_out_dir))
+            else:
+                os.mkdir(self.gem5_out_dir)
+
+            # We need to redirect the standard output and standard error for the
+            # gem5 process to a file so that we can debug when things go wrong.
+            f = os.path.join(self.gem5_out_dir, 'stdout')
+            self.stdout_file = open(f, 'w')
+            f = os.path.join(self.gem5_out_dir, 'stderr')
+            self.stderr_file = open(f, 'w')
+            # We need to keep this so we can check which port to use for the
+            # telnet connection.
+            self.stderr_filename = f
+
+            # Start gem5 simulation
+            self.logger.info("Starting the gem5 simulator")
+
+            command_line = "{} --outdir={} {} {}".format(self.gem5args_binary,
+                                                         self.gem5_out_dir,
+                                                         self.gem5args_args,
+                                                         self.gem5args_virtio)
+            self.logger.debug("gem5 command line: {}".format(command_line))
+            self.gem5 = subprocess.Popen(command_line.split(),
+                                         stdout=self.stdout_file,
+                                         stderr=self.stderr_file)
+
+        else:
+            # The simulation should already be running
+            # Need to dig up the (1) gem5 simulation in question (2) its input
+            # and output directories (3) virtio setting
+            self._intercept_existing_gem5()
+
+        # As the gem5 simulation is running now or was already running
+        # we now need to find out which telnet port it uses
+        self._intercept_telnet_port()
+
+    def _intercept_existing_gem5(self):
+        """
+        Intercept the information about a running gem5 simulation
+        e.g. pid, input directory etc
+        """
+        self.logger("This functionality is not yet implemented")
+        raise TargetError()
+
+    def _intercept_telnet_port(self):
+        """
+        Intercept the telnet port of a running gem5 simulation
+        """
+
+        if self.gem5 is None:
+            raise TargetError('The platform has no gem5 simulation! '
+                              'Something went wrong')
+        while self.gem5_port is None:
+            # Check that gem5 is running!
+            if self.gem5.poll():
+                raise TargetError("The gem5 process has crashed with error code {}!".format(self.gem5.poll()))
+
+            # Open the stderr file
+            with open(self.stderr_filename, 'r') as f:
+                for line in f:
+                    m = re.search(r"Listening for system connection on port (?P<port>\d+)", line)
+                    if m:
+                        port = int(m.group('port'))
+                        if port >= 3456 and port < 5900:
+                            self.gem5_port = port
+                            break
+                    # Check if the sockets are not disabled
+                    m = re.search(r"Sockets disabled, not accepting terminal connections", line)
+                    if m:
+                        raise TargetError("The sockets have been disabled!"
+                                          "Pass --listener-mode=on to gem5")
+                else:
+                    time.sleep(1)
+
+    def init_target_connection(self, target):
+        """
+        Update the type of connection in the target from here
+        """
+        if target.os == 'linux':
+            target.conn_cls = LinuxGem5Connection
+        else:
+            target.conn_cls = AndroidGem5Connection
+
+    def setup(self, target):
+        """
+        Deploy m5 if not yet installed
+        """
+        m5_path = target.get_installed('m5')
+        if m5_path is None:
+            m5_path = self._deploy_m5(target)
+        target.conn.m5_path = m5_path
+
+        # Set the terminal settings for the connection to gem5
+        self._resize_shell(target)
+
+    def update_from_target(self, target):
+        """
+        Set the m5 path and if not yet installed, deploy m5
+        Overwrite certain methods in the target that either can be done
+        more efficiently by gem5 or don't exist in gem5
+        """
+        m5_path = target.get_installed('m5')
+        if m5_path is None:
+            m5_path = self._deploy_m5(target)
+        target.conn.m5_path = m5_path
+
+        # Overwrite the following methods (monkey-patching)
+        self.logger.debug("Overwriting the 'capture_screen' method in target")
+        # Housekeeping to prevent recursion
+        setattr(target, 'target_impl_capture_screen', target.capture_screen)
+        target.capture_screen = types.MethodType(_overwritten_capture_screen, target)
+        self.logger.debug("Overwriting the 'reset' method in target")
+        target.reset = types.MethodType(_overwritten_reset, target)
+        self.logger.debug("Overwriting the 'reboot' method in target")
+        target.reboot = types.MethodType(_overwritten_reboot, target)
+
+        # Call the general update_from_target implementation
+        super(Gem5SimulationPlatform, self).update_from_target(target)
+
+    def gem5_capture_screen(self, filepath):
+        file_list = os.listdir(self.gem5_out_dir)
+        screen_caps = []
+        for f in file_list:
+            if '.bmp' in f:
+                screen_caps.append(f)
+
+        successful_capture = False
+        if len(screen_caps) == 1:
+            # Bail out if we do not have image, and resort to the slower, built
+            # in method.
+            try:
+                import Image
+                gem5_image = os.path.join(self.gem5_out_dir, screen_caps[0])
+                temp_image = os.path.join(self.gem5_out_dir, "file.png")
+                im = Image.open(gem5_image)
+                im.save(temp_image, "PNG")
+                shutil.copy(temp_image, filepath)
+                os.remove(temp_image)
+                gem5_logger.info("capture_screen: using gem5 screencap")
+                successful_capture = True
+
+            except (shutil.Error, ImportError, IOError):
+                pass
+
+        return successful_capture
+
+    def _deploy_m5(self, target):
+        # m5 is not yet installed so install it
+        host_executable = os.path.join(PACKAGE_BIN_DIRECTORY,
+                                       target.abi, 'm5')
+        return target.install(host_executable)
+
+    def _resize_shell(self, target):
+        """
+        Resize the shell to avoid line wrapping issues.
+
+        """
+        # Try and avoid line wrapping as much as possible.
+        target.execute('{} stty columns 1024'.format(target.busybox))
+        target.execute('reset', check_exit_code=False)
+
+# Methods that will be monkey-patched onto the target
+def _overwritten_reset(self):
+    raise TargetError('Resetting is not allowed on gem5 platforms!')
+
+def _overwritten_reboot(self):
+    raise TargetError('Rebooting is not allowed on gem5 platforms!')
+
+def _overwritten_capture_screen(self, filepath):
+    connection_screencapped = self.platform.gem5_capture_screen(filepath)
+    if connection_screencapped == False:
+        # The connection was not able to capture the screen so use the target
+        # implementation
+        self.logger.debug('{} was not able to screen cap, using the original target implementation'.format(self.platform.__class__.__name__))
+        self.target_impl_capture_screen(filepath)
+
+

devlib/target.py

@@ -152,7 +152,22 @@ class Target(object):
                  conn_cls=None,
                  ):
         self.connection_settings = connection_settings or {}
-        self.platform = platform or Platform()
+        # Set self.platform: either it's given directly (by platform argument)
+        # or it's given in the connection_settings argument
+        # If neither, create default Platform()
+        if platform is None:
+            self.platform = self.connection_settings.get('platform', Platform())
+        else:
+            self.platform = platform
+        # Check if the user hasn't given two different platforms
+        if 'platform' in self.connection_settings:
+            if connection_settings['platform'] is not platform:
+                raise TargetError('Platform specified in connection_settings '
+                                   '({}) differs from that directly passed '
+                                   '({})!)'
+                                   .format(connection_settings['platform'],
+                                    self.platform))
+        self.connection_settings['platform'] = self.platform
         self.working_directory = working_directory
         self.executables_directory = executables_directory
         self.modules = modules or []
@@ -222,6 +237,9 @@ class Target(object):
         for host_exe in (executables or []):  # pylint: disable=superfluous-parens
             self.install(host_exe)
 
+        # Check for platform dependent setup procedures
+        self.platform.setup(self)
+
         # Initialize modules which requires Buxybox (e.g. shutil dependent tasks)
         self._update_modules('setup')
 

devlib/utils/android.py

@@ -186,7 +186,7 @@ class AdbConnection(object):
             self.ls_command = 'ls'
         logger.info("ls command is set to {}".format(self.ls_command))
 
-    def __init__(self, device=None, timeout=None):
+    def __init__(self, device=None, timeout=None, platform=None):
         self.timeout = timeout if timeout is not None else self.default_timeout
         if device is None:
             device = adb_get_device(timeout=timeout)
@@ -216,7 +216,8 @@ class AdbConnection(object):
         command = "pull '{}' '{}'".format(source, dest)
         return adb_command(self.device, command, timeout=timeout)
 
-    def execute(self, command, timeout=None, check_exit_code=False, as_root=False):
+    def execute(self, command, timeout=None, check_exit_code=False,
+                as_root=False, strip_colors=True):
         return adb_shell(self.device, command, timeout, check_exit_code,
                          as_root, self.newline_separator)
 

devlib/utils/ssh.py

@@ -22,6 +22,7 @@ import re
 import threading
 import tempfile
 import shutil
+import socket
 import time
 
 import pexpect
@@ -34,14 +35,16 @@ from pexpect import EOF, TIMEOUT, spawn
 
 from devlib.exception import HostError, TargetError, TimeoutError
 from devlib.utils.misc import which, strip_bash_colors, escape_single_quotes, check_output
+from devlib.utils.types import boolean
 
 
 ssh = None
 scp = None
 sshpass = None
 
-logger = logging.getLogger('ssh')
 
+logger = logging.getLogger('ssh')
+gem5_logger = logging.getLogger('gem5-connection')
 
 def ssh_get_shell(host, username, password=None, keyfile=None, port=None, timeout=10, telnet=False, original_prompt=None):
     _check_env()
@@ -91,18 +94,20 @@ class TelnetPxssh(pxssh.pxssh):
 
         spawn._spawn(self, cmd)  # pylint: disable=protected-access
 
-        if password is None:
-            i = self.expect([self.original_prompt, 'Login timed out'], timeout=login_timeout)
-        else:
+        try:
             i = self.expect('(?i)(?:password)', timeout=login_timeout)
             if i == 0:
                 self.sendline(password)
                 i = self.expect([self.original_prompt, 'Login incorrect'], timeout=login_timeout)
-            else:
-                raise pxssh.ExceptionPxssh('could not log in: did not see a password prompt')
-
             if i:
                 raise pxssh.ExceptionPxssh('could not log in: password was incorrect')
+        except TIMEOUT:
+            if not password:
+                # No password promt before TIMEOUT & no password provided
+                # so assume everything is okay
+                pass
+            else:
+                raise pxssh.ExceptionPxssh('could not log in: did not see a password prompt')
 
         if not self.sync_original_prompt(sync_multiplier):
             self.close()
@@ -155,6 +160,7 @@ class SshConnection(object):
                  telnet=False,
                  password_prompt=None,
                  original_prompt=None,
+                 platform=None
                  ):
         self.host = host
         self.username = username
@@ -175,7 +181,8 @@ class SshConnection(object):
         source = '{}@{}:{}'.format(self.username, self.host, source)
         return self._scp(source, dest, timeout)
 
-    def execute(self, command, timeout=None, check_exit_code=True, as_root=False, strip_colors=True):
+    def execute(self, command, timeout=None, check_exit_code=True,
+                as_root=False, strip_colors=True): #pylint: disable=unused-argument
         try:
             with self.lock:
                 output = self._execute_and_wait_for_prompt(command, timeout, as_root, strip_colors)
@@ -286,7 +293,7 @@ class TelnetConnection(SshConnection):
                  timeout=None,
                  password_prompt=None,
                  original_prompt=None,
-                 ):
+                 platform=None):
         self.host = host
         self.username = username
         self.password = password
@@ -299,6 +306,503 @@ class TelnetConnection(SshConnection):
         self.conn = ssh_get_shell(host, username, password, None, port, timeout, True, original_prompt)
 
 
+class Gem5Connection(TelnetConnection):
+
+    def __init__(self,
+                 platform,
+                 host=None,
+                 username=None,
+                 password=None,
+                 port=None,
+                 timeout=None,
+                 password_prompt=None,
+                 original_prompt=None,
+                 ):
+        if host is not None:
+            host_system = socket.gethostname()
+            if host_system != host:
+                raise TargetError("Gem5Connection can only connect to gem5 "
+                                   "simulations on your current host, which "
+                                   "differs from the one given {}!"
+                                   .format(host_system, host))
+        if username is not None and username != 'root':
+            raise ValueError('User should be root in gem5!')
+        if password is not None and password != '':
+            raise ValueError('No password needed in gem5!')
+        self.username = 'root'
+        self.is_rooted = True
+        self.password = None
+        self.port = None
+        # Long timeouts to account for gem5 being slow
+        # Can be overriden if the given timeout is longer
+        self.default_timeout = 3600
+        if timeout is not None:
+            if timeout > self.default_timeout:
+                logger.info('Overwriting the default timeout of gem5 ({})'
+                                 ' to {}'.format(self.default_timeout, timeout))
+                self.default_timeout = timeout
+            else:
+                logger.info('Ignoring the given timeout --> gem5 needs longer timeouts')
+        self.ready_timeout = self.default_timeout * 3
+        # Counterpart in gem5_interact_dir
+        self.gem5_input_dir = '/mnt/host/'
+        # Location of m5 binary in the gem5 simulated system
+        self.m5_path = None
+        # Actual telnet connection to gem5 simulation
+        self.conn = None
+        # Flag to indicate the gem5 device is ready to interact with the
+        # outer world
+        self.ready = False
+        # Lock file to prevent multiple connections to same gem5 simulation
+        # (gem5 does not allow this)
+        self.lock_directory = '/tmp/'
+        self.lock_file_name = None # Will be set once connected to gem5
+
+        # These parameters will be set by either the method to connect to the
+        # gem5 platform or directly to the gem5 simulation
+        # Intermediate directory to push things to gem5 using VirtIO
+        self.gem5_interact_dir = None
+        # Directory to store output  from gem5 on the host
+        self.gem5_out_dir = None
+        # Actual gem5 simulation
+        self.gem5simulation = None
+
+        # Connect to gem5
+        if platform:
+            self._connect_gem5_platform(platform)
+
+        # Wait for boot
+        self._wait_for_boot()
+
+        # Mount the virtIO to transfer files in/out gem5 system
+        self._mount_virtio()
+
+    def set_hostinteractdir(self, indir):
+        logger.info('Setting hostinteractdir  from {} to {}'
+                    .format(self.gem5_input_dir, indir))
+        self.gem5_input_dir = indir
+
+    def push(self, source, dest, timeout=None):
+        """
+        Push a file to the gem5 device using VirtIO
+
+        The file to push to the device is copied to the temporary directory on
+        the host, before being copied within the simulation to the destination.
+        Checks, in the form of 'ls' with error code checking, are performed to
+        ensure that the file is copied to the destination.
+        """
+        # First check if the connection is set up to interact with gem5
+        self._check_ready()
+
+        filename = os.path.basename(source)
+        logger.debug("Pushing {} to device.".format(source))
+        logger.debug("gem5interactdir: {}".format(self.gem5_interact_dir))
+        logger.debug("dest: {}".format(dest))
+        logger.debug("filename: {}".format(filename))
+
+        # We need to copy the file to copy to the temporary directory
+        self._move_to_temp_dir(source)
+
+        # Dest in gem5 world is a file rather than directory
+        if os.path.basename(dest) != filename:
+            dest = os.path.join(dest, filename)
+        # Back to the gem5 world
+        self._gem5_shell("ls -al {}{}".format(self.gem5_input_dir, filename))
+        self._gem5_shell("cat '{}''{}' > '{}'".format(self.gem5_input_dir,
+                                                     filename,
+                                                     dest))
+        self._gem5_shell("sync")
+        self._gem5_shell("ls -al {}".format(dest))
+        self._gem5_shell("ls -al {}".format(self.gem5_input_dir))
+        logger.debug("Push complete.")
+
+    def pull(self, source, dest, timeout=0): #pylint: disable=unused-argument
+        """
+        Pull a file from the gem5 device using m5 writefile
+
+        The file is copied to the local directory within the guest as the m5
+        writefile command assumes that the file is local. The file is then
+        written out to the host system using writefile, prior to being moved to
+        the destination on the host.
+        """
+        # First check if the connection is set up to interact with gem5
+        self._check_ready()
+
+        filename = os.path.basename(source)
+
+        logger.debug("pull_file {} {}".format(source, filename))
+        # We don't check the exit code here because it is non-zero if the source
+        # and destination are the same. The ls below will cause an error if the
+        # file was not where we expected it to be.
+        if os.path.dirname(source) != os.getcwd():
+            self._gem5_shell("cat '{}' > '{}'".format(source, filename))
+        self._gem5_shell("sync")
+        self._gem5_shell("ls -la {}".format(filename))
+        logger.debug('Finished the copy in the simulator')
+        self._gem5_util("writefile {}".format(filename))
+
+        if 'cpu' not in filename:
+            while not os.path.exists(os.path.join(self.gem5_out_dir, filename)):
+                time.sleep(1)
+
+        # Perform the local move
+        shutil.move(os.path.join(self.gem5_out_dir, filename), dest)
+        logger.debug("Pull complete.")
+
+    def execute(self, command, timeout=1000, check_exit_code=True,
+                as_root=False, strip_colors=True):
+        """
+        Execute a command on the gem5 platform
+        """
+        # First check if the connection is set up to interact with gem5
+        self._check_ready()
+
+        output = self._gem5_shell(command, as_root=as_root)
+        if strip_colors:
+            output = strip_bash_colors(output)
+        return output
+
+    def background(self, command, stdout=subprocess.PIPE,
+                   stderr=subprocess.PIPE, as_root=False):
+        # First check if the connection is set up to interact with gem5
+        self._check_ready()
+
+        # Create the logfile for stderr/stdout redirection
+        command_name = command.split(' ')[0].split('/')[-1]
+        redirection_file = 'BACKGROUND_{}.log'.format(command_name)
+        trial = 0
+        while os.path.isfile(redirection_file):
+            # Log file already exists so add to name
+           redirection_file = 'BACKGROUND_{}{}.log'.format(command_name, trial)
+           trial += 1
+
+        # Create the command to pass on to gem5 shell
+        complete_command = '{} >> {} 2>&1 &'.format(command, redirection_file)
+        output = self._gem5_shell(complete_command, as_root=as_root)
+        output = strip_bash_colors(output)
+        gem5_logger.info('STDERR/STDOUT of background command will be '
+                         'redirected to {}. Use target.pull() to '
+                         'get this file'.format(redirection_file))
+        return output
+
+    def close(self):
+        """
+        Close and disconnect from the gem5 simulation. Additionally, we remove
+        the temporary directory used to pass files into the simulation.
+        """
+        gem5_logger.info("Gracefully terminating the gem5 simulation.")
+        try:
+            self._gem5_util("exit")
+            self.gem5simulation.wait()
+        except EOF:
+            pass
+        gem5_logger.info("Removing the temporary directory")
+        try:
+            shutil.rmtree(self.gem5_interact_dir)
+        except OSError:
+            gem5_logger.warn("Failed to remove the temporary directory!")
+
+        # Delete the lock file
+        os.remove(self.lock_file_name)
+
+    # Functions only to be called by the Gem5 connection itself
+    def _connect_gem5_platform(self, platform):
+        port = platform.gem5_port
+        gem5_simulation = platform.gem5
+        gem5_interact_dir = platform.gem5_interact_dir
+        gem5_out_dir = platform.gem5_out_dir
+
+        self.connect_gem5(port, gem5_simulation, gem5_interact_dir, gem5_out_dir)
+
+    # This function connects to the gem5 simulation
+    def connect_gem5(self, port, gem5_simulation, gem5_interact_dir,
+                      gem5_out_dir):
+        """
+        Connect to the telnet port of the gem5 simulation.
+
+        We connect, and wait for the prompt to be found. We do not use a timeout
+        for this, and wait for the prompt in a while loop as the gem5 simulation
+        can take many hours to reach a prompt when booting the system. We also
+        inject some newlines periodically to try and force gem5 to show a
+        prompt. Once the prompt has been found, we replace it with a unique
+        prompt to ensure that we are able to match it properly. We also disable
+        the echo as this simplifies parsing the output when executing commands
+        on the device.
+        """
+        host = socket.gethostname()
+        gem5_logger.info("Connecting to the gem5 simulation on port {}".format(port))
+
+        # Check if there is no on-going connection yet
+        lock_file_name = '{}{}_{}.LOCK'.format(self.lock_directory, host, port)
+        if os.path.isfile(lock_file_name):
+            # There is already a connection to this gem5 simulation
+            raise TargetError('There is already a connection to the gem5 '
+                              'simulation using port {} on {}!'
+                              .format(port, host))
+
+        # Connect to the gem5 telnet port. Use a short timeout here.
+        attempts = 0
+        while attempts < 10:
+            attempts += 1
+            try:
+                self.conn = TelnetPxssh(original_prompt=None)
+                self.conn.login(host, self.username, port=port,
+                                login_timeout=10, auto_prompt_reset=False)
+                break
+            except pxssh.ExceptionPxssh:
+                pass
+        else:
+            gem5_simulation.kill()
+            raise TargetError("Failed to connect to the gem5 telnet session.")
+
+        gem5_logger.info("Connected! Waiting for prompt...")
+
+        # Create the lock file
+        self.lock_file_name = lock_file_name
+        open(self.lock_file_name, 'w').close() # Similar to touch
+        gem5_logger.info("Created lock file {} to prevent reconnecting to "
+                         "same simulation".format(self.lock_file_name))
+
+        # We need to find the prompt. It might be different if we are resuming
+        # from a checkpoint. Therefore, we test multiple options here.
+        prompt_found = False
+        while not prompt_found:
+            try:
+                self._login_to_device()
+            except TIMEOUT:
+                pass
+            try:
+                # Try and force a prompt to be shown
+                self.conn.send('\n')
+                self.conn.expect([r'# ', self.conn.UNIQUE_PROMPT, r'\[PEXPECT\][\\\$\#]+ '], timeout=60)
+                prompt_found = True
+            except TIMEOUT:
+                pass
+
+        gem5_logger.info("Successfully logged in")
+        gem5_logger.info("Setting unique prompt...")
+
+        self.conn.set_unique_prompt()
+        self.conn.prompt()
+        gem5_logger.info("Prompt found and replaced with a unique string")
+
+        # We check that the prompt is what we think it should be. If not, we
+        # need to update the regex we use to match.
+        self._find_prompt()
+
+        self.conn.setecho(False)
+        self._sync_gem5_shell()
+
+        # Fully connected to gem5 simulation
+        self.gem5_interact_dir = gem5_interact_dir
+        self.gem5_out_dir = gem5_out_dir
+        self.gem5simulation = gem5_simulation
+
+        # Ready for interaction now
+        self.ready = True
+
+    def _login_to_device(self):
+        """
+        Login to device, will be overwritten if there is an actual login
+        """
+        pass
+
+    def _find_prompt(self):
+        prompt = r'\[PEXPECT\][\\\$\#]+ '
+        synced = False
+        while not synced:
+            self.conn.send('\n')
+            i = self.conn.expect([prompt, self.conn.UNIQUE_PROMPT, r'[\$\#] '], timeout=self.default_timeout)
+            if i == 0:
+                synced = True
+            elif i == 1:
+                prompt = self.conn.UNIQUE_PROMPT
+                synced = True
+            else:
+                prompt = re.sub(r'\$', r'\\\$', self.conn.before.strip() + self.conn.after.strip())
+                prompt = re.sub(r'\#', r'\\\#', prompt)
+                prompt = re.sub(r'\[', r'\[', prompt)
+                prompt = re.sub(r'\]', r'\]', prompt)
+
+        self.conn.PROMPT = prompt
+
+    def _sync_gem5_shell(self):
+        """
+        Synchronise with the gem5 shell.
+
+        Write some unique text to the gem5 device to allow us to synchronise
+        with the shell output. We actually get two prompts so we need to match
+        both of these.
+        """
+        gem5_logger.debug("Sending Sync")
+        self.conn.send("echo \*\*sync\*\*\n")
+        self.conn.expect(r"\*\*sync\*\*", timeout=self.default_timeout)
+        self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)
+        self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)
+
+    def _gem5_util(self, command):
+        """ Execute a gem5 utility command using the m5 binary on the device """
+        if self.m5_path is None:
+            raise TargetError('Path to m5 binary on simulated system  is not set!')
+        self._gem5_shell('{} {}'.format(self.m5_path, command))
+
+    def _gem5_shell(self, command, as_root=False, timeout=None, check_exit_code=True, sync=True):  # pylint: disable=R0912
+        """
+        Execute a command in the gem5 shell
+
+        This wraps the telnet connection to gem5 and processes the raw output.
+
+        This method waits for the shell to return, and then will try and
+        separate the output from the command from the command itself. If this
+        fails, warn, but continue with the potentially wrong output.
+
+        The exit code is also checked by default, and non-zero exit codes will
+        raise a TargetError.
+        """
+        if sync:
+            self._sync_gem5_shell()
+
+        gem5_logger.debug("gem5_shell command: {}".format(command))
+
+        # Send the actual command
+        self.conn.send("{}\n".format(command))
+
+        # Wait for the response. We just sit here and wait for the prompt to
+        # appear, as gem5 might take a long time to provide the output. This
+        # avoids timeout issues.
+        command_index = -1
+        while command_index == -1:
+            if self.conn.prompt():
+                output = re.sub(r' \r([^\n])', r'\1', self.conn.before)
+                output = re.sub(r'[\b]', r'', output)
+                # Deal with line wrapping
+                output = re.sub(r'[\r].+?<', r'', output)
+                command_index = output.find(command)
+
+                # If we have -1, then we cannot match the command, but the
+                # prompt has returned. Hence, we have a bit of an issue. We
+                # warn, and return the whole output.
+                if command_index == -1:
+                    gem5_logger.warn("gem5_shell: Unable to match command in "
+                                     "command output. Expect parsing errors!")
+                    command_index = 0
+
+        output = output[command_index + len(command):].strip()
+
+        # It is possible that gem5 will echo the command. Therefore, we need to
+        # remove that too!
+        command_index = output.find(command)
+        if command_index != -1:
+            output = output[command_index + len(command):].strip()
+
+        gem5_logger.debug("gem5_shell output: {}".format(output))
+
+        # We get a second prompt. Hence, we need to eat one to make sure that we
+        # stay in sync. If we do not do this, we risk getting out of sync for
+        # slower simulations.
+        self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)
+
+        if check_exit_code:
+            exit_code_text = self._gem5_shell('echo $?', as_root=as_root,
+                                             timeout=timeout, check_exit_code=False,
+                                             sync=False)
+            try:
+                exit_code = int(exit_code_text.split()[0])
+                if exit_code:
+                    message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}'
+                    raise TargetError(message.format(exit_code, command, output))
+            except (ValueError, IndexError):
+                gem5_logger.warning('Could not get exit code for "{}",\ngot: "{}"'.format(command, exit_code_text))
+
+        return output
+
+    def _mount_virtio(self):
+        """
+        Mount the VirtIO device in the simulated system.
+        """
+        gem5_logger.info("Mounting VirtIO device in simulated system")
+
+        self._gem5_shell('su -c "mkdir -p {}" root'.format(self.gem5_input_dir))
+        mount_command = "mount -t 9p -o trans=virtio,version=9p2000.L,aname={} gem5 {}".format(self.gem5_interact_dir, self.gem5_input_dir)
+        self._gem5_shell(mount_command)
+
+    def _move_to_temp_dir(self, source):
+        """
+        Move a file to the temporary directory on the host for copying to the
+        gem5 device
+        """
+        command = "cp {} {}".format(source, self.gem5_interact_dir)
+        gem5_logger.debug("Local copy command: {}".format(command))
+        subprocess.call(command.split())
+        subprocess.call("sync".split())
+
+    def _check_ready(self):
+        """
+        Check if the gem5 platform is ready
+        """
+        if not self.ready:
+            raise TargetError('Gem5 is not ready to interact yet')
+
+    def _wait_for_boot(self):
+        pass
+
+    def _probe_file(self, filepath):
+        """
+        Internal method to check if the target has a certain file
+        """
+        command = 'if [ -e \'{}\' ]; then echo 1; else echo 0; fi'
+        output = self.execute(command.format(filepath), as_root=self.is_rooted)
+        return boolean(output.strip())
+
+
+class LinuxGem5Connection(Gem5Connection):
+
+    def _login_to_device(self):
+        gem5_logger.info("Trying to log in to gem5 device")
+        login_prompt = ['login:', 'AEL login:', 'username:', 'aarch64-gem5 login:']
+        login_password_prompt = ['password:']
+        # Wait for the login prompt
+        prompt = login_prompt + [self.conn.UNIQUE_PROMPT]
+        i = self.conn.expect(prompt, timeout=10)
+        # Check if we are already at a prompt, or if we need to log in.
+        if i < len(prompt) - 1:
+            self.conn.sendline("{}".format(self.username))
+            password_prompt = login_password_prompt + [r'# ', self.conn.UNIQUE_PROMPT]
+            j = self.conn.expect(password_prompt, timeout=self.default_timeout)
+            if j < len(password_prompt) - 2:
+                self.conn.sendline("{}".format(self.password))
+                self.conn.expect([r'# ', self.conn.UNIQUE_PROMPT], timeout=self.default_timeout)
+
+
+
+class AndroidGem5Connection(Gem5Connection):
+
+    def _wait_for_boot(self):
+        """
+        Wait for the system to boot
+
+        We monitor the sys.boot_completed and service.bootanim.exit system
+        properties to determine when the system has finished booting. In the
+        event that we cannot coerce the result of service.bootanim.exit to an
+        integer, we assume that the boot animation was disabled and do not wait
+        for it to finish.
+
+        """
+        gem5_logger.info("Waiting for Android to boot...")
+        while True:
+            booted = False
+            anim_finished = True  # Assume boot animation was disabled on except
+            try:
+                booted = (int('0' + self._gem5_shell('getprop sys.boot_completed', check_exit_code=False).strip()) == 1)
+                anim_finished = (int(self._gem5_shell('getprop service.bootanim.exit', check_exit_code=False).strip()) == 1)
+            except ValueError:
+                pass
+            if booted and anim_finished:
+                break
+            time.sleep(60)
+
+        gem5_logger.info("Android booted")
+
 def _give_password(password, command):
     if not sshpass:
         raise HostError('Must have sshpass installed on the host in order to use password-based auth.')

doc/connection.rst

@@ -48,7 +48,7 @@ class that implements the following methods.
    :param timeout: Timeout (in seconds) for the execution of the command. If
        specified, an exception will be raised if execution does not complete
        with the specified period.
-   :param check_exit_code: If ``True`` the exit code (on connected device) 
+   :param check_exit_code: If ``True`` the exit code (on connected device)
        from execution of the command will be checked, and an exception will be
        raised if it is not ``0``.
    :param as_root: The command will be executed as root. This will fail on
@@ -68,9 +68,9 @@ class that implements the following methods.
        unrooted connected devices.
 
    .. note:: This **will block the connection** until the command completes.
-   
+
 .. note:: The above methods are directly wrapped by :class:`Target` methods,
-          however note that some of the defaults are different. 
+          however note that some of the defaults are different.
 
 .. method:: cancel_running_command(self)
 
@@ -104,7 +104,7 @@ Connection Types
                    combination. To see connected devices, you can run ``adb
                    devices`` on the host.
     :param timeout: Connection timeout in seconds. If a connection to the device
-                    is not esblished within this period, :class:`HostError` 
+                    is not esblished within this period, :class:`HostError`
                     is raised.
 
 
@@ -120,10 +120,10 @@ Connection Types
                      .. note:: In order to user password-based authentication,
                                ``sshpass`` utility must be installed on the
                                system.
-    
+
     :param keyfile: Path to the SSH private key to be used for the connection.
 
-                    .. note:: ``keyfile`` and ``password`` can't be specified 
+                    .. note:: ``keyfile`` and ``password`` can't be specified
                               at the same time.
 
     :param port: TCP port on which SSH server is litening on the remoted device.
@@ -174,9 +174,67 @@ Connection Types
 
 
     :param keep_password: If this is ``True`` (the default) user's password will
-                          be cached in memory after it is first requested. 
+                          be cached in memory after it is first requested.
     :param unrooted: If set to ``True``, the platform will be assumed to be
                      unrooted without testing for root. This is useful to avoid
                      blocking on password request in scripts.
     :param password: Specify password on connection creation rather than
                      prompting for it.
+
+
+.. class:: Gem5Connection(platform, host=None, username=None, password=None,\
+                          timeout=None, password_prompt=None,\
+                          original_prompt=None)
+
+    A connection to a gem5 simulation using a local Telnet connection.
+
+    .. note:: Some of the following input parameters are optional and will be ignored during
+              initialisation. They were kept to keep the anology with a :class:`TelnetConnection`
+              (i.e. ``host``, `username``, ``password``, ``port``,
+              ``password_prompt`` and ``original_promp``)
+
+
+    :param host: Host on which the gem5 simulation is running
+
+                     .. note:: Even thought the input parameter for the ``host``
+                               will be ignored, the gem5 simulation needs to on
+                               the same host as the user as the user is
+                               currently on, so if the host given as input
+                               parameter is not the same as the actual host, a
+                               ``TargetError`` will be raised to prevent
+                               confusion.
+
+    :param username: Username in the simulated system
+    :param password: No password required in gem5 so does not need to be set
+    :param port: Telnet port to connect to gem5. This does not need to be set
+                 at initialisation as this will either be determined by the
+                 :class:`Gem5SimulationPlatform` or can be set using the
+                 :func:`connect_gem5` method
+    :param timeout: Timeout for the connection in seconds. Gem5 has high
+                    latencies so unless the timeout given by the user via
+                    this input parameter is higher than the default one
+                    (3600 seconds), this input parameter will be ignored.
+    :param password_prompt: A string with password prompt
+    :param original_prompt: A regex for the shell prompt
+
+There are two classes that inherit from :class:`Gem5Connection`:
+:class:`AndroidGem5Connection` and :class:`LinuxGem5Connection`.
+They inherit *almost* all methods from the parent class, without altering them.
+The only methods discussed belows are those that will be overwritten by the
+:class:`LinuxGem5Connection` and :class:`AndroidGem5Connection` respectively.
+
+.. class:: LinuxGem5Connection
+
+    A connection to a gem5 simulation that emulates a Linux system.
+
+.. method:: _login_to_device(self)
+
+    Login to the gem5 simulated system.
+
+.. class:: AndroidGem5Connection
+
+    A connection to a gem5 simulation that emulates an Android system.
+
+.. method:: _wait_for_boot(self)
+
+    Wait for the gem5 simulated system to have booted and finished the booting animation.

doc/index.rst

@@ -21,6 +21,7 @@ Contents:
    instrumentation
    platform
    connection
+   platform
 
 Indices and tables
 ==================

doc/platform.rst

@@ -110,3 +110,62 @@ support additional configuration:
                           just that the services needed to establish a
                           connection (e.g. sshd or adbd) are up.
 
+
+.. _gem5-platform:
+
+Gem5 Simulation Platform
+------------------------
+
+By initialising a Gem5SimulationPlatform, devlib will start a gem5 simulation (based upon the
+arguments the user provided) and then connect to it using :class:`Gem5Connection`.
+Using the methods discussed above, some methods of the :class:`Target` will be altered
+slightly to better suit gem5.
+
+.. class:: Gem5SimulationPlatform(name, host_output_dir, gem5_bin, gem5_args, gem5_virtio, gem5_telnet_port=None)
+
+    During initialisation the gem5 simulation will be kicked off (based upon the arguments
+    provided by the user) and the telnet port used by the gem5 simulation will be intercepted
+    and stored for use by the :class:`Gem5Connection`.
+
+    :param name: Platform name
+
+    :param host_output_dir: Path on the host where the gem5 outputs will be placed (e.g. stats file)
+
+    :param gem5_bin: gem5 binary
+
+    :param gem5_args: Arguments to be passed onto gem5 such as config file etc.
+
+    :param gem5_virtio: Arguments to be passed onto gem5 in terms of the virtIO device used
+                        to transfer files between the host and the gem5 simulated system.
+
+    :param gem5_telnet_port: Not yet in use as it would be used in future implementations
+                             of devlib in which the user could use the platform to pick
+                             up an existing and running simulation.
+
+
+.. method:: Gem5SimulationPlatform.init_target_connection([target])
+
+    Based upon the OS defined in the :class:`Target`, the type of :class:`Gem5Connection`
+    will be set (:class:`AndroidGem5Connection` or :class:`AndroidGem5Connection`).
+
+.. method:: Gem5SimulationPlatform.update_from_target([target])
+
+    This method provides specific setup procedures for a gem5 simulation. First of all, the m5
+    binary will be installed on the guest (if it is not present). Secondly, three methods
+    in the :class:`Target` will be monkey-patched:
+
+            - **reboot**: this is not supported in gem5
+            - **reset**: this is not supported in gem5
+            - **capture_screen**: gem5 might already have screencaps so the
+              monkey-patched method will first try to
+              transfer the existing screencaps.
+              In case that does not work, it will fall back
+              to the original :class:`Target` implementation
+              of :func:`capture_screen`.
+
+    Finally, it will call the parent implementation of :func:`update_from_target`.
+
+.. method:: Gem5SimulationPlatform.setup([target])
+
+    The m5 binary be installed, if not yet installed on the gem5 simulated system.
+    It will also resize the gem5 shell, to avoid line wrapping issues.