mirror of
https://github.com/ARM-software/devlib.git
synced 2025-01-31 02:00:45 +00:00
marcbonnici
GitHub
commit
8abdfdc1ef
@ -7,6 +7,7 @@ from devlib.module import get_module, register_module
|
||||
|
||||
from devlib.platform import Platform
|
||||
from devlib.platform.arm import TC2, Juno, JunoEnergyInstrument
|
||||
from devlib.platform.gem5 import Gem5SimulationPlatform
|
||||
|
||||
from devlib.instrument import Instrument, InstrumentChannel, Measurement, MeasurementsCsv
|
||||
from devlib.instrument import MEASUREMENT_TYPES, INSTANTANEOUS, CONTINUOUS
|
||||
@ -19,4 +20,4 @@ from devlib.trace.ftrace import FtraceCollector
|
||||
|
||||
from devlib.host import LocalConnection
|
||||
from devlib.utils.android import AdbConnection
|
||||
from devlib.utils.ssh import SshConnection, TelnetConnection
|
||||
from devlib.utils.ssh import SshConnection, TelnetConnection, Gem5Connection
|
||||
|
||||
BIN
devlib/bin/arm64/m5
Executable file
BIN
devlib/bin/arm64/m5
Executable file
Binary file not shown.
BIN
devlib/bin/armeabi/m5
Executable file
BIN
devlib/bin/armeabi/m5
Executable file
Binary file not shown.
@ -49,7 +49,8 @@ class LocalConnection(object):
|
||||
else:
|
||||
shutil.copy(source, dest)
|
||||
|
||||
def execute(self, command, timeout=None, check_exit_code=True, as_root=False):
|
||||
def execute(self, command, timeout=None, check_exit_code=True,
|
||||
as_root=False, strip_colors=True):
|
||||
self.logger.debug(command)
|
||||
if as_root:
|
||||
if self.unrooted:
|
||||
|
||||
@ -48,6 +48,11 @@ class Platform(object):
|
||||
self.name = self.model
|
||||
self._validate()
|
||||
|
||||
def setup(self, target):
|
||||
# May be overwritten by subclasses to provide platform-specific
|
||||
# setup procedures.
|
||||
pass
|
||||
|
||||
def _set_core_clusters_from_core_names(self):
|
||||
self.core_clusters = []
|
||||
clusters = []
|
||||
|
||||
282
devlib/platform/gem5.py
Normal file
282
devlib/platform/gem5.py
Normal file
@ -0,0 +1,282 @@
|
||||
# Copyright 2016 ARM Limited
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import shutil
|
||||
import time
|
||||
import types
|
||||
|
||||
from devlib.exception import TargetError
|
||||
from devlib.host import PACKAGE_BIN_DIRECTORY
|
||||
from devlib.platform import Platform
|
||||
from devlib.utils.ssh import AndroidGem5Connection, LinuxGem5Connection
|
||||
|
||||
class Gem5SimulationPlatform(Platform):
|
||||
|
||||
def __init__(self, name, host_output_dir, gem5_bin, gem5_args, gem5_virtio,
|
||||
gem5_telnet_port=None):
|
||||
|
||||
# First call the parent class
|
||||
super(Gem5SimulationPlatform, self).__init__(name=name)
|
||||
|
||||
# Start setting up the gem5 parameters/directories
|
||||
# The gem5 subprocess
|
||||
self.gem5 = None
|
||||
self.gem5_port = gem5_telnet_port or None
|
||||
self.stats_directory = host_output_dir
|
||||
self.gem5_out_dir = os.path.join(self.stats_directory, "gem5")
|
||||
self.gem5_interact_dir = '/tmp' # Host directory
|
||||
self.executable_dir = None # Device directory
|
||||
self.working_dir = None # Device directory
|
||||
self.stdout_file = None
|
||||
self.stderr_file = None
|
||||
self.stderr_filename = None
|
||||
if self.gem5_port is None:
|
||||
# Allows devlib to pick up already running simulations
|
||||
self.start_gem5_simulation = True
|
||||
else:
|
||||
self.start_gem5_simulation = False
|
||||
|
||||
# Find the first one that does not exist. Ensures that we do not re-use
|
||||
# the directory used by someone else.
|
||||
for i in xrange(sys.maxint):
|
||||
directory = os.path.join(self.gem5_interact_dir, "wa_{}".format(i))
|
||||
try:
|
||||
os.stat(directory)
|
||||
continue
|
||||
except OSError:
|
||||
break
|
||||
self.gem5_interact_dir = directory
|
||||
self.logger.debug("Using {} as the temporary directory."
|
||||
.format(self.gem5_interact_dir))
|
||||
|
||||
# Parameters passed onto gem5
|
||||
self.gem5args_binary = gem5_bin
|
||||
self.gem5args_args = gem5_args
|
||||
self.gem5args_virtio = gem5_virtio
|
||||
self._check_gem5_command()
|
||||
|
||||
# Start the interaction with gem5
|
||||
self._start_interaction_gem5()
|
||||
|
||||
def _check_gem5_command(self):
|
||||
"""
|
||||
Check if the command to start gem5 makes sense
|
||||
"""
|
||||
if self.gem5args_binary is None:
|
||||
raise TargetError('Please specify a gem5 binary.')
|
||||
if self.gem5args_args is None:
|
||||
raise TargetError('Please specify the arguments passed on to gem5.')
|
||||
self.gem5args_virtio = str(self.gem5args_virtio).format(self.gem5_interact_dir)
|
||||
if self.gem5args_virtio is None:
|
||||
raise TargetError('Please specify arguments needed for virtIO.')
|
||||
|
||||
def _start_interaction_gem5(self):
|
||||
"""
|
||||
Starts the interaction of devlib with gem5.
|
||||
"""
|
||||
|
||||
# First create the input and output directories for gem5
|
||||
if self.start_gem5_simulation:
|
||||
# Create the directory to send data to/from gem5 system
|
||||
self.logger.info("Creating temporary directory for interaction "
|
||||
" with gem5 via virtIO: {}"
|
||||
.format(self.gem5_interact_dir))
|
||||
os.mkdir(self.gem5_interact_dir)
|
||||
|
||||
# Create the directory for gem5 output (stats files etc)
|
||||
if not os.path.exists(self.stats_directory):
|
||||
os.mkdir(self.stats_directory)
|
||||
if os.path.exists(self.gem5_out_dir):
|
||||
raise TargetError("The gem5 stats directory {} already "
|
||||
"exists.".format(self.gem5_out_dir))
|
||||
else:
|
||||
os.mkdir(self.gem5_out_dir)
|
||||
|
||||
# We need to redirect the standard output and standard error for the
|
||||
# gem5 process to a file so that we can debug when things go wrong.
|
||||
f = os.path.join(self.gem5_out_dir, 'stdout')
|
||||
self.stdout_file = open(f, 'w')
|
||||
f = os.path.join(self.gem5_out_dir, 'stderr')
|
||||
self.stderr_file = open(f, 'w')
|
||||
# We need to keep this so we can check which port to use for the
|
||||
# telnet connection.
|
||||
self.stderr_filename = f
|
||||
|
||||
# Start gem5 simulation
|
||||
self.logger.info("Starting the gem5 simulator")
|
||||
|
||||
command_line = "{} --outdir={} {} {}".format(self.gem5args_binary,
|
||||
self.gem5_out_dir,
|
||||
self.gem5args_args,
|
||||
self.gem5args_virtio)
|
||||
self.logger.debug("gem5 command line: {}".format(command_line))
|
||||
self.gem5 = subprocess.Popen(command_line.split(),
|
||||
stdout=self.stdout_file,
|
||||
stderr=self.stderr_file)
|
||||
|
||||
else:
|
||||
# The simulation should already be running
|
||||
# Need to dig up the (1) gem5 simulation in question (2) its input
|
||||
# and output directories (3) virtio setting
|
||||
self._intercept_existing_gem5()
|
||||
|
||||
# As the gem5 simulation is running now or was already running
|
||||
# we now need to find out which telnet port it uses
|
||||
self._intercept_telnet_port()
|
||||
|
||||
def _intercept_existing_gem5(self):
|
||||
"""
|
||||
Intercept the information about a running gem5 simulation
|
||||
e.g. pid, input directory etc
|
||||
"""
|
||||
self.logger("This functionality is not yet implemented")
|
||||
raise TargetError()
|
||||
|
||||
def _intercept_telnet_port(self):
|
||||
"""
|
||||
Intercept the telnet port of a running gem5 simulation
|
||||
"""
|
||||
|
||||
if self.gem5 is None:
|
||||
raise TargetError('The platform has no gem5 simulation! '
|
||||
'Something went wrong')
|
||||
while self.gem5_port is None:
|
||||
# Check that gem5 is running!
|
||||
if self.gem5.poll():
|
||||
raise TargetError("The gem5 process has crashed with error code {}!".format(self.gem5.poll()))
|
||||
|
||||
# Open the stderr file
|
||||
with open(self.stderr_filename, 'r') as f:
|
||||
for line in f:
|
||||
m = re.search(r"Listening for system connection on port (?P<port>\d+)", line)
|
||||
if m:
|
||||
port = int(m.group('port'))
|
||||
if port >= 3456 and port < 5900:
|
||||
self.gem5_port = port
|
||||
break
|
||||
# Check if the sockets are not disabled
|
||||
m = re.search(r"Sockets disabled, not accepting terminal connections", line)
|
||||
if m:
|
||||
raise TargetError("The sockets have been disabled!"
|
||||
"Pass --listener-mode=on to gem5")
|
||||
else:
|
||||
time.sleep(1)
|
||||
|
||||
def init_target_connection(self, target):
|
||||
"""
|
||||
Update the type of connection in the target from here
|
||||
"""
|
||||
if target.os == 'linux':
|
||||
target.conn_cls = LinuxGem5Connection
|
||||
else:
|
||||
target.conn_cls = AndroidGem5Connection
|
||||
|
||||
def setup(self, target):
|
||||
"""
|
||||
Deploy m5 if not yet installed
|
||||
"""
|
||||
m5_path = target.get_installed('m5')
|
||||
if m5_path is None:
|
||||
m5_path = self._deploy_m5(target)
|
||||
target.conn.m5_path = m5_path
|
||||
|
||||
# Set the terminal settings for the connection to gem5
|
||||
self._resize_shell(target)
|
||||
|
||||
def update_from_target(self, target):
|
||||
"""
|
||||
Set the m5 path and if not yet installed, deploy m5
|
||||
Overwrite certain methods in the target that either can be done
|
||||
more efficiently by gem5 or don't exist in gem5
|
||||
"""
|
||||
m5_path = target.get_installed('m5')
|
||||
if m5_path is None:
|
||||
m5_path = self._deploy_m5(target)
|
||||
target.conn.m5_path = m5_path
|
||||
|
||||
# Overwrite the following methods (monkey-patching)
|
||||
self.logger.debug("Overwriting the 'capture_screen' method in target")
|
||||
# Housekeeping to prevent recursion
|
||||
setattr(target, 'target_impl_capture_screen', target.capture_screen)
|
||||
target.capture_screen = types.MethodType(_overwritten_capture_screen, target)
|
||||
self.logger.debug("Overwriting the 'reset' method in target")
|
||||
target.reset = types.MethodType(_overwritten_reset, target)
|
||||
self.logger.debug("Overwriting the 'reboot' method in target")
|
||||
target.reboot = types.MethodType(_overwritten_reboot, target)
|
||||
|
||||
# Call the general update_from_target implementation
|
||||
super(Gem5SimulationPlatform, self).update_from_target(target)
|
||||
|
||||
def gem5_capture_screen(self, filepath):
|
||||
file_list = os.listdir(self.gem5_out_dir)
|
||||
screen_caps = []
|
||||
for f in file_list:
|
||||
if '.bmp' in f:
|
||||
screen_caps.append(f)
|
||||
|
||||
successful_capture = False
|
||||
if len(screen_caps) == 1:
|
||||
# Bail out if we do not have image, and resort to the slower, built
|
||||
# in method.
|
||||
try:
|
||||
import Image
|
||||
gem5_image = os.path.join(self.gem5_out_dir, screen_caps[0])
|
||||
temp_image = os.path.join(self.gem5_out_dir, "file.png")
|
||||
im = Image.open(gem5_image)
|
||||
im.save(temp_image, "PNG")
|
||||
shutil.copy(temp_image, filepath)
|
||||
os.remove(temp_image)
|
||||
gem5_logger.info("capture_screen: using gem5 screencap")
|
||||
successful_capture = True
|
||||
|
||||
except (shutil.Error, ImportError, IOError):
|
||||
pass
|
||||
|
||||
return successful_capture
|
||||
|
||||
def _deploy_m5(self, target):
|
||||
# m5 is not yet installed so install it
|
||||
host_executable = os.path.join(PACKAGE_BIN_DIRECTORY,
|
||||
target.abi, 'm5')
|
||||
return target.install(host_executable)
|
||||
|
||||
def _resize_shell(self, target):
|
||||
"""
|
||||
Resize the shell to avoid line wrapping issues.
|
||||
|
||||
"""
|
||||
# Try and avoid line wrapping as much as possible.
|
||||
target.execute('{} stty columns 1024'.format(target.busybox))
|
||||
target.execute('reset', check_exit_code=False)
|
||||
|
||||
# Methods that will be monkey-patched onto the target
|
||||
def _overwritten_reset(self):
|
||||
raise TargetError('Resetting is not allowed on gem5 platforms!')
|
||||
|
||||
def _overwritten_reboot(self):
|
||||
raise TargetError('Rebooting is not allowed on gem5 platforms!')
|
||||
|
||||
def _overwritten_capture_screen(self, filepath):
|
||||
connection_screencapped = self.platform.gem5_capture_screen(filepath)
|
||||
if connection_screencapped == False:
|
||||
# The connection was not able to capture the screen so use the target
|
||||
# implementation
|
||||
self.logger.debug('{} was not able to screen cap, using the original target implementation'.format(self.platform.__class__.__name__))
|
||||
self.target_impl_capture_screen(filepath)
|
||||
|
||||
|
||||
@ -152,7 +152,22 @@ class Target(object):
|
||||
conn_cls=None,
|
||||
):
|
||||
self.connection_settings = connection_settings or {}
|
||||
self.platform = platform or Platform()
|
||||
# Set self.platform: either it's given directly (by platform argument)
|
||||
# or it's given in the connection_settings argument
|
||||
# If neither, create default Platform()
|
||||
if platform is None:
|
||||
self.platform = self.connection_settings.get('platform', Platform())
|
||||
else:
|
||||
self.platform = platform
|
||||
# Check if the user hasn't given two different platforms
|
||||
if 'platform' in self.connection_settings:
|
||||
if connection_settings['platform'] is not platform:
|
||||
raise TargetError('Platform specified in connection_settings '
|
||||
'({}) differs from that directly passed '
|
||||
'({})!)'
|
||||
.format(connection_settings['platform'],
|
||||
self.platform))
|
||||
self.connection_settings['platform'] = self.platform
|
||||
self.working_directory = working_directory
|
||||
self.executables_directory = executables_directory
|
||||
self.modules = modules or []
|
||||
@ -222,6 +237,9 @@ class Target(object):
|
||||
for host_exe in (executables or []): # pylint: disable=superfluous-parens
|
||||
self.install(host_exe)
|
||||
|
||||
# Check for platform dependent setup procedures
|
||||
self.platform.setup(self)
|
||||
|
||||
# Initialize modules which requires Buxybox (e.g. shutil dependent tasks)
|
||||
self._update_modules('setup')
|
||||
|
||||
|
||||
@ -186,7 +186,7 @@ class AdbConnection(object):
|
||||
self.ls_command = 'ls'
|
||||
logger.info("ls command is set to {}".format(self.ls_command))
|
||||
|
||||
def __init__(self, device=None, timeout=None):
|
||||
def __init__(self, device=None, timeout=None, platform=None):
|
||||
self.timeout = timeout if timeout is not None else self.default_timeout
|
||||
if device is None:
|
||||
device = adb_get_device(timeout=timeout)
|
||||
@ -216,7 +216,8 @@ class AdbConnection(object):
|
||||
command = "pull '{}' '{}'".format(source, dest)
|
||||
return adb_command(self.device, command, timeout=timeout)
|
||||
|
||||
def execute(self, command, timeout=None, check_exit_code=False, as_root=False):
|
||||
def execute(self, command, timeout=None, check_exit_code=False,
|
||||
as_root=False, strip_colors=True):
|
||||
return adb_shell(self.device, command, timeout, check_exit_code,
|
||||
as_root, self.newline_separator)
|
||||
|
||||
|
||||
@ -22,6 +22,7 @@ import re
|
||||
import threading
|
||||
import tempfile
|
||||
import shutil
|
||||
import socket
|
||||
import time
|
||||
|
||||
import pexpect
|
||||
@ -34,14 +35,16 @@ from pexpect import EOF, TIMEOUT, spawn
|
||||
|
||||
from devlib.exception import HostError, TargetError, TimeoutError
|
||||
from devlib.utils.misc import which, strip_bash_colors, escape_single_quotes, check_output
|
||||
from devlib.utils.types import boolean
|
||||
|
||||
|
||||
ssh = None
|
||||
scp = None
|
||||
sshpass = None
|
||||
|
||||
logger = logging.getLogger('ssh')
|
||||
|
||||
logger = logging.getLogger('ssh')
|
||||
gem5_logger = logging.getLogger('gem5-connection')
|
||||
|
||||
def ssh_get_shell(host, username, password=None, keyfile=None, port=None, timeout=10, telnet=False, original_prompt=None):
|
||||
_check_env()
|
||||
@ -91,18 +94,20 @@ class TelnetPxssh(pxssh.pxssh):
|
||||
|
||||
spawn._spawn(self, cmd) # pylint: disable=protected-access
|
||||
|
||||
if password is None:
|
||||
i = self.expect([self.original_prompt, 'Login timed out'], timeout=login_timeout)
|
||||
else:
|
||||
try:
|
||||
i = self.expect('(?i)(?:password)', timeout=login_timeout)
|
||||
if i == 0:
|
||||
self.sendline(password)
|
||||
i = self.expect([self.original_prompt, 'Login incorrect'], timeout=login_timeout)
|
||||
else:
|
||||
raise pxssh.ExceptionPxssh('could not log in: did not see a password prompt')
|
||||
|
||||
if i:
|
||||
raise pxssh.ExceptionPxssh('could not log in: password was incorrect')
|
||||
except TIMEOUT:
|
||||
if not password:
|
||||
# No password promt before TIMEOUT & no password provided
|
||||
# so assume everything is okay
|
||||
pass
|
||||
else:
|
||||
raise pxssh.ExceptionPxssh('could not log in: did not see a password prompt')
|
||||
|
||||
if not self.sync_original_prompt(sync_multiplier):
|
||||
self.close()
|
||||
@ -155,6 +160,7 @@ class SshConnection(object):
|
||||
telnet=False,
|
||||
password_prompt=None,
|
||||
original_prompt=None,
|
||||
platform=None
|
||||
):
|
||||
self.host = host
|
||||
self.username = username
|
||||
@ -175,7 +181,8 @@ class SshConnection(object):
|
||||
source = '{}@{}:{}'.format(self.username, self.host, source)
|
||||
return self._scp(source, dest, timeout)
|
||||
|
||||
def execute(self, command, timeout=None, check_exit_code=True, as_root=False, strip_colors=True):
|
||||
def execute(self, command, timeout=None, check_exit_code=True,
|
||||
as_root=False, strip_colors=True): #pylint: disable=unused-argument
|
||||
try:
|
||||
with self.lock:
|
||||
output = self._execute_and_wait_for_prompt(command, timeout, as_root, strip_colors)
|
||||
@ -286,7 +293,7 @@ class TelnetConnection(SshConnection):
|
||||
timeout=None,
|
||||
password_prompt=None,
|
||||
original_prompt=None,
|
||||
):
|
||||
platform=None):
|
||||
self.host = host
|
||||
self.username = username
|
||||
self.password = password
|
||||
@ -299,6 +306,503 @@ class TelnetConnection(SshConnection):
|
||||
self.conn = ssh_get_shell(host, username, password, None, port, timeout, True, original_prompt)
|
||||
|
||||
|
||||
class Gem5Connection(TelnetConnection):
|
||||
|
||||
def __init__(self,
|
||||
platform,
|
||||
host=None,
|
||||
username=None,
|
||||
password=None,
|
||||
port=None,
|
||||
timeout=None,
|
||||
password_prompt=None,
|
||||
original_prompt=None,
|
||||
):
|
||||
if host is not None:
|
||||
host_system = socket.gethostname()
|
||||
if host_system != host:
|
||||
raise TargetError("Gem5Connection can only connect to gem5 "
|
||||
"simulations on your current host, which "
|
||||
"differs from the one given {}!"
|
||||
.format(host_system, host))
|
||||
if username is not None and username != 'root':
|
||||
raise ValueError('User should be root in gem5!')
|
||||
if password is not None and password != '':
|
||||
raise ValueError('No password needed in gem5!')
|
||||
self.username = 'root'
|
||||
self.is_rooted = True
|
||||
self.password = None
|
||||
self.port = None
|
||||
# Long timeouts to account for gem5 being slow
|
||||
# Can be overriden if the given timeout is longer
|
||||
self.default_timeout = 3600
|
||||
if timeout is not None:
|
||||
if timeout > self.default_timeout:
|
||||
logger.info('Overwriting the default timeout of gem5 ({})'
|
||||
' to {}'.format(self.default_timeout, timeout))
|
||||
self.default_timeout = timeout
|
||||
else:
|
||||
logger.info('Ignoring the given timeout --> gem5 needs longer timeouts')
|
||||
self.ready_timeout = self.default_timeout * 3
|
||||
# Counterpart in gem5_interact_dir
|
||||
self.gem5_input_dir = '/mnt/host/'
|
||||
# Location of m5 binary in the gem5 simulated system
|
||||
self.m5_path = None
|
||||
# Actual telnet connection to gem5 simulation
|
||||
self.conn = None
|
||||
# Flag to indicate the gem5 device is ready to interact with the
|
||||
# outer world
|
||||
self.ready = False
|
||||
# Lock file to prevent multiple connections to same gem5 simulation
|
||||
# (gem5 does not allow this)
|
||||
self.lock_directory = '/tmp/'
|
||||
self.lock_file_name = None # Will be set once connected to gem5
|
||||
|
||||
# These parameters will be set by either the method to connect to the
|
||||
# gem5 platform or directly to the gem5 simulation
|
||||
# Intermediate directory to push things to gem5 using VirtIO
|
||||
self.gem5_interact_dir = None
|
||||
# Directory to store output from gem5 on the host
|
||||
self.gem5_out_dir = None
|
||||
# Actual gem5 simulation
|
||||
self.gem5simulation = None
|
||||
|
||||
# Connect to gem5
|
||||
if platform:
|
||||
self._connect_gem5_platform(platform)
|
||||
|
||||
# Wait for boot
|
||||
self._wait_for_boot()
|
||||
|
||||
# Mount the virtIO to transfer files in/out gem5 system
|
||||
self._mount_virtio()
|
||||
|
||||
def set_hostinteractdir(self, indir):
|
||||
logger.info('Setting hostinteractdir from {} to {}'
|
||||
.format(self.gem5_input_dir, indir))
|
||||
self.gem5_input_dir = indir
|
||||
|
||||
def push(self, source, dest, timeout=None):
|
||||
"""
|
||||
Push a file to the gem5 device using VirtIO
|
||||
|
||||
The file to push to the device is copied to the temporary directory on
|
||||
the host, before being copied within the simulation to the destination.
|
||||
Checks, in the form of 'ls' with error code checking, are performed to
|
||||
ensure that the file is copied to the destination.
|
||||
"""
|
||||
# First check if the connection is set up to interact with gem5
|
||||
self._check_ready()
|
||||
|
||||
filename = os.path.basename(source)
|
||||
logger.debug("Pushing {} to device.".format(source))
|
||||
logger.debug("gem5interactdir: {}".format(self.gem5_interact_dir))
|
||||
logger.debug("dest: {}".format(dest))
|
||||
logger.debug("filename: {}".format(filename))
|
||||
|
||||
# We need to copy the file to copy to the temporary directory
|
||||
self._move_to_temp_dir(source)
|
||||
|
||||
# Dest in gem5 world is a file rather than directory
|
||||
if os.path.basename(dest) != filename:
|
||||
dest = os.path.join(dest, filename)
|
||||
# Back to the gem5 world
|
||||
self._gem5_shell("ls -al {}{}".format(self.gem5_input_dir, filename))
|
||||
self._gem5_shell("cat '{}''{}' > '{}'".format(self.gem5_input_dir,
|
||||
filename,
|
||||
dest))
|
||||
self._gem5_shell("sync")
|
||||
self._gem5_shell("ls -al {}".format(dest))
|
||||
self._gem5_shell("ls -al {}".format(self.gem5_input_dir))
|
||||
logger.debug("Push complete.")
|
||||
|
||||
def pull(self, source, dest, timeout=0): #pylint: disable=unused-argument
|
||||
"""
|
||||
Pull a file from the gem5 device using m5 writefile
|
||||
|
||||
The file is copied to the local directory within the guest as the m5
|
||||
writefile command assumes that the file is local. The file is then
|
||||
written out to the host system using writefile, prior to being moved to
|
||||
the destination on the host.
|
||||
"""
|
||||
# First check if the connection is set up to interact with gem5
|
||||
self._check_ready()
|
||||
|
||||
filename = os.path.basename(source)
|
||||
|
||||
logger.debug("pull_file {} {}".format(source, filename))
|
||||
# We don't check the exit code here because it is non-zero if the source
|
||||
# and destination are the same. The ls below will cause an error if the
|
||||
# file was not where we expected it to be.
|
||||
if os.path.dirname(source) != os.getcwd():
|
||||
self._gem5_shell("cat '{}' > '{}'".format(source, filename))
|
||||
self._gem5_shell("sync")
|
||||
self._gem5_shell("ls -la {}".format(filename))
|
||||
logger.debug('Finished the copy in the simulator')
|
||||
self._gem5_util("writefile {}".format(filename))
|
||||
|
||||
if 'cpu' not in filename:
|
||||
while not os.path.exists(os.path.join(self.gem5_out_dir, filename)):
|
||||
time.sleep(1)
|
||||
|
||||
# Perform the local move
|
||||
shutil.move(os.path.join(self.gem5_out_dir, filename), dest)
|
||||
logger.debug("Pull complete.")
|
||||
|
||||
def execute(self, command, timeout=1000, check_exit_code=True,
|
||||
as_root=False, strip_colors=True):
|
||||
"""
|
||||
Execute a command on the gem5 platform
|
||||
"""
|
||||
# First check if the connection is set up to interact with gem5
|
||||
self._check_ready()
|
||||
|
||||
output = self._gem5_shell(command, as_root=as_root)
|
||||
if strip_colors:
|
||||
output = strip_bash_colors(output)
|
||||
return output
|
||||
|
||||
def background(self, command, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.PIPE, as_root=False):
|
||||
# First check if the connection is set up to interact with gem5
|
||||
self._check_ready()
|
||||
|
||||
# Create the logfile for stderr/stdout redirection
|
||||
command_name = command.split(' ')[0].split('/')[-1]
|
||||
redirection_file = 'BACKGROUND_{}.log'.format(command_name)
|
||||
trial = 0
|
||||
while os.path.isfile(redirection_file):
|
||||
# Log file already exists so add to name
|
||||
redirection_file = 'BACKGROUND_{}{}.log'.format(command_name, trial)
|
||||
trial += 1
|
||||
|
||||
# Create the command to pass on to gem5 shell
|
||||
complete_command = '{} >> {} 2>&1 &'.format(command, redirection_file)
|
||||
output = self._gem5_shell(complete_command, as_root=as_root)
|
||||
output = strip_bash_colors(output)
|
||||
gem5_logger.info('STDERR/STDOUT of background command will be '
|
||||
'redirected to {}. Use target.pull() to '
|
||||
'get this file'.format(redirection_file))
|
||||
return output
|
||||
|
||||
def close(self):
|
||||
"""
|
||||
Close and disconnect from the gem5 simulation. Additionally, we remove
|
||||
the temporary directory used to pass files into the simulation.
|
||||
"""
|
||||
gem5_logger.info("Gracefully terminating the gem5 simulation.")
|
||||
try:
|
||||
self._gem5_util("exit")
|
||||
self.gem5simulation.wait()
|
||||
except EOF:
|
||||
pass
|
||||
gem5_logger.info("Removing the temporary directory")
|
||||
try:
|
||||
shutil.rmtree(self.gem5_interact_dir)
|
||||
except OSError:
|
||||
gem5_logger.warn("Failed to remove the temporary directory!")
|
||||
|
||||
# Delete the lock file
|
||||
os.remove(self.lock_file_name)
|
||||
|
||||
# Functions only to be called by the Gem5 connection itself
|
||||
def _connect_gem5_platform(self, platform):
|
||||
port = platform.gem5_port
|
||||
gem5_simulation = platform.gem5
|
||||
gem5_interact_dir = platform.gem5_interact_dir
|
||||
gem5_out_dir = platform.gem5_out_dir
|
||||
|
||||
self.connect_gem5(port, gem5_simulation, gem5_interact_dir, gem5_out_dir)
|
||||
|
||||
# This function connects to the gem5 simulation
|
||||
def connect_gem5(self, port, gem5_simulation, gem5_interact_dir,
|
||||
gem5_out_dir):
|
||||
"""
|
||||
Connect to the telnet port of the gem5 simulation.
|
||||
|
||||
We connect, and wait for the prompt to be found. We do not use a timeout
|
||||
for this, and wait for the prompt in a while loop as the gem5 simulation
|
||||
can take many hours to reach a prompt when booting the system. We also
|
||||
inject some newlines periodically to try and force gem5 to show a
|
||||
prompt. Once the prompt has been found, we replace it with a unique
|
||||
prompt to ensure that we are able to match it properly. We also disable
|
||||
the echo as this simplifies parsing the output when executing commands
|
||||
on the device.
|
||||
"""
|
||||
host = socket.gethostname()
|
||||
gem5_logger.info("Connecting to the gem5 simulation on port {}".format(port))
|
||||
|
||||
# Check if there is no on-going connection yet
|
||||
lock_file_name = '{}{}_{}.LOCK'.format(self.lock_directory, host, port)
|
||||
if os.path.isfile(lock_file_name):
|
||||
# There is already a connection to this gem5 simulation
|
||||
raise TargetError('There is already a connection to the gem5 '
|
||||
'simulation using port {} on {}!'
|
||||
.format(port, host))
|
||||
|
||||
# Connect to the gem5 telnet port. Use a short timeout here.
|
||||
attempts = 0
|
||||
while attempts < 10:
|
||||
attempts += 1
|
||||
try:
|
||||
self.conn = TelnetPxssh(original_prompt=None)
|
||||
self.conn.login(host, self.username, port=port,
|
||||
login_timeout=10, auto_prompt_reset=False)
|
||||
break
|
||||
except pxssh.ExceptionPxssh:
|
||||
pass
|
||||
else:
|
||||
gem5_simulation.kill()
|
||||
raise TargetError("Failed to connect to the gem5 telnet session.")
|
||||
|
||||
gem5_logger.info("Connected! Waiting for prompt...")
|
||||
|
||||
# Create the lock file
|
||||
self.lock_file_name = lock_file_name
|
||||
open(self.lock_file_name, 'w').close() # Similar to touch
|
||||
gem5_logger.info("Created lock file {} to prevent reconnecting to "
|
||||
"same simulation".format(self.lock_file_name))
|
||||
|
||||
# We need to find the prompt. It might be different if we are resuming
|
||||
# from a checkpoint. Therefore, we test multiple options here.
|
||||
prompt_found = False
|
||||
while not prompt_found:
|
||||
try:
|
||||
self._login_to_device()
|
||||
except TIMEOUT:
|
||||
pass
|
||||
try:
|
||||
# Try and force a prompt to be shown
|
||||
self.conn.send('\n')
|
||||
self.conn.expect([r'# ', self.conn.UNIQUE_PROMPT, r'\[PEXPECT\][\\\$\#]+ '], timeout=60)
|
||||
prompt_found = True
|
||||
except TIMEOUT:
|
||||
pass
|
||||
|
||||
gem5_logger.info("Successfully logged in")
|
||||
gem5_logger.info("Setting unique prompt...")
|
||||
|
||||
self.conn.set_unique_prompt()
|
||||
self.conn.prompt()
|
||||
gem5_logger.info("Prompt found and replaced with a unique string")
|
||||
|
||||
# We check that the prompt is what we think it should be. If not, we
|
||||
# need to update the regex we use to match.
|
||||
self._find_prompt()
|
||||
|
||||
self.conn.setecho(False)
|
||||
self._sync_gem5_shell()
|
||||
|
||||
# Fully connected to gem5 simulation
|
||||
self.gem5_interact_dir = gem5_interact_dir
|
||||
self.gem5_out_dir = gem5_out_dir
|
||||
self.gem5simulation = gem5_simulation
|
||||
|
||||
# Ready for interaction now
|
||||
self.ready = True
|
||||
|
||||
def _login_to_device(self):
|
||||
"""
|
||||
Login to device, will be overwritten if there is an actual login
|
||||
"""
|
||||
pass
|
||||
|
||||
def _find_prompt(self):
|
||||
prompt = r'\[PEXPECT\][\\\$\#]+ '
|
||||
synced = False
|
||||
while not synced:
|
||||
self.conn.send('\n')
|
||||
i = self.conn.expect([prompt, self.conn.UNIQUE_PROMPT, r'[\$\#] '], timeout=self.default_timeout)
|
||||
if i == 0:
|
||||
synced = True
|
||||
elif i == 1:
|
||||
prompt = self.conn.UNIQUE_PROMPT
|
||||
synced = True
|
||||
else:
|
||||
prompt = re.sub(r'\$', r'\\\$', self.conn.before.strip() + self.conn.after.strip())
|
||||
prompt = re.sub(r'\#', r'\\\#', prompt)
|
||||
prompt = re.sub(r'\[', r'\[', prompt)
|
||||
prompt = re.sub(r'\]', r'\]', prompt)
|
||||
|
||||
self.conn.PROMPT = prompt
|
||||
|
||||
def _sync_gem5_shell(self):
|
||||
"""
|
||||
Synchronise with the gem5 shell.
|
||||
|
||||
Write some unique text to the gem5 device to allow us to synchronise
|
||||
with the shell output. We actually get two prompts so we need to match
|
||||
both of these.
|
||||
"""
|
||||
gem5_logger.debug("Sending Sync")
|
||||
self.conn.send("echo \*\*sync\*\*\n")
|
||||
self.conn.expect(r"\*\*sync\*\*", timeout=self.default_timeout)
|
||||
self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)
|
||||
self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)
|
||||
|
||||
def _gem5_util(self, command):
|
||||
""" Execute a gem5 utility command using the m5 binary on the device """
|
||||
if self.m5_path is None:
|
||||
raise TargetError('Path to m5 binary on simulated system is not set!')
|
||||
self._gem5_shell('{} {}'.format(self.m5_path, command))
|
||||
|
||||
def _gem5_shell(self, command, as_root=False, timeout=None, check_exit_code=True, sync=True): # pylint: disable=R0912
|
||||
"""
|
||||
Execute a command in the gem5 shell
|
||||
|
||||
This wraps the telnet connection to gem5 and processes the raw output.
|
||||
|
||||
This method waits for the shell to return, and then will try and
|
||||
separate the output from the command from the command itself. If this
|
||||
fails, warn, but continue with the potentially wrong output.
|
||||
|
||||
The exit code is also checked by default, and non-zero exit codes will
|
||||
raise a TargetError.
|
||||
"""
|
||||
if sync:
|
||||
self._sync_gem5_shell()
|
||||
|
||||
gem5_logger.debug("gem5_shell command: {}".format(command))
|
||||
|
||||
# Send the actual command
|
||||
self.conn.send("{}\n".format(command))
|
||||
|
||||
# Wait for the response. We just sit here and wait for the prompt to
|
||||
# appear, as gem5 might take a long time to provide the output. This
|
||||
# avoids timeout issues.
|
||||
command_index = -1
|
||||
while command_index == -1:
|
||||
if self.conn.prompt():
|
||||
output = re.sub(r' \r([^\n])', r'\1', self.conn.before)
|
||||
output = re.sub(r'[\b]', r'', output)
|
||||
# Deal with line wrapping
|
||||
output = re.sub(r'[\r].+?<', r'', output)
|
||||
command_index = output.find(command)
|
||||
|
||||
# If we have -1, then we cannot match the command, but the
|
||||
# prompt has returned. Hence, we have a bit of an issue. We
|
||||
# warn, and return the whole output.
|
||||
if command_index == -1:
|
||||
gem5_logger.warn("gem5_shell: Unable to match command in "
|
||||
"command output. Expect parsing errors!")
|
||||
command_index = 0
|
||||
|
||||
output = output[command_index + len(command):].strip()
|
||||
|
||||
# It is possible that gem5 will echo the command. Therefore, we need to
|
||||
# remove that too!
|
||||
command_index = output.find(command)
|
||||
if command_index != -1:
|
||||
output = output[command_index + len(command):].strip()
|
||||
|
||||
gem5_logger.debug("gem5_shell output: {}".format(output))
|
||||
|
||||
# We get a second prompt. Hence, we need to eat one to make sure that we
|
||||
# stay in sync. If we do not do this, we risk getting out of sync for
|
||||
# slower simulations.
|
||||
self.conn.expect([self.conn.UNIQUE_PROMPT, self.conn.PROMPT], timeout=self.default_timeout)
|
||||
|
||||
if check_exit_code:
|
||||
exit_code_text = self._gem5_shell('echo $?', as_root=as_root,
|
||||
timeout=timeout, check_exit_code=False,
|
||||
sync=False)
|
||||
try:
|
||||
exit_code = int(exit_code_text.split()[0])
|
||||
if exit_code:
|
||||
message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}'
|
||||
raise TargetError(message.format(exit_code, command, output))
|
||||
except (ValueError, IndexError):
|
||||
gem5_logger.warning('Could not get exit code for "{}",\ngot: "{}"'.format(command, exit_code_text))
|
||||
|
||||
return output
|
||||
|
||||
def _mount_virtio(self):
|
||||
"""
|
||||
Mount the VirtIO device in the simulated system.
|
||||
"""
|
||||
gem5_logger.info("Mounting VirtIO device in simulated system")
|
||||
|
||||
self._gem5_shell('su -c "mkdir -p {}" root'.format(self.gem5_input_dir))
|
||||
mount_command = "mount -t 9p -o trans=virtio,version=9p2000.L,aname={} gem5 {}".format(self.gem5_interact_dir, self.gem5_input_dir)
|
||||
self._gem5_shell(mount_command)
|
||||
|
||||
def _move_to_temp_dir(self, source):
|
||||
"""
|
||||
Move a file to the temporary directory on the host for copying to the
|
||||
gem5 device
|
||||
"""
|
||||
command = "cp {} {}".format(source, self.gem5_interact_dir)
|
||||
gem5_logger.debug("Local copy command: {}".format(command))
|
||||
subprocess.call(command.split())
|
||||
subprocess.call("sync".split())
|
||||
|
||||
def _check_ready(self):
|
||||
"""
|
||||
Check if the gem5 platform is ready
|
||||
"""
|
||||
if not self.ready:
|
||||
raise TargetError('Gem5 is not ready to interact yet')
|
||||
|
||||
def _wait_for_boot(self):
|
||||
pass
|
||||
|
||||
def _probe_file(self, filepath):
|
||||
"""
|
||||
Internal method to check if the target has a certain file
|
||||
"""
|
||||
command = 'if [ -e \'{}\' ]; then echo 1; else echo 0; fi'
|
||||
output = self.execute(command.format(filepath), as_root=self.is_rooted)
|
||||
return boolean(output.strip())
|
||||
|
||||
|
||||
class LinuxGem5Connection(Gem5Connection):
|
||||
|
||||
def _login_to_device(self):
|
||||
gem5_logger.info("Trying to log in to gem5 device")
|
||||
login_prompt = ['login:', 'AEL login:', 'username:', 'aarch64-gem5 login:']
|
||||
login_password_prompt = ['password:']
|
||||
# Wait for the login prompt
|
||||
prompt = login_prompt + [self.conn.UNIQUE_PROMPT]
|
||||
i = self.conn.expect(prompt, timeout=10)
|
||||
# Check if we are already at a prompt, or if we need to log in.
|
||||
if i < len(prompt) - 1:
|
||||
self.conn.sendline("{}".format(self.username))
|
||||
password_prompt = login_password_prompt + [r'# ', self.conn.UNIQUE_PROMPT]
|
||||
j = self.conn.expect(password_prompt, timeout=self.default_timeout)
|
||||
if j < len(password_prompt) - 2:
|
||||
self.conn.sendline("{}".format(self.password))
|
||||
self.conn.expect([r'# ', self.conn.UNIQUE_PROMPT], timeout=self.default_timeout)
|
||||
|
||||
|
||||
|
||||
class AndroidGem5Connection(Gem5Connection):
|
||||
|
||||
def _wait_for_boot(self):
|
||||
"""
|
||||
Wait for the system to boot
|
||||
|
||||
We monitor the sys.boot_completed and service.bootanim.exit system
|
||||
properties to determine when the system has finished booting. In the
|
||||
event that we cannot coerce the result of service.bootanim.exit to an
|
||||
integer, we assume that the boot animation was disabled and do not wait
|
||||
for it to finish.
|
||||
|
||||
"""
|
||||
gem5_logger.info("Waiting for Android to boot...")
|
||||
while True:
|
||||
booted = False
|
||||
anim_finished = True # Assume boot animation was disabled on except
|
||||
try:
|
||||
booted = (int('0' + self._gem5_shell('getprop sys.boot_completed', check_exit_code=False).strip()) == 1)
|
||||
anim_finished = (int(self._gem5_shell('getprop service.bootanim.exit', check_exit_code=False).strip()) == 1)
|
||||
except ValueError:
|
||||
pass
|
||||
if booted and anim_finished:
|
||||
break
|
||||
time.sleep(60)
|
||||
|
||||
gem5_logger.info("Android booted")
|
||||
|
||||
def _give_password(password, command):
|
||||
if not sshpass:
|
||||
raise HostError('Must have sshpass installed on the host in order to use password-based auth.')
|
||||
|
||||
@ -180,3 +180,61 @@ Connection Types
|
||||
blocking on password request in scripts.
|
||||
:param password: Specify password on connection creation rather than
|
||||
prompting for it.
|
||||
|
||||
|
||||
.. class:: Gem5Connection(platform, host=None, username=None, password=None,\
|
||||
timeout=None, password_prompt=None,\
|
||||
original_prompt=None)
|
||||
|
||||
A connection to a gem5 simulation using a local Telnet connection.
|
||||
|
||||
.. note:: Some of the following input parameters are optional and will be ignored during
|
||||
initialisation. They were kept to keep the anology with a :class:`TelnetConnection`
|
||||
(i.e. ``host``, `username``, ``password``, ``port``,
|
||||
``password_prompt`` and ``original_promp``)
|
||||
|
||||
|
||||
:param host: Host on which the gem5 simulation is running
|
||||
|
||||
.. note:: Even thought the input parameter for the ``host``
|
||||
will be ignored, the gem5 simulation needs to on
|
||||
the same host as the user as the user is
|
||||
currently on, so if the host given as input
|
||||
parameter is not the same as the actual host, a
|
||||
``TargetError`` will be raised to prevent
|
||||
confusion.
|
||||
|
||||
:param username: Username in the simulated system
|
||||
:param password: No password required in gem5 so does not need to be set
|
||||
:param port: Telnet port to connect to gem5. This does not need to be set
|
||||
at initialisation as this will either be determined by the
|
||||
:class:`Gem5SimulationPlatform` or can be set using the
|
||||
:func:`connect_gem5` method
|
||||
:param timeout: Timeout for the connection in seconds. Gem5 has high
|
||||
latencies so unless the timeout given by the user via
|
||||
this input parameter is higher than the default one
|
||||
(3600 seconds), this input parameter will be ignored.
|
||||
:param password_prompt: A string with password prompt
|
||||
:param original_prompt: A regex for the shell prompt
|
||||
|
||||
There are two classes that inherit from :class:`Gem5Connection`:
|
||||
:class:`AndroidGem5Connection` and :class:`LinuxGem5Connection`.
|
||||
They inherit *almost* all methods from the parent class, without altering them.
|
||||
The only methods discussed belows are those that will be overwritten by the
|
||||
:class:`LinuxGem5Connection` and :class:`AndroidGem5Connection` respectively.
|
||||
|
||||
.. class:: LinuxGem5Connection
|
||||
|
||||
A connection to a gem5 simulation that emulates a Linux system.
|
||||
|
||||
.. method:: _login_to_device(self)
|
||||
|
||||
Login to the gem5 simulated system.
|
||||
|
||||
.. class:: AndroidGem5Connection
|
||||
|
||||
A connection to a gem5 simulation that emulates an Android system.
|
||||
|
||||
.. method:: _wait_for_boot(self)
|
||||
|
||||
Wait for the gem5 simulated system to have booted and finished the booting animation.
|
||||
|
||||
@ -21,6 +21,7 @@ Contents:
|
||||
instrumentation
|
||||
platform
|
||||
connection
|
||||
platform
|
||||
|
||||
Indices and tables
|
||||
==================
|
||||
|
||||
@ -110,3 +110,62 @@ support additional configuration:
|
||||
just that the services needed to establish a
|
||||
connection (e.g. sshd or adbd) are up.
|
||||
|
||||
|
||||
.. _gem5-platform:
|
||||
|
||||
Gem5 Simulation Platform
|
||||
------------------------
|
||||
|
||||
By initialising a Gem5SimulationPlatform, devlib will start a gem5 simulation (based upon the
|
||||
arguments the user provided) and then connect to it using :class:`Gem5Connection`.
|
||||
Using the methods discussed above, some methods of the :class:`Target` will be altered
|
||||
slightly to better suit gem5.
|
||||
|
||||
.. class:: Gem5SimulationPlatform(name, host_output_dir, gem5_bin, gem5_args, gem5_virtio, gem5_telnet_port=None)
|
||||
|
||||
During initialisation the gem5 simulation will be kicked off (based upon the arguments
|
||||
provided by the user) and the telnet port used by the gem5 simulation will be intercepted
|
||||
and stored for use by the :class:`Gem5Connection`.
|
||||
|
||||
:param name: Platform name
|
||||
|
||||
:param host_output_dir: Path on the host where the gem5 outputs will be placed (e.g. stats file)
|
||||
|
||||
:param gem5_bin: gem5 binary
|
||||
|
||||
:param gem5_args: Arguments to be passed onto gem5 such as config file etc.
|
||||
|
||||
:param gem5_virtio: Arguments to be passed onto gem5 in terms of the virtIO device used
|
||||
to transfer files between the host and the gem5 simulated system.
|
||||
|
||||
:param gem5_telnet_port: Not yet in use as it would be used in future implementations
|
||||
of devlib in which the user could use the platform to pick
|
||||
up an existing and running simulation.
|
||||
|
||||
|
||||
.. method:: Gem5SimulationPlatform.init_target_connection([target])
|
||||
|
||||
Based upon the OS defined in the :class:`Target`, the type of :class:`Gem5Connection`
|
||||
will be set (:class:`AndroidGem5Connection` or :class:`AndroidGem5Connection`).
|
||||
|
||||
.. method:: Gem5SimulationPlatform.update_from_target([target])
|
||||
|
||||
This method provides specific setup procedures for a gem5 simulation. First of all, the m5
|
||||
binary will be installed on the guest (if it is not present). Secondly, three methods
|
||||
in the :class:`Target` will be monkey-patched:
|
||||
|
||||
- **reboot**: this is not supported in gem5
|
||||
- **reset**: this is not supported in gem5
|
||||
- **capture_screen**: gem5 might already have screencaps so the
|
||||
monkey-patched method will first try to
|
||||
transfer the existing screencaps.
|
||||
In case that does not work, it will fall back
|
||||
to the original :class:`Target` implementation
|
||||
of :func:`capture_screen`.
|
||||
|
||||
Finally, it will call the parent implementation of :func:`update_from_target`.
|
||||
|
||||
.. method:: Gem5SimulationPlatform.setup([target])
|
||||
|
||||
The m5 binary be installed, if not yet installed on the gem5 simulated system.
|
||||
It will also resize the gem5 shell, to avoid line wrapping issues.
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user