diff --git a/wlauto/devices/android/gem5/__init__.py b/wlauto/devices/android/gem5/__init__.py new file mode 100644 index 00000000..b95e12b3 --- /dev/null +++ b/wlauto/devices/android/gem5/__init__.py @@ -0,0 +1,801 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Original implementation by Rene de Jong. Updated by Sascha Bischoff. + +import logging +import os +import re +import shutil +import socket +import subprocess +import sys +import tarfile +import time +from pexpect import EOF, TIMEOUT + +from wlauto import AndroidDevice, settings, Parameter +from wlauto.core import signal as sig +from wlauto.utils import ssh +from wlauto.exceptions import DeviceError, ConfigError + + +class Gem5Device(AndroidDevice): + """ + Implements gem5 Android device. + + This class allows a user to connect WA to a simulation using gem5. The + connection to the device is made using the telnet connection of the + simulator, and is used for all commands. The simulator does not have ADB + support, and therefore we need to fall back to using standard shell + commands. + + Files are copied into the simulation using a VirtIO 9P device in gem5. Files + are copied out of the simulated environment using the m5 writefile command + within the simulated system. + + When starting the workload run, the simulator is automatically started by + Workload Automation, and a connection to the simulator is established. WA + will then wait for Android to boot on the simulated system (which can take + hours), prior to executing any other commands on the device. It is also + possible to resume from a checkpoint when starting the simulation. To do + this, please append the relevant checkpoint commands from the gem5 + simulation script to the gem5_discription argument in the agenda. + + Host system requirements: + * VirtIO support. We rely on diod on the host system. This can be + installed on ubuntu using the following command: + + sudo apt-get install diod + + Guest requirements: + * VirtIO support. We rely on VirtIO to move files into the simulation. + Please make sure that the following are set in the kernel + configuration: + + CONFIG_NET_9P=y + + CONFIG_NET_9P_VIRTIO=y + + CONFIG_9P_FS=y + + CONFIG_9P_FS_POSIX_ACL=y + + CONFIG_9P_FS_SECURITY=y + + CONFIG_VIRTIO_BLK=y + + * m5 binary. Please make sure that the m5 binary is on the device and + can by found in the path. + * Busybox. Due to restrictions, we assume that busybox is installed in + the guest system, and can be found in the path. + """ + + name = 'gem5_android' + default_working_directory = '/data/' + has_gpu = False + platform = 'android' + path_module = 'posixpath' + default_package_data_directory = '/data/data/' + default_binaries_directory = '/system/bin' + default_external_storage_directory = '/data/data/' + default_adb_name = None + + # gem5 can be very slow. Hence, we use some very long timeouts! + delay = 3600 + long_delay = 3 * delay + ready_timeout = long_delay + default_timeout = delay + + parameters = [ + Parameter('core_names', default=[], override=True), + Parameter('core_clusters', default=[], override=True), + Parameter('gem5_description', kind=str, default='', override=True, + description="Command line passed to the gem5 simulation. This command line is used " + "to set up the simulated system, and should be the same as used for a standard " + "gem5 simulation without workload automation. Note that this is simulation script " + "specific and will hence need to be tailored to each particular use case."), + Parameter('virtio_command', kind=str, default='', override=True, + description="gem5 VirtIO command line used to enable the VirtIO device in the " + "simulated system. At the very least, the root parameter of the VirtIO9PDiod device " + "must be exposed on the command line. Please set this root mount to {}, as it will " + "be replaced with the directory used by Workload Automation at runtime."), + Parameter('temp_dir', kind=str, default='', override=True, + description="Temporary directory used to pass files into the gem5 simulation. " + "Workload Automation will automatically create a directory in this folder, and " + "will remove it again once the simulation completes."), + Parameter('checkpoint_post_boot', kind=bool, default=False, mandatory=False, override=True, + description="This parameter tells Workload Automation to create a checkpoint of " + "the simulated system once the guest system has finished booting. This checkpoint " + "can then be used at a later stage by other WA runs to avoid booting the guest system" + " a second time. Set to True to take a checkpoint of the simulated system post boot."), + Parameter('run_delay', kind=int, default=0, mandatory=False, override=True, + description="This sets the time that the system should sleep in the simulated system prior" + " to running and workloads or taking checkpoints. This allows the system to quieten down" + " prior to running the workloads. When this is combined with the checkpoint_post_boot" + " option, it allows the checkpoint to be created post-sleep, and therefore the set of " + "workloads resuming from this checkpoint will not be required to sleep.") + ] + + # Overwritten from Device. For documentation, see corresponding method in + # Device. + + @property + def is_rooted(self): + # gem5 is always rooted + return True + + def __init__(self, **kwargs): + self.logger = logging.getLogger('gem5Device') + + super(Gem5Device, self).__init__(**kwargs) + self.adb_name = kwargs.get('adb_name') or self.default_adb_name + self.binaries_directory = kwargs.get('binaries_directory', self.default_binaries_directory) + self.package_data_directory = kwargs.get('package_data_directory', self.default_package_data_directory) + self.external_storage_directory = kwargs.get('external_storage_directory', + self.default_external_storage_directory) + self.logcat_poll_period = kwargs.get('logcat_poll_period') + + # The gem5 subprocess + self.gem5 = None + + self.gem5_port = -1 + self.busybox = None + self._is_initialized = False + self._is_ready = False + self._just_rebooted = False + self._is_rooted = None + self._available_frequencies = {} + self._available_governors = {} + self._available_governor_tunables = {} + self._number_of_cores = None + self._logcat_poller = None + self.gem5outdir = os.path.join(settings.output_directory, "gem5") + self.stdout_file = None + self.stderr_file = None + self.stderr_filename = None + self.checkpoint = False + self.sckt = None + + if not kwargs.get('gem5_description'): + raise ConfigError('Please specify the system configuration with gem5_description') + self.gem5_args = kwargs.get('gem5_description') + + if kwargs.get('temp_dir'): + self.temp_dir = kwargs.get('temp_dir') + else: + self.logger.info("No temporary directory passed in. Defaulting to /tmp") + self.temp_dir = '/tmp' + + if kwargs.get('checkpoint_post_boot'): + self.checkpoint = True + + if kwargs.get('run_delay'): + self.run_delay = kwargs.get('run_delay') + + # Find the first one that does not exist. Ensures that we do not re-use + # the directory used by someone else. + for i in xrange(sys.maxint): + directory = os.path.join(self.temp_dir, "wa_{}".format(i)) + try: + os.stat(directory) + continue + except OSError: + break + self.temp_dir = directory + self.logger.info("Using {} as the temporary directory.".format(self.temp_dir)) + + if not kwargs.get('virtio_command'): + raise ConfigError('Please specify the VirtIO command specific to your script, ending with the root parameter of the device.') + + self.gem5_vio_arg = kwargs.get('virtio_command').format(self.temp_dir) + self.logger.debug("gem5 VirtIO command: {}".format(self.gem5_vio_arg)) + + # Start the gem5 simulation when WA starts a run using a signal. + sig.connect(self.init_gem5, sig.RUN_START) + + def init_gem5(self, _): + """ + Start gem5, find out the telnet port and connect to the simulation. + + We first create the temporary directory used by VirtIO to pass files + into the simulation, as well as the gem5 output directory.We then create + files for the standard output and error for the gem5 process. The gem5 + process then is started. + """ + self.logger.info("Creating temporary directory: {}".format(self.temp_dir)) + os.mkdir(self.temp_dir) + os.mkdir(self.gem5outdir) + + # We need to redirect the standard output and standard error for the + # gem5 process to a file so that we can debug when things go wrong. + f = os.path.join(self.gem5outdir, 'stdout') + self.stdout_file = open(f, 'w') + f = os.path.join(self.gem5outdir, 'stderr') + self.stderr_file = open(f, 'w') + # We need to keep this so we can check which port to use for the telnet + # connection. + self.stderr_filename = f + + self.start_gem5() + + def start_gem5(self): + """ + Starts the gem5 simulator, and parses the output to get the telnet port. + """ + self.logger.info("Starting the gem5 simulator") + + command_line = "./build/ARM/gem5.fast --outdir={}/gem5 {} {}".format(settings.output_directory, + self.gem5_args, + self.gem5_vio_arg) + self.logger.debug("gem5 command line: {}".format(command_line)) + self.gem5 = subprocess.Popen(command_line.split(), stdout=self.stdout_file, stderr=self.stderr_file) + + while self.gem5_port == -1: + # Check that gem5 is running! + if self.gem5.poll(): + raise DeviceError("The gem5 process has crashed with error code {}!".format(self.gem5.poll())) + + # Open the stderr file + f = open(self.stderr_filename, 'r') + for line in f: + m = re.search(r"Listening\ for\ system\ connection\ on\ port\ (?P\d+)", line) + if m: + port = int(m.group('port')) + if port >= 3456 and port < 5900: + self.gem5_port = port + f.close() + break + else: + time.sleep(1) + f.close() + + def connect(self): # pylint: disable=R0912 + """ + Connect to the gem5 simulation and wait for Android to boot. Then, + create checkpoints, and mount the VirtIO device. + """ + self.connect_gem5() + + self.logger.info("Waiting for Android to boot...") + while True: + try: + available = (1 == int('0' + self.gem5_shell('getprop sys.boot_completed', check_exit_code=False))) + if available: + break + except (DeviceError, ValueError): + pass + time.sleep(60) + + self.logger.info("Android booted") + + if self.run_delay: + self.logger.info("Sleeping for {} seconds in the guest".format(self.run_delay)) + self.gem5_shell("sleep {}".format(self.run_delay)) + + if self.checkpoint: + self.checkpoint_gem5() + + self.mount_virtio() + self.logger.info("Creating the working directory in the simulated system") + self.gem5_shell('mkdir -p {}'.format(self.working_directory)) + self._is_ready = True + + def connect_gem5(self): # pylint: disable=R0912 + """ + Connect to the telnet port of the gem5 simulation. + + We connect, and wait for the prompt to be found. We do not use a timeout + for this, and wait for the prompt in a while loop as the gem5 simulation + can take many hours to reach a prompt when booting the system. We also + inject some newlines periodically to try and force gem5 to show a + prompt. Once the prompt has been found, we replace it with a unique + prompt to ensure that we are able to match it properly. We also disable + the echo as this simplifies parsing the output when executing commands + on the device. + """ + self.logger.info("Connecting to the gem5 simulation on port " + str(self.gem5_port)) + host = socket.gethostname() + port = self.gem5_port + + # Connect to the gem5 telnet port. Use a short timeout here. + self.sckt = ssh.TelnetConnection() + self.sckt.login(host, 'None', port=port, auto_prompt_reset=False, login_timeout=10) + + self.logger.info("Connected! Waiting for prompt...") + + # We need to find the prompt. It might be different if we are resuming + # from a checkpoint. Therefore, we test multiple options here. + prompt_found = False + while not prompt_found: + try: + # Try and force a prompt to be shown + self.sckt.send('\n') + self.sckt.expect([r'# ', r'\[PEXPECT\]\$'], timeout=self.delay) + prompt_found = True + except TIMEOUT: + pass + + self.logger.info("Setting unique prompt...") + + self.sckt.set_unique_prompt() + self.sckt.prompt() + self.logger.info("Prompt found and replaced with a unique string") + + self.sckt.setecho(False) + self.sync_gem5_shell() + + def mount_virtio(self): + """ + Mount the VirtIO device in the simulated system. + + We cannot assume any state for the VirtIO device in gem5 as it is not + serialised when checkpointing the system. Therefore, we unbind and + rebind the VirtIO device to force the driver to re-initialize the + device, prior to using it. We then mount the folder on the host system + using the VirtIo device. + """ + self.logger.info("Mounting VirtIO device in simulated system") + + # We always unbind, then re-bind the device. This ensures that the + # driver is re-loaded and that the device is re-initialized. Hence, this + # should work for both checkpointed and non-checkpointed gem5 systems. + vio_info = self.gem5_shell('ls /sys/bus/pci/drivers/virtio-pci/').strip().split() + mounts = [] + for f in vio_info: + if len(f.split(':')) > 1: + mounts.append(f.strip()) + + # Unbind and rebind all of the + for mount in mounts: + self.gem5_shell('echo -n "{}" > /sys/bus/pci/drivers/virtio-pci/unbind'.format(mount)) + self.gem5_shell('echo -n "{}" > /sys/bus/pci/drivers/virtio-pci/bind'.format(mount)) + + mount_command = "mount -t 9p -o trans=virtio,version=9p2000.L,aname={} gem5 /mnt/obb".format(self.temp_dir) + self.gem5_shell('busybox {}'.format(mount_command)) + + def disconnect(self): + """ + Close and disconnect from the gem5 simulation. Additionally, we remove + the temporary directory used to pass files into the simulation. + """ + self.logger.info("Gracefully terminating the gem5 simulation.") + try: + self.gem5_util("exit") + self.gem5.wait() + except EOF: + pass + self.logger.info("Removing the temporary directory") + try: + shutil.rmtree(self.temp_dir) + except OSError: + self.logger.warn("Failed to remove the temporary directory!") + + def close(self): + if self._logcat_poller: + self._logcat_poller.stop() + + def reset(self): + self.logger.warn("Attempt to restart the gem5 device. This is not supported!") + + def init(self): + pass + + def push_file(self, source, dest, as_root=False): # pylint: disable=W0221 + """ + Push a file to the gem5 device using VirtIO + + The file to push to the device is copied to the temporary directory on + the host, before being copied within the simulation to the destination. + Checks, in the form of 'ls' with error code checking, are performed to + ensure that the file is copied to the destination. + """ + filename = source.split('/')[-1] + self.logger.debug("Pushing {} to device.".format(source)) + self.logger.debug("temp_dir: {}".format(self.temp_dir)) + self.logger.debug("dest: {}".format(dest)) + self.logger.debug("filename: {}".format(filename)) + + # We need to copy the file to copy to the temporary directory + self.move_to_temp_dir(source) + + # Back to the gem5 world + self.gem5_shell("ls -al /mnt/obb/{}".format(filename)) + self.gem5_shell("busybox cp /mnt/obb/{} {}".format(filename, dest)) + self.gem5_shell("busybox sync") + self.gem5_shell("ls -al {}".format(dest)) + self.gem5_shell("ls -al /mnt/obb/") + self.logger.debug("Push complete.") + + def pull_file(self, source, dest, as_root=False): # pylint: disable=W0221 + """ + Pull a file from the gem5 device using m5 writefile + + First, we check the extension of the file to be copied. If the file ends + in .gz, then gem5 wrongly assumes that it should create a gzipped output + stream, which results in a gem5 error. Therefore, we rename the file on + the local device prior to the writefile command when required. Next, the + file is copied to the local directory within the guest as the m5 + writefile command assumes that the file is local. The file is then + written out to the host system using writefile, prior to being moved to + the destination on the host. + """ + filename = source.split('/')[-1] + self.logger.debug("Pulling {} from device.".format(filename)) + + # gem5 assumes that files ending in .gz are gzip-compressed. We need to + # work around this, else gem5 panics on us. Rename the file for use in + # gem5 + if filename[-3:] == '.gz': + filename += '.fugem5' + + self.logger.debug("pull_file {} {}".format(source, filename)) + # We don't check the exit code here because it is non-zero if the source + # and destination are the same. The ls below will cause an error if the + # file was not where we expected it to be. + self.gem5_shell("busybox cp {} {}".format(source, filename), check_exit_code=False) + self.gem5_shell("busybox sync") + self.gem5_shell("ls -la {}".format(filename)) + self.logger.debug('Finished the copy in the simulator') + self.gem5_util("writefile {}".format(filename)) + + if 'cpu' not in filename: + while not os.path.exists(os.path.join(self.gem5outdir, filename)): + time.sleep(1) + + # Perform the local move + shutil.move(os.path.join(self.gem5outdir, filename), dest) + self.logger.debug("Pull complete.") + + def delete_file(self, filepath, as_root=False): # pylint: disable=W0221 + """ Delete a file on the device """ + self._check_ready() + self.gem5_shell("rm '{}'".format(filepath)) + + def file_exists(self, filepath): + """ Check if a file exists """ + self._check_ready() + output = self.gem5_shell('if [ -e \'{}\' ]; then echo 1; else echo 0; fi'.format(filepath)) + try: + if int(output): + return True + else: + return False + except ValueError: + if output: + return True + else: + return False + + def install(self, filepath, timeout=default_timeout): # pylint: disable=W0221 + """ Install an APK or a normal executable """ + ext = os.path.splitext(filepath)[1].lower() + if ext == '.apk': + return self.install_apk(filepath, timeout) + else: + return self.install_executable(filepath) + + def install_apk(self, filepath, timeout=default_timeout): # pylint: disable=W0221 + """ + Install an APK on the gem5 device + + The APK is pushed to the device. Then the file and folder permissions + are changed to ensure that the APK can be correctly installed. The APK + is then installed on the device using 'pm'. + """ + self._check_ready() + self.logger.info("Installing {}".format(filepath)) + ext = os.path.splitext(filepath)[1].lower() + if ext == '.apk': + filename = os.path.basename(filepath) + on_device_path = os.path.join('/data/local/tmp', filename) + self.push_file(filepath, on_device_path) + # We need to make sure that the folder permissions are set + # correctly, else the APK does not install correctly. + self.gem5_shell('busybox chmod 775 /data/local/tmp') + self.gem5_shell('busybox chmod 774 {}'.format(on_device_path)) + self.logger.debug("Actually installing the APK: {}".format(on_device_path)) + return self.gem5_shell("pm install {}".format(on_device_path)) + else: + raise DeviceError('Can\'t install {}: unsupported format.'.format(filepath)) + + def install_executable(self, filepath, with_name=None): + """ Install an executable """ + executable_name = os.path.basename(filepath) + on_device_file = self.path.join(self.working_directory, executable_name) + on_device_executable = self.path.join(self.binaries_directory, executable_name) + self.push_file(filepath, on_device_file) + self.execute('busybox cp {} {}'.format(on_device_file, on_device_executable)) + self.execute('busybox chmod 0777 {}'.format(on_device_executable)) + return on_device_executable + + def uninstall(self, package): + self._check_ready() + self.gem5_shell("pm uninstall {}".format(package)) + + def execute(self, command, timeout=default_timeout, check_exit_code=True, background=False, + as_root=False, busybox=False, **kwargs): + self._check_ready() + if as_root and not self.is_rooted: + raise DeviceError('Attempting to execute "{}" as root on unrooted device.'.format(command)) + if busybox: + if not self.is_rooted: + raise DeviceError('Attempting to execute "{}" with busybox. '.format(command) + + 'Busybox can only be deployed to rooted devices.') + command = ' '.join([self.busybox, command]) + if background: + self.logger.debug("Attempt to execute in background. Not supported in gem5, hence ignored.") + return self.gem5_shell(command, as_root=as_root) + + def dump_logcat(self, outfile, filter_spec=None): + """ Extract logcat from simulation """ + self.logger.info("Extracting logcat from the simulated system") + filename = outfile.split('/')[-1] + command = 'logcat -d > {}'.format(filename) + self.gem5_shell(command) + self.pull_file("{}".format(filename), outfile) + + def clear_logcat(self): + """Clear (flush) logcat log.""" + if self._logcat_poller: + return self._logcat_poller.clear_buffer() + else: + return self.gem5_shell('logcat -c') + + def disable_selinux(self): + """ Disable SELinux. Overridden as parent implementation uses ADB """ + api_level = int(self.gem5_shell('getprop ro.build.version.sdk').strip()) + + # SELinux was added in Android 4.3 (API level 18). Trying to + # 'getenforce' in earlier versions will produce an error. + if api_level >= 18: + se_status = self.execute('getenforce', as_root=True).strip() + if se_status == 'Enforcing': + self.execute('setenforce 0', as_root=True) + + def get_properties(self, context): # pylint: disable=R0801 + """ Get the property files from the device """ + for propfile in self.property_files: + try: + normname = propfile.lstrip(self.path.sep).replace(self.path.sep, '.') + outfile = os.path.join(context.host_working_directory, normname) + if self.is_file(propfile): + self.execute('cat {} > {}'.format(propfile, normname)) + self.pull_file(normname, outfile) + elif self.is_directory(propfile): + self.get_directory(context, propfile) + continue + else: + continue + except DeviceError: + # We pull these files "opportunistically", so if a pull fails + # (e.g. we don't have permissions to read the file), just note + # it quietly (not as an error/warning) and move on. + self.logger.debug('Could not pull property file "{}"'.format(propfile)) + + # This is duplicated from the AndroidDevice class, as we want to run + # this code without executing the BaseLinuxDevice implementation + props = {} + props['android_id'] = self.get_android_id() + buildprop_file = os.path.join(context.host_working_directory, 'build.prop') + if not os.path.isfile(buildprop_file): + self.pull_file('/system/build.prop', context.host_working_directory) + self._update_build_properties(buildprop_file, props) + context.add_run_artifact('build_properties', buildprop_file, 'export') + + dumpsys_window_file = os.path.join(context.host_working_directory, 'window.dumpsys') + self.execute('{} > {}'.format('dumpsys window', 'window.dumpsys')) + self.pull_file('window.dumpsys', dumpsys_window_file) + context.add_run_artifact('dumpsys_window', dumpsys_window_file, 'meta') + return props + + def get_directory(self, context, directory): + """ Pull a directory from the device """ + normname = directory.lstrip(self.path.sep).replace(self.path.sep, '.') + outdir = os.path.join(context.host_working_directory, normname) + temp_file = os.path.join(context.host_working_directory, "{}.tar".format(normname)) + # Check that the folder exists + self.gem5_shell("ls -la {}".format(directory)) + # Compress the folder + try: + self.gem5_shell("{} tar -cvf {}.tar {}".format(self.busybox, normname, directory)) + except DeviceError: + self.logger.debug("Failed to run tar command on device! Not pulling {}".format(directory)) + return + self.pull_file(normname, temp_file) + f = tarfile.open(temp_file, 'r') + os.mkdir(outdir) + f.extractall(outdir) + os.remove(temp_file) + + def get_pids_of(self, process_name): + """ Returns a list of PIDs of all processes with the specified name. """ + result = self.gem5_shell('ps | busybox grep {}'.format(process_name), check_exit_code=False).strip() + if result and 'not found' not in result and len(result.split('\n')) > 2: + return [int(x.split()[1]) for x in result.split('\n')] + else: + return [] + + def disable_screen_lock(self): + """ + Attempts to disable he screen lock on the device. + + Overridden here as otherwise we have issues with too many backslashes. + """ + lockdb = '/data/system/locksettings.db' + sqlcommand = "update locksettings set value=\'0\' where name=\'screenlock.disabled\';" + self.execute('sqlite3 {} "{}"'.format(lockdb, sqlcommand), as_root=True) + + # gem5 should dump out a framebuffer. We can use this if it exists. Failing + # that, fall back to the parent class implementation. + def capture_screen(self, filepath): + file_list = os.listdir(self.gem5outdir) + screen_caps = [] + for f in file_list: + if '.bmp' in f: + screen_caps.append(f) + + if len(screen_caps) == 1: + # Bail out if we do not have image, and resort to the slower, built + # in method. + try: + import Image + gem5_image = os.path.join(self.gem5outdir, screen_caps[0]) + temp_image = os.path.join(self.gem5outdir, "file.png") + im = Image.open(gem5_image) + im.save(temp_image, "PNG") + shutil.copy(temp_image, filepath) + os.remove(temp_image) + self.logger.debug("capture_screen: using gem5 screencap") + return + except (shutil.Error, ImportError, IOError): + pass + + # If we didn't manage to do the above, call the parent class. + self.logger.debug("capture_screen: falling back to parent class implementation") + super(Gem5Device, self).capture_screen(filepath) + + # gem5 might be slow. Hence, we need to make the ping timeout very long. + def ping(self): + self.logger.debug("Pinging gem5 to see if it is still alive") + self.gem5_shell('ls /', timeout=self.longdelay) + + # Additional Android-specific methods. + def forward_port(self, from_port, to_port): + raise DeviceError('we do not need forwarding') + + # Internal methods: do not use outside of the class. + + def _check_ready(self): + """ + Check if the device is ready. + + As this is gem5, we just assume that the device is ready once we have + connected to the gem5 simulation, and updated the prompt. + """ + if not self._is_ready: + raise DeviceError('Device not ready.') + + def gem5_shell(self, command, as_root=False, timeout=None, check_exit_code=True, sync=True): # pylint: disable=R0912 + """ + Execute a command in the gem5 shell + + This wraps the telnet connection to gem5 and processes the raw output. + + This method waits for the shell to return, and then will try and + separate the output from the command from the command itself. If this + fails, warn, but continue with the potentially wrong output. + + The exit code is also checked by default, and non-zero exit codes will + raise a DeviceError. + """ + conn = self.sckt + if sync: + self.sync_gem5_shell() + + self.logger.debug("gem5_shell command: {}".format(command)) + + # Send the actual command + conn.send("{}\n".format(command)) + + # Wait for the response. We just sit here and wait for the prompt to + # appear, as gem5 might take a long time to provide the output. This + # avoids timeout issues. + command_index = -1 + while command_index == -1: + if conn.prompt(): + output = re.sub(r' \r([^\n])', r'\1', conn.before) + command_index = output.find(command) + # The command was probably too long and wrapped. This bit of + # code is a total hack! + if command_index == -1: + # We need to remove the backspace characters! + output = re.sub(r'[\b]', r'', output) + while True: + first_cr = output.find('\r') + first_lt = output.find('<') + if first_cr == -1 or first_lt == -1: + break + if first_cr < first_lt: + new_output = output[:first_cr] + new_output += output[first_lt + 1:] + output = new_output + command_index = output.find(command) + else: + break + + # If we STILL have -1, then we cannot match the command, but the + # prompt has returned. Hence, we have a bit of an issue. We + # warn, and return the whole output. + if command_index == -1: + self.logger.warn("gem5_shell: Unable to match command in command output. Expect parsing errors!") + command_index = 0 + + output = output[command_index + len(command):].strip() + + # It is possible that gem5 will echo the command. Therefore, we need to + # remove that too! + command_index = output.find(command) + if command_index != -1: + output = output[command_index + len(command):].strip() + + self.logger.debug("gem5_shell output: {}".format(output)) + + # We get a second prompt. Hence, we need to eat one to make sure that we + # stay in sync. If we do not do this, we risk getting out of sync for + # slower simulations. + self.sckt.expect(r'\[PEXPECT\]\$', timeout=10000) + + if check_exit_code: + exit_code_text = self.gem5_shell('echo $?', as_root=as_root, timeout=timeout, check_exit_code=False, sync=False) + try: + exit_code = int(exit_code_text.split()[0]) + if exit_code: + message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}' + raise DeviceError(message.format(exit_code, command, output)) + except (ValueError, IndexError): + self.logger.warning('Could not get exit code for "{}",\ngot: "{}"'.format(command, exit_code_text)) + + return output + + def gem5_util(self, command): + """ Execute a gem5 utility command using the m5 binary on the device """ + self.gem5_shell('/sbin/m5 ' + command) + + def sync_gem5_shell(self): + """ + Synchronise with the gem5 shell. + + Write some unique text to the gem5 device to allow us to synchronise + with the shell output. We actually get two prompts so we need to match both of these. + """ + self.sckt.send("echo \*\*sync\*\*\n") + self.sckt.expect(r"\*\*sync\*\*", timeout=self.delay) + self.sckt.expect(r'\[PEXPECT\]\$', timeout=self.delay) + self.sckt.expect(r'\[PEXPECT\]\$', timeout=self.delay) + + def move_to_temp_dir(self, source): + """ Move a file to the temporary directory on the host for copying to the gem5 device """ + command = "cp {} {}".format(source, self.temp_dir) + self.logger.debug("Local copy command: {}".format(command)) + subprocess.call(command.split()) + subprocess.call("sync".split()) + + def checkpoint_gem5(self, end_simulation=False): + """ Checkpoint the gem5 simulation, storing all system state """ + self.logger.info("Taking a post-boot checkpoint") + self.gem5_util("checkpoint") + if end_simulation: + self.disconnect()