From a6382b730b505dce400f31ae83c4b76f26b9e821 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Mon, 2 Nov 2015 10:09:59 +0000 Subject: [PATCH 1/9] TelnetConnection: - Allowed telnet connections without a password. This is required as part of the upcoming Gem5Device, which uses a password-less telnet connection to communicate with the device. --- wlauto/utils/ssh.py | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/wlauto/utils/ssh.py b/wlauto/utils/ssh.py index 31d5b3d7..3b1ac9c4 100644 --- a/wlauto/utils/ssh.py +++ b/wlauto/utils/ssh.py @@ -62,15 +62,20 @@ class TelnetConnection(pxssh.pxssh): cmd = 'telnet -l {} {} {}'.format(username, server, port) spawn._spawn(self, cmd) # pylint: disable=protected-access - i = self.expect('(?i)(?:password)', timeout=login_timeout) - if i == 0: - self.sendline(password) - i = self.expect([original_prompt, 'Login incorrect'], timeout=login_timeout) - else: - raise pxssh.ExceptionPxssh('could not log in: did not see a password prompt') - - if i: - raise pxssh.ExceptionPxssh('could not log in: password was incorrect') + try: + i = self.expect('(?i)(?:password)', timeout=login_timeout) + if i == 0: + self.sendline(password) + i = self.expect([original_prompt, 'Login incorrect'], timeout=login_timeout) + if i: + raise pxssh.ExceptionPxssh('could not log in: password was incorrect') + except TIMEOUT: + if not password: + # There was no password prompt before TIMEOUT, and we didn't + # have a password to enter. Assume everything is OK. + pass + else: + raise pxssh.ExceptionPxssh('could not log in: did not see a password prompt') if not self.sync_original_prompt(sync_multiplier): self.close() From 96a6179355fea84c5f213217d87df12e341bb8ab Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Mon, 2 Nov 2015 10:15:34 +0000 Subject: [PATCH 2/9] Gem5Device: Add a gem5 device for Android - Implementation of a gem5 device which allows simulated systems to be used in the place of a real device. Currently, only Android is supported. - The gem5 simulation is started automatically based on a command line passed in via the agenda. The correct telnet port to connect on is extracted from the standard error from the gem5 process. - Resuming from gem5 checkpoints is supported, and can be specified as part of the gem5 system description. Additionally, the agenda option checkpoint_post_boot can be used to create a checkpoint automatically once the system has booted. This can then by used for subsequent runs to avoid booting the system a second time. - The Gem5Device waits for Android to finish booting, before sending commands to the simulated device. Additionally, the device supports a sleep option, which will sleep in the simulated system for a number of seconds, prior to running the workload. This ensures that the system can quieten down, prior to running the workload. - The Gem5Device relies of VirtIO to pull files into the simulated environment, and therefire diod support is required on the host system. Additionally, VirtIO 9P support is required in the guest system kernel. - The m5 writefile binary and gem5 pseudo instruction are used to extract files from the simulated environment. --- wlauto/devices/android/gem5/__init__.py | 801 ++++++++++++++++++++++++ 1 file changed, 801 insertions(+) create mode 100644 wlauto/devices/android/gem5/__init__.py diff --git a/wlauto/devices/android/gem5/__init__.py b/wlauto/devices/android/gem5/__init__.py new file mode 100644 index 00000000..b95e12b3 --- /dev/null +++ b/wlauto/devices/android/gem5/__init__.py @@ -0,0 +1,801 @@ +# Copyright 2014-2015 ARM Limited +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Original implementation by Rene de Jong. Updated by Sascha Bischoff. + +import logging +import os +import re +import shutil +import socket +import subprocess +import sys +import tarfile +import time +from pexpect import EOF, TIMEOUT + +from wlauto import AndroidDevice, settings, Parameter +from wlauto.core import signal as sig +from wlauto.utils import ssh +from wlauto.exceptions import DeviceError, ConfigError + + +class Gem5Device(AndroidDevice): + """ + Implements gem5 Android device. + + This class allows a user to connect WA to a simulation using gem5. The + connection to the device is made using the telnet connection of the + simulator, and is used for all commands. The simulator does not have ADB + support, and therefore we need to fall back to using standard shell + commands. + + Files are copied into the simulation using a VirtIO 9P device in gem5. Files + are copied out of the simulated environment using the m5 writefile command + within the simulated system. + + When starting the workload run, the simulator is automatically started by + Workload Automation, and a connection to the simulator is established. WA + will then wait for Android to boot on the simulated system (which can take + hours), prior to executing any other commands on the device. It is also + possible to resume from a checkpoint when starting the simulation. To do + this, please append the relevant checkpoint commands from the gem5 + simulation script to the gem5_discription argument in the agenda. + + Host system requirements: + * VirtIO support. We rely on diod on the host system. This can be + installed on ubuntu using the following command: + + sudo apt-get install diod + + Guest requirements: + * VirtIO support. We rely on VirtIO to move files into the simulation. + Please make sure that the following are set in the kernel + configuration: + + CONFIG_NET_9P=y + + CONFIG_NET_9P_VIRTIO=y + + CONFIG_9P_FS=y + + CONFIG_9P_FS_POSIX_ACL=y + + CONFIG_9P_FS_SECURITY=y + + CONFIG_VIRTIO_BLK=y + + * m5 binary. Please make sure that the m5 binary is on the device and + can by found in the path. + * Busybox. Due to restrictions, we assume that busybox is installed in + the guest system, and can be found in the path. + """ + + name = 'gem5_android' + default_working_directory = '/data/' + has_gpu = False + platform = 'android' + path_module = 'posixpath' + default_package_data_directory = '/data/data/' + default_binaries_directory = '/system/bin' + default_external_storage_directory = '/data/data/' + default_adb_name = None + + # gem5 can be very slow. Hence, we use some very long timeouts! + delay = 3600 + long_delay = 3 * delay + ready_timeout = long_delay + default_timeout = delay + + parameters = [ + Parameter('core_names', default=[], override=True), + Parameter('core_clusters', default=[], override=True), + Parameter('gem5_description', kind=str, default='', override=True, + description="Command line passed to the gem5 simulation. This command line is used " + "to set up the simulated system, and should be the same as used for a standard " + "gem5 simulation without workload automation. Note that this is simulation script " + "specific and will hence need to be tailored to each particular use case."), + Parameter('virtio_command', kind=str, default='', override=True, + description="gem5 VirtIO command line used to enable the VirtIO device in the " + "simulated system. At the very least, the root parameter of the VirtIO9PDiod device " + "must be exposed on the command line. Please set this root mount to {}, as it will " + "be replaced with the directory used by Workload Automation at runtime."), + Parameter('temp_dir', kind=str, default='', override=True, + description="Temporary directory used to pass files into the gem5 simulation. " + "Workload Automation will automatically create a directory in this folder, and " + "will remove it again once the simulation completes."), + Parameter('checkpoint_post_boot', kind=bool, default=False, mandatory=False, override=True, + description="This parameter tells Workload Automation to create a checkpoint of " + "the simulated system once the guest system has finished booting. This checkpoint " + "can then be used at a later stage by other WA runs to avoid booting the guest system" + " a second time. Set to True to take a checkpoint of the simulated system post boot."), + Parameter('run_delay', kind=int, default=0, mandatory=False, override=True, + description="This sets the time that the system should sleep in the simulated system prior" + " to running and workloads or taking checkpoints. This allows the system to quieten down" + " prior to running the workloads. When this is combined with the checkpoint_post_boot" + " option, it allows the checkpoint to be created post-sleep, and therefore the set of " + "workloads resuming from this checkpoint will not be required to sleep.") + ] + + # Overwritten from Device. For documentation, see corresponding method in + # Device. + + @property + def is_rooted(self): + # gem5 is always rooted + return True + + def __init__(self, **kwargs): + self.logger = logging.getLogger('gem5Device') + + super(Gem5Device, self).__init__(**kwargs) + self.adb_name = kwargs.get('adb_name') or self.default_adb_name + self.binaries_directory = kwargs.get('binaries_directory', self.default_binaries_directory) + self.package_data_directory = kwargs.get('package_data_directory', self.default_package_data_directory) + self.external_storage_directory = kwargs.get('external_storage_directory', + self.default_external_storage_directory) + self.logcat_poll_period = kwargs.get('logcat_poll_period') + + # The gem5 subprocess + self.gem5 = None + + self.gem5_port = -1 + self.busybox = None + self._is_initialized = False + self._is_ready = False + self._just_rebooted = False + self._is_rooted = None + self._available_frequencies = {} + self._available_governors = {} + self._available_governor_tunables = {} + self._number_of_cores = None + self._logcat_poller = None + self.gem5outdir = os.path.join(settings.output_directory, "gem5") + self.stdout_file = None + self.stderr_file = None + self.stderr_filename = None + self.checkpoint = False + self.sckt = None + + if not kwargs.get('gem5_description'): + raise ConfigError('Please specify the system configuration with gem5_description') + self.gem5_args = kwargs.get('gem5_description') + + if kwargs.get('temp_dir'): + self.temp_dir = kwargs.get('temp_dir') + else: + self.logger.info("No temporary directory passed in. Defaulting to /tmp") + self.temp_dir = '/tmp' + + if kwargs.get('checkpoint_post_boot'): + self.checkpoint = True + + if kwargs.get('run_delay'): + self.run_delay = kwargs.get('run_delay') + + # Find the first one that does not exist. Ensures that we do not re-use + # the directory used by someone else. + for i in xrange(sys.maxint): + directory = os.path.join(self.temp_dir, "wa_{}".format(i)) + try: + os.stat(directory) + continue + except OSError: + break + self.temp_dir = directory + self.logger.info("Using {} as the temporary directory.".format(self.temp_dir)) + + if not kwargs.get('virtio_command'): + raise ConfigError('Please specify the VirtIO command specific to your script, ending with the root parameter of the device.') + + self.gem5_vio_arg = kwargs.get('virtio_command').format(self.temp_dir) + self.logger.debug("gem5 VirtIO command: {}".format(self.gem5_vio_arg)) + + # Start the gem5 simulation when WA starts a run using a signal. + sig.connect(self.init_gem5, sig.RUN_START) + + def init_gem5(self, _): + """ + Start gem5, find out the telnet port and connect to the simulation. + + We first create the temporary directory used by VirtIO to pass files + into the simulation, as well as the gem5 output directory.We then create + files for the standard output and error for the gem5 process. The gem5 + process then is started. + """ + self.logger.info("Creating temporary directory: {}".format(self.temp_dir)) + os.mkdir(self.temp_dir) + os.mkdir(self.gem5outdir) + + # We need to redirect the standard output and standard error for the + # gem5 process to a file so that we can debug when things go wrong. + f = os.path.join(self.gem5outdir, 'stdout') + self.stdout_file = open(f, 'w') + f = os.path.join(self.gem5outdir, 'stderr') + self.stderr_file = open(f, 'w') + # We need to keep this so we can check which port to use for the telnet + # connection. + self.stderr_filename = f + + self.start_gem5() + + def start_gem5(self): + """ + Starts the gem5 simulator, and parses the output to get the telnet port. + """ + self.logger.info("Starting the gem5 simulator") + + command_line = "./build/ARM/gem5.fast --outdir={}/gem5 {} {}".format(settings.output_directory, + self.gem5_args, + self.gem5_vio_arg) + self.logger.debug("gem5 command line: {}".format(command_line)) + self.gem5 = subprocess.Popen(command_line.split(), stdout=self.stdout_file, stderr=self.stderr_file) + + while self.gem5_port == -1: + # Check that gem5 is running! + if self.gem5.poll(): + raise DeviceError("The gem5 process has crashed with error code {}!".format(self.gem5.poll())) + + # Open the stderr file + f = open(self.stderr_filename, 'r') + for line in f: + m = re.search(r"Listening\ for\ system\ connection\ on\ port\ (?P\d+)", line) + if m: + port = int(m.group('port')) + if port >= 3456 and port < 5900: + self.gem5_port = port + f.close() + break + else: + time.sleep(1) + f.close() + + def connect(self): # pylint: disable=R0912 + """ + Connect to the gem5 simulation and wait for Android to boot. Then, + create checkpoints, and mount the VirtIO device. + """ + self.connect_gem5() + + self.logger.info("Waiting for Android to boot...") + while True: + try: + available = (1 == int('0' + self.gem5_shell('getprop sys.boot_completed', check_exit_code=False))) + if available: + break + except (DeviceError, ValueError): + pass + time.sleep(60) + + self.logger.info("Android booted") + + if self.run_delay: + self.logger.info("Sleeping for {} seconds in the guest".format(self.run_delay)) + self.gem5_shell("sleep {}".format(self.run_delay)) + + if self.checkpoint: + self.checkpoint_gem5() + + self.mount_virtio() + self.logger.info("Creating the working directory in the simulated system") + self.gem5_shell('mkdir -p {}'.format(self.working_directory)) + self._is_ready = True + + def connect_gem5(self): # pylint: disable=R0912 + """ + Connect to the telnet port of the gem5 simulation. + + We connect, and wait for the prompt to be found. We do not use a timeout + for this, and wait for the prompt in a while loop as the gem5 simulation + can take many hours to reach a prompt when booting the system. We also + inject some newlines periodically to try and force gem5 to show a + prompt. Once the prompt has been found, we replace it with a unique + prompt to ensure that we are able to match it properly. We also disable + the echo as this simplifies parsing the output when executing commands + on the device. + """ + self.logger.info("Connecting to the gem5 simulation on port " + str(self.gem5_port)) + host = socket.gethostname() + port = self.gem5_port + + # Connect to the gem5 telnet port. Use a short timeout here. + self.sckt = ssh.TelnetConnection() + self.sckt.login(host, 'None', port=port, auto_prompt_reset=False, login_timeout=10) + + self.logger.info("Connected! Waiting for prompt...") + + # We need to find the prompt. It might be different if we are resuming + # from a checkpoint. Therefore, we test multiple options here. + prompt_found = False + while not prompt_found: + try: + # Try and force a prompt to be shown + self.sckt.send('\n') + self.sckt.expect([r'# ', r'\[PEXPECT\]\$'], timeout=self.delay) + prompt_found = True + except TIMEOUT: + pass + + self.logger.info("Setting unique prompt...") + + self.sckt.set_unique_prompt() + self.sckt.prompt() + self.logger.info("Prompt found and replaced with a unique string") + + self.sckt.setecho(False) + self.sync_gem5_shell() + + def mount_virtio(self): + """ + Mount the VirtIO device in the simulated system. + + We cannot assume any state for the VirtIO device in gem5 as it is not + serialised when checkpointing the system. Therefore, we unbind and + rebind the VirtIO device to force the driver to re-initialize the + device, prior to using it. We then mount the folder on the host system + using the VirtIo device. + """ + self.logger.info("Mounting VirtIO device in simulated system") + + # We always unbind, then re-bind the device. This ensures that the + # driver is re-loaded and that the device is re-initialized. Hence, this + # should work for both checkpointed and non-checkpointed gem5 systems. + vio_info = self.gem5_shell('ls /sys/bus/pci/drivers/virtio-pci/').strip().split() + mounts = [] + for f in vio_info: + if len(f.split(':')) > 1: + mounts.append(f.strip()) + + # Unbind and rebind all of the + for mount in mounts: + self.gem5_shell('echo -n "{}" > /sys/bus/pci/drivers/virtio-pci/unbind'.format(mount)) + self.gem5_shell('echo -n "{}" > /sys/bus/pci/drivers/virtio-pci/bind'.format(mount)) + + mount_command = "mount -t 9p -o trans=virtio,version=9p2000.L,aname={} gem5 /mnt/obb".format(self.temp_dir) + self.gem5_shell('busybox {}'.format(mount_command)) + + def disconnect(self): + """ + Close and disconnect from the gem5 simulation. Additionally, we remove + the temporary directory used to pass files into the simulation. + """ + self.logger.info("Gracefully terminating the gem5 simulation.") + try: + self.gem5_util("exit") + self.gem5.wait() + except EOF: + pass + self.logger.info("Removing the temporary directory") + try: + shutil.rmtree(self.temp_dir) + except OSError: + self.logger.warn("Failed to remove the temporary directory!") + + def close(self): + if self._logcat_poller: + self._logcat_poller.stop() + + def reset(self): + self.logger.warn("Attempt to restart the gem5 device. This is not supported!") + + def init(self): + pass + + def push_file(self, source, dest, as_root=False): # pylint: disable=W0221 + """ + Push a file to the gem5 device using VirtIO + + The file to push to the device is copied to the temporary directory on + the host, before being copied within the simulation to the destination. + Checks, in the form of 'ls' with error code checking, are performed to + ensure that the file is copied to the destination. + """ + filename = source.split('/')[-1] + self.logger.debug("Pushing {} to device.".format(source)) + self.logger.debug("temp_dir: {}".format(self.temp_dir)) + self.logger.debug("dest: {}".format(dest)) + self.logger.debug("filename: {}".format(filename)) + + # We need to copy the file to copy to the temporary directory + self.move_to_temp_dir(source) + + # Back to the gem5 world + self.gem5_shell("ls -al /mnt/obb/{}".format(filename)) + self.gem5_shell("busybox cp /mnt/obb/{} {}".format(filename, dest)) + self.gem5_shell("busybox sync") + self.gem5_shell("ls -al {}".format(dest)) + self.gem5_shell("ls -al /mnt/obb/") + self.logger.debug("Push complete.") + + def pull_file(self, source, dest, as_root=False): # pylint: disable=W0221 + """ + Pull a file from the gem5 device using m5 writefile + + First, we check the extension of the file to be copied. If the file ends + in .gz, then gem5 wrongly assumes that it should create a gzipped output + stream, which results in a gem5 error. Therefore, we rename the file on + the local device prior to the writefile command when required. Next, the + file is copied to the local directory within the guest as the m5 + writefile command assumes that the file is local. The file is then + written out to the host system using writefile, prior to being moved to + the destination on the host. + """ + filename = source.split('/')[-1] + self.logger.debug("Pulling {} from device.".format(filename)) + + # gem5 assumes that files ending in .gz are gzip-compressed. We need to + # work around this, else gem5 panics on us. Rename the file for use in + # gem5 + if filename[-3:] == '.gz': + filename += '.fugem5' + + self.logger.debug("pull_file {} {}".format(source, filename)) + # We don't check the exit code here because it is non-zero if the source + # and destination are the same. The ls below will cause an error if the + # file was not where we expected it to be. + self.gem5_shell("busybox cp {} {}".format(source, filename), check_exit_code=False) + self.gem5_shell("busybox sync") + self.gem5_shell("ls -la {}".format(filename)) + self.logger.debug('Finished the copy in the simulator') + self.gem5_util("writefile {}".format(filename)) + + if 'cpu' not in filename: + while not os.path.exists(os.path.join(self.gem5outdir, filename)): + time.sleep(1) + + # Perform the local move + shutil.move(os.path.join(self.gem5outdir, filename), dest) + self.logger.debug("Pull complete.") + + def delete_file(self, filepath, as_root=False): # pylint: disable=W0221 + """ Delete a file on the device """ + self._check_ready() + self.gem5_shell("rm '{}'".format(filepath)) + + def file_exists(self, filepath): + """ Check if a file exists """ + self._check_ready() + output = self.gem5_shell('if [ -e \'{}\' ]; then echo 1; else echo 0; fi'.format(filepath)) + try: + if int(output): + return True + else: + return False + except ValueError: + if output: + return True + else: + return False + + def install(self, filepath, timeout=default_timeout): # pylint: disable=W0221 + """ Install an APK or a normal executable """ + ext = os.path.splitext(filepath)[1].lower() + if ext == '.apk': + return self.install_apk(filepath, timeout) + else: + return self.install_executable(filepath) + + def install_apk(self, filepath, timeout=default_timeout): # pylint: disable=W0221 + """ + Install an APK on the gem5 device + + The APK is pushed to the device. Then the file and folder permissions + are changed to ensure that the APK can be correctly installed. The APK + is then installed on the device using 'pm'. + """ + self._check_ready() + self.logger.info("Installing {}".format(filepath)) + ext = os.path.splitext(filepath)[1].lower() + if ext == '.apk': + filename = os.path.basename(filepath) + on_device_path = os.path.join('/data/local/tmp', filename) + self.push_file(filepath, on_device_path) + # We need to make sure that the folder permissions are set + # correctly, else the APK does not install correctly. + self.gem5_shell('busybox chmod 775 /data/local/tmp') + self.gem5_shell('busybox chmod 774 {}'.format(on_device_path)) + self.logger.debug("Actually installing the APK: {}".format(on_device_path)) + return self.gem5_shell("pm install {}".format(on_device_path)) + else: + raise DeviceError('Can\'t install {}: unsupported format.'.format(filepath)) + + def install_executable(self, filepath, with_name=None): + """ Install an executable """ + executable_name = os.path.basename(filepath) + on_device_file = self.path.join(self.working_directory, executable_name) + on_device_executable = self.path.join(self.binaries_directory, executable_name) + self.push_file(filepath, on_device_file) + self.execute('busybox cp {} {}'.format(on_device_file, on_device_executable)) + self.execute('busybox chmod 0777 {}'.format(on_device_executable)) + return on_device_executable + + def uninstall(self, package): + self._check_ready() + self.gem5_shell("pm uninstall {}".format(package)) + + def execute(self, command, timeout=default_timeout, check_exit_code=True, background=False, + as_root=False, busybox=False, **kwargs): + self._check_ready() + if as_root and not self.is_rooted: + raise DeviceError('Attempting to execute "{}" as root on unrooted device.'.format(command)) + if busybox: + if not self.is_rooted: + raise DeviceError('Attempting to execute "{}" with busybox. '.format(command) + + 'Busybox can only be deployed to rooted devices.') + command = ' '.join([self.busybox, command]) + if background: + self.logger.debug("Attempt to execute in background. Not supported in gem5, hence ignored.") + return self.gem5_shell(command, as_root=as_root) + + def dump_logcat(self, outfile, filter_spec=None): + """ Extract logcat from simulation """ + self.logger.info("Extracting logcat from the simulated system") + filename = outfile.split('/')[-1] + command = 'logcat -d > {}'.format(filename) + self.gem5_shell(command) + self.pull_file("{}".format(filename), outfile) + + def clear_logcat(self): + """Clear (flush) logcat log.""" + if self._logcat_poller: + return self._logcat_poller.clear_buffer() + else: + return self.gem5_shell('logcat -c') + + def disable_selinux(self): + """ Disable SELinux. Overridden as parent implementation uses ADB """ + api_level = int(self.gem5_shell('getprop ro.build.version.sdk').strip()) + + # SELinux was added in Android 4.3 (API level 18). Trying to + # 'getenforce' in earlier versions will produce an error. + if api_level >= 18: + se_status = self.execute('getenforce', as_root=True).strip() + if se_status == 'Enforcing': + self.execute('setenforce 0', as_root=True) + + def get_properties(self, context): # pylint: disable=R0801 + """ Get the property files from the device """ + for propfile in self.property_files: + try: + normname = propfile.lstrip(self.path.sep).replace(self.path.sep, '.') + outfile = os.path.join(context.host_working_directory, normname) + if self.is_file(propfile): + self.execute('cat {} > {}'.format(propfile, normname)) + self.pull_file(normname, outfile) + elif self.is_directory(propfile): + self.get_directory(context, propfile) + continue + else: + continue + except DeviceError: + # We pull these files "opportunistically", so if a pull fails + # (e.g. we don't have permissions to read the file), just note + # it quietly (not as an error/warning) and move on. + self.logger.debug('Could not pull property file "{}"'.format(propfile)) + + # This is duplicated from the AndroidDevice class, as we want to run + # this code without executing the BaseLinuxDevice implementation + props = {} + props['android_id'] = self.get_android_id() + buildprop_file = os.path.join(context.host_working_directory, 'build.prop') + if not os.path.isfile(buildprop_file): + self.pull_file('/system/build.prop', context.host_working_directory) + self._update_build_properties(buildprop_file, props) + context.add_run_artifact('build_properties', buildprop_file, 'export') + + dumpsys_window_file = os.path.join(context.host_working_directory, 'window.dumpsys') + self.execute('{} > {}'.format('dumpsys window', 'window.dumpsys')) + self.pull_file('window.dumpsys', dumpsys_window_file) + context.add_run_artifact('dumpsys_window', dumpsys_window_file, 'meta') + return props + + def get_directory(self, context, directory): + """ Pull a directory from the device """ + normname = directory.lstrip(self.path.sep).replace(self.path.sep, '.') + outdir = os.path.join(context.host_working_directory, normname) + temp_file = os.path.join(context.host_working_directory, "{}.tar".format(normname)) + # Check that the folder exists + self.gem5_shell("ls -la {}".format(directory)) + # Compress the folder + try: + self.gem5_shell("{} tar -cvf {}.tar {}".format(self.busybox, normname, directory)) + except DeviceError: + self.logger.debug("Failed to run tar command on device! Not pulling {}".format(directory)) + return + self.pull_file(normname, temp_file) + f = tarfile.open(temp_file, 'r') + os.mkdir(outdir) + f.extractall(outdir) + os.remove(temp_file) + + def get_pids_of(self, process_name): + """ Returns a list of PIDs of all processes with the specified name. """ + result = self.gem5_shell('ps | busybox grep {}'.format(process_name), check_exit_code=False).strip() + if result and 'not found' not in result and len(result.split('\n')) > 2: + return [int(x.split()[1]) for x in result.split('\n')] + else: + return [] + + def disable_screen_lock(self): + """ + Attempts to disable he screen lock on the device. + + Overridden here as otherwise we have issues with too many backslashes. + """ + lockdb = '/data/system/locksettings.db' + sqlcommand = "update locksettings set value=\'0\' where name=\'screenlock.disabled\';" + self.execute('sqlite3 {} "{}"'.format(lockdb, sqlcommand), as_root=True) + + # gem5 should dump out a framebuffer. We can use this if it exists. Failing + # that, fall back to the parent class implementation. + def capture_screen(self, filepath): + file_list = os.listdir(self.gem5outdir) + screen_caps = [] + for f in file_list: + if '.bmp' in f: + screen_caps.append(f) + + if len(screen_caps) == 1: + # Bail out if we do not have image, and resort to the slower, built + # in method. + try: + import Image + gem5_image = os.path.join(self.gem5outdir, screen_caps[0]) + temp_image = os.path.join(self.gem5outdir, "file.png") + im = Image.open(gem5_image) + im.save(temp_image, "PNG") + shutil.copy(temp_image, filepath) + os.remove(temp_image) + self.logger.debug("capture_screen: using gem5 screencap") + return + except (shutil.Error, ImportError, IOError): + pass + + # If we didn't manage to do the above, call the parent class. + self.logger.debug("capture_screen: falling back to parent class implementation") + super(Gem5Device, self).capture_screen(filepath) + + # gem5 might be slow. Hence, we need to make the ping timeout very long. + def ping(self): + self.logger.debug("Pinging gem5 to see if it is still alive") + self.gem5_shell('ls /', timeout=self.longdelay) + + # Additional Android-specific methods. + def forward_port(self, from_port, to_port): + raise DeviceError('we do not need forwarding') + + # Internal methods: do not use outside of the class. + + def _check_ready(self): + """ + Check if the device is ready. + + As this is gem5, we just assume that the device is ready once we have + connected to the gem5 simulation, and updated the prompt. + """ + if not self._is_ready: + raise DeviceError('Device not ready.') + + def gem5_shell(self, command, as_root=False, timeout=None, check_exit_code=True, sync=True): # pylint: disable=R0912 + """ + Execute a command in the gem5 shell + + This wraps the telnet connection to gem5 and processes the raw output. + + This method waits for the shell to return, and then will try and + separate the output from the command from the command itself. If this + fails, warn, but continue with the potentially wrong output. + + The exit code is also checked by default, and non-zero exit codes will + raise a DeviceError. + """ + conn = self.sckt + if sync: + self.sync_gem5_shell() + + self.logger.debug("gem5_shell command: {}".format(command)) + + # Send the actual command + conn.send("{}\n".format(command)) + + # Wait for the response. We just sit here and wait for the prompt to + # appear, as gem5 might take a long time to provide the output. This + # avoids timeout issues. + command_index = -1 + while command_index == -1: + if conn.prompt(): + output = re.sub(r' \r([^\n])', r'\1', conn.before) + command_index = output.find(command) + # The command was probably too long and wrapped. This bit of + # code is a total hack! + if command_index == -1: + # We need to remove the backspace characters! + output = re.sub(r'[\b]', r'', output) + while True: + first_cr = output.find('\r') + first_lt = output.find('<') + if first_cr == -1 or first_lt == -1: + break + if first_cr < first_lt: + new_output = output[:first_cr] + new_output += output[first_lt + 1:] + output = new_output + command_index = output.find(command) + else: + break + + # If we STILL have -1, then we cannot match the command, but the + # prompt has returned. Hence, we have a bit of an issue. We + # warn, and return the whole output. + if command_index == -1: + self.logger.warn("gem5_shell: Unable to match command in command output. Expect parsing errors!") + command_index = 0 + + output = output[command_index + len(command):].strip() + + # It is possible that gem5 will echo the command. Therefore, we need to + # remove that too! + command_index = output.find(command) + if command_index != -1: + output = output[command_index + len(command):].strip() + + self.logger.debug("gem5_shell output: {}".format(output)) + + # We get a second prompt. Hence, we need to eat one to make sure that we + # stay in sync. If we do not do this, we risk getting out of sync for + # slower simulations. + self.sckt.expect(r'\[PEXPECT\]\$', timeout=10000) + + if check_exit_code: + exit_code_text = self.gem5_shell('echo $?', as_root=as_root, timeout=timeout, check_exit_code=False, sync=False) + try: + exit_code = int(exit_code_text.split()[0]) + if exit_code: + message = 'Got exit code {}\nfrom: {}\nOUTPUT: {}' + raise DeviceError(message.format(exit_code, command, output)) + except (ValueError, IndexError): + self.logger.warning('Could not get exit code for "{}",\ngot: "{}"'.format(command, exit_code_text)) + + return output + + def gem5_util(self, command): + """ Execute a gem5 utility command using the m5 binary on the device """ + self.gem5_shell('/sbin/m5 ' + command) + + def sync_gem5_shell(self): + """ + Synchronise with the gem5 shell. + + Write some unique text to the gem5 device to allow us to synchronise + with the shell output. We actually get two prompts so we need to match both of these. + """ + self.sckt.send("echo \*\*sync\*\*\n") + self.sckt.expect(r"\*\*sync\*\*", timeout=self.delay) + self.sckt.expect(r'\[PEXPECT\]\$', timeout=self.delay) + self.sckt.expect(r'\[PEXPECT\]\$', timeout=self.delay) + + def move_to_temp_dir(self, source): + """ Move a file to the temporary directory on the host for copying to the gem5 device """ + command = "cp {} {}".format(source, self.temp_dir) + self.logger.debug("Local copy command: {}".format(command)) + subprocess.call(command.split()) + subprocess.call("sync".split()) + + def checkpoint_gem5(self, end_simulation=False): + """ Checkpoint the gem5 simulation, storing all system state """ + self.logger.info("Taking a post-boot checkpoint") + self.gem5_util("checkpoint") + if end_simulation: + self.disconnect() From 3bf114cf48513b8857bd38417ab2083871d73d3e Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Mon, 2 Nov 2015 11:42:33 +0000 Subject: [PATCH 3/9] Gem5Device: Improve shell command matching - Replace ugly while True loop with a simple regex substitution achieving the same thing. This is required to match the command in the shell output when the command wraps around due to the length of the command. --- wlauto/devices/android/gem5/__init__.py | 22 ++++------------------ 1 file changed, 4 insertions(+), 18 deletions(-) diff --git a/wlauto/devices/android/gem5/__init__.py b/wlauto/devices/android/gem5/__init__.py index b95e12b3..75862e02 100644 --- a/wlauto/devices/android/gem5/__init__.py +++ b/wlauto/devices/android/gem5/__init__.py @@ -717,26 +717,12 @@ class Gem5Device(AndroidDevice): while command_index == -1: if conn.prompt(): output = re.sub(r' \r([^\n])', r'\1', conn.before) + output = re.sub(r'[\b]', r'', output) + # Deal with line wrapping + output = re.sub(r'[\r].+?<', r'', output) command_index = output.find(command) - # The command was probably too long and wrapped. This bit of - # code is a total hack! - if command_index == -1: - # We need to remove the backspace characters! - output = re.sub(r'[\b]', r'', output) - while True: - first_cr = output.find('\r') - first_lt = output.find('<') - if first_cr == -1 or first_lt == -1: - break - if first_cr < first_lt: - new_output = output[:first_cr] - new_output += output[first_lt + 1:] - output = new_output - command_index = output.find(command) - else: - break - # If we STILL have -1, then we cannot match the command, but the + # If we have -1, then we cannot match the command, but the # prompt has returned. Hence, we have a bit of an issue. We # warn, and return the whole output. if command_index == -1: From 29abd290f4c0b2666db0bad4c0f7397c570f37a5 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Mon, 2 Nov 2015 16:30:13 +0000 Subject: [PATCH 4/9] Gem5Device: Fix style issues - Fix up style issues --- wlauto/devices/android/gem5/__init__.py | 92 ++++++++++++++++--------- 1 file changed, 59 insertions(+), 33 deletions(-) diff --git a/wlauto/devices/android/gem5/__init__.py b/wlauto/devices/android/gem5/__init__.py index 75862e02..9f998faf 100644 --- a/wlauto/devices/android/gem5/__init__.py +++ b/wlauto/devices/android/gem5/__init__.py @@ -103,30 +103,41 @@ class Gem5Device(AndroidDevice): Parameter('core_names', default=[], override=True), Parameter('core_clusters', default=[], override=True), Parameter('gem5_description', kind=str, default='', override=True, - description="Command line passed to the gem5 simulation. This command line is used " - "to set up the simulated system, and should be the same as used for a standard " - "gem5 simulation without workload automation. Note that this is simulation script " - "specific and will hence need to be tailored to each particular use case."), + description="Command line passed to the gem5 simulation. This" + " command line is used to set up the simulated system, and " + "should be the same as used for a standard gem5 simulation " + "without workload automation. Note that this is simulation " + "script specific and will hence need to be tailored to each " + "particular use case."), Parameter('virtio_command', kind=str, default='', override=True, - description="gem5 VirtIO command line used to enable the VirtIO device in the " - "simulated system. At the very least, the root parameter of the VirtIO9PDiod device " - "must be exposed on the command line. Please set this root mount to {}, as it will " - "be replaced with the directory used by Workload Automation at runtime."), + description="gem5 VirtIO command line used to enable the " + "VirtIO device in the simulated system. At the very least, " + "the root parameter of the VirtIO9PDiod device must be " + "exposed on the command line. Please set this root mount to " + "{}, as it will be replaced with the directory used by " + "Workload Automation at runtime."), Parameter('temp_dir', kind=str, default='', override=True, - description="Temporary directory used to pass files into the gem5 simulation. " - "Workload Automation will automatically create a directory in this folder, and " - "will remove it again once the simulation completes."), - Parameter('checkpoint_post_boot', kind=bool, default=False, mandatory=False, override=True, - description="This parameter tells Workload Automation to create a checkpoint of " - "the simulated system once the guest system has finished booting. This checkpoint " - "can then be used at a later stage by other WA runs to avoid booting the guest system" - " a second time. Set to True to take a checkpoint of the simulated system post boot."), - Parameter('run_delay', kind=int, default=0, mandatory=False, override=True, - description="This sets the time that the system should sleep in the simulated system prior" - " to running and workloads or taking checkpoints. This allows the system to quieten down" - " prior to running the workloads. When this is combined with the checkpoint_post_boot" - " option, it allows the checkpoint to be created post-sleep, and therefore the set of " - "workloads resuming from this checkpoint will not be required to sleep.") + description="Temporary directory used to pass files into the " + "gem5 simulation. Workload Automation will automatically " + "create a directory in this folder, and will remove it again " + "once the simulation completes."), + Parameter('checkpoint_post_boot', kind=bool, default=False, + mandatory=False, override=True, description="This parameter " + "tells Workload Automation to create a checkpoint of the " + "simulated system once the guest system has finished booting." + " This checkpoint can then be used at a later stage by other " + "WA runs to avoid booting the guest system a second time. Set" + " to True to take a checkpoint of the simulated system post " + "boot."), + Parameter('run_delay', kind=int, default=0, mandatory=False, + override=True, description="This sets the time that the " + "system should sleep in the simulated system prior to " + "running and workloads or taking checkpoints. This allows " + "the system to quieten down prior to running the workloads. " + "When this is combined with the checkpoint_post_boot" + " option, it allows the checkpoint to be created post-sleep," + " and therefore the set of workloads resuming from this " + "checkpoint will not be required to sleep.") ] # Overwritten from Device. For documentation, see corresponding method in @@ -198,7 +209,9 @@ class Gem5Device(AndroidDevice): self.logger.info("Using {} as the temporary directory.".format(self.temp_dir)) if not kwargs.get('virtio_command'): - raise ConfigError('Please specify the VirtIO command specific to your script, ending with the root parameter of the device.') + raise ConfigError('Please specify the VirtIO command specific to ' + 'your script, ending with the root parameter of ' + 'the device.') self.gem5_vio_arg = kwargs.get('virtio_command').format(self.temp_dir) self.logger.debug("gem5 VirtIO command: {}".format(self.gem5_vio_arg)) @@ -241,7 +254,9 @@ class Gem5Device(AndroidDevice): self.gem5_args, self.gem5_vio_arg) self.logger.debug("gem5 command line: {}".format(command_line)) - self.gem5 = subprocess.Popen(command_line.split(), stdout=self.stdout_file, stderr=self.stderr_file) + self.gem5 = subprocess.Popen(command_line.split(), + stdout=self.stdout_file, + stderr=self.stderr_file) while self.gem5_port == -1: # Check that gem5 is running! @@ -306,13 +321,14 @@ class Gem5Device(AndroidDevice): the echo as this simplifies parsing the output when executing commands on the device. """ - self.logger.info("Connecting to the gem5 simulation on port " + str(self.gem5_port)) + self.logger.info("Connecting to the gem5 simulation on port {}".format(self.gem5_port)) host = socket.gethostname() port = self.gem5_port # Connect to the gem5 telnet port. Use a short timeout here. self.sckt = ssh.TelnetConnection() - self.sckt.login(host, 'None', port=port, auto_prompt_reset=False, login_timeout=10) + self.sckt.login(host, 'None', port=port, auto_prompt_reset=False, + login_timeout=10) self.logger.info("Connected! Waiting for prompt...") @@ -388,7 +404,8 @@ class Gem5Device(AndroidDevice): self._logcat_poller.stop() def reset(self): - self.logger.warn("Attempt to restart the gem5 device. This is not supported!") + self.logger.warn("Attempt to restart the gem5 device. This is not " + "supported!") def init(self): pass @@ -536,7 +553,8 @@ class Gem5Device(AndroidDevice): 'Busybox can only be deployed to rooted devices.') command = ' '.join([self.busybox, command]) if background: - self.logger.debug("Attempt to execute in background. Not supported in gem5, hence ignored.") + self.logger.debug("Attempt to execute in background. Not supported " + "in gem5, hence ignored.") return self.gem5_shell(command, as_root=as_root) def dump_logcat(self, outfile, filter_spec=None): @@ -622,7 +640,8 @@ class Gem5Device(AndroidDevice): def get_pids_of(self, process_name): """ Returns a list of PIDs of all processes with the specified name. """ - result = self.gem5_shell('ps | busybox grep {}'.format(process_name), check_exit_code=False).strip() + result = self.gem5_shell('ps | busybox grep {}'.format(process_name), + check_exit_code=False).strip() if result and 'not found' not in result and len(result.split('\n')) > 2: return [int(x.split()[1]) for x in result.split('\n')] else: @@ -726,7 +745,8 @@ class Gem5Device(AndroidDevice): # prompt has returned. Hence, we have a bit of an issue. We # warn, and return the whole output. if command_index == -1: - self.logger.warn("gem5_shell: Unable to match command in command output. Expect parsing errors!") + self.logger.warn("gem5_shell: Unable to match command in " + "command output. Expect parsing errors!") command_index = 0 output = output[command_index + len(command):].strip() @@ -745,7 +765,9 @@ class Gem5Device(AndroidDevice): self.sckt.expect(r'\[PEXPECT\]\$', timeout=10000) if check_exit_code: - exit_code_text = self.gem5_shell('echo $?', as_root=as_root, timeout=timeout, check_exit_code=False, sync=False) + exit_code_text = self.gem5_shell('echo $?', as_root=as_root, + timeout=timeout, check_exit_code=False, + sync=False) try: exit_code = int(exit_code_text.split()[0]) if exit_code: @@ -765,7 +787,8 @@ class Gem5Device(AndroidDevice): Synchronise with the gem5 shell. Write some unique text to the gem5 device to allow us to synchronise - with the shell output. We actually get two prompts so we need to match both of these. + with the shell output. We actually get two prompts so we need to match + both of these. """ self.sckt.send("echo \*\*sync\*\*\n") self.sckt.expect(r"\*\*sync\*\*", timeout=self.delay) @@ -773,7 +796,10 @@ class Gem5Device(AndroidDevice): self.sckt.expect(r'\[PEXPECT\]\$', timeout=self.delay) def move_to_temp_dir(self, source): - """ Move a file to the temporary directory on the host for copying to the gem5 device """ + """ + Move a file to the temporary directory on the host for copying to the + gem5 device + """ command = "cp {} {}".format(source, self.temp_dir) self.logger.debug("Local copy command: {}".format(command)) subprocess.call(command.split()) From 3a8eed106243977ed8515d058013eb7c030eba8f Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Wed, 4 Nov 2015 10:38:50 +0000 Subject: [PATCH 5/9] Gem5Device: Allowed the gem5 binary to be specified in the agenda - Added the gem5_binary option to the agenda which allows a different gem5 binary to be specified. This allows WA to be used with different levels of gem5 debugging. as well as allowing non-standard gem5 binary names and locations. --- wlauto/devices/android/gem5/__init__.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/wlauto/devices/android/gem5/__init__.py b/wlauto/devices/android/gem5/__init__.py index 9f998faf..99708d6e 100644 --- a/wlauto/devices/android/gem5/__init__.py +++ b/wlauto/devices/android/gem5/__init__.py @@ -102,6 +102,10 @@ class Gem5Device(AndroidDevice): parameters = [ Parameter('core_names', default=[], override=True), Parameter('core_clusters', default=[], override=True), + Parameter('gem5_binary', kind=str, default='./build/ARM/gem5.fast', + override=True, mandatory=False, + description="Command used to execute gem5. Adjust according " + "to needs."), Parameter('gem5_description', kind=str, default='', override=True, description="Command line passed to the gem5 simulation. This" " command line is used to set up the simulated system, and " @@ -180,6 +184,8 @@ class Gem5Device(AndroidDevice): self.checkpoint = False self.sckt = None + self.gem5_binary = kwargs.get('gem5_binary') + if not kwargs.get('gem5_description'): raise ConfigError('Please specify the system configuration with gem5_description') self.gem5_args = kwargs.get('gem5_description') @@ -250,9 +256,10 @@ class Gem5Device(AndroidDevice): """ self.logger.info("Starting the gem5 simulator") - command_line = "./build/ARM/gem5.fast --outdir={}/gem5 {} {}".format(settings.output_directory, - self.gem5_args, - self.gem5_vio_arg) + command_line = "{} --outdir={}/gem5 {} {}".format(self.gem5_binary, + settings.output_directory, + self.gem5_args, + self.gem5_vio_arg) self.logger.debug("gem5 command line: {}".format(command_line)) self.gem5 = subprocess.Popen(command_line.split(), stdout=self.stdout_file, From ee4764adc4c03193d065a0ceb344410671148625 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Tue, 10 Nov 2015 12:52:50 +0000 Subject: [PATCH 6/9] Gem5Device: Addressed review comments - Replaced hard-coded pexpect expect string with UNIQUE_PROMPT. - Changed the capture_screen debug to a warning to make sure that the user knows when it happens. - Fixed the logic for checking when a file exists. Previously, if the output could not correctly be processed (ValueError) then we just assumed that the file existed if there was any output at all. This is clearly not a good default. Changed to default to False if it was not able to process the output as this seems to be the safest option. - Changed ad hoc filename extraction to use os.path.basename. - Removed the processing of some kwargs and defaults that are handled by the parent class. - Stopped overriding some paramaters which were purely defined in the Gem5Device. --- wlauto/devices/android/gem5/__init__.py | 98 +++++++------------------ 1 file changed, 27 insertions(+), 71 deletions(-) diff --git a/wlauto/devices/android/gem5/__init__.py b/wlauto/devices/android/gem5/__init__.py index 99708d6e..cd6b7162 100644 --- a/wlauto/devices/android/gem5/__init__.py +++ b/wlauto/devices/android/gem5/__init__.py @@ -28,7 +28,7 @@ from pexpect import EOF, TIMEOUT from wlauto import AndroidDevice, settings, Parameter from wlauto.core import signal as sig -from wlauto.utils import ssh +from wlauto.utils import ssh, types from wlauto.exceptions import DeviceError, ConfigError @@ -84,14 +84,9 @@ class Gem5Device(AndroidDevice): """ name = 'gem5_android' - default_working_directory = '/data/' has_gpu = False platform = 'android' path_module = 'posixpath' - default_package_data_directory = '/data/data/' - default_binaries_directory = '/system/bin' - default_external_storage_directory = '/data/data/' - default_adb_name = None # gem5 can be very slow. Hence, we use some very long timeouts! delay = 3600 @@ -103,30 +98,30 @@ class Gem5Device(AndroidDevice): Parameter('core_names', default=[], override=True), Parameter('core_clusters', default=[], override=True), Parameter('gem5_binary', kind=str, default='./build/ARM/gem5.fast', - override=True, mandatory=False, - description="Command used to execute gem5. Adjust according " - "to needs."), - Parameter('gem5_description', kind=str, default='', override=True, + mandatory=False, description="Command used to execute gem5. " + "Adjust according to needs."), + Parameter('gem5_args', kind=types.arguments, mandatory=True, description="Command line passed to the gem5 simulation. This" " command line is used to set up the simulated system, and " "should be the same as used for a standard gem5 simulation " "without workload automation. Note that this is simulation " "script specific and will hence need to be tailored to each " "particular use case."), - Parameter('virtio_command', kind=str, default='', override=True, + Parameter('gem5_vio_args', kind=types.arguments, mandatory=True, + constraint=lambda x: "{}" in str(x), description="gem5 VirtIO command line used to enable the " "VirtIO device in the simulated system. At the very least, " "the root parameter of the VirtIO9PDiod device must be " "exposed on the command line. Please set this root mount to " "{}, as it will be replaced with the directory used by " "Workload Automation at runtime."), - Parameter('temp_dir', kind=str, default='', override=True, + Parameter('temp_dir', kind=str, default='/tmp', description="Temporary directory used to pass files into the " "gem5 simulation. Workload Automation will automatically " "create a directory in this folder, and will remove it again " "once the simulation completes."), - Parameter('checkpoint_post_boot', kind=bool, default=False, - mandatory=False, override=True, description="This parameter " + Parameter('checkpoint', kind=bool, default=False, + mandatory=False, description="This parameter " "tells Workload Automation to create a checkpoint of the " "simulated system once the guest system has finished booting." " This checkpoint can then be used at a later stage by other " @@ -134,7 +129,8 @@ class Gem5Device(AndroidDevice): " to True to take a checkpoint of the simulated system post " "boot."), Parameter('run_delay', kind=int, default=0, mandatory=False, - override=True, description="This sets the time that the " + constraint=lambda x: x >= 0, + description="This sets the time that the " "system should sleep in the simulated system prior to " "running and workloads or taking checkpoints. This allows " "the system to quieten down prior to running the workloads. " @@ -156,27 +152,10 @@ class Gem5Device(AndroidDevice): self.logger = logging.getLogger('gem5Device') super(Gem5Device, self).__init__(**kwargs) - self.adb_name = kwargs.get('adb_name') or self.default_adb_name - self.binaries_directory = kwargs.get('binaries_directory', self.default_binaries_directory) - self.package_data_directory = kwargs.get('package_data_directory', self.default_package_data_directory) - self.external_storage_directory = kwargs.get('external_storage_directory', - self.default_external_storage_directory) - self.logcat_poll_period = kwargs.get('logcat_poll_period') # The gem5 subprocess self.gem5 = None - self.gem5_port = -1 - self.busybox = None - self._is_initialized = False - self._is_ready = False - self._just_rebooted = False - self._is_rooted = None - self._available_frequencies = {} - self._available_governors = {} - self._available_governor_tunables = {} - self._number_of_cores = None - self._logcat_poller = None self.gem5outdir = os.path.join(settings.output_directory, "gem5") self.stdout_file = None self.stderr_file = None @@ -184,24 +163,6 @@ class Gem5Device(AndroidDevice): self.checkpoint = False self.sckt = None - self.gem5_binary = kwargs.get('gem5_binary') - - if not kwargs.get('gem5_description'): - raise ConfigError('Please specify the system configuration with gem5_description') - self.gem5_args = kwargs.get('gem5_description') - - if kwargs.get('temp_dir'): - self.temp_dir = kwargs.get('temp_dir') - else: - self.logger.info("No temporary directory passed in. Defaulting to /tmp") - self.temp_dir = '/tmp' - - if kwargs.get('checkpoint_post_boot'): - self.checkpoint = True - - if kwargs.get('run_delay'): - self.run_delay = kwargs.get('run_delay') - # Find the first one that does not exist. Ensures that we do not re-use # the directory used by someone else. for i in xrange(sys.maxint): @@ -212,19 +173,16 @@ class Gem5Device(AndroidDevice): except OSError: break self.temp_dir = directory - self.logger.info("Using {} as the temporary directory.".format(self.temp_dir)) - - if not kwargs.get('virtio_command'): - raise ConfigError('Please specify the VirtIO command specific to ' - 'your script, ending with the root parameter of ' - 'the device.') - - self.gem5_vio_arg = kwargs.get('virtio_command').format(self.temp_dir) - self.logger.debug("gem5 VirtIO command: {}".format(self.gem5_vio_arg)) + self.logger.debug("Using {} as the temporary directory.".format(self.temp_dir)) # Start the gem5 simulation when WA starts a run using a signal. sig.connect(self.init_gem5, sig.RUN_START) + def validate(self): + # Assemble the virtio args + self.gem5_vio_args = str(self.gem5_vio_args).format(self.temp_dir) + self.logger.debug("gem5 VirtIO command: {}".format(self.gem5_vio_args)) + def init_gem5(self, _): """ Start gem5, find out the telnet port and connect to the simulation. @@ -259,7 +217,7 @@ class Gem5Device(AndroidDevice): command_line = "{} --outdir={}/gem5 {} {}".format(self.gem5_binary, settings.output_directory, self.gem5_args, - self.gem5_vio_arg) + self.gem5_vio_args) self.logger.debug("gem5 command line: {}".format(command_line)) self.gem5 = subprocess.Popen(command_line.split(), stdout=self.stdout_file, @@ -346,7 +304,7 @@ class Gem5Device(AndroidDevice): try: # Try and force a prompt to be shown self.sckt.send('\n') - self.sckt.expect([r'# ', r'\[PEXPECT\]\$'], timeout=self.delay) + self.sckt.expect([r'# ', self.sckt.UNIQUE_PROMPT], timeout=self.delay) prompt_found = True except TIMEOUT: pass @@ -426,7 +384,7 @@ class Gem5Device(AndroidDevice): Checks, in the form of 'ls' with error code checking, are performed to ensure that the file is copied to the destination. """ - filename = source.split('/')[-1] + filename = os.path.basename(source) self.logger.debug("Pushing {} to device.".format(source)) self.logger.debug("temp_dir: {}".format(self.temp_dir)) self.logger.debug("dest: {}".format(dest)) @@ -498,10 +456,8 @@ class Gem5Device(AndroidDevice): else: return False except ValueError: - if output: - return True - else: - return False + # If we cannot process the output, assume that there is no file + return False def install(self, filepath, timeout=default_timeout): # pylint: disable=W0221 """ Install an APK or a normal executable """ @@ -690,7 +646,7 @@ class Gem5Device(AndroidDevice): pass # If we didn't manage to do the above, call the parent class. - self.logger.debug("capture_screen: falling back to parent class implementation") + self.logger.warning("capture_screen: falling back to parent class implementation") super(Gem5Device, self).capture_screen(filepath) # gem5 might be slow. Hence, we need to make the ping timeout very long. @@ -700,7 +656,7 @@ class Gem5Device(AndroidDevice): # Additional Android-specific methods. def forward_port(self, from_port, to_port): - raise DeviceError('we do not need forwarding') + self.logger.warning('Tried to forward port. We do not need any forwarding.') # Internal methods: do not use outside of the class. @@ -769,7 +725,7 @@ class Gem5Device(AndroidDevice): # We get a second prompt. Hence, we need to eat one to make sure that we # stay in sync. If we do not do this, we risk getting out of sync for # slower simulations. - self.sckt.expect(r'\[PEXPECT\]\$', timeout=10000) + self.sckt.expect(self.sckt.UNIQUE_PROMPT, timeout=10000) if check_exit_code: exit_code_text = self.gem5_shell('echo $?', as_root=as_root, @@ -799,8 +755,8 @@ class Gem5Device(AndroidDevice): """ self.sckt.send("echo \*\*sync\*\*\n") self.sckt.expect(r"\*\*sync\*\*", timeout=self.delay) - self.sckt.expect(r'\[PEXPECT\]\$', timeout=self.delay) - self.sckt.expect(r'\[PEXPECT\]\$', timeout=self.delay) + self.sckt.expect(self.sckt.UNIQUE_PROMPT, timeout=self.delay) + self.sckt.expect(self.sckt.UNIQUE_PROMPT, timeout=self.delay) def move_to_temp_dir(self, source): """ From 95f17702d74d06d8f61bc710c53472d25d2f92ed Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Tue, 10 Nov 2015 13:25:41 +0000 Subject: [PATCH 7/9] AndroidDevice: Move the processing of Android properties to an internal method - Move the processing of Android properties to an internal method. This allows the Android properties to be extracted without extracting those of the Linux device. - Redirect the output from 'dumpsys window' to a file and pull the file as opposed to extracting the output from the terminal. This is more reliable in the event that another process writes to the shell. --- wlauto/common/android/device.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/wlauto/common/android/device.py b/wlauto/common/android/device.py index 17998e6d..ad9e7685 100644 --- a/wlauto/common/android/device.py +++ b/wlauto/common/android/device.py @@ -490,6 +490,11 @@ class AndroidDevice(BaseLinuxDevice): # pylint: disable=W0223 def get_properties(self, context): """Captures and saves the information from /system/build.prop and /proc/version""" props = super(AndroidDevice, self).get_properties(context) + props.update(self._get_android_properties(context)) + return props + + def _get_android_properties(self, context): + props = {} props['android_id'] = self.get_android_id() buildprop_file = os.path.join(context.host_working_directory, 'build.prop') if not os.path.isfile(buildprop_file): @@ -498,9 +503,8 @@ class AndroidDevice(BaseLinuxDevice): # pylint: disable=W0223 context.add_run_artifact('build_properties', buildprop_file, 'export') dumpsys_window_file = os.path.join(context.host_working_directory, 'window.dumpsys') - dumpsys_window_output = self.execute('dumpsys window') - with open(dumpsys_window_file, 'w') as wfh: - wfh.write(dumpsys_window_output) + self.execute('{} > {}'.format('dumpsys window', 'window.dumpsys')) + self.pull_file('window.dumpsys', dumpsys_window_file) context.add_run_artifact('dumpsys_window', dumpsys_window_file, 'meta') return props From 672c74c76caf48ce9859cab2ec26cd519dd2698d Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Tue, 10 Nov 2015 13:28:25 +0000 Subject: [PATCH 8/9] Gem5Device: Avoid duplicate get_properties code - Remove the duplicated get_properties code by calling the internal _get_android_properties method directly. --- wlauto/devices/android/gem5/__init__.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/wlauto/devices/android/gem5/__init__.py b/wlauto/devices/android/gem5/__init__.py index cd6b7162..5a63d0f2 100644 --- a/wlauto/devices/android/gem5/__init__.py +++ b/wlauto/devices/android/gem5/__init__.py @@ -566,21 +566,9 @@ class Gem5Device(AndroidDevice): # it quietly (not as an error/warning) and move on. self.logger.debug('Could not pull property file "{}"'.format(propfile)) - # This is duplicated from the AndroidDevice class, as we want to run - # this code without executing the BaseLinuxDevice implementation - props = {} - props['android_id'] = self.get_android_id() - buildprop_file = os.path.join(context.host_working_directory, 'build.prop') - if not os.path.isfile(buildprop_file): - self.pull_file('/system/build.prop', context.host_working_directory) - self._update_build_properties(buildprop_file, props) - context.add_run_artifact('build_properties', buildprop_file, 'export') - - dumpsys_window_file = os.path.join(context.host_working_directory, 'window.dumpsys') - self.execute('{} > {}'.format('dumpsys window', 'window.dumpsys')) - self.pull_file('window.dumpsys', dumpsys_window_file) - context.add_run_artifact('dumpsys_window', dumpsys_window_file, 'meta') - return props + # As we do not call the super class, we explicitly need to + # process the Android properties. + props = self._get_android_properties(context) def get_directory(self, context, directory): """ Pull a directory from the device """ From 55f6ef4a5e485de3c1ba0751273d3ec3c06a2363 Mon Sep 17 00:00:00 2001 From: Sascha Bischoff Date: Thu, 5 Nov 2015 14:12:19 +0000 Subject: [PATCH 9/9] Gem5Device: Remove VirtIO device rebinding to align with gem5 - Remove the unbind and rebind for the VirtIO 9P mount method as gem5 now checkpoints the basic state of the device. This allows us to just mount it assuming that checkpoint have been created correctly. --- wlauto/devices/android/gem5/__init__.py | 38 ++++++------------------- 1 file changed, 9 insertions(+), 29 deletions(-) diff --git a/wlauto/devices/android/gem5/__init__.py b/wlauto/devices/android/gem5/__init__.py index 5a63d0f2..7eb3303e 100644 --- a/wlauto/devices/android/gem5/__init__.py +++ b/wlauto/devices/android/gem5/__init__.py @@ -318,35 +318,6 @@ class Gem5Device(AndroidDevice): self.sckt.setecho(False) self.sync_gem5_shell() - def mount_virtio(self): - """ - Mount the VirtIO device in the simulated system. - - We cannot assume any state for the VirtIO device in gem5 as it is not - serialised when checkpointing the system. Therefore, we unbind and - rebind the VirtIO device to force the driver to re-initialize the - device, prior to using it. We then mount the folder on the host system - using the VirtIo device. - """ - self.logger.info("Mounting VirtIO device in simulated system") - - # We always unbind, then re-bind the device. This ensures that the - # driver is re-loaded and that the device is re-initialized. Hence, this - # should work for both checkpointed and non-checkpointed gem5 systems. - vio_info = self.gem5_shell('ls /sys/bus/pci/drivers/virtio-pci/').strip().split() - mounts = [] - for f in vio_info: - if len(f.split(':')) > 1: - mounts.append(f.strip()) - - # Unbind and rebind all of the - for mount in mounts: - self.gem5_shell('echo -n "{}" > /sys/bus/pci/drivers/virtio-pci/unbind'.format(mount)) - self.gem5_shell('echo -n "{}" > /sys/bus/pci/drivers/virtio-pci/bind'.format(mount)) - - mount_command = "mount -t 9p -o trans=virtio,version=9p2000.L,aname={} gem5 /mnt/obb".format(self.temp_dir) - self.gem5_shell('busybox {}'.format(mount_command)) - def disconnect(self): """ Close and disconnect from the gem5 simulation. Additionally, we remove @@ -762,3 +733,12 @@ class Gem5Device(AndroidDevice): self.gem5_util("checkpoint") if end_simulation: self.disconnect() + + def mount_virtio(self): + """ + Mount the VirtIO device in the simulated system. + """ + self.logger.info("Mounting VirtIO device in simulated system") + + mount_command = "mount -t 9p -o trans=virtio,version=9p2000.L,aname={} gem5 /mnt/obb".format(self.temp_dir) + self.gem5_shell('busybox {}'.format(mount_command))