sascha/devlib
1
0
Fork 0
mirror of https://github.com/ARM-software/devlib.git synced 2025-09-23 04:11:54 +01:00
Code Issues Releases Wiki Activity

target: Add Target.{push,pull}(globbing=False) parameter

Browse Source 
When globbing=True, the source is interpreted as a globbing pattern on
the target and is expanded before pulling the files or folders.

This also aligns the behaviour of all targets:
    * adb connection was supported a limited form of globbing by default
      (only on the last component of the path)
    * SCP was supporting a limited form of globbing
    * GEM5 was not supporting globbing at all
    * paramiko was not supporting globbing at all

Also fix a race condition on push/pull as root, where pushing/pulling
the same file from multiple threads would have ended up using the same
temporary file.
This commit is contained in:
douglas-raillard-arm
2020-06-16 11:56:25 +01:00
committed by Marc Bonnici
parent 07bbf902ba
commit 24e6de67ae
6 changed files with 259 additions and 131 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
devlib/utils/android.py
View File

@@ -19,6 +19,7 @@ Utility functions for working with Android devices through adb.
"""
# pylint: disable=E1103
import glob
import os
import re
import sys
@@ -288,28 +289,24 @@ class AdbConnection(ConnectionBase):
self._setup_ls()
self._setup_su()
def push(self, source, dest, timeout=None):
def _push_pull(self, action, sources, dest, timeout):
if timeout is None:
timeout = self.timeout
command = "push {} {}".format(quote(source), quote(dest))
if not os.path.exists(source):
raise HostError('No such file "{}"'.format(source))
return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
def pull(self, source, dest, timeout=None):
if timeout is None:
timeout = self.timeout
# Pull all files matching a wildcard expression
if os.path.isdir(dest) and \
('*' in source or '?' in source):
command = 'shell {} {}'.format(self.ls_command, source)
output = adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
for line in output.splitlines():
command = "pull {} {}".format(quote(line.strip()), quote(dest))
adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
return
command = "pull {} {}".format(quote(source), quote(dest))
return adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
paths = sources + [dest]
# Quote twice to avoid expansion by host shell, then ADB globbing
do_quote = lambda x: quote(glob.escape(x))
paths = ' '.join(map(do_quote, paths))
command = "{} {}".format(action, paths)
adb_command(self.device, command, timeout=timeout, adb_server=self.adb_server)
def push(self, sources, dest, timeout=None):
return self._push_pull('push', sources, dest, timeout)
def pull(self, sources, dest, timeout=None):
return self._push_pull('pull', sources, dest, timeout)
# pylint: disable=unused-argument
def execute(self, command, timeout=None, check_exit_code=False,

145
devlib/utils/ssh.py
View File

@@ -14,6 +14,7 @@
#
import glob
import os
import stat
import logging
@@ -299,15 +300,21 @@ class SshConnectionBase(ConnectionBase):
self.options = {}
logger.debug('Logging in {}@{}'.format(username, host))
def push(self, source, dest, timeout=30):
dest = '{}@{}:{}'.format(self.username, self.host, dest)
return self._scp(source, dest, timeout)
def push(self, sources, dest, timeout=30):
# Quote the destination as SCP would apply globbing too
dest = '{}@{}:{}'.format(self.username, self.host, quote(dest))
paths = sources + [dest]
return self._scp(paths, timeout)
def pull(self, source, dest, timeout=30):
source = '{}@{}:{}'.format(self.username, self.host, source)
return self._scp(source, dest, timeout)
def pull(self, sources, dest, timeout=30):
# First level of escaping for the remote shell
sources = ' '.join(map(quote, sources))
# All the sources are merged into one scp parameter
sources = '{}@{}:{}'.format(self.username, self.host, sources)
paths = [sources, dest]
self._scp(paths, timeout)
def _scp(self, source, dest, timeout=30):
def _scp(self, paths, timeout=30):
# NOTE: the version of scp in Ubuntu 12.04 occasionally (and bizarrely)
# fails to connect to a device if port is explicitly specified using -P
# option, even if it is the default port, 22. To minimize this problem,
@@ -316,12 +323,12 @@ class SshConnectionBase(ConnectionBase):
keyfile_string = '-i {}'.format(quote(self.keyfile)) if self.keyfile else ''
options = " ".join(["-o {}={}".format(key, val)
for key, val in self.options.items()])
command = '{} {} -r {} {} {} {}'.format(scp,
paths = ' '.join(map(quote, paths))
command = '{} {} -r {} {} {}'.format(scp,
options,
keyfile_string,
port_string,
quote(source),
quote(dest))
paths)
command_redacted = command
logger.debug(command)
if self.password:
@@ -534,21 +541,23 @@ class SshConnection(SshConnectionBase):
# Maybe that was a directory, so retry as such
cls._pull_folder(sftp, src, dst)
def push(self, source, dest, timeout=30):
def push(self, sources, dest, timeout=30):
# If using scp, use implementation from base class
if self.use_scp:
super().push(source, dest, timeout)
super().push(sources, dest, timeout)
else:
with _handle_paramiko_exceptions(), self._get_sftp(timeout) as sftp:
self._push_path(sftp, source, dest)
for source in sources:
self._push_path(sftp, source, dest)
def pull(self, source, dest, timeout=30):
def pull(self, sources, dest, timeout=30):
# If using scp, use implementation from base class
if self.use_scp:
super().pull(source, dest, timeout)
super().pull(sources, dest, timeout)
else:
with _handle_paramiko_exceptions(), self._get_sftp(timeout) as sftp:
self._pull_path(sftp, source, dest)
for source in sources:
self._pull_path(sftp, source, dest)
def execute(self, command, timeout=None, check_exit_code=True,
as_root=False, strip_colors=True, will_succeed=False): #pylint: disable=unused-argument
@@ -1012,7 +1021,7 @@ class Gem5Connection(TelnetConnection):
.format(self.gem5_input_dir, indir))
self.gem5_input_dir = indir
def push(self, source, dest, timeout=None):
def push(self, sources, dest, timeout=None):
"""
Push a file to the gem5 device using VirtIO
@@ -1024,28 +1033,29 @@ class Gem5Connection(TelnetConnection):
# First check if the connection is set up to interact with gem5
self._check_ready()
filename = os.path.basename(source)
logger.debug("Pushing {} to device.".format(source))
logger.debug("gem5interactdir: {}".format(self.gem5_interact_dir))
logger.debug("dest: {}".format(dest))
logger.debug("filename: {}".format(filename))
for source in sources:
filename = os.path.basename(source)
logger.debug("Pushing {} to device.".format(source))
logger.debug("gem5interactdir: {}".format(self.gem5_interact_dir))
logger.debug("dest: {}".format(dest))
logger.debug("filename: {}".format(filename))
# We need to copy the file to copy to the temporary directory
self._move_to_temp_dir(source)
# We need to copy the file to copy to the temporary directory
self._move_to_temp_dir(source)
# Dest in gem5 world is a file rather than directory
if os.path.basename(dest) != filename:
dest = os.path.join(dest, filename)
# Back to the gem5 world
filename = quote(self.gem5_input_dir + filename)
self._gem5_shell("ls -al {}".format(filename))
self._gem5_shell("cat {} > {}".format(filename, quote(dest)))
self._gem5_shell("sync")
self._gem5_shell("ls -al {}".format(quote(dest)))
self._gem5_shell("ls -al {}".format(quote(self.gem5_input_dir)))
logger.debug("Push complete.")
# Dest in gem5 world is a file rather than directory
if os.path.basename(dest) != filename:
dest = os.path.join(dest, filename)
# Back to the gem5 world
filename = quote(self.gem5_input_dir + filename)
self._gem5_shell("ls -al {}".format(filename))
self._gem5_shell("cat {} > {}".format(filename, quote(dest)))
self._gem5_shell("sync")
self._gem5_shell("ls -al {}".format(quote(dest)))
self._gem5_shell("ls -al {}".format(quote(self.gem5_input_dir)))
logger.debug("Push complete.")
def pull(self, source, dest, timeout=0): #pylint: disable=unused-argument
def pull(self, sources, dest, timeout=0): #pylint: disable=unused-argument
"""
Pull a file from the gem5 device using m5 writefile
@@ -1057,40 +1067,41 @@ class Gem5Connection(TelnetConnection):
# First check if the connection is set up to interact with gem5
self._check_ready()
result = self._gem5_shell("ls {}".format(source))
files = strip_bash_colors(result).split()
for source in sources:
result = self._gem5_shell("ls {}".format(source))
files = strip_bash_colors(result).split()
for filename in files:
dest_file = os.path.basename(filename)
logger.debug("pull_file {} {}".format(filename, dest_file))
# writefile needs the file to be copied to be in the current
# working directory so if needed, copy to the working directory
# We don't check the exit code here because it is non-zero if the
# source and destination are the same. The ls below will cause an
# error if the file was not where we expected it to be.
if os.path.isabs(source):
if os.path.dirname(source) != self.execute('pwd',
check_exit_code=False):
self._gem5_shell("cat {} > {}".format(quote(filename),
quote(dest_file)))
self._gem5_shell("sync")
self._gem5_shell("ls -la {}".format(dest_file))
logger.debug('Finished the copy in the simulator')
self._gem5_util("writefile {}".format(dest_file))
for filename in files:
dest_file = os.path.basename(filename)
logger.debug("pull_file {} {}".format(filename, dest_file))
# writefile needs the file to be copied to be in the current
# working directory so if needed, copy to the working directory
# We don't check the exit code here because it is non-zero if the
# source and destination are the same. The ls below will cause an
# error if the file was not where we expected it to be.
if os.path.isabs(source):
if os.path.dirname(source) != self.execute('pwd',
check_exit_code=False):
self._gem5_shell("cat {} > {}".format(quote(filename),
quote(dest_file)))
self._gem5_shell("sync")
self._gem5_shell("ls -la {}".format(dest_file))
logger.debug('Finished the copy in the simulator')
self._gem5_util("writefile {}".format(dest_file))
if 'cpu' not in filename:
while not os.path.exists(os.path.join(self.gem5_out_dir,
dest_file)):
time.sleep(1)
if 'cpu' not in filename:
while not os.path.exists(os.path.join(self.gem5_out_dir,
dest_file)):
time.sleep(1)
# Perform the local move
if os.path.exists(os.path.join(dest, dest_file)):
logger.warning(
'Destination file {} already exists!'\
.format(dest_file))
else:
shutil.move(os.path.join(self.gem5_out_dir, dest_file), dest)
logger.debug("Pull complete.")
# Perform the local move
if os.path.exists(os.path.join(dest, dest_file)):
logger.warning(
'Destination file {} already exists!'\
.format(dest_file))
else:
shutil.move(os.path.join(self.gem5_out_dir, dest_file), dest)
logger.debug("Pull complete.")
def execute(self, command, timeout=1000, check_exit_code=True,
as_root=False, strip_colors=True, will_succeed=False):