LinuxTarget: fixing reboot sequence

When issuing a target.reboot(), the reset was immediately followed by a
boot() (for platforms that have it) and an attempt to connect. When
issuing a soft reset, it's possible the target is still shutting down
when the attempt to connect is made. This results in the connection
succeeding but being severed shortly thereafter.

This introduces a delay after the reset to the reboot sequence, giving
the target time to shutdown and improves the handling of EOF's that
result from failed reconnection attempts (while still being with the
allotted timeout period.

devlib/target.py

@@ -228,6 +228,13 @@ class Target(object):
                           '(in which case, a hard_reset module must be installed)'
                 raise TargetError(message)
             self.reset()
+            # Wait a fixed delay before starting polling to give the target time to
+            # shut down, otherwise, might create the connection while it's still shutting
+            # down resulting in subsequenct connection failing.
+            self.logger.debug('Waiting for target to power down...')
+            reset_delay = 20
+            time.sleep(reset_delay)
+            timeout = max(timeout - reset_delay, 10)
         if self.has('boot'):
             self.boot()  # pylint: disable=no-member
         if connect:

devlib/utils/ssh.py

@@ -22,6 +22,7 @@ import re
 import threading
 import tempfile
 import shutil
+import time
 
 import pexpect
 from distutils.version import StrictVersion as V
@@ -44,19 +45,28 @@ logger = logging.getLogger('ssh')
 
 def ssh_get_shell(host, username, password=None, keyfile=None, port=None, timeout=10, telnet=False):
     _check_env()
+    start_time = time.time()
+    while True:
         if telnet:
             if keyfile:
                 raise ValueError('keyfile may not be used with a telnet connection.')
             conn = TelnetConnection()
         else:  # ssh
             conn = pxssh.pxssh()
+
         try:
             if keyfile:
                 conn.login(host, username, ssh_key=keyfile, port=port, login_timeout=timeout)
             else:
                 conn.login(host, username, password, port=port, login_timeout=timeout)
+            break
         except EOF:
-        raise TargetError('Could not connect to {}; is the host name correct?'.format(host))
+            timeout -= time.time() - start_time
+            if timeout <= 0:
+                message = 'Could not connect to {}; is the host name correct?'
+                raise TargetError(message.format(host))
+            time.sleep(5)
+
     conn.setwinsize(500,200)
     conn.sendline('')
     conn.prompt()
@@ -151,6 +161,7 @@ class SshConnection(object):
         return self._scp(source, dest, timeout)
 
     def execute(self, command, timeout=None, check_exit_code=True, as_root=False, strip_colors=True):
+        try:
             with self.lock:
                 output = self._execute_and_wait_for_prompt(command, timeout, as_root, strip_colors)
                 if check_exit_code:
@@ -163,8 +174,11 @@ class SshConnection(object):
                     except (ValueError, IndexError):
                         logger.warning('Could not get exit code for "{}",\ngot: "{}"'.format(command, exit_code_text))
                 return output
+        except EOF:
+            raise TargetError('Connection lost.')
 
     def background(self, command, stdout=subprocess.PIPE, stderr=subprocess.PIPE):
+        try:
             port_string = '-p {}'.format(self.port) if self.port else ''
             keyfile_string = '-i {}'.format(self.keyfile) if self.keyfile else ''
             command = '{} {} {} {}@{} {}'.format(ssh, keyfile_string, port_string, self.username, self.host, command)
@@ -172,6 +186,8 @@ class SshConnection(object):
             if self.password:
                 command = _give_password(self.password, command)
             return subprocess.Popen(command, stdout=stdout, stderr=stderr, shell=True)
+        except EOF:
+            raise TargetError('Connection lost.')
 
     def close(self):
         logger.debug('Logging out {}@{}'.format(self.username, self.host))