Fixing reboot on Linux devices [part 2]

- connect() to device before issuing the initial reboot, as soft reset
  requires a device connection.
- boot() has been implemented to wait properly for the device to reboot
  after reset.
- port now defaults to 22 rather than being left unset, as need
  something to connect to when polling for device after reboot.
- Only use -P option for scp when port is *not* 22; as that option
  appears to cause intermittent issues with default scp on Ubuntu 12.04

wlauto/common/linux/device.py

@@ -16,6 +16,8 @@
 # pylint: disable=E1101
 import os
 import re
+import time
+import socket
 from collections import namedtuple
 from subprocess import CalledProcessError
 
@@ -743,12 +745,14 @@ class LinuxDevice(BaseLinuxDevice):
         Parameter('username', mandatory=True, description='User name for the account on the device.'),
         Parameter('password', description='Password for the account on the device (for password-based auth).'),
         Parameter('keyfile', description='Keyfile to be used for key-based authentication.'),
-        Parameter('port', kind=int, description='SSH port number on the device.'),
+        Parameter('port', kind=int, default=22, description='SSH port number on the device.'),
         Parameter('password_prompt', default='[sudo] password',
                   description='Prompt presented by sudo when requesting the password.'),
 
         Parameter('use_telnet', kind=boolean, default=False,
                   description='Optionally, telnet may be used instead of ssh, though this is discouraged.'),
+        Parameter('boot_timeout', kind=int, default=120,
+                  description='How long to try to connect to the device after a reboot.'),
 
         Parameter('working_directory', default=None,
                   description='''
@@ -823,6 +827,19 @@ class LinuxDevice(BaseLinuxDevice):
 
     def boot(self, **kwargs):
         self.reset()
+        self.logger.debug('Waiting for device...')
+        start_time = time.time()
+        while (time.time() - start_time) < self.boot_timeout:
+            try:
+                s = socket.create_connection((self.host, self.port), timeout=5)
+                s.close()
+                break
+            except socket.timeout:
+                pass
+            except socket.error:
+                time.sleep(5)
+        else:
+            raise DeviceError('Could not connect to {} after reboot'.format(self.host))
 
     def connect(self):  # NOQA pylint: disable=R0912
         self.shell = SshShell(password_prompt=self.password_prompt, timeout=self.default_timeout)

wlauto/core/execution.py

@@ -511,13 +511,7 @@ class Runner(object):
 
     def _initialize_run(self):
         self.context.run_info.start_time = datetime.utcnow()
-        if self.context.reboot_policy.perform_initial_boot:
+        self._connect_to_device()
-            self.logger.info('\tBooting device')
-            with self._signal_wrap('INITIAL_BOOT'):
-                self._reboot_device()
-        else:
-            self.logger.info('Connecting to device')
-            self.device.connect()
         self.logger.info('Initializing device')
         self.device.initialize(self.context)
 
@@ -529,6 +523,24 @@ class Runner(object):
         if instrumentation.check_failures():
             raise InstrumentError('Detected failure(s) during instrumentation initialization.')
 
+    def _connect_to_device(self):
+        if self.context.reboot_policy.perform_initial_boot:
+            try:
+                self.device.connect()
+            except DeviceError:  # device may be offline
+                if self.device.can('reset_power'):
+                    self.device.hard_reset()
+                else:
+                    raise DeviceError('Cannot connect to device for initial reboot; '
+                                      'and device does not support hard reset.')
+            else:  # successfully connected
+                self.logger.info('\tBooting device')
+                with self._signal_wrap('INITIAL_BOOT'):
+                    self._reboot_device()
+        else:
+            self.logger.info('Connecting to device')
+            self.device.connect()
+
     def _init_job(self):
         self.current_job.result.status = IterationResult.RUNNING
         self.context.next_job(self.current_job)

wlauto/utils/ssh.py

@@ -168,7 +168,11 @@ class SshShell(object):
         return output
 
     def _scp(self, source, dest, timeout=30):
-        port_string = '-P {}'.format(self.port) if self.port else ''
+        # NOTE: the version of scp in Ubuntu 12.04 occasionally (and bizarrely)
+        # fails to connect to a device if port is explicitly specified using -P
+        # option, even if it is the default port, 22. To minimize this problem,
+        # only specify -P for scp if the port is *not* the default.
+        port_string = '-P {}'.format(self.port) if (self.port and self.port != 22) else ''
         keyfile_string = '-i {}'.format(self.keyfile) if self.keyfile else ''
         command = '{} -r {} {} {} {}'.format(scp, keyfile_string, port_string, source, dest)
         pass_string = ''