fw/instrument: add hostside decorator

Add a decorator to indicate that a callback runs entirely host-side and
does not rely on a connection to the target. This  means it will be
invoked even if the target was detected to be unresponsive.

doc/source/instrument_method_map.template

@@ -19,3 +19,16 @@ were registered). The table below shows the mapping of the decorator to the
 corresponding priority:
 
 $priority_prefixes
+
+
+Unresponsive Targets
+~~~~~~~~~~~~~~~~~~~~
+
+If a target is believed to be unresponsive, instrument callbacks will be
+disabled to prevent a cascade of errors and potential corruptions of state, as
+it is generally assumed that instrument callbacks will want to do something with
+the target.
+
+If your callback only does something with the host, and does not require an
+active target connection, you can decorate it with ``@hostside`` decorator to
+ensure it gets invoked even if the target becomes unresponsive.

wa/__init__.py

@@ -8,7 +8,7 @@ from wa.framework.exception import (CommandError, ConfigError, HostError, Instru
                                     TargetNotRespondingError, TimeoutError, ToolError,
                                     ValidationError, WAError, WorkloadError, WorkerThreadError)
 from wa.framework.instrument import (Instrument, extremely_slow, very_slow, slow, normal, fast,
-                                     very_fast, extremely_fast)
+                                     very_fast, extremely_fast, hostside)
 from wa.framework.output import RunOutput, discover_wa_outputs
 from wa.framework.output_processor import OutputProcessor
 from wa.framework.plugin import Plugin, Parameter, Alias

wa/framework/instrument.py

@@ -186,6 +186,22 @@ very_fast = priority(signal.CallbackPriority.very_high)
 extremely_fast = priority(signal.CallbackPriority.extremely_high)
 
 
+def hostside(func):
+    """
+    Used as a hint that the callback only performs actions on the
+    host and does not rely on an active connection to the target.
+    This means the callback will be invoked even if the target is
+    thought to be unresponsive.
+
+    """
+    func.is_hostside = True
+    return func
+
+
+def is_hostside(func):
+    return getattr(func, 'is_hostside', False)
+
+
 installed = []
 
 
@@ -240,12 +256,13 @@ class ManagedCallback(object):
     def __init__(self, instrument, callback):
         self.instrument = instrument
         self.callback = callback
+        self.is_hostside = is_hostside(callback)
 
     def __call__(self, context):
         if self.instrument.is_enabled:
             try:
-                if not context.tm.is_responsive:
+                if not context.tm.is_responsive and not self.is_hostside:
-                    logger.debug("Target unreponsive; skipping callback {}".format(self.callback))
+                    logger.debug("Target unresponsive; skipping callback {}".format(self.callback))
                     return
                 self.callback(context)
             except (KeyboardInterrupt, TargetNotRespondingError, TimeoutError):  # pylint: disable=W0703
@@ -313,8 +330,9 @@ def install(instrument, context):
             raise ValueError(message.format(attr_name, arg_num))
 
         priority = get_priority(attr)
-        logger.debug('\tConnecting %s to %s with priority %s(%d)', attr.__name__,
+        hostside = ' [hostside]' if is_hostside(attr) else ''
-                     SIGNAL_MAP[attr_name], priority.name, priority.value)
+        logger.debug('\tConnecting %s to %s with priority %s(%d)%s', attr.__name__,
+                     SIGNAL_MAP[attr_name], priority.name, priority.value, hostside)
 
         mc = ManagedCallback(instrument, attr)
         _callbacks.append(mc)