Initial commit of open source Workload Automation.

wlauto/core/instrumentation.py

@@ -0,0 +1,374 @@
+#    Copyright 2013-2015 ARM Limited
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+
+"""
+Adding New Instrument
+=====================
+
+Any new instrument should be a subclass of Instrument and it must have a name.
+When a new instrument is added to Workload Automation, the methods of the new
+instrument will be found automatically and hooked up to the supported signals.
+Once a signal is broadcasted, the corresponding registered method is invoked.
+
+Each method in Instrument must take two arguments, which are self and context.
+Supported signals can be found in [... link to signals ...] To make
+implementations easier and common, the basic steps to add new instrument is
+similar to the steps to add new workload.
+
+Hence, the following methods are sufficient to implement to add new instrument:
+
+    - setup: This method is invoked after the workload is setup. All the
+       necessary setups should go inside this method. Setup, includes operations
+       like, pushing the files to the target device, install them, clear logs,
+       etc.
+    - start: It is invoked just before the workload start execution. Here is
+       where instrument measures start being registered/taken.
+    - stop: It is invoked just after the workload execution stops. The measures
+       should stop being taken/registered.
+    - update_result: It is invoked after the workload updated its result.
+       update_result is where the taken measures are added to the result so it
+       can be processed by Workload Automation.
+    - teardown is invoked after the workload is teared down. It is a good place
+       to clean any logs generated by the instrument.
+
+For example, to add an instrument which will trace device errors, we subclass
+Instrument and overwrite the variable name.::
+
+        #BINARY_FILE = os.path.join(os.path.dirname(__file__), 'trace')
+        class TraceErrorsInstrument(Instrument):
+
+            name = 'trace-errors'
+
+            def __init__(self, device):
+                super(TraceErrorsInstrument, self).__init__(device)
+                self.trace_on_device = os.path.join(self.device.working_directory, 'trace')
+
+We then declare and implement the aforementioned methods. For the setup method,
+we want to push the file to the target device and then change the file mode to
+755 ::
+
+    def setup(self, context):
+        self.device.push_file(BINARY_FILE, self.device.working_directory)
+        self.device.execute('chmod 755 {}'.format(self.trace_on_device))
+
+Then we implemented the start method, which will simply run the file to start
+tracing. ::
+
+    def start(self, context):
+        self.device.execute('{} start'.format(self.trace_on_device))
+
+Lastly, we need to stop tracing once the workload stops and this happens in the
+stop method::
+
+    def stop(self, context):
+        self.device.execute('{} stop'.format(self.trace_on_device))
+
+The generated result can be updated inside update_result, or if it is trace, we
+just pull the file to the host device. context has a result variable which
+has add_metric method. It can be used to add the instrumentation results metrics
+to the final result for the workload. The method can be passed 4 params, which
+are metric key, value, unit and lower_is_better, which is a boolean. ::
+
+    def update_result(self, context):
+        # pull the trace file to the device
+        result = os.path.join(self.device.working_directory, 'trace.txt')
+        self.device.pull_file(result, context.working_directory)
+
+        # parse the file if needs to be parsed, or add result to
+        # context.result
+
+At the end, we might want to delete any files generated by the instrumentation
+and the code to clear these file goes in teardown method. ::
+
+    def teardown(self, context):
+        self.device.delete_file(os.path.join(self.device.working_directory, 'trace.txt'))
+
+"""
+
+import logging
+import inspect
+from collections import OrderedDict
+
+import wlauto.core.signal as signal
+from wlauto.core.extension import Extension
+from wlauto.exceptions import WAError, DeviceNotRespondingError, TimeoutError
+from wlauto.utils.misc import get_traceback, isiterable
+
+
+logger = logging.getLogger('instrumentation')
+
+
+# Maps method names onto signals the should be registered to.
+# Note: the begin/end signals are paired -- if a begin_ signal is sent,
+#       then the corresponding end_ signal is guaranteed to also be sent.
+# Note: using OrderedDict to preserve logical ordering for the table generated
+#       in the documentation
+SIGNAL_MAP = OrderedDict([
+    # Below are "aliases" for some of the more common signals to allow
+    # instrumentation to have similar structure to workloads
+    ('initialize', signal.RUN_INIT),
+    ('setup', signal.SUCCESSFUL_WORKLOAD_SETUP),
+    ('start', signal.BEFORE_WORKLOAD_EXECUTION),
+    ('stop', signal.AFTER_WORKLOAD_EXECUTION),
+    ('process_workload_result', signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE),
+    ('update_result', signal.AFTER_WORKLOAD_RESULT_UPDATE),
+    ('teardown', signal.AFTER_WORKLOAD_TEARDOWN),
+    ('finalize', signal.RUN_FIN),
+
+    ('on_run_start', signal.RUN_START),
+    ('on_run_end', signal.RUN_END),
+    ('on_workload_spec_start', signal.WORKLOAD_SPEC_START),
+    ('on_workload_spec_end', signal.WORKLOAD_SPEC_END),
+    ('on_iteration_start', signal.ITERATION_START),
+    ('on_iteration_end', signal.ITERATION_END),
+
+    ('before_initial_boot', signal.BEFORE_INITIAL_BOOT),
+    ('on_successful_initial_boot', signal.SUCCESSFUL_INITIAL_BOOT),
+    ('after_initial_boot', signal.AFTER_INITIAL_BOOT),
+    ('before_first_iteration_boot', signal.BEFORE_FIRST_ITERATION_BOOT),
+    ('on_successful_first_iteration_boot', signal.SUCCESSFUL_FIRST_ITERATION_BOOT),
+    ('after_first_iteration_boot', signal.AFTER_FIRST_ITERATION_BOOT),
+    ('before_boot', signal.BEFORE_BOOT),
+    ('on_successful_boot', signal.SUCCESSFUL_BOOT),
+    ('after_boot', signal.AFTER_BOOT),
+
+    ('on_spec_init', signal.SPEC_INIT),
+    ('on_run_init', signal.RUN_INIT),
+    ('on_iteration_init', signal.ITERATION_INIT),
+
+    ('before_workload_setup', signal.BEFORE_WORKLOAD_SETUP),
+    ('on_successful_workload_setup', signal.SUCCESSFUL_WORKLOAD_SETUP),
+    ('after_workload_setup', signal.AFTER_WORKLOAD_SETUP),
+    ('before_workload_execution', signal.BEFORE_WORKLOAD_EXECUTION),
+    ('on_successful_workload_execution', signal.SUCCESSFUL_WORKLOAD_EXECUTION),
+    ('after_workload_execution', signal.AFTER_WORKLOAD_EXECUTION),
+    ('before_workload_result_update', signal.BEFORE_WORKLOAD_RESULT_UPDATE),
+    ('on_successful_workload_result_update', signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE),
+    ('after_workload_result_update', signal.AFTER_WORKLOAD_RESULT_UPDATE),
+    ('before_workload_teardown', signal.BEFORE_WORKLOAD_TEARDOWN),
+    ('on_successful_workload_teardown', signal.SUCCESSFUL_WORKLOAD_TEARDOWN),
+    ('after_workload_teardown', signal.AFTER_WORKLOAD_TEARDOWN),
+
+    ('before_overall_results_processing', signal.BEFORE_OVERALL_RESULTS_PROCESSING),
+    ('on_successful_overall_results_processing', signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING),
+    ('after_overall_results_processing', signal.AFTER_OVERALL_RESULTS_PROCESSING),
+
+    ('on_error', signal.ERROR_LOGGED),
+    ('on_warning', signal.WARNING_LOGGED),
+])
+
+PRIORITY_MAP = OrderedDict([
+    ('very_fast_', 20),
+    ('fast_', 10),
+    ('normal_', 0),
+    ('slow_', -10),
+    ('very_slow_', -20),
+])
+
+installed = []
+
+
+def is_installed(instrument):
+    if isinstance(instrument, Instrument):
+        if instrument in installed:
+            return True
+        if instrument.name in [i.name for i in installed]:
+            return True
+    elif isinstance(instrument, type):
+        if instrument in [i.__class__ for i in installed]:
+            return True
+    else:  # assume string
+        if instrument in [i.name for i in installed]:
+            return True
+    return False
+
+
+failures_detected = False
+
+
+def reset_failures():
+    global failures_detected  # pylint: disable=W0603
+    failures_detected = False
+
+
+def check_failures():
+    result = failures_detected
+    reset_failures()
+    return result
+
+
+class ManagedCallback(object):
+    """
+    This wraps instruments' callbacks to ensure that errors do interfer
+    with run execution.
+
+    """
+
+    def __init__(self, instrument, callback):
+        self.instrument = instrument
+        self.callback = callback
+
+    def __call__(self, context):
+        if self.instrument.is_enabled:
+            try:
+                self.callback(context)
+            except (KeyboardInterrupt, DeviceNotRespondingError, TimeoutError):  # pylint: disable=W0703
+                raise
+            except Exception as e:  # pylint: disable=W0703
+                logger.error('Error in insturment {}'.format(self.instrument.name))
+                global failures_detected  # pylint: disable=W0603
+                failures_detected = True
+                if isinstance(e, WAError):
+                    logger.error(e)
+                else:
+                    tb = get_traceback()
+                    logger.error(tb)
+                    logger.error('{}({})'.format(e.__class__.__name__, e))
+                if not context.current_iteration:
+                    # Error occureed outside of an iteration (most likely
+                    # during intial setup or teardown). Since this would affect
+                    # the rest of the run, mark the instument as broken so that
+                    # it doesn't get re-enabled for subsequent iterations.
+                    self.instrument.is_broken = True
+                disable(self.instrument)
+
+
+# Need this to keep track of callbacks, because the dispatcher only keeps
+# weak references, so if the callbacks aren't referenced elsewhere, they will
+# be deallocated before they've had a chance to be invoked.
+_callbacks = []
+
+
+def install(instrument):
+    """
+    This will look for methods (or any callable members) with specific names
+    in the instrument and hook them up to the corresponding signals.
+
+    :param instrument: Instrument instance to install.
+
+    """
+    logger.debug('Installing instrument %s.', instrument)
+    if is_installed(instrument):
+        raise ValueError('Instrument {} is already installed.'.format(instrument.name))
+    for attr_name in dir(instrument):
+        priority = 0
+        stripped_attr_name = attr_name
+        for key, value in PRIORITY_MAP.iteritems():
+            if attr_name.startswith(key):
+                stripped_attr_name = attr_name[len(key):]
+                priority = value
+                break
+        if stripped_attr_name in SIGNAL_MAP:
+            attr = getattr(instrument, attr_name)
+            if not callable(attr):
+                raise ValueError('Attribute {} not callable in {}.'.format(attr_name, instrument))
+            arg_num = len(inspect.getargspec(attr).args)
+            if not arg_num == 2:
+                raise ValueError('{} must take exactly 2 arguments; {} given.'.format(attr_name, arg_num))
+
+            logger.debug('\tConnecting %s to %s', attr.__name__, SIGNAL_MAP[stripped_attr_name])
+            mc = ManagedCallback(instrument, attr)
+            _callbacks.append(mc)
+            signal.connect(mc, SIGNAL_MAP[stripped_attr_name], priority=priority)
+    installed.append(instrument)
+
+
+def uninstall(instrument):
+    instrument = get_instrument(instrument)
+    installed.remove(instrument)
+
+
+def validate():
+    for instrument in installed:
+        instrument.validate()
+
+
+def get_instrument(inst):
+    if isinstance(inst, Instrument):
+        return inst
+    for installed_inst in installed:
+        if installed_inst.name == inst:
+            return installed_inst
+    raise ValueError('Instrument {} is not installed'.format(inst))
+
+
+def disable_all():
+    for instrument in installed:
+        _disable_instrument(instrument)
+
+
+def enable_all():
+    for instrument in installed:
+        _enable_instrument(instrument)
+
+
+def enable(to_enable):
+    if isiterable(to_enable):
+        for inst in to_enable:
+            _enable_instrument(inst)
+    else:
+        _enable_instrument(to_enable)
+
+
+def disable(to_disable):
+    if isiterable(to_disable):
+        for inst in to_disable:
+            _disable_instrument(inst)
+    else:
+        _disable_instrument(to_disable)
+
+
+def _enable_instrument(inst):
+    inst = get_instrument(inst)
+    if not inst.is_broken:
+        logger.debug('Enabling instrument {}'.format(inst.name))
+        inst.is_enabled = True
+    else:
+        logger.debug('Not enabling broken instrument {}'.format(inst.name))
+
+
+def _disable_instrument(inst):
+    inst = get_instrument(inst)
+    if inst.is_enabled:
+        logger.debug('Disabling instrument {}'.format(inst.name))
+        inst.is_enabled = False
+
+
+def get_enabled():
+    return [i for i in installed if i.is_enabled]
+
+
+def get_disabled():
+    return [i for i in installed if not i.is_enabled]
+
+
+class Instrument(Extension):
+    """
+    Base class for instrumentation implementations.
+    """
+
+    def __init__(self, device, **kwargs):
+        super(Instrument, self).__init__(**kwargs)
+        self.device = device
+        self.is_enabled = True
+        self.is_broken = False
+
+    def __str__(self):
+        return self.name
+
+    def __repr__(self):
+        return 'Instrument({})'.format(self.name)
+