workload-automation/wlauto/core/instrumentation.py

#    Copyright 2013-2015 ARM Limited
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#


"""
Adding New Instrument
=====================

Any new instrument should be a subclass of Instrument and it must have a name.
When a new instrument is added to Workload Automation, the methods of the new
instrument will be found automatically and hooked up to the supported signals.
Once a signal is broadcasted, the corresponding registered method is invoked.

Each method in Instrument must take two arguments, which are self and context.
Supported signals can be found in [... link to signals ...] To make
implementations easier and common, the basic steps to add new instrument is
similar to the steps to add new workload.

Hence, the following methods are sufficient to implement to add new instrument:

    - setup: This method is invoked after the workload is setup. All the
       necessary setups should go inside this method. Setup, includes operations
       like, pushing the files to the target device, install them, clear logs,
       etc.
    - start: It is invoked just before the workload start execution. Here is
       where instrument measures start being registered/taken.
    - stop: It is invoked just after the workload execution stops. The measures
       should stop being taken/registered.
    - update_result: It is invoked after the workload updated its result.
       update_result is where the taken measures are added to the result so it
       can be processed by Workload Automation.
    - teardown is invoked after the workload is teared down. It is a good place
       to clean any logs generated by the instrument.

For example, to add an instrument which will trace device errors, we subclass
Instrument and overwrite the variable name.::

        #BINARY_FILE = os.path.join(os.path.dirname(__file__), 'trace')
        class TraceErrorsInstrument(Instrument):

            name = 'trace-errors'

            def __init__(self, device):
                super(TraceErrorsInstrument, self).__init__(device)
                self.trace_on_device = os.path.join(self.device.working_directory, 'trace')

We then declare and implement the aforementioned methods. For the setup method,
we want to push the file to the target device and then change the file mode to
755 ::

    def setup(self, context):
        self.device.push_file(BINARY_FILE, self.device.working_directory)
        self.device.execute('chmod 755 {}'.format(self.trace_on_device))

Then we implemented the start method, which will simply run the file to start
tracing. ::

    def start(self, context):
        self.device.execute('{} start'.format(self.trace_on_device))

Lastly, we need to stop tracing once the workload stops and this happens in the
stop method::

    def stop(self, context):
        self.device.execute('{} stop'.format(self.trace_on_device))

The generated result can be updated inside update_result, or if it is trace, we
just pull the file to the host device. context has a result variable which
has add_metric method. It can be used to add the instrumentation results metrics
to the final result for the workload. The method can be passed 4 params, which
are metric key, value, unit and lower_is_better, which is a boolean. ::

    def update_result(self, context):
        # pull the trace file to the device
        result = os.path.join(self.device.working_directory, 'trace.txt')
        self.device.pull_file(result, context.working_directory)

        # parse the file if needs to be parsed, or add result to
        # context.result

At the end, we might want to delete any files generated by the instrumentation
and the code to clear these file goes in teardown method. ::

    def teardown(self, context):
        self.device.delete_file(os.path.join(self.device.working_directory, 'trace.txt'))

"""

import logging
import inspect
from collections import OrderedDict

import wlauto.core.signal as signal
from wlauto.core.extension import Extension
from wlauto.exceptions import WAError, DeviceNotRespondingError, TimeoutError
from wlauto.utils.misc import get_traceback, isiterable
from wlauto.utils.types import identifier


logger = logging.getLogger('instrumentation')


# Maps method names onto signals the should be registered to.
# Note: the begin/end signals are paired -- if a begin_ signal is sent,
#       then the corresponding end_ signal is guaranteed to also be sent.
# Note: using OrderedDict to preserve logical ordering for the table generated
#       in the documentation
SIGNAL_MAP = OrderedDict([
    # Below are "aliases" for some of the more common signals to allow
    # instrumentation to have similar structure to workloads
    ('initialize', signal.RUN_INIT),
    ('setup', signal.SUCCESSFUL_WORKLOAD_SETUP),
    ('start', signal.BEFORE_WORKLOAD_EXECUTION),
    ('stop', signal.AFTER_WORKLOAD_EXECUTION),
    ('process_workload_result', signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE),
    ('update_result', signal.AFTER_WORKLOAD_RESULT_UPDATE),
    ('teardown', signal.AFTER_WORKLOAD_TEARDOWN),
    ('finalize', signal.RUN_FIN),

    ('on_run_start', signal.RUN_START),
    ('on_run_end', signal.RUN_END),
    ('on_workload_spec_start', signal.WORKLOAD_SPEC_START),
    ('on_workload_spec_end', signal.WORKLOAD_SPEC_END),
    ('on_iteration_start', signal.ITERATION_START),
    ('on_iteration_end', signal.ITERATION_END),

    ('before_initial_boot', signal.BEFORE_INITIAL_BOOT),
    ('on_successful_initial_boot', signal.SUCCESSFUL_INITIAL_BOOT),
    ('after_initial_boot', signal.AFTER_INITIAL_BOOT),
    ('before_first_iteration_boot', signal.BEFORE_FIRST_ITERATION_BOOT),
    ('on_successful_first_iteration_boot', signal.SUCCESSFUL_FIRST_ITERATION_BOOT),
    ('after_first_iteration_boot', signal.AFTER_FIRST_ITERATION_BOOT),
    ('before_boot', signal.BEFORE_BOOT),
    ('on_successful_boot', signal.SUCCESSFUL_BOOT),
    ('after_boot', signal.AFTER_BOOT),

    ('on_spec_init', signal.SPEC_INIT),
    ('on_run_init', signal.RUN_INIT),
    ('on_iteration_init', signal.ITERATION_INIT),

    ('before_workload_setup', signal.BEFORE_WORKLOAD_SETUP),
    ('on_successful_workload_setup', signal.SUCCESSFUL_WORKLOAD_SETUP),
    ('after_workload_setup', signal.AFTER_WORKLOAD_SETUP),
    ('before_workload_execution', signal.BEFORE_WORKLOAD_EXECUTION),
    ('on_successful_workload_execution', signal.SUCCESSFUL_WORKLOAD_EXECUTION),
    ('after_workload_execution', signal.AFTER_WORKLOAD_EXECUTION),
    ('before_workload_result_update', signal.BEFORE_WORKLOAD_RESULT_UPDATE),
    ('on_successful_workload_result_update', signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE),
    ('after_workload_result_update', signal.AFTER_WORKLOAD_RESULT_UPDATE),
    ('before_workload_teardown', signal.BEFORE_WORKLOAD_TEARDOWN),
    ('on_successful_workload_teardown', signal.SUCCESSFUL_WORKLOAD_TEARDOWN),
    ('after_workload_teardown', signal.AFTER_WORKLOAD_TEARDOWN),

    ('before_overall_results_processing', signal.BEFORE_OVERALL_RESULTS_PROCESSING),
    ('on_successful_overall_results_processing', signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING),
    ('after_overall_results_processing', signal.AFTER_OVERALL_RESULTS_PROCESSING),

    ('on_error', signal.ERROR_LOGGED),
    ('on_warning', signal.WARNING_LOGGED),
])

PRIORITY_MAP = OrderedDict([
    ('very_fast_', 20),
    ('fast_', 10),
    ('normal_', 0),
    ('slow_', -10),
    ('very_slow_', -20),
])

installed = []


def is_installed(instrument):
    if isinstance(instrument, Instrument):
        if instrument in installed:
            return True
        if instrument.name in [i.name for i in installed]:
            return True
    elif isinstance(instrument, type):
        if instrument in [i.__class__ for i in installed]:
            return True
    else:  # assume string
        if identifier(instrument) in [identifier(i.name) for i in installed]:
            return True
    return False


failures_detected = False


def reset_failures():
    global failures_detected  # pylint: disable=W0603
    failures_detected = False


def check_failures():
    result = failures_detected
    reset_failures()
    return result


class ManagedCallback(object):
    """
    This wraps instruments' callbacks to ensure that errors do interfer
    with run execution.

    """

    def __init__(self, instrument, callback):
        self.instrument = instrument
        self.callback = callback

    def __call__(self, context):
        if self.instrument.is_enabled:
            try:
                self.callback(context)
            except (KeyboardInterrupt, DeviceNotRespondingError, TimeoutError):  # pylint: disable=W0703
                raise
            except Exception as e:  # pylint: disable=W0703
                logger.error('Error in insturment {}'.format(self.instrument.name))
                global failures_detected  # pylint: disable=W0603
                failures_detected = True
                if isinstance(e, WAError):
                    logger.error(e)
                else:
                    tb = get_traceback()
                    logger.error(tb)
                    logger.error('{}({})'.format(e.__class__.__name__, e))
                if not context.current_iteration:
                    # Error occureed outside of an iteration (most likely
                    # during intial setup or teardown). Since this would affect
                    # the rest of the run, mark the instument as broken so that
                    # it doesn't get re-enabled for subsequent iterations.
                    self.instrument.is_broken = True
                disable(self.instrument)


# Need this to keep track of callbacks, because the dispatcher only keeps
# weak references, so if the callbacks aren't referenced elsewhere, they will
# be deallocated before they've had a chance to be invoked.
_callbacks = []


def install(instrument):
    """
    This will look for methods (or any callable members) with specific names
    in the instrument and hook them up to the corresponding signals.

    :param instrument: Instrument instance to install.

    """
    logger.debug('Installing instrument %s.', instrument)
    if is_installed(instrument):
        raise ValueError('Instrument {} is already installed.'.format(instrument.name))
    for attr_name in dir(instrument):
        priority = 0
        stripped_attr_name = attr_name
        for key, value in PRIORITY_MAP.iteritems():
            if attr_name.startswith(key):
                stripped_attr_name = attr_name[len(key):]
                priority = value
                break
        if stripped_attr_name in SIGNAL_MAP:
            attr = getattr(instrument, attr_name)
            if not callable(attr):
                raise ValueError('Attribute {} not callable in {}.'.format(attr_name, instrument))
            arg_num = len(inspect.getargspec(attr).args)
            if not arg_num == 2:
                raise ValueError('{} must take exactly 2 arguments; {} given.'.format(attr_name, arg_num))

            logger.debug('\tConnecting %s to %s', attr.__name__, SIGNAL_MAP[stripped_attr_name])
            mc = ManagedCallback(instrument, attr)
            _callbacks.append(mc)
            signal.connect(mc, SIGNAL_MAP[stripped_attr_name], priority=priority)
    installed.append(instrument)


def uninstall(instrument):
    instrument = get_instrument(instrument)
    installed.remove(instrument)


def validate():
    for instrument in installed:
        instrument.validate()


def get_instrument(inst):
    if isinstance(inst, Instrument):
        return inst
    for installed_inst in installed:
        if identifier(installed_inst.name) == identifier(inst):
            return installed_inst
    raise ValueError('Instrument {} is not installed'.format(inst))


def disable_all():
    for instrument in installed:
        _disable_instrument(instrument)


def enable_all():
    for instrument in installed:
        _enable_instrument(instrument)


def enable(to_enable):
    if isiterable(to_enable):
        for inst in to_enable:
            _enable_instrument(inst)
    else:
        _enable_instrument(to_enable)


def disable(to_disable):
    if isiterable(to_disable):
        for inst in to_disable:
            _disable_instrument(inst)
    else:
        _disable_instrument(to_disable)


def _enable_instrument(inst):
    inst = get_instrument(inst)
    if not inst.is_broken:
        logger.debug('Enabling instrument {}'.format(inst.name))
        inst.is_enabled = True
    else:
        logger.debug('Not enabling broken instrument {}'.format(inst.name))


def _disable_instrument(inst):
    inst = get_instrument(inst)
    if inst.is_enabled:
        logger.debug('Disabling instrument {}'.format(inst.name))
        inst.is_enabled = False


def get_enabled():
    return [i for i in installed if i.is_enabled]


def get_disabled():
    return [i for i in installed if not i.is_enabled]


class Instrument(Extension):
    """
    Base class for instrumentation implementations.
    """

    def __init__(self, device, **kwargs):
        super(Instrument, self).__init__(**kwargs)
        self.device = device
        self.is_enabled = True
        self.is_broken = False

    def __str__(self):
        return self.name

    def __repr__(self):
        return 'Instrument({})'.format(self.name)