mirror of
https://github.com/ARM-software/workload-automation.git
synced 2025-01-19 12:24:32 +00:00
376 lines
14 KiB
Python
376 lines
14 KiB
Python
# Copyright 2013-2015 ARM Limited
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
#
|
|
|
|
|
|
"""
|
|
Adding New Instrument
|
|
=====================
|
|
|
|
Any new instrument should be a subclass of Instrument and it must have a name.
|
|
When a new instrument is added to Workload Automation, the methods of the new
|
|
instrument will be found automatically and hooked up to the supported signals.
|
|
Once a signal is broadcasted, the corresponding registered method is invoked.
|
|
|
|
Each method in Instrument must take two arguments, which are self and context.
|
|
Supported signals can be found in [... link to signals ...] To make
|
|
implementations easier and common, the basic steps to add new instrument is
|
|
similar to the steps to add new workload.
|
|
|
|
Hence, the following methods are sufficient to implement to add new instrument:
|
|
|
|
- setup: This method is invoked after the workload is setup. All the
|
|
necessary setups should go inside this method. Setup, includes operations
|
|
like, pushing the files to the target device, install them, clear logs,
|
|
etc.
|
|
- start: It is invoked just before the workload start execution. Here is
|
|
where instrument measures start being registered/taken.
|
|
- stop: It is invoked just after the workload execution stops. The measures
|
|
should stop being taken/registered.
|
|
- update_result: It is invoked after the workload updated its result.
|
|
update_result is where the taken measures are added to the result so it
|
|
can be processed by Workload Automation.
|
|
- teardown is invoked after the workload is teared down. It is a good place
|
|
to clean any logs generated by the instrument.
|
|
|
|
For example, to add an instrument which will trace device errors, we subclass
|
|
Instrument and overwrite the variable name.::
|
|
|
|
#BINARY_FILE = os.path.join(os.path.dirname(__file__), 'trace')
|
|
class TraceErrorsInstrument(Instrument):
|
|
|
|
name = 'trace-errors'
|
|
|
|
def __init__(self, device):
|
|
super(TraceErrorsInstrument, self).__init__(device)
|
|
self.trace_on_device = os.path.join(self.device.working_directory, 'trace')
|
|
|
|
We then declare and implement the aforementioned methods. For the setup method,
|
|
we want to push the file to the target device and then change the file mode to
|
|
755 ::
|
|
|
|
def setup(self, context):
|
|
self.device.push_file(BINARY_FILE, self.device.working_directory)
|
|
self.device.execute('chmod 755 {}'.format(self.trace_on_device))
|
|
|
|
Then we implemented the start method, which will simply run the file to start
|
|
tracing. ::
|
|
|
|
def start(self, context):
|
|
self.device.execute('{} start'.format(self.trace_on_device))
|
|
|
|
Lastly, we need to stop tracing once the workload stops and this happens in the
|
|
stop method::
|
|
|
|
def stop(self, context):
|
|
self.device.execute('{} stop'.format(self.trace_on_device))
|
|
|
|
The generated result can be updated inside update_result, or if it is trace, we
|
|
just pull the file to the host device. context has a result variable which
|
|
has add_metric method. It can be used to add the instrumentation results metrics
|
|
to the final result for the workload. The method can be passed 4 params, which
|
|
are metric key, value, unit and lower_is_better, which is a boolean. ::
|
|
|
|
def update_result(self, context):
|
|
# pull the trace file to the device
|
|
result = os.path.join(self.device.working_directory, 'trace.txt')
|
|
self.device.pull_file(result, context.working_directory)
|
|
|
|
# parse the file if needs to be parsed, or add result to
|
|
# context.result
|
|
|
|
At the end, we might want to delete any files generated by the instrumentation
|
|
and the code to clear these file goes in teardown method. ::
|
|
|
|
def teardown(self, context):
|
|
self.device.delete_file(os.path.join(self.device.working_directory, 'trace.txt'))
|
|
|
|
"""
|
|
|
|
import logging
|
|
import inspect
|
|
from collections import OrderedDict
|
|
|
|
import wlauto.core.signal as signal
|
|
from wlauto.core.extension import Extension
|
|
from wlauto.exceptions import WAError, DeviceNotRespondingError, TimeoutError
|
|
from wlauto.utils.misc import get_traceback, isiterable
|
|
from wlauto.utils.types import identifier
|
|
|
|
|
|
logger = logging.getLogger('instrumentation')
|
|
|
|
|
|
# Maps method names onto signals the should be registered to.
|
|
# Note: the begin/end signals are paired -- if a begin_ signal is sent,
|
|
# then the corresponding end_ signal is guaranteed to also be sent.
|
|
# Note: using OrderedDict to preserve logical ordering for the table generated
|
|
# in the documentation
|
|
SIGNAL_MAP = OrderedDict([
|
|
# Below are "aliases" for some of the more common signals to allow
|
|
# instrumentation to have similar structure to workloads
|
|
('initialize', signal.RUN_INIT),
|
|
('setup', signal.SUCCESSFUL_WORKLOAD_SETUP),
|
|
('start', signal.BEFORE_WORKLOAD_EXECUTION),
|
|
('stop', signal.AFTER_WORKLOAD_EXECUTION),
|
|
('process_workload_result', signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE),
|
|
('update_result', signal.AFTER_WORKLOAD_RESULT_UPDATE),
|
|
('teardown', signal.AFTER_WORKLOAD_TEARDOWN),
|
|
('finalize', signal.RUN_FIN),
|
|
|
|
('on_run_start', signal.RUN_START),
|
|
('on_run_end', signal.RUN_END),
|
|
('on_workload_spec_start', signal.WORKLOAD_SPEC_START),
|
|
('on_workload_spec_end', signal.WORKLOAD_SPEC_END),
|
|
('on_iteration_start', signal.ITERATION_START),
|
|
('on_iteration_end', signal.ITERATION_END),
|
|
|
|
('before_initial_boot', signal.BEFORE_INITIAL_BOOT),
|
|
('on_successful_initial_boot', signal.SUCCESSFUL_INITIAL_BOOT),
|
|
('after_initial_boot', signal.AFTER_INITIAL_BOOT),
|
|
('before_first_iteration_boot', signal.BEFORE_FIRST_ITERATION_BOOT),
|
|
('on_successful_first_iteration_boot', signal.SUCCESSFUL_FIRST_ITERATION_BOOT),
|
|
('after_first_iteration_boot', signal.AFTER_FIRST_ITERATION_BOOT),
|
|
('before_boot', signal.BEFORE_BOOT),
|
|
('on_successful_boot', signal.SUCCESSFUL_BOOT),
|
|
('after_boot', signal.AFTER_BOOT),
|
|
|
|
('on_spec_init', signal.SPEC_INIT),
|
|
('on_run_init', signal.RUN_INIT),
|
|
('on_iteration_init', signal.ITERATION_INIT),
|
|
|
|
('before_workload_setup', signal.BEFORE_WORKLOAD_SETUP),
|
|
('on_successful_workload_setup', signal.SUCCESSFUL_WORKLOAD_SETUP),
|
|
('after_workload_setup', signal.AFTER_WORKLOAD_SETUP),
|
|
('before_workload_execution', signal.BEFORE_WORKLOAD_EXECUTION),
|
|
('on_successful_workload_execution', signal.SUCCESSFUL_WORKLOAD_EXECUTION),
|
|
('after_workload_execution', signal.AFTER_WORKLOAD_EXECUTION),
|
|
('before_workload_result_update', signal.BEFORE_WORKLOAD_RESULT_UPDATE),
|
|
('on_successful_workload_result_update', signal.SUCCESSFUL_WORKLOAD_RESULT_UPDATE),
|
|
('after_workload_result_update', signal.AFTER_WORKLOAD_RESULT_UPDATE),
|
|
('before_workload_teardown', signal.BEFORE_WORKLOAD_TEARDOWN),
|
|
('on_successful_workload_teardown', signal.SUCCESSFUL_WORKLOAD_TEARDOWN),
|
|
('after_workload_teardown', signal.AFTER_WORKLOAD_TEARDOWN),
|
|
|
|
('before_overall_results_processing', signal.BEFORE_OVERALL_RESULTS_PROCESSING),
|
|
('on_successful_overall_results_processing', signal.SUCCESSFUL_OVERALL_RESULTS_PROCESSING),
|
|
('after_overall_results_processing', signal.AFTER_OVERALL_RESULTS_PROCESSING),
|
|
|
|
('on_error', signal.ERROR_LOGGED),
|
|
('on_warning', signal.WARNING_LOGGED),
|
|
])
|
|
|
|
PRIORITY_MAP = OrderedDict([
|
|
('very_fast_', 20),
|
|
('fast_', 10),
|
|
('normal_', 0),
|
|
('slow_', -10),
|
|
('very_slow_', -20),
|
|
])
|
|
|
|
installed = []
|
|
|
|
|
|
def is_installed(instrument):
|
|
if isinstance(instrument, Instrument):
|
|
if instrument in installed:
|
|
return True
|
|
if instrument.name in [i.name for i in installed]:
|
|
return True
|
|
elif isinstance(instrument, type):
|
|
if instrument in [i.__class__ for i in installed]:
|
|
return True
|
|
else: # assume string
|
|
if identifier(instrument) in [identifier(i.name) for i in installed]:
|
|
return True
|
|
return False
|
|
|
|
|
|
failures_detected = False
|
|
|
|
|
|
def reset_failures():
|
|
global failures_detected # pylint: disable=W0603
|
|
failures_detected = False
|
|
|
|
|
|
def check_failures():
|
|
result = failures_detected
|
|
reset_failures()
|
|
return result
|
|
|
|
|
|
class ManagedCallback(object):
|
|
"""
|
|
This wraps instruments' callbacks to ensure that errors do interfer
|
|
with run execution.
|
|
|
|
"""
|
|
|
|
def __init__(self, instrument, callback):
|
|
self.instrument = instrument
|
|
self.callback = callback
|
|
|
|
def __call__(self, context):
|
|
if self.instrument.is_enabled:
|
|
try:
|
|
self.callback(context)
|
|
except (KeyboardInterrupt, DeviceNotRespondingError, TimeoutError): # pylint: disable=W0703
|
|
raise
|
|
except Exception as e: # pylint: disable=W0703
|
|
logger.error('Error in insturment {}'.format(self.instrument.name))
|
|
global failures_detected # pylint: disable=W0603
|
|
failures_detected = True
|
|
if isinstance(e, WAError):
|
|
logger.error(e)
|
|
else:
|
|
tb = get_traceback()
|
|
logger.error(tb)
|
|
logger.error('{}({})'.format(e.__class__.__name__, e))
|
|
if not context.current_iteration:
|
|
# Error occureed outside of an iteration (most likely
|
|
# during intial setup or teardown). Since this would affect
|
|
# the rest of the run, mark the instument as broken so that
|
|
# it doesn't get re-enabled for subsequent iterations.
|
|
self.instrument.is_broken = True
|
|
disable(self.instrument)
|
|
|
|
|
|
# Need this to keep track of callbacks, because the dispatcher only keeps
|
|
# weak references, so if the callbacks aren't referenced elsewhere, they will
|
|
# be deallocated before they've had a chance to be invoked.
|
|
_callbacks = []
|
|
|
|
|
|
def install(instrument):
|
|
"""
|
|
This will look for methods (or any callable members) with specific names
|
|
in the instrument and hook them up to the corresponding signals.
|
|
|
|
:param instrument: Instrument instance to install.
|
|
|
|
"""
|
|
logger.debug('Installing instrument %s.', instrument)
|
|
if is_installed(instrument):
|
|
raise ValueError('Instrument {} is already installed.'.format(instrument.name))
|
|
for attr_name in dir(instrument):
|
|
priority = 0
|
|
stripped_attr_name = attr_name
|
|
for key, value in PRIORITY_MAP.iteritems():
|
|
if attr_name.startswith(key):
|
|
stripped_attr_name = attr_name[len(key):]
|
|
priority = value
|
|
break
|
|
if stripped_attr_name in SIGNAL_MAP:
|
|
attr = getattr(instrument, attr_name)
|
|
if not callable(attr):
|
|
raise ValueError('Attribute {} not callable in {}.'.format(attr_name, instrument))
|
|
arg_num = len(inspect.getargspec(attr).args)
|
|
if not arg_num == 2:
|
|
raise ValueError('{} must take exactly 2 arguments; {} given.'.format(attr_name, arg_num))
|
|
|
|
logger.debug('\tConnecting %s to %s', attr.__name__, SIGNAL_MAP[stripped_attr_name])
|
|
mc = ManagedCallback(instrument, attr)
|
|
_callbacks.append(mc)
|
|
signal.connect(mc, SIGNAL_MAP[stripped_attr_name], priority=priority)
|
|
installed.append(instrument)
|
|
|
|
|
|
def uninstall(instrument):
|
|
instrument = get_instrument(instrument)
|
|
installed.remove(instrument)
|
|
|
|
|
|
def validate():
|
|
for instrument in installed:
|
|
instrument.validate()
|
|
|
|
|
|
def get_instrument(inst):
|
|
if isinstance(inst, Instrument):
|
|
return inst
|
|
for installed_inst in installed:
|
|
if identifier(installed_inst.name) == identifier(inst):
|
|
return installed_inst
|
|
raise ValueError('Instrument {} is not installed'.format(inst))
|
|
|
|
|
|
def disable_all():
|
|
for instrument in installed:
|
|
_disable_instrument(instrument)
|
|
|
|
|
|
def enable_all():
|
|
for instrument in installed:
|
|
_enable_instrument(instrument)
|
|
|
|
|
|
def enable(to_enable):
|
|
if isiterable(to_enable):
|
|
for inst in to_enable:
|
|
_enable_instrument(inst)
|
|
else:
|
|
_enable_instrument(to_enable)
|
|
|
|
|
|
def disable(to_disable):
|
|
if isiterable(to_disable):
|
|
for inst in to_disable:
|
|
_disable_instrument(inst)
|
|
else:
|
|
_disable_instrument(to_disable)
|
|
|
|
|
|
def _enable_instrument(inst):
|
|
inst = get_instrument(inst)
|
|
if not inst.is_broken:
|
|
logger.debug('Enabling instrument {}'.format(inst.name))
|
|
inst.is_enabled = True
|
|
else:
|
|
logger.debug('Not enabling broken instrument {}'.format(inst.name))
|
|
|
|
|
|
def _disable_instrument(inst):
|
|
inst = get_instrument(inst)
|
|
if inst.is_enabled:
|
|
logger.debug('Disabling instrument {}'.format(inst.name))
|
|
inst.is_enabled = False
|
|
|
|
|
|
def get_enabled():
|
|
return [i for i in installed if i.is_enabled]
|
|
|
|
|
|
def get_disabled():
|
|
return [i for i in installed if not i.is_enabled]
|
|
|
|
|
|
class Instrument(Extension):
|
|
"""
|
|
Base class for instrumentation implementations.
|
|
"""
|
|
|
|
def __init__(self, device, **kwargs):
|
|
super(Instrument, self).__init__(**kwargs)
|
|
self.device = device
|
|
self.is_enabled = True
|
|
self.is_broken = False
|
|
|
|
def __str__(self):
|
|
return self.name
|
|
|
|
def __repr__(self):
|
|
return 'Instrument({})'.format(self.name)
|
|
|