From 4b5d85ce4070f8546667da2fc0971d5fd334cd19 Mon Sep 17 00:00:00 2001 From: Marc Bonnici Date: Thu, 7 Jun 2018 17:08:26 +0100 Subject: [PATCH] doc: Add WA execution model. Add brief overview of WA's execution model. --- doc/source/developer_reference.rst | 6 + .../developer_reference/WA_Execution.svg | 2 + .../WA_Signal_Dispatch.svg | 2 + .../developer_reference/execution_model.rst | 152 ++++++++++++++++++ 4 files changed, 162 insertions(+) create mode 100644 doc/source/developer_reference/WA_Execution.svg create mode 100644 doc/source/developer_reference/WA_Signal_Dispatch.svg create mode 100644 doc/source/developer_reference/execution_model.rst diff --git a/doc/source/developer_reference.rst b/doc/source/developer_reference.rst index e0e64c13..aad91653 100644 --- a/doc/source/developer_reference.rst +++ b/doc/source/developer_reference.rst @@ -10,8 +10,14 @@ Developer Reference -------------------------------------------------------------------------------- +.. include:: developer_reference/execution_model.rst + +----------------- .. include:: developer_reference/output_processing_api.rst + +----------------- + .. include:: developer_reference/writing_plugins.rst ----------------- diff --git a/doc/source/developer_reference/WA_Execution.svg b/doc/source/developer_reference/WA_Execution.svg new file mode 100644 index 00000000..a886e6e4 --- /dev/null +++ b/doc/source/developer_reference/WA_Execution.svg @@ -0,0 +1,2 @@ + +
initialize run
initialize run
initialize target
initialize target
initialize job queue
initialize job queue
initialize workloads
initialize workloads
configure target
configure target
setup workload
setup workload
run workload
run workload
extract data from target
extract data from target
update WA output
update WA output
teardown workload
teardown workload
end job execution
end job execution
finalize workloads
finalize workloads
finalize run
finalize run
extract instruments data
extract instruments data
initialize instruments
initialize instruments
setup instruments
setup instruments
start instruments
start instruments
stop instruments
stop instruments
teardown instruments
teardown instruments
finalize instruments
finalize instruments
finalize processor
finalize processor
process run output
process run output
process job output
process job output
initialize processors
initialize processors
job
job
signal dispatch
signal dispatch
core framwork
core framwork
workload
workload
instruments
instruments
output processing
output processing
initialize instruments
initialize instruments
initialize output processors
initialize output processors \ No newline at end of file diff --git a/doc/source/developer_reference/WA_Signal_Dispatch.svg b/doc/source/developer_reference/WA_Signal_Dispatch.svg new file mode 100644 index 00000000..0cd2e9eb --- /dev/null +++ b/doc/source/developer_reference/WA_Signal_Dispatch.svg @@ -0,0 +1,2 @@ + +
initialize run
initialize run
initialize target
initialize target
initialize job queue
initialize job queue
initialize workloads
initialize workloads
configure target
configure target
setup workload
setup workload
run workload
run workload
extract data from target
extract data from target
update WA output
update WA output
teardown workload
teardown workload
end job execution
end job execution
finalize workloads
finalize workloads
finalize run
finalize run
TARGET_DISCONNECT
TARGET_DISCONNECT
OVERALL_RESULTS_PROCESSING
OVERALL_RESULTS_PROCESSING
REBOOT
REBOOT
AFTER_RUN_OUTPUT_PROCESSED
AFTER_RUN_OUTPUT_PROCESSED
AFTER_JOB
AFTER_JOB
RUN_STARTED
RUN_STARTED
RUN_INITIALIZED
RUN_INITIALIZED
RUN_FINALIZED
RUN_FINALIZED
RUN_COMPLETED
RUN_COMPLETED
JOB_STARTED
JOB_STARTED
JOB_COMPLETED
JOB_COMPLETED
WORKLOAD_INITIALIZED
WORKLOAD_INITIALIZED
WORKLOAD_SETUP
WORKLOAD_SETUP
WORKLOAD_EXECUTION
WORKLOAD_EXECUTION
WORKLOAD_RESULT_EXTRACTION
WORKLOAD_RESULT_EXTRACTION
WORKLOAD_OUTPUT_UPDATE
WORKLOAD_OUTPUT_UPDATE
WORKLOAD_TEARDOWN
WORKLOAD_TEARDOWN
WORKLOAD_FINALIZED
WORKLOAD_FINALIZED
TARGET_CONNECT
TARGET_CONNECT
JOB_TARGET_CONFIG
JOB_TARGET_CONFIG
BEFORE_JOB
BEFORE_JOB
BEFORE_RUN_OUTPUT_PROCESSED
BEFORE_RUN_OUTPUT_PROCESSED
REBOOT
REBOOT
initialize instruments
initialize instruments
initialize output processors
initialize output processors
initialize run
initialize run
core framwork stage
core framwork stage
workload stage
workload stage
individual signal
individual signal
conditional signal
conditional signal
paired signal
paired signal
conditional paired signal
conditional paired signal
RUN_ABORTED
RUN_ABORTED
RUN_FAILED
RUN_FAILED
JOB_ABORTED
JOB_ABORTED
JOB_RESTARTED
JOB_RESTARTED
WARNING_LOGGED
WARNING_LOGGED
ERROR_LOGGED
ERROR_LOGGED
Conditional Signals
Conditional Signals \ No newline at end of file diff --git a/doc/source/developer_reference/execution_model.rst b/doc/source/developer_reference/execution_model.rst new file mode 100644 index 00000000..f5e9f3a4 --- /dev/null +++ b/doc/source/developer_reference/execution_model.rst @@ -0,0 +1,152 @@ +Framework Overview +================== + +Execution Model +--------------- + +At the high level, the execution model looks as follows: + +.. image:: developer_reference/WA_Execution.svg + :scale: 100 % + +After some initial setup, the framework initializes the device, loads and +initialized instruments and output processors and begins executing jobs defined +by the workload specs in the agenda. Each job executes in basic stages: + +initialize + Perform any once-per-run initialization of a workload instance, i.e. + binary resource resolution. +setup + Initial setup for the workload is performed. E.g. required assets are + deployed to the devices, required services or applications are launched, + etc. Run time configuration of the device for the workload is also + performed at this time. +run + This is when the workload actually runs. This is defined as the part of + the workload that is to be measured. Exactly what happens at this stage + depends entirely on the workload. +extract results + Extract any results that have been generated during the execution of the + workload from the device and back to that target. Any files pulled from + the devices should be added as artifacts to the run context. +update output + Perform any required parsing and processing of any collected results and + add any generated metrics to the run context. +teardown + Final clean up is performed, e.g. applications may closed, files + generated during execution deleted, etc. + +Signals are dispatched (see :ref:`below `) at each stage of +workload execution, which installed instruments can hook into in order to +collect measurements, alter workload execution, etc. Instruments implementation +usually mirrors that of workloads, defining initialization, setup, teardown and +output processing stages for a particular instrument. Instead of a ``run`` +method instruments usually implement ``start`` and ``stop`` methods instead +which triggered just before and just after a workload run. However, the signal +dispatch mechanism gives a high degree of flexibility to instruments allowing +them to hook into almost any stage of a WA run (apart from the very early +initialization). + +Metrics and artifacts generated by workloads and instruments are accumulated by +the framework and are then passed to active output processors. This happens +after each individual workload execution and at the end of the run. A output +processor may chose to act at either or both of these points. + + +Control Flow +------------ + +This section goes into more detail explaining the relationship between the major +components of the framework and how control passes between them during a run. It +will only go through the major transitions and interactions and will not attempt +to describe every single thing that happens. + +.. note:: This is the control flow for the ``wa run`` command which is the main + functionality of WA. Other commands are much simpler and most of what + is described below does not apply to them. + +#. :class:`wa.framework.entrypoint` parses the command from the arguments, creates a + :class:`wa.framework.configuration.execution.ConfigManager` and executes the run + command (:class:`wa.commands.run.RunCommand`) passing it the ConfigManger. +#. Run command initializes the output directory and creates a + :class:`wa.framework.configuration.parsers.AgendaParser` and will parser an + agenda and populate the ConfigManger based on the command line arguments. + Finally it instantiates a :class:`wa.framework.execution.Executor` and + passes it the completed ConfigManager. +#. The Executor uses the ConfigManager to create a + :class:`wa.framework.configuration.core.RunConfiguration` and fully defines the + configuration for the run (which will be serialised into ``__meta`` subdirectory + under the output directory). +#. The Executor proceeds to instantiate a TargetManager, used to handle the + device connection and configuration, and a + :class:`wa.framework.execution.ExecutionContext` which is used to track the + current state of the run execution and also serves as a means of + communication between the core framework and plugins. After this any required + instruments and output processors are initialized and installed. +#. Finally, the Executor instantiates a :class:`wa.framework.execution.Runner`, + initializes its job queue with workload specs from the RunConfiguraiton, and + kicks it off. +#. The Runner performs the run time configuration of the device and goes + through the workload specs (in the order defined by ``execution_order`` + setting), running each spec according to the execution model described in the + previous section and sending signals (see below) at appropriate points during + execution. +#. At the end of the run, the control is briefly passed back to the Executor, + which outputs a summary for the run. + + +.. _signal_dispatch: + +Signal Dispatch +--------------- + +WA uses the `louie `_ (formerly, +pydispatcher) library for signal dispatch. Callbacks can be registered for +signals emitted during the run. WA uses a version of louie that has been +modified to introduce priority to registered callbacks (so that callbacks that +are know to be slow can be registered with a lower priority and therefore do not +interfere with other callbacks). + +This mechanism is abstracted for instruments. Methods of an +:class:`wa.framework.Instrument` subclass automatically get hooked to +appropriate signals based on their names when the instrument is "installed" +for the run. Priority can then be specified by adding ``extremely_fast``, +``very_fast``, ``fast`` , ``slow``, ``very_slow`` or ``extremely_slow`` +:ref:`decorators ` to the method definitions. + +The full list of method names and the signals they map to may be viewed +:ref:`here `. + +Signal dispatching mechanism may also be used directly, for example to +dynamically register callbacks at runtime or allow plugins other than +``Instruments`` to access stages of the run they are normally not aware of. + +Signals can be either paired or non paired signals. Non paired signals are one +off signals that are sent to indicate special events or transitions in execution +stages have occurred for example ``TARGET_CONNECTED``. Paired signals are used to +signify the start and end of a particular event. If the start signal has been +sent the end signal is guaranteed to also be sent, whether the operation was a +successes or not, however in the case of correct operation an additional success +signal will also be sent. For example in the event of a successful reboot of the +the device, the following signals will be sent ``BEFORE_REBOOT``, +``SUCCESSFUL_REBOOT`` and ``AFTER_REBOOT``. + +An overview of what signals are sent at which point during execution can be seen +below. Most of the paired signals have been removed from the diagram for clarity +and shown as being dispatched from a particular stage of execution, however in +reality these signals will be sent just before and just after these stages are +executed. As mentioned above for each of these signals there will be at least 2 +and up to 3 signals sent. If the "BEFORE_X" signal (sent just before the stage +is ran) is sent then the "AFTER_X" (sent just after the stage is ran) signal is +guaranteed to also be sent, and under normal operation a "SUCCESSFUL_X" signal +is also sent just after stage has been completed. The diagram also lists the +conditional signals that can be sent at any time during execution if something +unexpected happens, for example an error occurs or the user aborts the run. + +.. image:: developer_reference/WA_Signal_Dispatch.svg + :scale: 100 % + +See Also +-------- + + - :ref:`Instrumentation Signal-Method Mapping `.