From 45d8be52280eafbaa32d7c86388d58a624052d67 Mon Sep 17 00:00:00 2001 From: Sergei Trofimov Date: Fri, 24 Nov 2017 08:23:00 +0000 Subject: [PATCH] framework/execution: ensure output is always processed Ensure that job output is processed even if a workload fails. This is because output processing includes things like extracting logs, which we still want to happen on failure. Job status is now also set correctly when an error occurs during output processing rather than actual running of the workload. Previously, the status would be correctly set to PARTIAL in the inner except clause, but the exception is then re-raised, and the status was "upgraded" to FAILED in the outer except clause. --- wa/framework/execution.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/wa/framework/execution.py b/wa/framework/execution.py index f7d906cc..72a85099 100644 --- a/wa/framework/execution.py +++ b/wa/framework/execution.py @@ -464,28 +464,30 @@ class Runner(object): job.setup(context) try: - with signal.wrap('JOB_EXECUTION', self): - job.run(context) try: - with signal.wrap('JOB_OUTPUT_PROCESSED', self): - job.process_output(context) - self.pm.process_job_output(context) - self.pm.export_job_output(context) - except Exception: - job.set_status(Status.PARTIAL) - raise + with signal.wrap('JOB_EXECUTION', self): + job.run(context) + except Exception as e: + job.set_status(Status.FAILED) + if not getattr(e, 'logged', None): + log.log_error(e, self.logger) + e.logged = True + raise e + finally: + try: + with signal.wrap('JOB_OUTPUT_PROCESSED', self): + job.process_output(context) + self.pm.process_job_output(context) + self.pm.export_job_output(context) + except Exception: + job.set_status(Status.PARTIAL) + raise except KeyboardInterrupt: job.set_status(Status.ABORTED) self.logger.info('Got CTRL-C. Aborting.') raise - except Exception as e: - job.set_status(Status.FAILED) - if not getattr(e, 'logged', None): - log.log_error(e, self.logger) - e.logged = True - raise e finally: # If setup was successfully completed, teardown must # run even if the job failed