diff --git a/wa/framework/instrumentation.py b/wa/framework/instrumentation.py
index b3ce49af..4df437b3 100644
--- a/wa/framework/instrumentation.py
+++ b/wa/framework/instrumentation.py
@@ -275,7 +275,10 @@ class ManagedCallback(object):
                 if isinstance(e, WorkloadError):
                     context.set_status('FAILED')
                 else:
-                    context.set_status('PARTIAL')
+                    if context.current_job:
+                        context.set_status('PARTIAL')
+                    else:
+                        raise
 
 
 # Need this to keep track of callbacks, because the dispatcher only keeps