Handle retry failed

wa/framework/configuration/core.py

@@ -20,9 +20,8 @@ from collections import OrderedDict, defaultdict
 from wa.framework.exception import ConfigError, NotFoundError
 from wa.framework.configuration.tree import SectionNode
 from wa.utils.misc import (get_article, merge_config_values)
-from wa.utils.types import (identifier, integer, boolean,
-                                list_of_strings, toggle_set,
-                                obj_dict)
+from wa.utils.types import (identifier, integer, boolean, list_of_strings, 
+                            list_of, toggle_set, obj_dict, enum)
 from wa.utils.serializer import is_pod
 
 # Mapping for kind conversion; see docs for convert_types below
@@ -32,17 +31,9 @@ KIND_MAP = {
     dict: OrderedDict,
 }
 
-ITERATION_STATUS = [
-    'NOT_STARTED',
-    'RUNNING',
+JobStatus = enum(['NEW', 'LOADED', 'PENDING', 'RUNNING', 
+                  'OK', 'FAILED', 'PARTIAL', 'ABORTED', 'SKIPPED'])
 
-    'OK',
-    'NONCRITICAL',
-    'PARTIAL',
-    'FAILED',
-    'ABORTED',
-    'SKIPPED',
-]
 
 ##########################
 ### CONFIG POINT TYPES ###
@@ -716,9 +707,9 @@ class RunConfiguration(Configuration):
                            This setting defines what specific Device subclass will be used to interact
                            the connected device. Obviously, this must match your setup.
                            '''),
-        ConfigurationPoint('retry_on_status', kind=status_list,
+        ConfigurationPoint('retry_on_status', kind=list_of(JobStatus),
                            default=['FAILED', 'PARTIAL'],
-                           allowed_values=ITERATION_STATUS,
+                           allowed_values=JobStatus.values,
                            description='''
                            This is list of statuses on which a job will be cosidered to have failed and
                            will be automatically retried up to ``max_retries`` times. This defaults to
@@ -736,10 +727,10 @@ class RunConfiguration(Configuration):
                            ``"ABORTED"``
                            The user interupted the workload
                            '''),
-        ConfigurationPoint('max_retries', kind=int, default=3,
+        ConfigurationPoint('max_retries', kind=int, default=2,
                            description='''
                            The maximum number of times failed jobs will be retried before giving up. If
-                           not set, this will default to ``3``.
+                           not set.
 
                            .. note:: this number does not include the original attempt
                            '''),

wa/framework/configuration/execution.py

@@ -4,10 +4,11 @@ from itertools import izip_longest, groupby, chain
 
 from wa.framework import pluginloader
 from wa.framework.configuration.core import (MetaConfiguration, RunConfiguration,
-                                             JobGenerator, settings)
+                                             JobGenerator, JobStatus, settings)
 from wa.framework.configuration.parsers import ConfigParser
 from wa.framework.configuration.plugin_cache import PluginCache
 from wa.framework.exception import NotFoundError
+from wa.framework.job import Job
 from wa.utils.types import enum
 
 
@@ -29,59 +30,6 @@ class CombinedConfig(object):
                 'run_config': self.run_config.to_pod()}
 
 
-JobStatus = enum(['NEW', 'LOADED', 'PENDING', 'RUNNING', 
-                  'OK', 'FAILED', 'PARTIAL', 'ABORTED', 'SKIPPED'])
-
-class Job(object):
-
-    @property
-    def id(self):
-        return self.spec.id
-
-    @property
-    def output_name(self):
-        return '{}-{}-{}'.format(self.id, self.spec.label, self.iteration)
-
-    def __init__(self, spec, iteration, context):
-        self.logger = logging.getLogger('job')
-        self.spec = spec
-        self.iteration = iteration
-        self.context = context
-        self.status = JobStatus.NEW
-        self.workload = None
-        self.output = None
-
-    def load(self, target, loader=pluginloader):
-        self.logger.debug('Loading job {}'.format(self.id))
-        self.workload = loader.get_workload(self.spec.workload_name,
-                                            target,
-                                            **self.spec.workload_parameters)
-        self.workload.init_resources(self.context)
-        self.workload.validate()
-        self.status = JobStatus.LOADED
-
-    def initialize(self, context):
-        self.logger.info('Initializing job {}'.format(self.id))
-        self.status = JobStatus.PENDING
-
-    def configure_target(self, context):
-        self.logger.info('Configuring target for job {}'.format(self.id))
-
-    def setup(self, context):
-        self.logger.info('Setting up job {}'.format(self.id))
-
-    def run(self, context):
-        self.logger.info('Running job {}'.format(self.id))
-
-    def process_output(self, context):
-        self.looger.info('Processing output for job {}'.format(self.id))
-
-    def teardown(self, context):
-        self.logger.info('Tearing down job {}'.format(self.id))
-
-    def finalize(self, context):
-        self.logger.info('Finalizing job {}'.format(self.id))
-
 class ConfigManager(object):
     """
     Represents run-time state of WA. Mostly used as a container for loaded