sascha/workload-automation
1
0
Fork 0
mirror of https://github.com/ARM-software/workload-automation.git synced 2025-09-02 11:22:41 +01:00
Code Issues Releases Wiki Activity

Add support for Python 3

Browse Source 
Add support for running under Python 3, while maintaining compatibility
with Python 2.

See http://python-future.org/compatible_idioms.html for more details
behind these changes.
This commit is contained in:
Sergei Trofimov
2018-05-30 13:58:49 +01:00
committed by Marc Bonnici
parent c3ddb31d4d
commit b3de85455a
53 changed files with 377 additions and 384 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
wa/framework/configuration/core.py
View File

@@ -95,11 +95,11 @@ class RebootPolicy(object):
__repr__ = __str__
def __cmp__(self, other):
def __eq__(self, other):
if isinstance(other, RebootPolicy):
return cmp(self.policy, other.policy)
return self.policy == other.policy
else:
return cmp(self.policy, other)
return self.policy == other
def to_pod(self):
return self.policy
@@ -127,7 +127,7 @@ class LoggingConfig(dict):
def __init__(self, config=None):
dict.__init__(self)
if isinstance(config, dict):
config = {identifier(k.lower()): v for k, v in config.iteritems()}
config = {identifier(k.lower()): v for k, v in config.items()}
self['regular_format'] = config.pop('regular_format', self.defaults['regular_format'])
self['verbose_format'] = config.pop('verbose_format', self.defaults['verbose_format'])
self['file_format'] = config.pop('file_format', self.defaults['file_format'])
@@ -135,9 +135,9 @@ class LoggingConfig(dict):
self['color'] = config.pop('color', self.defaults['color'])
if config:
message = 'Unexpected logging configuration parameters: {}'
raise ValueError(message.format(bad_vals=', '.join(config.keys())))
raise ValueError(message.format(bad_vals=', '.join(list(config.keys()))))
elif config is None:
for k, v in self.defaults.iteritems():
for k, v in self.defaults.items():
self[k] = v
else:
raise ValueError(config)
@@ -360,7 +360,7 @@ class Configuration(object):
cfg_point.set_value(instance, value)
if pod:
msg = 'Invalid entry(ies) for "{}": "{}"'
raise ValueError(msg.format(cls.name, '", "'.join(pod.keys())))
raise ValueError(msg.format(cls.name, '", "'.join(list(pod.keys()))))
return instance
def __init__(self):
@@ -380,7 +380,7 @@ class Configuration(object):
def update_config(self, values, check_mandatory=True):
for k, v in values.iteritems():
for k, v in values.items():
self.set(k, v, check_mandatory=check_mandatory)
def validate(self):
@@ -824,7 +824,7 @@ class JobSpec(Configuration):
def update_config(self, source, check_mandatory=True):
self._sources.append(source)
values = source.config
for k, v in values.iteritems():
for k, v in values.items():
if k == "id":
continue
elif k.endswith('_parameters'):
@@ -849,7 +849,7 @@ class JobSpec(Configuration):
if not config:
continue
for name, cfg_point in cfg_points.iteritems():
for name, cfg_point in cfg_points.items():
if name in config:
value = config.pop(name)
cfg_point.set_value(workload_params, value,
@@ -873,7 +873,7 @@ class JobSpec(Configuration):
runtime_parameters[source] = global_runtime_params[source]
# Add runtime parameters from JobSpec
for source, values in self.to_merge['runtime_parameters'].iteritems():
for source, values in self.to_merge['runtime_parameters'].items():
runtime_parameters[source] = values
# Merge
@@ -884,9 +884,9 @@ class JobSpec(Configuration):
for source in self._sources[1:]]) # ignore first id, "global"
# ensure *_parameters are always obj_dict's
self.boot_parameters = obj_dict((self.boot_parameters or {}).items())
self.runtime_parameters = obj_dict((self.runtime_parameters or {}).items())
self.workload_parameters = obj_dict((self.workload_parameters or {}).items())
self.boot_parameters = obj_dict(list((self.boot_parameters or {}).items()))
self.runtime_parameters = obj_dict(list((self.runtime_parameters or {}).items()))
self.workload_parameters = obj_dict(list((self.workload_parameters or {}).items()))
if self.label is None:
self.label = self.workload_name
@@ -903,7 +903,7 @@ class JobGenerator(object):
self._read_augmentations = True
if self._enabled_instruments is None:
self._enabled_instruments = []
for entry in self._enabled_augmentations.merge_with(self.disabled_augmentations).values():
for entry in list(self._enabled_augmentations.merge_with(self.disabled_augmentations).values()):
entry_cls = self.plugin_cache.get_plugin_class(entry)
if entry_cls.kind == 'instrument':
self._enabled_instruments.append(entry)
@@ -914,7 +914,7 @@ class JobGenerator(object):
self._read_augmentations = True
if self._enabled_processors is None:
self._enabled_processors = []
for entry in self._enabled_augmentations.merge_with(self.disabled_augmentations).values():
for entry in list(self._enabled_augmentations.merge_with(self.disabled_augmentations).values()):
entry_cls = self.plugin_cache.get_plugin_class(entry)
if entry_cls.kind == 'output_processor':
self._enabled_processors.append(entry)
@@ -934,7 +934,7 @@ class JobGenerator(object):
self.job_spec_template.name = "globally specified job spec configuration"
self.job_spec_template.id = "global"
# Load defaults
for cfg_point in JobSpec.configuration.itervalues():
for cfg_point in JobSpec.configuration.values():
cfg_point.set_value(self.job_spec_template, check_mandatory=False)
self.root_node = SectionNode(self.job_spec_template)
@@ -996,7 +996,7 @@ class JobGenerator(object):
break
else:
continue
self.update_augmentations(job_spec.augmentations.values())
self.update_augmentations(list(job_spec.augmentations.values()))
specs.append(job_spec)
return specs

16
wa/framework/configuration/execution.py
View File

@@ -1,5 +1,7 @@
import random
from itertools import izip_longest, groupby, chain
from itertools import groupby, chain
from future.moves.itertools import zip_longest
from wa.framework.configuration.core import (MetaConfiguration, RunConfiguration,
JobGenerator, Status, settings)
@@ -157,8 +159,8 @@ def permute_by_iteration(specs):
all_tuples = []
for spec in chain(*groups):
all_tuples.append([(spec, i + 1)
for i in xrange(spec.iterations)])
for t in chain(*map(list, izip_longest(*all_tuples))):
for i in range(spec.iterations)])
for t in chain(*list(map(list, zip_longest(*all_tuples)))):
if t is not None:
yield t
@@ -183,8 +185,8 @@ def permute_by_section(specs):
all_tuples = []
for spec in chain(*groups):
all_tuples.append([(spec, i + 1)
for i in xrange(spec.iterations)])
for t in chain(*map(list, izip_longest(*all_tuples))):
for i in range(spec.iterations)])
for t in chain(*list(map(list, zip_longest(*all_tuples)))):
if t is not None:
yield t
@@ -196,7 +198,7 @@ def permute_randomly(specs):
"""
result = []
for spec in specs:
for i in xrange(1, spec.iterations + 1):
for i in range(1, spec.iterations + 1):
result.append((spec, i))
random.shuffle(result)
for t in result:
@@ -214,5 +216,5 @@ permute_map = {
def permute_iterations(specs, exec_order):
if exec_order not in permute_map:
msg = 'Unknown execution order "{}"; must be in: {}'
raise ValueError(msg.format(exec_order, permute_map.keys()))
raise ValueError(msg.format(exec_order, list(permute_map.keys())))
return permute_map[exec_order](specs)

17
wa/framework/configuration/parsers.py
View File

@@ -21,6 +21,7 @@ from wa.framework.exception import ConfigError
from wa.utils import log
from wa.utils.serializer import json, read_pod, SerializerSyntaxError
from wa.utils.types import toggle_set, counter
from functools import reduce
logger = logging.getLogger('config')
@@ -47,27 +48,27 @@ class ConfigParser(object):
merge_augmentations(raw)
# Get WA core configuration
for cfg_point in state.settings.configuration.itervalues():
for cfg_point in state.settings.configuration.values():
value = pop_aliased_param(cfg_point, raw)
if value is not None:
logger.debug('Setting meta "{}" to "{}"'.format(cfg_point.name, value))
state.settings.set(cfg_point.name, value)
# Get run specific configuration
for cfg_point in state.run_config.configuration.itervalues():
for cfg_point in state.run_config.configuration.values():
value = pop_aliased_param(cfg_point, raw)
if value is not None:
logger.debug('Setting run "{}" to "{}"'.format(cfg_point.name, value))
state.run_config.set(cfg_point.name, value)
# Get global job spec configuration
for cfg_point in JobSpec.configuration.itervalues():
for cfg_point in JobSpec.configuration.values():
value = pop_aliased_param(cfg_point, raw)
if value is not None:
logger.debug('Setting global "{}" to "{}"'.format(cfg_point.name, value))
state.jobs_config.set_global_value(cfg_point.name, value)
for name, values in raw.iteritems():
for name, values in raw.items():
# Assume that all leftover config is for a plug-in or a global
# alias it is up to PluginCache to assert this assumption
logger.debug('Caching "{}" with "{}"'.format(name, values))
@@ -106,7 +107,7 @@ class AgendaParser(object):
if raw:
msg = 'Invalid top level agenda entry(ies): "{}"'
raise ConfigError(msg.format('", "'.join(raw.keys())))
raise ConfigError(msg.format('", "'.join(list(raw.keys()))))
sect_ids, wkl_ids = self._collect_ids(sections, global_workloads)
self._process_global_workloads(state, global_workloads, wkl_ids)
@@ -301,7 +302,7 @@ def _construct_valid_entry(raw, seen_ids, prefix, jobs_config):
merge_augmentations(raw)
# Validate all workload_entry
for name, cfg_point in JobSpec.configuration.iteritems():
for name, cfg_point in JobSpec.configuration.items():
value = pop_aliased_param(cfg_point, raw)
if value is not None:
value = cfg_point.kind(value)
@@ -317,7 +318,7 @@ def _construct_valid_entry(raw, seen_ids, prefix, jobs_config):
# error if there are unknown workload_entry
if raw:
msg = 'Invalid entry(ies) in "{}": "{}"'
raise ConfigError(msg.format(workload_entry['id'], ', '.join(raw.keys())))
raise ConfigError(msg.format(workload_entry['id'], ', '.join(list(raw.keys()))))
return workload_entry
@@ -339,7 +340,7 @@ def _collect_valid_id(entry_id, seen_ids, entry_type):
def _get_workload_entry(workload):
if isinstance(workload, basestring):
if isinstance(workload, str):
workload = {'name': workload}
elif not isinstance(workload, dict):
raise ConfigError('Invalid workload entry: "{}"')

20
wa/framework/configuration/plugin_cache.py
View File

@@ -90,11 +90,11 @@ class PluginCache(object):
msg = 'configuration provided for unknown plugin "{}"'
raise ConfigError(msg.format(plugin_name))
if not hasattr(values, 'iteritems'):
if not hasattr(values, 'items'):
msg = 'Plugin configuration for "{}" not a dictionary ({} is {})'
raise ConfigError(msg.format(plugin_name, repr(values), type(values)))
for name, value in values.iteritems():
for name, value in values.items():
if (plugin_name not in GENERIC_CONFIGS and
name not in self.get_plugin_parameters(plugin_name)):
msg = "'{}' is not a valid parameter for '{}'"
@@ -124,7 +124,7 @@ class PluginCache(object):
for source in self.sources:
if source not in plugin_config:
continue
for name, value in plugin_config[source].iteritems():
for name, value in plugin_config[source].items():
cfg_points[name].set_value(config, value=value)
else:
# A more complicated merge that involves priority of sources and
@@ -136,7 +136,7 @@ class PluginCache(object):
def get_plugin(self, name, kind=None, *args, **kwargs):
config = self.get_plugin_config(name)
kwargs = dict(config.items() + kwargs.items())
kwargs = dict(list(config.items()) + list(kwargs.items()))
return self.loader.get_plugin(name, kind=kind, *args, **kwargs)
def get_plugin_class(self, name, kind=None):
@@ -154,18 +154,18 @@ class PluginCache(object):
def _set_plugin_defaults(self, plugin_name, config):
cfg_points = self.get_plugin_parameters(plugin_name)
for cfg_point in cfg_points.itervalues():
for cfg_point in cfg_points.values():
cfg_point.set_value(config, check_mandatory=False)
try:
_, alias_params = self.resolve_alias(plugin_name)
for name, value in alias_params.iteritems():
for name, value in alias_params.items():
cfg_points[name].set_value(config, value)
except NotFoundError:
pass
def _set_from_global_aliases(self, plugin_name, config):
for alias, param in self._global_alias_map[plugin_name].iteritems():
for alias, param in self._global_alias_map[plugin_name].items():
if alias in self.global_alias_values:
for source in self.sources:
if source not in self.global_alias_values[alias]:
@@ -230,7 +230,7 @@ class PluginCache(object):
# Validate final configuration
merged_config.name = specific_name
for cfg_point in ms.cfg_points.itervalues():
for cfg_point in ms.cfg_points.values():
cfg_point.validate(merged_config, check_mandatory=is_final)
def __getattr__(self, name):
@@ -285,7 +285,7 @@ class MergeState(object):
def update_config_from_source(final_config, source, state):
if source in state.generic_config:
final_config.name = state.generic_name
for name, cfg_point in state.cfg_points.iteritems():
for name, cfg_point in state.cfg_points.items():
if name in state.generic_config[source]:
if name in state.seen_specific_config:
msg = ('"{generic_name}" configuration "{config_name}" has '
@@ -307,7 +307,7 @@ def update_config_from_source(final_config, source, state):
if source in state.specific_config:
final_config.name = state.specific_name
for name, cfg_point in state.cfg_points.iteritems():
for name, cfg_point in state.cfg_points.items():
if name in state.specific_config[source]:
state.seen_specific_config[name].append(str(source))
value = state.specific_config[source].pop(name)

2
wa/framework/configuration/tree.py
View File

@@ -39,7 +39,7 @@ class JobSpecSource(object):
def _log_self(self):
logger.debug('Creating {} node'.format(self.kind))
with log.indentcontext():
for key, value in self.config.iteritems():
for key, value in self.config.items():
logger.debug('"{}" to "{}"'.format(key, value))