Add support for Python 3

Add support for running under Python 3, while maintaining compatibility
with Python 2.

See http://python-future.org/compatible_idioms.html for more details
behind these changes.

doc/source/changes.rst

@@ -19,8 +19,8 @@ For more information on migrating from WA2 to WA3 please see the
 
 Not all of WA2 extensions have been ported for the initial 3.0.0 release. We
 have ported the ones we believe to be most widely used and useful. The porting
-work will continue, and more of WA2's extensions will be in the future releases,
+work will continue, and more of WA2's extensions will be in the future releases.
-however we do not intend to port absolutely everything, as some things we
+However, we do not intend to port absolutely everything, as some things we
 believe to be no longer useful.
 
 .. note:: If there a particular WA2 extension you would like to see in WA3 that
@@ -31,6 +31,15 @@ believe to be no longer useful.
 New Features
 ~~~~~~~~~~~~
 
+- Python 3 support. WA now runs on both Python 2 and Python 3.
+
+  .. warning:: Python 2 support should now be considered depricated. Python 2
+               will still be fully supported up to the next major release
+               (v3.1). After that, Python 2 will be supported for existing
+               functionality, however there will be no guarantee that newly
+               added functionality would be compatible with Python 2. Support
+               for Python 2 will be dropped completely after release v3.2.
+
 - There is a new Output API which can be used to aid in post processing a
   run's output. For more information please see :ref:`output_processing_api`.
 - All "augmentations" can now be enabled on a per workload basis (in WA2 this

setup.py

@@ -81,9 +81,10 @@ params = dict(
         'pyYAML',  # YAML-formatted agenda parsing
         'requests',  # Fetch assets over HTTP
         'devlib>=0.0.4',  # Interacting with devices
-        'louie',  # callbacks dispatch
+        'louie-latest',  # callbacks dispatch
         'wrapt',  # better decorators
         'pandas>=0.13.1',  # Data analysis and manipulation
+        'future',  # Python 2-3 compatiblity
     ],
     dependency_links=['https://github.com/ARM-software/devlib/tarball/master#egg=devlib-0.0.4'],
 
@@ -104,7 +105,7 @@ params = dict(
     ],
 )
 
-all_extras = list(chain(params['extras_require'].itervalues()))
+all_extras = list(chain(iter(params['extras_require'].values())))
 params['extras_require']['everything'] = all_extras
 
 setup(**params)

wa/commands/create.py

@@ -94,9 +94,9 @@ class CreateWorkloadSubcommand(SubCommand):
         self.parser.add_argument('-f', '--force', action='store_true',
                                  help='Create the new workload even if a workload with the specified ' +
                                       'name already exists.')
-        self.parser.add_argument('-k', '--kind', metavar='KIND', default='basic', choices=create_funcs.keys(),
+        self.parser.add_argument('-k', '--kind', metavar='KIND', default='basic', choices=list(create_funcs.keys()),
                                  help='The type of workload to be created. The available options ' +
-                                      'are: {}'.format(', '.join(create_funcs.keys())))
+                                      'are: {}'.format(', '.join(list(create_funcs.keys()))))
 
     def execute(self, state, args):  # pylint: disable=R0201
         where = args.path or 'local'
@@ -179,7 +179,7 @@ def create_workload(name, kind='basic', where='local', check_name=True, **kwargs
     except KeyError:
         raise CommandError('Unknown workload type: {}'.format(kind))
 
-    print 'Workload created in {}'.format(workload_dir)
+    print('Workload created in {}'.format(workload_dir))
 
 
 def create_template_workload(path, name, kind, class_name):

wa/commands/list.py

@@ -71,7 +71,7 @@ def list_targets():
     output = DescriptionListFormatter()
     for target in targets:
         output.add_item(target.description or '', target.name)
-    print output.format_data()
+    print(output.format_data())
 
 
 def list_plugins(args, filters):
@@ -80,7 +80,7 @@ def list_plugins(args, filters):
         filtered_results = []
         for result in results:
             passed = True
-            for k, v in filters.iteritems():
+            for k, v in filters.items():
                 if getattr(result, k) != v:
                     passed = False
                     break
@@ -95,7 +95,7 @@ def list_plugins(args, filters):
         output = DescriptionListFormatter()
         for result in sorted(filtered_results, key=lambda x: x.name):
             output.add_item(get_summary(result), result.name)
-        print output.format_data()
+        print(output.format_data())
 
 
 def check_platform(plugin, platform):

wa/commands/revent.py

@@ -24,6 +24,10 @@ from wa.framework.target.manager import TargetManager
 from wa.utils.revent import ReventRecorder
 
 
+if sys.version_info[0] == 3:
+    raw_input = input
+
+
 class RecordCommand(Command):
 
     name = 'record'
@@ -146,7 +150,7 @@ class RecordCommand(Command):
         if os.path.exists(host_path):
             msg = 'Revent file \'{}\' already exists, overwrite? [y/n]'
             self.logger.info(msg.format(revent_file_name))
-            if raw_input('') == 'y':
+            if input('') == 'y':
                 os.remove(host_path)
             else:
                 msg = 'Did not pull and overwrite \'{}\''
@@ -222,7 +226,7 @@ class RecordCommand(Command):
         if not file_name:
             file_name = '{}.revent'.format(self.target.model)
         if not output_path:
-            output_path = os.getcwdu()
+            output_path = os.getcwd()
 
         return output_path, file_name
 

wa/commands/show.py

@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 #
+import sys
 from subprocess import call, Popen, PIPE
 
 from wa import Command
@@ -66,7 +67,11 @@ class ShowCommand(Command):
 
         if which('pandoc'):
             p = Popen(['pandoc', '-f', 'rst', '-t', 'man'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
-            output, _ = p.communicate(rst_output)
+            if sys.version_info[0] == 3:
+                output, _ = p.communicate(rst_output.encode(sys.stdin.encoding))
+                output = output.decode(sys.stdout.encoding)
+            else:
+                output, _ = p.communicate(rst_output)
 
             # Make sure to double escape back slashes
             output = output.replace('\\', '\\\\\\')
@@ -78,7 +83,7 @@ class ShowCommand(Command):
 
             call('echo "{}" | man -l -'.format(escape_double_quotes(output)), shell=True)
         else:
-            print rst_output
+            print(rst_output)
 
 
 def get_target_description(name):

wa/framework/configuration/core.py

@@ -95,11 +95,11 @@ class RebootPolicy(object):
 
     __repr__ = __str__
 
-    def __cmp__(self, other):
+    def __eq__(self, other):
         if isinstance(other, RebootPolicy):
-            return cmp(self.policy, other.policy)
+            return self.policy == other.policy
         else:
-            return cmp(self.policy, other)
+            return self.policy == other
 
     def to_pod(self):
         return self.policy
@@ -127,7 +127,7 @@ class LoggingConfig(dict):
     def __init__(self, config=None):
         dict.__init__(self)
         if isinstance(config, dict):
-            config = {identifier(k.lower()): v for k, v in config.iteritems()}
+            config = {identifier(k.lower()): v for k, v in config.items()}
             self['regular_format'] = config.pop('regular_format', self.defaults['regular_format'])
             self['verbose_format'] = config.pop('verbose_format', self.defaults['verbose_format'])
             self['file_format'] = config.pop('file_format', self.defaults['file_format'])
@@ -135,9 +135,9 @@ class LoggingConfig(dict):
             self['color'] = config.pop('color', self.defaults['color'])
             if config:
                 message = 'Unexpected logging configuration parameters: {}'
-                raise ValueError(message.format(bad_vals=', '.join(config.keys())))
+                raise ValueError(message.format(bad_vals=', '.join(list(config.keys()))))
         elif config is None:
-            for k, v in self.defaults.iteritems():
+            for k, v in self.defaults.items():
                 self[k] = v
         else:
             raise ValueError(config)
@@ -360,7 +360,7 @@ class Configuration(object):
                 cfg_point.set_value(instance, value)
         if pod:
             msg = 'Invalid entry(ies) for "{}": "{}"'
-            raise ValueError(msg.format(cls.name, '", "'.join(pod.keys())))
+            raise ValueError(msg.format(cls.name, '", "'.join(list(pod.keys()))))
         return instance
 
     def __init__(self):
@@ -380,7 +380,7 @@ class Configuration(object):
 
 
     def update_config(self, values, check_mandatory=True):
-        for k, v in values.iteritems():
+        for k, v in values.items():
             self.set(k, v, check_mandatory=check_mandatory)
 
     def validate(self):
@@ -824,7 +824,7 @@ class JobSpec(Configuration):
     def update_config(self, source, check_mandatory=True):
         self._sources.append(source)
         values = source.config
-        for k, v in values.iteritems():
+        for k, v in values.items():
             if k == "id":
                 continue
             elif k.endswith('_parameters'):
@@ -849,7 +849,7 @@ class JobSpec(Configuration):
             if not config:
                 continue
 
-            for name, cfg_point in cfg_points.iteritems():
+            for name, cfg_point in cfg_points.items():
                 if name in config:
                     value = config.pop(name)
                     cfg_point.set_value(workload_params, value,
@@ -873,7 +873,7 @@ class JobSpec(Configuration):
                 runtime_parameters[source] = global_runtime_params[source]
 
         # Add runtime parameters from JobSpec
-        for source, values in self.to_merge['runtime_parameters'].iteritems():
+        for source, values in self.to_merge['runtime_parameters'].items():
             runtime_parameters[source] = values
 
         # Merge
@@ -884,9 +884,9 @@ class JobSpec(Configuration):
                             for source in self._sources[1:]])  # ignore first id, "global"
 
         # ensure *_parameters are always obj_dict's
-        self.boot_parameters = obj_dict((self.boot_parameters or {}).items())
+        self.boot_parameters = obj_dict(list((self.boot_parameters or {}).items()))
-        self.runtime_parameters = obj_dict((self.runtime_parameters or {}).items())
+        self.runtime_parameters = obj_dict(list((self.runtime_parameters or {}).items()))
-        self.workload_parameters = obj_dict((self.workload_parameters or {}).items())
+        self.workload_parameters = obj_dict(list((self.workload_parameters or {}).items()))
 
         if self.label is None:
             self.label = self.workload_name
@@ -903,7 +903,7 @@ class JobGenerator(object):
         self._read_augmentations = True
         if self._enabled_instruments is None:
             self._enabled_instruments = []
-            for entry in self._enabled_augmentations.merge_with(self.disabled_augmentations).values():
+            for entry in list(self._enabled_augmentations.merge_with(self.disabled_augmentations).values()):
                 entry_cls = self.plugin_cache.get_plugin_class(entry)
                 if entry_cls.kind == 'instrument':
                     self._enabled_instruments.append(entry)
@@ -914,7 +914,7 @@ class JobGenerator(object):
         self._read_augmentations = True
         if self._enabled_processors is None:
             self._enabled_processors = []
-            for entry in self._enabled_augmentations.merge_with(self.disabled_augmentations).values():
+            for entry in list(self._enabled_augmentations.merge_with(self.disabled_augmentations).values()):
                 entry_cls = self.plugin_cache.get_plugin_class(entry)
                 if entry_cls.kind == 'output_processor':
                     self._enabled_processors.append(entry)
@@ -934,7 +934,7 @@ class JobGenerator(object):
         self.job_spec_template.name = "globally specified job spec configuration"
         self.job_spec_template.id = "global"
         # Load defaults
-        for cfg_point in JobSpec.configuration.itervalues():
+        for cfg_point in JobSpec.configuration.values():
             cfg_point.set_value(self.job_spec_template, check_mandatory=False)
 
         self.root_node = SectionNode(self.job_spec_template)
@@ -996,7 +996,7 @@ class JobGenerator(object):
                             break
                     else:
                         continue
-                self.update_augmentations(job_spec.augmentations.values())
+                self.update_augmentations(list(job_spec.augmentations.values()))
                 specs.append(job_spec)
         return specs
 

wa/framework/configuration/execution.py

@@ -1,5 +1,7 @@
 import random
-from itertools import izip_longest, groupby, chain
+from itertools import groupby, chain
+
+from future.moves.itertools import zip_longest
 
 from wa.framework.configuration.core import (MetaConfiguration, RunConfiguration,
                                              JobGenerator, Status, settings)
@@ -157,8 +159,8 @@ def permute_by_iteration(specs):
     all_tuples = []
     for spec in chain(*groups):
         all_tuples.append([(spec, i + 1)
-                           for i in xrange(spec.iterations)])
+                           for i in range(spec.iterations)])
-    for t in chain(*map(list, izip_longest(*all_tuples))):
+    for t in chain(*list(map(list, zip_longest(*all_tuples)))):
         if t is not None:
             yield t
 
@@ -183,8 +185,8 @@ def permute_by_section(specs):
     all_tuples = []
     for spec in chain(*groups):
         all_tuples.append([(spec, i + 1)
-                           for i in xrange(spec.iterations)])
+                           for i in range(spec.iterations)])
-    for t in chain(*map(list, izip_longest(*all_tuples))):
+    for t in chain(*list(map(list, zip_longest(*all_tuples)))):
         if t is not None:
             yield t
 
@@ -196,7 +198,7 @@ def permute_randomly(specs):
     """
     result = []
     for spec in specs:
-        for i in xrange(1, spec.iterations + 1):
+        for i in range(1, spec.iterations + 1):
             result.append((spec, i))
     random.shuffle(result)
     for t in result:
@@ -214,5 +216,5 @@ permute_map = {
 def permute_iterations(specs, exec_order):
     if exec_order not in permute_map:
         msg = 'Unknown execution order "{}"; must be in: {}'
-        raise ValueError(msg.format(exec_order, permute_map.keys()))
+        raise ValueError(msg.format(exec_order, list(permute_map.keys())))
     return permute_map[exec_order](specs)

wa/framework/configuration/parsers.py

@@ -21,6 +21,7 @@ from wa.framework.exception import ConfigError
 from wa.utils import log
 from wa.utils.serializer import json, read_pod, SerializerSyntaxError
 from wa.utils.types import toggle_set, counter
+from functools import reduce
 
 
 logger = logging.getLogger('config')
@@ -47,27 +48,27 @@ class ConfigParser(object):
             merge_augmentations(raw)
 
             # Get WA core configuration
-            for cfg_point in state.settings.configuration.itervalues():
+            for cfg_point in state.settings.configuration.values():
                 value = pop_aliased_param(cfg_point, raw)
                 if value is not None:
                     logger.debug('Setting meta "{}" to "{}"'.format(cfg_point.name, value))
                     state.settings.set(cfg_point.name, value)
 
             # Get run specific configuration
-            for cfg_point in state.run_config.configuration.itervalues():
+            for cfg_point in state.run_config.configuration.values():
                 value = pop_aliased_param(cfg_point, raw)
                 if value is not None:
                     logger.debug('Setting run "{}" to "{}"'.format(cfg_point.name, value))
                     state.run_config.set(cfg_point.name, value)
 
             # Get global job spec configuration
-            for cfg_point in JobSpec.configuration.itervalues():
+            for cfg_point in JobSpec.configuration.values():
                 value = pop_aliased_param(cfg_point, raw)
                 if value is not None:
                     logger.debug('Setting global "{}" to "{}"'.format(cfg_point.name, value))
                     state.jobs_config.set_global_value(cfg_point.name, value)
 
-            for name, values in raw.iteritems():
+            for name, values in raw.items():
                 # Assume that all leftover config is for a plug-in or a global
                 # alias it is up to PluginCache to assert this assumption
                 logger.debug('Caching "{}" with "{}"'.format(name, values))
@@ -106,7 +107,7 @@ class AgendaParser(object):
 
             if raw:
                 msg = 'Invalid top level agenda entry(ies): "{}"'
-                raise ConfigError(msg.format('", "'.join(raw.keys())))
+                raise ConfigError(msg.format('", "'.join(list(raw.keys()))))
 
             sect_ids, wkl_ids = self._collect_ids(sections, global_workloads)
             self._process_global_workloads(state, global_workloads, wkl_ids)
@@ -301,7 +302,7 @@ def _construct_valid_entry(raw, seen_ids, prefix, jobs_config):
     merge_augmentations(raw)
 
     # Validate all workload_entry
-    for name, cfg_point in JobSpec.configuration.iteritems():
+    for name, cfg_point in JobSpec.configuration.items():
         value = pop_aliased_param(cfg_point, raw)
         if value is not None:
             value = cfg_point.kind(value)
@@ -317,7 +318,7 @@ def _construct_valid_entry(raw, seen_ids, prefix, jobs_config):
     # error if there are unknown workload_entry
     if raw:
         msg = 'Invalid entry(ies) in "{}": "{}"'
-        raise ConfigError(msg.format(workload_entry['id'], ', '.join(raw.keys())))
+        raise ConfigError(msg.format(workload_entry['id'], ', '.join(list(raw.keys()))))
 
     return workload_entry
 
@@ -339,7 +340,7 @@ def _collect_valid_id(entry_id, seen_ids, entry_type):
 
 
 def _get_workload_entry(workload):
-    if isinstance(workload, basestring):
+    if isinstance(workload, str):
         workload = {'name': workload}
     elif not isinstance(workload, dict):
         raise ConfigError('Invalid workload entry: "{}"')

wa/framework/configuration/plugin_cache.py

@@ -90,11 +90,11 @@ class PluginCache(object):
             msg = 'configuration provided for unknown plugin "{}"'
             raise ConfigError(msg.format(plugin_name))
 
-        if not hasattr(values, 'iteritems'):
+        if not hasattr(values, 'items'):
             msg = 'Plugin configuration for "{}" not a dictionary ({} is {})'
             raise ConfigError(msg.format(plugin_name, repr(values), type(values)))
 
-        for name, value in values.iteritems():
+        for name, value in values.items():
             if (plugin_name not in GENERIC_CONFIGS and
                     name not in self.get_plugin_parameters(plugin_name)):
                 msg = "'{}' is not a valid parameter for '{}'"
@@ -124,7 +124,7 @@ class PluginCache(object):
             for source in self.sources:
                 if source not in plugin_config:
                     continue
-                for name, value in plugin_config[source].iteritems():
+                for name, value in plugin_config[source].items():
                     cfg_points[name].set_value(config, value=value)
         else:
             # A more complicated merge that involves priority of sources and
@@ -136,7 +136,7 @@ class PluginCache(object):
 
     def get_plugin(self, name, kind=None, *args, **kwargs):
         config = self.get_plugin_config(name)
-        kwargs = dict(config.items() + kwargs.items())
+        kwargs = dict(list(config.items()) + list(kwargs.items()))
         return self.loader.get_plugin(name, kind=kind, *args, **kwargs)
 
     def get_plugin_class(self, name, kind=None):
@@ -154,18 +154,18 @@ class PluginCache(object):
 
     def _set_plugin_defaults(self, plugin_name, config):
         cfg_points = self.get_plugin_parameters(plugin_name)
-        for cfg_point in cfg_points.itervalues():
+        for cfg_point in cfg_points.values():
             cfg_point.set_value(config, check_mandatory=False)
 
         try:
             _, alias_params = self.resolve_alias(plugin_name)
-            for name, value in alias_params.iteritems():
+            for name, value in alias_params.items():
                 cfg_points[name].set_value(config, value)
         except NotFoundError:
             pass
 
     def _set_from_global_aliases(self, plugin_name, config):
-        for alias, param in self._global_alias_map[plugin_name].iteritems():
+        for alias, param in self._global_alias_map[plugin_name].items():
             if alias in self.global_alias_values:
                 for source in self.sources:
                     if source not in self.global_alias_values[alias]:
@@ -230,7 +230,7 @@ class PluginCache(object):
 
         # Validate final configuration
         merged_config.name = specific_name
-        for cfg_point in ms.cfg_points.itervalues():
+        for cfg_point in ms.cfg_points.values():
             cfg_point.validate(merged_config, check_mandatory=is_final)
 
     def __getattr__(self, name):
@@ -285,7 +285,7 @@ class MergeState(object):
 def update_config_from_source(final_config, source, state):
     if source in state.generic_config:
         final_config.name = state.generic_name
-        for name, cfg_point in state.cfg_points.iteritems():
+        for name, cfg_point in state.cfg_points.items():
             if name in state.generic_config[source]:
                 if name in state.seen_specific_config:
                     msg = ('"{generic_name}" configuration "{config_name}" has '
@@ -307,7 +307,7 @@ def update_config_from_source(final_config, source, state):
 
     if source in state.specific_config:
         final_config.name = state.specific_name
-        for name, cfg_point in state.cfg_points.iteritems():
+        for name, cfg_point in state.cfg_points.items():
             if name in state.specific_config[source]:
                 state.seen_specific_config[name].append(str(source))
                 value = state.specific_config[source].pop(name)

wa/framework/configuration/tree.py

@@ -39,7 +39,7 @@ class JobSpecSource(object):
     def _log_self(self):
         logger.debug('Creating {} node'.format(self.kind))
         with log.indentcontext():
-            for key, value in self.config.iteritems():
+            for key, value in self.config.items():
                 logger.debug('"{}" to "{}"'.format(key, value))
 
 

wa/framework/exception.py

@@ -20,7 +20,11 @@ from wa.utils.misc import get_traceback
 
 class WAError(Exception):
     """Base class for all Workload Automation exceptions."""
-    pass
+    @property
+    def message(self):
+        if self.args:
+            return self.args[0]
+        return ''
 
 
 class NotFoundError(WAError):

wa/framework/execution.py

@@ -464,7 +464,7 @@ class Runner(object):
             self.logger.info('Skipping remaining jobs.')
             self.context.skip_remaining_jobs()
         except Exception as e:
-            message = e.message if e.message else str(e)
+            message = e.args[0] if e.args else str(e)
             log.log_error(e, self.logger)
             self.logger.error('Skipping remaining jobs due to "{}".'.format(e))
             self.context.skip_remaining_jobs()

wa/framework/getters.py

@@ -18,7 +18,7 @@
 This module contains the standard set of resource getters used by Workload Automation.
 
 """
-import httplib
+import http.client
 import json
 import logging
 import os
@@ -233,13 +233,17 @@ class Http(ResourceGetter):
             return {}
         index_url = urljoin(self.url, 'index.json')
         response = self.geturl(index_url)
-        if response.status_code != httplib.OK:
+        if response.status_code != http.client.OK:
             message = 'Could not fetch "{}"; recieved "{} {}"'
             self.logger.error(message.format(index_url,
                                              response.status_code,
                                              response.reason))
             return {}
-        return json.loads(response.content)
+        if sys.version_info[0] == 3:
+            content = response.content.decode('utf-8')
+        else:
+            content = response.content
+        return json.loads(content)
 
     def download_asset(self, asset, owner_name):
         url = urljoin(self.url, owner_name, asset['path'])
@@ -252,7 +256,7 @@ class Http(ResourceGetter):
                 return local_path
         self.logger.debug('Downloading {}'.format(url))
         response = self.geturl(url, stream=True)
-        if response.status_code != httplib.OK:
+        if response.status_code != http.client.OK:
             message = 'Could not download asset "{}"; recieved "{} {}"'
             self.logger.warning(message.format(url,
                                                response.status_code,
@@ -275,7 +279,7 @@ class Http(ResourceGetter):
         if not assets:
             return None
         asset_map = {a['path']: a for a in assets}
-        paths = get_path_matches(resource, asset_map.keys())
+        paths = get_path_matches(resource, list(asset_map.keys()))
         local_paths = []
         for path in paths:
             local_paths.append(self.download_asset(asset_map[path],
@@ -292,7 +296,7 @@ class Http(ResourceGetter):
 
         asset_map = {a['path']: a for a in assets}
         if resource.kind in ['jar', 'revent']:
-            path = get_generic_resource(resource, asset_map.keys())
+            path = get_generic_resource(resource, list(asset_map.keys()))
             if path:
                 return asset_map[path]
         elif resource.kind == 'executable':

wa/framework/host.py

@@ -90,7 +90,7 @@ def convert_wa2_agenda(filepath, output_path):
                             default=True),
             ])
 
-    for param in orig_agenda.keys():
+    for param in list(orig_agenda.keys()):
         for cfg_point in config_points:
             if param == cfg_point.name or param in cfg_point.aliases:
                 if cfg_point.name == 'augmentations':
@@ -105,7 +105,7 @@ def convert_wa2_agenda(filepath, output_path):
 
         # Convert plugin configuration
         output.write("# Plugin Configuration\n")
-        for param in orig_agenda.keys():
+        for param in list(orig_agenda.keys()):
             if pluginloader.has_plugin(param):
                 entry = {param: orig_agenda.pop(param)}
                 yaml.dump(format_parameter(entry), output, default_flow_style=False)
@@ -114,7 +114,7 @@ def convert_wa2_agenda(filepath, output_path):
         # Write any additional aliased parameters into new config
         plugin_cache = PluginCache()
         output.write("# Additional global aliases\n")
-        for param in orig_agenda.keys():
+        for param in list(orig_agenda.keys()):
             if plugin_cache.is_global_alias(param):
                 entry = {param: orig_agenda.pop(param)}
                 yaml.dump(format_parameter(entry), output, default_flow_style=False)
@@ -123,7 +123,7 @@ def convert_wa2_agenda(filepath, output_path):
 
 def format_parameter(param):
     if isinstance(param, dict):
-         return {identifier(k) : v for k, v in param.iteritems()}
+         return {identifier(k) : v for k, v in param.items()}
     else:
         return param
 

wa/framework/instrument.py

@@ -165,7 +165,7 @@ def priority(priority):
     def decorate(func):
         def wrapper(*args, **kwargs):
             return func(*args, **kwargs)
-        wrapper.func_name = func.func_name
+        wrapper.__name__ = func.__name__
         if priority in signal.CallbackPriority.levels:
             wrapper.priority = signal.CallbackPriority(priority)
         else:
@@ -255,7 +255,7 @@ class ManagedCallback(object):
                 global failures_detected  # pylint: disable=W0603
                 failures_detected = True
                 log_error(e, logger)
-                context.add_event(e.message)
+                context.add_event(e.args[0] if e.args else str(e))
                 if isinstance(e, WorkloadError):
                     context.set_status('FAILED')
                 elif isinstance(e, TargetError) or isinstance(e, TimeoutError):
@@ -268,7 +268,7 @@ class ManagedCallback(object):
 
     def __repr__(self):
         text = 'ManagedCallback({}, {})'
-        return text.format(self.instrument.name, self.callback.im_func.func_name)
+        return text.format(self.instrument.name, self.callback.__func__.__name__)
 
     __str__ = __repr__
 

wa/framework/job.py

@@ -85,7 +85,7 @@ class Job(object):
             enabled_instruments = set(i.name for i in instrument.get_enabled())
             enabled_output_processors = set(p.name for p in pm.get_enabled())
 
-            for augmentation in self.spec.augmentations.values():
+            for augmentation in list(self.spec.augmentations.values()):
                 augmentation_cls = context.cm.plugin_cache.get_plugin_class(augmentation)
                 if augmentation_cls.kind == 'instrument':
                     instruments_to_enable.add(augmentation)

wa/framework/output.py

@@ -10,7 +10,7 @@ from wa.framework.configuration.execution import CombinedConfig
 from wa.framework.exception import HostError
 from wa.framework.run import RunState, RunInfo
 from wa.framework.target.info import TargetInfo
-from wa.utils.misc import touch, ensure_directory_exists
+from wa.utils.misc import touch, ensure_directory_exists, isiterable
 from wa.utils.serializer import write_pod, read_pod, is_pod
 from wa.utils.types import enum, numeric
 
@@ -229,7 +229,7 @@ class RunOutput(Output):
         if os.path.isfile(self.jobsfile):
             self.job_specs = self.read_job_specs()
 
-        for job_state in self.state.jobs.itervalues():
+        for job_state in self.state.jobs.values():
             job_path = os.path.join(self.basepath, job_state.output_name)
             job = JobOutput(job_path, job_state.id,
                             job_state.label, job_state.iteration,
@@ -387,14 +387,14 @@ class Result(object):
         if key not in self.metadata:
             return self.add_metadata(key, *args)
 
-        if hasattr(self.metadata[key], 'iteritems'):
+        if hasattr(self.metadata[key], 'items'):
             if len(args) == 2:
                 self.metadata[key][args[0]] = args[1]
             elif len(args) > 2:  # assume list of key-value pairs
                 for k, v in args:
                     self.metadata[key][k] = v
-            elif hasattr(args[0], 'iteritems'):
+            elif hasattr(args[0], 'items'):
-                for k, v in args[0].iteritems():
+                for k, v in args[0].items():
                     self.metadata[key][k] = v
             else:
                 raise ValueError('Invalid value for key "{}": {}'.format(key, args))

wa/framework/plugin.py

@@ -25,6 +25,8 @@ from collections import OrderedDict, defaultdict
 from itertools import chain
 from copy import copy
 
+from future.utils import with_metaclass
+
 from wa.framework.configuration.core import settings, ConfigurationPoint as Parameter
 from wa.framework.exception import (NotFoundError, PluginLoaderError, TargetError,
                                     ValidationError, ConfigError, HostError)
@@ -34,7 +36,10 @@ from wa.utils.misc import (ensure_directory_exists as _d, walk_modules, load_cla
 from wa.utils.types import identifier
 
 
-MODNAME_TRANS = string.maketrans(':/\\.', '____')
+if sys.version_info[0] == 3:
+    MODNAME_TRANS = str.maketrans(':/\\.', '____')
+else:
+    MODNAME_TRANS = string.maketrans(':/\\.', '____')
 
 
 class AttributeCollection(object):
@@ -50,7 +55,7 @@ class AttributeCollection(object):
 
     @property
     def values(self):
-        return self._attrs.values()
+        return list(self._attrs.values())
 
     def __init__(self, attrcls):
         self._attrcls = attrcls
@@ -61,7 +66,7 @@ class AttributeCollection(object):
         if p.name in self._attrs:
             if p.override:
                 newp = copy(self._attrs[p.name])
-                for a, v in p.__dict__.iteritems():
+                for a, v in p.__dict__.items():
                     if v is not None:
                         setattr(newp, a, v)
                 if not hasattr(newp, "_overridden"):
@@ -77,7 +82,7 @@ class AttributeCollection(object):
     append = add
 
     def __str__(self):
-        return 'AC({})'.format(map(str, self._attrs.values()))
+        return 'AC({})'.format(list(map(str, list(self._attrs.values()))))
 
     __repr__ = __str__
 
@@ -212,14 +217,14 @@ class PluginMeta(type):
         if hasattr(cls, 'aliases'):
             aliases, cls.aliases = cls.aliases, AliasCollection()
             for alias in aliases:
-                if isinstance(alias, basestring):
+                if isinstance(alias, str):
                     alias = Alias(alias)
                 alias.validate(cls)
                 alias.plugin_name = cls.name
                 cls.aliases.add(alias)
 
 
-class Plugin(object):
+class Plugin(with_metaclass(PluginMeta, object)):
     """
     Base class for all WA plugins. An plugin is basically a plug-in.  It
     extends the functionality of WA in some way. Plugins are discovered and
@@ -230,7 +235,6 @@ class Plugin(object):
     ``~/.workload_automation/``.
 
     """
-    __metaclass__ = PluginMeta
 
     kind = None
     name = None
@@ -334,7 +338,7 @@ class Plugin(object):
     can = has
 
     def _load_module(self, loader, module_spec):
-        if isinstance(module_spec, basestring):
+        if isinstance(module_spec, str):
             name = module_spec
             params = {}
         elif isinstance(module_spec, dict):
@@ -342,7 +346,7 @@ class Plugin(object):
                 msg = 'Invalid module spec: {}; dict must have exctly one key -- '\
                       'the module name.'
                 raise ValueError(msg.format(module_spec))
-            name, params = module_spec.items()[0]
+            name, params = list(module_spec.items())[0]
         else:
             message = 'Invalid module spec: {}; must be a string or a one-key dict.'
             raise ValueError(message.format(module_spec))
@@ -491,7 +495,7 @@ class PluginLoader(object):
 
         """
         name, base_kwargs = self.resolve_alias(name)
-        kwargs = OrderedDict(chain(base_kwargs.iteritems(), kwargs.iteritems()))
+        kwargs = OrderedDict(chain(iter(base_kwargs.items()), iter(kwargs.items())))
         cls = self.get_plugin_class(name, kind)
         plugin = cls(*args, **kwargs)
         return plugin
@@ -514,10 +518,10 @@ class PluginLoader(object):
 
         """
         if kind is None:
-            return self.plugins.values()
+            return list(self.plugins.values())
         if kind not in self.kind_map:
             raise ValueError('Unknown plugin type: {}'.format(kind))
-        return self.kind_map[kind].values()
+        return list(self.kind_map[kind].values())
 
     def has_plugin(self, name, kind=None):
         """
@@ -625,7 +629,7 @@ class PluginLoader(object):
             modname = os.path.splitext(filepath[1:])[0].translate(MODNAME_TRANS)
             module = imp.load_source(modname, filepath)
             self._discover_in_module(module)
-        except (SystemExit, ImportError), e:
+        except (SystemExit, ImportError) as e:
             if self.keep_going:
                 self.logger.warning('Failed to load {}'.format(filepath))
                 self.logger.warning('Got: {}'.format(e))
@@ -639,7 +643,7 @@ class PluginLoader(object):
     def _discover_in_module(self, module):  # NOQA pylint: disable=too-many-branches
         self.logger.debug('Checking module %s', module.__name__)
         with log.indentcontext():
-            for obj in vars(module).itervalues():
+            for obj in vars(module).values():
                 if inspect.isclass(obj):
                     if not issubclass(obj, Plugin):
                         continue

wa/framework/pluginloader.py

@@ -21,7 +21,7 @@ class __LoaderWrapper(object):
     def kinds(self):
         if not self._loader:
             self.reset()
-        return self._loader.kind_map.keys()
+        return list(self._loader.kind_map.keys())
 
     @property
     def kind_map(self):

wa/framework/resource.py

@@ -287,7 +287,7 @@ def loose_version_matching(config_version, apk_version):
     if len(apk_version) < len(config_version):
         return False  # More specific version requested than available
 
-    for i in xrange(len(config_version)):
+    for i in range(len(config_version)):
         if config_version[i] != apk_version[i]:
             return False
     return True

wa/framework/run.py

@@ -76,7 +76,7 @@ class RunState(object):
 
     @property
     def num_completed_jobs(self):
-        return sum(1 for js in self.jobs.itervalues()
+        return sum(1 for js in self.jobs.values()
                    if js.status > Status.RUNNING)
 
     def __init__(self):
@@ -95,7 +95,7 @@ class RunState(object):
 
     def get_status_counts(self):
         counter = Counter()
-        for job_state in self.jobs.itervalues():
+        for job_state in self.jobs.values():
             counter[job_state.status] += 1
         return counter
 
@@ -103,7 +103,7 @@ class RunState(object):
         return OrderedDict(
             status=str(self.status),
             timestamp=self.timestamp,
-            jobs=[j.to_pod() for j in self.jobs.itervalues()],
+            jobs=[j.to_pod() for j in self.jobs.values()],
         )
 
 

wa/framework/target/descriptor.py

@@ -28,7 +28,7 @@ def list_target_descriptions(loader=pluginloader):
                 raise PluginLoaderError(msg.format(desc.name, prev_dtor.name,
                                                    descriptor.name))
             targets[desc.name] = desc
-    return targets.values()
+    return list(targets.values())
 
 
 def get_target_description(name, loader=pluginloader):
@@ -47,11 +47,11 @@ def instantiate_target(tdesc, params, connect=None, extra_platform_params=None):
     tp, pp, cp = {}, {}, {}
 
     for supported_params, new_params in (target_params, tp), (platform_params, pp), (conn_params, cp):
-        for name, value in supported_params.iteritems():
+        for name, value in supported_params.items():
             if value.default and name == value.name:
                 new_params[name] = value.default
 
-    for name, value in params.iteritems():
+    for name, value in params.items():
         if name in target_params:
             tp[name] = value
         elif name in platform_params:
@@ -64,7 +64,7 @@ def instantiate_target(tdesc, params, connect=None, extra_platform_params=None):
             msg = 'Unexpected parameter for {}: {}'
             raise ValueError(msg.format(tdesc.name, name))
 
-    for pname, pval in (extra_platform_params or {}).iteritems():
+    for pname, pval in (extra_platform_params or {}).items():
         if pname in pp:
             raise RuntimeError('Platform parameter clash: {}'.format(pname))
         pp[pname] = pval
@@ -121,7 +121,7 @@ class TargetDescription(object):
             vals = []
         elif isiterable(vals):
             if hasattr(vals, 'values'):
-                vals = v.values()
+                vals = list(v.values())
         else:
             msg = '{} must be iterable; got "{}"'
             raise ValueError(msg.format(attr, vals))
@@ -453,11 +453,11 @@ class DefaultTargetDescriptor(TargetDescriptor):
 
     def get_descriptions(self):
         result = []
-        for target_name, target_tuple in TARGETS.iteritems():
+        for target_name, target_tuple in TARGETS.items():
             (target, conn), target_params = self._get_item(target_tuple)
             assistant = ASSISTANTS[target_name]
             conn_params =  CONNECTION_PARAMS[conn]
-            for platform_name, platform_tuple in PLATFORMS.iteritems():
+            for platform_name, platform_tuple in PLATFORMS.items():
                 (platform, plat_conn), platform_params = self._get_item(platform_tuple)
                 name = '{}_{}'.format(platform_name, target_name)
                 td = TargetDescription(name, self)
@@ -484,11 +484,11 @@ class DefaultTargetDescriptor(TargetDescriptor):
             return cls, params
 
         param_map = OrderedDict((p.name, copy(p)) for p in params)
-        for name, value in defaults.iteritems():
+        for name, value in defaults.items():
             if name not in param_map:
                 raise ValueError('Unexpected default "{}"'.format(name))
             param_map[name].default = value
-        return cls, param_map.values()
+        return cls, list(param_map.values())
 
 
 
@@ -522,7 +522,7 @@ def create_target_description(name, *args, **kwargs):
 def _get_target_defaults(target):
     specificity = 0
     res = ('linux', TARGETS['linux'])  # fallback to a generic linux target
-    for name, ttup in TARGETS.iteritems():
+    for name, ttup in TARGETS.items():
         if issubclass(target, ttup[0][0]):
             new_spec =  len(inspect.getmro(ttup[0][0]))
             if new_spec > specificity:
@@ -540,7 +540,7 @@ def add_description_for_target(target, description=None, **kwargs):
     if 'platform' not in kwargs:
         kwargs['platform'] = Platform
     if 'platform_params' not in kwargs:
-        for (plat, conn), params, _ in PLATFORMS.itervalues():
+        for (plat, conn), params, _ in PLATFORMS.values():
             if plat == kwargs['platform']:
                 kwargs['platform_params'] = params
                 if conn is not None and kwargs['conn'] is None:

wa/framework/target/info.py

@@ -10,7 +10,7 @@ def cpuinfo_from_pod(pod):
     cpuinfo.sections = pod['cpuinfo']
     lines = []
     for section in cpuinfo.sections:
-        for key, value in section.iteritems():
+        for key, value in section.items():
             line = '{}: {}'.format(key, value)
             lines.append(line)
         lines.append('')
@@ -35,7 +35,7 @@ def kernel_config_from_pod(pod):
     config = KernelConfig('')
     config._config = pod['kernel_config']
     lines = []
-    for key, value in config._config.iteritems():
+    for key, value in config._config.items():
         if value == 'n':
             lines.append('# {} is not set'.format(key))
         else:

wa/framework/target/runtime_config.py

@@ -33,7 +33,7 @@ class RuntimeConfig(Plugin):
 
     @property
     def supported_parameters(self):
-        return self._runtime_params.values()
+        return list(self._runtime_params.values())
 
     @property
     def core_names(self):
@@ -166,12 +166,12 @@ class HotplugRuntimeConfig(RuntimeConfig):
 
     def validate_parameters(self):
         if len(self.num_cores) == self.target.number_of_cpus:
-            if all(v is False for v in self.num_cores.values()):
+            if all(v is False for v in list(self.num_cores.values())):
                 raise ValueError('Cannot set number of all cores to 0')
 
     def commit(self):
         '''Online all CPUs required in order before then off-lining'''
-        num_cores = sorted(self.num_cores.iteritems())
+        num_cores = sorted(self.num_cores.items())
         for cpu, online in num_cores:
             if online:
                 self.target.hotplug.online(cpu)
@@ -190,7 +190,7 @@ class SysfileValuesRuntimeConfig(RuntimeConfig):
     #pylint: disable=unused-argument
     @staticmethod
     def set_sysfile(obj, value, core):
-        for path, value in value.iteritems():
+        for path, value in value.items():
             verify = True
             if path.endswith('!'):
                 verify = False
@@ -222,7 +222,7 @@ class SysfileValuesRuntimeConfig(RuntimeConfig):
         return
 
     def commit(self):
-        for path, (value, verify) in self.sysfile_values.iteritems():
+        for path, (value, verify) in self.sysfile_values.items():
             self.target.write_value(path, value, verify=verify)
 
     def clear(self):
@@ -255,7 +255,7 @@ class FreqValue(object):
                 raise TargetError(msg.format(value))
         elif isinstance(value, int) and value in self.values:
             return value
-        elif isinstance(value, basestring):
+        elif isinstance(value, str):
             value = caseless_string(value)
             if value in ['min', 'max']:
                 return value
@@ -675,7 +675,7 @@ class IdleStateValue(object):
         if self.values is None:
             return value
 
-        if isinstance(value, basestring):
+        if isinstance(value, str):
             value = caseless_string(value)
             if value == 'all':
                 return [state[0] for state in self.values]

wa/framework/target/runtime_parameter_manager.py

@@ -39,7 +39,7 @@ class RuntimeParameterManager(object):
     def merge_runtime_parameters(self, parameters):
         merged_params = obj_dict()
         for source in parameters:
-            for name, value in parameters[source].iteritems():
+            for name, value in parameters[source].items():
                 cp = self.get_cfg_point(name)
                 cp.set_value(merged_params, value)
         return dict(merged_params)
@@ -60,7 +60,7 @@ class RuntimeParameterManager(object):
 
     # Stores a set of parameters performing isolated validation when appropriate
     def set_runtime_parameters(self, parameters):
-        for name, value in parameters.iteritems():
+        for name, value in parameters.items():
             cfg = self.get_config_for_name(name)
             if cfg is None:
                 msg = 'Unsupported runtime parameter: "{}"'
@@ -74,14 +74,14 @@ class RuntimeParameterManager(object):
 
     def get_config_for_name(self, name):
         name = caseless_string(name)
-        for k, v in self.runtime_params.iteritems():
+        for k, v in self.runtime_params.items():
             if name == k:
                 return v.rt_config
         return None
 
     def get_cfg_point(self, name):
         name = caseless_string(name)
-        for k, v in self.runtime_params.iteritems():
+        for k, v in self.runtime_params.items():
             if name == k:
                 return v.cfg_point
         raise ConfigError('Unknown runtime parameter: {}'.format(name))

wa/framework/version.py

@@ -14,6 +14,7 @@
 #
 
 import os
+import sys
 from collections import namedtuple
 from subprocess import Popen, PIPE
 
@@ -45,4 +46,7 @@ def get_commit():
     p.wait()
     if p.returncode:
         return None
-    return std[:8]
+    if sys.version_info[0] == 3:
+        return std[:8].decode(sys.stdout.encoding)
+    else:
+        return std[:8]

wa/instruments/dmesg.py

@@ -31,7 +31,7 @@ class DmesgInstrument(Instrument):
     name = 'dmesg'
 
     parameters = [
-        Parameter('loglevel', kind=int, allowed_values=range(8),
+        Parameter('loglevel', kind=int, allowed_values=list(range(8)),
                   description='Set loglevel for console output.')
     ]
 

wa/instruments/energy_measurement.py

@@ -15,7 +15,7 @@
 
 
 # pylint: disable=W0613,E1101
-from __future__ import division
+
 from collections import defaultdict
 import os
 import shutil
@@ -413,9 +413,9 @@ class EnergyMeasurement(Instrument):
         self.params = obj_dict()
 
         instrument_parameters = {identifier(k): v
-                                 for k, v in self.instrument_parameters.iteritems()}
+                                 for k, v in self.instrument_parameters.items()}
         supported_params = self.backend.get_parameters()
-        for name, param in supported_params.iteritems():
+        for name, param in supported_params.items():
             value = instrument_parameters.pop(name, None)
             param.set_value(self.params, value)
         if instrument_parameters:
@@ -426,7 +426,7 @@ class EnergyMeasurement(Instrument):
     def initialize(self, context):
         self.instruments = self.backend.get_instruments(self.target, context.run_output.metadir, **self.params)
 
-        for instrument in self.instruments.itervalues():
+        for instrument in self.instruments.values():
             if not (instrument.mode & CONTINUOUS):
                 msg = '{} instrument does not support continuous measurement collection'
                 raise ConfigError(msg.format(self.instrument))
@@ -436,26 +436,26 @@ class EnergyMeasurement(Instrument):
             # Check that the expeccted channels exist.
             # If there are multiple Instruments, they were all constructed with
             # the same channels param, so check them all.
-            for instrument in self.instruments.itervalues():
+            for instrument in self.instruments.values():
                 if not instrument.get_channels(channel):
                     raise ConfigError('No channels found for "{}"'.format(channel))
 
     def setup(self, context):
-        for instrument in self.instruments.itervalues():
+        for instrument in self.instruments.values():
             instrument.reset(sites=self.sites,
                              kinds=self.kinds,
                              channels=self.channels)
 
     def start(self, context):
-        for instrument in self.instruments.itervalues():
+        for instrument in self.instruments.values():
             instrument.start()
 
     def stop(self, context):
-        for instrument in self.instruments.itervalues():
+        for instrument in self.instruments.values():
             instrument.stop()
 
     def update_output(self, context):
-        for device, instrument in self.instruments.iteritems():
+        for device, instrument in self.instruments.items():
             # Append the device key to the filename and artifact name, unless
             # it's None (as it will be for backends with only 1
             # devce/instrument)
@@ -501,7 +501,7 @@ class EnergyMeasurement(Instrument):
         #  the devlib instrument, before we potentially appended a device key to
         #  it)
         if len(self.instruments) > 1:
-            for name, metrics in metrics_by_name.iteritems():
+            for name, metrics in metrics_by_name.items():
                 units = metrics[0].units
                 value = sum(m.value for m in metrics)
                 context.add_metric(name, value, units)

wa/instruments/hwmon.py

@@ -58,11 +58,11 @@ class HwmonInstrument(Instrument):
         measurements_before = {m.channel.label: m for m in self.before}
         measurements_after = {m.channel.label: m for m in self.after}
 
-        if measurements_before.keys() != measurements_after.keys():
+        if list(measurements_before.keys()) != list(measurements_after.keys()):
             self.logger.warning(
                 'hwmon before/after measurements returned different entries!')
 
-        for label, measurement_after in measurements_after.iteritems():
+        for label, measurement_after in measurements_after.items():
             if label not in measurements_before:
                 continue # We've already warned about this
             measurement_before = measurements_before[label]

wa/instruments/misc.py

@@ -30,16 +30,17 @@ import re
 import logging
 import time
 import tarfile
-from itertools import izip, izip_longest
 from subprocess import CalledProcessError
 
-from devlib.exception import TargetError
+from future.moves.itertools import zip_longest
 
+from devlib.exception import TargetError
 from devlib.utils.android import ApkInfo
 
 from wa import Instrument, Parameter, very_fast
 from wa.framework.exception import ConfigError
 from wa.framework.instrument import slow
+from wa.utils.diff import diff_sysfs_dirs, diff_interrupt_files
 from wa.utils.misc import as_relative, diff_tokens, write_table
 from wa.utils.misc import ensure_file_directory_exists as _f
 from wa.utils.misc import ensure_directory_exists as _d
@@ -112,7 +113,7 @@ class SysfsExtractor(Instrument):
             _d(os.path.join(context.output_directory, 'diff', self._local_dir(d)))
             for d in self.paths
         ]
-        self.device_and_host_paths = zip(self.paths, before_dirs, after_dirs, diff_dirs)
+        self.device_and_host_paths = list(zip(self.paths, before_dirs, after_dirs, diff_dirs))
 
         if self.use_tmpfs:
             for d in self.paths:
@@ -177,7 +178,7 @@ class SysfsExtractor(Instrument):
                 self.logger.error('sysfs files were not pulled from the device.')
                 self.device_and_host_paths.remove(paths)  # Path is removed to skip diffing it
         for _, before_dir, after_dir, diff_dir in self.device_and_host_paths:
-            _diff_sysfs_dirs(before_dir, after_dir, diff_dir)
+            diff_sysfs_dirs(before_dir, after_dir, diff_dir)
 
     def teardown(self, context):
         self._one_time_setup_done = []
@@ -280,7 +281,7 @@ class InterruptStatsInstrument(Instrument):
     def update_output(self, context):
         # If workload execution failed, the after_file may not have been created.
         if os.path.isfile(self.after_file):
-            _diff_interrupt_files(self.before_file, self.after_file, _f(self.diff_file))
+            diff_interrupt_files(self.before_file, self.after_file, _f(self.diff_file))
 
 
 class DynamicFrequencyInstrument(SysfsExtractor):
@@ -307,83 +308,3 @@ class DynamicFrequencyInstrument(SysfsExtractor):
         super(DynamicFrequencyInstrument, self).validate()
         if not self.tmpfs_mount_point.endswith('-cpufreq'):  # pylint: disable=access-member-before-definition
             self.tmpfs_mount_point += '-cpufreq'
-
-
-def _diff_interrupt_files(before, after, result):  # pylint: disable=R0914
-    output_lines = []
-    with open(before) as bfh:
-        with open(after) as ofh:
-            for bline, aline in izip(bfh, ofh):
-                bchunks = bline.strip().split()
-                while True:
-                    achunks = aline.strip().split()
-                    if achunks[0] == bchunks[0]:
-                        diffchunks = ['']
-                        diffchunks.append(achunks[0])
-                        diffchunks.extend([diff_tokens(b, a) for b, a
-                                           in zip(bchunks[1:], achunks[1:])])
-                        output_lines.append(diffchunks)
-                        break
-                    else:  # new category appeared in the after file
-                        diffchunks = ['>'] + achunks
-                        output_lines.append(diffchunks)
-                        try:
-                            aline = ofh.next()
-                        except StopIteration:
-                            break
-
-    # Offset heading columns by one to allow for row labels on subsequent
-    # lines.
-    output_lines[0].insert(0, '')
-
-    # Any "columns" that do not have headings in the first row are not actually
-    # columns -- they are a single column where space-spearated words got
-    # split. Merge them back together to prevent them from being
-    # column-aligned by write_table.
-    table_rows = [output_lines[0]]
-    num_cols = len(output_lines[0])
-    for row in output_lines[1:]:
-        table_row = row[:num_cols]
-        table_row.append(' '.join(row[num_cols:]))
-        table_rows.append(table_row)
-
-    with open(result, 'w') as wfh:
-        write_table(table_rows, wfh)
-
-
-def _diff_sysfs_dirs(before, after, result):  # pylint: disable=R0914
-    before_files = []
-    os.path.walk(before,
-                 lambda arg, dirname, names: arg.extend([os.path.join(dirname, f) for f in names]),
-                 before_files
-                 )
-    before_files = filter(os.path.isfile, before_files)
-    files = [os.path.relpath(f, before) for f in before_files]
-    after_files = [os.path.join(after, f) for f in files]
-    diff_files = [os.path.join(result, f) for f in files]
-
-    for bfile, afile, dfile in zip(before_files, after_files, diff_files):
-        if not os.path.isfile(afile):
-            logger.debug('sysfs_diff: {} does not exist or is not a file'.format(afile))
-            continue
-
-        with open(bfile) as bfh, open(afile) as afh:  # pylint: disable=C0321
-            with open(_f(dfile), 'w') as dfh:
-                for i, (bline, aline) in enumerate(izip_longest(bfh, afh), 1):
-                    if aline is None:
-                        logger.debug('Lines missing from {}'.format(afile))
-                        break
-                    bchunks = re.split(r'(\W+)', bline)
-                    achunks = re.split(r'(\W+)', aline)
-                    if len(bchunks) != len(achunks):
-                        logger.debug('Token length mismatch in {} on line {}'.format(bfile, i))
-                        dfh.write('xxx ' + bline)
-                        continue
-                    if ((len([c for c in bchunks if c.strip()]) == len([c for c in achunks if c.strip()]) == 2) and
-                            (bchunks[0] == achunks[0])):
-                        # if there are only two columns and the first column is the
-                        # same, assume it's a "header" column and do not diff it.
-                        dchunks = [bchunks[0]] + [diff_tokens(b, a) for b, a in zip(bchunks[1:], achunks[1:])]
-                    else:
-                        dchunks = [diff_tokens(b, a) for b, a in zip(bchunks, achunks)]
-                    dfh.write(''.join(dchunks))

wa/instruments/trace_cmd.py

@@ -15,7 +15,7 @@
 
 
 # pylint: disable=W0613,E1101
-from __future__ import division
+
 import os
 
 from devlib import FtraceCollector

wa/output_processors/cpustates.py

@@ -14,9 +14,10 @@
 #
 
 import os
-import csv
 from collections import OrderedDict
 
+from devlib.utils.csvutil import csvwriter
+
 from wa import OutputProcessor, Parameter
 from wa.utils.types import list_of_strings
 from wa.utils.cpustates import report_power_stats
@@ -29,14 +30,14 @@ def _get_cpustates_description():
     """
     output_lines = []
     lines = iter(report_power_stats.__doc__.split('\n'))
-    line = lines.next()
+    line = next(lines)
     while True:
         try:
             if line.strip().startswith(':param'):
                 while line.strip():
-                    line = lines.next()
+                    line = next(lines)
             output_lines.append(line)
-            line = lines.next()
+            line = next(lines)
         except StopIteration:
             break
     return '\n'.join(output_lines)
@@ -105,7 +106,7 @@ class CpuStatesProcessor(OutputProcessor):
             split_wfi_states=self.split_wfi_states,
         )
 
-        for report in reports.itervalues():
+        for report in reports.values():
             output.add_artifact(report.name, report.filepath, kind='data')
 
         iteration_id = (output.id, output.label, output.iteration)
@@ -118,7 +119,7 @@ class CpuStatesProcessor(OutputProcessor):
 
         parallel_rows = []
         powerstate_rows = []
-        for iteration_id, reports in self.iteration_reports.iteritems():
+        for iteration_id, reports in self.iteration_reports.items():
             job_id, workload, iteration = iteration_id
             parallel_report = reports['parallel-stats']
             powerstate_report = reports['power-state-stats']
@@ -132,8 +133,7 @@ class CpuStatesProcessor(OutputProcessor):
                                            for s in stats])
 
         outpath = output.get_path('parallel-stats.csv')
-        with open(outpath, 'w') as wfh:
+        with csvwriter(outpath) as writer:
-            writer = csv.writer(wfh)
             writer.writerow(['id', 'workload', 'iteration', 'cluster',
                              'number_of_cores', 'total_time',
                              '%time', '%running_time'])
@@ -141,8 +141,7 @@ class CpuStatesProcessor(OutputProcessor):
         output.add_artifact('run-parallel-stats', outpath, kind='export')
 
         outpath = output.get_path('power-state-stats.csv')
-        with open(outpath, 'w') as wfh:
+        with csvwriter(outpath) as writer:
-            writer = csv.writer(wfh)
             headers = ['id', 'workload', 'iteration', 'state']
             headers += ['{} CPU{}'.format(c, i)
                         for i, c in enumerate(powerstate_report.core_names)]

wa/output_processors/csvproc.py

@@ -1,4 +1,6 @@
-import csv
+import sys
+
+from devlib.utils.csvutil import csvwriter
 
 from wa import OutputProcessor, Parameter
 from wa.framework.exception import ConfigError
@@ -64,7 +66,7 @@ class CsvReportProcessor(OutputProcessor):
             classifiers = set([])
             for out in outputs:
                 for metric in out.metrics:
-                    classifiers.update(metric.classifiers.keys())
+                    classifiers.update(list(metric.classifiers.keys()))
             extra_columns = list(classifiers)
         elif self.extra_columns:
             extra_columns = self.extra_columns
@@ -72,8 +74,7 @@ class CsvReportProcessor(OutputProcessor):
             extra_columns = []
 
         outfile = output.get_path('results.csv')
-        with open(outfile, 'wb') as wfh:
+        with csvwriter(outfile) as writer:
-            writer = csv.writer(wfh)
             writer.writerow(['id', 'workload', 'iteration', 'metric', ] +
                             extra_columns + ['value', 'units'])
 

wa/output_processors/status.py

@@ -49,8 +49,8 @@ class StatusTxtReporter(OutputProcessor):
             txt = '{}/{} iterations completed without error\n'
             wfh.write(txt.format(counter[Status.OK], len(output.jobs)))
             wfh.write('\n')
-            status_lines = [map(str, [o.id, o.label, o.iteration, o.status,
+            status_lines = [list(map(str, [o.id, o.label, o.iteration, o.status,
-                                      o.event_summary])
+                                      o.event_summary]))
                             for o in output.jobs]
             write_table(status_lines, wfh, align='<<>><')
 

wa/utils/cpustates.py

@@ -13,16 +13,17 @@
 # limitations under the License.
 #
 
-from __future__ import division
+
 import os
 import sys
-import csv
 import re
 import logging
 from ctypes import c_int32
 from collections import defaultdict
 import argparse
 
+from devlib.utils.csvutil import create_writer, csvwriter
+
 from wa.utils.trace_cmd import TraceCmdParser, trace_has_marker, TRACE_MARKER_START, TRACE_MARKER_STOP
 
 
@@ -114,7 +115,7 @@ class SystemPowerState(object):
         self.timestamp = None
         self.cpus = []
         idle_state = -1 if no_idle else None
-        for _ in xrange(num_cores):
+        for _ in range(num_cores):
             self.cpus.append(CpuPowerState(idle_state=idle_state))
 
     def copy(self):
@@ -331,8 +332,7 @@ class PowerStateTransitions(object):
 
     def __init__(self, output_directory):
         self.filepath = os.path.join(output_directory, 'state-transitions-timeline.csv')
-        self._wfh = open(self.filepath, 'w')
+        self.writer, self._wfh = create_writer(self.filepath)
-        self.writer = csv.writer(self._wfh)
         headers = ['timestamp', 'cpu_id', 'frequency', 'idle_state']
         self.writer.writerow(headers)
 
@@ -360,8 +360,7 @@ class PowerStateTimeline(object):
     def __init__(self, output_directory, cpus):
         self.filepath = os.path.join(output_directory, 'power-state-timeline.csv')
         self.idle_state_names = {cpu.id: [s.name for s in cpu.cpuidle.states] for cpu in cpus}
-        self._wfh = open(self.filepath, 'w')
+        self.writer, self._wfh = create_writer(self.filepath)
-        self.writer = csv.writer(self._wfh)
         headers = ['ts'] + ['{} CPU{}'.format(cpu.name, cpu.id) for cpu in cpus]
         self.writer.writerow(headers)
 
@@ -405,7 +404,7 @@ class ParallelStats(object):
                 clusters.append(cpu.cpufreq.related_cpus)
 
         for i, clust in enumerate(clusters):
-            self.clusters[i] = set(clust)
+            self.clusters[str(i)] = set(clust)
         self.clusters['all'] = set([cpu.id for cpu in cpus])
 
         self.first_timestamp = None
@@ -419,7 +418,7 @@ class ParallelStats(object):
             delta = timestamp - self.last_timestamp
             active_cores = [i for i, c in enumerate(self.previous_states)
                             if c and c[0] == -1]
-            for cluster, cluster_cores in self.clusters.iteritems():
+            for cluster, cluster_cores in self.clusters.items():
                 clust_active_cores = len(cluster_cores.intersection(active_cores))
                 self.parallel_times[cluster][clust_active_cores] += delta
                 if clust_active_cores:
@@ -438,7 +437,7 @@ class ParallelStats(object):
         total_time = self.last_timestamp - self.first_timestamp
         for cluster in sorted(self.parallel_times):
             running_time = self.running_times[cluster]
-            for n in xrange(len(self.clusters[cluster]) + 1):
+            for n in range(len(self.clusters[cluster]) + 1):
                 time = self.parallel_times[cluster][n]
                 time_pc = time / total_time
                 if not self.use_ratios:
@@ -474,8 +473,7 @@ class ParallelReport(object):
         self.values.append(value)
 
     def write(self):
-        with open(self.filepath, 'w') as wfh:
+        with csvwriter(self.filepath) as writer:
-            writer = csv.writer(wfh)
             writer.writerow(['cluster', 'number_of_cores', 'total_time', '%time', '%running_time'])
             writer.writerows(self.values)
 
@@ -520,7 +518,7 @@ class PowerStateStats(object):
         total_time = self.last_timestamp - self.first_timestamp
         state_stats = defaultdict(lambda: [None] * len(self.core_names))
 
-        for cpu, states in self.cpu_states.iteritems():
+        for cpu, states in self.cpu_states.items():
             for state in states:
                 time = states[state]
                 time_pc = time / total_time
@@ -543,8 +541,7 @@ class PowerStateStatsReport(object):
         self.precision = precision
 
     def write(self):
-        with open(self.filepath, 'w') as wfh:
+        with csvwriter(self.filepath) as writer:
-            writer = csv.writer(wfh)
             headers = ['state'] + ['{} CPU{}'.format(c, i)
                                    for i, c in enumerate(self.core_names)]
             writer.writerow(headers)
@@ -561,8 +558,7 @@ class CpuUtilizationTimeline(object):
 
     def __init__(self, output_directory, cpus):
         self.filepath = os.path.join(output_directory, 'utilization-timeline.csv')
-        self._wfh = open(self.filepath, 'w')
+        self.writer, self._wfh = create_writer(self.filepath)
-        self.writer = csv.writer(self._wfh)
 
         headers = ['ts'] + ['{} CPU{}'.format(cpu.name, cpu.id) for cpu in cpus]
         self.writer.writerow(headers)

wa/utils/diff.py

@@ -1,11 +1,18 @@
-from wa.utils.misc import write_table
+import os
+import re
+
+from future.moves.itertools import zip_longest
+
+from wa.utils.misc import as_relative, diff_tokens, write_table
+from wa.utils.misc import ensure_file_directory_exists as _f
+from wa.utils.misc import ensure_directory_exists as _d
 
 
 def diff_interrupt_files(before, after, result):  # pylint: disable=R0914
     output_lines = []
     with open(before) as bfh:
         with open(after) as ofh:
-            for bline, aline in izip(bfh, ofh):
+            for bline, aline in zip(bfh, ofh):
                 bchunks = bline.strip().split()
                 while True:
                     achunks = aline.strip().split()
@@ -20,7 +27,7 @@ def diff_interrupt_files(before, after, result):  # pylint: disable=R0914
                         diffchunks = ['>'] + achunks
                         output_lines.append(diffchunks)
                         try:
-                            aline = ofh.next()
+                            aline = next(ofh)
                         except StopIteration:
                             break
 
@@ -45,11 +52,9 @@ def diff_interrupt_files(before, after, result):  # pylint: disable=R0914
 
 def diff_sysfs_dirs(before, after, result):  # pylint: disable=R0914
     before_files = []
-    os.path.walk(before,
+    for root, dirs, files in os.walk(before):
-                 lambda arg, dirname, names: arg.extend([os.path.join(dirname, f) for f in names]),
+        before_files.extend([os.path.join(root, f) for f in files])
-                 before_files
+    before_files = list(filter(os.path.isfile, before_files))
-                 )
-    before_files = filter(os.path.isfile, before_files)
     files = [os.path.relpath(f, before) for f in before_files]
     after_files = [os.path.join(after, f) for f in files]
     diff_files = [os.path.join(result, f) for f in files]
@@ -61,7 +66,7 @@ def diff_sysfs_dirs(before, after, result):  # pylint: disable=R0914
 
         with open(bfile) as bfh, open(afile) as afh:  # pylint: disable=C0321
             with open(_f(dfile), 'w') as dfh:
-                for i, (bline, aline) in enumerate(izip_longest(bfh, afh), 1):
+                for i, (bline, aline) in enumerate(zip_longest(bfh, afh), 1):
                     if aline is None:
                         logger.debug('Lines missing from {}'.format(afile))
                         break

wa/utils/doc.py

@@ -164,16 +164,16 @@ def format_simple_table(rows, headers=None, align='>', show_borders=True, border
     """Formats a simple table."""
     if not rows:
         return ''
-    rows = [map(str, r) for r in rows]
+    rows = [list(map(str, r)) for r in rows]
     num_cols = len(rows[0])
 
     # cycle specified alignments until we have num_cols of them. This is
     # consitent with how such cases are handled in R, pandas, etc.
     it = cycle(align)
-    align = [it.next() for _ in xrange(num_cols)]
+    align = [next(it) for _ in range(num_cols)]
 
-    cols = zip(*rows)
+    cols = list(zip(*rows))
-    col_widths = [max(map(len, c)) for c in cols]
+    col_widths = [max(list(map(len, c))) for c in cols]
     if headers:
         col_widths = [max(len(h), cw) for h, cw in zip(headers, col_widths)]
     row_format = ' '.join(['{:%s%s}' % (align[i], w) for i, w in enumerate(col_widths)])
@@ -259,12 +259,12 @@ def indent(text, spaces=4):
 
 
 def format_literal(lit):
-    if isinstance(lit, basestring):
+    if isinstance(lit, str):
         return '``\'{}\'``'.format(lit)
     elif hasattr(lit, 'pattern'):  # regex
         return '``r\'{}\'``'.format(lit.pattern)
     elif isinstance(lit, dict):
-        content = indent(',\n'.join("{}: {}".format(key,val) for (key,val) in lit.iteritems()))
+        content = indent(',\n'.join("{}: {}".format(key,val) for (key,val) in lit.items()))
         return '::\n\n{}'.format(indent('{{\n{}\n}}'.format(content)))
     else:
         return '``{}``'.format(lit)
@@ -287,7 +287,7 @@ def get_params_rst(parameters):
             text += indent('\nconstraint: ``{}``\n'.format(get_type_name(param.constraint)))
         if param.default:
             value = param.default
-            if isinstance(value, basestring) and value.startswith(USER_HOME):
+            if isinstance(value, str) and value.startswith(USER_HOME):
                 value = value.replace(USER_HOME, '~')
             text += indent('\ndefault: {}\n'.format(format_literal(value)))
         text += '\n'
@@ -298,7 +298,7 @@ def get_aliases_rst(aliases):
     text = ''
     for alias in aliases:
         param_str = ', '.join(['{}={}'.format(n, format_literal(v))
-                               for n, v in alias.params.iteritems()])
+                               for n, v in alias.params.items()])
         text += '{}\n{}\n\n'.format(alias.name, indent(param_str))
     return text
 

wa/utils/exec_control.py

@@ -12,7 +12,7 @@ def activate_environment(name):
     #pylint: disable=W0603
     global __active_environment
 
-    if name not in __environments.keys():
+    if name not in list(__environments.keys()):
         init_environment(name)
     __active_environment = name
 
@@ -24,7 +24,7 @@ def init_environment(name):
     :raises: ``ValueError`` if an environment with name ``name``
              already exists.
     """
-    if name in __environments.keys():
+    if name in list(__environments.keys()):
         msg = "Environment {} already exists".format(name)
         raise ValueError(msg)
     __environments[name] = []
@@ -39,7 +39,7 @@ def reset_environment(name=None):
     """
 
     if name is not None:
-        if name not in __environments.keys():
+        if name not in list(__environments.keys()):
             msg = "Environment {} does not exist".format(name)
             raise ValueError(msg)
         __environments[name] = []
@@ -75,7 +75,7 @@ def once_per_class(method):
         if __active_environment is None:
             activate_environment('default')
 
-        func_id = repr(method.func_name) + repr(args[0].__class__)
+        func_id = repr(method.__name__) + repr(args[0].__class__)
 
         if func_id in __environments[__active_environment]:
             return

wa/utils/log.py

@@ -128,13 +128,13 @@ def disable(logs):
 
 
 def __enable_logger(logger):
-    if isinstance(logger, basestring):
+    if isinstance(logger, str):
         logger = logging.getLogger(logger)
     logger.propagate = True
 
 
 def __disable_logger(logger):
-    if isinstance(logger, basestring):
+    if isinstance(logger, str):
         logger = logging.getLogger(logger)
     logger.propagate = False
 

wa/utils/misc.py

@@ -18,7 +18,7 @@
 Miscellaneous functions that don't fit anywhere else.
 
 """
-from __future__ import division
+
 import os
 import sys
 import re
@@ -30,9 +30,13 @@ import traceback
 import logging
 import random
 import hashlib
+import sys
 from datetime import datetime, timedelta
 from operator import mul
-from StringIO import StringIO
+if sys.version_info[0] == 3:
+    from io import StringIO
+else:
+    from io import BytesIO as StringIO
 from itertools import chain, cycle
 from distutils.spawn import find_executable
 
@@ -83,11 +87,11 @@ def diff_tokens(before_token, after_token):
 def prepare_table_rows(rows):
     """Given a list of lists, make sure they are prepared to be formatted into a table
     by making sure each row has the same number of columns and stringifying all values."""
-    rows = [map(str, r) for r in rows]
+    rows = [list(map(str, r)) for r in rows]
-    max_cols = max(map(len, rows))
+    max_cols = max(list(map(len, rows)))
     for row in rows:
         pad = max_cols - len(row)
-        for _ in xrange(pad):
+        for _ in range(pad):
             row.append('')
     return rows
 
@@ -102,10 +106,10 @@ def write_table(rows, wfh, align='>', headers=None):  # pylint: disable=R0914
     # cycle specified alignments until we have max_cols of them. This is
     # consitent with how such cases are handled in R, pandas, etc.
     it = cycle(align)
-    align = [it.next() for _ in xrange(num_cols)]
+    align = [next(it) for _ in range(num_cols)]
 
-    cols = zip(*rows)
+    cols = list(zip(*rows))
-    col_widths = [max(map(len, c)) for c in cols]
+    col_widths = [max(list(map(len, c))) for c in cols]
     row_format = ' '.join(['{:%s%s}' % (align[i], w) for i, w in enumerate(col_widths)])
     row_format += '\n'
 
@@ -144,7 +148,7 @@ def _check_remove_item(the_list, item):
     """Helper function for merge_lists that implements checking wether an items
     should be removed from the list and doing so if needed. Returns ``True`` if
     the item has been removed and ``False`` otherwise."""
-    if not isinstance(item, basestring):
+    if not isinstance(item, str):
         return False
     if not item.startswith('~'):
         return False
@@ -275,7 +279,7 @@ def get_article(word):
 
 def get_random_string(length):
     """Returns a random ASCII string of the specified length)."""
-    return ''.join(random.choice(string.ascii_letters + string.digits) for _ in xrange(length))
+    return ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(length))
 
 
 class LoadSyntaxError(Exception):
@@ -307,9 +311,9 @@ def load_struct_from_python(filepath=None, text=None):
             while modname in sys.modules:  # highly unlikely, but...
                 modname = get_random_string(RAND_MOD_NAME_LEN)
             mod = imp.new_module(modname)
-            exec text in mod.__dict__  # pylint: disable=exec-used
+            exec(text, mod.__dict__)  # pylint: disable=exec-used
         return dict((k, v)
-                    for k, v in mod.__dict__.iteritems()
+                    for k, v in mod.__dict__.items()
                     if not k.startswith('_'))
     except SyntaxError as e:
         raise LoadSyntaxError(e.message, filepath, e.lineno)
@@ -404,7 +408,7 @@ def istextfile(fileobj, blocksize=512):
 def categorize(v):
     if hasattr(v, 'merge_with') and hasattr(v, 'merge_into'):
         return 'o'
-    elif hasattr(v, 'iteritems'):
+    elif hasattr(v, 'items'):
         return 'm'
     elif isiterable(v):
         return 's'
@@ -515,13 +519,14 @@ def merge_sequencies(s1, s2):
     return type(s2)(unique(chain(s1, s2)))
 
 
+
 def merge_maps(m1, m2):
-    return type(m2)(chain(m1.iteritems(), m2.iteritems()))
+    return type(m2)(chain(iter(m1.items()), iter(m2.items())))
 
 
 def merge_dicts_simple(base, other):
     result = base.copy()
-    for key, value in (other or {}).iteritems():
+    for key, value in (other or {}).items():
         result[key] = merge_config_values(result.get(key), value)
     return result
 
@@ -534,11 +539,11 @@ def touch(path):
 def get_object_name(obj):
     if hasattr(obj, 'name'):
         return obj.name
-    elif hasattr(obj, 'im_func'):
+    elif hasattr(obj, '__func__') and hasattr(obj, '__self__'):
-        return '{}.{}'.format(get_object_name(obj.im_class),
+        return '{}.{}'.format(get_object_name(obj.__self__.__class__),
-                              obj.im_func.func_name)
+                              obj.__func__.__name__)
     elif hasattr(obj, 'func_name'):
-        return obj.func_name
+        return obj.__name__
     elif hasattr(obj, '__name__'):
         return obj.__name__
     elif hasattr(obj, '__class__'):
@@ -557,7 +562,7 @@ def resolve_cpus(name, target):
         - 'all' - returns all cpus
         - '' - Empty name will also return all cpus
     """
-    cpu_list = range(target.number_of_cpus)
+    cpu_list = list(range(target.number_of_cpus))
 
     # Support for passing cpu no directly
     if isinstance(name, int):

wa/utils/revent.py

@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 
-from __future__ import division
+
 import os
 import struct
 import signal
@@ -88,7 +88,7 @@ class UinputDeviceInfo(object):
         self.abs_bits = bytearray(parts[3])
         self.num_absinfo = parts[4]
         self.absinfo = [absinfo(*read_struct(fh, absinfo_struct))
-                        for _ in xrange(self.num_absinfo)]
+                        for _ in range(self.num_absinfo)]
 
     def __str__(self):
         return 'UInputInfo({})'.format(self.__dict__)
@@ -145,7 +145,7 @@ class ReventRecording(object):
             if self.stream:
                 events = self._iter_events()
                 try:
-                    first = last = events.next()
+                    first = last = next(events)
                 except StopIteration:
                     self._duration = 0
                 for last in events:
@@ -230,7 +230,7 @@ class ReventRecording(object):
 
     def _read_devices(self, fh):
         num_devices, = read_struct(fh, u32_struct)
-        for _ in xrange(num_devices):
+        for _ in range(num_devices):
             self.device_paths.append(read_string(fh))
 
     def _read_gamepad_info(self, fh):
@@ -243,7 +243,7 @@ class ReventRecording(object):
             raise RuntimeError(msg)
         self.fh.seek(self._events_start)
         if self.version >= 2:
-            for _ in xrange(self.num_events):
+            for _ in range(self.num_events):
                 yield ReventEvent(self.fh)
         else:
             file_size = os.path.getsize(self.filepath)

wa/utils/serializer.py

@@ -71,7 +71,7 @@ POD_TYPES = [
     dict,
     set,
     str,
-    unicode,
+    str,
     int,
     float,
     bool,
@@ -104,7 +104,7 @@ class WAJSONDecoder(_json.JSONDecoder):
         d = _json.JSONDecoder.decode(self, s, **kwargs)
 
         def try_parse_object(v):
-            if isinstance(v, basestring):
+            if isinstance(v, str):
                 if v.startswith('REGEX:'):
                     _, flags, pattern = v.split(':', 2)
                     return re.compile(pattern, int(flags or 0))
@@ -122,8 +122,8 @@ class WAJSONDecoder(_json.JSONDecoder):
 
         def load_objects(d):
             pairs = []
-            for k, v in d.iteritems():
+            for k, v in d.items():
-                if hasattr(v, 'iteritems'):
+                if hasattr(v, 'items'):
                     pairs.append((k, load_objects(v)))
                 elif isiterable(v):
                     pairs.append((k, [try_parse_object(i) for i in v]))
@@ -160,13 +160,13 @@ class json(object):
 
 
 _mapping_tag = _yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG
-_regex_tag = u'tag:wa:regex'
+_regex_tag = 'tag:wa:regex'
-_level_tag = u'tag:wa:level'
+_level_tag = 'tag:wa:level'
-_cpu_mask_tag = u'tag:wa:cpu_mask'
+_cpu_mask_tag = 'tag:wa:cpu_mask'
 
 
 def _wa_dict_representer(dumper, data):
-    return dumper.represent_mapping(_mapping_tag, data.iteritems())
+    return dumper.represent_mapping(_mapping_tag, iter(data.items()))
 
 
 def _wa_regex_representer(dumper, data):
@@ -248,17 +248,17 @@ class python(object):
     def loads(s, *args, **kwargs):
         pod = {}
         try:
-            exec s in pod  # pylint: disable=exec-used
+            exec(s, pod)  # pylint: disable=exec-used
         except SyntaxError as e:
             raise SerializerSyntaxError(e.message, e.lineno)
-        for k in pod.keys():
+        for k in list(pod.keys()):
             if k.startswith('__'):
                 del pod[k]
         return pod
 
 
 def read_pod(source, fmt=None):
-    if isinstance(source, basestring):
+    if isinstance(source, str):
         with open(source) as fh:
             return _read_pod(fh, fmt)
     elif hasattr(source, 'read') and (hasattr(source, 'name') or fmt):
@@ -269,7 +269,7 @@ def read_pod(source, fmt=None):
 
 
 def write_pod(pod, dest, fmt=None):
-    if isinstance(dest, basestring):
+    if isinstance(dest, str):
         with open(dest, 'w') as wfh:
             return _write_pod(pod, wfh, fmt)
     elif hasattr(dest, 'write') and (hasattr(dest, 'name') or fmt):
@@ -323,8 +323,8 @@ def _write_pod(pod, wfh, fmt=None):
 def is_pod(obj):
     if  type(obj) not in POD_TYPES:
         return False
-    if hasattr(obj, 'iteritems'):
+    if hasattr(obj, 'items'):
-        for k, v in obj.iteritems():
+        for k, v in obj.items():
             if not (is_pod(k) and is_pod(v)):
                 return False
     elif isiterable(obj):

wa/utils/terminalsize.py

@@ -89,5 +89,5 @@ def _get_terminal_size_linux():
 
 if __name__ == "__main__":
     sizex, sizey = get_terminal_size()
-    print 'width =', sizex, 'height =', sizey
+    print('width =', sizex, 'height =', sizey)
 

wa/utils/trace_cmd.py

@@ -114,7 +114,7 @@ class DroppedEventsEvent(object):
 def try_convert_to_numeric(v):
     try:
         if isiterable(v):
-            return map(numeric, v)
+            return list(map(numeric, v))
         else:
             return numeric(v)
     except ValueError:
@@ -153,13 +153,13 @@ def regex_body_parser(regex, flags=0):
     If regex is a pre-compiled object, flags will be ignored.
 
     """
-    if isinstance(regex, basestring):
+    if isinstance(regex, str):
         regex = re.compile(regex, flags)
 
     def regex_parser_func(event, text):
         match = regex.search(text)
         if match:
-            for k, v in match.groupdict().iteritems():
+            for k, v in match.groupdict().items():
                 try:
                     event.fields[k] = int(v)
                 except ValueError:
@@ -321,7 +321,7 @@ class TraceCmdParser(object):
                         continue
 
                 body_parser = EVENT_PARSER_MAP.get(event_name, default_body_parser)
-                if isinstance(body_parser, basestring) or isinstance(body_parser, re._pattern_type):  # pylint: disable=protected-access
+                if isinstance(body_parser, str) or isinstance(body_parser, re._pattern_type):  # pylint: disable=protected-access
                     body_parser = regex_body_parser(body_parser)
                 yield TraceCmdEvent(parser=body_parser, **match.groupdict())
 

wa/utils/types.py

@@ -29,10 +29,19 @@ import os
 import re
 import numbers
 import shlex
+import sys
 from bisect import insort
-from urllib import quote, unquote
+if sys.version_info[0] == 3:
+    from urllib.parse import quote, unquote
+    from past.builtins import basestring
+    long = int
+else:
+    from urllib import quote, unquote
 from collections import defaultdict, MutableMapping
 from copy import copy
+from functools import total_ordering
+
+from future.utils import with_metaclass
 
 from devlib.utils.types import identifier, boolean, integer, numeric, caseless_string
 
@@ -47,7 +56,7 @@ def list_of_strs(value):
     """
     if not isiterable(value):
         raise ValueError(value)
-    return map(str, value)
+    return list(map(str, value))
 
 list_of_strings = list_of_strs
 
@@ -59,7 +68,7 @@ def list_of_ints(value):
     """
     if not isiterable(value):
         raise ValueError(value)
-    return map(int, value)
+    return list(map(int, value))
 
 list_of_integers = list_of_ints
 
@@ -72,7 +81,7 @@ def list_of_numbers(value):
     """
     if not isiterable(value):
         raise ValueError(value)
-    return map(numeric, value)
+    return list(map(numeric, value))
 
 
 def list_of_bools(value, interpret_strings=True):
@@ -88,9 +97,9 @@ def list_of_bools(value, interpret_strings=True):
     if not isiterable(value):
         raise ValueError(value)
     if interpret_strings:
-        return map(boolean, value)
+        return list(map(boolean, value))
     else:
-        return map(bool, value)
+        return list(map(bool, value))
 
 
 def list_of(type_):
@@ -98,16 +107,16 @@ def list_of(type_):
     attempts to convert all elements in the passed value to the specifed
     ``type_``, raising ``ValueError`` on error."""
     def __init__(self, values):
-        list.__init__(self, map(type_, values))
+        list.__init__(self, list(map(type_, values)))
 
     def append(self, value):
         list.append(self, type_(value))
 
     def extend(self, other):
-        list.extend(self, map(type_, other))
+        list.extend(self, list(map(type_, other)))
 
     def from_pod(cls, pod):
-        return cls(map(type_, pod))
+        return cls(list(map(type_, pod)))
 
     def _to_pod(self):
         return self
@@ -132,7 +141,7 @@ def list_or_string(value):
     a one-element list with stringified value will be returned.
 
     """
-    if isinstance(value, basestring):
+    if isinstance(value, str):
         return [value]
     else:
         try:
@@ -147,11 +156,11 @@ def list_or_caseless_string(value):
     not iterable a one-element list with stringified value will be returned.
 
     """
-    if isinstance(value, basestring):
+    if isinstance(value, str):
         return [caseless_string(value)]
     else:
         try:
-            return map(caseless_string, value)
+            return list(map(caseless_string, value))
         except ValueError:
             return [caseless_string(value)]
 
@@ -229,8 +238,8 @@ class arguments(list):
 
     def __init__(self, value=None):
         if isiterable(value):
-            super(arguments, self).__init__(map(str, value))
+            super(arguments, self).__init__(list(map(str, value)))
-        elif isinstance(value, basestring):
+        elif isinstance(value, str):
             posix = os.name != 'nt'
             super(arguments, self).__init__(shlex.split(value, posix=posix))
         elif value is None:
@@ -242,7 +251,7 @@ class arguments(list):
         return super(arguments, self).append(str(value))
 
     def extend(self, values):
-        return super(arguments, self).extend(map(str, values))
+        return super(arguments, self).extend(list(map(str, values)))
 
     def __str__(self):
         return ' '.join(self)
@@ -288,7 +297,7 @@ class prioritylist(object):
         self.__delitem__(index)
 
     def _priority_index(self, element):
-        for priority, elements in self.elements.iteritems():
+        for priority, elements in self.elements.items():
             if element in elements:
                 return (priority, elements.index(element))
         raise IndexError(element)
@@ -333,7 +342,7 @@ class prioritylist(object):
             else:
                 index_range = [index]
         elif isinstance(index, slice):
-            index_range = range(index.start or 0, index.stop, index.step or 1)
+            index_range = list(range(index.start or 0, index.stop, index.step or 1))
         else:
             raise ValueError('Invalid index {}'.format(index))
         current_global_offset = 0
@@ -391,7 +400,7 @@ class toggle_set(set):
     def __init__(self, *args):
         if args:
             value = args[0]
-            if isinstance(value, basestring):
+            if isinstance(value, str):
                 msg = 'invalid type for toggle_set: "{}"'
                 raise TypeError(msg.format(type(value)))
         set.__init__(self, *args)
@@ -507,12 +516,15 @@ class obj_dict(MutableMapping):
             raise AttributeError("No such attribute: " + name)
 
     def __getattr__(self, name):
+        if 'dict' not in self.__dict__:
+            raise AttributeError("No such attribute: " + name)
         if name in self.__dict__['dict']:
             return self.__dict__['dict'][name]
         else:
             raise AttributeError("No such attribute: " + name)
 
 
+@total_ordering
 class level(object):
     """
     A level has a name and behaves like a string when printed, however it also
@@ -538,11 +550,8 @@ class level(object):
     def __repr__(self):
         return '{}({})'.format(self.name, self.value)
 
-    def __cmp__(self, other):
+    def __hash__(self):
-        if isinstance(other, level):
+        return hash(self.name)
-            return cmp(self.value, other.value)
-        else:
-            return cmp(self.value, other)
 
     def __eq__(self, other):
         if isinstance(other, level):
@@ -552,13 +561,24 @@ class level(object):
         else:
             return self.value == other
 
-    def __ne__(self, other):
+    def __lt__(self, other):
         if isinstance(other, level):
-            return self.value != other.value
+            return self.value < other.value
         elif isinstance(other, basestring):
-            return self.name != other
+            return self.name < other
         else:
-            return self.value != other
+            return self.value < other
+
+
+class _EnumMeta(type):
+
+    def __str__(cls):
+        return str(cls.levels)
+
+    def __getattr__(self, name):
+        name = name.lower()
+        if name in self.__dict__:
+                return self.__dict__[name]
 
 
 def enum(args, start=0, step=1):
@@ -583,11 +603,7 @@ def enum(args, start=0, step=1):
 
     """
 
-    class Enum(object):
+    class Enum(with_metaclass(_EnumMeta, object)):
-
-        class __metaclass__(type):
-            def __str__(cls):
-                return str(cls.levels)
 
         @classmethod
         def from_pod(cls, pod):
@@ -642,14 +658,14 @@ class ParameterDict(dict):
     # Function to determine the appropriate prefix based on the parameters type
     @staticmethod
     def _get_prefix(obj):
-        if isinstance(obj, basestring):
+        if isinstance(obj, str):
             prefix = 's'
         elif isinstance(obj, float):
             prefix = 'f'
-        elif isinstance(obj, long):
-            prefix = 'd'
         elif isinstance(obj, bool):
             prefix = 'b'
+        elif isinstance(obj, long):
+            prefix = 'i'
         elif isinstance(obj, int):
             prefix = 'i'
         elif obj is None:
@@ -686,7 +702,7 @@ class ParameterDict(dict):
             elif value_type == 'b':
                 return boolean(value)
             elif value_type == 'd':
-                return long(value)
+                return int(value)
             elif value_type == 'f':
                 return float(value)
             elif value_type == 'i':
@@ -700,7 +716,7 @@ class ParameterDict(dict):
             raise ValueError('Unknown {} {}'.format(type(string), string))
 
     def __init__(self, *args, **kwargs):
-        for k, v in kwargs.iteritems():
+        for k, v in kwargs.items():
             self.__setitem__(k, v)
         dict.__init__(self, *args)
 
@@ -714,7 +730,7 @@ class ParameterDict(dict):
         return dict.__contains__(self, self._encode(item))
 
     def __iter__(self):
-        return iter((k, self._decode(v)) for (k, v) in self.items())
+        return iter((k, self._decode(v)) for (k, v) in list(self.items()))
 
     def iteritems(self):
         return self.__iter__()
@@ -730,7 +746,10 @@ class ParameterDict(dict):
         return (key, self._decode(value))
 
     def iter_encoded_items(self):
-        return dict.iteritems(self)
+        if sys.version_info[0] == 3:
+            return dict.items(self)
+        else:
+            return dict.iteritems(self)
 
     def get_encoded_value(self, name):
         return dict.__getitem__(self, name)
@@ -743,7 +762,7 @@ class ParameterDict(dict):
             if isinstance(d, ParameterDict):
                 dict.update(self, d)
             else:
-                for k, v in d.iteritems():
+                for k, v in d.items():
                     self[k] = v
 
 
@@ -762,7 +781,7 @@ class cpu_mask(object):
         self._mask = 0
         if isinstance(cpus, int):
             self._mask = cpus
-        elif isinstance(cpus, basestring):
+        elif isinstance(cpus, str):
             if cpus[:2] == '0x' or cpus[:2] == '0X':
                 self._mask = int(cpus, 16)
             else:

wa/workloads/exoplayer/__init__.py

@@ -18,7 +18,13 @@
 import re
 import os
 import time
-import urllib
+
+from future.standard_library import install_aliases
+install_aliases()
+
+import urllib.request
+import urllib.parse
+import urllib.error
 
 from wa import ApkWorkload, Parameter, ConfigError, WorkloadError
 from wa.framework.configuration.core import settings
@@ -81,7 +87,7 @@ class ExoPlayer(ApkWorkload):
                   Playback duration of the video file. This becomes the duration of the workload.
                   If provided must be shorter than the length of the media.
                   """),
-        Parameter('format', allowed_values=DOWNLOAD_URLS.keys(),
+        Parameter('format', allowed_values=list(DOWNLOAD_URLS.keys()),
                   description="""
                   Specifies which format video file to play. Default is {}
                   """.format(default_format)),
@@ -137,7 +143,7 @@ class ExoPlayer(ApkWorkload):
                 filename = '{}_{}'.format(format_resolution, os.path.basename(url))
                 filepath = os.path.join(self.video_directory, filename)
                 self.logger.info('Downloading {} to {}...'.format(url, filepath))
-                urllib.urlretrieve(url, filepath)
+                urllib.request.urlretrieve(url, filepath)
                 return filepath
             else:
                 if len(files) > 1:
@@ -172,7 +178,7 @@ class ExoPlayer(ApkWorkload):
         self.play_cmd = 'am start -a {} -d "file://{}"'.format(self.action,
                                                                self.device_video_file)
 
-        self.monitor = self.target.get_logcat_monitor(REGEXPS.values())
+        self.monitor = self.target.get_logcat_monitor(list(REGEXPS.values()))
         self.monitor.start()
 
     def run(self, context):

wa/workloads/hackbench/__init__.py

@@ -84,7 +84,7 @@ class Hackbench(Workload):
         results_file = context.get_artifact_path('hackbench-results')
         with open(results_file) as fh:
             for line in fh:
-                for label, (regex, units) in regex_map.iteritems():
+                for label, (regex, units) in regex_map.items():
                     match = regex.search(line)
                     if match:
                         context.add_metric(label, float(match.group(1)), units)

wa/workloads/jankbench/__init__.py

@@ -15,7 +15,7 @@
 
 # pylint: disable=E1101,W0201,E0203
 
-from __future__ import division
+
 import os
 import re
 import select
@@ -23,6 +23,7 @@ import json
 import threading
 import sqlite3
 import subprocess
+import sys
 from copy import copy
 
 import pandas as pd
@@ -143,7 +144,7 @@ class Jankbench(ApkWorkload):
 
             for test_name, rep in results.index:
                 test_results = results.ix[test_name, rep]
-                for metric, value in test_results.iteritems():
+                for metric, value in test_results.items():
                     context.add_metric(metric, value, units=None, lower_is_better=True,
                                        classifiers={'test_name': test_name, 'rep': rep})
 
@@ -222,6 +223,8 @@ class JankbenchRunMonitor(threading.Thread):
                 ready, _, _ = select.select([proc.stdout, proc.stderr], [], [], 2)
                 if ready:
                     line = ready[0].readline()
+                    if sys.version_info[0] == 3:
+                        line = line.decode(sys.stdout.encoding)
                     if self.regex.search(line):
                         self.run_ended.set()
 

wa/workloads/meabo/__init__.py

@@ -145,7 +145,7 @@ class Meabo(Workload):
             Controls which phases to run.
             ''',
             constraint=lambda x: all(0 < v <=10 for v in x),
-            default=range(1, 11),
+            default=list(range(1, 11)),
         ),
         Parameter(
             'threads',

wa/workloads/openssl/__init__.py

@@ -102,7 +102,7 @@ class Openssl(Workload):
 
             parts =  line.split(':')
             if parts[0] == '+F':  # evp ciphers
-                for bs, value  in zip(BLOCK_SIZES, map(float, parts[3:])):
+                for bs, value  in zip(BLOCK_SIZES, list(map(float, parts[3:]))):
                     value = value / 2**20  # to MB
                     context.add_metric('score', value, 'MB/s',
                                        classifiers={'block_size': bs})

wa/workloads/sysbench/__init__.py

@@ -135,16 +135,16 @@ class Sysbench(Workload):
 
         with open(self.host_results_file) as fh:
             find_line_with('General statistics:', fh)
-            extract_metric('total time', fh.next(), context.output)
+            extract_metric('total time', next(fh), context.output)
-            extract_metric('total number of events', fh.next(), context.output, lower_is_better=False)
+            extract_metric('total number of events', next(fh), context.output, lower_is_better=False)
             find_line_with('response time:', fh)
-            extract_metric('min', fh.next(), context.output, 'response time ')
+            extract_metric('min', next(fh), context.output, 'response time ')
-            extract_metric('avg', fh.next(), context.output, 'response time ')
+            extract_metric('avg', next(fh), context.output, 'response time ')
-            extract_metric('max', fh.next(), context.output, 'response time ')
+            extract_metric('max', next(fh), context.output, 'response time ')
-            extract_metric('approx.  95 percentile', fh.next(), context.output)
+            extract_metric('approx.  95 percentile', next(fh), context.output)
             find_line_with('Threads fairness:', fh)
-            extract_threads_fairness_metric('events', fh.next(), context.output)
+            extract_threads_fairness_metric('events', next(fh), context.output)
-            extract_threads_fairness_metric('execution time', fh.next(), context.output)
+            extract_threads_fairness_metric('execution time', next(fh), context.output)
 
     def teardown(self, context):
         self.target.remove(self.target_results_file)
@@ -155,7 +155,7 @@ class Sysbench(Workload):
 
     def _build_command(self, **parameters):
         param_strings = ['--{}={}'.format(k.replace('_', '-'), v)
-                         for k, v in parameters.iteritems()]
+                         for k, v in parameters.items()]
         if self.file_test_mode:
             param_strings.append('--file-test-mode={}'.format(self.file_test_mode))
         sysbench_command = '{} {} {} run'.format(self.target_binary, ' '.join(param_strings), self.cmd_params)

wa/workloads/vellamo/__init__.py

@@ -17,7 +17,7 @@ import os
 import json
 import re
 
-from HTMLParser import HTMLParser
+from html.parser import HTMLParser
 
 from wa import ApkUiautoWorkload, Parameter
 from wa.utils.types import list_of_strs
@@ -48,7 +48,7 @@ class Vellamo(ApkUiautoWorkload):
         '3.0': ['Browser', 'Metal', 'Multi'],
         '3.2.4': ['Browser', 'Metal', 'Multi'],
     }
-    valid_versions = benchmark_types.keys()
+    valid_versions = list(benchmark_types.keys())
     summary_metrics = None
 
     parameters = [
@@ -119,7 +119,7 @@ class Vellamo(ApkUiautoWorkload):
                     benchmark.name = benchmark.name.replace(' ', '_')
                     context.add_metric('{}_Total'.format(benchmark.name),
                                                                 benchmark.score)
-                    for name, score in benchmark.metrics.items():
+                    for name, score in list(benchmark.metrics.items()):
                         name = name.replace(' ', '_')
                         context.add_metric('{}_{}'.format(benchmark.name,
                                                                  name), score)