Add support for Python 3

Add support for running under Python 3, while maintaining compatibility
with Python 2.

See http://python-future.org/compatible_idioms.html for more details
behind these changes.

wa/instruments/misc.py

@@ -30,16 +30,17 @@ import re
 import logging
 import time
 import tarfile
-from itertools import izip, izip_longest
 from subprocess import CalledProcessError
 
-from devlib.exception import TargetError
+from future.moves.itertools import zip_longest
 
+from devlib.exception import TargetError
 from devlib.utils.android import ApkInfo
 
 from wa import Instrument, Parameter, very_fast
 from wa.framework.exception import ConfigError
 from wa.framework.instrument import slow
+from wa.utils.diff import diff_sysfs_dirs, diff_interrupt_files
 from wa.utils.misc import as_relative, diff_tokens, write_table
 from wa.utils.misc import ensure_file_directory_exists as _f
 from wa.utils.misc import ensure_directory_exists as _d
@@ -112,7 +113,7 @@ class SysfsExtractor(Instrument):
             _d(os.path.join(context.output_directory, 'diff', self._local_dir(d)))
             for d in self.paths
         ]
-        self.device_and_host_paths = zip(self.paths, before_dirs, after_dirs, diff_dirs)
+        self.device_and_host_paths = list(zip(self.paths, before_dirs, after_dirs, diff_dirs))
 
         if self.use_tmpfs:
             for d in self.paths:
@@ -177,7 +178,7 @@ class SysfsExtractor(Instrument):
                 self.logger.error('sysfs files were not pulled from the device.')
                 self.device_and_host_paths.remove(paths)  # Path is removed to skip diffing it
         for _, before_dir, after_dir, diff_dir in self.device_and_host_paths:
-            _diff_sysfs_dirs(before_dir, after_dir, diff_dir)
+            diff_sysfs_dirs(before_dir, after_dir, diff_dir)
 
     def teardown(self, context):
         self._one_time_setup_done = []
@@ -280,7 +281,7 @@ class InterruptStatsInstrument(Instrument):
     def update_output(self, context):
         # If workload execution failed, the after_file may not have been created.
         if os.path.isfile(self.after_file):
-            _diff_interrupt_files(self.before_file, self.after_file, _f(self.diff_file))
+            diff_interrupt_files(self.before_file, self.after_file, _f(self.diff_file))
 
 
 class DynamicFrequencyInstrument(SysfsExtractor):
@@ -307,83 +308,3 @@ class DynamicFrequencyInstrument(SysfsExtractor):
         super(DynamicFrequencyInstrument, self).validate()
         if not self.tmpfs_mount_point.endswith('-cpufreq'):  # pylint: disable=access-member-before-definition
             self.tmpfs_mount_point += '-cpufreq'
-
-
-def _diff_interrupt_files(before, after, result):  # pylint: disable=R0914
-    output_lines = []
-    with open(before) as bfh:
-        with open(after) as ofh:
-            for bline, aline in izip(bfh, ofh):
-                bchunks = bline.strip().split()
-                while True:
-                    achunks = aline.strip().split()
-                    if achunks[0] == bchunks[0]:
-                        diffchunks = ['']
-                        diffchunks.append(achunks[0])
-                        diffchunks.extend([diff_tokens(b, a) for b, a
-                                           in zip(bchunks[1:], achunks[1:])])
-                        output_lines.append(diffchunks)
-                        break
-                    else:  # new category appeared in the after file
-                        diffchunks = ['>'] + achunks
-                        output_lines.append(diffchunks)
-                        try:
-                            aline = ofh.next()
-                        except StopIteration:
-                            break
-
-    # Offset heading columns by one to allow for row labels on subsequent
-    # lines.
-    output_lines[0].insert(0, '')
-
-    # Any "columns" that do not have headings in the first row are not actually
-    # columns -- they are a single column where space-spearated words got
-    # split. Merge them back together to prevent them from being
-    # column-aligned by write_table.
-    table_rows = [output_lines[0]]
-    num_cols = len(output_lines[0])
-    for row in output_lines[1:]:
-        table_row = row[:num_cols]
-        table_row.append(' '.join(row[num_cols:]))
-        table_rows.append(table_row)
-
-    with open(result, 'w') as wfh:
-        write_table(table_rows, wfh)
-
-
-def _diff_sysfs_dirs(before, after, result):  # pylint: disable=R0914
-    before_files = []
-    os.path.walk(before,
-                 lambda arg, dirname, names: arg.extend([os.path.join(dirname, f) for f in names]),
-                 before_files
-                 )
-    before_files = filter(os.path.isfile, before_files)
-    files = [os.path.relpath(f, before) for f in before_files]
-    after_files = [os.path.join(after, f) for f in files]
-    diff_files = [os.path.join(result, f) for f in files]
-
-    for bfile, afile, dfile in zip(before_files, after_files, diff_files):
-        if not os.path.isfile(afile):
-            logger.debug('sysfs_diff: {} does not exist or is not a file'.format(afile))
-            continue
-
-        with open(bfile) as bfh, open(afile) as afh:  # pylint: disable=C0321
-            with open(_f(dfile), 'w') as dfh:
-                for i, (bline, aline) in enumerate(izip_longest(bfh, afh), 1):
-                    if aline is None:
-                        logger.debug('Lines missing from {}'.format(afile))
-                        break
-                    bchunks = re.split(r'(\W+)', bline)
-                    achunks = re.split(r'(\W+)', aline)
-                    if len(bchunks) != len(achunks):
-                        logger.debug('Token length mismatch in {} on line {}'.format(bfile, i))
-                        dfh.write('xxx ' + bline)
-                        continue
-                    if ((len([c for c in bchunks if c.strip()]) == len([c for c in achunks if c.strip()]) == 2) and
-                            (bchunks[0] == achunks[0])):
-                        # if there are only two columns and the first column is the
-                        # same, assume it's a "header" column and do not diff it.
-                        dchunks = [bchunks[0]] + [diff_tokens(b, a) for b, a in zip(bchunks[1:], achunks[1:])]
-                    else:
-                        dchunks = [diff_tokens(b, a) for b, a in zip(bchunks, achunks)]
-                    dfh.write(''.join(dchunks))