Add support for Python 3

Add support for running on Python 3 while maintaining Python 2
compatibility.

devlib/utils/android.py

@@ -27,7 +27,8 @@ import logging
 import re
 import threading
 import tempfile
-import Queue
+import queue
+import sys
 from collections import defaultdict
 
 from devlib.exception import TargetError, HostError, DevlibError
@@ -88,7 +89,7 @@ class AndroidProperties(object):
         self._properties = dict(re.findall(r'\[(.*?)\]:\s+\[(.*?)\]', text))
 
     def iteritems(self):
-        return self._properties.iteritems()
+        return iter(self._properties.items())
 
     def __iter__(self):
         return iter(self._properties)
@@ -140,6 +141,8 @@ class ApkInfo(object):
         logger.debug(' '.join(command))
         try:
             output = subprocess.check_output(command, stderr=subprocess.STDOUT)
+            if sys.version_info[0] == 3:
+                output = output.decode(sys.stdout.encoding)
         except subprocess.CalledProcessError as e:
             raise HostError('Error parsing APK file {}. `aapt` says:\n{}'
                             .format(apk_path, e.output))
@@ -160,7 +163,7 @@ class ApkInfo(object):
                 mapped_abis = []
                 for apk_abi in apk_abis:
                     found = False
-                    for abi, architectures in ABI_MAP.iteritems():
+                    for abi, architectures in ABI_MAP.items():
                         if apk_abi in architectures:
                             mapped_abis.append(abi)
                             found = True

devlib/utils/csvutil.py

@@ -0,0 +1,85 @@
+'''
+Due to the change in the nature of "binary mode" when opening files in
+Python 3, the way files need to be opened for ``csv.reader`` and ``csv.writer``
+is different from Python 2.
+
+The functions in this module are intended to hide these differences allowing
+the rest of the code to create csv readers/writers without worrying about which
+Python version it is running under.
+
+First up are ``csvwriter`` and ``csvreader`` context mangers that handle the
+opening and closing of the underlying file. These are intended to replace the
+most common usage pattern
+
+.. code-block:: python
+
+    with open(filepath, 'wb') as wfh:  # or open(filepath, 'w', newline='') in Python 3
+        writer = csv.writer(wfh)
+        writer.writerows(data)
+
+
+with
+
+.. code-block:: python
+
+    with csvwriter(filepath) as writer:
+        writer.writerows(data)
+
+
+``csvreader`` works in an analogous way. ``csvreader`` and ``writer`` can take
+additional arguments which will be passed directly to the
+``csv.reader``/``csv.writer`` calls.
+
+In some cases, it is desirable not to use a context manager (e.g. if the
+reader/writer is intended to be returned from the function that creates it. For
+such cases, alternative functions, ``create_reader`` and ``create_writer``,
+exit. These return a two-tuple, with the created reader/writer as the first
+element, and the corresponding ``FileObject`` as the second. It is the
+responsibility of the calling code to ensure that the file is closed properly.
+
+'''
+import csv
+import sys
+from contextlib import contextmanager
+
+
+@contextmanager
+def csvwriter(filepath, *args, **kwargs):
+    if sys.version_info[0] == 3:
+        wfh = open(filepath, 'w', newline='')
+    else:
+        wfh = open(filepath, 'wb')
+
+    try:
+        yield csv.writer(wfh, *args, **kwargs)
+    finally:
+        wfh.close()
+
+
+@contextmanager
+def csvreader(filepath, *args, **kwargs):
+    if sys.version_info[0] == 3:
+        fh = open(filepath, 'r', newline='')
+    else:
+        fh = open(filepath, 'rb')
+
+    try:
+        yield csv.reader(fh, *args, **kwargs)
+    finally:
+        fh.close()
+
+
+def create_writer(filepath, *args, **kwargs):
+    if sys.version_info[0] == 3:
+        wfh = open(filepath, 'w', newline='')
+    else:
+        wfh = open(filepath, 'wb')
+    return csv.writer(wfh, *args, **kwargs), wfh
+
+
+def create_reader(filepath, *args, **kwargs):
+    if sys.version_info[0] == 3:
+        fh = open(filepath, 'r', newline='')
+    else:
+        fh = open(filepath, 'rb')
+    return csv.reader(fh, *args, **kwargs), fh

devlib/utils/gem5.py

@@ -45,7 +45,7 @@ def iter_statistics_dump(stats_file):
                 k = res.group("key")
                 vtext = res.group("value")
                 try:
-                    v = map(numeric, vtext.split())
+                    v = list(map(numeric, vtext.split()))
                     cur_dump[k] = v[0] if len(v)==1 else set(v)
                 except ValueError:
                     msg = 'Found non-numeric entry in gem5 stats ({}: {})'

devlib/utils/misc.py

@@ -36,8 +36,10 @@ from itertools import groupby
 from functools import partial
 
 import wrapt
+from past.builtins import basestring
 
 from devlib.exception import HostError, TimeoutError
+from functools import reduce
 
 
 # ABI --> architectures list
@@ -176,6 +178,9 @@ def check_output(command, timeout=None, ignore=None, inputtext=None, **kwargs):
 
     try:
         output, error = process.communicate(inputtext)
+        if sys.version_info[0] == 3:
+            output = output.decode(sys.stdout.encoding)
+            error =  error.decode(sys.stderr.encoding)
     finally:
         if timeout:
             timer.cancel()
@@ -185,7 +190,7 @@ def check_output(command, timeout=None, ignore=None, inputtext=None, **kwargs):
         if retcode == -9:  # killed, assume due to timeout callback
             raise TimeoutError(command, output='\n'.join([output, error]))
         elif ignore != 'all' and retcode not in ignore:
-            raise subprocess.CalledProcessError(retcode, command, output='\n'.join([output, error]))
+            raise subprocess.CalledProcessError(retcode, command, output='\n'.join([str(output), str(error)]))
     return output, error
 
 
@@ -257,8 +262,8 @@ def _merge_two_dicts(base, other, list_duplicates='all', match_types=False,  # p
                      dict_type=dict, should_normalize=True, should_merge_lists=True):
     """Merge dicts normalizing their keys."""
     merged = dict_type()
-    base_keys = base.keys()
-    other_keys = other.keys()
+    base_keys = list(base.keys())
+    other_keys = list(other.keys())
     norm = normalize if should_normalize else lambda x, y: x
 
     base_only = []
@@ -390,7 +395,7 @@ def normalize(value, dict_type=dict):
     no surrounding whitespace, underscore-delimited strings."""
     if isinstance(value, dict):
         normalized = dict_type()
-        for k, v in value.iteritems():
+        for k, v in value.items():
             key = k.strip().lower().replace(' ', '_')
             normalized[key] = normalize(v, dict_type)
         return normalized
@@ -431,7 +436,7 @@ def getch(count=1):
     """Read ``count`` characters from standard input."""
     if os.name == 'nt':
         import msvcrt  # pylint: disable=F0401
-        return ''.join([msvcrt.getch() for _ in xrange(count)])
+        return ''.join([msvcrt.getch() for _ in range(count)])
     else:  # assume Unix
         import tty  # NOQA
         import termios  # NOQA
@@ -509,7 +514,7 @@ def strip_bash_colors(text):
 
 def get_random_string(length):
     """Returns a random ASCII string of the specified length)."""
-    return ''.join(random.choice(string.ascii_letters + string.digits) for _ in xrange(length))
+    return ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(length))
 
 
 class LoadSyntaxError(Exception):
@@ -526,7 +531,10 @@ class LoadSyntaxError(Exception):
 
 RAND_MOD_NAME_LEN = 30
 BAD_CHARS = string.punctuation + string.whitespace
-TRANS_TABLE = string.maketrans(BAD_CHARS, '_' * len(BAD_CHARS))
+if sys.version_info[0] == 3:
+    TRANS_TABLE = str.maketrans(BAD_CHARS, '_' * len(BAD_CHARS))
+else:
+    TRANS_TABLE = string.maketrans(BAD_CHARS, '_' * len(BAD_CHARS))
 
 
 def to_identifier(text):
@@ -555,8 +563,8 @@ def ranges_to_list(ranges_string):
     values = []
     for rg in ranges_string.split(','):
         if '-' in rg:
-            first, last = map(int, rg.split('-'))
-            values.extend(xrange(first, last + 1))
+            first, last = list(map(int, rg.split('-')))
+            values.extend(range(first, last + 1))
         else:
             values.append(int(rg))
     return values
@@ -565,8 +573,8 @@ def ranges_to_list(ranges_string):
 def list_to_ranges(values):
     """Converts a list, e.g ``[0,2,3,4]``, into a sysfs-style ranges string, e.g. ``"0,2-4"``"""
     range_groups = []
-    for _, g in groupby(enumerate(values), lambda (i, x): i - x):
-        range_groups.append(map(itemgetter(1), g))
+    for _, g in groupby(enumerate(values), lambda i_x: i_x[0] - i_x[1]):
+        range_groups.append(list(map(itemgetter(1), g)))
     range_strings = []
     for group in range_groups:
         if len(group) == 1:
@@ -589,7 +597,7 @@ def mask_to_list(mask):
     """Converts the specfied integer bitmask into a list of
     indexes of bits that are set in the mask."""
     size = len(bin(mask)) - 2  # because of "0b"
-    return [size - i - 1 for i in xrange(size)
+    return [size - i - 1 for i in range(size)
             if mask & (1 << size - i - 1)]
 
 
@@ -634,7 +642,7 @@ def memoized(wrapped, instance, args, kwargs):
     def memoize_wrapper(*args, **kwargs):
         id_string = func_id + ','.join([__get_memo_id(a) for a in  args])
         id_string += ','.join('{}={}'.format(k, v)
-                              for k, v in kwargs.iteritems())
+                              for k, v in kwargs.items())
         if id_string not in __memo_cache:
             __memo_cache[id_string] = wrapped(*args, **kwargs)
         return __memo_cache[id_string]

devlib/utils/parse_aep.py

@@ -67,7 +67,7 @@ class AepParser(object):
         virtual = {}
 
         # Create an entry for each virtual parent
-        for supply in topo.iterkeys():
+        for supply in topo.keys():
             index = topo[supply]['index']
             # Don't care of hidden columns
             if hide[index]:
@@ -85,11 +85,11 @@ class AepParser(object):
 
         # Remove parent with 1 child as they don't give more information than their
         # child
-        for supply in virtual.keys():
+        for supply in list(virtual.keys()):
             if len(virtual[supply]) == 1:
                 del virtual[supply];
 
-        for supply in virtual.keys():
+        for supply in list(virtual.keys()):
             # Add label, hide and duplicate columns for virtual domains
             hide.append(0)
             duplicate.append(1)
@@ -166,9 +166,9 @@ class AepParser(object):
     @staticmethod
     def add_virtual_data(data, virtual):
         # write virtual domain
-        for parent in virtual.iterkeys():
+        for parent in virtual.keys():
             power = 0
-            for child in virtual[parent].values():
+            for child in list(virtual[parent].values()):
                 try:
                     power += data[child]
                 except IndexError:
@@ -440,7 +440,7 @@ class AepParser(object):
 
 
         # Create an entry for each virtual parent
-        for supply in topo.iterkeys():
+        for supply in topo.keys():
             # Parent is in the topology
             parent = topo[supply]['parent']
             if parent in topo:
@@ -454,15 +454,15 @@ class AepParser(object):
 
         # Remove parent with 1 child as they don't give more information than their
         # child
-        for supply in virtual.keys():
+        for supply in list(virtual.keys()):
             if len(virtual[supply]) == 1:
                 del virtual[supply];
 
         topo_list = ['']*(1+len(topo)+len(virtual))
         topo_list[0] = 'time'
-        for chnl in topo.iterkeys():
+        for chnl in topo.keys():
             topo_list[topo[chnl]['index']] = chnl
-        for chnl in virtual.iterkeys():
+        for chnl in virtual.keys():
             index +=1
             topo_list[index] = chnl
 
@@ -495,7 +495,7 @@ if __name__ == '__main__':
     try:
         opts, args = getopt.getopt(sys.argv[1:], "i:vo:s:l:t:")
     except getopt.GetoptError as err:
-        print str(err) # will print something like "option -a not recognized"
+        print(str(err)) # will print something like "option -a not recognized"
         sys.exit(2)
 
     for o, a in opts:
@@ -513,7 +513,7 @@ if __name__ == '__main__':
         if o == "-t":
             topofile = a
             parser = AepParser()
-            print parser.topology_from_config(topofile)
+            print(parser.topology_from_config(topofile))
             exit(0)
 
     parser = AepParser()

devlib/utils/rendering.py

@@ -1,4 +1,3 @@
-import csv
 import logging
 import os
 import re
@@ -11,6 +10,7 @@ from collections import namedtuple, OrderedDict
 from distutils.version import LooseVersion
 
 from devlib.exception  import WorkerThreadError, TargetNotRespondingError, TimeoutError
+from devlib.utils.csvutil import csvwriter
 
 
 logger = logging.getLogger('rendering')
@@ -53,7 +53,7 @@ class FrameCollector(threading.Thread):
                 wfh.close()
         except (TargetNotRespondingError, TimeoutError):  # pylint: disable=W0703
             raise
-        except Exception, e:  # pylint: disable=W0703
+        except Exception as e:  # pylint: disable=W0703
             logger.warning('Exception on collector thread: {}({})'.format(e.__class__.__name__, e))
             self.exc = WorkerThreadError(self.name, sys.exc_info())
         logger.debug('Surface flinger frame data collection stopped.')
@@ -93,8 +93,7 @@ class FrameCollector(threading.Thread):
                 indexes.append(self.header.index(c))
             frames = [[f[i] for i in indexes] for f in self.frames]
             header = columns
-        with open(outfile, 'w') as wfh:
-            writer = csv.writer(wfh)
+        with csvwriter(outfile) as writer:
             if header:
                 writer.writerow(header)
             writer.writerows(frames)
@@ -142,7 +141,7 @@ class SurfaceFlingerFrameCollector(FrameCollector):
     def _process_trace_line(self, line):
         parts = line.split()
         if len(parts) == 3:
-            frame = SurfaceFlingerFrame(*map(int, parts))
+            frame = SurfaceFlingerFrame(*list(map(int, parts)))
             if not frame.frame_ready_time:
                 return # "null" frame
             if frame.frame_ready_time <= self.last_ready_time:
@@ -167,7 +166,7 @@ def read_gfxinfo_columns(target):
     for line in lines:
         if line.startswith('---PROFILEDATA---'):
             break
-    columns_line = lines.next()
+    columns_line = next(lines)
     return columns_line.split(',')[:-1]  # has a trailing ','
 
 
@@ -202,11 +201,11 @@ class GfxinfoFrameCollector(FrameCollector):
                         found = True
                         break
 
-                fh.next()  # headers
+                next(fh)  # headers
                 for line in fh:
                     if line.startswith('---PROFILEDATA---'):
                         break
-                    entries = map(int, line.strip().split(',')[:-1])  # has a trailing ','
+                    entries = list(map(int, line.strip().split(',')[:-1]))  # has a trailing ','
                     if entries[1] <= last_vsync:
                         continue  # repeat frame
                     last_vsync = entries[1]
@@ -240,14 +239,14 @@ def gfxinfo_get_last_dump(filepath):
         fh_iter = _file_reverse_iter(fh)
         try:
             while True:
-                buf = fh_iter.next()
+                buf = next(fh_iter)
                 ix = buf.find('** Graphics')
                 if ix >= 0:
                     return buf[ix:] + record
 
                 ix = buf.find(' **\n')
                 if ix >= 0:
-                    buf =  fh_iter.next() + buf
+                    buf =  next(fh_iter) + buf
                     ix = buf.find('** Graphics')
                     if ix < 0:
                         msg = '"{}" appears to be corrupted'

devlib/utils/ssh.py

@@ -23,6 +23,7 @@ import threading
 import tempfile
 import shutil
 import socket
+import sys
 import time
 
 import pexpect
@@ -236,7 +237,7 @@ class SshConnection(object):
     def cancel_running_command(self):
         # simulate impatiently hitting ^C until command prompt appears
         logger.debug('Sending ^C')
-        for _ in xrange(self.max_cancel_attempts):
+        for _ in range(self.max_cancel_attempts):
             self.conn.sendline(chr(3))
             if self.conn.prompt(0.1):
                 return True
@@ -263,7 +264,10 @@ class SshConnection(object):
         timed_out = self._wait_for_prompt(timeout)
         # the regex removes line breaks potential introduced when writing
         # command to shell.
-        output = process_backspaces(self.conn.before)
+        if sys.version_info[0] == 3:
+            output = process_backspaces(self.conn.before.decode(sys.stdout.encoding))
+        else:
+            output = process_backspaces(self.conn.before)
         output = re.sub(r'\r([^\n])', r'\1', output)
         if '\r\n' in output: # strip the echoed command
             output = output.split('\r\n', 1)[1]
@@ -604,7 +608,7 @@ class Gem5Connection(TelnetConnection):
                 break
             except pxssh.ExceptionPxssh:
                 pass
-            except EOF, err:
+            except EOF as err:
                 self._gem5_EOF_handler(gem5_simulation, gem5_out_dir, err)
         else:
             gem5_simulation.kill()
@@ -626,7 +630,7 @@ class Gem5Connection(TelnetConnection):
                 self._login_to_device()
             except TIMEOUT:
                 pass
-            except EOF, err:
+            except EOF as err:
                 self._gem5_EOF_handler(gem5_simulation, gem5_out_dir, err)
 
             try:
@@ -636,7 +640,7 @@ class Gem5Connection(TelnetConnection):
                 prompt_found = True
             except TIMEOUT:
                 pass
-            except EOF, err:
+            except EOF as err:
                 self._gem5_EOF_handler(gem5_simulation, gem5_out_dir, err)
 
         gem5_logger.info("Successfully logged in")

devlib/utils/types.py

@@ -26,6 +26,9 @@ is not the best language to use for configuration.
 
 """
 import math
+from functools import total_ordering
+
+from past.builtins import basestring
 
 from devlib.utils.misc import isiterable, to_identifier, ranges_to_list, list_to_mask
 
@@ -88,6 +91,7 @@ def numeric(value):
     return fvalue
 
 
+@total_ordering
 class caseless_string(str):
     """
     Just like built-in Python string except case-insensitive on comparisons. However, the
@@ -100,13 +104,13 @@ class caseless_string(str):
             other = other.lower()
         return self.lower() == other
 
-    def __ne__(self, other):
-        return not self.__eq__(other)
-
-    def __cmp__(self, other):
-        if isinstance(basestring, other):
+    def __lt__(self, other):
+        if isinstance(other, basestring):
             other = other.lower()
-        return cmp(self.lower(), other)
+        return self.lower() < other
+
+    def __hash__(self):
+        return hash(self.lower())
 
     def format(self, *args, **kwargs):
         return caseless_string(super(caseless_string, self).format(*args, **kwargs))

devlib/utils/uefi.py

@@ -19,6 +19,8 @@ import time
 import logging
 from copy import copy
 
+from past.builtins import basestring
+
 from devlib.utils.serial_port import write_characters, TIMEOUT
 from devlib.utils.types import boolean
 
@@ -193,14 +195,14 @@ class UefiMenu(object):
         is not in the current menu, ``LookupError`` will be raised."""
         if not self.prompt:
             self.read_menu(timeout)
-        return self.options.items()
+        return list(self.options.items())
 
     def get_option_index(self, text, timeout=default_timeout):
         """Returns the menu index of the specified option text (uses regex matching). If the option
         is not in the current menu, ``LookupError`` will be raised."""
         if not self.prompt:
             self.read_menu(timeout)
-        for k, v in self.options.iteritems():
+        for k, v in self.options.items():
             if re.search(text, v):
                 return k
         raise LookupError(text)