Issue #783: python3 compatible - introduced theano.compat.

setup.py

@@ -181,7 +181,7 @@ def do_setup():
           license=LICENSE,
           platforms=PLATFORMS,
           packages=find_packages(),
-          install_requires=['numpy>=1.5.0', 'scipy>=0.7.2', 'six>=1.2.0'],
+          install_requires=['numpy>=1.5.0', 'scipy>=0.7.2'],
           package_data={
               '': ['*.txt', '*.rst', '*.cu', '*.cuh', '*.c', '*.sh',
                    'ChangeLog'],

theano/compat/__init__.py

+"""Code supporting compatibility across versions of Python.
+
+"""
+
+# Python 3.x compatibility
+from theano.compat.six import PY3, b, BytesIO, next, get_unbound_function
+from theano.compat.six.moves import reload_module as reload, configparser
+
+if PY3:
+
+    from operator import truediv as operator_div
+
+
+    # In python 3.x, when an exception is reraised it saves original
+    # exception in its args, therefore in order to find the actual
+    # message, we need to unpack arguments recurcively.
+    def exc_message(e):
+        msg = e.args[0]
+        if isinstance(msg, Exception):
+            return exc_message(msg)
+        return msg
+
+    def cmp(a, b):
+        """Return -1 if x<y, 0 if x==y, 1 if x > y."""
+        return (a > b) - (a < b)
+
+
+else:
+
+    from operator import div as operator_div
+
+
+    def exc_message(e):
+        return e[0]
+
+    cmp = cmp

theano/compat/six.py

+"""Utilities for writing code that runs on Python 2 and 3"""
+
+# Copyright (c) 2010-2012 Benjamin Peterson
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of
+# this software and associated documentation files (the "Software"), to deal in
+# the Software without restriction, including without limitation the rights to
+# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+# the Software, and to permit persons to whom the Software is furnished to do so,
+# subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+# FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+# COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+# IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+import operator
+import sys
+import types
+
+__author__ = "Benjamin Peterson <benjamin@python.org>"
+__version__ = "1.2.0"
+
+
+# True if we are running on Python 3.
+PY3 = sys.version_info[0] == 3
+
+if PY3:
+    string_types = str,
+    integer_types = int,
+    class_types = type,
+    text_type = str
+    binary_type = bytes
+
+    MAXSIZE = sys.maxsize
+else:
+    string_types = basestring,
+    integer_types = (int, long)
+    class_types = (type, types.ClassType)
+    text_type = unicode
+    binary_type = str
+
+    if sys.platform.startswith("java"):
+        # Jython always uses 32 bits.
+        MAXSIZE = int((1 << 31) - 1)
+    else:
+        # It's possible to have sizeof(long) != sizeof(Py_ssize_t).
+        class X(object):
+            def __len__(self):
+                return 1 << 31
+        try:
+            len(X())
+        except OverflowError:
+            # 32-bit
+            MAXSIZE = int((1 << 31) - 1)
+        else:
+            # 64-bit
+            MAXSIZE = int((1 << 63) - 1)
+            del X
+
+
+def _add_doc(func, doc):
+    """Add documentation to a function."""
+    func.__doc__ = doc
+
+
+def _import_module(name):
+    """Import module, returning the module after the last dot."""
+    __import__(name)
+    return sys.modules[name]
+
+
+class _LazyDescr(object):
+
+    def __init__(self, name):
+        self.name = name
+
+    def __get__(self, obj, tp):
+        result = self._resolve()
+        setattr(obj, self.name, result)
+        # This is a bit ugly, but it avoids running this again.
+        delattr(tp, self.name)
+        return result
+
+
+class MovedModule(_LazyDescr):
+
+    def __init__(self, name, old, new=None):
+        super(MovedModule, self).__init__(name)
+        if PY3:
+            if new is None:
+                new = name
+            self.mod = new
+        else:
+            self.mod = old
+
+    def _resolve(self):
+        return _import_module(self.mod)
+
+
+class MovedAttribute(_LazyDescr):
+
+    def __init__(self, name, old_mod, new_mod, old_attr=None, new_attr=None):
+        super(MovedAttribute, self).__init__(name)
+        if PY3:
+            if new_mod is None:
+                new_mod = name
+            self.mod = new_mod
+            if new_attr is None:
+                if old_attr is None:
+                    new_attr = name
+                else:
+                    new_attr = old_attr
+            self.attr = new_attr
+        else:
+            self.mod = old_mod
+            if old_attr is None:
+                old_attr = name
+            self.attr = old_attr
+
+    def _resolve(self):
+        module = _import_module(self.mod)
+        return getattr(module, self.attr)
+
+
+
+class _MovedItems(types.ModuleType):
+    """Lazy loading of moved objects"""
+
+
+_moved_attributes = [
+    MovedAttribute("cStringIO", "cStringIO", "io", "StringIO"),
+    MovedAttribute("filter", "itertools", "builtins", "ifilter", "filter"),
+    MovedAttribute("input", "__builtin__", "builtins", "raw_input", "input"),
+    MovedAttribute("map", "itertools", "builtins", "imap", "map"),
+    MovedAttribute("reload_module", "__builtin__", "imp", "reload"),
+    MovedAttribute("reduce", "__builtin__", "functools"),
+    MovedAttribute("StringIO", "StringIO", "io"),
+    MovedAttribute("xrange", "__builtin__", "builtins", "xrange", "range"),
+    MovedAttribute("zip", "itertools", "builtins", "izip", "zip"),
+
+    MovedModule("builtins", "__builtin__"),
+    MovedModule("configparser", "ConfigParser"),
+    MovedModule("copyreg", "copy_reg"),
+    MovedModule("http_cookiejar", "cookielib", "http.cookiejar"),
+    MovedModule("http_cookies", "Cookie", "http.cookies"),
+    MovedModule("html_entities", "htmlentitydefs", "html.entities"),
+    MovedModule("html_parser", "HTMLParser", "html.parser"),
+    MovedModule("http_client", "httplib", "http.client"),
+    MovedModule("email_mime_multipart", "email.MIMEMultipart", "email.mime.multipart"),
+    MovedModule("email_mime_text", "email.MIMEText", "email.mime.text"),
+    MovedModule("email_mime_base", "email.MIMEBase", "email.mime.base"),
+    MovedModule("BaseHTTPServer", "BaseHTTPServer", "http.server"),
+    MovedModule("CGIHTTPServer", "CGIHTTPServer", "http.server"),
+    MovedModule("SimpleHTTPServer", "SimpleHTTPServer", "http.server"),
+    MovedModule("cPickle", "cPickle", "pickle"),
+    MovedModule("queue", "Queue"),
+    MovedModule("reprlib", "repr"),
+    MovedModule("socketserver", "SocketServer"),
+    MovedModule("tkinter", "Tkinter"),
+    MovedModule("tkinter_dialog", "Dialog", "tkinter.dialog"),
+    MovedModule("tkinter_filedialog", "FileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_scrolledtext", "ScrolledText", "tkinter.scrolledtext"),
+    MovedModule("tkinter_simpledialog", "SimpleDialog", "tkinter.simpledialog"),
+    MovedModule("tkinter_tix", "Tix", "tkinter.tix"),
+    MovedModule("tkinter_constants", "Tkconstants", "tkinter.constants"),
+    MovedModule("tkinter_dnd", "Tkdnd", "tkinter.dnd"),
+    MovedModule("tkinter_colorchooser", "tkColorChooser",
+                "tkinter.colorchooser"),
+    MovedModule("tkinter_commondialog", "tkCommonDialog",
+                "tkinter.commondialog"),
+    MovedModule("tkinter_tkfiledialog", "tkFileDialog", "tkinter.filedialog"),
+    MovedModule("tkinter_font", "tkFont", "tkinter.font"),
+    MovedModule("tkinter_messagebox", "tkMessageBox", "tkinter.messagebox"),
+    MovedModule("tkinter_tksimpledialog", "tkSimpleDialog",
+                "tkinter.simpledialog"),
+    MovedModule("urllib_robotparser", "robotparser", "urllib.robotparser"),
+    MovedModule("winreg", "_winreg"),
+]
+for attr in _moved_attributes:
+    setattr(_MovedItems, attr.name, attr)
+del attr
+
+moves = sys.modules[__name__ + ".moves"] = _MovedItems("moves")
+
+
+def add_move(move):
+    """Add an item to six.moves."""
+    setattr(_MovedItems, move.name, move)
+
+
+def remove_move(name):
+    """Remove item from six.moves."""
+    try:
+        delattr(_MovedItems, name)
+    except AttributeError:
+        try:
+            del moves.__dict__[name]
+        except KeyError:
+            raise AttributeError("no such move, %r" % (name,))
+
+
+if PY3:
+    _meth_func = "__func__"
+    _meth_self = "__self__"
+
+    _func_code = "__code__"
+    _func_defaults = "__defaults__"
+
+    _iterkeys = "keys"
+    _itervalues = "values"
+    _iteritems = "items"
+else:
+    _meth_func = "im_func"
+    _meth_self = "im_self"
+
+    _func_code = "func_code"
+    _func_defaults = "func_defaults"
+
+    _iterkeys = "iterkeys"
+    _itervalues = "itervalues"
+    _iteritems = "iteritems"
+
+
+try:
+    advance_iterator = next
+except NameError:
+    def advance_iterator(it):
+        return it.next()
+next = advance_iterator
+
+
+try:
+    callable = callable
+except NameError:
+    def callable(obj):
+        return any("__call__" in klass.__dict__ for klass in type(obj).__mro__)
+
+
+if PY3:
+    def get_unbound_function(unbound):
+        return unbound
+
+    Iterator = object
+else:
+    def get_unbound_function(unbound):
+        return unbound.im_func
+
+    class Iterator(object):
+
+        def next(self):
+            return type(self).__next__(self)
+
+    callable = callable
+_add_doc(get_unbound_function,
+         """Get the function out of a possibly unbound function""")
+
+
+get_method_function = operator.attrgetter(_meth_func)
+get_method_self = operator.attrgetter(_meth_self)
+get_function_code = operator.attrgetter(_func_code)
+get_function_defaults = operator.attrgetter(_func_defaults)
+
+
+def iterkeys(d):
+    """Return an iterator over the keys of a dictionary."""
+    return iter(getattr(d, _iterkeys)())
+
+def itervalues(d):
+    """Return an iterator over the values of a dictionary."""
+    return iter(getattr(d, _itervalues)())
+
+def iteritems(d):
+    """Return an iterator over the (key, value) pairs of a dictionary."""
+    return iter(getattr(d, _iteritems)())
+
+
+if PY3:
+    def b(s):
+        return s.encode("latin-1")
+    def u(s):
+        return s
+    if sys.version_info[1] <= 1:
+        def int2byte(i):
+            return bytes((i,))
+    else:
+        # This is about 2x faster than the implementation above on 3.2+
+        int2byte = operator.methodcaller("to_bytes", 1, "big")
+    import io
+    StringIO = io.StringIO
+    BytesIO = io.BytesIO
+else:
+    def b(s):
+        return s
+    def u(s):
+        return unicode(s, "unicode_escape")
+    int2byte = chr
+    import StringIO
+    StringIO = BytesIO = StringIO.StringIO
+_add_doc(b, """Byte literal""")
+_add_doc(u, """Text literal""")
+
+
+if PY3:
+    import builtins
+    exec_ = getattr(builtins, "exec")
+
+
+    def reraise(tp, value, tb=None):
+        if value.__traceback__ is not tb:
+            raise value.with_traceback(tb)
+        raise value
+
+
+    print_ = getattr(builtins, "print")
+    del builtins
+
+else:
+    def exec_(_code_, _globs_=None, _locs_=None):
+        """Execute code in a namespace."""
+        if _globs_ is None:
+            frame = sys._getframe(1)
+            _globs_ = frame.f_globals
+            if _locs_ is None:
+                _locs_ = frame.f_locals
+            del frame
+        elif _locs_ is None:
+            _locs_ = _globs_
+        exec("""exec _code_ in _globs_, _locs_""")
+
+
+    exec_("""def reraise(tp, value, tb=None):
+    raise tp, value, tb
+""")
+
+
+    def print_(*args, **kwargs):
+        """The new-style print function."""
+        fp = kwargs.pop("file", sys.stdout)
+        if fp is None:
+            return
+        def write(data):
+            if not isinstance(data, basestring):
+                data = str(data)
+            fp.write(data)
+        want_unicode = False
+        sep = kwargs.pop("sep", None)
+        if sep is not None:
+            if isinstance(sep, unicode):
+                want_unicode = True
+            elif not isinstance(sep, str):
+                raise TypeError("sep must be None or a string")
+        end = kwargs.pop("end", None)
+        if end is not None:
+            if isinstance(end, unicode):
+                want_unicode = True
+            elif not isinstance(end, str):
+                raise TypeError("end must be None or a string")
+        if kwargs:
+            raise TypeError("invalid keyword arguments to print()")
+        if not want_unicode:
+            for arg in args:
+                if isinstance(arg, unicode):
+                    want_unicode = True
+                    break
+        if want_unicode:
+            newline = unicode("\n")
+            space = unicode(" ")
+        else:
+            newline = "\n"
+            space = " "
+        if sep is None:
+            sep = space
+        if end is None:
+            end = newline
+        for i, arg in enumerate(args):
+            if i:
+                write(sep)
+            write(arg)
+        write(end)
+
+_add_doc(reraise, """Reraise an exception.""")
+
+
+def with_metaclass(meta, base=object):
+    """Create a base class with a metaclass."""
+    return meta("NewBase", (base,), {})

theano/compile/debugmode.py

@@ -10,10 +10,10 @@ from itertools import izip
 from StringIO import StringIO
 
 import numpy
-import six
 
 import theano
 from theano import gof
+from theano.compat import get_unbound_function
 from theano.gof import FunctionGraph,graph, utils, link, ops_with_inner_function
 from theano.gof.link import raise_with_op
 from theano.gof.cc import CLinker
@@ -1564,8 +1564,8 @@ class _VariableEquivalenceTracker(object):
 #List of default version of make thunk.
 #This is needed to know if the user overrided it.
 #The GpuOp will be added here when theano.sandbox.cuda is imported.
-default_make_thunk = [six.get_unbound_function(theano.gof.Op.make_thunk),
-                      six.get_unbound_function(theano.gof.OpenMPOp.make_thunk)]
+default_make_thunk = [get_unbound_function(theano.gof.Op.make_thunk),
+                      get_unbound_function(theano.gof.OpenMPOp.make_thunk)]
 
 
 class _Linker(gof.link.LocalLinker):

theano/compile/tests/test_function_module.py

@@ -632,7 +632,7 @@ class T_picklefunction(unittest.TestCase):
 
         f = theano.function([x], theano.tensor.dot(x, y))
 
-        from six import BytesIO
+        from theano.compat import BytesIO
         fp = BytesIO()
         p = cPickle.Pickler(fp, 2)
         p.persistent_id = pers_save

theano/compile/tests/test_module.py

@@ -4,27 +4,14 @@
 __docformat__ = "restructuredtext en"
 
 import cPickle, numpy, unittest
+
 from theano import config
+from theano.compat import exc_message
 from theano.compile.module import *
 from theano.compile.function_module import AliasedMemoryError
 import theano.tensor as T
 import sys, copy
 import theano
-from six import PY3
-
-# Copied from theano/tensor/tests/test_basic.py
-if PY3:
-    # In python 3.x, when an exception is reraised it saves original
-    # exception in its args, therefore in order to find the actual
-    # message, we need to unpack arguments recurcively.
-    def exc_message(e):
-        msg = e.args[0]
-        if isinstance(msg, Exception):
-            return exc_message(msg)
-        return msg
-else:
-    def exc_message(e):
-        return e[0]
 
 #TODO: add test for module.make(member=init_value)
 class T_module(unittest.TestCase):

theano/configparser.py

@@ -7,11 +7,10 @@ import os
 import sys
 import warnings
 
-from six.moves import configparser as ConfigParser
-
 import StringIO
 
 import theano
+from theano.compat import configparser as ConfigParser
 
 _logger = logging.getLogger('theano.configparser')
 

theano/gof/cc.py

@@ -9,11 +9,12 @@ import os
 import StringIO
 import sys
 from itertools import izip
-from six import PY3
 
 import numpy
 
-if sys.version_info[:2] >= (3, 0):
+from theano.compat import PY3
+
+if PY3:
     import hashlib
 
     def hash_from_code(msg):

theano/gof/cmodule.py

@@ -13,13 +13,13 @@ import subprocess
 import sys
 import tempfile
 import time
-from six import PY3, b, next
 
 import distutils.sysconfig
 
 import numpy.distutils  # TODO: TensorType should handle this
 
 import theano
+from theano.compat import PY3, b, next
 from theano.gof.utils import flatten
 from theano.configparser import config
 from theano.gof.cc import hash_from_code

theano/gof/cutils.py

-import os
-import sys
-from six import PY3
+import os, sys
+
+from theano.compat import PY3
 from theano.gof.compilelock import get_lock, release_lock
 from theano import config
 

theano/gof/lazylinker_c.py

 import errno
 import os, logging, sys
-from six.moves import reload_module as reload
 
 import theano
 from theano import config
+from theano.compat import reload
 from theano.gof.compilelock import get_lock, release_lock
 from theano.gof import cmodule
 

theano/gof/sched.py

 from theano.gof.graph import list_of_nodes
 from theano.gof.python25 import any, defaultdict
-try:
-    cmp
-except NameError:
-    def cmp(a, b):
-        return (a > b) - (a < b)
+from theano.compat import cmp
 
 ## {{{ http://code.activestate.com/recipes/578231/ (r1)
 # Copyright (c) Oren Tirosh 2012

theano/gof/tests/test_optdb.py

 from unittest import TestCase
-from six import PY3
-if PY3:
-    # In python 3.x, when an exception is reraised it saves original
-    # exception in its args, therefore in order to find the actual
-    # message, we need to unpack arguments recurcively.
-    def exc_message(e):
-        msg = e.args[0]
-        if isinstance(msg, Exception):
-            return exc_message(msg)
-        return msg
-else:
-    def exc_message(e):
-        return e[0]
 
+from theano.compat import exc_message
 from theano.gof.optdb import opt, DB
 
 

theano/gof/tests/test_sched.py

@@ -5,12 +5,7 @@ import theano
 from theano import tensor
 from theano.gof.graph import io_toposort
 from theano.gof.python25 import any
-try:
-    cmp
-except NameError:
-    def cmp(a, b):
-        return (a > b) - (a < b)
-
+from theano.compat import cmp
 
 def test_dependence():
     dependence = make_dependence_cmp()

theano/sandbox/cuda/__init__.py

@@ -6,9 +6,8 @@ import shutil
 import stat
 import sys
 
-import six
-
 import theano
+from theano.compat import get_unbound_function
 from theano.compile import optdb
 from theano.gof.cmodule import get_lib_extension
 from theano.gof.compilelock import get_lock, release_lock
@@ -243,7 +242,7 @@ class GpuOp(theano.gof.Op):
         return super(GpuOp, self).make_thunk(node, storage_map,
                                              compute_map, no_recycling)
 
-theano.compile.debugmode.default_make_thunk.append(six.get_unbound_function(GpuOp.make_thunk))
+theano.compile.debugmode.default_make_thunk.append(get_unbound_function(GpuOp.make_thunk))
 
 # We must do those import to be able to create the full doc when
 # nvcc is not available

theano/scalar/basic.py

@@ -18,10 +18,10 @@ from copy import copy
 from itertools import imap
 from textwrap import dedent
 
-from six import PY3
 import numpy
 
 import theano
+from theano.compat import PY3
 from theano import gof
 from theano.gof import Op, utils, Variable, Constant, Type, Apply, FunctionGraph
 from theano.gof.python25 import partial, all, any

theano/scan_module/scan_perform_ext.py

 import os, logging, sys
-from six.moves import reload_module as reload
 
 import theano
 from theano import config
+from theano.compat import reload
 from theano.gof.compilelock import get_lock, release_lock
 from theano.gof import cmodule
 

theano/sparse/sandbox/test_sp.py

@@ -2,7 +2,6 @@ from nose.plugins.skip import SkipTest
 import sys
 import time
 import unittest
-from six import next
 
 import theano.sparse
 if not theano.sparse.enable_sparse:
@@ -15,6 +14,7 @@ import numpy
 
 from theano import function, tensor
 import theano
+from theano.compat import next
 from theano.sparse.sandbox import sp
 from theano.sparse.tests.test_basic import random_lil
 from theano.tests import unittest_tools as utt

theano/tensor/basic.py

@@ -6,11 +6,11 @@ import sys
 import warnings
 from itertools import izip
 
-from six import PY3
 import numpy
 #from copy import copy as python_copy
 
 import theano
+from theano.compat import PY3
 from theano.configparser import config
 from theano import gof
 from theano.gof import Apply, Constant, Op, Type, Variable

theano/tensor/tests/test_basic.py

@@ -13,29 +13,13 @@ from itertools import izip
 import __builtin__
 builtin_min = __builtin__.min
 
-from six import PY3
-if PY3:
-    operator_div = operator.truediv
-    # In python 3.x, when an exception is reraised it saves original
-    # exception in its args, therefore in order to find the actual
-    # message, we need to unpack arguments recurcively.
-    def exc_message(e):
-        msg = e.args[0]
-        if isinstance(msg, Exception):
-            return exc_message(msg)
-        return msg
-else:
-    operator_div = operator.div
-    def exc_message(e):
-        return e[0]
-
-
 from nose.plugins.skip import SkipTest
 import numpy
 from numpy.testing import dec, assert_array_equal, assert_allclose
 from numpy.testing.noseclasses import KnownFailureTest
 
 import theano
+from theano.compat import PY3, exc_message, operator_div
 from theano import compile, config, function, gof, tensor, shared
 from theano.compile import DeepCopyOp
 from theano.compile.mode import get_default_mode

theano/tensor/tests/test_blas.py

@@ -2,23 +2,10 @@
 #import traceback
 import itertools
 import sys
-# Copied from tensor/tests/test_basic.py.
-from six import PY3
-if PY3:
-    # In python 3.x, when an exception is reraised it saves original
-    # exception in its args, therefore in order to find the actual
-    # message, we need to unpack arguments recurcively.
-    def exc_message(e):
-        msg = e.args[0]
-        if isinstance(msg, Exception):
-            return exc_message(msg)
-        return msg
-else:
-    def exc_message(e):
-        return e[0]
 
 import theano.tensor as T
 from theano import tensor
+from theano.compat import PY3, exc_message
 from theano.gof.python25 import product as itertools_product
 from theano.gof.python25 import any
 from theano.printing import pp