merge

theano/compile/function_module.py

@@ -6,7 +6,7 @@ __docformat__ = "restructuredtext en"
 import copy_reg
 import cPickle
 
-import sys
+import sys, time, copy
 
 if sys.version_info[:2] >= (2,5):
   from functools import partial
@@ -14,8 +14,6 @@ if sys.version_info[:2] >= (2,5):
 import numpy
 import theano.gof
 #from theano import gof
-import copy
-import time
 import mode as mode_module
 from io import *
 
@@ -713,7 +711,9 @@ class FunctionMaker(object):
         optimizer, linker = mode.optimizer, copy.copy(mode.linker)
 
         # optimize the env
+        t0 = time.time()
         optimizer(env)
+        _logger.debug('Optimizing took %f seconds' % (time.time() - t0))
 
         # initialize the linker
         if not hasattr(linker, 'accept'):
@@ -784,7 +784,9 @@ class FunctionMaker(object):
 
 
         # Get a function instance
+        t0 = time.time()
         _fn, _i, _o = self.linker.make_thunk(input_storage = input_storage_lists)
+        _logger.debug('Linking took %f seconds' % (time.time() - t0))
         fn = self.function_builder(_fn, _i, _o, self.indices, self.outputs, defaults, self.unpack_single, self.return_none, self)
         return fn
 

theano/compile/sandbox/pfunc.py

@@ -137,7 +137,8 @@ def pfunc(params, outputs=None, mode=None, updates=[], givens=[]):
     #  - replace some update expressions (but update keys remain)
     new_updates = {}
     for (store_into, update_val) in iter_over_pairs(updates):
-        assert isinstance(store_into, SharedVariable)
+        if not isinstance(store_into, SharedVariable):
+            raise TypeError('update target must be a SharedVariable', store_into)
         update_val = v_clone(store_into.filter_update(update_val))
         if update_val.type != store_into.type:
             raise TypeError('an update must have the same type as the original shared variable', 

theano/gof/sandbox/typeattr.txt

+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+!!  This is a working proposal, not done.
+!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
+
+Type Attributes: Proposal for Propagating extra type information in graphs
+==========================================================================
+
+For various reasons, we would often like to know more about arguments to an Op than just their
+type.
+
+In a very obvious way, it is essential to know the size of various inputs to be able to
+reorganize graphs for minimal runtime.
+
+Also, to generate better C code, there are many reasons why we would like to know the size, the
+shape, and the numeric range of the arguments.  Algebraic properties such as positive
+semi-definiteness might be interesting to add down the road, but are not necessary for our work
+now.
+
+To generate better C code:
+
+  - On GPU, shape and strides have a big impact on the choice of algorithm for many ops.
+
+  - knowing the shape and stride we can generate faster GPU code that uses fewer registers
+    (parallelizes better)
+    
+  - Conv needs to know the shape (and possibly even the stride) of the arguments
+
+  - Gemm needs to know the strides (it currently should deal with things by copying internally)
+
+  - Reduce needs to know the strides
+
+  - Many scalar ops need to know if NaN or Inf might be coming in on input.
+
+  - Loop-unrolling methods need to know the shape
+
+  - knowing strides in advance means that array indexes can be pre-computed
+
+
+Attempt 1: Adding things to TensorType
+--------------------------------------
+
+One way to include this information would be to add it into the the TensorType class.  This
+failed though, because throughout the library there are checks of type equality.  When one
+TensorType Variable might have any shape, but another TensorType Variable must have a
+particular shape, they are neither of equal type nor unequal type.... one is a special case
+allowed by the other.  I tried to add a .shape attribute to the TensorType that would default
+to a tuple of 1 and None, where 1 was in broadcastable positions, and None meant 'unknown'.
+However I did not see a correct way of incorporating this shape into the __eq__ and __hash__
+functions.
+
+
+Lessons learned from Attempt 1
+------------------------------
+
+The meaning of a Type is that it identifies a set of values that are allowed.  Membership in
+that set is defined by filter(x, strict=True).
+
+TypeAttributes are typically unknown in current graphs, and we can represent that with a
+special UNKNOWN singleton or with None.
+
+Setting TypeAttributes in a Type to be non-None must not add to that Membership set, but can
+(and typically will) reduce the membership set.   For example, the set of vectors that may
+contain NaN is smaller than the set of vectors that do not contain NaN.
+
+Two types are equal when their membership sets are equal.  Equality of all attributes implies
+equality, but not vice versa since an attribute might make another one irrelevant.  ***HOW CAN
+WE IMPLEMENT THIS?***
+
+We should allow an env to replace a variable with one of a different (non-equal) type, as long
+as the new type's membership set is a subclass of the original.  In other words, we can add
+restrictions to a type during optimization.
+
+Furthermore, I think we should add a restrict_type env operation, similar to replace.  This
+operation would swap the type of a given variable for one that is stricter (or equally strict).
+
+It is ok to add new attributes at any time, even after graphs have been constructed, as long as
+things are implemented such that the new attribute is UNKNOWN for all variables that were
+created before the attribute was introduced.  This way, a user can
+just make up an attribute and use it for a special variable in his graph.  This should not
+interfere with anything.  Type Attributes might supercede the 'tags' of variables in this way.
+
+
+Challenge learned from Attempt 1: asymmetry in type-checking
+-------------------------------------------------------------
+
+Many make_node implementations and optimizations (especially local ones) follow patterns like:
+
+.. code-block:: python
+
+    if node.inputs[0].type == tensor.lvector:
+        ... 
+    
+What this typically means is  "if the set of potential values for inputs[0] is a subset of the
+ones identified by lvector".  But this meaning in terms of subsets is not symmetric, so it
+requires reading through a lot of our code, and rethinking and rewriting many cases.
+
+
+Challenge learned from Attempt 1: equality checking with user-defined Attributes
+---------------------------------------------------------------------------------
+
+If a user makes up a new attribute, how is he/she supposed to define the way in which that
+attribute restricts the type set?
+
+This is required in order to support important API functions:
+
+  - type0.is_subset_of(type1)
+
+  - type0 == type1
+
+In the absence of any attributes, or in the case where attributes are known ahead of time, 
+the Type class can simply use the programmers knowledge of the meaning of the attributes to
+define the is_subset_of function correctly.
+
+I think that if a user adds a new attribute at runtime, then the options are very limited: the
+user-defined attribute either restricts the membership set or it doesn't.   If it does restrict
+the set, then we can use the cmp() function on the attribute value as the definition of how
+much it restricts the set.
+
+If cmp() is not sufficient, then a Type subclass is necessary.  That subclass will override the
+is_subset_of and __eq__ functions as required.
+
+"""
+

theano/scalar/basic.py

@@ -228,12 +228,17 @@ int8 = Scalar('int8')
 int16 = Scalar('int16')
 int32 = Scalar('int32')
 int64 = Scalar('int64')
+uint8 = Scalar('uint8')
+uint16 = Scalar('uint16')
+uint32 = Scalar('uint32')
+uint64 = Scalar('uint64')
 float32 = Scalar('float32')
 float64 = Scalar('float64')
 complex64 = Scalar('complex64')
 complex128 = Scalar('complex128')
 
 int_types = int8, int16, int32, int64
+uint_types = uint8, uint16, uint32, uint64
 float_types = float32, float64
 complex_types = complex64, complex128
 
@@ -888,11 +893,7 @@ class Cast(UnaryScalarOp):
     def impl(self, input):
         return self.ctor(input)
     def c_code(self, node, name, (x, ), (z, ), sub):
-        #HACK: we assume that x has the form 'VARNAME_i', 
-        #      and we need the varname to get the dtype.
-        assert (len(x) > 2) and (x[-2:] == '_i')
-        varname = x[:-2]
-        return "%s = (dtype_%s)%s;" % (z, varname, x)
+        return "%s = (%s)%s;" % (z, node.outputs[0].type.dtype_specs()[1], x)
     def grad(self, (x, ), (gz, )):
         if x.type in grad_types:
           return [cast(gz, x.type.dtype)]
@@ -901,7 +902,7 @@ class Cast(UnaryScalarOp):
     def c_code_cache_version(self):
         s = super(Cast, self).c_code_cache_version()
         if s:
-            return (2,) + s
+            return (3,) + s
         else:
             return s
 
@@ -909,15 +910,24 @@ convert_to_int8 = Cast(int8, name='convert_to_int8')
 convert_to_int16 = Cast(int16, name='convert_to_int16')
 convert_to_int32 = Cast(int32, name='convert_to_int32')
 convert_to_int64 = Cast(int64, name='convert_to_int64')
+convert_to_uint8 = Cast(uint8, name='convert_to_uint8')
+convert_to_uint16 = Cast(uint16, name='convert_to_uint16')
+convert_to_uint32 = Cast(uint32, name='convert_to_uint32')
+convert_to_uint64 = Cast(uint64, name='convert_to_uint64')
 convert_to_float32 = Cast(float32, name='convert_to_float32')
 convert_to_float64 = Cast(float64, name='convert_to_float64')
 convert_to_complex64 = Cast(complex64, name='convert_to_complex64')
 convert_to_complex128 = Cast(complex128, name='convert_to_complex128')
 
-_cast_mapping = {'int8': convert_to_int8,
+_cast_mapping = {
+           'int8': convert_to_int8,
            'int16': convert_to_int16,
            'int32': convert_to_int32,
            'int64': convert_to_int64,
+           'uint8': convert_to_uint8,
+           'uint16': convert_to_uint16,
+           'uint32': convert_to_uint32,
+           'uint64': convert_to_uint64,
            'float32': convert_to_float32,
            'float64': convert_to_float64,
            'complex64': convert_to_complex64,

theano/tensor/basic.py

@@ -253,6 +253,11 @@ class TensorType(Type):
     When this is True, strict filtering rejects data containing NaN or Inf entries. (Used in `DebugMode`)
     """
 
+    use_shape = False
+    """
+    This should be removed (hardcoded to be False) after AISTATS09
+    """
+
     def __init__(self, dtype, broadcastable, name = None, shape=None):
         """Initialize self.dtype and self.broadcastable.
 
@@ -305,10 +310,11 @@ class TensorType(Type):
             if self.filter_checks_isfinite and (not numpy.all(numpy.isfinite(data))):
                 raise TypeError("non-finite elements not allowed")
 
-            for si, di in zip(self.shape, data.shape):
-                if not (si is None or si == di):
-                    raise TypeError('%s requires ndarray with shape matching %s (got %s)'%(
-                        self, self.shape, data.shape))
+            if TensorType.use_shape:
+                for si, di in zip(self.shape, data.shape):
+                    if not (si is None or si == di):
+                        raise TypeError('%s requires ndarray with shape matching %s (got %s)'%(
+                            self, self.shape, data.shape))
             return data
         else:
             data = numpy.asarray(data, dtype = self.dtype)
@@ -347,9 +353,13 @@ class TensorType(Type):
 
     def __eq__(self, other):
         """Compare True iff other is the same kind of TensorType"""
-        return type(self) == type(other) and other.dtype == self.dtype \
-                and other.broadcastable == self.broadcastable \
-                and other.shape == self.shape
+        if TensorType.use_shape:
+            return type(self) == type(other) and other.dtype == self.dtype \
+                    and other.broadcastable == self.broadcastable \
+                    and other.shape == self.shape
+        else:
+            return type(self) == type(other) and other.dtype == self.dtype \
+                    and other.broadcastable == self.broadcastable
 
     @staticmethod
     def values_eq(a, b):
@@ -420,7 +430,10 @@ class TensorType(Type):
 
     def __hash__(self):
         """Hash equal for same kinds of TensorType"""
-        return hashtype(self) ^ hash(self.dtype) ^ hash(self.broadcastable) ^ hash(self.shape)
+        if TensorType.use_shape:
+            return hashtype(self) ^ hash(self.dtype) ^ hash(self.broadcastable) ^ hash(self.shape)
+        else:
+            return hashtype(self) ^ hash(self.dtype) ^ hash(self.broadcastable)
 
     ndim = property(lambda self: len(self.broadcastable), doc = "number of dimensions")
     """Number of dimensions
@@ -1043,6 +1056,18 @@ _convert_to_int32 = _conversion(elemwise.Elemwise(scal.convert_to_int32), 'int32
 _convert_to_int64 = _conversion(elemwise.Elemwise(scal.convert_to_int64), 'int64')
 """Cast to 64-bit integer"""
 
+_convert_to_uint8  = _conversion(elemwise.Elemwise(scal.convert_to_uint8), 'uint8')
+"""Cast to unsigned 8-bit integer"""
+    
+_convert_to_uint16 = _conversion(elemwise.Elemwise(scal.convert_to_uint16), 'uint16')
+"""Cast to unsigned 16-bit integer"""
+
+_convert_to_uint32 = _conversion(elemwise.Elemwise(scal.convert_to_uint32), 'uint32')
+"""Cast to unsigned 32-bit integer"""
+
+_convert_to_uint64 = _conversion(elemwise.Elemwise(scal.convert_to_uint64), 'uint64')
+"""Cast to unsigned 64-bit integer"""
+
 _convert_to_float32 = _conversion(elemwise.Elemwise(scal.convert_to_float32), 'float32')
 """Cast to single-precision floating point"""
 
@@ -1055,10 +1080,15 @@ _convert_to_complex64  = _conversion(elemwise.Elemwise(scal.convert_to_complex64
 _convert_to_complex128 = _conversion(elemwise.Elemwise(scal.convert_to_complex128), 'complex128')
 """Cast to double-precision complex"""
 
-_cast_mapping = {'int8': _convert_to_int8,
+_cast_mapping = {
+           'int8': _convert_to_int8,
            'int16': _convert_to_int16,
            'int32': _convert_to_int32,
            'int64': _convert_to_int64,
+           'uint8': _convert_to_uint8,
+           'uint16': _convert_to_uint16,
+           'uint32': _convert_to_uint32,
+           'uint64': _convert_to_uint64,
            'float32': _convert_to_float32,
            'float64': _convert_to_float64,
            'complex64': _convert_to_complex64,

theano/tensor/nnet.py

@@ -582,7 +582,7 @@ class CrossentropySoftmaxArgmax1HotWithBias(gof.Op):
         }
         if (%(x)s->dimensions[0] != %(y_idx)s->dimensions[0])
         {
-            PyErr_Format(PyExc_ValueError, "number of rows in x (%%i) does not match length of y (%%i)",
+            PyErr_Format(PyExc_ValueError, "number of rows in x (%%zi) does not match length of y (%%zi)",
             %(x)s->dimensions[0], %(y_idx)s->dimensions[0]);
             %(fail)s;
         }
@@ -633,7 +633,7 @@ class CrossentropySoftmaxArgmax1HotWithBias(gof.Op):
 
 
     def c_code_cache_version(self):
-        return (4,) + SoftmaxWithBias.c_code_cache_version()
+        return (5,) + SoftmaxWithBias.c_code_cache_version()
     def c_code(self, node, name, (x, b, y_idx), (nll, sm, am), sub):
         y_idx_type = node.inputs[2].type.dtype_specs()[1]
         am_type = y_idx_type

theano/tensor/tests/test_casting.py

@@ -26,7 +26,7 @@ class test_casting(unittest.TestCase):
         assert 0
 
     def test_basic(self):
-        for type1 in ['int8', 'int16', 'int32', 'int64', 'float32', 'float64']:
+        for type1 in ['uint8', 'uint16', 'uint32', 'uint64', 'int8', 'int16', 'int32', 'int64', 'float32', 'float64']:
             x = TensorType(dtype = type1, broadcastable = (False, )).make_variable()
             for type2, converter in zip(['int8', 'int16', 'int32', 'int64', 'float32', 'float64'],
                                         [_convert_to_int8, _convert_to_int16,