Merge branch 'ynd_sparse' of https://github.com/dwf/Theano into sparse

theano/gof/python25.py

@@ -91,5 +91,17 @@ if sys.version_info[:2] < (2,6):
             for j in range(i+1, r):
                 indices[j] = indices[j-1] + 1
             yield tuple(pool[i] for i in indices)
+
+
+    def product(*args, **kwds):
+        # product('ABCD', 'xy') --> Ax Ay Bx By Cx Cy Dx Dy
+        # product(range(2), repeat=3) --> 000 001 010 011 100 101 110 111
+        pools = map(tuple, args) * kwds.get('repeat', 1)
+        result = [[]]
+        for pool in pools:
+            result = [x+[y] for x in result for y in pool]
+        for prod in result:
+            yield tuple(prod)
+
 else:
-    from itertools import combinations
+    from itertools import combinations, product

theano/sparse/basic.py

@@ -1451,8 +1451,11 @@ class StructuredDotGradCSR(gof.Op):
         }
 
         """% dict(locals(), **sub)
+
+
 sdg_csr = StructuredDotGradCSR()
 
+
 class Dot(gof.op.Op):
     """
     Operation for efficiently calculating the dot product when
@@ -1490,7 +1493,9 @@ class Dot(gof.op.Op):
         return gof.Apply(self, [x, y], [tensor.tensor(dtype=dtype_out,
                          broadcastable=(False, False))])
 
-    def perform(self, node, (x, y), (out, )):
+    def perform(self, node, inputs, out):
+        x, y = inputs
+        out = out[0]
         x_is_sparse = _is_sparse(x)
         y_is_sparse = _is_sparse(y)
 
@@ -1506,23 +1511,31 @@ class Dot(gof.op.Op):
 
     def grad(self, (x, y), (gz,)):
         assert _is_sparse_variable(x) or _is_sparse_variable(y)
+        rval = []
 
-        rval = [
-            tensor.dot(gz, y.T) if _is_dense_variable(y) else dot(gz, y.T),
-            tensor.dot(x.T, gz) if _is_dense_variable(x) else dot(x.T, gz)
-        ]
+        if _is_dense_variable(y):
+            rval.append(tensor.dot(gz, y.T))
+        else:
+            rval.append(dot(gz, y.T))
+        if _is_dense_variable(x):
+            rval.append(tensor.dot(x.T, gz))
+        else:
+            rval.append(dot(x.T, gz))
 
         return rval
 _dot = Dot()
 
+
 def dot(x, y):
     """
     Operation for efficiently calculating the dot product when
     one or all operands is sparse. Supported format are CSC and CSR.
     The output of the operation is dense.
     """
-    if hasattr(x, 'getnnz'): x = as_sparse_variable(x)
-    if hasattr(y, 'getnnz'): y = as_sparse_variable(y)
+    if hasattr(x, 'getnnz'):
+        x = as_sparse_variable(x)
+    if hasattr(y, 'getnnz'):
+        y = as_sparse_variable(y)
 
     x_is_sparse_variable = _is_sparse_variable(x)
     y_is_sparse_variable = _is_sparse_variable(y)
@@ -1572,12 +1585,13 @@ class Usmm(gof.op.Op):
             # We should use Dot22 and Gemm in that case.
             raise TypeError(x)
 
-        dtype_out = scalar.upcast(alpha.type.dtype, x.type.dtype, y.type.dtype, z.type.dtype)
+        dtype_out = scalar.upcast(alpha.type.dtype, x.type.dtype,
+                                  y.type.dtype, z.type.dtype)
         alpha = tensor.as_tensor_variable(alpha)
         z = tensor.as_tensor_variable(z)
 
         assert z.ndim == 2
-        assert alpha.type.broadcastable == (True,)* alpha.ndim
+        assert alpha.type.broadcastable == (True,) * alpha.ndim
         if not _is_sparse_variable(x):
             x = tensor.as_tensor_variable(x)
             assert x.ndim == 2
@@ -1585,7 +1599,9 @@ class Usmm(gof.op.Op):
             y = tensor.as_tensor_variable(y)
             assert y.ndim == 2
 
-        return gof.Apply(self, [alpha, x, y, z], [tensor.tensor(dtype=dtype_out, broadcastable=(False, False))])
+        return gof.Apply(self, [alpha, x, y, z],
+                         [tensor.tensor(dtype=dtype_out,
+                                        broadcastable=(False, False))])
 
     def perform(self, node, (alpha, x, y, z), (out, )):
         x_is_sparse = _is_sparse(x)
@@ -1609,6 +1625,7 @@ class Usmm(gof.op.Op):
         out[0] = rval
 usmm = Usmm()
 
+
 class UsmmCscDense(gof.Op):
     """
     Performs the expression is alpha * x y + z
@@ -1621,16 +1638,20 @@ class UsmmCscDense(gof.Op):
     def __init__(self, inplace):
         self.inplace = inplace
         if inplace:
-            self.destroy_map={ 0 : [6] }
+            self.destroy_map = {0: [6]}
+
     def __str__(self):
         if self.inplace:
             return 'UsmmCscDense{inplace}'
         else:
             return 'UsmmCscDense{no_inplace}'
+
     def __eq__(self, other):
         return (type(self) == type(other)) and self.inplace == other.inplace
+
     def __hash__(self):
         return hash(type(self)) ^ self.inplace
+
     def infer_shape(self, node, shapes):
         xshp, yshp = shapes
         x, y = node.inputs
@@ -1643,6 +1664,7 @@ class UsmmCscDense(gof.Op):
         if x.ndim == 1 and y.ndim == 1:
             return [()]
         raise NotImplementedError()
+
     def make_node(self, alpha, x_val, x_ind, x_ptr, x_nrows, y, z):
         alpha = tensor.as_tensor_variable(alpha)
         x_val = tensor.as_tensor_variable(x_val)
@@ -1682,9 +1704,6 @@ class UsmmCscDense(gof.Op):
                 [tensor.tensor(dtype_out, (False, y.type.broadcastable[1]))])
         return r
 
-    #def perform(self, node, (alpha, x_val, x_ind, x_ptr, x_nrows, y, z), (out,)):
-    #    raise NotImplemented()
-    
     def c_support_code(self):
         return blas.blas_header_text()
 
@@ -1700,9 +1719,12 @@ class UsmmCscDense(gof.Op):
     def c_header_dirs(self):
         return blas.ldflags(libs=False, include_dir=True)
 
-    def c_code(self, node, name, (alpha, x_val, x_ind, x_ptr, x_nrows, y, z), (zn,), sub):
+    def c_code(self, node, name, inputs, outputs, sub):
+        alpha, x_val, x_ind, x_ptr, x_nrows, y, z = inputs
+        zn = outputs[0]
         if node.inputs[1].type.dtype in ('complex64', 'complex128'):
-            raise NotImplementedError('Complex types are not supported for x_val')
+            raise NotImplementedError('Complex types are not supported for '
+                                      'x_val')
         if node.inputs[5].type.dtype in ('complex64', 'complex128'):
             raise NotImplementedError('Complex types are not supported for y')
         if node.inputs[6].type.dtype != node.outputs[0].type.dtype:
@@ -1714,12 +1736,12 @@ class UsmmCscDense(gof.Op):
         else:
             conv_type = "double"
             axpy = "daxpy_"
-        
-        typenum_alpha = node.inputs[0].type.dtype_specs()[-1] # retrieve dtype number
-        typenum_x_val = node.inputs[1].type.dtype_specs()[-1] # retrieve dtype number
-        typenum_y = node.inputs[5].type.dtype_specs()[-1] # retrieve dtype number
-        typenum_z = node.inputs[6].type.dtype_specs()[-1] # retrieve dtype number
-        typenum_zn = node.outputs[0].type.dtype_specs()[-1] # retrieve dtype number
+        # retrieve dtype numbers
+        typenum_alpha = node.inputs[0].type.dtype_specs()[-1]
+        typenum_x_val = node.inputs[1].type.dtype_specs()[-1]
+        typenum_y = node.inputs[5].type.dtype_specs()[-1]
+        typenum_z = node.inputs[6].type.dtype_specs()[-1]
+        typenum_zn = node.outputs[0].type.dtype_specs()[-1]
 
         inplace = int(self.inplace)
 
@@ -1839,15 +1861,24 @@ class UsmmCscDense(gof.Op):
                 }
             }
         }
-        """% dict(locals(), **sub)
+        """ % dict(locals(), **sub)
 
         return rval
+
+
 usmm_csc_dense = UsmmCscDense(inplace=False)
 usmm_csc_dense_inplace = UsmmCscDense(inplace=True)
 
-local_usmm = gof.opt.PatternSub((tensor.sub, 'z', (tensor.mul, {'pattern' : 'alpha', 'constraint' : lambda expr: numpy.all(expr.type.broadcastable) },
+
+local_usmm = gof.opt.PatternSub(
+    (tensor.sub, 'z',
+     (tensor.mul,
+      {'pattern': 'alpha',
+       'constraint': lambda expr: numpy.all(expr.type.broadcastable)},
     (_dot, 'x', 'y'))),
     (usmm, (tensor.neg, 'alpha'), 'x', 'y', 'z'))
+
+
 register_specialize(local_usmm, name="local_usmm")
 
 
@@ -1863,15 +1894,18 @@ def local_usmm_csx(node):
             if x.type.format == 'csc':
                 x_val, x_ind, x_ptr, x_shape = csm_properties(x)
                 x_nsparse = x_shape[0]
-                dtype_out = scalar.upcast(alpha.type.dtype, x.type.dtype, y.type.dtype, z.type.dtype)
+                dtype_out = scalar.upcast(alpha.type.dtype, x.type.dtype,
+                                          y.type.dtype, z.type.dtype)
                 # Sparse cast is not implemented.
                 if y.type.dtype != dtype_out:
                     return False
 
-                return [usmm_csc_dense(alpha, x_val, x_ind, x_ptr, x_nsparse, y, z)]
+                return [usmm_csc_dense(alpha, x_val, x_ind, x_ptr,
+                                       x_nsparse, y, z)]
     return False
 register_specialize(local_usmm_csx)
 
+
 @gof.local_optimizer([usmm_csc_dense])
 def local_usmm_csc_dense_inplace(node):
     if node.op == usmm_csc_dense:

theano/sparse/tests/test_basic.py

@@ -12,6 +12,8 @@ except ImportError:
 import theano
 from theano import compile, config
 from theano.sparse import enable_sparse
+from theano.gof.python25 import product
+
 if enable_sparse == False:
     raise SkipTest('Optional package sparse disabled')
 
@@ -534,7 +536,6 @@ class DotTests(unittest.TestCase):
         self.y_csr = scipy.sparse.csr_matrix(numpy.random.binomial(1, 0.5, y_size), dtype=theano.config.floatX)
         self.y_csc = scipy.sparse.csc_matrix(numpy.random.binomial(1, 0.5, y_size), dtype=theano.config.floatX)
 
-    
     def test_csr_dense(self):
         x = theano.sparse.csr_matrix('x')
         y = theano.tensor.matrix('y')
@@ -542,7 +543,7 @@ class DotTests(unittest.TestCase):
         f_a = theano.function([x, y], theano.sparse.dot(x, y))
         f_b = lambda x, y: x * y
 
-        assert abs(f_a(self.x_csr, self.y) - f_b(self.x_csr, self.y)).max() < 10**-4
+        assert abs(f_a(self.x_csr, self.y) - f_b(self.x_csr, self.y)).max() < 1e-4
 
     def test_csc_dense(self):
         x = theano.sparse.csc_matrix('x')
@@ -551,7 +552,9 @@ class DotTests(unittest.TestCase):
         f_a = theano.function([x, y], theano.sparse.dot(x, y))
         f_b = lambda x, y: x * y
 
-        assert abs(f_a(self.x_csc, self.y) - f_b(self.x_csc, self.y)).max() < 10**-4
+        assert (abs(f_a(self.x_csc, self.y) - f_b(self.x_csc, self.y)).max()
+                < 1e-4)
+
     def test_sparse_sparse(self):
         for d1, d2 in [('float32', 'float32'),
                        ('float32', 'float64'),
@@ -571,7 +574,7 @@ class DotTests(unittest.TestCase):
 
                 vx = getattr(self,'x_'+x_f).astype(d1)
                 vy = getattr(self,'y_'+y_f).astype(d2)
-                assert abs(f_a(vx, vy) - f_b(vx, vy)).max() < 10**-4
+                assert abs(f_a(vx, vy) - f_b(vx, vy)).max() < 1e-4
 
 
 class UsmmTests(unittest.TestCase):
@@ -591,28 +594,28 @@ class UsmmTests(unittest.TestCase):
             else:
                 return theano.sparse.matrix(format, name, dtype=dtype)
 
-        for dtype1 in ['float32', 'float64']:
-            for dtype2 in ['float32', 'float64']:
-                for dtype3 in ['float32', 'float64']:
-                    for dtype4 in ['float32', 'float64']:
-                        for format1 in ['dense', 'csc','csr']:
-                            for format2 in ['dense', 'csc','csr']:
+        params = product(*([['float32', 'float64']] * 4 +
+                           [['dense', 'csc', 'csr']] * 2))
+
+        for dtype1, dtype2, dtype3, dtype4, format1, format2 in params:
             if format1 == 'dense' and format2 == 'dense':
                 # Usmm won't be used!
                 continue
             x = mat(format1, 'x', dtype1)
             y = mat(format2, 'y', dtype2)
             a = theano.tensor.scalar('a', dtype=dtype3)
-                                z = theano.tensor.shared(numpy.asarray(self.z,dtype=dtype4).copy())
+            z = theano.tensor.shared(
+                numpy.asarray(self.z, dtype=dtype4).copy()
+            )
 
             f_b = lambda z, a, x, y: z - a * (x * y)
-                                x_data = numpy.asarray(self.x, dtype = dtype1)
+            x_data = numpy.asarray(self.x, dtype=dtype1)
             if format1 != 'dense':
                 x_data = as_sparse_format(x_data, format1)
-                                y_data = numpy.asarray(self.y, dtype = dtype2)
+            y_data = numpy.asarray(self.y, dtype=dtype2)
             if format2 != 'dense':
                 y_data = as_sparse_format(y_data, format2)
-                                z_data = numpy.asarray(self.z, dtype = dtype3)
+            z_data = numpy.asarray(self.z, dtype=dtype3)
 
             f_b_out = f_b(z_data, 1, x_data, y_data)
 
@@ -623,26 +626,37 @@ class UsmmTests(unittest.TestCase):
             mode = theano.compile.mode.get_default_mode().excluding('fusion')
 
             if inplace:
+                updates = {z: z - a * theano.sparse.dot(x, y)}
                 f_a = theano.function([a, x, y], [],
-                                                          updates={ z : z - a * theano.sparse.dot(x, y)},
-                                                          mode = mode)
+                                      updates=updates,
+                                      mode=mode)
                 f_a(1, x_data, y_data)
-                                    assert abs(z.get_value(borrow=True) - f_b_out).max() < 10**-4
+                assert abs(z.get_value(borrow=True) - f_b_out).max() < 1e-4
             else:
-                                    f_a = theano.function([a, x, y], z - a * theano.sparse.dot(x, y),
-                                                          mode = mode)
+                f_a = theano.function([a, x, y],
+                                      z - a * theano.sparse.dot(x, y),
+                                      mode=mode)
                 f_a_out = f_a(1, x_data, y_data)
-                                    assert abs(f_a_out - f_b_out).max() < 10**-4
+                assert abs(f_a_out - f_b_out).max() < 1e-4
             topo = f_a.maker.env.toposort()
-                                if (y.type.dtype == theano.scalar.upcast(dtype1, dtype2, dtype3, dtype4)
-                                    and format1=='csc' and format2=='dense'):
-
-                                    assert sum([isinstance(node.op, tensor.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.basic.Cast) for node in topo])==len(topo)-5
-                                    topo = [node for node in topo if not(isinstance(node.op, tensor.Elemwise) and isinstance(node.op.scalar_op, theano.scalar.basic.Cast))]
-                                    assert len(topo)==5, topo
+            up = theano.scalar.upcast(dtype1, dtype2, dtype3, dtype4)
+            if y.type.dtype == up and format1 == 'csc' and format2 == 'dense':
+                assert (sum([isinstance(node.op, tensor.Elemwise) and
+                             isinstance(node.op.scalar_op,
+                                        theano.scalar.basic.Cast)
+                             for node in topo]) == len(topo) - 5)
+                new_topo = []
+                for node in topo:
+                    if not isinstance(node.op, tensor.Elemwise) and \
+                       isinstance(node.op.scalar_op, theano.scalar.basic.Cast):
+                        new_topo.append(node)
+                topo = new_topo
+                assert len(topo) == 5, topo
                 # Usmm is tested at the same time in debugmode
-                                    # Check if the optimization local_usmm and local_usmm_csx is applied
-                                    assert isinstance(topo[0].op, theano.sparse.basic.CSMProperties)
+                # Check if the optimization local_usmm and local_usmm_csx is
+                # applied
+                assert isinstance(topo[0].op,
+                                  theano.sparse.basic.CSMProperties)
                 assert isinstance(topo[1].op, theano.tensor.DimShuffle)
                 assert isinstance(topo[2].op, theano.tensor.Subtensor)
                 assert topo[3].op == theano.tensor.neg