Merge pull request #2509 from nouiz/tests

theano/sandbox/cuda/tests/test_opt.py

@@ -16,14 +16,14 @@ from theano.tests import unittest_tools as utt
 
 import theano.sandbox.cuda as cuda
 
-if cuda.cuda_available == False:
+if not cuda.cuda_available:
     raise SkipTest('Optional package cuda disabled')
 
 from theano.sandbox.cuda import basic_ops
 from theano.sandbox.cuda.type import CudaNdarrayType
 from theano.scalar.basic_scipy import erfinv
 
-if theano.config.mode=='FAST_COMPILE':
+if theano.config.mode == 'FAST_COMPILE':
     mode_with_gpu = theano.compile.mode.get_mode('FAST_RUN').including('gpu')
     mode_without_gpu = theano.compile.mode.get_mode('FAST_RUN').excluding('gpu')
 else:
@@ -34,14 +34,14 @@ else:
 def test_no_shared_var_graph():
     """Test that the InputToGpuOptimizer optimizer make graph that don't have shared variable compiled too.
     """
-    a=tensor.fmatrix()
-    b=tensor.fmatrix()
-    f = theano.function([a,b],[a+b], mode=mode_with_gpu)
+    a = tensor.fmatrix()
+    b = tensor.fmatrix()
+    f = theano.function([a, b], [a + b], mode=mode_with_gpu)
     l = f.maker.fgraph.toposort()
-    assert len(l)==4
-    assert numpy.any(isinstance(x.op,cuda.GpuElemwise) for x in l)
-    assert numpy.any(isinstance(x.op,cuda.GpuFromHost) for x in l)
-    assert numpy.any(isinstance(x.op,cuda.HostFromGpu) for x in l)
+    assert len(l) == 4
+    assert numpy.any(isinstance(x.op, cuda.GpuElemwise) for x in l)
+    assert numpy.any(isinstance(x.op, cuda.GpuFromHost) for x in l)
+    assert numpy.any(isinstance(x.op, cuda.HostFromGpu) for x in l)
 
 
 def test_local_assert():
@@ -66,8 +66,6 @@ def test_int_pow():
     op_names = [n.op.__class__.__name__ for n in f.maker.fgraph.toposort()]
     assert op_names == ['GpuElemwise', 'GpuCAReduce', 'HostFromGpu']
 
-    #theano.printing.debugprint(f)
-
 
 def test_gpualloc():
     '''
@@ -95,12 +93,12 @@ class Test_local_elemwise_alloc(test_opt.Test_local_elemwise_alloc):
         super(Test_local_elemwise_alloc, self).setUp()
         self.fast_run_mode = mode_with_gpu
 
-        #self.vec = tensor.vector('vec', dtype=dtype)
-        #self.mat = tensor.matrix('mat', dtype=dtype)
-        #self.tens = tensor.tensor3('tens', dtype=dtype)
+        # self.vec = tensor.vector('vec', dtype=dtype)
+        # self.mat = tensor.matrix('mat', dtype=dtype)
+        # self.tens = tensor.tensor3('tens', dtype=dtype)
 
-        #self.alloc_wo_dep = basic_ops.gpu_alloc(self.vec, 2, 2)
-        #self.alloc_w_dep = basic_ops.gpu_alloc(self.vec, *self.mat.shape)
+        # self.alloc_wo_dep = basic_ops.gpu_alloc(self.vec, 2, 2)
+        # self.alloc_w_dep = basic_ops.gpu_alloc(self.vec, *self.mat.shape)
 
         self.alloc_wo_dep = basic_ops.gpu_alloc(self.vec, 2, 2)
         self.alloc_w_dep = basic_ops.gpu_alloc(self.vec, *self.mat.shape)
@@ -166,7 +164,7 @@ def test_alloc_memset_0():
 
 
 def test_gpuspecifyshape():
-    x = cuda.shared_constructor(numpy.ones(3,dtype='float32'), 'x')
+    x = cuda.shared_constructor(numpy.ones(3, dtype='float32'), 'x')
     m = theano.tensor.specify_shape(x + numpy.float32(1), (3,))
     f = theano.function([], updates=[(x, m * numpy.float32(2))],
                         mode=mode_with_gpu)
@@ -174,41 +172,44 @@ def test_gpuspecifyshape():
     assert not numpy.any([isinstance(x.op, cuda.HostFromGpu) for x in l])
 
 
-
 def test_softmax():
     x = tensor.fmatrix()
 
-    f = theano.function([x],tensor.nnet.nnet.Softmax()(x), mode=mode_with_gpu)
-    f2 = theano.function([x],tensor.nnet.nnet.Softmax()(x), mode=mode_without_gpu)
-    assert isinstance(f.maker.fgraph.toposort()[1].op,cuda.nnet.GpuSoftmax)
-    xv=numpy.random.rand(7,8).astype('float32')
-    assert numpy.allclose(f(xv),f2(xv))
+    f = theano.function([x], tensor.nnet.nnet.Softmax()(x),
+                        mode=mode_with_gpu.excluding('cudnn'))
+    f2 = theano.function([x], tensor.nnet.nnet.Softmax()(x),
+                         mode=mode_without_gpu)
+    assert isinstance(f.maker.fgraph.toposort()[1].op, cuda.nnet.GpuSoftmax)
+    xv = numpy.random.rand(7, 8).astype('float32')
+    assert numpy.allclose(f(xv), f2(xv))
 
 
 def test_softmax_with_bias():
     x = tensor.fmatrix()
     b = tensor.fvector()
 
-    f = theano.function([x,b],tensor.nnet.nnet.SoftmaxWithBias()(x,b), mode=mode_with_gpu)
-    f2 = theano.function([x,b],tensor.nnet.nnet.SoftmaxWithBias()(x,b), mode=mode_without_gpu)
-    assert isinstance(f.maker.fgraph.toposort()[2].op,cuda.nnet.GpuSoftmaxWithBias)
-    xv=numpy.random.rand(7,8).astype('float32')
-    bv=numpy.random.rand(8).astype('float32')
-    assert numpy.allclose(f(xv,bv),f2(xv,bv))
+    f = theano.function([x, b], tensor.nnet.nnet.SoftmaxWithBias()(x, b),
+                        mode=mode_with_gpu)
+    f2 = theano.function([x, b], tensor.nnet.nnet.SoftmaxWithBias()(x, b),
+                         mode=mode_without_gpu)
+    assert isinstance(f.maker.fgraph.toposort()[2].op,
+                      cuda.nnet.GpuSoftmaxWithBias)
+    xv = numpy.random.rand(7, 8).astype('float32')
+    bv = numpy.random.rand(8).astype('float32')
+    assert numpy.allclose(f(xv, bv), f2(xv, bv))
+
 
 def test_opt_gpujoin_onlyajoin():
     # from a bug in normal sampling
-    _a = numpy.asarray([[1,2],[3,4]],dtype='float32')
-    _b = numpy.asarray([[5,6,7],[8,9,10]],dtype='float32')
+    _a = numpy.asarray([[1, 2], [3, 4]], dtype='float32')
+    _b = numpy.asarray([[5, 6, 7], [8, 9, 10]], dtype='float32')
     a = cuda.shared_constructor(_a)
     b = cuda.shared_constructor(_b)
 
-    c = tensor.join(1,a,b)
+    c = tensor.join(1, a, b)
 
     f = theano.function([], c, mode=mode_with_gpu)
 
-    #theano.printing.debugprint(f)
-
     f()
 
     graph_nodes = f.maker.fgraph.toposort()
@@ -216,35 +217,32 @@ def test_opt_gpujoin_onlyajoin():
     assert isinstance(graph_nodes[-1].op, cuda.HostFromGpu)
     assert isinstance(graph_nodes[-2].op, cuda.GpuJoin)
 
-    assert numpy.all(f() == numpy.concatenate([_a,_b], axis=1))
-
+    assert numpy.all(f() == numpy.concatenate([_a, _b], axis=1))
 
 
 def test_opt_gpujoin_joinvectors_elemwise_then_minusone():
     # from a bug in gpu normal sampling
-    _a = numpy.asarray([1,2,3,4],dtype='float32')
-    _b = numpy.asarray([5,6,7,8],dtype='float32')
+    _a = numpy.asarray([1, 2, 3, 4], dtype='float32')
+    _b = numpy.asarray([5, 6, 7, 8], dtype='float32')
     a = cuda.shared_constructor(_a)
     b = cuda.shared_constructor(_b)
 
     a_prime = tensor.cos(a)
     b_prime = tensor.sin(b)
 
-    c = tensor.join(0,a_prime,b_prime)
+    c = tensor.join(0, a_prime, b_prime)
 
     d = c[:-1]
 
     f = theano.function([], d, mode=mode_with_gpu)
 
-    #theano.printing.debugprint(f)
-
     graph_nodes = f.maker.fgraph.toposort()
 
     assert isinstance(graph_nodes[-1].op, cuda.HostFromGpu)
     assert isinstance(graph_nodes[-2].op, cuda.GpuSubtensor)
     assert isinstance(graph_nodes[-3].op, cuda.GpuJoin)
 
-    concat = numpy.concatenate([numpy.cos(_a),numpy.sin(_b)],axis=1)
+    concat = numpy.concatenate([numpy.cos(_a), numpy.sin(_b)], axis=1)
     concat = concat[:-1]
 
     assert numpy.allclose(numpy.asarray(f()), concat)
@@ -312,19 +310,20 @@ def test_local_split():
     # Check equality
     assert all([(cpu == gpu).all() for cpu, gpu in zip(cpu_res, gpu_res)])
 
+
 def test_print_op():
     """ Test that print ops don't block gpu optimization"""
     b = tensor.fmatrix()
-    f = theano.function([b],theano.printing.Print()(b)*2, mode=mode_with_gpu)
-    #theano.printing.debugprint(f)
-    #print f.maker.fgraph.toposort()
+    f = theano.function([b], theano.printing.Print()(b)*2, mode=mode_with_gpu)
+    # theano.printing.debugprint(f)
+    # print f.maker.fgraph.toposort()
 #[GpuFromHost(<TensorType(float32, matrix)>), <theano.printing.Print object at 0x3581210>(GpuFromHost.0), GpuElemwise{mul}(CudaNdarray{[[ 2.]]}, <theano.printing.Print object at 0x3581210>.0), HostFromGpu(GpuElemwise{mul}.0)]
     topo = f.maker.fgraph.toposort()
     assert topo[0].op == cuda.gpu_from_host
     assert isinstance(topo[1].op, theano.printing.Print)
     assert isinstance(topo[2].op, cuda.GpuElemwise)
     assert topo[3].op == cuda.host_from_gpu
-    f(numpy.random.random((5,5)).astype('float32'))
+    f(numpy.random.random((5, 5)).astype('float32'))
 
 
 def test_huge_elemwise_fusion():
@@ -348,14 +347,11 @@ def test_huge_elemwise_fusion():
     f = pfunc(vars, [reduce(operator.sub, vars)], mode=mode_with_gpu)
 
     topo = f.maker.fgraph.toposort()
-    #theano.printing.debugprint(f)
-    #for i, node in enumerate(topo):
-    #    print >> sys.stdout, i, node
     assert len(topo) == len_topo
     assert sum([isinstance(node.op, cuda.GpuElemwise) for node in topo]) == 2
     assert isinstance(topo[-3].op.scalar_op, theano.scalar.basic.Sub)
     assert isinstance(topo[-2].op.scalar_op, theano.scalar.basic.Composite)
-    #let debugmode catch errors
+    # let debugmode catch errors
     gen = lambda: theano._asarray(numpy.random.rand(*shape), dtype='float32')
     f(*[gen() for i in range(nb_in)])
 
@@ -368,11 +364,10 @@ def test_huge_elemwise_fusion():
     f = pfunc(vars, [vars[0] - vars[1] - vars[2] - vars[3] - vars[4] -
                      vars[5] - vars[6]], mode=mode_with_gpu)
     topo = f.maker.fgraph.toposort()
-    #theano.printing.debugprint(f)
     assert len(topo) == 1
     assert sum([isinstance(node.op, cuda.GpuElemwise) for node in topo]) == 0
     assert sum([isinstance(node.op, tensor.Elemwise) for node in topo]) == 1
-    #let debugmode catch errors
+    # let debugmode catch errors
     gen = lambda: theano._asarray(numpy.random.rand(*shape), dtype='float32')
     f(gen(), gen(), gen(), gen(), gen(), gen(), gen())
 
@@ -402,14 +397,14 @@ def test_huge_elemwise_fusion():
                     out = cuda.gpu_from_host(out)
                 f = pfunc([], [out], mode=mode_with_gpu)
                 topo = f.maker.fgraph.toposort()
-                #print shape, nb_var, use_tan, len(topo)
+                # print shape, nb_var, use_tan, len(topo)
                 assert (sum([isinstance(node.op, cuda.GpuElemwise)
                              for node in topo]) == len(topo) or
-                        (nb_var == 1 and use_tan == False))
+                        (nb_var == 1 and use_tan is False))
                 assert sum([isinstance(node.op, tensor.Elemwise)
                             for node in topo]) == 0
 
-                #let debugmode catch errors
+                # let debugmode catch errors
                 f()
 
 
@@ -428,7 +423,6 @@ def test_local_gpu_elemwise_0():
     # Due to optimization order, this composite is created when all
     # the op are on the gpu.
     f = theano.function([a, b, c], [a + b + c], mode=mode_with_gpu)
-    #theano.printing.debugprint(f)
     topo = f.maker.fgraph.toposort()
     assert sum(isinstance(node.op, cuda.GpuElemwise) for node in topo) == 1
     assert sum(isinstance(node.op, tensor.Elemwise) for node in topo) == 1
@@ -442,7 +436,6 @@ def test_local_gpu_elemwise_0():
     out_s = theano.scalar.Composite([a_s, b_s, c_s], [a_s + b_s + c_s])
     out_op = tensor.Elemwise(out_s)
     f = theano.function([a, b, c], [out_op(a, b, c)], mode=mode_with_gpu)
-    #theano.printing.debugprint(f)
     topo = f.maker.fgraph.toposort()
     assert sum(isinstance(node.op, cuda.GpuElemwise) for node in topo) == 1
     assert sum(isinstance(node.op, tensor.Elemwise) for node in topo) == 1
@@ -462,7 +455,7 @@ def test_elemwise_fusion():
         print >> sys.stdout, i, node
     assert len(topo) == 4
     assert isinstance(topo[2].op.scalar_op, theano.scalar.basic.Composite)
-    #let debugmode catch errors
+    # let debugmode catch errors
     f(theano._asarray(numpy.random.rand(*shape), dtype='float32'),
       theano._asarray(numpy.random.rand(*shape), dtype='float32'))
 
@@ -479,6 +472,7 @@ class TestIfElse(theano.tests.test_ifelse.test_ifelse):
     def get_ifelse(self, n):
         return theano.ifelse.IfElse(n, gpu=True, as_view=True)
 
+
 def test_incsubtensor_mixed():
 
     # This catches a bug that occurred when incrementing
@@ -491,14 +485,14 @@ def test_incsubtensor_mixed():
     # fail.
     X = tensor.fmatrix()
     Y = tensor.dmatrix()
-    Z = tensor.inc_subtensor(X[0:1,0:1],Y)
-    f = theano.function([X,Y], Z, mode=mode_with_gpu)
+    Z = tensor.inc_subtensor(X[0:1, 0:1], Y)
+    f = theano.function([X, Y], Z, mode=mode_with_gpu)
     packed, = f.maker.fgraph.inputs[1].clients
     client, idx = packed
     print client
     assert isinstance(client.op, tensor.Elemwise)
     assert isinstance(client.op.scalar_op, theano.scalar.Cast)
-    packed ,= client.outputs[0].clients
+    packed, = client.outputs[0].clients
     client, idx = packed
     assert isinstance(client.op, cuda.GpuFromHost)
 
@@ -507,11 +501,13 @@ def test_erfinvgpu():
     """ Test that local_gpu_elemwise_0 replaces Erfinv with ErfinvGPU """
     x = tensor.fmatrix()
     f = theano.function([x], tensor.Elemwise(erfinv)(x), mode=mode_with_gpu)
-    f2 = theano.function([x], tensor.Elemwise(erfinv)(x), mode=mode_without_gpu)
+    f2 = theano.function([x], tensor.Elemwise(erfinv)(x),
+                         mode=mode_without_gpu)
     assert isinstance(f.maker.fgraph.toposort()[1].op, cuda.GpuElemwise)
-    assert isinstance(f.maker.fgraph.toposort()[1].op.scalar_op, cuda.elemwise.ErfinvGPU)
-    xv=numpy.random.rand(7,8).astype('float32')
-    assert numpy.allclose(f(xv),f2(xv))
+    assert isinstance(f.maker.fgraph.toposort()[1].op.scalar_op,
+                      cuda.elemwise.ErfinvGPU)
+    xv = numpy.random.rand(7, 8).astype('float32')
+    assert numpy.allclose(f(xv), f2(xv))
 
 
 def test_local_gpu_dot_to_dot22dot():