Adding test for scan and comments in the doc.

doc/tutorial/loop.txt

@@ -119,11 +119,11 @@ The full documentation can be found in the library: :ref:`Scan <lib_scan>`.
   # define tensor variable
   X = T.matrix("X")
   results, updates = theano.scan(lambda x_i:T.sqrt((x_i**2).sum()), sequences=[X.T])
-  compute_norm_lines = theano.function(inputs = [X], outputs=[results])
+  compute_norm_cols = theano.function(inputs = [X], outputs=[results])
   
   # test value
   x = np.diag(np.arange(1,6),1)
-  print compute_norm_lines(x)[0]
+  print compute_norm_cols(x)[0]
   
   # comparison with numpy
   print np.sqrt((x**2).sum(0))
@@ -178,18 +178,20 @@ The full documentation can be found in the library: :ref:`Scan <lib_scan>`.
   w = 0.5*np.ones((2,2))
   u = 0.5*(np.ones((2,2))-np.eye(2))
   v = 0.5*np.ones((2,2))
-  n = 100
+  n = 10
   b = np.ones((2))
   
   print compute_seq2(x,u,v,w,b,n)
   
   # comparison with numpy
-  x_res = np.zeros((100,2))
-  x_res[0] = x[1].dot(v) + np.tanh(x[1].dot(w) + b)
-  x_res[1] = x_res[0].dot(v) + np.tanh(x_res[0].dot(w) + b)
-  x_res[2] = x_res[0].dot(u) + x_res[1].dot(v) + np.tanh(x_res[1].dot(w) + b)
-  for i in range(2,100):
-    x_res[i] = (x_res[i-2].dot(u) + x_res[i-1].dot(v) + np.tanh(x_res[i-1].dot(w) + b))
+  x_res = numpy.zeros((10,2))
+  x_res[0] = x[0].dot(u) + x[1].dot(v) + numpy.tanh(x[1].dot(w) + b)
+  x_res[1] = x[1].dot(u) + x_res[0].dot(v) + numpy.tanh(x_res[0].dot(w) + b)
+  x_res[2] = x_res[0].dot(u) + x_res[1].dot(v) \
+            + numpy.tanh(x_res[1].dot(w) + b)
+  for i in range(2,10):
+    x_res[i] = (x_res[i-2].dot(u) + x_res[i-1].dot(v) \
+            + numpy.tanh(x_res[i-1].dot(w) + b))
 
 **Scan Example: Computing the Jacobian of y = tanh(v*A) wrt x**
 
@@ -214,6 +216,8 @@ The full documentation can be found in the library: :ref:`Scan <lib_scan>`.
   # compare with numpy
   print ((1 - np.tanh(x.dot(w))**2)*w).T
 
+Note that we need to iterate over the indices of ``y`` and not over the elements of ``y``. The reason is that scan create a placeholder variable for its internal function and this placeholder variable does not have the same dependencies than the variables that will replace it. 
+
 **Scan Example: Accumulate number of loop during a scan**
 
 .. code-block:: python
@@ -223,6 +227,7 @@ The full documentation can be found in the library: :ref:`Scan <lib_scan>`.
   
   # define shared variables
   k = theano.shared(0)
+  n_sym = T.iscalar("n_sym")
   
   results, updates = theano.scan(lambda:{k:(k+1)}, n_steps=n_sym)
   accumulator = theano.function([n_sym], [], updates=updates, allow_input_downcast = True)
@@ -239,8 +244,8 @@ The full documentation can be found in the library: :ref:`Scan <lib_scan>`.
   import numpy as np
   
   # define tensor variables
-  v = T.vector()
-  A = T.matrix()
+  X = T.matrix("X")
+  W = T.matrix("W")
   b_sym = T.vector("b_sym")
   
   # define shared random stream
@@ -256,7 +261,7 @@ The full documentation can be found in the library: :ref:`Scan <lib_scan>`.
   
   print compute_with_bnoise(x, w, b)
 
-
+Note that if you want to use a random variable ``d`` that will not be updated through scan loops, you should pass this variable as a ``non_sequences`` arguments. 
 
 **Scan Example: Computing pow(A,k)**
 

theano/tests/test_tutorial.py

@@ -1136,3 +1136,213 @@ class T_graphstructures(unittest.TestCase):
         assert e.owner.inputs[1] is mul_variable
         assert e.owner.inputs[1].owner.inputs[0] is y
         assert e.owner.inputs[1].owner.inputs[1] is z
+
+class T_scan(unittest.TestCase):
+    ## All tests here belong to
+    ## http://deeplearning.net/software/theano/tutorial/loop.html
+    ## Theano/doc/tutorial/loop.txt
+    ## Any change you do here also add it to the tutorial !
+    
+    def test_elemwise(self):
+        
+          # defining the tensor variables
+          X = T.matrix("X")
+          W = T.matrix("W")
+          b_sym = T.vector("b_sym")
+  
+          results, updates = theano.scan(lambda v:T.tanh(T.dot(v,W)+b_sym), \
+                    sequences=X)
+          compute_elementwise = theano.function(inputs = [X, W, b_sym], \
+                    outputs=[results])
+  
+          # test values
+          x = numpy.eye(2)
+          w = numpy.ones((2,2))
+          b = numpy.ones((2))
+          b[1] = 2
+  
+          print "Scan results:", compute_elementwise(x, w, b)[0]
+  
+          # comparison with numpy
+          print "Numpy results:", numpy.tanh(x.dot(w) + b)
+    
+    def test_sequence(self):
+        
+          # define tensor variables
+          X = T.vector("X")
+          W = T.matrix("W")
+          b_sym = T.vector("b_sym")
+          U = T.matrix("U")
+          Y = T.matrix("Y")
+          V = T.matrix("V")
+          P = T.matrix("P")
+  
+          results, updates = theano.scan(lambda \
+                        y,p,x_tm1:T.tanh(T.dot(x_tm1,W) + \
+                        T.dot(y,U)+T.dot(p,V)), \
+                    sequences=[Y,P[::-1]], outputs_info=[X])
+            
+          compute_seq = theano.function(inputs = [X, W, Y, U, P, V], \
+                    outputs=[results])
+  
+          # test values
+          x = numpy.zeros((2))
+          x[1] = 1
+          w = numpy.ones((2,2))
+          y = numpy.ones((5,2))
+          y[0,:] = -3
+          u = numpy.ones((2,2))
+          p = numpy.ones((5,2))
+          p[0,:] = 3
+          v = numpy.ones((2,2))
+  
+          print "Scan results", compute_seq(x,w,y,u,p,v)[0]
+  
+          # comparison with numpy
+          x_res = numpy.zeros((5,2))
+          x_res[0] = numpy.tanh(x.dot(w) + y[0].dot(u) + p[4].dot(v))
+          for i in range(1,5):
+            x_res[i] = numpy.tanh(x_res[i-1].dot(w) \
+                        + y[i].dot(u) + p[4-i].dot(v))
+        
+          print "Numpy results:", x_res
+    
+    def test_norm(self):
+          # define tensor variable
+          X = T.matrix("X")
+          results, updates = theano.scan(lambda x_i:T.sqrt((x_i**2).sum()), \
+                            sequences=[X])
+          compute_norm_lines = theano.function(inputs = [X], outputs=[results])
+          
+          results, updates = theano.scan(lambda x_i:T.sqrt((x_i**2).sum()), \
+                            sequences=[X.T])
+          compute_norm_cols = theano.function(inputs = [X], outputs=[results])
+  
+          # test value
+          x = numpy.diag(numpy.arange(1,6),1)
+          print "Scan results:", compute_norm_lines(x)[0], \
+                            compute_norm_cols(x)[0]
+  
+          # comparison with numpy
+          print "Numpy results:", numpy.sqrt((x**2).sum(1)), \
+                            numpy.sqrt((x**2).sum(0))
+    
+    def test_trace(self):
+        
+          # define tensor variable
+          X = T.matrix("X")
+          results, updates = theano.scan(lambda i, j, t_f:T.cast(X[i,j] + \
+                                t_f, theano.config.floatX), \
+                            sequences=[T.arange(X.shape[0]), \
+                                T.arange(X.shape[1])], \
+                            outputs_info=numpy.zeros_like(0., \
+                                dtype=theano.config.floatX))
+            
+          result = results[-1]
+          compute_trace = theano.function(inputs = [X], outputs=[result])
+  
+          # test value
+          x = numpy.eye(5)
+          x[0] = numpy.arange(5)
+          print "Scan results:", compute_trace(x)[0]
+  
+          # comparison with numpy
+          print "Numpy results:", numpy.diagonal(x).sum()
+
+    def test_taps(self):
+    
+          # define tensor variables
+          X = T.matrix("X")
+          W = T.matrix("W")
+          b_sym = T.vector("b_sym")
+          U = T.matrix("U")
+          V = T.matrix("V")
+          n_sym = T.iscalar("n_sym")
+
+          results, updates = theano.scan(lambda x_tm2,x_tm1:T.dot(x_tm2,U) \
+                        + T.dot(x_tm1,V) + T.tanh(T.dot(x_tm1,W) + b_sym), \
+                    n_steps=n_sym, \
+                    outputs_info=[dict(initial = X, taps = [-2,-1])])
+        
+          compute_seq2 = theano.function(inputs = [X, U, V, W, b_sym, \
+                    n_sym], outputs=[results])
+  
+          # test values
+          x = numpy.zeros((2,2)) 
+          # the initial value must be able to return x[-2]
+          x[1,1] = 1
+          w = 0.5*numpy.ones((2,2))
+          u = 0.5*(numpy.ones((2,2))-numpy.eye(2))
+          v = 0.5*numpy.ones((2,2))
+          n = 10
+          b = numpy.ones((2))
+  
+          print "Scan results:", compute_seq2(x,u,v,w,b,n)
+  
+          # comparison with numpy
+          x_res = numpy.zeros((10,2))
+          x_res[0] = x[0].dot(u) + x[1].dot(v) + numpy.tanh(x[1].dot(w) + b)
+          x_res[1] = x[1].dot(u) + x_res[0].dot(v) + numpy.tanh(x_res[0].dot(w) + b)
+          x_res[2] = x_res[0].dot(u) + x_res[1].dot(v) \
+                    + numpy.tanh(x_res[1].dot(w) + b)
+          for i in range(2,10):
+            x_res[i] = (x_res[i-2].dot(u) + x_res[i-1].dot(v) \
+                    + numpy.tanh(x_res[i-1].dot(w) + b))
+          
+          print "Numpy results:", x_res
+
+    def test_jacobian(self):
+    
+          # define tensor variables
+          v = T.vector()
+          A = T.matrix()
+          y = T.tanh(T.dot(v,A))
+          results, updates = theano.scan(lambda i:T.grad(y[i], v), \
+                    sequences = [T.arange(y.shape[0])])
+          compute_jac_t = theano.function([A,v], [results], \
+                allow_input_downcast = True) # shape (d_out, d_in)
+  
+          # test values
+          x = numpy.eye(5)[0]
+          w = numpy.eye(5,3)
+          w[2] = numpy.ones((3))
+          print "Scan results:", compute_jac_t(w,x)[0]
+  
+          # compare with numpy
+          print "Numpy results:", ((1 - numpy.tanh(x.dot(w))**2)*w).T
+
+    def test_accumulator(self):
+          # define shared variables
+          k = theano.shared(0)
+          n_sym = T.iscalar("n_sym")
+  
+          results, updates = theano.scan(lambda:{k:(k+1)}, n_steps=n_sym)
+          accumulator = theano.function([n_sym], [], updates=updates, \
+                allow_input_downcast = True)
+  
+          print "Before 5 steps:", k.get_value()
+          accumulator(5)
+          print "After 5 steps:", k.get_value()
+    
+    def test_random(self):
+          # define tensor variables
+          X = T.matrix("X")
+          W = T.matrix("W")
+          b_sym = T.vector("b_sym")
+  
+          # define shared random stream
+          trng = T.shared_randomstreams.RandomStreams(1234)
+          d=trng.binomial(size=W[1].shape)
+  
+          results, updates = theano.scan(lambda v:T.tanh(T.dot(v,W) \
+                        + b_sym)*d, sequences=X)
+          compute_with_bnoise = theano.function(inputs = [X, W, b_sym], \
+                        outputs=[results], \
+                        updates=updates, \
+                        allow_input_downcast = True)
+          x = numpy.eye(10,2)
+          w = numpy.ones((2,2))
+          b = numpy.ones((2))
+  
+          print compute_with_bnoise(x, w, b)
+