Added usecases for scan tutorial

doc/tutorial/loop.txt

@@ -24,6 +24,240 @@ Scan
 
 The full documentation can be found in the library: :ref:`Scan <lib_scan>`.
 
+**Scan Example: Computing tanh(x(t)*W + b) elementwise**
+
+.. code-block:: python
+  import theano
+  import theano.tensor as T
+  import numpy as np
+  
+  # defining the tensor variables
+  X = T.matrix("X")
+  W = T.matrix("W")
+  b_sym = T.vector("b_sym")
+  
+  results, updates = theano.scan(lambda v:T.tanh(T.dot(v,W)+b_sym), sequences=X)
+  compute_elementwise = theano.function(inputs = [X, W, b_sym], outputs=[results])
+  
+  # test values
+  x = np.eye(2)
+  w = np.ones((2,2))
+  b = np.ones((2))
+  b[1] = 2
+  
+  print compute_elementwise(x, w, b)[0]
+  
+  # comparison with numpy
+  print np.tanh(x.dot(w) + b)
+  
+
+**Scan Example: Computing the sequence x(t) = tanh(x(t-1)*W + y(t)*U + p(T-t)*V)**
+
+.. code-block:: python
+  import theano
+  import theano.tensor as T
+  import numpy as np
+  
+  # define tensor variables
+  X = T.vector("X")
+  W = T.matrix("W")
+  b_sym = T.vector("b_sym")
+  U = T.matrix("U")
+  Y = T.matrix("Y")
+  V = T.matrix("V")
+  P = T.matrix("P")
+  
+  results, updates = theano.scan(lambda y,p,x_tm1:T.tanh(T.dot(x_tm1,W)+T.dot(y,U)+T.dot(p,V)), \
+                                 sequences=[Y,P[::-1]], outputs_info=[X])
+  compute_seq = theano.function(inputs = [X, W, Y, U, P, V], outputs=[results])
+  
+  # test values
+  x = np.zeros((2))
+  x[1] = 1
+  w = np.ones((2,2))
+  y = np.ones((5,2))
+  y[0,:] = -3
+  u = np.ones((2,2))
+  p = np.ones((5,2))
+  p[0,:] = 3
+  v = np.ones((2,2))
+  
+  print compute_seq(x,w,y,u,p,v)[0]
+  
+  # comparison with numpy
+  x_res = np.zeros((5,2))
+  x_res[0] = np.tanh(x.dot(w) + y[0].dot(u) + p[4].dot(v))
+  for i in range(1,5):
+    x_res[i] = np.tanh(x_res[i-1].dot(w) + y[i].dot(u) + p[4-i].dot(v))
+
+**Scan Example: Computing norms of lines of X**
+
+.. code-block:: python
+  import theano
+  import theano.tensor as T
+  import numpy as np
+  
+  # define tensor variable
+  X = T.matrix("X")
+  results, updates = theano.scan(lambda x_i:T.sqrt((x_i**2).sum()), sequences=[X])
+  compute_norm_lines = theano.function(inputs = [X], outputs=[results])
+  
+  # test value
+  x = np.diag(np.arange(1,6),1)
+  print compute_norm_lines(x)[0]
+  
+  # comparison with numpy
+  print np.sqrt((x**2).sum(1))
+
+**Scan Example: Computing norms of columns of X**
+
+.. code-block:: python
+  import theano
+  import theano.tensor as T
+  import numpy as np
+  
+  # define tensor variable
+  X = T.matrix("X")
+  results, updates = theano.scan(lambda x_i:T.sqrt((x_i**2).sum()), sequences=[X.T])
+  compute_norm_lines = theano.function(inputs = [X], outputs=[results])
+  
+  # test value
+  x = np.diag(np.arange(1,6),1)
+  print compute_norm_lines(x)[0]
+  
+  # comparison with numpy
+  print np.sqrt((x**2).sum(0))
+
+**Scan Example: Computing trace of X**
+
+.. code-block:: python
+  import theano
+  import theano.tensor as T
+  import numpy as np
+  floatX = "float32"
+  
+  # define tensor variable
+  X = T.matrix("X")
+  results, updates = theano.scan(lambda i, j, t_f:T.cast(X[i,j]+t_f, floatX), \
+                    sequences=[T.arange(X.shape[0]), T.arange(X.shape[1])], \
+                    outputs_info=np.zeros_like(0., dtype=floatX))
+  result = results[-1]
+  compute_trace = theano.function(inputs = [X], outputs=[result])
+  
+  # test value
+  x = np.eye(5)
+  x[0] = np.arange(5)
+  compute_trace(x)[0]
+  
+  # comparison with numpy
+  print np.diagonal(x).sum()
+
+**Scan Example: Computing the sequence x(t) = x(t-2)*U + x(t-1)*V +  tanh(x(t-1)*W  + b)**
+
+.. code-block:: python
+  import theano
+  import theano.tensor as T
+  import numpy as np
+  
+  # define tensor variables
+  X = T.matrix("X")
+  W = T.matrix("W")
+  b_sym = T.vector("b_sym")
+  U = T.matrix("U")
+  V = T.matrix("V")
+  n_sym = T.iscalar("n_sym")
+
+  results, updates = theano.scan(lambda x_tm2,x_tm1:T.dot(x_tm2,U) + T.dot(x_tm1,V) \
+                                + T.tanh(T.dot(x_tm1,W) + b_sym), \
+                      n_steps=n_sym, outputs_info=[dict(initial = X, taps = [-2,-1])])
+  compute_seq2 = theano.function(inputs = [X, U, V, W, b_sym, n_sym], outputs=[results])
+  
+  # test values
+  x = np.zeros((2,2)) # the initial value must be able to return x[-2]
+  x[1,1] = 1
+  w = 0.5*np.ones((2,2))
+  u = 0.5*(np.ones((2,2))-np.eye(2))
+  v = 0.5*np.ones((2,2))
+  n = 100
+  b = np.ones((2))
+  
+  print compute_seq2(x,u,v,w,b,n)
+  
+  # comparison with numpy
+  x_res = np.zeros((100,2))
+  x_res[0] = x[1].dot(v) + np.tanh(x[1].dot(w) + b)
+  x_res[1] = x_res[0].dot(v) + np.tanh(x_res[0].dot(w) + b)
+  x_res[2] = x_res[0].dot(u) + x_res[1].dot(v) + np.tanh(x_res[1].dot(w) + b)
+  for i in range(2,100):
+    x_res[i] = (x_res[i-2].dot(u) + x_res[i-1].dot(v) + np.tanh(x_res[i-1].dot(w) + b))
+
+**Scan Example: Computing the Jacobian of y = tanh(v*A) wrt x**
+
+.. code-block:: python
+  import theano
+  import theano.tensor as T
+  import numpy as np
+  
+  # define tensor variables
+  v = T.vector()
+  A = T.matrix()
+  y = T.tanh(T.dot(v,A))
+  results, updates = theano.scan(lambda i:T.grad(y[i], v), sequences = [T.arange(y.shape[0])])
+  compute_jac_t = theano.function([A,v], [results], allow_input_downcast = True) # shape (d_out, d_in)
+  
+  # test values
+  x = np.eye(5)[0]
+  w = np.eye(5,3)
+  w[2] = np.ones((3))
+  print compute_jac_t(w,x)[0]
+  
+  # compare with numpy
+  print ((1 - np.tanh(x.dot(w))**2)*w).T
+
+**Scan Example: Accumulate number of loop during a scan**
+
+.. code-block:: python
+  import theano
+  import theano.tensor as T
+  import numpy as np
+  
+  # define shared variables
+  k = theano.shared(0)
+  
+  results, updates = theano.scan(lambda:{k:(k+1)}, n_steps=n_sym)
+  accumulator = theano.function([n_sym], [], updates=updates, allow_input_downcast = True)
+  
+  k.get_value()
+  accumulator(5)
+  k.get_value()
+
+**Scan Example: Computing tanh(v*W + b)*d where b is binomial**
+
+.. code-block:: python
+  import theano
+  import theano.tensor as T
+  import numpy as np
+  
+  # define tensor variables
+  v = T.vector()
+  A = T.matrix()
+  b_sym = T.vector("b_sym")
+  
+  # define shared random stream
+  trng = T.shared_randomstreams.RandomStreams(1234)
+  d=trng.binomial(size=W[1].shape)
+  
+  results, updates = theano.scan(lambda v:T.tanh(T.dot(v,W)+b_sym)*d, sequences=X)
+  compute_with_bnoise = theano.function(inputs = [X, W, b_sym], outputs=[results], \
+                            updates=updates, allow_input_downcast = True)
+  x = np.eye(10,2)
+  w = np.ones((2,2))
+  b = np.ones((2))
+  
+  print compute_with_bnoise(x, w, b)
+
+
+
 **Scan Example: Computing pow(A,k)**
 
 .. code-block:: python