Correct Theanos's tutorial: round of corrections, add exercises and their links

doc/library/scan.txt

@@ -136,26 +136,41 @@ arange must have its length specified at creation time.
 Simple accumulation into a scalar, ditching lamba
 -------------------------------------------------
 
-This should be fairly self-explanatory.
+Although this example would seem almost self-explanatory, it stresses a
+pitfall to be careful of: the initial output state that is supplied, that is 
+``output_info``, must be of a **shape similar to that of the output variable**
+generated at each iteration and moreover, it **must not involve an implicit
+downcast** of the latter. 
 
 .. code-block:: python
 
-    up_to = T.iscalar("up_to")
 
-    # define a named function, rather than using lambda
-    def accumulate_by_adding(arange_val, sum_to_date):
-        return sum_to_date + arange_val
+import numpy as np
+import theano
+import theano.tensor as T
 
-    scan_result, scan_updates = theano.scan(fn=accumulate_by_adding,
-                                            outputs_info=T.as_tensor_variable(0),
-                                            sequences=T.arange(up_to))
-    triangular_sequence = theano.function(inputs=[up_to], outputs=scan_result)
+up_to = T.iscalar("up_to")
 
-    # test
-    some_num = 15
-    print triangular_sequence(some_num)
-    print [n * (n + 1) // 2 for n in xrange(some_num)]
+# define a named function, rather than using lambda
+def accumulate_by_adding(arange_val, sum_to_date):
+    return sum_to_date + arange_val
+seq = T.arange(up_to)
+
+# An unauthorized implicit downcast from the dtype of 'seq', to that of 
+# 'T.as_tensor_variable(0)' which is of dtype 'int8' by default would occur
+# if this instruction were to be used instead of the next one:
+# outputs_info = T.as_tensor_variable(0)
+
+outputs_info = T.as_tensor_variable(np.asarray(0, seq.dtype))
+scan_result, scan_updates = theano.scan(fn=accumulate_by_adding,
+                                        outputs_info=outputs_info,
+                                        sequences=seq)
+triangular_sequence = theano.function(inputs=[up_to], outputs=scan_result)
 
+# test
+some_num = 15
+print triangular_sequence(some_num)
+print [n * (n + 1) // 2 for n in xrange(some_num)]
 
 
 Another simple example

doc/tutorial/adding.txt

@@ -183,4 +183,9 @@ with NumPy arrays may be found here: :ref:`tensor creation<libdoc_tensor_creatio
   
 Modify and execute this code to compute this expression: a**2 + b**2 + 2*a*b.
 
+
+.. TODO: repair this link
+
+:download:`Solution<../adding_solution_1.py>`
+
 -------------------------------------------

doc/tutorial/adding_solution_1.py

+
+# Theano tutorial
+# Solution to Exercise in section 'Baby Steps - Algebra'
+
+import theano
+a = theano.tensor.vector()  # declare variable
+b = theano.tensor.vector()  # declare variable
+out = a ** 2 + b ** 2 + 2 * a * b  # build symbolic expression
+f = theano.function([a, b], out)   # compile function
+print f([1, 2], [4, 5])  # prints [ 25.  49.]
+

doc/tutorial/debug_faq.txt

@@ -144,7 +144,7 @@ The ``compute_test_value`` mechanism works as follows:
   which do not implement a ``perform`` method.
 
 
-How do I Print an Intermediate Value in a Function/Method?
+"How do I Print an Intermediate Value in a Function/Method?"
 ----------------------------------------------------------
 
 Theano provides a 'Print' op to do this.
@@ -259,8 +259,8 @@ Use your imagination :)
 This can be a really powerful debugging tool. Note the call to *fn* inside the call to
 *print_eval*; without it, the graph wouldn't get computed at all!
 
-How to Use pdb ?
-----------------
+How to Use pdb
+--------------
 
 In the majority of cases, you won't be executing from the interactive shell
 but from a set of Python scripts. In such cases, the use of the Python

doc/tutorial/extending_theano.txt

@@ -10,8 +10,8 @@ Theano Graphs
 
 - Theano works with symbolic graphs.
 - Those graphs are bi-partite graphs (graph with 2 types of nodes).
-- The 2 types of nodes are Apply and Variable nodes.
-- Each Apply node has a link to the op that it executes.
+- The two types of nodes are ``Apply`` and ``Variable`` nodes.
+- Each ``Apply`` node has a link to the op that it executes.
 
 Inputs and Outputs are lists of Theano variables.
 
@@ -21,24 +21,25 @@ Inputs and Outputs are lists of Theano variables.
 .. note::
 
     This tutorial does not cover how to make an op that returns a view or
-    modify the values in its inputs. So all
-    Ops created with the instructions here MUST return newly allocated
+    modifies the values in its inputs. Thus, all ops created with the 
+    instructions described here MUST return newly allocated
     memory or reuse the memory provided in the parameter
-    output_storage of the :func:`perform` function. See :ref:`views_and_inplace`
-    for explanation of how to do this.
+    ``output_storage`` of the :func:`perform` function. See :ref:`views_and_inplace`
+    for an explanation on how to do this.
 
-    If your Op returns a view or change the value on its inputs
-    without doing as said in that page, Theano will run, but will
-    return good results for some graphs, but bad results for others.
+    If your op returns a view or changes the value of its inputs
+    without doing as prescribed in that page, Theano will run, but will
+    return correct results for some graphs and wrong results for others.
 
-    It is recommented that you run your tests in DebugMode (Theano flag
-    mode=DebugMode) that checks if your Op behaves correctly in this
+    It is recommended that you run your tests in DebugMode (Theano *flag*
+    ``mode=DebugMode``) since it verifies if your op behaves correctly in this
     regard.
 
 .. note::
 
-   See the :ref:`dev_start_guide` for information about git, github, the
-   development workflow and how to make a quality contribution.
+   See the :ref:`dev_start_guide` for information regarding the versioning
+   framework, namely about *git* and *GitHub*, regarding the development workflow and
+   how to make a quality contribution.
 
 
 Op Contract
@@ -90,7 +91,7 @@ Op Contract
 
 .. ../extending/op.txt
 
-There are 2 mandatory methods that one needs to implement.
+There are two mandatory methods that one needs to implement.
 The first one is :func:`make_node`. The second one 
 would describe the computations that are required to be done
 at run time. Currently there are 2 different possibilites:
@@ -105,7 +106,7 @@ a ``thunk``: a standalone function that when called will do the wanted computati
 This is useful if you want to generate code and compile it yourself. For
 example, this allows you to use PyCUDA to compile GPU code.
 
-Also there are 2 methods that are highly recommended to be implemented. They are
+Also there are two methods whose implementations are highly recommended. They are
 needed in order to merge duplicate computations involving your op. So if you
 do not want Theano to execute your op multiple times with the same inputs,
 do implement them. Those methods are :func:`__eq__` and
@@ -182,9 +183,9 @@ You can try it as follows:
 How To Test it
 --------------
 
-Theano has some functions to simplify testing. These help test the
+Theano has some functionalities to simplify testing. These help test the
 ``infer_shape``, ``grad`` and ``R_op`` methods. Put the following code
-in a file and execute it with the ``nosetests`` program.
+in a file and execute it with the ``theano-nose`` program.
 
 **Basic Tests**
 
@@ -304,10 +305,24 @@ To perform your tests, you may select either one of the three following methods:
 
 **theano-nose**
 
-The method of choice to run every test located in a specific folder
-or in Theano's path is to run the file ``theano-nose`` which is by default located in
-the ``theano/bin`` folder. If a folder is specified on the command line, every test
-it contains will be conducted. Otherwise, every test found in Theano's path will be.
+The method of choice to conduct tests is to run the file ``theano-nose``. In a regular
+Theano installation, the latter will be on the operating system's path and directly accessible.
+Otherwise, it can be accessed in the ``Theano/bin`` folder. The following command
+lines may be used for the corresponding purposes:
+
+* ``theano-nose``: Run every test found in Theano's path.
+
+* ``theano-nose folder_name``: Run every test found in the folder *folder_name*.
+
+* ``theano-nose test_file.py``: Run every test found in the file *test_file.py*.
+
+The following are particularly useful for development purposes since they call for
+particular classes or even for particular tests: 
+
+* ``theano-nose test_file.py:test_DoubleRop``: Run every test found inside the class *test_DoubleRop*.
+
+* ``theano-nose test_file.py:test_DoubleRop.test_double_op``: Run only the test *test_double_op*
+  in the class *test_DoubleRop*.
 
 Help with the use and functionalities of ``theano-nose`` may be obtained by running
 it with the command line parameter ``--help (-h)``. 
@@ -315,18 +330,10 @@ it with the command line parameter ``--help (-h)``.
 **nosetests**
 
 The command ``nosetests`` can also be used.  Although it lacks the useful 
-functionalities that ``theano-nose`` provides, you can use ``nosetests`` similarly
-to ``theano-nose`` to run all tests in Theano's
-path or in a specific folder. Nevertheless, ``nosetests`` is particularly useful for development
-purposes since it allows to request for specific tests.  For instance, the following command
-lines may be used for these respective purposes:
-
-* ``nosetests test_file.py``: Run every test found in the file *test_file.py*.
+functionalities that ``theano-nose`` provides, ``nosetests`` can be called similarly
+to ``theano-nose`` from any folder in Python's path like so:
 
-* ``nosetests test_file.py:test_DoubleRop``: Run every test found inside the class *test_DoubleRop*.
-
-* ``nosetests test_file.py:test_DoubleRop.test_double_op``: Run only the test *test_double_op*
-  in the class *test_DoubleRop*.
+``nosetests [suffix similar to the above]``.
 
 More documentation on ``nosetests`` is available here:
 `nosetests <http://readthedocs.org/docs/nose/en/latest/>`_.
@@ -351,12 +358,18 @@ file containing a specific test of interest and run the file. In this example, t
 **Exercise**
 
 
-Run the code in the file double_op.py.
+Run the code of the *DoubleOp* example above.
 
 Modify and execute to compute: x * y.
 
-Modify and execute the example to return 2 outputs: x + y and x - y
-(our current element-wise fusion generates computation with only 1 output).
+Modify and execute the example to return two outputs: x + y and x - y.
+
+You can omit the Rop functions. Try to implement the testing apparatus described above.
+
+(Notice that Theano's current *elemwise fusion* optimization is
+only applicable to computations involving a single output. Hence, to gain
+efficiency over the basic solution that is asked here, the two operations would
+have to be jointly optimized explicitly in the code.)
 
 SciPy
 -----
@@ -401,6 +414,11 @@ don't forget to call the parent ``setUp`` function.
 For more details see :ref:`random_value_in_tests`.
 
 
+.. TODO: repair this link
+
+:download:`Solution<../extending_theano_solution_1.py>`
+
+-------------------------------------------
 
 
 **A Final Note:**

doc/tutorial/extending_theano_solution_1.py

+
+# Theano tutorial
+# Solution to Exercise in section 'Extending Theano'
+
+
+import theano
+
+
+# 1. Op returns x * y
+
+class ProdOp(theano.Op):
+    def __eq__(self, other):
+        return type(self) == type(other)
+
+    def __hash__(self):
+        return hash(type(self))
+
+    def __str__(self):
+        return self.__class__.__name__
+
+    def make_node(self, x, y):
+        x = theano.tensor.as_tensor_variable(x)
+        y = theano.tensor.as_tensor_variable(y)
+        outdim = x.ndim
+        output = (theano.tensor.TensorType
+                  (dtype=theano.scalar.upcast(x.dtype, y.dtype),
+                      broadcastable=[False] * outdim)())
+        return theano.Apply(self, inputs=[x, y], outputs=[output])
+
+    def perform(self, node, inputs, output_storage):
+        x, y = inputs
+        z = output_storage[0]
+        z[0] = x * y
+
+    def infer_shape(self, node, i0_shapes):
+        return [i0_shapes[0]]
+
+    def grad(self, inputs, output_grads):
+        return [output_grads[0] * inputs[1], output_grads[0] * inputs[0]]
+
+
+# 2. Op returns x + y and x - y
+
+class SumDiffOp(theano.Op):
+    def __eq__(self, other):
+        return type(self) == type(other)
+
+    def __hash__(self):
+        return hash(type(self))
+
+    def __str__(self):
+        return self.__class__.__name__
+
+    def make_node(self, x, y):
+        x = theano.tensor.as_tensor_variable(x)
+        y = theano.tensor.as_tensor_variable(y)
+        outdim = x.ndim
+        output1 = (theano.tensor.TensorType
+                  (dtype=theano.scalar.upcast(x.dtype, y.dtype),
+                      broadcastable=[False] * outdim)())
+        output2 = (theano.tensor.TensorType
+                  (dtype=theano.scalar.upcast(x.dtype, y.dtype),
+                      broadcastable=[False] * outdim)())
+        return theano.Apply(self, inputs=[x, y], outputs=[output1, output2])
+
+    def perform(self, node, inputs, output_storage):
+        x, y = inputs
+        z1, z2 = output_storage
+        z1[0] = x + y
+        z2[0] = x - y
+
+    def infer_shape(self, node, i0_shapes):
+        return [i0_shapes[0], i0_shapes[0]]
+
+    def grad(self, inputs, output_grads):
+        return [output_grads[0] + output_grads[1],
+                output_grads[0] - output_grads[1]]
+
+
+# 3. Testing apparatus
+
+import numpy
+from theano.gof import Op, Apply
+from theano import tensor, function, printing
+from theano.tests import unittest_tools as utt
+
+
+class TestOp(utt.InferShapeTester):
+
+    rng = numpy.random.RandomState(43)
+
+    def setUp(self):
+
+        super(TestOp, self).setUp()
+        # adapt the choice of the next instruction to the op under test
+        self.op_class = ProdOp  # case 1
+        #self.op_class = SumDiffOp  # case 2
+
+    def test_perform(self):
+
+        x = theano.tensor.matrix()
+        y = theano.tensor.matrix()
+        f = theano.function([x, y], self.op_class()(x, y))
+        import numpy
+        x_val = numpy.random.rand(5, 4)
+        y_val = numpy.random.rand(5, 4)
+        out = f(x_val, y_val)
+        # adapt the choice of the next instruction to the op under test
+        assert numpy.allclose(x_val * y_val, out)  # case 1
+        #assert numpy.allclose([x_val + y_val, x_val - y_val], out)  # case 2
+
+    def test_gradient(self):
+
+        utt.verify_grad(self.op_class(), [numpy.random.rand(5, 4),
+                                numpy.random.rand(5, 4)],
+                        n_tests=1, rng=TestOp.rng)
+
+    def test_infer_shape(self):
+
+        x = tensor.dmatrix()
+        y = tensor.dmatrix()
+
+        # adapt the choice of the next instruction to the op under test
+       
+        self._compile_and_check([x, y], [self.op_class()(x, y)],  # case 1
+                                [numpy.random.rand(5, 6),
+                                 numpy.random.rand(5, 6)],
+                                self.op_class)
+        """
+        
+        self._compile_and_check([x, y], self.op_class()(x, y),  # case 2
+                                [numpy.random.rand(5, 6),
+                                 numpy.random.rand(5, 6)],
+                                self.op_class)
+        """
+
+if __name__ == "__main__":
+
+    t = TestOp('setUp')
+    t.setUp()
+    t.test_perform()
+    # comment out next instruction in case 2 since autotesting of
+    # gradient of multiple output functions is not implemented yet
+    t.test_gradient() # enable in case 1, disable in case 2
+    t.test_infer_shape()

doc/tutorial/faq.txt

@@ -15,7 +15,7 @@ be implemented on Theano variables:
 
    TypeError: object of type 'TensorVariable' has no len()
 
-Python requires that *__len__* returns an integer, yet it cannot be done as Theano's symbolic variables. However, `var.shape[0]` can be used as a workaround.
+Python requires that *__len__* returns an integer, yet it cannot be done as Theano's are symbolic variables. However, `var.shape[0]` can be used as a workaround.
 
 This error message cannot be made more explicit because the relevant aspects of Python's 
 internals cannot be modified.
@@ -36,3 +36,18 @@ Related Projects
 
 We try to list in this `wiki page <https://github.com/Theano/Theano/wiki/Related-projects>`_ other Theano related projects.
 
+
+"What are Theano's Limitations?"
+-------------------------------
+
+Theano offers a good amount of flexibility, but has some limitations too.
+You must answer for yourself the following question: How can my algorithm be cleverly written 
+so as to make the most of what Theano can do?
+
+Here is a list of some of the known limitations:
+
+- *While*- or *for*-Loops within an expression graph are supported, but only via
+  the :func:`theano.scan` op (which puts restrictions on how the loop body can
+  interact with the rest of the graph).
+
+- Neither *goto* nor *recursion* is supported or planned within expression graphs.

doc/tutorial/gradients.txt

@@ -98,7 +98,7 @@ Computing the Jacobian
 ======================
 
 In Theano's parlance, the term *Jacobian* designates the tensor comprising the
-first differences of the output of a function with respect to its inputs.
+first partial derivatives of the output of a function with respect to its inputs.
 (This is a generalization of to the so-called Jacobian matrix in Mathematics.) 
 Theano implements the :func:`theano.gradient.jacobian` macro that does all
 that is needed to compute the Jacobian. The following text explains how
@@ -115,7 +115,7 @@ do is to loop over the entries in *y* and compute the gradient of
     manner all kinds of recurrent equations. While creating
     symbolic loops (and optimizing them for performance) is a hard task,
     effort is being done for improving the performance of ``scan``. We 
-    shall return to ``scan`` later in this tutorial.
+    shall return to :ref:`scan<tutloop>` later in this tutorial.
 
 >>> x = T.dvector('x')
 >>> y = x**2

doc/tutorial/index.txt

@@ -18,7 +18,8 @@ of Theano. Let us import that subpackage under a handy name like
 If that succeeded you are ready for the tutorial, otherwise check your
 installation (see :ref:`install`).
 
-Throughout the tutorial, bear in mind that there is a :ref:`glossary` to help
+Throughout the tutorial, bear in mind that there is a :ref:`glossary` as well
+as *index* and *modules* links in the upper-right corner of each page to help
 you out.
 
 .. toctree::

doc/tutorial/loop.txt

@@ -32,9 +32,12 @@ The full documentation can be found in the library: :ref:`Scan <lib_scan>`.
   import theano.tensor as T
   theano.config.warn.subtensor_merge_bug = False
 
-  k = T.iscalar("k"); A = T.vector("A")
+  k = T.iscalar("k")
+  A = T.vector("A")
+
+  def inner_fct(prior_result, A):
+      return prior_result * A
 
-  def inner_fct(prior_result, A): return prior_result * A
   # Symbolic description of the result
   result, updates = theano.scan(fn=inner_fct,
                               outputs_info=T.ones_like(A),
@@ -61,7 +64,8 @@ The full documentation can be found in the library: :ref:`Scan <lib_scan>`.
   theano.config.warn.subtensor_merge_bug = False	
 
   coefficients = theano.tensor.vector("coefficients")
-  x = T.scalar("x"); max_coefficients_supported = 10000
+  x = T.scalar("x")
+  max_coefficients_supported = 10000
 
   # Generate the components of the polynomial
   full_range=theano.tensor.arange(max_coefficients_supported)
@@ -70,6 +74,7 @@ The full documentation can be found in the library: :ref:`Scan <lib_scan>`.
                                   outputs_info=None,
                                   sequences=[coefficients, full_range],
                                   non_sequences=x)
+
   polynomial = components.sum()
   calculate_polynomial = theano.function(inputs=[coefficients, x],
                                        outputs=polynomial)
@@ -89,4 +94,9 @@ Run both examples.
 Modify and execute the polynomial example to have the reduction done by ``scan``.
 
 
+.. TODO: repair this link as well as the code in the target file
+
+:download:`Solution<../loop_solution_1.py>`
+
+
 -------------------------------------------

doc/tutorial/loop_solution_1.py

+
+# Theano tutorial
+# Solution to Exercise in section 'Loop'
+
+"""
+
+# 1. First example (runs satisfactorily)
+
+import theano
+import theano.tensor as T
+
+theano.config.warn.subtensor_merge_bug = False
+
+k = T.iscalar("k")
+A = T.vector("A")
+
+
+def inner_fct(prior_result, A):
+    return prior_result * A
+
+# Symbolic description of the result
+result, updates = theano.scan(fn=inner_fct,
+                            outputs_info=T.ones_like(A),
+                            non_sequences=A, n_steps=k)
+
+# Scan has provided us with A**1 through A**k.  Keep only the last
+# value. Scan notices this and does not waste memory saving them.
+final_result = result[-1]
+
+power = theano.function(inputs=[A, k], outputs=final_result,
+                      updates=updates)
+
+print power(range(10), 2)
+# [  0.   1.   4.   9.  16.  25.  36.  49.  64.  81.]
+
+
+# 2. Second example (runs satisfactorily)
+
+import numpy
+import theano
+import theano.tensor as T
+
+coefficients = theano.tensor.vector("coefficients")
+x = T.scalar("x")
+max_coefficients_supported = 10000
+
+# Generate the components of the polynomial
+full_range = theano.tensor.arange(max_coefficients_supported)
+components, updates = theano.scan(fn=lambda coeff, power, free_var:
+                                   coeff * (free_var ** power),
+                                outputs_info=None,
+                                sequences=[coefficients, full_range],
+                                non_sequences=x)
+#polynomial = components.sum()
+polynomial1 = components
+calculate_polynomial1 = theano.function(inputs=[coefficients, x],
+                                     outputs=polynomial)
+
+test_coeff = numpy.asarray([1, 0, 2], dtype=numpy.float32)
+print calculate_polynomial1(test_coeff, 3)
+# 19.0
+"""
+
+
+# 3. Reduction performed inside scan
+
+# TODO: repair this code: yields 56.0 instead of 19.0
+
+import numpy
+import theano
+import theano.tensor as T
+
+theano.config.warn.subtensor_merge_bug = False
+
+coefficients = theano.tensor.vector("coefficients")
+x = T.scalar("x")
+max_coefficients_supported = 10000
+
+# Generate the components of the polynomial
+full_range = theano.tensor.arange(max_coefficients_supported)
+
+
+outputs_info = T.as_tensor_variable(numpy.asarray(0, 'float64'))
+
+components, updates = theano.scan(fn=lambda prior_value, coeff, power, free_var:
+                                 prior_value + (coeff * (free_var ** power)),
+                                outputs_info=outputs_info,
+                                sequences=[coefficients, full_range],
+                                non_sequences=x)
+                                
+polynomial = components[-1]
+calculate_polynomial = theano.function(inputs=[coefficients, x],
+                                     outputs=polynomial, updates=updates)
+
+test_coeff = numpy.asarray([1, 0, 2], dtype=numpy.float32)
+print calculate_polynomial(test_coeff, 3)
+# 19.0
+

doc/tutorial/modes.txt

@@ -82,16 +82,15 @@ Consider the logistic regression:
     predict = theano.function(inputs=[x], outputs=prediction,
                 name = "predict")
 
-    if any( [x.op.__class__.__name__=='Gemv' for x in
+    if any([x.op.__class__.__name__ in ['Gemv', 'CGemv', 'Gemm', 'CGemm'] for x in
     train.maker.fgraph.toposort()]):
-        print 'Used the cpu'
-    elif any( [x.op.__class__.__name__=='GpuGemm' for x in
+	print 'Used the cpu'
+    elif any([x.op.__class__.__name__ in ['GpuGemm', 'GpuGemv'] for x in
     train.maker.fgraph.toposort()]):
-        print 'Used the gpu'
+	print 'Used the gpu'
     else:
-        print 'ERROR, not able to tell if theano used the cpu or the gpu'
-        print train.maker.fgraph.toposort()
-
+	print 'ERROR, not able to tell if theano used the cpu or the gpu'
+	print train.maker.fgraph.toposort()
 
     for i in range(training_steps):
         pred, err = train(D[0], D[1])
@@ -127,6 +126,10 @@ time the execution using the command line ``time python file.py``.
      * Insert manual cast around the mean operator (this involves division by length, which is an *int64*).
      * Notice that a new casting mechanism is being developed.
 
+.. TODO: repair this link
+
+:download:`Solution<../modes_solution_1.py>`
+
 -------------------------------------------
 
 Mode
@@ -142,7 +145,7 @@ Theano defines the following modes by name:
 - ``'FAST_COMPILE'``: Apply just a few graph optimizations and only use Python implementations.
 - ``'FAST_RUN'``: Apply all optimizations, and use C implementations where possible.
 - ``'DEBUG_MODE'``: Verify the correctness of all optimizations, and compare C and Python 
-    implementations. This mode can take much longer than the other modes,but can identify
+    implementations. This mode can take much longer than the other modes, but can identify
     several kinds of problems.
 - ``'PROFILE_MODE'``: Same optimization then FAST_RUN, put print some profiling information
 

doc/tutorial/modes_solution_1.py

+
+# Theano tutorial
+# Solution to Exercise in section 'Configuration Settings and Compiling Modes'
+
+import numpy
+import theano
+import theano.tensor as T
+
+theano.config.floatX = 'float32'
+
+rng = numpy.random
+
+N = 400
+feats = 784
+D = (rng.randn(N, feats).astype(theano.config.floatX),
+rng.randint(size=N, low=0, high=2).astype(theano.config.floatX))
+training_steps = 10000
+
+# Declare Theano symbolic variables
+x = T.matrix("x")
+y = T.vector("y")
+w = theano.shared(rng.randn(feats).astype(theano.config.floatX), name="w")
+b = theano.shared(numpy.asarray(0., dtype=theano.config.floatX), name="b")
+x.tag.test_value = D[0]
+y.tag.test_value = D[1]
+#print "Initial model:"
+#print w.get_value(), b.get_value()
+
+# Construct Theano expression graph
+p_1 = 1 / (1 + T.exp(-T.dot(x, w) - b))  # Probabily of having a one
+prediction = p_1 > 0.5  # The prediction that is done: 0 or 1
+xent = -y * T.log(p_1) - (1 - y) * T.log(1 - p_1)  # Cross-entropy
+cost = T.cast(xent.mean(), 'float32') + \
+       0.01 * (w ** 2).sum()  # The cost to optimize
+gw, gb = T.grad(cost, [w, b])
+
+# Compile expressions to functions
+train = theano.function(
+            inputs=[x, y],
+            outputs=[prediction, xent],
+            updates={w: w - 0.01 * gw, b: b - 0.01 * gb},
+            name="train")
+predict = theano.function(inputs=[x], outputs=prediction,
+            name="predict")
+
+if any([x.op.__class__.__name__ in ['Gemv', 'CGemv', 'Gemm', 'CGemm'] for x in
+train.maker.fgraph.toposort()]):
+    print 'Used the cpu'
+elif any([x.op.__class__.__name__ in ['GpuGemm', 'GpuGemv'] for x in
+train.maker.fgraph.toposort()]):
+    print 'Used the gpu'
+else:
+    print 'ERROR, not able to tell if theano used the cpu or the gpu'
+    print train.maker.fgraph.toposort()
+
+for i in range(training_steps):
+    pred, err = train(D[0], D[1])
+#print "Final model:"
+#print w.get_value(), b.get_value()
+
+print "target values for D"
+print D[1]
+
+print "prediction on D"
+print predict(D[0])
+

doc/tutorial/remarks.txt

@@ -5,21 +5,8 @@
 Some general Remarks
 =====================
 
-.. TODO: This discussion is awkward. Even with this beneficial reordering (28 July 2012)
-..  its purpose and message are for the moment unclear. 
-
-
-Limitations
------------
-
-Theano offers a good amount of flexibility, but has some limitations too.
-You must answer for yourself the following question: How can my algorithm be cleverly written 
-so as to make the most of what Theano can do?
-
-
-- *While*- or *for*-Loops within an expression graph are supported, but only via
-  the :func:`theano.scan` op (which puts restrictions on how the loop body can
-  interact with the rest of the graph).
-
-- Neither *goto* nor *recursion* is supported or planned within expression graphs.
+.. This section is reserved for remarks and discussions of a general scope regarding the
+.. nature and development of Theano.
+.. The FAQ section is dedicated to specifics, i.e. information regarding the what, how,
+.. why and if of Theano. 
 

doc/tutorial/sparse.txt

@@ -4,9 +4,6 @@
 Sparse
 ======
 
-Sparse Matrices
-===============
-
 In general, *sparse* matrices provide the same functionality as regular
 matrices. The difference lies in the way the elements of *sparse* matrices are 
 represented and stored in memory. Only the non-zero elements of the latter are stored.

doc/tutorial/symbolic_graphs.txt

@@ -18,18 +18,18 @@ The first step in writing Theano code is to write down all mathematical
 relations using symbolic placeholders (**variables**). When writing down 
 these expressions you use operations like ``+``, ``-``, ``**``,
 ``sum()``, ``tanh()``. All these are represented internally as **ops**. 
-An **op** represents a certain computation on some type of inputs
+An *op* represents a certain computation on some type of inputs
 producing some type of output. You can see it as a *function definition*
 in most programming languages. 
 
 Theano builds internally a graph structure composed of interconnected 
 **variable** nodes, **op** nodes and **apply** nodes. An 
-**apply** node represents the application of an **op** to some 
-**variables**. It is important to make the difference between the
-definition of a computation represented by an **op** and its application
-to some actual data which is represented by the **apply** node. For more
-detail about these building blocks see :ref:`variable`, :ref:`op`, 
-:ref:`apply`. Here is a an example of a graph:
+*apply* node represents the application of an *op* to some 
+*variables*. It is important to draw the difference between the
+definition of a computation represented by an *op* and its application
+to some actual data which is represented by the *apply* node. For more
+detail about these building blocks refer to :ref:`variable`, :ref:`op`, 
+:ref:`apply`. Here is an example of a graph:
 
 
 **Code**
@@ -54,9 +54,9 @@ detail about these building blocks see :ref:`variable`, :ref:`op`,
     WARNING: hyper-links and ref's seem to break the PDF build when placed
     into this figure caption.
 
-Arrows in this :ref:`figure <tutorial-graphfigure>` represent references to the 
+Arrows in this figure represent references to the 
 Python objects pointed at. The blue
-box is an :ref:`apply` node. Red boxes are :ref:`variable` nodes. Green
+box is an :ref:`Apply` node. Red boxes are :ref:`Variable` nodes. Green
 circles are :ref:`Ops <op>`. Purple boxes are :ref:`Types <type>`.
 
 
@@ -111,18 +111,18 @@ Automatic Differentiation
 
 Having the graph structure, computing automatic differentiation is
 simple. The only thing :func:`tensor.grad` has to do is to traverse the
-graph from the outputs back towards the inputs through all :ref:`apply`
-nodes (:ref:`apply` nodes are those that define which computations the
-graph does). For each such :ref:`apply` node, its  :ref:`op` defines 
-how to compute the gradient of the node's outputs with respect to its
-inputs. Note that if an :ref:`op` does not provide this information, 
-it is assumed that the gradient is not defined.
+graph from the outputs back towards the inputs through all *apply*
+nodes (*apply* nodes are those that define which computations the
+graph does). For each such *apply* node, its  *op* defines 
+how to compute the *gradient* of the node's outputs with respect to its
+inputs. Note that if an *op* does not provide this information, 
+it is assumed that the *gradient* is not defined.
 Using the 
 `chain rule <http://en.wikipedia.org/wiki/Chain_rule>`_ 
 these gradients can be composed in order to obtain the expression of the 
-gradient of the graph's output with respect to the graph's inputs .
+*gradient* of the graph's output with respect to the graph's inputs .
 
-A following section of this tutorial will examine the topic of differentiation
+A following section of this tutorial will examine the topic of :ref:`differentiation<tutcomputinggrads>`
 in greater detail.
 
 
@@ -142,7 +142,7 @@ identical subgraphs and ensure that the same values are not computed
 twice or reformulate parts of the graph to a GPU specific version.
 
 For example, one (simple) optimization that Theano uses is to replace 
-the pattern :math:`\frac{xy}{y}` by :math:`x`.
+the pattern :math:`\frac{xy}{y}` by *x.*
 
 Further information regarding the optimization
 :ref:`process<optimization>` and the specific :ref:`optimizations<optimizations>` that are applicable

theano/scan_module/scan.py

@@ -197,11 +197,12 @@ def scan(fn,
 
         * ``initial`` -- Theano variable that represents the initial
           state of a given output. In case the output is not computed
-          recursively (think of a map) and does not require a initial
-          state this field can be skiped. Given that only the previous
-          time step of the output is used by ``fn`` the initial state
-          should have the same shape as the output. If multiple time
-          taps are used, the initial state should have one extra
+          recursively (think of a map) and does not require an initial
+          state this field can be skipped. Given that (only) the previous
+          time step of the output is used by ``fn``, the initial state
+          **should have the same shape** as the output and **should not
+          involve a downcast** of the data type of the output. If multiple
+          time taps are used, the initial state should have one extra
           dimension that should cover all the possible taps. For example
           if we use ``-5``, ``-2`` and ``-1`` as past taps, at step 0,
           ``fn`` will require (by an abuse of notation) ``output[-5]``,

theano/tests/test_tutorial.py

@@ -797,6 +797,7 @@ class T_using_gpu(unittest.TestCase):
         rng = numpy.random.RandomState(22)
         x = shared(numpy.asarray(rng.rand(vlen), config.floatX))
         f = function([], T.exp(x))
+        # print f.maker.fgraph.toposort()
         t0 = time.time()
         for i in xrange(iters):
             r = f()
@@ -813,7 +814,6 @@ class T_using_gpu(unittest.TestCase):
             assert numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()])
 
 
-
     def test_using_gpu_2(self):
 
         if theano.config.device.find('gpu') > -1:
@@ -829,6 +829,7 @@ class T_using_gpu(unittest.TestCase):
             rng = numpy.random.RandomState(22)
             x = shared(numpy.asarray(rng.rand(vlen), config.floatX))
             f = function([], sandbox.cuda.basic_ops.gpu_from_host(T.exp(x)))
+            # print f.maker.fgraph.toposort()
             t0 = time.time()
             for i in xrange(iters):
                 r = f()
@@ -844,9 +845,6 @@ class T_using_gpu(unittest.TestCase):
             assert not numpy.any([isinstance(x.op, T.Elemwise) for x in f.maker.fgraph.toposort()])
 
 
-
-
-
     def test_using_gpu_3(self):
 
         if theano.config.device.find('gpu') >-1:
@@ -864,6 +862,7 @@ class T_using_gpu(unittest.TestCase):
             f = function([],
                     Out(sandbox.cuda.basic_ops.gpu_from_host(T.exp(x)),
                         borrow=True))
+            # print f.maker.fgraph.toposort()
             t0 = time.time()
             for i in xrange(iters):
                 r = f()