Merge pull request #714 from goodfeli/doc

theano/compile/function.py

@@ -77,6 +77,84 @@ def function(inputs, outputs=None, mode=None, updates=None, givens=None,
     another expression is undefined.  Replacements specified with givens are different from
     optimizations in that Var2 is not expected to be equivalent to Var1.
 
+
+    Internal documentation:
+
+        What happens when you call theano.function?
+           1. RemoveShared: shared variables are just an abstraction to make
+        things more convenient for the user. The shared variables are
+        transformed into implicit inputs and implicit outputs. The
+        optimizations don't see which variables are shared or not.
+           2. Env: determines whether a graph is valid. for example, suppose
+        you merge the two apply nodes in our example above, ie, do the
+        addition and the tanh at the same time. If you propose a merge that
+        changes the resulting dtype or broadcastable pattern of V4, the env
+        will detect this.
+                    inplace optimizations: say we have an apply node that
+        does + on V1 and V2, with output V3. We can change the output to be
+        V1, to use less memory. theano must be told that this optimization is
+        happening though, so that other parts of the graph are given the
+        correct (pre + or post + ) version of V1.
+                  env will raise an error if any of these types of
+        modifications causes an error
+                  env also adds a field called "clients" to all variables.
+        clients is a list of apply nodes that use the variable. this makes it
+        possible to traverse the graph in both directions. this is useful for
+        determining whether to do some optimizations. for example, a fusion
+        operation that removes V3 is not very helpful if V3 is also needed for
+        some other apply node. fusion operations result in a composite op that
+        takes a minigraph of theano scalars and uses this to do elemwise
+        operations on theano tensors
+         3. Optimization
+               How well do optimizations apply to new ops?
+                 Usually there are no optimizations for new ops. In fact, new
+        ops can disrupt patterns and break currently working optimizations.
+        Since the Print op, for example, is not known by any optimization,
+        setting a Print op in the middle of a pattern that is usually
+        optimized out will block the optimization. for example, log(1+x)
+        optimizes to log1p(x) but log(1+Print(x)) is unaffected by
+        optimizations.
+                 One exception is elemwise ops. If you implement your new op
+        as a scalar op then it will automatically work with all the elemwise
+        fusion machinery.
+
+                 Local optimizations try to replace some node in the graph
+        with a different node. In the case of log(1+x), we want to replace the
+        log node.
+
+                 def opt_log1p(node):
+                    if not isinstance(node.op,Elemwise):
+                       return
+                    if not isinstance(node.op.scalar_op, log,):
+                       return
+                    inp = node.inputs[0]
+                    if not inp.owner:
+                       return
+                    if not isinstance(inp.owner.op, add):
+                       return
+                    inp2 = inp.owner.inputs
+                    check that this has length 2, and that one of the inputs
+        is 1. assign the other input to x
+                    return log1p(x)
+
+
+         4. Linker
+               The linker uses a python loop to execute the code associated
+               with all the Apply nodes in the graph in the correct order.
+               the cvm is a linker that replaces this python loop with a c
+               loop to avoid continuously changing between python and c.
+               (It's currently a matter of disagreement as to whether there
+               is a cost of changing between
+               python and C, or whether the cvm is faster only because C code
+               is faster than python code)
+               the vm is a linker that was developed to prototype the cvm. it
+        was easier to develop the vm in python then translate it to c instead
+        of just writing it in c from scratch
+               cvm stands for c virtual machine.
+
+
+
+
     """
     #tuple are used in some tests, as we accepted them in the past
     #I prefer to allow it as they act the same as list for what they are used.

theano/gof/env.py

@@ -35,6 +35,13 @@ class Env(utils.object2):
     variable in the subgraph by another, e.g. replace (x + x).out by (2
     * x).out. This is the basis for optimization in theano.
 
+    This class is also reponsible for verifying that a graph is valid
+    (ie, all the dtypes and broadcast patterns are compatible with the
+    way the the Variables are used) and for annotating the Variables with
+    a .clients field that specifies which Apply nodes use the variable.
+    The .clients field combined with the .owner field and the Apply nodes'
+    .inputs field allows the graph to be traversed in both directions.
+
     It can also be "extended" using env.extend(some_object). See the
     toolbox and ext modules for common extensions.
 
@@ -43,7 +50,7 @@ class Env(utils.object2):
     - feature.on_attach(env)
         Called by extend. The feature has great freedom in what
         it can do with the env: it may, for example, add methods
-        to it dynicamically.
+        to it dynamically.
 
     - feature.on_detach(env)
         Called by remove_feature(feature).  Should remove any dynamically-added

theano/gof/graph.py

@@ -195,7 +195,7 @@ class Variable(utils.object2):
     """
     A :term:`Variable` is a node in an expression graph that represents a variable.
 
-    The inputs and outputs of every `Apply` are `Variable` instances.
+    The inputs and outputs of every `Apply` (theano.gof.Apply) are `Variable` instances.
     The input and output arguments to create a `function` are also `Variable` instances.
     A `Variable` is like a strongly-typed variable in some other languages; each `Variable` contains a
     reference to a `Type` instance that defines the kind of value the `Variable` can take in a
@@ -220,8 +220,23 @@ class Variable(utils.object2):
     - `Constant` (a subclass) which adds a default and un-replaceable :literal:`value`, and
       requires that owner is None
 
+    - `TensorVariable`
+
+    - `SharedTensorVariable`
+
+    - `SparseVariable`
+
+    - `CudaVariable`
+
+    - `RandomVariable`
+
     A Variable which is the output of a symbolic computation will have an owner != None.
 
+    Using the Variables' owner field and the Apply nodes' inputs fields, one can navigate a graph
+    from an output all the way to the inputs. The opposite direction is not possible until an
+    Env has annotated the Variables with the clients field, ie, before the compilation process
+    has begun a Variable does not know which Apply nodes take it as input.
+
     **Code Example**
 
     .. code-block:: python

theano/gof/unify.py

-
+"""
+If you have two expressions
+containing unification variables, these expressions can be "unified"
+if there exists an assignment to all unification variables such that
+    the two expressions are equal. For instance, [5, A, B] and [A, C, 9]
+    can be unified if A=C=5 and B=9, yielding [5, 5, 9]. [5, [A, B]] and
+    [A, [1, 2]] cannot be unified because there is no value for A that
+    satisfies the constraints. That's useful for pattern matching.
+"""
 from copy import copy
 
 from utils import *
@@ -10,8 +18,18 @@ from utils import *
 class Variable:
     """
     Serves as a base class of variables for the purpose of unification.
+    "Unification" here basically means matching two patterns, see the
+    module-level docstring.
+
     Behavior for unifying various types of variables should be added as
     overloadings of the 'unify' function.
+
+    Note: there are two Variable classes in theano and this is the
+    more rarely used one.
+    This class is used internally by the PatternSub optimization,
+    and possibly other subroutines that have to perform graph queries.
+    If that doesn't sound like what you're doing, the Variable class you
+    want is probably theano.gof.graph.Variable
     """
     def __init__(self, name = "?"):
         self.name = name
@@ -166,7 +184,7 @@ def unify_walk(a, b, U):
     be added if required)
 
     Here is a list of unification rules with their associated behavior:
-        
+
     """
     if a.__class__ != b.__class__:
         return False

theano/gof/vm.py

 """
 VMs that run Theano graph computations.
+A VM is not actually different from a Linker, we just decided
+VM was a better name at some point
 """
 import logging
 import sys
@@ -26,7 +28,16 @@ raise_with_op = link.raise_with_op
 
 class VM(object):
     """
-    A VM object evaluates a Theano program with its __call__ method.
+    A VM object's __call__ method evaluates a Theano program.
+
+    The Stack should be considered the reference VM/Linker implementation.
+    It can correctly evaluate all graphs and is the easiest to read. The CVM
+    is a port of Stack, and should have the same behavior, but run faster.
+    The CVM's code is harder to read though.
+
+    The other python VMs are maybe not necessary anymore, and don't take
+    advantage of lazy computation, though they still produce the correct
+    output for lazy nodes.
 
     Attributes:
 
@@ -184,6 +195,23 @@ class Stack(VM):
     This supports lazy evaluation of subtrees and partial
     computations of graphs when only some inputs have changed.
 
+    At a pseudo-code level, the basic idea is the following:
+
+    def recursively_evaluate(var):
+        if var is up to date:
+            return
+        if var.owner.inputs are up to date:
+            update var
+            return
+        for input in var.owner.unputs:
+            recursively_evaluate(var)
+
+    for output in outputs:
+        recursively_evaluate(output)
+
+    The actual logic is more complex to support intermediate
+    garbage collection, lazily-evaluated nodes, and better speed.
+
     """
 
     def __init__(self, nodes, thunks, pre_call_clear,

theano/printing.py

@@ -128,7 +128,15 @@ class Print(Op):
                   their return value printed.
 
     :note: WARNING. This can disable some optimizations!
-                    (speed and/pr stabilization)
+                    (speed and/or stabilization)
+
+            Detailed explanation:
+            As of 2012-06-21 the Print op is not known by any optimization.
+            Setting a Print op in the middle of a pattern that is usually
+            optimized out will block the optimization. for example, log(1+x)
+            optimizes to log1p(x) but log(1+Print(x)) is unaffected by
+            optimizations.
+
     """
     view_map = {0: [0]}