Merge pull request #921 from goodfeli/doc_topo

doc/extending/inplace.txt

@@ -62,7 +62,7 @@ purpose, you would set the ``view_map`` field as follows:
 
 What this means is that the first output (position 0) is a view of the
 first input (position 0). Even though the interface allows a list of
-inputs that are a view of a given output, this feature is currently
+inputs that are viewed by a given output, this feature is currently
 unsupported. Here are more examples:
 
 

theano/compile/debugmode.py

@@ -675,7 +675,7 @@ def _optcheck_fgraph(input_specs, output_specs, accept_inplace=False):
             features=[equivalence_tracker])
 
     if not accept_inplace:
-        for node in fgraph.nodes:
+        for node in fgraph.apply_nodes:
             if getattr(node.op, 'destroy_map', None):
                 raise TypeError("Graph must not contain inplace operations",
                                 node)

theano/compile/function_module.py

@@ -131,7 +131,7 @@ def std_fgraph(input_specs, output_specs, accept_inplace = False):
     inputs, outputs = gof.graph.clone(orig_inputs, orig_outputs)
     fgraph = gof.fg.FunctionGraph(inputs, outputs)
 
-    for node in fgraph.nodes:
+    for node in fgraph.apply_nodes:
         if getattr(node.op, 'destroy_map', None):
             if not accept_inplace:
                 raise TypeError("Graph must not contain inplace operations", node, node.op)

theano/compile/profilemode.py

@@ -520,7 +520,7 @@ class ProfileMode(Mode):
             print "Profile of Theano functions memory:"
             print "(This check only the output of each apply node. It don't check the temporary memory used by the op in the apply node.)"
             nb_skipped = 0
-            for fgraph,nodes_mem in fct_memory.iteritems():
+            for fgraph, nodes_mem in fct_memory.iteritems():
                 size_sum=sum([sum(val) for key,val in nodes_mem.iteritems()])
                 if size_sum < min_memory_size:
                     nb_skipped += 1

theano/compile/profiling.py

@@ -711,7 +711,7 @@ if 0: # old code still to be ported from ProfileMode
                     var_mem[out]=v
             print
             print "Profile of Theano functions memory:"
-            for fgraph,nodes_mem in fct_memory.iteritems():
+            for fgraph, nodes_mem in fct_memory.iteritems():
                 print "Theano fct:", [fct for fct in fct_call.keys() if fct.maker.fgraph is fgraph][0].name
                 size_sum=sum([sum(val) for key,val in nodes_mem.iteritems()])
                 print "    Max without gc, inplace and view (KB)",size_sum/1024

theano/gof/fg.py

@@ -79,10 +79,10 @@ class FunctionGraph(utils.object2):
         # so I probably am) this should be a set.
         self._features = []
 
-        # All nodes in the subgraph defined by inputs and outputs are cached in nodes
-        self.nodes = set()
+        # All apply nodes in the subgraph defined by inputs and outputs are cached in this field
+        self.apply_nodes = set()
 
-        # Ditto for variables
+        # Ditto for variable nodes
         self.variables = set()
 
         self.inputs = list(inputs)
@@ -151,13 +151,13 @@ class FunctionGraph(utils.object2):
         nodes and variables. If there are no features, this should set
         them back to what they were originally.
         """
-        for node in self.nodes:
-            del node.fgraph
-            del node.deps
+        for apply_node in self.apply_nodes:
+            del apply_node.fgraph
+            del apply_node.deps
         for variable in self.variables:
             del variable.fgraph
             del variable.clients
-        self.nodes = set()
+        self.apply_nodes = set()
         self.variables = set()
         self.inputs = None
         self.outputs = None
@@ -215,11 +215,11 @@ class FunctionGraph(utils.object2):
         if NullType is None:
             from null_type import NullType
         # Imports the owners of the variables
-        r_owner_done = set(self.nodes)
-        for node in [r.owner for r in variables if r.owner is not None]:
-            if node not in r_owner_done:
-                r_owner_done.add(node)
-                self.__import__(node)
+        r_owner_done = set(self.apply_nodes)
+        for apply_node in [r.owner for r in variables if r.owner is not None]:
+            if apply_node not in r_owner_done:
+                r_owner_done.add(apply_node)
+                self.__import__(apply_node)
         for r in variables:
             if r.owner is None and not isinstance(r, graph.Constant) and r not in self.inputs:
                 if isinstance(r.type,NullType):
@@ -229,7 +229,9 @@ class FunctionGraph(utils.object2):
                 self.__setup_r__(r)
             self.variables.add(r)
 
-    def __import__(self, node, check = True):
+    def __import__(self, apply_node, check = True):
+        node = apply_node
+
         # We import the nodes in topological order. We only are interested
         # in new nodes, so we use all variables we know of as if they were the input set.
         # (the functions in the graph module only use the input set to
@@ -311,9 +313,9 @@ class FunctionGraph(utils.object2):
                             r)
 
         for node in new_nodes:
-            assert node not in self.nodes
+            assert node not in self.apply_nodes
             self.__setup_node__(node)
-            self.nodes.add(node)
+            self.apply_nodes.add(node)
             for output in node.outputs:
                 self.__setup_r__(output)
                 self.variables.add(output)
@@ -336,8 +338,9 @@ class FunctionGraph(utils.object2):
             if not r.clients and r in self.variables:
                 self.variables.remove(r)
 
-    def __prune__(self, node):
-        if node not in self.nodes:
+    def __prune__(self, apply_node):
+        node = apply_node
+        if node not in self.apply_nodes:
             raise Exception("%s does not belong to this FunctionGraph and cannot be pruned." % node)
         assert node.fgraph is self
         # If node's outputs have no clients, removes it from the graph
@@ -348,7 +351,7 @@ class FunctionGraph(utils.object2):
             # Cannot prune an op which is an output or used somewhere
             if self.clients(output) or output in self.outputs: #output in self.outputs or self.clients(output):
                 return
-        self.nodes.remove(node)
+        self.apply_nodes.remove(node)
         self.variables.difference_update(node.outputs)
         self.execute_callbacks('on_prune', node)
 
@@ -446,21 +449,29 @@ class FunctionGraph(utils.object2):
         Adds a gof.toolbox.Feature to this function_graph
         and triggers its on_attach callback
         """
+
+        # Filter out literally identical features
         if feature in self._features:
             return # the feature is already present
 
-        #it would be nice if we could require a specific class instead of
-        #a "workalike" so we could do actual error checking
-        #if not isinstance(feature, toolbox.Feature):
-        #    raise TypeError("Expected gof.toolbox.Feature instance, got "+\
-        #            str(type(feature)))
-
+        # Filter out functionally identical features.
+        # Features may use their on_attach method to raise
+        # toolbox.AlreadyThere if they detect that some
+        # installed feature does the same thing already
         attach = getattr(feature, 'on_attach', None)
         if attach is not None:
             try:
                 attach(self)
             except toolbox.AlreadyThere:
                 return
+
+        #it would be nice if we could require a specific class instead of
+        #a "workalike" so we could do actual error checking
+        #if not isinstance(feature, toolbox.Feature):
+        #    raise TypeError("Expected gof.toolbox.Feature instance, got "+\
+        #            str(type(feature)))
+
+        # Add the feature
         self._features.append(feature)
 
     def remove_feature(self, feature):
@@ -490,6 +501,9 @@ class FunctionGraph(utils.object2):
             try:
                 fn = getattr(feature, name)
             except AttributeError:
+                # this is safe because there is no work done inside the
+                # try; the AttributeError reall must come from feature.${name}
+                # not existing
                 continue
 
             #####HORRIBLE OPTIONAL ARGUMENT HACK
@@ -532,12 +546,12 @@ class FunctionGraph(utils.object2):
         {node: predecessors} where predecessors is a list of nodes
         that should be computed before the key node.
         """
-        if len(self.nodes) < 2:
+        if len(self.apply_nodes) < 2:
             # optimization
             # when there are 0 or 1 nodes, no sorting is necessary
             # This special case happens a lot because the OpWiseCLinker produces
             # 1-element graphs.
-            return list(self.nodes)
+            return list(self.apply_nodes)
         fg = self
         ords = self.orderings()
         order = graph.io_toposort(fg.inputs, fg.outputs, ords)
@@ -569,26 +583,31 @@ class FunctionGraph(utils.object2):
         """WRITEME Same as len(self.clients(r))."""
         return len(self.clients(r))
 
-#     def edge(self, r):
-#         return r in self.inputs or r in self.orphans
+    def nodes_getter(self):
+        warnings.warn("FunctionGraph.nodes is deprecated, it has been renamed 'apply_nodes'",
+                stacklevel=2)
+        return self.apply_nodes
+
+    def nodes_setter(self, value):
+        warnings.warn("FunctionGraph.nodes is deprecated, it has been renamed 'apply_nodes'",
+                stacklevel=2)
+        self.apply_nodes = value
+
+    def nodes_deleter(self):
+        warnings.warn("FunctionGraph.nodes is deprecated, it has been renamed 'apply_nodes'",
+                stacklevel=2)
+        del self.apply_nodes
 
-#     def follow(self, r):
-#         node = r.owner
-#         if self.edge(r):
-#             return None
-#         else:
-#             if node is None:
-#                 raise Exception("what the fuck")
-#             return node.inputs
+    nodes = property(nodes_getter, nodes_setter, nodes_deleter)
 
     def check_integrity(self):
         """WRITEME
         Call this for a diagnosis if things go awry.
         """
         nodes = graph.ops(self.inputs, self.outputs)
-        if self.nodes != nodes:
-            missing = nodes.difference(self.nodes)
-            excess = self.nodes.difference(nodes)
+        if self.apply_nodes != nodes:
+            missing = nodes.difference(self.apply_nodes)
+            excess = self.apply_nodes.difference(nodes)
             raise Exception("The nodes are inappropriately cached. missing, in excess: ", missing, excess)
         for node in nodes:
             if node.fgraph is not self:

theano/gof/graph.py

@@ -21,7 +21,25 @@ is_same_graph_with_merge = None
 equal_computations = None
 
 
-class Apply(utils.object2):
+class Node(utils.object2):
+    """A Node in a theano graph.
+    Graphs contain two kinds of Nodes--
+    Variable and Apply.
+    Edges in the graph are not explicitly represented.
+    Instead each Node keeps track of its parents via
+    Variable.owner / Apply.inputs and its children
+    via Variable.clients / Apply.outputs.
+    """
+
+    def get_parents(self):
+        """ Return a list of the parents of this node.
+        Should return a copy--i.e., modifying the return
+        value should not modify the graph structure."""
+
+        raise NotImplementedError()
+
+
+class Apply(Node):
     """
     An :term:`Apply` instance is a node in an expression graph which represents the application
     of an `Op` to some input `Variable` nodes, producing some output `Variable` nodes.
@@ -202,6 +220,9 @@ class Apply(utils.object2):
             new_node.inputs = new_inputs
         return new_node
 
+    def get_parents(self):
+        return list( self.inputs )
+
     #convenience properties
     nin = property(lambda self: len(self.inputs), doc='same as len(self.inputs)')
     """property: Number of inputs"""
@@ -210,7 +231,7 @@ class Apply(utils.object2):
     """property: Number of outputs"""
 
 
-class Variable(utils.object2):
+class Variable(Node):
     """
     A :term:`Variable` is a node in an expression graph that represents a variable.
 
@@ -364,6 +385,11 @@ class Variable(utils.object2):
         raise NotImplementedError('Subclasses of Variable must provide __ge__',
                                   self.__class__.__name__)
 
+    def get_parents(self):
+        if self.owner is not None:
+            return [ self.owner ]
+        return [ ]
+
     def env_getter(self):
         warnings.warn("Variable.env is deprecated, it has been renamed 'fgraph'",
                 stacklevel=2)
@@ -726,13 +752,26 @@ def general_toposort(r_out, deps, debug_print=False):
     return rlist
 
 
-def io_toposort(i, o, orderings=None):
+def io_toposort(inputs, outputs, orderings=None):
     """WRITEME
+
+    inputs: a list or tuple of Variable instances
+    outputs: a list or tuple of Variable instances
+
+    orderings: a dictionary
+                key: Apply instance
+                value: list of Apply instance
+
+                it is important that the value be
+                a container with a deterministic iteration
+                order. no sets allowed!
+
     """
     if orderings is None:
         orderings = {}
+
     #the inputs are used only here in the function that decides what 'predecessors' to explore
-    iset = set(i)
+    iset = set(inputs)
 
     def deps(obj):
         rval = []
@@ -747,7 +786,7 @@ def io_toposort(i, o, orderings=None):
             assert not orderings.get(obj, [])
         return rval
 
-    topo = general_toposort(o, deps)
+    topo = general_toposort(outputs, deps)
     return [o for o in topo if isinstance(o, Apply)]
 
 

theano/gof/opt.py

@@ -162,7 +162,7 @@ class SeqOptimizer(Optimizer, list):
         l = []
         if fgraph.profile:
             validate_before = fgraph.profile.validate_time
-        nb_node_before = len(fgraph.nodes)
+        nb_node_before = len(fgraph.apply_nodes)
         sub_profs = []
         for optimizer in self:
             try:
@@ -184,7 +184,7 @@ class SeqOptimizer(Optimizer, list):
             print "SeqOptimizer",
             if hasattr(self,"name"): print self.name,
             elif hasattr(self,"__name__"): print self.__name__,
-            print " time %.3fs for %d/%d nodes before/after optimization"%(sum(l),nb_node_before,len(fgraph.nodes))
+            print " time %.3fs for %d/%d nodes before/after optimization"%(sum(l),nb_node_before,len(fgraph.apply_nodes))
             print " time %.3fs for validate " % (
                 fgraph.profile.validate_time - validate_before)
             ll=[]
@@ -208,7 +208,7 @@ class SeqOptimizer(Optimizer, list):
         else:
             validate_time = None
         return (self, l, validate_time, nb_node_before,
-                len(fgraph.nodes), sub_profs)
+                len(fgraph.apply_nodes), sub_profs)
 
     def __eq__(self, other):
         #added to override the list's __eq__ implementation
@@ -1503,7 +1503,7 @@ class EquilibriumOptimizer(NavigatorOptimizer):
                                 max_use_abort = True
                                 opt_name = (getattr(lopt, "name", None)
                                             or getattr(lopt, "__name__", ""))
-                            if node not in fgraph.nodes:
+                            if node not in fgraph.apply_nodes:
                                 # go to next node
                                 break
             finally:

theano/gof/sandbox/equilibrium.py

@@ -73,7 +73,7 @@ if 0:
             tasks = defaultdict(list)
 
             if self.max_use_ratio is not None:
-                max_uses = self.max_use_ratio * len(fgraph.nodes)
+                max_uses = self.max_use_ratio * len(fgraph.apply_nodes)
                 runs = defaultdict(int)
             else:
                 runs = None
@@ -91,7 +91,7 @@ if 0:
                     self.backtrack(new_r.owner, tasks)
 
     #         # == NOT IDEAL == #
-    #         for node in fgraph.nodes:
+    #         for node in fgraph.apply_nodes:
     #             importer(node)
 
 

theano/gof/toolbox.py

@@ -7,6 +7,9 @@ import graph
 
 
 class AlreadyThere(Exception):
+    """Raised by a Feature's on_attach callback method if the FunctionGraph
+    attempting to attach the feature already has a functionally identical
+    feature."""
     pass
 
 
@@ -32,13 +35,18 @@ class Feature(object):
 
     def on_attach(self, function_graph):
         """
-        Called by FunctionGraph.attach_feature, the method that attaches the feature
-        to the FunctionGraph. Since this is called after the FunctionGraph
-        is initially populated, this is where you should run checks on the
-        initial contents of the FunctionGraph.
-        The feature has great freedom in what
-        it can do with the function_graph: it may, for example, add methods
-        to it dynamically.
+        Called by FunctionGraph.attach_feature, the method that attaches
+        the feature to the FunctionGraph. Since this is called after the
+        FunctionGraph is initially populated, this is where you should
+        run checks on the initial contents of the FunctionGraph.
+
+        The on_attach method may raise the AlreadyThere exception to cancel
+        the attach operation if it detects that another Feature instance
+        implementing the same functionality is already atttached to the
+        FunctionGraph.
+
+        The feature has great freedom in what it can do with the
+        function_graph: it may, for example, add methods to it dynamically.
         """
 
     def on_detach(self, function_graph):
@@ -219,7 +227,7 @@ class ReplaceValidate(History, Validator):
         """
         chk = fgraph.replace_all_validate(replacements, reason)
         for rm in remove:
-            if rm in fgraph.nodes or rm in fgraph.variables:
+            if rm in fgraph.apply_nodes or rm in fgraph.variables:
                 fgraph.revert(chk)
                 if warn:
                     out = sys.stderr

theano/sandbox/cuda/tests/test_basic_ops.py

@@ -1002,7 +1002,7 @@ def test_many_arg_elemwise():
                     #assert that the test was done on the gpu.
                     if mode is mode_with_gpu:
                         assert any([isinstance(node.op, cuda.GpuElemwise)
-                                    for node in f.maker.fgraph.nodes])
+                                    for node in f.maker.fgraph.apply_nodes])
 
                     #test the optijmization local_gpu_elemwise_1
                     f = theano.function(
@@ -1013,7 +1013,7 @@ def test_many_arg_elemwise():
                     #assert that the test was done on the gpu.
                     if mode is mode_with_gpu:
                         assert any([isinstance(node.op, cuda.GpuElemwise)
-                                    for node in f.maker.fgraph.nodes])
+                                    for node in f.maker.fgraph.apply_nodes])
                     assert numpy.allclose(out, outputs[-1])
 
                 results_gpu, results_cpu = outputs

theano/scalar/basic.py

@@ -2667,7 +2667,7 @@ class Composite(ScalarOp):
     def init_fgraph(self):
         fgraph = FunctionGraph(*gof.graph.clone(self.inputs, self.outputs))
         gof.MergeOptimizer().optimize(fgraph)
-        for node in fgraph.nodes:
+        for node in fgraph.apply_nodes:
             if not isinstance(node.op, ScalarOp):
                 raise ValueError("The fgraph to Composite must be exclusively"
                                  " composed of ScalarOp instances.")

theano/tensor/blas.py

@@ -1382,7 +1382,7 @@ class GemmOptimizer(Optimizer):
                                    (theano.scalar.Add, theano.scalar.Sub,
                                     theano.scalar.Neg, theano.scalar.Mul))):
                     continue
-                if not node in fgraph.nodes:
+                if not node in fgraph.apply_nodes:
                     # This mean that we already removed this node from
                     # the graph
                     continue

theano/tensor/opt.py

@@ -176,7 +176,7 @@ def inplace_elemwise_optimizer_op(OP):
         # We execute `validate` after this number of change.
         check_each_change = config.tensor.insert_inplace_optimizer_validate_nb
         if check_each_change == -1:
-            if len(fgraph.nodes) > 500:
+            if len(fgraph.apply_nodes) > 500:
                 check_each_change = 10
             else:
                 check_each_change = 1
@@ -4596,7 +4596,7 @@ class FusionOptimizer(Optimizer):
             did_something = False
             for node in nodelist:
                 # Don't try to fuse node that have already been fused.
-                if node in fgraph.nodes:
+                if node in fgraph.apply_nodes:
                     new_outputs = self.optimizer(node)
                     if new_outputs:
                         assert len(new_outputs) == len(node.outputs)

theano/tensor/tests/test_blas.py

@@ -478,7 +478,7 @@ def just_gemm(i, o, ishapes=[(4, 3), (3, 5), (4, 5), (), ()],
                 mode='FAST_RUN',
                 on_unused_input='ignore')
         nb_gemm = 0
-        for node in f.maker.fgraph.nodes:
+        for node in f.maker.fgraph.apply_nodes:
             if node.op == T.dot:
                 raise Failure('dot not changed to gemm_inplace in graph')
             if node.op == _dot22:
@@ -488,7 +488,7 @@ def just_gemm(i, o, ishapes=[(4, 3), (3, 5), (4, 5), (), ()],
         assert nb_gemm == expected_nb_gemm, (nb_gemm, expected_nb_gemm)
         g = inplace_func(i, o, mode=compile.Mode(linker='py', optimizer=None),
                 allow_input_downcast=True, on_unused_input='ignore')
-        for node in g.maker.fgraph.nodes:
+        for node in g.maker.fgraph.apply_nodes:
             if node.op == gemm_inplace:
                 raise Exception('gemm_inplace in original graph')
 
@@ -561,14 +561,14 @@ def test_gemm_opt_double_gemm():
     try:
         f = inplace_func([Param(ii, mutable=True) for ii in i], o,
                 mode='FAST_RUN', on_unused_input='ignore')
-        for node in f.maker.fgraph.nodes:
+        for node in f.maker.fgraph.apply_nodes:
             if node.op == T.dot:
                 raise Failure('dot in graph')
             if node.op == _dot22:
                 raise Failure('_dot22 in graph')
         g = inplace_func(i, o, mode=compile.Mode(linker='py', optimizer=None),
                 on_unused_input='ignore')
-        #for node in g.maker.fgraph.nodes:
+        #for node in g.maker.fgraph.apply_nodes:
         #    if node.op == gemm_inplace: raise Failure('gemm_inplace in graph')
 
         rng = numpy.random.RandomState(unittest_tools.fetch_seed(234))
@@ -760,11 +760,11 @@ def test_gemm_opt_vector_stuff():
     u, v = T.vector(), T.vector()
 
     f = inplace_func([a, u, v], a + T.dot(u, v), mode='FAST_RUN')
-    if gemm_inplace in [n.op for n in f.maker.fgraph.nodes]:
+    if gemm_inplace in [n.op for n in f.maker.fgraph.apply_nodes]:
         raise Failure('gemm_inplace in graph')
 
     f = inplace_func([a, u, X, Y], a * u + T.dot(X, Y), mode='FAST_RUN')
-    if (gemm_inplace in [n.op for n in f.maker.fgraph.nodes]):
+    if (gemm_inplace in [n.op for n in f.maker.fgraph.apply_nodes]):
         raise Failure('gemm_inplace in graph')
 
 
@@ -823,16 +823,16 @@ def test_inplace0():
 
     f = inplace_func([Z, b, R, S],
             [Z * (Z + b * T.dot(R, S).T)], mode='FAST_RUN')
-    if (gemm_inplace in [n.op for n in f.maker.fgraph.nodes]):
+    if (gemm_inplace in [n.op for n in f.maker.fgraph.apply_nodes]):
         print pp(f.maker.fgraph.outputs[0])
         raise Failure('gemm_inplace in graph')
-    assert gemm_no_inplace in [n.op for n in f.maker.fgraph.nodes]
+    assert gemm_no_inplace in [n.op for n in f.maker.fgraph.apply_nodes]
 
     # gemm_inplace should be inserted here, to work in-place on Z*c
     f = inplace_func([X, Y, Z, a, b, R, S, c],
             [Z * (c * Z + a * T.dot(X, Y) + b * T.dot(R, S).T)],
             mode='FAST_RUN')
-    if (not gemm_inplace in [n.op for n in f.maker.fgraph.nodes]):
+    if (not gemm_inplace in [n.op for n in f.maker.fgraph.apply_nodes]):
         theano.printing.debugprint(f)
         raise Failure('no gemm_inplace in graph')
 
@@ -844,7 +844,7 @@ def test_inplace1():
             [Z + Z + T.dot(X, Y)], mode='FAST_RUN')
     #theano.printing.debugprint(f)
     # it doesn't work inplace because we didn't mark Z as mutable input
-    assert [n.op for n in f.maker.fgraph.nodes] == [gemm_no_inplace]
+    assert [n.op for n in f.maker.fgraph.apply_nodes] == [gemm_no_inplace]
 
 
 def test_dot22():

theano/tensor/tests/test_naacl09.py

@@ -590,7 +590,7 @@ def test_naacl_model(iters_per_unsup=3, iters_per_sup=3,
     #print input_pretraining_gradients[4].owner.inputs[1].owner.inputs
     #sys.exit()
 
-    #print "PROGRAM LEN %i HASH %i"% (len(m.pretraining_update.maker.fgraph.nodes), reduce(lambda a, b: hash(a) ^ hash(b),prog_str))
+    #print "PROGRAM LEN %i HASH %i"% (len(m.pretraining_update.maker.fgraph.apply_nodes), reduce(lambda a, b: hash(a) ^ hash(b),prog_str))
 
     rng = N.random.RandomState(unittest_tools.fetch_seed(23904))