Added MonitorMode as a debugging tool

doc/tutorial/debug_faq.txt

@@ -168,7 +168,7 @@ Theano provides a 'Print' op to do this.
 Since Theano runs your program in a topological order, you won't have precise
 control over the order in which multiple ``Print()`` ops are evaluted.  For a more
 precise inspection of what's being computed where, when, and how, see the discussion
-:ref:`faq_wraplinker`.
+:ref:`faq_monitormode`.
 
 .. warning::
 
@@ -216,48 +216,79 @@ Tips:
   of type *float64*.
 
 
-.. _faq_wraplinker:
+.. _faq_monitormode:
 
-"How do I Step through a Compiled Function with the WrapLinker?"
-----------------------------------------------------------------
+"How do I Step through a Compiled Function?"
+--------------------------------------------
 
-This is not exactly a FAQ, but the doc is here for now...
-It's pretty easy to roll-your-own evaluation mode.
-Check out this one:
+You can use ``MonitorMode`` to inspect the inputs and outputs of each
+node being executed when the function is called. The code snipped below
+shows how to print all inputs and outputs:
 
 .. code-block:: python
 
-    class PrintEverythingMode(Mode):
-        def __init__(self):
-            def print_eval(i, node, fn):
+    import theano
+
+    def inspect_inputs(i, node, fn):
         print i, node, [input[0] for input in fn.inputs],
-                fn()
+
+    def inspect_outputs(i, node, fn):
         print [output[0] for output in fn.outputs]
-            wrap_linker = theano.gof.WrapLinkerMany([theano.gof.OpWiseCLinker()], [print_eval])
-            super(PrintEverythingMode, self).__init__(wrap_linker, optimizer='fast_run')
 
-When you use ``mode=PrintEverythingMode()`` as the mode for ``Function`` or ``Method``,
-then you should see [potentially a lot of] output.  Every ``Apply`` node will be printed out,
-along with its position in the graph, the arguments to the functions ``perform`` or
-``c_code`` and the output it computed.  
+    x = theano.tensor.dscalar('x')
+    f = theano.function([x], [5 * x],
+                        mode=theano.compile.MonitorMode(
+                            pre_func=inspect_inputs,
+                            post_func=inspect_outputs))
+    f(3)
 
->>> x = T.dscalar('x')
->>> f = function([x], [5 * x], mode=PrintEverythingMode())
->>> f(3)
->>> # print: 0 Elemwise{mul,no_inplace}(5, x) [array(5, dtype=int8), array(3.0)] [array(15.0)]
->>> # print: [array(15.0)]
+    # The code will print the following:
+    #   0 Elemwise{mul,no_inplace}(TensorConstant{5.0}, x) [array(5.0), array(3.0)] [array(15.0)]
 
+When using these ``inspect_inputs`` and ``inspect_outputs`` functions
+with ``MonitorMode``, you should see [potentially a lot of] printed output.
+Every ``Apply`` node will be printed out,
+along with its position in the graph, the arguments to the functions ``perform`` or
+``c_code`` and the output it computed.
 Admittedly, this may be a huge amount of
 output to read through if you are using big tensors... but you can choose to
-put logic inside of the *print_eval* function that would, for example, print 
+add logic that would, for instance, print
 something out only if a certain kind of op were used, at a certain program
 position, or only if a particular value showed up in one of the inputs or outputs.
-Use your imagination :)
+A typical example is to detect when NaN values are added into computations, which
+can be achieved as follows:
+
+.. code-block:: python
+
+    import numpy
+
+    import theano
+
+    def detect_nan(i, node, fn):
+        for output in fn.outputs:
+            if numpy.isnan(output[0]).any():
+                print '*** NaN detected ***'
+                theano.printing.debugprint(node)
+                print 'Inputs : %s' % [input[0] for input in fn.inputs]
+                print 'Outputs: %s' % [output[0] for output in fn.outputs]
+                break
+
+    x = theano.tensor.dscalar('x')
+    f = theano.function([x], [theano.tensor.log(x) * x],
+                        mode=theano.compile.MonitorMode(
+                            post_func=detect_nan))
+    f(0)  # log(0) * 0 = -inf * 0 = NaN
+
+    # The code above will print:
+    #   *** NaN detected ***
+    #   Elemwise{Composite{[mul(log(i0), i0)]}} [@A] ''
+    #    |x [@B]
+    #   Inputs : [array(0.0)]
+    #   Outputs: [array(nan)]
+
 
 .. TODO: documentation for link.WrapLinkerMany
 
-This can be a really powerful debugging tool. Note the call to *fn* inside the call to
-*print_eval*; without it, the graph wouldn't get computed at all!
 
 How to Use pdb
 --------------

doc/tutorial/modes.txt

@@ -153,6 +153,13 @@ short name        Full constructor
 ``ProfileMode``   ``compile.profilemode.ProfileMode()``                           C implementations where available, all available graph transformations, print profile information.
 ================= =============================================================== ===============================================================================
 
+.. Note::
+
+    For debugging purpose, there also exists a ``MonitorMode`` (which has no
+    short name). It can be used to step through the execution of a function:
+    see :ref:`the debugging FAQ<faq_monitormode>` for details.
+
+
 Linkers
 =======
 

theano/compile/__init__.py

@@ -21,6 +21,8 @@ from module import *
 import debugmode   # register DEBUG_MODE
 from debugmode import DebugMode
 
+from monitormode import MonitorMode
+
 from profilemode import ProfileMode
 
 from theano.compile.sharedvalue import shared, shared_constructor, SharedVariable

theano/compile/monitormode.py

+# Note: this code was initially copied from the 'pyutools' package by its
+# original author, and re-licensed under Theano's license.
+
+
+import theano
+from theano.compile.mode import Mode
+
+
+class MonitorMode(Mode):
+
+    """
+    `MonitorMode` is a debug mode to easily step through function execution.
+
+    Its default behavior is to behave like the 'FAST_RUN' mode. By providing
+    either a `pre_func` (called before a node is executed) or a `post_func`
+    (called after a node is executed) monitoring function, the user can inspect
+    node behavior.
+
+    A typical use case is to detect the introduction of NaN values in a graph.
+    For an example of such a use case, see doc/tutorial/debug_faq.txt.
+    """
+
+    def __init__(self, pre_func=None, post_func=None, optimizer='fast_run'):
+        """
+        Constructor.
+
+        :param pre_func: A function to call before executing a thunk, with
+            arguments:
+                - the thunk index
+                - the Apply node
+                - the thunk to be called
+
+        :param post_func: A function to call after executing a thunk, with the
+            same three arguments as `pre_func`.
+
+        :param optimizer: The optimizer to use. One may use for instance
+            'fast_compile' to skip optimizations.
+        """
+        self.pre_func = pre_func
+        self.post_func = post_func
+        wrap_linker = theano.gof.WrapLinkerMany([theano.gof.OpWiseCLinker()],
+                                                [self.eval])
+        super(MonitorMode, self).__init__(wrap_linker, optimizer=optimizer)
+
+    def eval(self, i, node, fn):
+        """
+        The method that calls the thunk `fn`.
+        """
+        if self.pre_func is not None:
+            self.pre_func(i, node, fn)
+        fn()
+        if self.post_func is not None:
+            self.post_func(i, node, fn)

theano/compile/tests/test_monitormode.py

+import numpy
+
+import theano
+
+
+def test_detect_nan():
+    """
+    Test the code snippet example that detects NaN values.
+    """
+    nan_detected = [False]
+
+    def detect_nan(i, node, fn):
+        for output in fn.outputs:
+            if numpy.isnan(output[0]).any():
+                print '*** NaN detected ***'
+                theano.printing.debugprint(node)
+                print 'Inputs : %s' % [input[0] for input in fn.inputs]
+                print 'Outputs: %s' % [output[0] for output in fn.outputs]
+                nan_detected[0] = True
+                break
+
+    x = theano.tensor.dscalar('x')
+    f = theano.function([x], [theano.tensor.log(x) * x],
+                        mode=theano.compile.MonitorMode(
+                            post_func=detect_nan))
+    f(0)  # log(0) * 0 = -inf * 0 = NaN
+    assert nan_detected[0]