Merge pull request #4079 from nouiz/stack_trace

theano/compile/function.py

@@ -120,6 +120,8 @@ def function(inputs, outputs=None, mode=None, updates=None, givens=None,
     profile: None, True, or ProfileStats instance
         Accumulate profiling information into a given ProfileStats instance.
         If argument is `True` then a new ProfileStats instance will be used.
+        If argument is a string, a new ProfileStats instance will be created
+        with that string as its ``message`` attribute.
         This profiling object will be available via self.profile.
     on_unused_input
         What to do if a variable in the 'inputs' list is not used in the graph.

theano/gof/link.py

@@ -169,7 +169,7 @@ def raise_with_op(node, thunk=None, exc_info=None, storage_map=None):
     # Print node backtraces
     tr = getattr(node.outputs[0].tag, 'trace', [])
     if type(tr) is list and len(tr) > 0:
-        detailed_err_msg += "\nBacktrace when the node is created:\n"
+        detailed_err_msg += "\nBacktrace when the node is created(use Theano flag traceback.limit=N to make it longer):\n"
 
         # Print separate message for each element in the list of batcktraces
         sio = StringIO()

theano/gof/tests/test_utils.py

@@ -62,3 +62,18 @@ def test_hash_from_dict():
 
     # List are not hashable. So they are transformed into tuple.
     assert hash_from_dict({0: (0,)}) == hash_from_dict({0: [0]})
+
+
+def test_stack_trace():
+    orig = theano.config.traceback.limit
+    try:
+        theano.config.traceback.limit = 1
+        v = theano.tensor.vector()
+        assert len(v.tag.trace) == 1
+        assert len(v.tag.trace[0]) == 1
+        theano.config.traceback.limit = 2
+        v = theano.tensor.vector()
+        assert len(v.tag.trace) == 1
+        assert len(v.tag.trace[0]) == 2
+    finally:
+        theano.config.traceback.limit = orig

theano/gof/utils.py

 from __future__ import print_function
 import linecache
-import traceback
 import sys
 
 import numpy
@@ -10,14 +9,20 @@ from theano import config
 from theano.compat import OrderedDict, PY3
 
 
-def simple_extract_stack(f=None, limit=None):
-    """
-    This is traceback.extract_stack from python 2.7 with this change:
+def simple_extract_stack(f=None, limit=None, skips=[]):
+    """This is traceback.extract_stack from python 2.7 with this change:
 
     - Comment the update of the cache.
+    - Skip internal stack trace level.
+
+    The update of the cache call os.stat to verify is the cache is up
+    to date.  This take too much time on cluster.
+
+    limit - The number of stack level we want to return. If None, mean
+    all what we can.
 
-    This is because this update cause an call to os.stat to get the
-    line content. This cause too much long on cluster.
+    skips - partial path of stack level we don't want to keep and count.
+        When we find one level that isn't skipped, we stop skipping.
 
     """
     if f is None:
@@ -28,7 +33,7 @@ def simple_extract_stack(f=None, limit=None):
     if limit is None:
         if hasattr(sys, 'tracebacklimit'):
             limit = sys.tracebacklimit
-    list = []
+    trace = []
     n = 0
     while f is not None and (limit is None or n < limit):
         lineno = f.f_lineno
@@ -41,20 +46,28 @@ def simple_extract_stack(f=None, limit=None):
             line = line.strip()
         else:
             line = None
-        list.append((filename, lineno, name, line))
         f = f.f_back
-        n = n + 1
-    list.reverse()
-    return list
 
-if sys.version_info[:2] > (3, 4):
-    # I enable my implementation only for some python version just to
-    # be sure the Python internal do not change. If this work with
-    # other python version, you can enable it.
-    simple_extract_stack = traceback.extract_stack  # noqa
+        # Just skip inner level
+        if len(trace) == 0:
+            rm = False
+            for p in skips:
+                # Julian: I added the 'tests' exception together with
+                # Arnaud.  Otherwise, we'd lose the stack trace during
+                # in our test cases (e.g. in test_opt.py). We're not
+                # sure this is the right way to do it though.
+                if p in filename and 'tests' not in filename:
+                    rm = True
+                    break
+            if rm:
+                continue
+        trace.append((filename, lineno, name, line))
+        n = n + 1
+    trace.reverse()
+    return trace
 
 
-def add_tag_trace(thing, user_line=1):
+def add_tag_trace(thing, user_line=None):
     """
     Add tag.trace to an node or variable.
 
@@ -73,42 +86,24 @@ def add_tag_trace(thing, user_line=1):
     we look.
 
     """
-    limit = config.traceback.limit
-    if limit == -1:
-        limit = None
-    tr = simple_extract_stack(limit=limit)[:-1]
-
+    if user_line is None:
+        user_line = config.traceback.limit
+
+    if user_line == -1:
+        user_line = None
+    skips = ["theano/tensor/", "theano\\tensor\\",
+             "theano/compile/", "theano\\compile\\",
+             "theano/gof/", "theano\\gof\\",
+             "theano/scalar/basic.py", "theano\\scalar\\basic.py",
+             "theano/sandbox/", "theano\\sandbox\\",
+             "theano/scan_module/", "theano\\scan_module\\",
+             "theano/sparse/", "theano\\sparse\\",
+             "theano/typed_list/", "theano\\typed_list\\"]
+    tr = simple_extract_stack(limit=user_line, skips=skips)
     # Different python version use different sementic for
     # limit. python 2.7 include the call to extrack_stack. The -1 get
     # rid of it.
 
-    # Get rid of Theano internal
-    while tr:
-        file_path = tr[-1][0]
-        rm = False
-        for p in ["theano/tensor/", "theano\\tensor\\",
-                  "theano/compile/", "theano\\compile\\",
-                  "theano/gof/", "theano\\gof\\",
-                  "theano/scalar/basic.py", "theano\\scalar\\basic.py",
-                  "theano/sandbox/", "theano\\sandbox\\",
-                  "theano/scan_module/", "theano\\scan_module\\",
-                  "theano/sparse/", "theano\\sparse\\",
-                  "theano/typed_list/", "theano\\typed_list\\",
-                  ]:
-            # Julian: I added the 'tests' exception together with Arnaud.
-            # Otherwise, we'd lose the stack trace during in our test cases
-            # (e.g. in test_opt.py). We're not sure this is the right way to
-            # do it though.
-            if p in file_path and 'tests' not in file_path:
-                tr = tr[:-1]
-                rm = True
-                break
-        if not rm:
-            break
-    # Keep only the most recent stack level.
-    # The order is from the oldest to the newest
-    if len(tr) > user_line:
-        tr = tr[-user_line:]
     if tr:
         thing.tag.trace = [tr]
     else: