Merge pull request #4376 from adbrebs/fix_stack_check_nnet

theano/tensor/nnet/nnet.py

@@ -905,9 +905,11 @@ def softmax_simplifier(numerators, denominators):
                                 matching_denom = denominator
                                 break
         if matching_denom:
+            softmax = softmax_op(x)
+            copy_stack_trace(numerator, softmax)
             numerators.remove(numerator)
             denominators.remove(matching_denom)
-            numerators.append(softmax_op(x))
+            numerators.append(softmax)
 
     return numerators, denominators
 opt.local_mul_canonizer.add_simplifier(softmax_simplifier, 'softmax_simplifier')

theano/tensor/nnet/sigm.py

@@ -612,6 +612,7 @@ def local_exp_over_1_plus_exp(node):
             else:
                 # case: 1/(1+exp(x))
                 sigmoids.append(sigmoid(-t))
+            copy_stack_trace(node.outputs[0], sigmoids[-1])
 
         if not sigmoids:  # we didn't find any.  abort
             return
@@ -625,12 +626,17 @@ def local_exp_over_1_plus_exp(node):
         if num_neg ^ denom_neg:
             new_num = -new_num
 
+        copy_stack_trace(num, new_num)
+
         if len(denom_rest) == 0:
             return [new_num]
         elif len(denom_rest) == 1:
-            return [new_num / denom_rest[0]]
+            out = new_num / denom_rest[0]
         else:
-            return [new_num / tensor.mul(*denom_rest)]
+            out = new_num / tensor.mul(*denom_rest)
+
+        copy_stack_trace(node.outputs[0], out)
+        return [out]
 
 
 def parse_mul_tree(root):
@@ -923,6 +929,7 @@ def local_sigm_times_exp(node):
     exp(x) * sigm(-x) -> sigm(x)
     exp(-x) * sigm(x) -> sigm(-x)
 
+    todo: add stack traces to the intermediate variables
     """
     # Bail early if it is not a multiplication.
     if node.op != tensor.mul:

theano/tensor/nnet/tests/test_abstract_conv.py

@@ -7,6 +7,7 @@ from nose.tools import assert_raises
 
 import theano
 from theano import tensor
+from theano.gof.opt import check_stack_trace
 from theano.tests import unittest_tools as utt
 from theano.tensor.nnet import corr, abstract_conv as conv
 from theano.tensor.nnet.abstract_conv import get_conv_output_shape
@@ -98,7 +99,7 @@ class BaseTestConv2d(unittest.TestCase):
     def run_fwd(self, inputs_shape, filters_shape, ref=conv_corr,
                 subsample=(1, 1), verify_grad=True, mode=None,
                 border_mode='valid', filter_flip=True, provide_shape=False,
-                target_op=None):
+                target_op=None, check_trace=False):
         inputs_val = numpy.random.random(inputs_shape).astype('float32')
         filters_val = numpy.random.random(filters_shape).astype('float32')
 
@@ -133,8 +134,9 @@ class BaseTestConv2d(unittest.TestCase):
         if target_op is not None:
             assert any([isinstance(n.op, target_op) for n
                         in f.maker.fgraph.toposort()])
+            if check_trace:
+                self.assertTrue(check_stack_trace(f, ops_to_check=target_op))
 
-        self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace'))
         res_ref = numpy.array(f_ref())
         res = numpy.array(f())
         utt.assert_allclose(res_ref, res)
@@ -148,7 +150,7 @@ class BaseTestConv2d(unittest.TestCase):
     def run_gradweight(self, inputs_shape, filters_shape, output_shape,
                        ref=conv_corr_gw, subsample=(1, 1), filter_flip=True,
                        verify_grad=True, mode=None, border_mode='valid',
-                       provide_shape=False, target_op=None):
+                       provide_shape=False, target_op=None, check_trace=False):
 
         inputs_val = numpy.random.random(inputs_shape).astype('float32')
         output_val = numpy.random.random(output_shape).astype('float32')
@@ -177,12 +179,13 @@ class BaseTestConv2d(unittest.TestCase):
                     subsample=subsample,
                     conv_mode=conv_mode)
         f = theano.function([], c, mode=mode)
-        self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace'))
         f_ref = theano.function([], c_ref, mode='FAST_RUN')
 
         if target_op is not None:
             assert any([isinstance(n.op, target_op) for n
                         in f.maker.fgraph.toposort()])
+            if check_trace:
+                self.assertTrue(check_stack_trace(f, ops_to_check=target_op))
 
         res_ref = numpy.array(f_ref())
         res = numpy.array(f())
@@ -201,7 +204,7 @@ class BaseTestConv2d(unittest.TestCase):
     def run_gradinput(self, inputs_shape, filters_shape, output_shape,
                       ref=conv_corr_gi, subsample=(1, 1), filter_flip=True,
                       verify_grad=True, mode=None, border_mode='valid',
-                      provide_shape=False, target_op=None):
+                      provide_shape=False, target_op=None, check_trace=False):
 
         output_val = numpy.random.random(output_shape).astype('float32')
         filters_val = numpy.random.random(filters_shape).astype('float32')
@@ -227,12 +230,13 @@ class BaseTestConv2d(unittest.TestCase):
                     border_mode=border_mode, subsample=subsample,
                     conv_mode=conv_mode)
         f = theano.function([], c, mode=mode)
-        self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace'))
         f_ref = theano.function([], c_ref, mode='FAST_RUN')
 
         if target_op is not None:
             assert any([isinstance(n.op, target_op) for n
                         in f.maker.fgraph.toposort()])
+            if check_trace:
+                self.assertTrue(check_stack_trace(f, ops_to_check=target_op))
 
         res_ref = numpy.array(f_ref())
         res = numpy.array(f())
@@ -291,15 +295,18 @@ class TestCorrConv2d(BaseTestConv2d):
             raise SkipTest("Need blas to test conv2d")
         self.run_fwd(inputs_shape=i, filters_shape=f, subsample=s,
                      verify_grad=True, provide_shape=provide_shape,
-                     border_mode=b, filter_flip=flip, target_op=CorrMM)
+                     border_mode=b, filter_flip=flip, target_op=CorrMM,
+                     check_trace=True)
         self.run_gradweight(inputs_shape=i, filters_shape=f,
                             output_shape=o, subsample=s, verify_grad=True,
                             provide_shape=provide_shape, border_mode=b,
-                            filter_flip=flip, target_op=CorrMM_gradWeights)
+                            filter_flip=flip, target_op=CorrMM_gradWeights,
+                            check_trace=True)
         self.run_gradinput(inputs_shape=i, filters_shape=f,
                            output_shape=o, subsample=s, verify_grad=True,
                            provide_shape=provide_shape, border_mode=b,
-                           filter_flip=flip, target_op=CorrMM_gradInputs)
+                           filter_flip=flip, target_op=CorrMM_gradInputs,
+                           check_trace=True)
 
 
 class TestCpuConv2d(BaseTestConv2d):
@@ -343,7 +350,8 @@ class TestCpuConv2d(BaseTestConv2d):
             self.run_fwd(inputs_shape=i, filters_shape=f, subsample=s,
                          verify_grad=(gradweight_OK and gradinput_OK),
                          mode=mode, provide_shape=provide_shape,
-                         border_mode=b, filter_flip=flip, target_op=ConvOp)
+                         border_mode=b, filter_flip=flip, target_op=ConvOp,
+                         check_trace=True)
         else:
             self.assertRaises(AssertionError,
                               self.run_fwd,
@@ -354,7 +362,8 @@ class TestCpuConv2d(BaseTestConv2d):
                               mode=mode,
                               provide_shape=provide_shape,
                               border_mode=b,
-                              filter_flip=flip)
+                              filter_flip=flip,
+                              check_trace=True)
 
         if gradweight_OK:
             if not theano.config.blas.ldflags:
@@ -364,7 +373,8 @@ class TestCpuConv2d(BaseTestConv2d):
                                 verify_grad=False, mode=mode,
                                 provide_shape=provide_shape, border_mode=b,
                                 filter_flip=flip,
-                                target_op=(ConvOp, ConvGrad3D))
+                                target_op=(ConvOp, ConvGrad3D),
+                                check_trace=True)
         else:
             self.assertRaises(AssertionError,
                               self.run_gradweight,
@@ -376,7 +386,8 @@ class TestCpuConv2d(BaseTestConv2d):
                               mode=mode,
                               provide_shape=provide_shape,
                               border_mode=b,
-                              filter_flip=flip)
+                              filter_flip=flip,
+                              check_trace=True)
 
         if gradinput_OK:
             if not theano.config.blas.ldflags:
@@ -386,7 +397,8 @@ class TestCpuConv2d(BaseTestConv2d):
                                verify_grad=False, mode=mode,
                                provide_shape=provide_shape, border_mode=b,
                                filter_flip=flip,
-                               target_op=(ConvOp, ConvTransp3D))
+                               target_op=(ConvOp, ConvTransp3D),
+                               check_trace=True)
         else:
             self.assertRaises(AssertionError,
                               self.run_gradinput,
@@ -398,7 +410,8 @@ class TestCpuConv2d(BaseTestConv2d):
                               mode=mode,
                               provide_shape=provide_shape,
                               border_mode=b,
-                              filter_flip=flip)
+                              filter_flip=flip,
+                              check_trace=True)
 
 
 def test_constant_shapes():

theano/tensor/nnet/tests/test_conv3d2d.py

@@ -10,6 +10,7 @@ except ImportError:
 from six.moves import xrange
 
 import theano
+from theano.gof.opt import check_stack_trace
 from theano.tensor.nnet.conv3d2d import *
 import theano.tests.unittest_tools as utt
 
@@ -73,10 +74,11 @@ def pyconv3d(signals, filters):
                 r_i += o_i[Tf2:o_i_sh0-Tf2, Hf2:-Hf2, Wf2:-Wf2]
     return rval
 
+
 def check_diagonal_subtensor_view_traces(fn):
-    for apply_node in fn.maker.fgraph.apply_nodes:
-        if isinstance(apply_node.op, (DiagonalSubtensor, IncDiagonalSubtensor)):
-            assert hasattr(apply_node.outputs[0].tag, 'trace')
+    assert check_stack_trace(
+        fn, ops_to_check=(DiagonalSubtensor, IncDiagonalSubtensor))
+
 
 def test_conv3d(mode=mode_without_gpu, shared=theano.tensor._shared):
     if ndimage is None:
@@ -150,7 +152,6 @@ def test_conv3d(mode=mode_without_gpu, shared=theano.tensor._shared):
     newconv3d = theano.function([], [],
                                 updates={s_output: out},
                                 mode=mode)
-    check_diagonal_subtensor_view_traces(newconv3d)
 
     t0 = time.time()
     newconv3d()
@@ -162,7 +163,6 @@ def test_conv3d(mode=mode_without_gpu, shared=theano.tensor._shared):
                                           (s_signals, gsignals)],
                                  mode=mode,
                                  name='grad')
-    check_diagonal_subtensor_view_traces(gnewconv3d)
 
     t0 = time.time()
     gnewconv3d()

theano/tensor/nnet/tests/test_nnet.py

@@ -10,6 +10,7 @@ from theano import config
 from theano import tensor as T
 from theano import tensor
 from theano import gof
+from theano.gof.opt import check_stack_trace
 from theano.tests import unittest_tools as utt
 from theano import printing
 from theano.tensor.nnet import (categorical_crossentropy,
@@ -150,8 +151,7 @@ class T_SoftmaxWithBias(utt.InferShapeTester):
         b = theano.shared(numpy.float32(numpy.random.randn()))
         sm = T.nnet.softmax(a + b)
         f = theano.function([], sm)
-        self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace'))
-        print('f.maker.fgraph.outputs[0]: {0}'.format(f.maker.fgraph.outputs[0], ))
+        assert check_stack_trace(f, ops_to_check='last')
 
     def test_infer_shape(self):
         admat = matrix()
@@ -256,9 +256,10 @@ class T_LogSoftmax(utt.InferShapeTester):
         sm = tensor.nnet.softmax(x)
         logsm = tensor.log(sm)
         f = theano.function([x], logsm)
-        self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace'))
         assert isinstance(f.maker.fgraph.outputs[0].owner.op,
                           theano.tensor.nnet.nnet.LogSoftmax)
+        assert check_stack_trace(
+            f, ops_to_check=theano.tensor.nnet.nnet.LogSoftmax)
 
     def test_local_softmax_grad_optimization_and_big_input(self):
         """Test the Logsoftmax's grad substitution.
@@ -272,7 +273,8 @@ class T_LogSoftmax(utt.InferShapeTester):
         m.check_isfinite = False
         # some inputs that are large to make the gradient explode in the non
         # optimized case
-        a = numpy.exp(10 * numpy.random.rand(5, 10).astype(theano.config.floatX))
+        a = numpy.exp(
+            10 * numpy.random.rand(5, 10).astype(theano.config.floatX))
 
         def myfunc(x):
             sm = tensor.nnet.softmax(x)
@@ -280,8 +282,9 @@ class T_LogSoftmax(utt.InferShapeTester):
             return logsm
         # We set step to 0.1 because for big values we need a big epsilon
         utt.verify_grad(myfunc, [a], eps=0.1, mode=m)
-        f = theano.function([], myfunc(a))
-        self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace'))
+        sa = theano.shared(a)
+        f = theano.function([], myfunc(sa))
+        self.assertTrue(check_stack_trace(f, ops_to_check='all'))
 
 
 class T_SoftmaxGrad(utt.InferShapeTester):
@@ -659,7 +662,9 @@ class T_CrossentropyCategorical1Hot(utt.InferShapeTester):
         fgraph = gof.FunctionGraph(
                 [x, one_of_n],
                 [g_x])
-        self.assertTrue(hasattr(fgraph.outputs[0].tag, 'trace'))
+        assert check_stack_trace(
+            fgraph, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx,
+                                  softmax_op])
 
         # print 'BEFORE'
         # for node in fgraph.toposort():
@@ -755,7 +760,9 @@ class T_CrossentropyCategorical1Hot(utt.InferShapeTester):
         for expr in expressions:
             # Verify the optimizer worked on the expressions
             f = theano.function([x, y], expr, mode=mode)
-            self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace'))
+            # todo: only the first output of the op has a stack trace
+            # assert check_stack_trace(
+            #     f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias)
             if verbose:
                 theano.printing.debugprint(f)
             try:
@@ -771,7 +778,9 @@ class T_CrossentropyCategorical1Hot(utt.InferShapeTester):
 
             # Also verify the gradient wrt x
             g = theano.function([x, y], T.grad(expr, x), mode=mode)
-            self.assertTrue(hasattr(g.maker.fgraph.outputs[0].tag, 'trace'))
+            assert check_stack_trace(
+                g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx,
+                                 softmax_op])
             if verbose:
                 theano.printing.debugprint(g)
             try:
@@ -794,7 +803,9 @@ class T_CrossentropyCategorical1Hot(utt.InferShapeTester):
 
         for expr in bias_expressions:
             f = theano.function([x, b, y], expr, mode=mode)
-            self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace'))
+            # todo: only the first output of the op has a stack trace
+            # assert check_stack_trace(
+            #     f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias)
             if verbose:
                 theano.printing.debugprint(f)
             try:
@@ -806,7 +817,9 @@ class T_CrossentropyCategorical1Hot(utt.InferShapeTester):
                 theano.printing.debugprint(f)
                 raise
             g = theano.function([x, b, y], T.grad(expr, x), mode=mode)
-            self.assertTrue(hasattr(g.maker.fgraph.outputs[0].tag, 'trace'))
+            assert check_stack_trace(
+                g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx,
+                                 softmax_with_bias])
             if verbose:
                 theano.printing.debugprint(g)
             try:
@@ -829,7 +842,9 @@ class T_CrossentropyCategorical1Hot(utt.InferShapeTester):
 
         for expr in mean_expressions:
             f = theano.function([x, y], expr, mode=mode)
-            self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace'))
+            # todo: only the first output of the op has a stack trace
+            # assert check_stack_trace(
+            #     f, ops_to_check=[crossentropy_softmax_argmax_1hot_with_bias])
             if verbose:
                 theano.printing.debugprint(f)
             try:
@@ -844,7 +859,9 @@ class T_CrossentropyCategorical1Hot(utt.InferShapeTester):
                 raise
 
             g = theano.function([x, y], T.grad(expr, x), mode=mode)
-            self.assertTrue(hasattr(g.maker.fgraph.outputs[0].tag, 'trace'))
+            assert check_stack_trace(
+                g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx,
+                                 softmax_op])
             if verbose:
                 theano.printing.debugprint(g)
             try:
@@ -868,7 +885,9 @@ class T_CrossentropyCategorical1Hot(utt.InferShapeTester):
 
         for expr in mean_bias_expressions:
             f = theano.function([x, b, y], expr, mode=mode)
-            self.assertTrue(hasattr(f.maker.fgraph.outputs[0].tag, 'trace'))
+            # todo: only the first output of the op has a stack trace
+            # assert check_stack_trace(
+            #     f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias)
             if verbose:
                 theano.printing.debugprint(f)
             try:
@@ -881,7 +900,9 @@ class T_CrossentropyCategorical1Hot(utt.InferShapeTester):
                 theano.printing.debugprint(f)
                 raise
             g = theano.function([x, b, y], T.grad(expr, x), mode=mode)
-            self.assertTrue(hasattr(g.maker.fgraph.outputs[0].tag, 'trace'))
+            assert check_stack_trace(
+                g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx,
+                                 softmax_with_bias])
             if verbose:
                 theano.printing.debugprint(g)
             try:
@@ -1287,6 +1308,8 @@ def test_argmax_pushdown():
         assert len(fgraph.toposort()) == 2  # an output_guard is second
         assert fgraph.toposort()[0].op == tensor.basic._max_and_argmax
         assert str(fgraph.toposort()[1].op) == 'OutputGuard'
+        assert check_stack_trace(
+            fgraph, ops_to_check=tensor.basic._max_and_argmax)
         x = tensor.matrix()
         # test that the max_and_argmax is not pushed down if the max is used
         out = tensor.max_and_argmax(
@@ -1295,7 +1318,6 @@ def test_argmax_pushdown():
         fgraph = gof.FunctionGraph(
                 [x],
                 [out])
-        assert hasattr(fgraph.outputs[0].tag, 'trace')
 
         backup = config.warn.argmax_pushdown_bug
         config.warn.argmax_pushdown_bug = False
@@ -1324,8 +1346,6 @@ def test_argmax_pushdown_bias():
     fgraph = gof.FunctionGraph(
             [x, b],
             [out])
-    f = theano.function([x, b], out)
-    assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
 
     theano.compile.mode.optdb.query(
             theano.compile.mode.OPT_FAST_RUN).optimize(fgraph)
@@ -1333,11 +1353,12 @@ def test_argmax_pushdown_bias():
     # print 'AFTER'
     # for node in fgraph.toposort():
     #    print node.op
+    types_to_check = (tensor.DimShuffle, tensor.Elemwise, tensor.MaxAndArgmax)
     assert len(fgraph.toposort()) == 4
-    assert isinstance(fgraph.toposort()[0].op, tensor.DimShuffle)
-    assert isinstance(fgraph.toposort()[1].op, tensor.Elemwise)
-    assert isinstance(fgraph.toposort()[2].op, tensor.MaxAndArgmax)
+    for i, type in enumerate(types_to_check):
+        assert isinstance(fgraph.toposort()[i].op, type)
     assert str(fgraph.toposort()[3].op) == 'OutputGuard'
+    assert check_stack_trace(fgraph, ops_to_check=types_to_check)
 
     x = tensor.matrix()
     b = tensor.vector()
@@ -1345,8 +1366,6 @@ def test_argmax_pushdown_bias():
     fgraph = gof.FunctionGraph(
             [x, b],
             [out])
-    f = theano.function([x, b], out)
-    assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
 
     backup = config.warn.argmax_pushdown_bug
     config.warn.argmax_pushdown_bug = False
@@ -1364,6 +1383,8 @@ def test_argmax_pushdown_bias():
     assert isinstance(fgraph.toposort()[1].op, tensor.CAReduce)
     assert isinstance(fgraph.toposort()[1].op.scalar_op, theano.scalar.Maximum)
     assert str(fgraph.toposort()[2].op) == 'OutputGuard'
+    assert check_stack_trace(
+        fgraph, ops_to_check=(SoftmaxWithBias, tensor.CAReduce))
 
 
 def test_asymptotic_32():
@@ -1437,7 +1458,7 @@ class Test_softmax_opt:
         # test that function contains softmax and no div.
         f = theano.function([c], p_y, mode=self.mode)
 
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+        assert check_stack_trace(f, ops_to_check=softmax_op)
 
         f_ops = [n.op for n in f.maker.fgraph.toposort()]
         # print '--- f ='
@@ -1454,7 +1475,7 @@ class Test_softmax_opt:
         # test that function contains softmax and no div.
         f = theano.function([c], p_y, mode=self.mode)
 
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+        assert check_stack_trace(f, ops_to_check=softmax_op)
 
         f_ops = [n.op for n in f.maker.fgraph.toposort()]
         # print '--- f ='
@@ -1474,7 +1495,6 @@ class Test_softmax_opt:
         config.warn.sum_div_dimshuffle_bug = False
         try:
             g = theano.function([c, w], T.grad((p_y * w).sum(), c))
-            hasattr(g.maker.fgraph.outputs[0].tag, 'trace')
         finally:
             config.warn.sum_div_dimshuffle_bug = backup
         g_ops = [n.op for n in g.maker.fgraph.toposort()]
@@ -1502,7 +1522,6 @@ class Test_softmax_opt:
         config.warn.sum_div_dimshuffle_bug = False
         try:
             g = theano.function([c], T.grad(p_y.sum(), c))
-            hasattr(g.maker.fgraph.outputs[0].tag, 'trace')
         finally:
             config.warn.sum_div_dimshuffle_bug = backup
         # printing.debugprint(g)
@@ -1515,7 +1534,6 @@ class Test_softmax_opt:
 
         # test that function contains softmax and no div.
         f = theano.function([c], p_y)
-        hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
         # printing.debugprint(f)
 
         # test that function contains softmax and no div.
@@ -1523,7 +1541,6 @@ class Test_softmax_opt:
         config.warn.sum_div_dimshuffle_bug = False
         try:
             g = theano.function([c], T.grad(p_y.sum(), c))
-            hasattr(g.maker.fgraph.outputs[0].tag, 'trace')
         finally:
             config.warn.sum_div_dimshuffle_bug = backup
         # printing.debugprint(g)
@@ -1563,7 +1580,7 @@ def test_stabilize_log_softmax():
     z = theano.tensor.log(y)
 
     f = theano.function([x], z, mode=mode)
-    assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+    assert check_stack_trace(f, ops_to_check='all')
 
     # check that the softmax has been optimized out
     for node in f.maker.fgraph.toposort():

theano/tensor/nnet/tests/test_opt.py

 from __future__ import absolute_import, print_function, division
 import theano
 from theano import tensor
-from theano.tensor.nnet.blocksparse import sparse_block_dot
+from theano.gof.opt import check_stack_trace
+from theano.tensor.nnet.blocksparse import (
+    sparse_block_dot, sparse_block_gemv_inplace, sparse_block_outer_inplace,
+    sparse_block_gemv, sparse_block_outer)
 
 
 def test_blocksparse_inplace_gemv_opt():
@@ -14,12 +17,13 @@ def test_blocksparse_inplace_gemv_opt():
     o = sparse_block_dot(W, h, iIdx, b, oIdx)
 
     f = theano.function([W, h, iIdx, b, oIdx], o)
-    assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
 
     if theano.config.mode == "FAST_COMPILE":
         assert not f.maker.fgraph.toposort()[-1].op.inplace
+        assert check_stack_trace(f, ops_to_check=[sparse_block_gemv])
     else:
         assert f.maker.fgraph.toposort()[-1].op.inplace
+        assert check_stack_trace(f, ops_to_check=[sparse_block_gemv_inplace])
 
 
 def test_blocksparse_inplace_outer_opt():
@@ -31,13 +35,12 @@ def test_blocksparse_inplace_outer_opt():
 
     o = sparse_block_dot(W, h, iIdx, b, oIdx)
 
-    theano.printing.debugprint(tensor.grad(o.sum(), wrt=W))
-
     f = theano.function([W, h, iIdx, b, oIdx],
                         [o, tensor.grad(o.sum(), wrt=W)])
-    assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
 
     if theano.config.mode == "FAST_COMPILE":
         assert not f.maker.fgraph.toposort()[-1].op.inplace
+        assert check_stack_trace(f, ops_to_check=sparse_block_outer)
     else:
         assert f.maker.fgraph.toposort()[-1].op.inplace
+        assert check_stack_trace(f, ops_to_check=sparse_block_outer_inplace)

theano/tensor/nnet/tests/test_sigm.py

@@ -8,6 +8,7 @@ import theano.tensor.inplace
 from theano.tensor import basic as tensor
 from theano import tensor as T
 from theano import config
+from theano.gof.opt import check_stack_trace
 from theano.tests import unittest_tools as utt
 from theano.tensor.nnet import (sigmoid, sigmoid_inplace,
                                 softplus, ultra_fast_sigmoid, hard_sigmoid)
@@ -126,40 +127,37 @@ class T_sigmoid_opts(unittest.TestCase):
 
             # tests inv_1_plus_exp
             f = theano.function([x], T.fill(x, 1.0) / (1 + T.exp(-x)), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+            # todo: solve issue #4589 first
+            # assert check_stack_trace(f, ops_to_check=sigmoid)
             assert [node.op for node in f.maker.fgraph.toposort()] == [sigmoid]
             f(data)
             f = theano.function([x], T.fill(x, 1.0) / (2 + T.exp(-x)), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid]
             f(data)
             f = theano.function([x], T.fill(x, 1.0) / (1 - T.exp(-x)), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid]
             f(data)
             f = theano.function([x], T.fill(x, 1.1) / (1 + T.exp(-x)), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid]
             f(data)
 
             # tests inv_1_plus_exp with neg
             f = theano.function([x], T.fill(x, -1.0) / (1 + T.exp(-x)), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+            # todo: solve issue #4589 first
+            # assert check_stack_trace(
+            #     f, ops_to_check=[sigmoid, theano.tensor.inplace.neg_inplace])
             assert [node.op for node in f.maker.fgraph.toposort()] == [sigmoid,
                     theano.tensor.inplace.neg_inplace]
             f(data)
             f = theano.function([x], T.fill(x, -1.0) / (1 - T.exp(-x)), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid,
                     theano.tensor.inplace.neg_inplace]
             f(data)
             f = theano.function([x], T.fill(x, -1.0) / (2 + T.exp(-x)), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid,
                     theano.tensor.inplace.neg_inplace]
             f(data)
             f = theano.function([x], T.fill(x, -1.1) / (1 + T.exp(-x)), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid,
                     theano.tensor.inplace.neg_inplace]
             f(data)
@@ -170,37 +168,33 @@ class T_sigmoid_opts(unittest.TestCase):
             # = - (sigm(x) * sigm(x))
             f = theano.function([x], (T.fill(x, -1.0) * T.exp(x)) /
                                 ((1 + T.exp(x)) * (1 + T.exp(-x))), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+            # todo: solve issue #4589 first
+            # assert check_stack_trace(f, ops_to_check=[sigmoid, T.mul])
             assert [node.op for node in f.maker.fgraph.toposort()] == [sigmoid,
                     T.mul]
             f(data)
             f = theano.function([x], (T.fill(x, -1.1) * T.exp(x)) /
                                 ((1 + T.exp(x)) * (1 + T.exp(-x))), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid,
                     T.mul, theano.tensor.inplace.neg_inplace]
             f(data)
             f = theano.function([x], (T.fill(x, -1.0) * T.exp(x)) /
                                 ((2 + T.exp(x)) * (1 + T.exp(-x))), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid,
                     T.mul, theano.tensor.inplace.neg_inplace]
             f(data)
             f = theano.function([x], (T.fill(x, -1.0) * T.exp(x)) /
                                 ((1 + T.exp(x)) * (2 + T.exp(-x))), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid,
                     T.mul, theano.tensor.inplace.neg_inplace]
             f(data)
             f = theano.function([x], (T.fill(x, -1.0) * T.exp(x)) /
                                 ((1 + T.exp(x)) * (1 + T.exp(x))), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid,
                     T.mul, theano.tensor.inplace.neg_inplace]
             f(data)
             f = theano.function([x], (T.fill(x, -1.0) * T.exp(x)) /
                                 ((1 + T.exp(x)) * (2 + T.exp(-x))), mode=m)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             assert [node.op for node in f.maker.fgraph.toposort()] != [sigmoid,
                     T.mul, theano.tensor.inplace.neg_inplace]
             f(data)
@@ -218,13 +212,13 @@ class T_sigmoid_opts(unittest.TestCase):
 
         # tests exp_over_1_plus_exp
         f = theano.function([x], 1 - T.exp(x) / (1 + T.exp(x)), mode=m)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+        assert check_stack_trace(f, ops_to_check=[tensor.neg, sigmoid_inplace])
         assert [node.op for node in f.maker.fgraph.toposort()] == [
             tensor.neg, sigmoid_inplace]
 
         # tests inv_1_plus_exp
         f = theano.function([x], 1 - T.fill(x, 1.0) / (1 + T.exp(-x)), mode=m)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+        assert check_stack_trace(f, ops_to_check=[tensor.neg, sigmoid_inplace])
         assert [node.op for node in f.maker.fgraph.toposort()] == [tensor.neg,
                 sigmoid_inplace]
 
@@ -241,25 +235,26 @@ class T_sigmoid_opts(unittest.TestCase):
         x, y = tensor.vectors('x', 'y')
 
         f = theano.function([x], sigmoid(-x) * tensor.exp(x), mode=m)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
         match(f, [sigmoid])
+        assert check_stack_trace(f, ops_to_check=sigmoid)
 
         f = theano.function([x], sigmoid(x) * tensor.exp(-x), mode=m)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
         match(f, [tensor.neg, sigmoid])
+        assert check_stack_trace(f, ops_to_check=sigmoid)
 
         f = theano.function([x], -(-(-(sigmoid(x)))) * tensor.exp(-x), mode=m)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
         match(f, [tensor.neg, sigmoid, tensor.neg])
+        # assert check_stack_trace(f, ops_to_check=sigmoid)
 
         f = theano.function(
                 [x, y],
                 (sigmoid(x) * sigmoid(-y) * -tensor.exp(-x) *
                  tensor.exp(x * y) * tensor.exp(y)),
                 mode=m)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
         match(f, [sigmoid, tensor.mul, tensor.neg, tensor.exp, sigmoid,
                   tensor.mul])
+        # assert check_stack_trace(f, ops_to_check=[sigmoid, tensor.mul,
+        #                                           tensor.exp])
 
     def test_perform_sigm_times_exp(self):
         """
@@ -318,7 +313,6 @@ class T_sigmoid_opts(unittest.TestCase):
         mode = self.get_mode()
         if not isinstance(mode, theano.compile.DebugMode):
             f = theano.function([x, lr], ux, mode=mode)
-            assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
             ux_v = f([[50]], 0.1)
             assert not numpy.isnan(ux_v)
 
@@ -328,14 +322,14 @@ class T_sigmoid_opts(unittest.TestCase):
 
         mode = self.get_mode('local_ultra_fast_sigmoid')
         f = theano.function([x], s, mode=mode)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+        assert check_stack_trace(f, ops_to_check=sigmoid)
         topo = f.maker.fgraph.toposort()
         assert len(topo) == 1
         assert topo[0].op == sigmoid
 
         mode = self.get_mode().including('local_ultra_fast_sigmoid')
         f = theano.function([x], s, mode=mode)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+        assert check_stack_trace(f, ops_to_check=ultra_fast_sigmoid)
         topo = f.maker.fgraph.toposort()
         assert topo[0].op == ultra_fast_sigmoid
         assert len(topo) == 1
@@ -347,18 +341,21 @@ class T_sigmoid_opts(unittest.TestCase):
 
         mode = self.get_mode('local_hard_sigmoid')
         f = theano.function([x], s, mode=mode)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+        assert check_stack_trace(f, ops_to_check=sigmoid)
         topo = f.maker.fgraph.toposort()
         assert topo[0].op == sigmoid
         assert len(topo) == 1
 
         mode = self.get_mode().including('local_hard_sigmoid')
         f = theano.function([x], s, mode=mode)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
         topo = f.maker.fgraph.toposort()
         assert not any([n.op == sigmoid for n in topo])
         ux_v = f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]])
 
+        mode2 = mode.excluding('fusion').excluding('inplace')
+        f2 = theano.function([x], s, mode=mode2)
+        self.assertTrue(check_stack_trace(f2, ops_to_check=theano.tensor.clip))
+
 
 class T_softplus_opts(unittest.TestCase):
     def setUp(self):
@@ -376,7 +373,11 @@ class T_softplus_opts(unittest.TestCase):
 
         out = T.log(sigmoid(x))
         f = theano.function([x], out, mode=self.m)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+
+        # Fix ticket #4581 first
+        # assert check_stack_trace(
+        #     f, ops_to_check=(theano.scalar.Neg,
+        #                      theano.tensor.nnet.sigm.ScalarSoftplus))
         topo = f.maker.fgraph.toposort()
         assert len(topo) == 3
         assert isinstance(topo[0].op.scalar_op, theano.scalar.Neg)
@@ -395,12 +396,14 @@ class T_softplus_opts(unittest.TestCase):
         assert isinstance(topo[0].op.scalar_op,
                           theano.tensor.nnet.sigm.ScalarSoftplus)
         assert isinstance(topo[1].op.scalar_op, theano.scalar.Neg)
+        # assert check_stack_trace(f, ops_to_check='all')
         f(numpy.random.rand(54, 11).astype(config.floatX))
 
         # Same test with a flatten
         out = T.log(1 - T.flatten(sigmoid(x)))
         f = theano.function([x], out, mode=self.m)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+
+        # assert check_stack_trace(f, ops_to_check='all')
         topo = f.maker.fgraph.toposort()
         assert len(topo) == 3
         assert tensor.is_flat(topo[0].outputs[0])
@@ -429,7 +432,9 @@ class T_softplus_opts(unittest.TestCase):
 
         out = T.log(1 + T.exp(x))
         f = theano.function([x], out, mode=self.m)
-        assert hasattr(f.maker.fgraph.outputs[0].tag, 'trace')
+
+        # Fix ticket #4581 first
+        # assert check_stack_trace(f, ops_to_check='all')
         topo = f.maker.fgraph.toposort()
         assert len(topo) == 1
         assert isinstance(topo[0].op.scalar_op,