Merge pull request #3416 from nouiz/bn_follow_up

doc/library/tensor/nnet/index.txt

@@ -19,3 +19,4 @@ and ops which are particular to neural networks and deep learning.
     conv
     nnet
     neighbours
+    bn

theano/compile/profiling.py

@@ -697,7 +697,7 @@ class ProfileStats(object):
         print('Time in all call to theano.grad() %es' %
               theano.gradient.grad_time, file=file)
         total_time = time.time() - theano_imported_time
-        print('Time since theano import %.3fs' % (total_time))
+        print('Time since theano import %.3fs' % (total_time), file=file)
 
     def summary_memory(self, file, N=None):
         fct_memory = {}  # fgraph->dict(node->[outputs size])

theano/gradient.py

@@ -1612,15 +1612,10 @@ def verify_grad(fun, pt, n_tests=2, rng=None, eps=None,
 
     # We allow input downcast in function, because numeric_grad works in the
     # most precise dtype used among the inputs, so we may need to cast some.
-    def function(inputs, output):
-        if mode is None:
-            f = compile.function(inputs, output, accept_inplace=True,
-                                 allow_input_downcast=True,
-                                 on_unused_input='ignore')
-        else:
-            f = compile.function(inputs, output, accept_inplace=True,
-                                 allow_input_downcast=True, mode=mode,
-                                 on_unused_input='ignore')
+    def function(inputs, output, name):
+        f = compile.function(inputs, output, accept_inplace=True,
+                             allow_input_downcast=True, mode=mode,
+                             on_unused_input='ignore', name=name)
         return f
 
     tensor_pt = [
@@ -1639,7 +1634,7 @@ def verify_grad(fun, pt, n_tests=2, rng=None, eps=None,
         # but this doesn't handle the case where not all the outputs are
         # differentiable... so I leave this as TODO for now -JB.
 
-    o_fn = function(tensor_pt, o_output)
+    o_fn = function(tensor_pt, o_output, name='gradient.py fwd')
     o_fn_out = o_fn(*[p.copy() for p in pt])
 
     if isinstance(o_fn_out, tuple) or isinstance(o_fn_out, list):
@@ -1663,12 +1658,13 @@ def verify_grad(fun, pt, n_tests=2, rng=None, eps=None,
     # This sum() is defined above, it's not the builtin sum.
     cost = theano.tensor.sum(t_r * o_output)
 
-    cost_fn = function(tensor_pt, cost)
+    cost_fn = function(tensor_pt, cost, name='gradient.py cost')
 
     symbolic_grad = grad(cost, tensor_pt,
                          disconnected_inputs='ignore')
 
-    grad_fn = function(tensor_pt, symbolic_grad)
+    grad_fn = function(tensor_pt, symbolic_grad,
+                       name='gradient.py symbolic grad')
 
     for test_num in xrange(n_tests):
         try:

theano/sandbox/cuda/opt.py

@@ -2146,7 +2146,7 @@ def local_gpualloc(node):
                        i.owner.op in [host_from_gpu, tensor.alloc]
                        for i in c.inputs[1:]])
                   for c, idx in node.outputs[0].clients]):
-            # if the client is a subtensor with input on gpu or alloc
+            # if the client is on gpu or alloc
             replace = True
         if replace and node.inputs[0].dtype != 'float32':
             replace = False

theano/scalar/basic.py

@@ -3329,6 +3329,8 @@ class Composite(ScalarOp):
     Composite depends on all the Ops in its graph having C code.
 
     """
+    init_param = ('inputs', 'outputs')
+
     def __str__(self):
         return self.name
 
@@ -3339,7 +3341,8 @@ class Composite(ScalarOp):
         This fct allow fix patch this.
 
         """
-        out = self.__class__(self.inputs, self.outputs)
+        d = dict([(k, getattr(self, k)) for k in self.init_param])
+        out = self.__class__(**d)
         if name:
             out.name = name
         else:

theano/tensor/basic.py

@@ -3030,7 +3030,7 @@ def mean(input, axis=None, dtype=None, op=False, keepdims=False,
 
     if dtype == 'float16' or (dtype is None and input.dtype == 'float16'):
         s = cast(s, 'float16')
-
+    s.name = 'mean'
     return s
 
 
@@ -3075,7 +3075,9 @@ def var(input, axis=None, keepdims=False):
     centered_input = input - mean_input
 
     # return the mean sqr
-    return mean((centered_input ** 2), axis, keepdims=keepdims)
+    v = mean((centered_input ** 2), axis, keepdims=keepdims)
+    v.name = 'var'
+    return v
 
 
 @constructor

theano/tensor/nnet/__init__.py

@@ -4,5 +4,6 @@ from .Conv3D import *
 from .ConvGrad3D import *
 from .ConvTransp3D import *
 from .sigm import (softplus, sigmoid, sigmoid_inplace,
-                  scalar_sigmoid, ultra_fast_sigmoid,
-                  hard_sigmoid)
+                   scalar_sigmoid, ultra_fast_sigmoid,
+                   hard_sigmoid)
+from .bn import batch_normalization

theano/tensor/nnet/bn.py

@@ -4,8 +4,10 @@ from theano.scalar import add, sub, true_div, mul
 
 
 class BNComposite(Composite):
+    init_param = ('dtype',)
 
     def __init__(self, dtype):
+        self.dtype = dtype
         x = theano.scalar.Scalar(dtype=dtype).make_variable()
         mean = theano.scalar.Scalar(dtype=dtype).make_variable()
         std = theano.scalar.Scalar(dtype=dtype).make_variable()
@@ -33,6 +35,8 @@ def batch_normalization(inputs, gamma, beta, mean, std,
     to a set of activations.
     Work also on GPU
 
+    .. versionadded:: 0.7.1
+
     Parameters
     ----------
     inputs : symbolic tensor

theano/tensor/nnet/nnet.py

@@ -2014,6 +2014,8 @@ def relu(x, alpha=0):
     """
     Compute the element-wise rectified linear activation function.
 
+    .. versionadded:: 0.7.1
+
     Parameters
     ----------
     x : symbolic tensor

theano/tensor/opt.py

@@ -357,7 +357,7 @@ def inplace_elemwise_optimizer_op(OP):
                             fgraph.validate()
                             chk = fgraph.checkpoint()
                             nb_change_no_validate = 0
-                    except (ValueError, TypeError, InconsistencyError) as e:
+                    except (ValueError, InconsistencyError) as e:
                         if check_each_change != 1 and not raised_warning:
                             print(("Some inplace optimization was not "
                                    "performed due to unexpected error:"),
@@ -2414,7 +2414,8 @@ def local_useless_subtensor(node):
     return [node.inputs[0]]
 
 
-@register_canonicalize
+# fast_compile to allow opt subtensor(cast{float32}(make_vector))
+@register_canonicalize('fast_compile')
 @gof.local_optimizer([Subtensor])
 def local_subtensor_lift(node):
     """