Added doc about ultra_fast_sigmoid.

doc/library/tensor/nnet/nnet.txt

@@ -14,7 +14,8 @@
    Returns the standard sigmoid nonlinearity applied to x
     :Parameters: *x* - symbolic Tensor (or compatible)
     :Return type: same as x
-    :Returns: element-wise sigmoid: :math:`sigmoid(x) = \frac{1}{1 + \exp(-x)}`. 
+    :Returns: element-wise sigmoid: :math:`sigmoid(x) = \frac{1}{1 + \exp(-x)}`.
+    :note: see :func:`ultra_fast_sigmoid` for a faster version
 
 Example:
 
@@ -26,6 +27,18 @@ Example:
 
 .. note:: The underlying code will return an exact 0 or 1 if an element of x is too small or too big.
 
+.. function:: ultra_fast_sigmoid(x)
+
+   Returns the standard sigmoid nonlinearity applied to x
+    :Parameters: *x* - symbolic Tensor (or compatible)
+    :Return type: same as x
+    :Returns: approximated element-wise sigmoid: :math:`sigmoid(x) = \frac{1}{1 + \exp(-x)}`.
+    :note: To automatically change all sigmoid op to this version, use
+      the Theano optimization ``local_ultra_fast_sigmoid``. This can be done
+      with the Theano flag ``optimizer_including=local_ultra_fast_sigmoid``.
+      This optimization is done late, so it shouldn't affect
+      stabilization optimization.
+
 .. function:: softplus(x)
 
    Returns the softplus nonlinearity applied to x

theano/tensor/nnet/sigm.py

@@ -192,6 +192,37 @@ pprint.assign(ultra_fast_sigmoid,
               printing.FunctionPrinter('ultra_fast_sigmoid'))
 
 
+#@opt.register_uncanonicalize
+@gof.local_optimizer([sigmoid])
+def local_ultra_fast_sigmoid(node):
+    """
+    When enabled, change all sigmoid to ultra_fast_sigmoid.
+
+    To example do mode.including('local_ultra_fast_sigmoid')
+    or use the Theano flag optimizer_including=local_ultra_fast_sigmoid
+
+    This speed up the sigmoid op by using an approximation.
+
+    This is done after the stabilization and specialize phase
+    to don't interact with them.
+
+    """
+    if (isinstance(node.op, tensor.Elemwise) and
+            node.op.scalar_op == scalar_sigmoid):
+        out = ultra_fast_sigmoid(node.inputs[0])
+        out2 = ultra_fast_sigmoid(node.inputs[0])
+
+        def values_eq_approx_remove_low_prec(a, b):
+            # atol is found by trial/error.
+            # Other test could fail without good reason.
+            return tensor.TensorType.values_eq_approx(a, b, atol=0.02)
+        # Let DebugMode know that there this opt approx the values.
+        out.values_eq_approx = values_eq_approx_remove_low_prec
+        return [out]
+theano.compile.optdb['uncanonicalize'].register("local_ultra_fast_sigmoid",
+                                                local_ultra_fast_sigmoid)
+
+
 class ScalarSoftplus(scalar.UnaryScalarOp):
     @staticmethod
     def static_impl(x):

theano/tensor/nnet/tests/test_sigm.py

@@ -289,6 +289,19 @@ class T_sigmoid_opts(unittest.TestCase):
             ux_v = f([[50]], 0.1)
             assert not numpy.isnan(ux_v)
 
+    def test_local_ultra_fast_sigmoid(self):
+        x = tensor.matrix('x')
+        s = sigmoid(x)
+
+        mode = self.get_mode('local_ultra_fast_sigmoid')
+        f = theano.function([x], s, mode=mode)
+        assert f.maker.fgraph.toposort()[0].op == sigmoid
+
+        mode = self.get_mode().including('local_ultra_fast_sigmoid')
+        f = theano.function([x], s, mode=mode)
+        assert f.maker.fgraph.toposort()[0].op == ultra_fast_sigmoid
+        ux_v = f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]])
+
 
 class T_softplus_opts(unittest.TestCase):
     def setUp(self):