Add, document and test hard_sigmoid.

doc/library/tensor/nnet/nnet.txt

@@ -15,30 +15,63 @@
     :Parameters: *x* - symbolic Tensor (or compatible)
     :Return type: same as x
     :Returns: element-wise sigmoid: :math:`sigmoid(x) = \frac{1}{1 + \exp(-x)}`.
-    :note: see :func:`ultra_fast_sigmoid` for a faster version
+    :note: see :func:`ultra_fast_sigmoid` or :func:`hard_sigmoid` for faster version.
+        Speed comparison for 100M float64 element on a Core2 Duo @ 3.16 GHz.
 
-Example:
+          - hard_sigmoid: 1.1s
+          - ultra_fast_sigmoid: 1.4s
+          - sigmoid (with amdlibm): 2.3s
+          - sigmoid (without amdlibm): 3.7s
 
-.. code-block:: python
+        Precision: sigmoid(without or without amdlibm) > ultra_fast_sigmoid > hard_sigmoid.
 
-    x,y,b = T.dvectors('x','y','b')
-    W = T.dmatrix('W')
-    y = T.nnet.sigmoid(T.dot(W,x) + b)
+   .. image:: sigmoid_prec.png
+
+   Example:
+
+   .. code-block:: python
 
-.. note:: The underlying code will return an exact 0 or 1 if an element of x is too small or too big.
+       x,y,b = T.dvectors('x','y','b')
+       W = T.dmatrix('W')
+       y = T.nnet.sigmoid(T.dot(W,x) + b)
+
+   .. note:: The underlying code will return an exact 0 or 1 if an
+      element of x is too small or too big.
 
 .. function:: ultra_fast_sigmoid(x)
 
-   Returns the standard sigmoid nonlinearity applied to x
+   Returns the *approximated* standard :func:`sigmoid` nonlinearity applied to x.
     :Parameters: *x* - symbolic Tensor (or compatible)
     :Return type: same as x
     :Returns: approximated element-wise sigmoid: :math:`sigmoid(x) = \frac{1}{1 + \exp(-x)}`.
-    :note: To automatically change all sigmoid op to this version, use
+    :note: To automatically change all :func:`sigmoid` op to this version, use
       the Theano optimization ``local_ultra_fast_sigmoid``. This can be done
       with the Theano flag ``optimizer_including=local_ultra_fast_sigmoid``.
       This optimization is done late, so it shouldn't affect
       stabilization optimization.
 
+   .. note:: The underlying code will return 0.00247262315663 as the
+       minimum value and 0.997527376843 as the maximum value. So it
+       never return 0 or 1.
+
+
+
+
+.. function:: hard_sigmoid(x)
+
+   Returns the *approximated* standard :func:`sigmoid` nonlinearity applied to x.
+    :Parameters: *x* - symbolic Tensor (or compatible)
+    :Return type: same as x
+    :Returns: approximated element-wise sigmoid: :math:`sigmoid(x) = \frac{1}{1 + \exp(-x)}`.
+    :note: To automatically change all :func:`sigmoid` op to this version, use
+      the Theano optimization ``local_hard_sigmoid``. This can be done
+      with the Theano flag ``optimizer_including=local_hard_sigmoid``.
+      This optimization is done late, so it shouldn't affect
+      stabilization optimization.
+
+   .. note:: The underlying code will return an exact 0 or 1 if an
+      element of x is too small or too big.
+
 .. function:: softplus(x)
 
    Returns the softplus nonlinearity applied to x

theano/tensor/nnet/__init__.py

@@ -4,4 +4,5 @@ from Conv3D import *
 from ConvGrad3D import *
 from ConvTransp3D import *
 from sigm import (softplus, sigmoid, sigmoid_inplace,
-                  scalar_sigmoid, ultra_fast_sigmoid)
+                  scalar_sigmoid, ultra_fast_sigmoid,
+                  hard_sigmoid)

theano/tensor/nnet/sigm.py

@@ -112,6 +112,42 @@ for i in xrange(750):
         """ % locals()
         raise theano.gof.utils.MethodNotDefined()
 
+    @staticmethod
+    def gen_graph():
+        """
+        This method was used to generate the graph: sigmoid_prec.png in the doc
+        """
+        import matplotlib
+        data = numpy.arange(-15, 15, .1)
+        val = 1/(1+numpy.exp(-data))
+
+        def hard_sigmoid(x):
+            return theano.tensor.nnet.hard_sigmoid(x)
+
+        def ultra_fast_sigmoid(x):
+            return theano.tensor.nnet.ultra_fast_sigmoid(x)
+
+        val_hard = hard_sigmoid(data).eval()
+        val_ultra = ultra_fast_sigmoid(data).eval()
+
+        import matplotlib.pyplot as plt
+        import os
+        fig = plt.figure()
+        ax = fig.add_subplot(111)
+        ax.plot(data, val)#, 'o-')
+        ax.plot(data, val_ultra)#, '-')
+        ax.plot(data, val_hard)#, '-')
+        ax.grid(True)
+        ax.legend(("sigmoid", "ultra_fast", "hard"), "upper left")
+        fname = os.path.join(os.path.dirname(theano.__file__), '..',
+                             'doc', 'library', 'tensor', 'nnet',
+                             'sigmoid_prec.png')
+        plt.savefig(fname)
+        print "New picture saved at", fname
+        print val_ultra.max()
+        print val_ultra.min()
+
+
 scalar_sigmoid = ScalarSigmoid(scalar.upgrade_to_float, name='scalar_sigmoid')
 sigmoid = elemwise.Elemwise(scalar_sigmoid, name='sigmoid')
 
@@ -210,7 +246,6 @@ def local_ultra_fast_sigmoid(node):
     if (isinstance(node.op, tensor.Elemwise) and
             node.op.scalar_op == scalar_sigmoid):
         out = ultra_fast_sigmoid(node.inputs[0])
-        out2 = ultra_fast_sigmoid(node.inputs[0])
 
         def values_eq_approx_remove_low_prec(a, b):
             # atol is found by trial/error.
@@ -223,6 +258,41 @@ theano.compile.optdb['uncanonicalize'].register("local_ultra_fast_sigmoid",
                                                 local_ultra_fast_sigmoid)
 
 
+def hard_sigmoid(x):
+    """An approximation of sigmoid.
+
+    More approximate and faster then ultra_fast_sigmoid.
+
+    Approx in 3 parts: 0, scaled linear, 1
+
+    Removing the slop and shift don't make it faster.
+
+    """
+    slop = 0.2
+    shift = 0.5
+    x = (x * 0.2) + shift
+    x = tensor.clip(x, 0, 1)
+    return x
+
+
+#@opt.register_uncanonicalize
+@gof.local_optimizer([sigmoid])
+def local_hard_sigmoid(node):
+    if (isinstance(node.op, tensor.Elemwise) and
+            node.op.scalar_op == scalar_sigmoid):
+        out = hard_sigmoid(node.inputs[0])
+
+        def values_eq_approx_remove_low_prec(a, b):
+            # atol is found by trial/error.
+            # Other test could fail without good reason.
+            return tensor.TensorType.values_eq_approx(a, b, atol=0.1)
+        # Let DebugMode know that there this opt approx the values.
+        out.values_eq_approx = values_eq_approx_remove_low_prec
+        return [out]
+theano.compile.optdb['uncanonicalize'].register("local_hard_sigmoid",
+                                                local_hard_sigmoid)
+
+
 class ScalarSoftplus(scalar.UnaryScalarOp):
     @staticmethod
     def static_impl(x):

theano/tensor/nnet/tests/test_sigm.py

@@ -9,7 +9,7 @@ from theano import tensor as T
 from theano import config
 from theano.tests import unittest_tools as utt
 from theano.tensor.nnet import (sigmoid, sigmoid_inplace,
-                                softplus, ultra_fast_sigmoid)
+                                softplus, ultra_fast_sigmoid, hard_sigmoid)
 from theano.tensor.nnet.sigm import (
     compute_mul, is_1pexp, parse_mul_tree, perform_sigm_times_exp,
     register_local_1msigmoid, simplify_mul,
@@ -46,6 +46,16 @@ UltraFastSigmoidTester = makeBroadcastTester(
 # This is an approx of the sigmoid. That is why we raise eps
     eps=5e-2)
 
+HardSigmoidTester = makeBroadcastTester(
+    op=hard_sigmoid,
+    expected=lambda inputs: check_floatX(
+        inputs, 1/(1+numpy.exp(-inputs))),
+    good=_good_broadcast_unary_normal_no_complex,
+    #grad=_grad_broadcast_unary_normal,
+    name='UltraFastSigmoidTester',
+# This is an approx of the sigmoid. That is why we raise eps
+    eps=1e-1)
+
 
 SoftplusTester = makeBroadcastTester(
     op=softplus,
@@ -295,11 +305,32 @@ class T_sigmoid_opts(unittest.TestCase):
 
         mode = self.get_mode('local_ultra_fast_sigmoid')
         f = theano.function([x], s, mode=mode)
-        assert f.maker.fgraph.toposort()[0].op == sigmoid
+        topo = f.maker.fgraph.toposort()
+        assert len(topo) == 1
+        assert topo[0].op == sigmoid
 
         mode = self.get_mode().including('local_ultra_fast_sigmoid')
         f = theano.function([x], s, mode=mode)
-        assert f.maker.fgraph.toposort()[0].op == ultra_fast_sigmoid
+        topo = f.maker.fgraph.toposort()
+        assert topo[0].op == ultra_fast_sigmoid
+        assert len(topo) == 1
+        ux_v = f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]])
+
+    def test_local_hard_sigmoid(self):
+        x = tensor.matrix('x')
+        s = sigmoid(x)
+
+        mode = self.get_mode('local_hard_sigmoid')
+        f = theano.function([x], s, mode=mode)
+        topo = f.maker.fgraph.toposort()
+        assert topo[0].op == sigmoid
+        assert len(topo) == 1
+
+        mode = self.get_mode().including('local_hard_sigmoid')
+        f = theano.function([x], s, mode=mode)
+        topo = f.maker.fgraph.toposort()
+        assert len(topo) > 1
+        assert not any([n.op == sigmoid for n in topo])
         ux_v = f([[-50, -10, -4, -1, 0, 1, 4, 10, 50]])