Merge pull request #6050 from vipulraheja/add-selu-activation

doc/library/tensor/nnet/nnet.txt

@@ -20,6 +20,8 @@
    - :func:`softmax`
    - :func:`softsign`
    - :func:`relu() <theano.tensor.nnet.relu>`
+   - :func:`elu() <theano.tensor.nnet.elu>`
+   - :func:`selu() <theano.tensor.nnet.selu>`
    - :func:`binary_crossentropy`
    - :func:`sigmoid_binary_crossentropy`
    - :func:`.categorical_crossentropy`
@@ -147,6 +149,10 @@
 
 .. autofunction:: theano.tensor.nnet.relu
 
+.. autofunction:: theano.tensor.nnet.elu
+
+.. autofunction:: theano.tensor.nnet.selu
+
 .. function:: binary_crossentropy(output,target)
 
    Computes the binary cross-entropy between a target and an output:

theano/tensor/nnet/__init__.py

@@ -18,7 +18,7 @@ from .nnet import (
     graph_merge_softmax_with_crossentropy_softmax, h_softmax,
     logsoftmax, logsoftmax_op, prepend_0_to_each_row, prepend_1_to_each_row,
     prepend_scalar_to_each_row, relu, softmax, softmax_grad, softmax_graph,
-    softmax_op, softmax_simplifier, softmax_with_bias, elu,
+    softmax_op, softmax_simplifier, softmax_with_bias, elu, selu,
     confusion_matrix, softsign)
 from . import opt
 from .conv import ConvOp

theano/tensor/nnet/nnet.py

@@ -2423,7 +2423,7 @@ def h_softmax(x, batch_size, n_outputs, n_classes, n_outputs_per_class,
 
 def elu(x, alpha=1):
     """
-    Compute the element-wise exponential linear activation function.
+    Compute the element-wise exponential linear activation function [2]_.
 
     .. versionadded:: 0.8.0
 
@@ -2441,13 +2441,38 @@ def elu(x, alpha=1):
 
     References
     -----
-    .. [1] Djork-Arne Clevert,  Thomas Unterthiner, Sepp Hochreiter
+    .. [2] Djork-Arne Clevert,  Thomas Unterthiner, Sepp Hochreiter
         "Fast and Accurate Deep Network Learning by
         Exponential Linear Units (ELUs)" <http://arxiv.org/abs/1511.07289>`.
     """
     return tensor.switch(x > 0, x, alpha * tensor.expm1(x))
 
 
+def selu(x):
+    """Compute the element-wise Scaled Exponential Linear unit [3]_.
+
+    .. versionadded:: 0.9.0
+
+    Parameters
+    ----------
+    x : symbolic tensor
+        Tensor to compute the activation function for.
+
+    Returns
+    -------
+    symbolic tensor
+        Element-wise scaled exponential linear activation function applied to `x`.
+
+    References
+    ----------
+    .. [3] Klambauer G, Unterthiner T, Mayr A, Hochreiter S.
+        "Self-Normalizing Neural Networks" <https://arxiv.org/abs/1706.02515>
+    """
+    alpha = 1.6732632423543772848170429916717
+    scale = 1.0507009873554804934193349852946
+    return scale * elu(x, alpha)
+
+
 class ScalarSoftsign(theano.scalar.UnaryScalarOp):
     """
     Softsign activation function

theano/tensor/nnet/tests/test_nnet.py

@@ -32,6 +32,7 @@ from theano.tensor.nnet import (categorical_crossentropy,
                                 relu,
                                 h_softmax,
                                 elu,
+                                selu,
                                 binary_crossentropy,
                                 sigmoid_binary_crossentropy,
                                 confusion_matrix)
@@ -1737,6 +1738,19 @@ def test_elu():
         utt.assert_allclose(y, np.where(X > 0, X, alpha * (np.exp(X) - 1)))
 
 
+def test_selu():
+    alpha = 1.6732632423543772848170429916717
+    scale = 1.0507009873554804934193349852946
+
+    x = matrix('x')
+    seed = theano.tests.unittest_tools.fetch_seed()
+    rng = np.random.RandomState(seed)
+    X = rng.randn(20, 30).astype(config.floatX)
+
+    y = selu(x).eval({x: X})
+    utt.assert_allclose(y, np.where(X > 0, scale * X, scale * alpha * (np.exp(X) - 1)))
+
+
 def test_binary_crossentropy_reshape():
     # Reported as https://github.com/Theano/Theano/issues/4086
     a = tensor.tensor4('a')