Revert to the boundaries for float32 when computing in float16 for now.

theano/tensor/nnet/sigm.py

@@ -68,9 +68,11 @@ class ScalarSigmoid(scalar.UnaryScalarOp):
         #                   (theano._asarray(1.0, dtype=dt) +
         #                    numpy.exp(-theano._asarray([i,-i], dtype=dt))))
 
-        if node.inputs[0].type == scalar.float16:
-            return """%(z)s = %(x)s < -11.0f ? 0.0 : %(x)s > 7.0f ? 1.0f : 1.0f /(1.0f + exp(-%(x)s));""" % locals()
-        elif node.inputs[0].type == scalar.float32:
+        # float16 limits: -11.0, 7.0f
+        # We use the float32 limits for float16 for now as the
+        # computation will happend in float32 anyway.
+        if (node.inputs[0].type == scalar.float32 or
+                node.inputs[0].type == scalar.float16):
             return """%(z)s = %(x)s < -88.0f ? 0.0 : %(x)s > 15.0f ? 1.0f : 1.0f /(1.0f + exp(-%(x)s));""" % locals()
         elif node.inputs[0].type == scalar.float64:
             return """%(z)s = %(x)s < -709.0 ? 0.0 : %(x)s > 19.0 ? 1.0 : 1.0 /(1.0+exp(-%(x)s));""" % locals()
@@ -335,9 +337,12 @@ class ScalarSoftplus(scalar.UnaryScalarOp):
         #  for i in xrange(750):
         #      print i, repr(numpy.log1p(numpy.exp(theano._asarray([i,-i], dtype=dt))))
         # the boundary checks prevent us from generating inf
-        if node.inputs[0].type == scalar.float16:
-            return """%(z)s = %(x)s < -17.0f ? 0.0 : %(x)s > 6.0f ? %(x)s : log1p(exp(%(x)s));""" % locals()
-        elif node.inputs[0].type == scalar.float32:
+
+        # float16 limits: -17.0, 6.0
+        # We use the float32 limits for float16 for now as the
+        # computation will happend in float32 anyway.
+        if (node.inputs[0].type == scalar.float32 or
+                node.inputs[0].type == scalar.float16):
             return """%(z)s = %(x)s < -103.0f ? 0.0 : %(x)s > 14.0f ? %(x)s : log1p(exp(%(x)s));""" % locals()
         elif node.inputs[0].type == scalar.float64:
             return """%(z)s = %(x)s < -745.0 ? 0.0 : %(x)s > 16.0 ? %(x)s : log1p(exp(%(x)s));""" % locals()