merge

theano/configdefaults.py

@@ -68,3 +68,12 @@ AddConfigVar('gpuelemwise.sync',
 AddConfigVar('traceback.limit',
              "The number of stack to trace. -1 mean all.",
              IntParam(5))
+
+
+###
+### To disable some warning about old bug that are fixed now.
+###
+
+AddConfigVar('warn.argmax_pushdown_bug',
+             "Warn if in past version of Theano we generated a bug with the optimisation theano.tensor.nnet.nnet.local_argmax_pushdown optimization. Was fixed 27 may 2010",
+             BoolParam(True))

theano/gof/cc.py

@@ -941,14 +941,25 @@ class CLinker(link.Linker):
         try:
             debug("LOCATION", location)
             c_compiler = self.c_compiler()
+            libs = self.libraries()
+            preargs = self.compile_args()
+            if c_compiler.__name__=='nvcc_module_compile_str' and config.lib.amdlibm:
+                #this lib don't work correctly with nvcc in device code.
+                if '<amdlibm.h>' in mod.includes:
+                    mod.includes.remove('<amdlibm.h>')
+                if '-DREPLACE_WITH_AMDLIBM' in preargs:
+                    preargs.remove('-DREPLACE_WITH_AMDLIBM')
+                if 'amdlibm' in libs:
+                    libs.remove('amdlibm')
+
             module = c_compiler(
                     module_name=mod.name,
                     src_code = mod.code(),
                     location=location,
                     include_dirs=self.header_dirs(),
                     lib_dirs=self.lib_dirs(),
-                    libs=self.libraries(),
-                    preargs=self.compile_args())
+                    libs=libs,
+                    preargs=preargs)
         finally:
             release_lock()
 

theano/gof/cmodule.py

@@ -353,7 +353,7 @@ class ModuleCache(object):
                     # If so, it should not have been deleted.  This should be considered a
                     # failure of the OTHER process, that deleted it.
                     if entry in self.module_from_name:
-                        error("The module %s that was loaded by this ModuleCache can no longer be read from file %s ... this could lead to problems." % (key,entry))
+                        warning("The module %s that was loaded by this ModuleCache can no longer be read from file %s ... this could lead to problems." % (key,entry))
                         del self.module_from_name[entry]
 
                     info("deleting ModuleCache entry", entry)
@@ -455,7 +455,9 @@ class ModuleCache(object):
         #debug('stats', self.stats, sum(self.stats))
         return rval
 
-    age_thresh_del = 60*60*24*31
+    age_thresh_del = 60*60*24*31#31 days
+    age_thresh_del_unversionned = 60*60*24*7#7 days
+
     """The default age threshold for `clear_old` (in seconds)
     """
     def clear_old(self, age_thresh_del=None): #default to a 31-day age_thresh_delold
@@ -519,7 +521,8 @@ class ModuleCache(object):
                 assert parent.startswith(os.path.join(self.dirname, 'tmp'))
                 info("clear_unversioned removing cache dir", parent)
                 _rmtree(parent)
-
+                
+        time_now = time.time()
         for filename in os.listdir(self.dirname):
             if filename.startswith('tmp'):
                 try:
@@ -528,13 +531,15 @@ class ModuleCache(object):
                 except IOError:
                     has_key = False
                 if not has_key:
-                    #TODO: Remove this, or at least wait one week
-                    # This cache dir is either in use by another process
-                    # (in that case, it will be removed by the code above),
-                    # or a remnant of a crashed process, in that case, it will
-                    # be removed by clear_old at some point.
-                    info("clear_unversioned removing cache dir", filename)
-                    _rmtree(os.path.join(self.dirname, filename))
+                    age = time_now - last_access_time(os.path.join(self.dirname, filename))
+                    #In normal case, the processus that created this directory will delete it
+                    #In case this processus crash, it won't be cleaned up.
+                    #As we don't know how to know if this directory is still used
+                    #we wait 1 weak and suppose that the processus crashed
+                    #and we do the clean up for it.
+                    if age > self.age_thresh_del_unversionned:
+                        info("clear_unversioned removing cache dir", filename)
+                        _rmtree(os.path.join(self.dirname, filename))
 
     def _on_atexit(self):
         self.refresh()

theano/sandbox/cuda/conv.cu

@@ -99,7 +99,7 @@ CudaNdarray_conv_valid(const CudaNdarray *img, const CudaNdarray * kern,
 
     if (verbose>1)
     {
-        printf("INFO: Running conv_valid version %d with inputs:\n",version);
+        printf("INFO: Running conv_valid version=%d, MACRO kern_width=%d with inputs:\n",version,THEANO_KERN_WID);
         printf("INFO:   img  dim: %i %i %i %i  img  stride: %i %i %i %i\n", 
                 CudaNdarray_HOST_DIMS(img)[0], CudaNdarray_HOST_DIMS(img)[1],CudaNdarray_HOST_DIMS(img)[2],CudaNdarray_HOST_DIMS(img)[3],
                 CudaNdarray_HOST_STRIDES(img)[0], CudaNdarray_HOST_STRIDES(img)[1],CudaNdarray_HOST_STRIDES(img)[2],CudaNdarray_HOST_STRIDES(img)[3]);
@@ -735,7 +735,7 @@ CudaNdarray_conv_full(const CudaNdarray *img, const CudaNdarray * kern, CudaNdar
 
     if (verbose>1)
     {
-        printf("INFO: Running conv_full version %d with inputs:\n",version);
+        printf("INFO: Running conv_full version=%d, MACRO kern_width=%d with inputs:\n",version,THEANO_KERN_WID);
         printf("INFO:   img  dim: %i %i %i %i  img  stride: %i %i %i %i\n", 
                 CudaNdarray_HOST_DIMS(img)[0], CudaNdarray_HOST_DIMS(img)[1],CudaNdarray_HOST_DIMS(img)[2],CudaNdarray_HOST_DIMS(img)[3],
                 CudaNdarray_HOST_STRIDES(img)[0], CudaNdarray_HOST_STRIDES(img)[1],CudaNdarray_HOST_STRIDES(img)[2],CudaNdarray_HOST_STRIDES(img)[3]);

theano/sandbox/rng_mrg.py

@@ -149,6 +149,8 @@ class mrg_uniform_base(Op):
         return Apply(self, 
                 [rstate, size], 
                 [rstate.type(), self.output_type()])
+    def c_code_cache_version(self):
+        return (1,)
 
 class mrg_uniform(mrg_uniform_base):
     #CPU VERSION

theano/sandbox/test_rng_mrg.py

-import sys, time
+import os, sys, time
 
 import numpy
 import theano
@@ -32,7 +32,8 @@ utt.seed_rng()
 # 12 streams
 # 7 substreams for each stream
 # 5 samples drawn from each substream
-java_samples = numpy.loadtxt('samples_MRG31k3p_12_7_5.txt')
+java_samples = numpy.loadtxt(os.path.join(os.path.split(theano.__file__)[0],
+                                          'sandbox','samples_MRG31k3p_12_7_5.txt'))
 
 
 def test_deterministic():

theano/tensor/nnet/nnet.py

@@ -2,8 +2,10 @@
 
 :note: TODO: factor this out into a neural-network toolbox.
 """
+import logging
 import numpy
 
+import theano
 from theano import gof
 from theano import printing
 from theano.tensor import basic as tensor
@@ -872,7 +874,15 @@ opt.register_specialize(local_crossentropy_to_crossentropy_with_softmax_grad)
 @opt.register_specialize
 @gof.local_optimizer([tensor._max_and_argmax])
 def local_argmax_pushdown(node):
-    if node.op == tensor._max_and_argmax:
+    if node.op == tensor._max_and_argmax and node.inputs[0].owner and \
+            len(node.outputs[0].clients)>0 and node.inputs[0].owner.op in \
+            (softmax, softplus, tensor.exp, tensor.log, tensor.tanh, sigmoid,
+             softmax_with_bias):
+        if theano.config.warn.argmax_pushdown_bug:
+            logging.getLogger('theano.tensor.nnet.nnet').warn("WARNING: their was a bug in Theano fixed the 27 may 2010 in this case. I.E. when we take the max of a softplus, softmax, exp, log, tanh, sigmoid, softmax_with_bias op, we where doing the max of the parent of the input. To remove this warning set the Theano flags 'warn.argmax_pushdown_bug' to False")
+
+
+    if node.op == tensor._max_and_argmax and node.inputs[0].owner and len(node.outputs[0].clients)==0:
         x_max, x_argmax = node.outputs
         x, axis = node.inputs
         #TODO: Make a list/set of monotonic ops...

theano/tensor/nnet/sigm.py

@@ -117,8 +117,15 @@ log1msigm_to_softplus = gof.PatternSub(
     (tensor.neg, (softplus, 'x')),
     allow_multiple_clients = True)
 
+log1pexp_to_softplus = gof.PatternSub(
+    (tensor.log1p, 
+     (tensor.exp, 'x')),
+    (softplus, 'x'),
+    allow_multiple_clients = True)
+
 opt.register_stabilize(logsigm_to_softplus, name = 'logsigm_to_softplus')
 opt.register_stabilize(log1msigm_to_softplus, name = 'log1msigm_to_softplus')
+opt.register_stabilize(log1pexp_to_softplus, name = 'log1pexp_to_softplus')
 
 def is_1pexp(t):
     # if t is of form (1+exp(x)), return x

theano/tensor/nnet/tests/test_nnet.py

@@ -773,9 +773,11 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase):
 def test_argmax_pushdown():
     x = tensor.dmatrix()
 
+    #test that the max_and_argmax is pushed down if the max is not used
+    out = tensor.max_and_argmax(softmax(tensor.exp(tensor.tanh(sigmoid(x)))))[1]
     env = gof.Env(
             [x],
-            [tensor.max(softmax(tensor.exp(tensor.tanh(sigmoid(x)))))])
+            [out])
 
     theano.compile.mode.optdb.query(
             theano.compile.mode.OPT_FAST_RUN).optimize(env)
@@ -785,27 +787,67 @@ def test_argmax_pushdown():
         #print node.op
     assert len(env.toposort()) == 2 # an output_guard is second
     assert env.toposort()[0].op == tensor._max_and_argmax
+    assert str(env.toposort()[1].op) == 'OutputGuard'
+    x = tensor.dmatrix()
+    #test that the max_and_argmax is not pushed down if the max is used
+    out = tensor.max_and_argmax(softmax(tensor.exp(tensor.tanh(sigmoid(x)))))[0]
+    env = gof.Env(
+            [x],
+            [out])
+
+    theano.compile.mode.optdb.query(
+            theano.compile.mode.OPT_FAST_RUN).optimize(env)
+
+    #print 'AFTER'
+    #for node in env.toposort():
+        #print node.op
+    assert len(env.toposort()) == 4 # an output_guard is second
+    assert isinstance(env.toposort()[0].op, tensor.Elemwise)
+    assert isinstance(env.toposort()[1].op, Softmax)
+    assert isinstance(env.toposort()[2].op, tensor.MaxAndArgmax)
+    assert str(env.toposort()[3].op) == 'OutputGuard'
+
 
 def test_argmax_pushdown_bias():
     x = tensor.dmatrix()
     b = tensor.dvector()
 
+    out = tensor.argmax(softmax_with_bias(x, b))
     env = gof.Env(
             [x,b],
-            [tensor.max(softmax_with_bias(x, b))])
+            [out])
 
     theano.compile.mode.optdb.query(
             theano.compile.mode.OPT_FAST_RUN).optimize(env)
 
-    print 'AFTER'
-    for node in env.toposort():
-        print node.op
+    #print 'AFTER'
+    #for node in env.toposort():
+    #    print node.op
     assert len(env.toposort()) == 4
     assert isinstance(env.toposort()[0].op, tensor.DimShuffle)
     assert isinstance(env.toposort()[1].op, tensor.Elemwise)
     assert isinstance(env.toposort()[2].op, tensor.MaxAndArgmax)
     assert str(env.toposort()[3].op) == 'OutputGuard'
 
+    x = tensor.dmatrix()
+    b = tensor.dvector()
+
+    out = tensor.max_and_argmax(softmax_with_bias(x, b))[0]
+    env = gof.Env(
+            [x,b],
+            [out])
+
+    theano.compile.mode.optdb.query(
+            theano.compile.mode.OPT_FAST_RUN).optimize(env)
+
+    #print 'AFTER'
+    #for node in env.toposort():
+    #    print node.op
+    assert len(env.toposort()) == 3
+    assert isinstance(env.toposort()[0].op, SoftmaxWithBias)
+    assert isinstance(env.toposort()[1].op, tensor.MaxAndArgmax)
+    assert str(env.toposort()[2].op) == 'OutputGuard'
+
 def test_asymptotic_32():
     """
     This test makes sure that our functions behave sensibly when huge values are present

theano/tensor/nnet/tests/test_sigm.py

@@ -83,3 +83,49 @@ class T_sigmoid_opts(unittest.TestCase):
         assert [node.op for node in f.maker.env.toposort()] == [tensor.neg, 
                 sigmoid_inplace]
 
+
+class T_softplus_opts(unittest.TestCase):
+    def setUp(self):
+        if theano.config.mode == 'FAST_COMPILE':
+            m = theano.compile.mode.get_mode('FAST_RUN')
+        else:
+            m = theano.compile.mode.get_default_mode().excluding('local_elemwise_fusion')
+        self.m = m
+        utt.seed_rng()
+    def test_logsigm_to_softplus(self):
+        x = T.vector()
+    
+        out = T.log(sigmoid(x))
+        f = theano.function([x],out,mode=self.m)
+        topo = f.maker.env.toposort()
+        print topo
+        assert len(topo)==3
+        assert isinstance(topo[0].op.scalar_op, theano.scalar.Neg)
+        assert isinstance(topo[1].op.scalar_op, theano.tensor.nnet.sigm.ScalarSoftplus)
+        assert isinstance(topo[2].op.scalar_op, theano.scalar.Neg)
+        f(numpy.random.rand(54))
+
+    def test_log1msigm_to_softplus(self):
+        x = T.vector()
+    
+        out = T.log(1-sigmoid(x))
+        f = theano.function([x],out,mode=self.m)
+        topo = f.maker.env.toposort()
+        assert len(topo)==2
+        assert isinstance(topo[0].op.scalar_op, theano.tensor.nnet.sigm.ScalarSoftplus)
+        assert isinstance(topo[1].op.scalar_op, theano.scalar.Neg)
+        f(numpy.random.rand(54))
+        
+    def test_log1pexp_to_softplus(self):
+        m = theano.config.mode
+        if m == 'FAST_COMPILE':
+            m = 'FAST_RUN'
+
+        x = T.vector()
+    
+        out = T.log(1+T.exp(x))
+        f = theano.function([x],out,mode=self.m)
+        topo = f.maker.env.toposort()
+        assert len(topo)==1
+        assert isinstance(topo[0].op.scalar_op,theano.tensor.nnet.sigm.ScalarSoftplus)
+        f(numpy.random.rand(54))