Merge pull request #430 from delallea/minor

NEWS.txt

@@ -2,11 +2,11 @@
 
 Since 0.5rc2
 
- * Fix a memory leak with shared variable(we kept a pointor to the original value)
- * The key in our cache now store the hash of constant and not the constant value itself. This is significat for big constant.
- * theano-cache list list key file bigger then 1M
- * theano-cache list print an histograme of the number of key per compiled module
- * theano-cache list print the number of compiled module per op class
+ * Fixed a memory leak with shared variable (we kept a pointer to the original value)
+ * The keys in our cache now store the hash of constants and not the constant values themselves. This is significantly more efficient for big constant arrays.
+ * 'theano-cache list' lists key files bigger than 1M
+ * 'theano-cache list' prints an histogram of the number of keys per compiled module
+ * 'theano-cache list' prints the number of compiled modules per op class
 
 =============
 Release Notes

bin/theano-cache

@@ -39,6 +39,6 @@ else:
     print 'Type "theano-cache clear" to erase the cache'
     print 'Type "theano-cache list" to print the cache content'
     print 'Type "theano-cache unlock" to unlock the cache directory'
-    print 'Type "theano-cache cleanup" to delete old key format'
+    print 'Type "theano-cache cleanup" to delete keys in the old format'
     sys.exit(1)
 

doc/NEWS.txt

 .. _NEWS:
 
+Since 0.5rc2
+
+ * Fixed a memory leak with shared variable (we kept a pointer to the original value)
+ * The keys in our cache now store the hash of constants and not the constant values themselves. This is significantly more efficient for big constant arrays.
+ * 'theano-cache list' lists key files bigger than 1M
+ * 'theano-cache list' prints an histogram of the number of keys per compiled module
+ * 'theano-cache list' prints the number of compiled modules per op class
+
 =============
 Release Notes
 =============

doc/install.txt

@@ -84,7 +84,7 @@ Benjamin J. McCann provides `installation documentation <http://www.benmccann.co
 Gentoo
 ~~~~~~
 
-Brian Vandenberg emailed `installation instruction on Gentoo <https://groups.google.com/d/msg/theano-dev/-8WCMn2FMR0/bJPasoZXaqoJ>`.
+Brian Vandenberg emailed `installation instructions on Gentoo <http://groups.google.com/d/msg/theano-dev/-8WCMn2FMR0/bJPasoZXaqoJ>`.
 
 
 .. _linux_basic:

theano/gof/cc.py

@@ -6,6 +6,7 @@ Defines Linkers that deal with C implementations.
 from copy import copy
 import re #for set_compiledir
 import os, sys, StringIO
+from itertools import izip
 
 
 if sys.version_info[:2] >= (2,5):
@@ -733,8 +734,8 @@ class CLinker(link.Linker):
                                     output_storage,
                                     keep_lock=keep_lock)
         return thunk, \
-            [link.Container(input, storage) for input, storage in zip(self.env.inputs, input_storage)], \
-            [link.Container(output, storage, True) for output, storage in zip(self.env.outputs, output_storage)], \
+            [link.Container(input, storage) for input, storage in izip(self.env.inputs, input_storage)], \
+            [link.Container(output, storage, True) for output, storage in izip(self.env.outputs, output_storage)], \
             error_storage
 
     def get_init_tasks(self):
@@ -906,11 +907,12 @@ class CLinker(link.Linker):
             if isinstance(i, graph.Constant): #orphans
                 if id(i) not in constant_ids:
                     isig = (i.signature(), topological_pos, i_idx)
-                    # If the Theano constant provide a strong hash
+                    # If the Theano constant provides a strong hash
                     # (no collision for transpose, 2, 1, 0, -1, -2,
                     # 2 element swapped...) we put this hash in the signature
-                    # instead of the value. This make the key file much smaller
-                    # for big constant. Before this, we saw key file up to 80M.
+                    # instead of the value. This makes the key file much
+                    # smaller for big constant arrays. Before this, we saw key
+                    # files up to 80M.
                     if hasattr(isig[0], "theano_hash"):
                         isig = (isig[0].theano_hash(), topological_pos, i_idx)
                     try:
@@ -1234,20 +1236,21 @@ class OpWiseCLinker(link.LocalLinker):
         self.no_recycling = no_recycling
         return self
 
-    def make_all(self, profiler = None, input_storage = None, output_storage = None):
+    def make_all(self, profiler=None, input_storage=None, output_storage=None):
 
         # The lock will be acquired when we compile the first
         # C code. We will keep the lock untill all the function
         # compilation will be finished. This allow to don't
         # require the lock when all c code are already compiled!
-        orig_n_lock = getattr(get_lock,"n_lock",0)
+        orig_n_lock = getattr(get_lock, "n_lock", 0)
         try:
 
             env = self.env
             order = env.toposort()
             no_recycling = self.no_recycling
 
-            input_storage, output_storage, storage_map = link.map_storage(env, order, input_storage, output_storage)
+            input_storage, output_storage, storage_map = link.map_storage(
+                                    env, order, input_storage, output_storage)
             if self.allow_gc:
                 computed, last_user = link.gc_helper(order)
                 post_thunk_old_storage = []
@@ -1265,7 +1268,7 @@ class OpWiseCLinker(link.LocalLinker):
                 # There are ops that don't have _op_use_c_code property
                 # for example ifelse (or any ops that come with their own
                 # make_thunk
-                old_value = getattr(node.op,'_op_use_c_code', False)
+                old_value = getattr(node.op, '_op_use_c_code', False)
                 try:
                     node.op._op_use_c_code = True
                     thunks += [node.op.make_thunk(node,
@@ -1280,31 +1283,37 @@ class OpWiseCLinker(link.LocalLinker):
                 if self.allow_gc:
                     post_thunk_old_storage.append([storage_map[input]
                         for input in node.inputs
-                        if (input in computed) and (input not in env.outputs) and node == last_user[input]])
+                        if ((input in computed) and
+                            (input not in env.outputs) and
+                            node == last_user[input])])
 
             if no_recycling is True:
                 no_recycling = storage_map.values()
                 no_recycling = utils.difference(no_recycling, input_storage)
             else:
-                no_recycling = [storage_map[r] for r in no_recycling if r not in env.inputs]
+                no_recycling = [storage_map[r]
+                                for r in no_recycling if r not in env.inputs]
 
             f = link.streamline(env, thunks, order,
                     post_thunk_old_storage,
-                    no_recycling = no_recycling,
-                    nice_errors = self.nice_errors)
+                    no_recycling=no_recycling,
+                    nice_errors=self.nice_errors)
 
             f.allow_gc = self.allow_gc
 
         finally:
             # Release lock on compilation directory.
-            if getattr(get_lock,"n_lock",0) > orig_n_lock:
+            if getattr(get_lock, "n_lock", 0) > orig_n_lock:
                 release_lock()
                 assert get_lock.n_lock == orig_n_lock
 
-        return f, [link.Container(input, storage) for input, storage in zip(env.inputs, input_storage)], \
-            [link.Container(output, storage, True) for output, storage in zip(env.outputs, output_storage)], \
-            thunks, order
-
+        return (f,
+                [link.Container(input, storage)
+                 for input, storage in izip(env.inputs, input_storage)],
+                [link.Container(output, storage, True)
+                 for output, storage in izip(env.outputs, output_storage)],
+                thunks,
+                order)
 
 
 def _default_checker(x, y):
@@ -1313,7 +1322,9 @@ def _default_checker(x, y):
     variables contain the same data using ==.
     """
     if x[0] != y[0]:
-        raise Exception("Output mismatch.", {'performlinker': x[0], 'clinker': y[0]})
+        raise Exception("Output mismatch.",
+                        {'performlinker': x[0], 'clinker': y[0]})
+
 
 class DualLinker(link.Linker):
     """WRITEME
@@ -1327,7 +1338,7 @@ class DualLinker(link.Linker):
     function.
     """
 
-    def __init__(self, checker = _default_checker):
+    def __init__(self, checker=_default_checker):
         """
         Initialize a DualLinker.
 
@@ -1353,10 +1364,11 @@ class DualLinker(link.Linker):
         self.env = None
         self.checker = checker
 
-    def accept(self, env, no_recycling = []):
+    def accept(self, env, no_recycling=[]):
         if self.env is not None and self.env is not env:
             return type(self)(self.checker).accept(env, no_recycling)
-            #raise Exception("Cannot accept from a Linker that is already tied to another Env.")
+            # raise Exception("Cannot accept from a Linker that is already "
+            #                 "tied to another Env.")
         self.env = env
         self.no_recycling = no_recycling
         return self
@@ -1366,26 +1378,31 @@ class DualLinker(link.Linker):
         env = self.env
         no_recycling = self.no_recycling
 
-        _f, i1, o1, thunks1, order1 = link.PerformLinker().accept(env, no_recycling = no_recycling).make_all(**kwargs)
+        _f, i1, o1, thunks1, order1 = link.PerformLinker().accept(env,
+                                no_recycling=no_recycling).make_all(**kwargs)
         kwargs.pop('input_storage', None)
-        _f, i2, o2, thunks2, order2 =      OpWiseCLinker().accept(env, no_recycling = no_recycling).make_all(**kwargs)
+        _f, i2, o2, thunks2, order2 = OpWiseCLinker().accept(env,
+                                no_recycling=no_recycling).make_all(**kwargs)
 
         def f():
-            for input1, input2 in zip(i1, i2):
-                # set the inputs to be the same in both branches
-                # the copy is necessary in order for inplace ops not to interfere
+            for input1, input2 in izip(i1, i2):
+                # Set the inputs to be the same in both branches.
+                # The copy is necessary in order for inplace ops not to
+                # interfere.
                 input2.storage[0] = copy(input1.storage[0])
-            for thunk1, thunk2, node1, node2 in zip(thunks1, thunks2, order1, order2):
-                for output, storage in zip(node1.outputs, thunk1.outputs):
+            for thunk1, thunk2, node1, node2 in izip(thunks1, thunks2,
+                                                     order1, order2):
+                for output, storage in izip(node1.outputs, thunk1.outputs):
                     if output in no_recycling:
                         storage[0] = None
-                for output, storage in zip(node2.outputs, thunk2.outputs):
+                for output, storage in izip(node2.outputs, thunk2.outputs):
                     if output in no_recycling:
                         storage[0] = None
                 try:
                     thunk1()
                     thunk2()
-                    for output1, output2 in zip(thunk1.outputs, thunk2.outputs):
+                    for output1, output2 in izip(thunk1.outputs,
+                                                 thunk2.outputs):
                         self.checker(output1, output2)
                 except Exception:
                     link.raise_with_op(node1)

theano/gof/compiledir.py

@@ -120,12 +120,13 @@ def flatten(a):
 
 
 def cleanup():
-    """ Delete old keys from the compiledir
+    """
+    Delete keys in old format from the compiledir.
 
-        We define old key as key that have an ndarray in them.
-        Now we use an hash in the keys of the constant data.
+    We define keys in old format as keys that have an ndarray in them.
+    Now we use a hash in the keys of the constant data.
 
-        If there is no key left for a compiled module, we delete the module.
+    If there is no key left for a compiled module, we delete the module.
     """
     compiledir = theano.config.compiledir
     for directory in os.listdir(compiledir):
@@ -144,7 +145,6 @@ def cleanup():
                                 break
                     if len(keydata.keys) == 0:
                         shutil.rmtree(os.path.join(compiledir, directory))
-                        pass
 
                 except EOFError:
                     print ("ERROR while reading this key file '%s'."

theano/sparse/utils.py

@@ -2,17 +2,17 @@ from theano.gof.cc import hash_from_code
 
 
 def hash_from_sparse(data):
-    # We need to hash the shapes as hash_from_code only hash
+    # We need to hash the shapes as hash_from_code only hashes
     # the data buffer. Otherwise, this will cause problem with shapes like:
     # (1, 0) and (2, 0)
     # We also need to add the dtype to make the distinction between
     # uint32 and int32 of zeros with the same shape.
 
-    # python hash are not strong, so I always use md5. To don't have a too long
-    # hash, I call it again on the contatenation of all part.
-    return (hash_from_code(hash_from_code(data.data) +
-                           hash_from_code(data.indices) +
-                           hash_from_code(data.indptr) +
-                           hash_from_code(str(data.shape)) +
-                           hash_from_code(str(data.dtype)) +
-                           hash_from_code(data.format)))
+    # Python hash is not strong, so I always use md5. To avoid having a too
+    # long hash, I call it again on the contatenation of all parts.
+    return hash_from_code(hash_from_code(data.data) +
+                          hash_from_code(data.indices) +
+                          hash_from_code(data.indptr) +
+                          hash_from_code(str(data.shape)) +
+                          hash_from_code(str(data.dtype)) +
+                          hash_from_code(data.format))

theano/tensor/utils.py

@@ -4,7 +4,7 @@ from theano.gof.cc import hash_from_code
 
 
 def hash_from_ndarray(data):
-    # We need to hash the shapes and strides as hash_from_code only hash
+    # We need to hash the shapes and strides as hash_from_code only hashes
     # the data buffer. Otherwise, this will cause problem with shapes like:
     # (1, 0) and (2, 0) and problem with inplace transpose.
     # We also need to add the dtype to make the distinction between
@@ -14,7 +14,7 @@ def hash_from_ndarray(data):
     # too long hash, I call it again on the concatenation of all parts.
     if not data.flags["C_CONTIGUOUS"] and not data.flags["F_CONTIGUOUS"]:
         data = numpy.ascontiguousarray(data)
-    return (hash_from_code(hash_from_code(data) +
-                           hash_from_code(str(data.shape)) +
-                           hash_from_code(str(data.strides)) +
-                           hash_from_code(str(data.dtype))))
+    return hash_from_code(hash_from_code(data) +
+                          hash_from_code(str(data.shape)) +
+                          hash_from_code(str(data.strides)) +
+                          hash_from_code(str(data.dtype)))