sort the histograph of keys and code file. Add an option to print the keys that…

theano/misc/check_duplicate_key.py

-import pickle
+import cPickle
 import os, sys
 
 import theano
 
 DISPLAY_DUPLICATE_KEYS = False
+DISPLAY_MOST_FREQUENT_DUPLICATE_CCODE = False
 
 dirs = []
 if len(sys.argv)>1:
@@ -16,13 +17,13 @@ keys = {} # key -> nb seen
 mods = {}
 for dir in dirs:
 
+    key = None
     try:
         f = open(os.path.join(dir, "key.pkl"))
         key = f.read()
         f.close()
         keys.setdefault(key, 0)
         keys[key]+=1
-        del key
         del f
     except IOError:
         #print dir, "don't have a key.pkl file"
@@ -34,8 +35,8 @@ for dir in dirs:
         f = open(path)
         mod = f.read()
         f.close()
-        mods.setdefault(mod, 0)
-        mods[mod]+=1
+        mods.setdefault(mod, ())
+        mods[mod]+=(key,)
         del mod
         del f
         del path
@@ -46,27 +47,42 @@ for dir in dirs:
 if DISPLAY_DUPLICATE_KEYS:
     for k, v in keys.iteritems():
         if v > 1:
-            print "Duplicate key (%i copies): %s" % (v, pickle.loads(k))
+            print "Duplicate key (%i copies): %s" % (v, cPickle.loads(k))
 
-nbs = {} # nb seen -> now many key
+nbs_keys = {} # nb seen -> now many key
 for val in keys.values():
-    nbs.setdefault(val, 0)
-    nbs[val]+=1
+    nbs_keys.setdefault(val, 0)
+    nbs_keys[val]+=1
 
-print "key.pkl histograph"
-print nbs
-
-nbs = {} # nb seen -> now many key
+nbs_mod = {} # nb seen -> how many key
+nbs_mod_to_key = {} #nb seen -> keys
 more_then_one = 0
-for val in mods.values():
-    nbs.setdefault(val, 0)
-    nbs[val]+=1
+for mod,kk in mods.iteritems():
+    val = len(kk)
+    nbs_mod.setdefault(val, 0)
+    nbs_mod[val]+=1
     if val>1:
         more_then_one += 1
+    nbs_mod_to_key[val] = kk
+
+if DISPLAY_MOST_FREQUENT_DUPLICATE_CCODE:
+    m = max(nbs_mod.keys())
+    print "The keys associated to the mod.{cpp,cu} with the most number of copy:"
+    for kk in nbs_mod_to_key[m]:
+        kk = cPickle.loads(kk)
+        print kk
+
+print "key.pkl histograph"
+l = nbs_keys.items()
+l.sort()
+print l
 
 print "mod.{cpp,cu} histogram"
-print nbs
-total = sum(mods.values())
+l = nbs_mod.items()
+l.sort()
+print l
+
+total = sum([len(k) for k in mods.values()])
 uniq = len(mods)
 useless = total - uniq
 print "mod.{cpp,cu} total:", total