Move compiledir and compilelock to compile module

bin/theano_cache.py

@@ -14,7 +14,7 @@ if sys.platform == "win32":
     os.environ["THEANO_FLAGS"] = theano_flags
 
 import theano
-import theano.gof.compiledir
+import theano.compile.compiledir
 from theano import config
 from theano.link.c.basic import get_module_cache
 
@@ -81,16 +81,16 @@ def main():
                 )
                 _logger.debug(f"Remaining elements ({len(items)}): {', '.join(items)}")
         elif sys.argv[1] == "list":
-            theano.gof.compiledir.print_compiledir_content()
+            theano.compile.compiledir.print_compiledir_content()
         elif sys.argv[1] == "cleanup":
-            theano.gof.compiledir.cleanup()
+            theano.compile.compiledir.cleanup()
             cache = get_module_cache(init_args=dict(do_refresh=False))
             cache.clear_old()
         elif sys.argv[1] == "unlock":
-            theano.gof.compilelock.force_unlock()
+            theano.compile.compilelock.force_unlock()
             print("Lock successfully removed!")
         elif sys.argv[1] == "purge":
-            theano.gof.compiledir.compiledir_purge()
+            theano.compile.compiledir.compiledir_purge()
         elif sys.argv[1] == "basecompiledir":
             # Simply print the base_compiledir
             print(theano.config.base_compiledir)
@@ -98,9 +98,9 @@ def main():
             print_help(exit_status=1)
     elif len(sys.argv) == 3 and sys.argv[1] == "basecompiledir":
         if sys.argv[2] == "list":
-            theano.gof.compiledir.basecompiledir_ls()
+            theano.compile.compiledir.basecompiledir_ls()
         elif sys.argv[2] == "purge":
-            theano.gof.compiledir.basecompiledir_purge()
+            theano.compile.compiledir.basecompiledir_purge()
         else:
             print_help(exit_status=1)
     else:

doc/extending/pipeline.txt

@@ -116,7 +116,7 @@ case if ``borrow`` was True, the thunk would be allowed to reuse (or
     crashes with paralell execution of some scripts). This mechanism is
     currently enabled by default, but if it causes any problem it may be
     disabled using the function
-    ``theano.gof.compilelock.set_lock_status(..)``.
+    ``theano.compile.compilelock.set_lock_status(..)``.
 
 Step 4 - Wrap the thunk in a pretty package
 ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^

theano/gof/compiledir.py → theano/compile/compiledir.py

+"""
+This module contains housekeeping functions for cleaning/purging the "compiledir".
+It is used by the "theano-cache" CLI tool, located in the /bin folder of the repository.
+"""
 import logging
 import os
 import pickle
@@ -10,7 +14,7 @@ from theano.configdefaults import config
 from theano.utils import flatten
 
 
-_logger = logging.getLogger("theano.gof.compiledir")
+_logger = logging.getLogger("theano.compile.compiledir")
 
 
 def cleanup():

theano/gof/compilelock.py → theano/compile/compilelock.py

@@ -5,7 +5,7 @@ in the same compilation directory (which can cause crashes).
 import atexit
 import logging
 import os
-import socket  # only used for gethostname()
+import socket
 import time
 from contextlib import contextmanager
 
@@ -14,16 +14,24 @@ import numpy as np
 from theano.configdefaults import config
 
 
-random = np.random.RandomState([2015, 8, 2])
+__all__ = [
+    "force_unlock",
+    "get_lock",
+    "lock",
+    "lock_ctx",
+    "release_lock",
+    "set_lock_status",
+]
 
-_logger = logging.getLogger("theano.gof.compilelock")
+
+_random = np.random.RandomState([2015, 8, 2])
+
+_logger = logging.getLogger("theano.compile.compilelock")
 # If the user provided a logging level, we don't want to override it.
 if _logger.level == logging.NOTSET:
     # INFO will show the "Refreshing lock" messages
     _logger.setLevel(logging.INFO)
 
-hostname = socket.gethostname()
-
 
 def force_unlock():
     """
@@ -201,6 +209,7 @@ def lock(tmp_dir, timeout=notset, min_wait=None, max_wait=None, verbosity=1):
 
     # Variable initialization.
     lock_file = os.path.join(tmp_dir, "lock")
+    hostname = socket.gethostname()
     my_pid = os.getpid()
     no_display = verbosity == 0
 
@@ -276,7 +285,7 @@ def lock(tmp_dir, timeout=notset, min_wait=None, max_wait=None, verbosity=1):
                     if verbosity <= 1:
                         no_display = True
                 nb_wait += 1
-                time.sleep(random.uniform(min_wait, max_wait))
+                time.sleep(_random.uniform(min_wait, max_wait))
 
             try:
                 os.mkdir(tmp_dir)
@@ -312,7 +321,7 @@ def lock(tmp_dir, timeout=notset, min_wait=None, max_wait=None, verbosity=1):
             nb_error += 1
             if nb_error > 10:
                 raise
-            time.sleep(random.uniform(min_wait, max_wait))
+            time.sleep(_random.uniform(min_wait, max_wait))
             continue
 
 
@@ -324,8 +333,8 @@ def refresh_lock(lock_file):
     """
     unique_id = "{}_{}_{}".format(
         os.getpid(),
-        "".join([str(random.randint(0, 9)) for i in range(10)]),
-        hostname,
+        "".join([str(_random.randint(0, 9)) for i in range(10)]),
+        socket.gethostname(),
     )
     try:
         with open(lock_file, "w") as lock_write:
@@ -381,7 +390,7 @@ class Unlocker:
                 with open(lock_file) as f:
                     owner = f.readlines()[0].strip()
                     pid, _, hname = owner.split("_")
-                    if pid != str(os.getpid()) or hname != hostname:
+                    if pid != str(os.getpid()) or hname != socket.gethostname():
                         return
             except Exception:
                 pass

theano/compile/function/types.py

@@ -1361,7 +1361,7 @@ class FunctionMaker:
         # This function is not finished
         import os.path
 
-        from theano.gof.compilelock import get_lock, release_lock
+        from theano.compile.compilelock import get_lock, release_lock
 
         graph_db_file = os.path.join(config.compiledir, "optimized_graphs.pkl")
 

theano/link/c/basic.py

@@ -11,9 +11,9 @@ from io import StringIO
 
 import numpy as np
 
+from theano.compile.compilelock import get_lock, release_lock
 from theano.configdefaults import config
 from theano.gof.callcache import CallCache
-from theano.gof.compilelock import get_lock, release_lock
 from theano.gof.graph import Constant, NoParams, io_toposort
 from theano.gof.graph import variables as get_variables
 from theano.gof.utils import MethodNotDefined

theano/link/c/cmodule.py

@@ -23,10 +23,10 @@ from io import BytesIO, StringIO
 import numpy.distutils
 
 import theano
-from theano.configdefaults import config, gcc_version_str
 
 # we will abuse the lockfile mechanism when reading and writing the registry
-from theano.gof import compilelock
+from theano.compile.compilelock import lock_ctx
+from theano.configdefaults import config, gcc_version_str
 from theano.link.c.exceptions import MissingGXX
 from theano.utils import (
     LOCAL_BITWIDTH,
@@ -1015,7 +1015,7 @@ class ModuleCache:
                     self.loaded_key_pkl.remove(pkl_file_to_remove)
 
         if to_delete or to_delete_empty:
-            with compilelock.lock_ctx():
+            with lock_ctx():
                 for a, kw in to_delete:
                     _rmtree(*a, **kw)
                 for a, kw in to_delete_empty:
@@ -1055,7 +1055,7 @@ class ModuleCache:
         if module_hash in self.module_hash_to_key_data:
             key_data = self.module_hash_to_key_data[module_hash]
             module = self._get_from_key(None, key_data)
-            with compilelock.lock_ctx(keep_lock=keep_lock):
+            with lock_ctx(keep_lock=keep_lock):
                 try:
                     key_data.add_key(key, save_pkl=bool(key[0]))
                     key_broken = False
@@ -1160,7 +1160,7 @@ class ModuleCache:
         if module is not None:
             return module
 
-        with compilelock.lock_ctx(keep_lock=keep_lock):
+        with lock_ctx(keep_lock=keep_lock):
             # 1) Maybe somebody else compiled it for us while we
             #    where waiting for the lock. Try to load it again.
             # 2) If other repo that import Theano have Theano ops defined,
@@ -1247,7 +1247,7 @@ class ModuleCache:
                 # same time.  This can happen as we read the cache
                 # without taking the lock.
                 if i == 2:
-                    with compilelock.lock_ctx():
+                    with lock_ctx():
                         with open(key_pkl, "rb") as f:
                             key_data = pickle.load(f)
                 time.sleep(2)
@@ -1334,7 +1334,7 @@ class ModuleCache:
         )
         if not too_old_to_use:
             return
-        with compilelock.lock_ctx():
+        with lock_ctx():
             # Update the age of modules that have been accessed by other
             # processes and get all module that are too old to use
             # (not loaded in self.entry_from_key).
@@ -1374,7 +1374,7 @@ class ModuleCache:
             See help of refresh() method.
 
         """
-        with compilelock.lock_ctx():
+        with lock_ctx():
             self.clear_old(age_thresh_del=-1.0, delete_if_problem=delete_if_problem)
             self.clear_unversioned(min_age=unversioned_min_age)
             if clear_base_files:
@@ -1391,7 +1391,7 @@ class ModuleCache:
         next time we clear the cache.
 
         """
-        with compilelock.lock_ctx():
+        with lock_ctx():
             for base_dir in ("cutils_ext", "lazylinker_ext", "scan_perform"):
                 to_delete = os.path.join(self.dirname, base_dir + ".delete.me")
                 if os.path.isdir(to_delete):

theano/link/c/cutils.py

@@ -2,8 +2,8 @@ import errno
 import os
 import sys
 
+from theano.compile.compilelock import get_lock, release_lock
 from theano.configdefaults import config
-from theano.gof.compilelock import get_lock, release_lock
 from theano.link.c import cmodule
 
 

theano/link/c/lazylinker_c.py

@@ -6,8 +6,8 @@ import warnings
 from importlib import reload
 
 import theano
+from theano.compile.compilelock import get_lock, release_lock
 from theano.configdefaults import config
-from theano.gof.compilelock import get_lock, release_lock
 from theano.link.c.cmodule import GCC_compiler
 
 

theano/scan/scan_perform_ext.py

@@ -17,8 +17,8 @@ from importlib import reload
 import numpy as np
 
 import theano
+from theano.compile.compilelock import get_lock, release_lock
 from theano.configdefaults import config
-from theano.gof.compilelock import get_lock, release_lock
 from theano.link.c import cmodule