Merge pull request #2310 from abergeron/compile_time

doc/library/config.txt

@@ -166,7 +166,7 @@ import theano and print the config variable, as in:
     Theano initialize the GPU device.  Newer version of PyCUDA
     (currently only in the trunk) don't have this restriction.
 
-.. attribute:: config.print_active_device
+.. attribute:: print_active_device
 
     Bool value: either ``True`` or ``False``
 
@@ -310,26 +310,26 @@ import theano and print the config variable, as in:
     Do the vm/cvm linkers profile the optimization phase when compiling a Theano function?
     It only works when profile=True.
 
-.. attribute:: profiling.n_apply
+.. attribute:: config.profiling.n_apply
 
     Positive int value, default: 20.
 
     The number of Apply nodes to print in the profiler output
 
-.. attribute:: profiling.n_ops
+.. attribute:: config.profiling.n_ops
 
     Positive int value, default: 20.
 
     The number of Ops to print in the profiler output
 
-.. attribute:: profiling.min_memory_size
+.. attribute:: config.profiling.min_memory_size
 
     Positive int value, default: 1024.
 
     For the memory profile, do not print Apply nodes if the size
     of their outputs (in bytes) is lower than this.
 
-.. attribute:: profiling.min_peak_memory
+.. attribute:: config.profiling.min_peak_memory
 
     Bool value: either True or False
 
@@ -338,7 +338,7 @@ import theano and print the config variable, as in:
     Does the memory profile print the min peak memory usage?
     It only works when profile=True, profile_memory=True
 
-.. attribute:: profiling.destination
+.. attribute:: config.profiling.destination
 
     String value: 'stderr', 'stdout', or a name of a
     file to be created
@@ -491,7 +491,7 @@ import theano and print the config variable, as in:
     Extra parameters to pass to gcc when compiling.  Extra include paths,
     library paths, configuration options, etc.
 
-.. attribute:: config.cxx
+.. attribute:: cxx
 
 
     Default: 'g++' if g++ is present. Empty string otherwise.
@@ -555,7 +555,33 @@ import theano and print the config variable, as in:
     This means files whose compilation failed are deleted.
     Set to True to keep those files in order to debug compilation errors.
 
-.. attribute:: config.DebugMode
+.. attribute:: compile
+
+   This section contains attributes which influence the compilation of
+   C code for ops.  Due to historical reasons many attributes outside
+   of this section also have an influence over compilation, most
+   notably 'cxx'.  This is not expected to change any time soon.
+
+.. attribute:: config.compile.timeout
+
+   Positive int value, default: :attr:`compile.wait` * 24
+
+   Time to wait before an unrefreshed lock is broken and stolen.  This
+   is in place to avoid manual cleanup of locks in case a process
+   crashed and left a lock in place.
+
+   The refresh time is automatically set to half the timeout value.
+
+.. attribute:: config.compile.wait
+
+   Positive int value, default: 5
+
+   Time to wait between attempts at grabbing the lock if the first
+   attempt is not successful. The actual time will be between
+   :attr:`compile.wait` and :attr:`compile.wait` * 2 to avoid a
+   crowding effect on lock.
+
+.. attribute:: DebugMode
 
     This section contains various attributes configuring the behaviour
     of mode :class:`~debugmode.DebugMode`. See directly this section
@@ -595,7 +621,7 @@ import theano and print the config variable, as in:
     Generate a warning when the destroy_map or view_map tell that an op work
     inplace, but the op did not reuse the input for its output.
 
-.. attribute:: config.numpy
+.. attribute:: numpy
 
     This section contains different attributes for configuring numpy's
     behaviour, described by `numpy.seterr
@@ -669,7 +695,7 @@ import theano and print the config variable, as in:
 
     This flag's value cannot be modified during the program execution.
 
-.. attribute:: config.compute_test_value
+.. attribute:: compute_test_value
 
     String Value: ``'off'``, ``'ignore'``, ``'warn'``, ``'raise'``.
 
@@ -693,13 +719,13 @@ import theano and print the config variable, as in:
           this Op
         - ``'raise'`` will raise an Exception
     
-.. attribute:: config.compute_test_value_opt
+.. attribute:: compute_test_value_opt
 
    As ``compute_test_value``, but it is the value used during Theano
    optimization phase. Theano user's do not need to use this. This is
    to help debug shape error in Theano optimization.
 
-.. attribute:: config.reoptimize_unpickled_function
+.. attribute:: reoptimize_unpickled_function
 
     Bool value, default: True
 
@@ -709,7 +735,7 @@ import theano and print the config variable, as in:
     reoptimized when being unpickled. Otherwise, skip the graph optimization and 
     use directly the optimized graph. 
 
-.. attribute:: config.exception_verbosity
+.. attribute:: exception_verbosity
 
     String Value: ``'low'``, ``'high'``.
 
@@ -749,13 +775,13 @@ import theano and print the config variable, as in:
     This is useful to debug in gdb modules compiled by Theano.
     The parameter -g is passed by default to g++.
 
-.. attribute:: cmodule.compilation_warning
+.. attribute:: config.cmodule.compilation_warning
 
     Bool value, default: False
 
     If True, will print compilation warnings.
 
-.. attribute:: cmodule.preload_cache'
+.. attribute:: config.cmodule.preload_cache
 
     Bool value, default: False
 

theano/configdefaults.py

@@ -132,29 +132,38 @@ AddConfigVar('mode',
                 'FAST_COMPILE', 'PROFILE_MODE', 'DEBUG_MODE'),
         in_c_key=False)
 
-param = StrParam("g++")
+param = "g++"
 
 # Test whether or not g++ is present: disable C code if it is not.
 try:
     rc = call_subprocess_Popen(['g++', '-v'])
 except OSError:
-    param = StrParam("")
+    param = ""
     rc = 1
 
 # On Mac we test for 'clang++' and use it by default
 if sys.platform == 'darwin':
     try:
         rc = call_subprocess_Popen(['clang++', '-v'])
-        param = StrParam("clang++")
+        param = "clang++"
     except OSError:
         pass
 
+# Try to find the full compiler path from the name
+if param != "":
+    import distutils.spawn
+    newp = distutils.spawn.find_executable(param)
+    if newp is not None:
+        param = newp
+    del newp
+    del distutils
+
 AddConfigVar('cxx',
              "The C++ compiler to use. Currently only g++ is"
              " supported, but supporting additional compilers should not be "
              "too difficult. "
              "If it is empty, no C++ code is compiled.",
-             param,
+             StrParam(param),
              in_c_key=False)
 del param
 

theano/gof/compilelock.py

@@ -9,38 +9,39 @@ import time
 import logging
 
 from theano import config
+from theano.configparser import AddConfigVar, IntParam
 
 _logger = logging.getLogger("theano.gof.compilelock")
 # If the user provided a logging level, we don't want to override it.
 if _logger.level == logging.NOTSET:
-    # INFO will show the the messages "Refreshing lock" message
+    # INFO will show the "Refreshing lock" messages
     _logger.setLevel(logging.INFO)
 
-# In seconds, time that a process will wait before deciding to override an
-# existing lock. An override only happens when the existing lock is held by
-# the same owner *and* has not been 'refreshed' by this owner for more than
-# 'timeout_before_override' seconds.
-timeout_before_override = 120
+AddConfigVar('compile.wait',
+             """Time to wait before retrying to aquire the compile lock.""",
+             IntParam(5, lambda i: i > 0, allow_override=False),
+             in_c_key=False)
 
-# In seconds, duration before a lock is refreshed. More precisely, the lock is
-# refreshed each time 'get_lock()' is called (typically for each file being
-# compiled) and the existing lock has not been refreshed in the past
-# 'refresh_every' seconds.
-refresh_every = 60
+def _timeout_default():
+    return config.compile.wait * 24
+
+AddConfigVar('compile.timeout',
+             """In seconds, time that a process will wait before deciding to
+override an existing lock. An override only happens when the existing
+lock is held by the same owner *and* has not been 'refreshed' by this
+owner for more than this period. Refreshes are done every half timeout
+period for running processes.""",
+             IntParam(_timeout_default, lambda i: i >= 0,
+                      allow_override=False),
+             in_c_key=False)
 
 
 def force_unlock():
     """
     Delete the compilation lock if someone else has it.
     """
-    global timeout_before_override
-    timeout_backup = timeout_before_override
-    timeout_before_override = 0
-    try:
-        get_lock(min_wait=0, max_wait=0.001)
-        release_lock()
-    finally:
-        timeout_before_override = timeout_backup
+    get_lock(min_wait=0, max_wait=0.001, timeout=0)
+    release_lock()
 
 
 def get_lock(lock_dir=None, **kw):
@@ -74,16 +75,17 @@ def get_lock(lock_dir=None, **kw):
     if get_lock.lock_is_enabled:
         # Only really try to acquire the lock if we do not have it already.
         if get_lock.n_lock == 0:
-            lock(get_lock.lock_dir, timeout=timeout_before_override, **kw)
+            lock(get_lock.lock_dir, **kw)
             atexit.register(Unlocker.unlock, get_lock.unlocker)
             # Store time at which the lock was set.
             get_lock.start_time = time.time()
         else:
-            # Check whether we need to 'refresh' the lock. We do this every
-            # 'refresh_every' seconds to ensure noone else tries to override
-            # our lock after their 'timeout_before_override' timeout period.
+            # Check whether we need to 'refresh' the lock. We do this
+            # every 'config.compile.timeout / 2' seconds to ensure
+            # no one else tries to override our lock after their
+            # 'config.compile.timeout' timeout period.
             now = time.time()
-            if now - get_lock.start_time > refresh_every:
+            if now - get_lock.start_time > config.compile.timeout/2:
                 lockpath = os.path.join(get_lock.lock_dir, 'lock')
                 _logger.info('Refreshing lock %s', str(lockpath))
                 refresh_lock(lockpath)
@@ -109,46 +111,57 @@ def set_lock_status(use_lock):
     by default). Disabling may make compilation slightly faster (but is not
     recommended for parallel execution).
 
-    @param use_lock: whether to use the compilation lock or not
-    @type  use_lock: bool
+    :param use_lock: whether to use the compilation lock or not
+    :type  use_lock: bool
     """
     get_lock.lock_is_enabled = use_lock
 
+# This is because None is a valid input for timeout
+notset = object()
 
-def lock(tmp_dir, timeout=120, min_wait=5, max_wait=10, verbosity=1):
+def lock(tmp_dir, timeout=notset, min_wait=None, max_wait=None, verbosity=1):
     """
     Obtain lock access by creating a given temporary directory (whose base will
     be created if needed, but will not be deleted after the lock is removed).
     If access is refused by the same lock owner during more than 'timeout'
     seconds, then the current lock is overridden. If timeout is None, then no
     timeout is performed.
+
     The lock is performed by creating a 'lock' file in 'tmp_dir' that contains
     a unique id identifying the owner of the lock (the process id, followed by
     a random string).
+
     When there is already a lock, the process sleeps for a random amount of
     time between min_wait and max_wait seconds before trying again.
+
     If 'verbosity' is >= 1, then a message will be displayed when we need to
     wait for the lock. If it is set to a value >1, then this message will be
     displayed each time we re-check for the presence of the lock. Otherwise it
     is displayed only when we notice the lock's owner has changed.
 
-    @param tmp_dir: lock directory that will be created when acquiring the lock
-    @type  tmp_dir: string
+    :param str tmp_dir: lock directory that will be created when
+                        acquiring the lock
 
-    @param timeout: time (in seconds) to wait before replacing an existing lock
-    @type  timeout: int or None
+    :param timeout: time (in seconds) to wait before replacing an
+                    existing lock (default config 'compile.timeout')
+    :type  timeout: int or None
 
-    @param min_wait: minimum time (in seconds) to wait before trying again to
-                     get the lock
-    @type  min_wait: int
+    :param int min_wait: minimum time (in seconds) to wait before
+                         trying again to get the lock
+                         (default config 'compile.wait')
 
-    @param max_wait: maximum time (in seconds) to wait before trying again to
-                     get the lock
-    @type  max_wait: int
+    :param int max_wait: maximum time (in seconds) to wait before
+                         trying again to get the lock
+                         (default 2 * min_wait)
 
-    @param verbosity: amount of feedback displayed to screen
-    @type  verbosity: int
+    :param int verbosity: amount of feedback displayed to screen (default 1)
     """
+    if min_wait is None:
+        min_wait = config.compile.wait
+    if max_wait is None:
+        max_wait = min_wait * 2
+    if timeout is notset:
+        timeout = config.compile.timeout
     # Create base of lock directory if required.
     base_lock = os.path.dirname(tmp_dir)
     if not os.path.isdir(base_lock):
@@ -207,7 +220,7 @@ def lock(tmp_dir, timeout=120, min_wait=5, max_wait=10, verbosity=1):
                     continue
                 if last_owner == read_owner:
                     if (timeout is not None and
-                            time.time() - time_start >= timeout):
+                        time.time() - time_start >= timeout):
                         # Timeout exceeded or locking process dead.
                         if not no_display:
                             if read_owner == 'failure':