Merge pull request #644 from nouiz/cache_key

bin/theano-cache

@@ -10,10 +10,9 @@ from theano.gof.cc import get_module_cache
 _logger = logging.getLogger('theano.bin.theano-cache')
 _logger.setLevel(logging.WARN)
 
-
 if len(sys.argv) == 1:
     print config.compiledir
-elif sys.argv[1] in ('clear'):
+elif sys.argv[1] == 'clear':
     # We skip the refresh on module cache creation because the refresh will
     # be done when calling clear afterwards.
     cache = get_module_cache(init_args=dict(do_refresh=False))
@@ -29,9 +28,9 @@ elif sys.argv[1] in ('clear'):
                         config.compiledir)
         _logger.debug('Remaining elements (%s): %s' %
                       (len(items), ', '.join(items)))
-elif sys.argv[1] in ('list'):
+elif sys.argv[1] == 'list':
     theano.gof.compiledir.print_compiledir_content()
-elif sys.argv[1] in ('cleanup'):
+elif sys.argv[1] == 'cleanup':
     theano.gof.compiledir.cleanup()
 elif sys.argv[1] == 'unlock':
     theano.gof.compilelock.force_unlock()

doc/library/compile/function.txt

@@ -81,7 +81,7 @@ Reference
         Initialize object attributes.
 
 
-.. function:: function(inputs, outputs, mode=None, updates=None, givens=None, accept_inplace=False, name=None)
+.. function:: function(inputs, outputs, mode=None, updates=None, givens=None, no_default_updates=False, accept_inplace=False, name=None, rebuild_strict=True, allow_input_downcast=None, profile=None, on_unused_input='raise')
 
     Return a callable object that will calculate `outputs` from `inputs`.
 
@@ -121,6 +121,30 @@ Reference
     :param name: an optional name for this function.
       The profile mode will print the time spent in this function.
 
+    :param rebuild_strict: True (Default) is the safer and better tested setting, in which case
+    `givens` must substitute new variables with the same Type as the variables they replace.
+    False is a you-better-know-what-you-are-doing setting, that permits `givens` to replace
+    variables with new variables of any Type.  The consequence of changing a Type is that all
+    results depending on that variable may have a different Type too (the graph is rebuilt from
+    inputs to outputs).  If one of the new types does not make sense for one of the Ops in the
+    graph, an Exception will be raised.
+
+    :type allow_input_downcast: Boolean or None
+    :param allow_input_downcast: True means that the values passed as
+    inputs when calling the function can be silently downcasted to fit
+    the dtype of the corresponding Variable, which may lose precision.
+    False means that it will only be cast to a more general, or
+    precise, type. None (default) is almost like False, but allows
+    downcasting of Python float scalars to floatX.
+
+    :type profile: None, True, or ProfileStats instance
+    :param profile: accumulate profiling information into a given ProfileStats
+    instance. If argument is `True` then a new ProfileStats instance will be
+    used.  This profiling object will be available via self.profile.
+
+    :param on_unused_input: What to do if a variable in the 'inputs' list is
+    not used in the graph. Possible values are 'raise', 'warn', and 'ignore'.
+
     :rtype: Function instance
 
     :returns: a callable object that will compute the outputs (given the inputs)

theano/compile/function.py

@@ -44,10 +44,6 @@ def function(inputs, outputs=None, mode=None, updates=None, givens=None,
 
     :param name: an optional name for this function. The profile mode will print the time spent in this function.
 
-    :rtype: Function instance
-    :returns: a callable object that will compute the outputs (given the inputs)
-    and update the implicit function arguments according to the `updates`.
-
     :param rebuild_strict: True (Default) is the safer and better tested setting, in which case
     `givens` must substitute new variables with the same Type as the variables they replace.
     False is a you-better-know-what-you-are-doing setting, that permits `givens` to replace
@@ -72,6 +68,10 @@ def function(inputs, outputs=None, mode=None, updates=None, givens=None,
     :param on_unused_input: What to do if a variable in the 'inputs' list is
     not used in the graph. Possible values are 'raise', 'warn', 'ignore' and None.
 
+    :rtype: Function instance
+    :returns: a callable object that will compute the outputs (given the inputs)
+    and update the implicit function arguments according to the `updates`.
+
     :note: Regarding givens: Be careful to make sure that these substitutions are
     independent--behaviour when Var1 of one pair appears in the graph leading to Var2 in
     another expression is undefined.  Replacements specified with givens are different from

theano/compile/function_module.py

@@ -1106,8 +1106,8 @@ class FunctionMaker(object):
                 blockers=[i.variable for i in inputs])
 
         msg = ("theano.function was asked to create a function computing "
-                "outputs given certain inputs, but one of the provided "
-                "input variables is not part of the computational graph "
+                "outputs given certain inputs, but the provided input "
+                "variable at index %i is not part of the computational graph "
                 "needed to compute the outputs: %s.\n%s")
         warn_msg = ("To make this warning into an error, you can pass the "
                 "parameter on_unused_input='raise' to theano.function. "
@@ -1119,9 +1119,9 @@ class FunctionMaker(object):
         for i in inputs:
             if ((i.variable not in used_inputs) and (i.update is None)):
                 if on_unused_input == 'warn':
-                    warnings.warn(msg % (i.variable, warn_msg), stacklevel=6)
+                    warnings.warn(msg % (inputs.index(i), i.variable, warn_msg), stacklevel=6)
                 elif on_unused_input == 'raise':
-                    raise UnusedInputError(msg % (i.variable, err_msg))
+                    raise UnusedInputError(msg % (inputs.index(i), i.variable, err_msg))
                 else:
                     raise ValueError(("Invalid value for keyword "
                         "on_unused_input of theano.function: '%s'. "

theano/gof/cc.py

@@ -11,6 +11,9 @@ import sys
 from itertools import izip
 
 
+import numpy
+
+
 if sys.version_info[:2] >= (2, 5):
     import hashlib
 
@@ -918,7 +921,7 @@ class CLinker(link.Linker):
         The signature has the following form:
         {{{
             'CLinker.cmodule_key', compilation args, libraries,
-            header_dirs, config md5,
+            header_dirs, numpy ABI version, config md5,
             (op0, input_signature0, output_signature0),
             (op1, input_signature1, output_signature1),
             ...
@@ -986,11 +989,12 @@ class CLinker(link.Linker):
                           compile_args=self.compile_args(),
                           libraries=self.libraries(),
                           header_dirs=self.header_dirs(),
+                          c_compiler=self.c_compiler(),
                           )
 
     @staticmethod
     def cmodule_key_(env, no_recycling, compile_args=None, libraries=None,
-                     header_dirs=None, insert_config_md5=True):
+                     header_dirs=None, insert_config_md5=True, c_compiler=None):
         """
         Do the actual computation of cmodule_key in a static method
         to allow it to be reused in scalar.Composite.__eq__
@@ -1032,6 +1036,13 @@ class CLinker(link.Linker):
             args = tuple(args)
             sig.append(args)
 
+        #We must always add the numpy ABI version here as
+        # DynamicModule always add the include <numpy/arrayobject.h>
+        sig.append('NPY_ABI_VERSION=0x%X' %
+                   numpy.core.multiarray._get_ndarray_c_version())
+        if c_compiler:
+            sig.append('c_compiler_str=' + c_compiler.version_str())
+
         # IMPORTANT: The 'md5' prefix is used to isolate the compilation
         # parameters from the rest of the key. If you want to add more key
         # elements, they should be before this md5 hash if and only if they

theano/gof/cmodule.py

@@ -25,6 +25,7 @@ from theano.gof.cc import hash_from_code
 
 # we will abuse the lockfile mechanism when reading and writing the registry
 import compilelock
+from compiledir import gcc_version_str
 
 from theano.configparser import AddConfigVar, BoolParam
 
@@ -314,6 +315,7 @@ def get_module_hash(src_code, key):
         2. The version part of the key.
         3. The compiler options defined in `key` (command line parameters and
            libraries to link against).
+        4. The NumPy ABI version.
     """
     # `to_hash` will contain any element such that we know for sure that if
     # it changes, then the module hash should be different.
@@ -347,6 +349,9 @@ def get_module_hash(src_code, key):
                 # This is the md5 hash of the config options. We can stop
                 # here.
                 break
+            elif (key_element.startswith('NPY_ABI_VERSION=0x') or
+                  key_element.startswith('c_compiler_str=')):
+                to_hash.append(key_element)
             else:
                 raise AssertionError(error_msg)
         else:
@@ -1403,30 +1408,15 @@ def std_lib_dirs():
     return std_lib_dirs_and_libs()[1]
 
 
-# Using the dummy file descriptors below is a workaround for a crash
-# experienced in an unusual Python 2.4.4 Windows environment with the default
-# None values.
-dummy_in = open(os.devnull)
-dummy_err = open(os.devnull, 'w')
-p = None
-try:
-    p = subprocess.Popen(['g++', '-dumpversion'], stdout=subprocess.PIPE,
-                         stdin=dummy_in.fileno(), stderr=dummy_err.fileno())
-    p.wait()
-    gcc_version_str = p.stdout.readline().strip()
-except OSError:
-    # Typically means gcc cannot be found.
-    gcc_version_str = 'GCC_NOT_FOUND'
-del p
-del dummy_in
-del dummy_err
-
-
 def gcc_version():
     return gcc_version_str
 
 
 class GCC_compiler(object):
+    @staticmethod
+    def version_str():
+        return "g++ " + gcc_version_str
+
     @staticmethod
     def compile_args():
         cxxflags = [flag for flag in config.gcc.cxxflags.split(' ') if flag]

theano/gof/compiledir.py

@@ -3,6 +3,7 @@ import errno
 import os
 import platform
 import re
+import subprocess
 import shutil
 import sys
 import textwrap
@@ -13,11 +14,30 @@ import theano
 from theano.configparser import config, AddConfigVar, ConfigParam, StrParam
 from theano.gof.utils import flatten
 
+# Using the dummy file descriptors below is a workaround for a crash
+# experienced in an unusual Python 2.4.4 Windows environment with the default
+# None values.
+dummy_in = open(os.devnull)
+dummy_err = open(os.devnull, 'w')
+p = None
+try:
+    p = subprocess.Popen(['g++', '-dumpversion'], stdout=subprocess.PIPE,
+                         stdin=dummy_in.fileno(), stderr=dummy_err.fileno())
+    p.wait()
+    gcc_version_str = p.stdout.readline().strip()
+except OSError:
+    # Typically means gcc cannot be found.
+    gcc_version_str = 'GCC_NOT_FOUND'
+del p
+del dummy_in
+del dummy_err
+
 compiledir_format_dict = {"platform": platform.platform(),
                           "processor": platform.processor(),
                           "python_version": platform.python_version(),
                           "theano_version": theano.__version__,
                           "numpy_version": numpy.__version__,
+                          "g++": gcc_version_str.replace(" ", "_"),
                          }
 compiledir_format_keys = ", ".join(compiledir_format_dict.keys())
 default_compiledir_format =\
@@ -115,8 +135,11 @@ def cleanup():
     """
     Delete keys in old format from the compiledir.
 
-    We define keys in old format as keys that have an ndarray in them.
-    Now we use a hash in the keys of the constant data.
+    Old clean up include key in old format:
+    1) keys that have an ndarray in them.
+       Now we use a hash in the keys of the constant data.
+    2) key that don't have the numpy ABI version in them
+    3) They do not have a compile version string
 
     If there is no key left for a compiled module, we delete the module.
     """
@@ -131,10 +154,20 @@ def cleanup():
                 try:
                     keydata = cPickle.load(file)
                     for key in list(keydata.keys):
+                        have_npy_abi_version = False
+                        have_c_compiler = False
                         for obj in flatten(key):
                             if isinstance(obj, numpy.ndarray):
                                 keydata.remove_key(key)
                                 break
+                            elif isinstance(obj, basestring):
+                                if obj.startswith('NPY_ABI_VERSION=0x'):
+                                    have_npy_abi_version = True
+                                elif obj.startswith('c_compiler_str='):
+                                    have_c_compiler = True
+
+                        if not have_npy_abi_version or not have_c_compiler:
+                            keydata.remove_key(key)
                     if len(keydata.keys) == 0:
                         shutil.rmtree(os.path.join(compiledir, directory))