numpydoc for theano/gof/cmodule.py

theano/gof/cmodule.py

-"""Generate and compile C modules for Python,
 """
+Generate and compile C modules for Python.
 
+"""
 from __future__ import print_function
 
 import atexit
@@ -81,16 +82,23 @@ import_time = 0
 class MissingGXX(Exception):
     """
     This error is raised when we try to generate c code,
-    but g++ is not available
+    but g++ is not available.
+
     """
+
     pass
 
 
 def debug_counter(name, every=1):
-    """Debug counter to know how often we go through some piece of code.
+    """
+    Debug counter to know how often we go through some piece of code.
 
-    This is a utility function one may use when debugging. Usage example:
+    This is a utility function one may use when debugging.
+
+    Example
+    -------
     debug_counter('I want to know how often I run this line')
+
     """
     setattr(debug_counter, name, getattr(debug_counter, name, 0) + 1)
     n = getattr(debug_counter, name)
@@ -99,27 +107,36 @@ def debug_counter(name, every=1):
 
 
 class ExtFunction(object):
-    """A C function to put into a DynamicModule """
+    """
+    A C function to put into a DynamicModule.
+
+    """
 
     name = ""
-    """string - function's name"""
+    """
+    str - function's name.
 
+    """
     code_block = ""
-    """string - the entire code for the function.
+    """
+    str - the entire code for the function.
 
     Has the form ``static PyObject* <name>([...]){ ... }
 
     See Python's C API Reference for how to write c functions for python
     modules.
-    """
 
-    method = ""
     """
-    str - calling method for this function (i.e. 'METH_VARARGS', 'METH_NOARGS')
+    method = ""
     """
+    str - calling method for this function (i.e. 'METH_VARARGS', 'METH_NOARGS').
 
+    """
     doc = ""
-    """str - documentation string for this function"""
+    """
+    str - documentation string for this function.
+
+    """
 
     def __init__(self, name, code_block, method, doc="undocumented"):
         self.name = name
@@ -132,6 +149,7 @@ class ExtFunction(object):
         Returns the signature for this function.
 
         It goes into the DynamicModule's method table.
+
         """
         return '\t{"%s", %s, %s, "%s"}' % (
             self.name, self.name, self.method, self.doc)
@@ -244,7 +262,10 @@ static struct PyModuleDef moduledef = {{
         return rval
 
     def list_code(self, ofile=sys.stdout):
-        """Print out the code with line numbers to `ofile` """
+        """
+        Print out the code with line numbers to `ofile`.
+
+        """
         for i, line in enumerate(self.code().split('\n')):
             print(('%4i' % (i + 1)), line, file=ofile)
         ofile.flush()
@@ -253,15 +274,21 @@ static struct PyModuleDef moduledef = {{
 
 
 def dlimport(fullpath, suffix=None):
-    """Dynamically load a .so, .pyd, .dll, or .py file
+    """
+    Dynamically load a .so, .pyd, .dll, or .py file.
 
-    :type fullpath: string
-    :param fullpath: a fully-qualified path do a compiled python module
-    :param suffix: a suffix to strip from the end of fullpath to get the
-        import name
-    :type suffix: string
+    Parameters
+    ----------
+    fullpath : str
+        A fully-qualified path do a compiled python module.
+    suffix : str
+        A suffix to strip from the end of fullpath to get the
+        import name.
 
-    :returns: the dynamically loaded module (from __import__)
+    Returns
+    -------
+    object
+        The dynamically loaded module (from __import__).
 
     """
     if not os.path.isabs(fullpath):
@@ -310,7 +337,8 @@ def dlimport_workdir(basedir):
     """
     Return a directory where you should put your .so file for dlimport
     to be able to load it, given a basedir which should normally be
-    config.compiledir
+    config.compiledir.
+
     """
     return tempfile.mkdtemp(dir=basedir)
 
@@ -319,6 +347,7 @@ def last_access_time(path):
     """
     Return the number of seconds since the epoch of the last access of a
     given file.
+
     """
     return os.stat(path)[stat.ST_ATIME]
 
@@ -327,6 +356,7 @@ def module_name_from_dir(dirname, err=True, files=None):
     """
     Scan the contents of a cache directory and return full path of the
     dynamic lib in it.
+
     """
     if files is None:
         files = os.listdir(dirname)
@@ -349,6 +379,7 @@ def is_same_entry(entry_1, entry_2):
         - They are equal.
         - Their real paths are equal.
         - They share the same temporary work directory and module file name.
+
     """
     if entry_1 == entry_2:
         return True
@@ -372,6 +403,7 @@ def get_module_hash(src_code, key):
         3. The compiler options defined in `key` (command line parameters and
            libraries to link against).
         4. The NumPy ABI version.
+
     """
     # `to_hash` will contain any element such that we know for sure that if
     # it changes, then the module hash should be different.
@@ -425,6 +457,7 @@ def get_safe_part(key):
     It is used to reduce the amount of key comparisons one has to go through
     in order to find broken keys (i.e. keys with bad implementations of __eq__
     or __hash__).
+
     """
     version = key[0]
     # This function should only be called on versioned keys.
@@ -442,35 +475,43 @@ def get_safe_part(key):
 
 
 class KeyData(object):
-
-    """Used to store the key information in the cache."""
-
-    def __init__(self, keys, module_hash, key_pkl, entry):
     """
-        Constructor.
-
-        :param keys: Set of keys that are associated to the exact same module.
-
-        :param module_hash: Hash identifying the module (it should hash both
-        the code and the compilation options).
-
-        :param key_pkl: Path to the file in which this KeyData object should be
+    Used to store the key information in the cache.
+
+    Parameters
+    ----------
+    keys
+        Set of keys that are associated to the exact same module.
+    module_hash
+        Hash identifying the module (it should hash both the code and the
+        compilation options).
+    key_pkl
+        Path to the file in which this KeyData object should be
         pickled.
+
     """
+
+    def __init__(self, keys, module_hash, key_pkl, entry):
         self.keys = keys
         self.module_hash = module_hash
         self.key_pkl = key_pkl
         self.entry = entry
 
     def add_key(self, key, save_pkl=True):
-        """Add a key to self.keys, and update pickled file if asked to."""
+        """
+        Add a key to self.keys, and update pickled file if asked to.
+
+        """
         assert key not in self.keys
         self.keys.add(key)
         if save_pkl:
             self.save_pkl()
 
     def remove_key(self, key, save_pkl=True):
-        """Remove a key from self.keys, and update pickled file if asked to."""
+        """
+        Remove a key from self.keys, and update pickled file if asked to.
+
+        """
         self.keys.remove(key)
         if save_pkl:
             self.save_pkl()
@@ -481,6 +522,7 @@ class KeyData(object):
 
         May raise a cPickle.PicklingError if such an exception is raised at
         pickle time (in which case a warning is also displayed).
+
         """
         # Note that writing in binary mode is important under Windows.
         try:
@@ -493,7 +535,10 @@ class KeyData(object):
             raise
 
     def get_entry(self):
-        """Return path to the module file."""
+        """
+        Return path to the module file.
+
+        """
         # TODO This method may be removed in the future (e.g. in 0.5) since
         # its only purpose is to make sure that old KeyData objects created
         # before the 'entry' field was added are properly handled.
@@ -508,6 +553,7 @@ class KeyData(object):
         Note that broken keys will not appear in the keys field, so we also
         manually look for keys associated to the same entry, unless
         do_manual_check is False.
+
         """
         entry = self.get_entry()
         for key in self.keys:
@@ -522,7 +568,8 @@ class KeyData(object):
 
 
 class ModuleCache(object):
-    """Interface to the cache of dynamically compiled modules on disk
+    """
+    Interface to the cache of dynamically compiled modules on disk.
 
     Note that this interface does not assume exclusive use of the cache
     directory. It is built to handle the case where multiple programs are also
@@ -569,43 +616,58 @@ class ModuleCache(object):
         - They share the same C code.
     These three elements uniquely identify a module, and are summarized
     in a single "module hash".
+
+    Parameters
+    ----------
+    check_for_broken_eq
+        A bad __eq__ implementation can break this cache mechanism.
+        This option turns on a not-too-expensive sanity check every
+        time a new key is added to the cache.
+
+    do_refresh : bool
+        If True, then the ``refresh`` method will be called
+        in the constructor.
+
     """
 
     dirname = ""
-    """The working directory that is managed by this interface"""
+    """
+    The working directory that is managed by this interface.
 
+    """
     module_from_name = {}
-    """maps a module filename to the loaded module object"""
+    """
+    Maps a module filename to the loaded module object.
 
+    """
     entry_from_key = {}
-    """Maps keys to the filename of a .so/.pyd.
     """
+    Maps keys to the filename of a .so/.pyd.
 
+    """
     similar_keys = {}
-    """Maps a part-of-key to all keys that share this same part."""
+    """
+    Maps a part-of-key to all keys that share this same part.
 
+    """
     module_hash_to_key_data = {}
-    """Maps a module hash to its corresponding KeyData object."""
+    """
+    Maps a module hash to its corresponding KeyData object.
 
+    """
     stats = []
     """
     A list with counters for the number of hits, loads, compiles issued by
-    module_from_key()
-    """
+    module_from_key().
 
+    """
     loaded_key_pkl = set()
-    """set of all key.pkl files that have been loaded.
     """
+    Set of all key.pkl files that have been loaded.
 
-    def __init__(self, dirname, check_for_broken_eq=True, do_refresh=True):
     """
-        :param check_for_broken_eq: A bad __eq__ implementation can break this
-        cache mechanism. This option turns on a not-too-expensive sanity check
-        every time a new key is added to the cache.
 
-        :param do_refresh: If True, then the ``refresh`` method will be called
-        in the constructor.
-        """
+    def __init__(self, dirname, check_for_broken_eq=True, do_refresh=True):
         self.dirname = dirname
         self.module_from_name = dict(self.module_from_name)
         self.entry_from_key = dict(self.entry_from_key)
@@ -624,11 +686,13 @@ class ModuleCache(object):
     The default age threshold (in seconds) for cache files we want to use.
 
     Older modules will be deleted in ``clear_old``.
+
     """
 
     def _get_module(self, name):
         """
         Fetch a compiled module from the loaded cache or the disk.
+
         """
         if name not in self.module_from_name:
             _logger.debug('loading name %s', name)
@@ -641,25 +705,31 @@ class ModuleCache(object):
 
     def refresh(self, age_thresh_use=None, delete_if_problem=False,
                 cleanup=True):
-        """Update cache data by walking the cache directory structure.
+        """
+        Update cache data by walking the cache directory structure.
 
         Load key.pkl files that have not been loaded yet.
         Remove entries which have been removed from the filesystem.
         Also, remove malformed cache directories.
 
-        :param age_thresh_use: Do not use modules olther than this.
-        Defaults to self.age_thresh_use.
-
-        :param delete_if_problem: If True, cache entries that meet one
-        of those two conditions are deleted:
+        Parameters
+        ----------
+        age_thresh_use
+            Do not use modules other than this. Defaults to self.age_thresh_use.
+        delete_if_problem : bool
+            If True, cache entries that meet one of those two conditions are
+            deleted:
             - Those for which unpickling the KeyData file fails with
               an unknown exception.
             - Duplicated modules, regardless of their age.
+        cleanup : bool
+            Do a cleanup of the cache removing expired and broken modules.
 
-        :param cleanup: Do a cleanup of the cache removing expired and
-        broken modules.
+        Returns
+        -------
+        list
+            A list of modules of age higher than age_thresh_use.
 
-        :returns: a list of modules of age higher than age_thresh_use.
         """
         if age_thresh_use is None:
             age_thresh_use = self.age_thresh_use
@@ -935,6 +1005,7 @@ class ModuleCache(object):
         and None otherwise.
 
         May raise ValueError if the key is malformed.
+
         """
         name = None
         if key is not None:
@@ -993,6 +1064,7 @@ class ModuleCache(object):
     def _add_to_cache(self, module, key, module_hash):
         """
         This function expects the compile lock to be held.
+
         """
         name = module.__file__
         _logger.debug("Adding module to cache %s %s",
@@ -1036,18 +1108,19 @@ class ModuleCache(object):
         """
         Return a module from the cache, compiling it if necessary.
 
-        :param key: The key object associated with the module. If this
-                    hits a match, we avoid compilation.
-
-        :param lnk: Usually a CLinker instance, but it can be any
-                    object that defines the `get_src_code()` and
-                    `compile_cmodule(location)` functions. The first
-                    one returns the source code of the module to
-                    load/compile and the second performs the actual
-                    compilation.
+        Parameters
+        ----------
+        key
+            The key object associated with the module. If this hits a match,
+            we avoid compilation.
+        lnk
+            Usually a CLinker instance, but it can be any object that defines
+            the `get_src_code()` and `compile_cmodule(location)` functions. The
+            first one returns the source code of the module to load/compile and
+            the second performs the actual compilation.
+        keep_lock : bool
+            If True, the compilation lock will not be released if taken.
 
-        :param keep_lock: If True, the compilation lock will not be
-                          released if taken.
         """
         # Is the module in the cache?
         module = self._get_from_key(key)
@@ -1123,8 +1196,13 @@ class ModuleCache(object):
         """
         Perform checks to detect broken __eq__ / __hash__ implementations.
 
-        :param key: The key to be checked.
-        :param key_pkl: Its associated pickled file containing a KeyData.
+        Parameters
+        ----------
+        key
+            The key to be checked.
+        key_pkl
+            Its associated pickled file containing a KeyData.
+
         """
         start_time = time.time()
         # Verify that when we reload the KeyData from the pickled file, the
@@ -1177,18 +1255,24 @@ class ModuleCache(object):
 
     age_thresh_del = 60 * 60 * 24 * 31  # 31 days
     age_thresh_del_unversioned = 60 * 60 * 24 * 7  # 7 days
+    """
+    The default age threshold for `clear_old` (in seconds).
 
-    """The default age threshold for `clear_old` (in seconds)
     """
+
     def clear_old(self, age_thresh_del=None, delete_if_problem=False):
         """
         Delete entries from the filesystem for cache entries that are too old.
 
-        :param age_thresh_del: Dynamic modules whose last access time is more
-        than ``age_thresh_del`` seconds ago will be erased. Defaults to 31-day
-        age if not provided.
+        Parameters
+        ----------
+        age_thresh_del
+            Dynamic modules whose last access time is more than
+            ``age_thresh_del`` seconds ago will be erased.
+            Defaults to 31-day age if not provided.
+        delete_if_problem
+            See help of refresh() method.
 
-        :param delete_if_problem: See help of refresh() method.
         """
         if age_thresh_del is None:
             age_thresh_del = self.age_thresh_del
@@ -1232,16 +1316,19 @@ class ModuleCache(object):
         """
         Clear all elements in the cache.
 
-        :param unversioned_min_age: Forwarded to `clear_unversioned`. In
-        particular, you can set it to -1 in order to delete all unversioned
-        cached modules regardless of their age.
-
-        :param clear_base_files: If True, then delete base directories
-        'cuda_ndarray', 'cutils_ext', 'lazylinker_ext' and 'scan_perform'
-        if they are present.
+        Parameters
+        ----------
+        unversioned_min_age
+            Forwarded to `clear_unversioned`. In particular, you can set it
+            to -1 in order to delete all unversioned cached modules regardless
+            of their age.
+        clear_base_files : bool
+            If True, then delete base directories 'cuda_ndarray', 'cutils_ext',
+            'lazylinker_ext' and 'scan_perform' if they are present.
             If False, those directories are left intact.
+        delete_if_problem
+            See help of refresh() method.
 
-        :param delete_if_problem: See help of refresh() method.
         """
         with compilelock.lock_ctx():
             self.clear_old(
@@ -1260,6 +1347,7 @@ class ModuleCache(object):
         some systems due to these modules being currently in use. Instead we
         rename them with the '.delete.me' extension, to mark them to be deleted
         next time we clear the cache.
+
         """
         with compilelock.lock_ctx():
             for base_dir in ('cuda_ndarray', 'cutils_ext', 'lazylinker_ext',
@@ -1287,8 +1375,12 @@ class ModuleCache(object):
         They are deleted both from the internal dictionaries and from the
         filesystem.
 
-        :param min_age: Minimum age to be deleted, in seconds. Defaults to
+        Parameters
+        ----------
+        min_age
+            Minimum age to be deleted, in seconds. Defaults to
             7-day age if not provided.
+
         """
         if min_age is None:
             min_age = self.age_thresh_del_unversioned
@@ -1409,8 +1501,13 @@ _module_cache = None
 
 def get_module_cache(dirname, init_args=None):
     """
-    :param init_args: If not None, the (k, v) pairs in this dictionary will
-    be forwarded to the ModuleCache constructor as keyword arguments.
+
+    Parameters
+    ----------
+    init_args
+        If not None, the (k, v) pairs in this dictionary will be forwarded to
+        the ModuleCache constructor as keyword arguments.
+
     """
     global _module_cache
     if init_args is None:
@@ -1429,7 +1526,10 @@ def get_module_cache(dirname, init_args=None):
 
 
 def get_lib_extension():
-    """Return the platform-dependent extension for compiled modules."""
+    """
+    Return the platform-dependent extension for compiled modules.
+
+    """
     if sys.platform in ['win32', 'cygwin']:
         return 'pyd'
     else:
@@ -1437,7 +1537,10 @@ def get_lib_extension():
 
 
 def get_gcc_shared_library_arg():
-    """Return the platform-dependent GCC argument for shared libraries."""
+    """
+    Return the platform-dependent GCC argument for shared libraries.
+
+    """
     if sys.platform == 'darwin':
         return '-dynamiclib'
     else:
@@ -1534,9 +1637,11 @@ def gcc_version():
 
 
 def gcc_llvm():
-    """ Detect if the g++ version used is the llvm one or not.
+    """
+    Detect if the g++ version used is the llvm one or not.
 
     It don't support all g++ parameters even if it support many of them.
+
     """
     if gcc_llvm.is_llvm is None:
         try:
@@ -1558,12 +1663,15 @@ gcc_llvm.is_llvm = None
 
 class Compiler(object):
     """
-    Meta compiler that offer some generic function
+    Meta compiler that offer some generic function.
+
     """
+
     @staticmethod
     def _try_compile_tmp(src_code, tmp_prefix='', flags=(),
                          try_run=False, output=False, compiler=None):
-        """Try to compile (and run) a test program.
+        """
+        Try to compile (and run) a test program.
 
         This is useful in various occasions, to check if libraries
         or compilers are behaving as expected.
@@ -1574,6 +1682,7 @@ class Compiler(object):
         If try_run is False, returns the compilation status.
         If try_run is True, returns a (compile_status, run_status) pair.
         If output is there, we append the stdout and stderr to the output.
+
         """
         if not compiler:
             return False
@@ -1631,12 +1740,13 @@ class Compiler(object):
     @staticmethod
     def _try_flags(flag_list, preambule="", body="",
                    try_run=False, output=False, compiler=None):
-        '''
+        """
         Try to compile a dummy file with these flags.
 
         Returns True if compilation was successful, False if there
         were errors.
-        '''
+
+        """
         if not compiler:
             return False
 
@@ -1933,33 +2043,38 @@ class GCC_compiler(Compiler):
                     include_dirs=None, lib_dirs=None, libs=None,
                     preargs=None, py_module=True, hide_symbols=True):
         """
-        :param module_name: string (this has been embedded in the src_code
-
-        :param src_code: a complete c or c++ source listing for the module
-
-        :param location: a pre-existing filesystem directory where the
-        cpp file and .so will be written
-
-        :param include_dirs: a list of include directory names (each
-        gets prefixed with -I)
-
-        :param lib_dirs: a list of library search path directory names
-        (each gets prefixed with -L)
-
-        :param libs: a list of libraries to link with (each gets
-        prefixed with -l)
-
-        :param preargs: a list of extra compiler arguments
-
-        :param py_module: if False, compile to a shared library, but do not
-            import it as a Python module.
 
-        :param hide_symbols: if True (the default) all symbols will be
-        hidden from the library symbol table (which means that other
-        objects can't use them.
+        Parameters
+        ----------
+        module_name : str
+            This has been embedded in the src_code.
+        src_code
+            A complete c or c++ source listing for the module.
+        location
+            A pre-existing filesystem directory where the cpp file and .so will
+            be written.
+        include_dirs
+            A list of include directory names (each gets prefixed with -I).
+        lib_dirs
+            A list of library search path directory names (each gets prefixed
+            with -L).
+        libs
+            A list of libraries to link with (each gets prefixed with -l).
+        preargs
+            A list of extra compiler arguments.
+        py_module
+            If False, compile to a shared library, but do not import it as a
+            Python module.
+        hide_symbols
+            If True (the default) all symbols will be hidden from the library
+            symbol table (which means that other objects can't use them).
+
+        Returns
+        -------
+        object
+            Dynamically-imported python module of the compiled code (unless
+            py_module is False, in that case returns None).
 
-        :returns: dynamically-imported python module of the compiled code.
-            (unless py_module is False, in that case returns None.)
         """
         # TODO: Do not do the dlimport in this function