Merge pull request #221 from michaelosthege/clean-config-importing

tests/gof/test_cmodule.py

@@ -5,11 +5,13 @@ But this one tests a current behavior that isn't good: the c_code isn't
 deterministic based on the input type and the op.
 """
 
+import logging
+from unittest.mock import patch
 
 import numpy as np
 
 import theano
-from theano.gof.cmodule import GCC_compiler
+from theano.gof.cmodule import GCC_compiler, default_blas_ldflags
 
 
 class MyOp(theano.compile.ops.DeepCopyOp):
@@ -68,3 +70,16 @@ def test_flag_detection():
     # but was not detected because that path is not usually taken,
     # so we test it here directly.
     GCC_compiler.try_flags(["-lblas"])
+
+
+@patch("theano.gof.cmodule.try_blas_flag", return_value=None)
+@patch("theano.gof.cmodule.sys")
+def test_default_blas_ldflags(sys_mock, try_blas_flag_mock, caplog):
+
+    sys_mock.version = "3.8.0 | packaged by conda-forge | (default, Nov 22 2019, 19:11:38) \n[GCC 7.3.0]"
+
+    with patch.dict("sys.modules", {"mkl": None}):
+        with caplog.at_level(logging.WARNING):
+            default_blas_ldflags()
+
+    assert "install mkl with" in caplog.text

tests/gof/test_compiledir.py

-from theano.configdefaults import short_platform
-
-
-def test_short_platform():
-    for r, p, a in [  # (release, platform, answer)
-        (
-            "3.2.0-70-generic",
-            "Linux-3.2.0-70-generic-x86_64-with-debian-wheezy-sid",
-            "Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
-        ),
-        (
-            "3.2.0-70.1-generic",
-            "Linux-3.2.0-70.1-generic-x86_64-with-debian-wheezy-sid",
-            "Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
-        ),
-        (
-            "3.2.0-70.1.2-generic",
-            "Linux-3.2.0-70.1.2-generic-x86_64-with-debian-wheezy-sid",
-            "Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
-        ),
-        (
-            "2.6.35.14-106.fc14.x86_64",
-            "Linux-2.6.35.14-106.fc14.x86_64-x86_64-with-fedora-14-Laughlin",
-            "Linux-2.6-fc14.x86_64-x86_64-with-fedora-14-Laughlin",
-        ),
-    ]:
-        o = short_platform(r, p)
-        assert o == a, (o, a)

tests/tensor/test_mpi.py

@@ -68,8 +68,8 @@ def test_mpi_roundtrip():
     )
     (stdout, stderr) = p.communicate()
 
-    result = theano.utils.decode(stdout)
-    assert "True" in result, theano.utils.decode(stderr)
+    result = stdout.decode()
+    assert "True" in result, stderr.decode()
 
 
 def test_mpi_send_wait_cmp():

tests/test_config.py

 """Test config options."""
 import configparser as stdlib_configparser
-import logging
-from unittest.mock import patch
 
 import pytest
 
 from theano import configdefaults, configparser
-from theano.configdefaults import default_blas_ldflags
+from theano.configdefaults import short_platform
 from theano.configparser import ConfigParam
 
 
@@ -98,19 +96,6 @@ def test_invalid_default():
     assert "test__test_invalid_default_b" not in root._flags_dict
 
 
-@patch("theano.configdefaults.try_blas_flag", return_value=None)
-@patch("theano.configdefaults.sys")
-def test_default_blas_ldflags(sys_mock, try_blas_flag_mock, caplog):
-
-    sys_mock.version = "3.8.0 | packaged by conda-forge | (default, Nov 22 2019, 19:11:38) \n[GCC 7.3.0]"
-
-    with patch.dict("sys.modules", {"mkl": None}):
-        with caplog.at_level(logging.WARNING):
-            default_blas_ldflags()
-
-    assert "install mkl with" in caplog.text
-
-
 def test_config_param_apply_and_validation():
     cp = ConfigParam(
         "TheDeFauLt",
@@ -269,3 +254,34 @@ def test_mode_apply():
         configdefaults.filter_mode(theano.compile.mode.FAST_COMPILE)
         == theano.compile.mode.FAST_COMPILE
     )
+
+
+class TestConfigHelperFunctions:
+    @pytest.mark.parametrize(
+        "release,platform,answer",
+        [
+            (
+                "3.2.0-70-generic",
+                "Linux-3.2.0-70-generic-x86_64-with-debian-wheezy-sid",
+                "Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
+            ),
+            (
+                "3.2.0-70.1-generic",
+                "Linux-3.2.0-70.1-generic-x86_64-with-debian-wheezy-sid",
+                "Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
+            ),
+            (
+                "3.2.0-70.1.2-generic",
+                "Linux-3.2.0-70.1.2-generic-x86_64-with-debian-wheezy-sid",
+                "Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
+            ),
+            (
+                "2.6.35.14-106.fc14.x86_64",
+                "Linux-2.6.35.14-106.fc14.x86_64-x86_64-with-fedora-14-Laughlin",
+                "Linux-2.6-fc14.x86_64-x86_64-with-fedora-14-Laughlin",
+            ),
+        ],
+    )
+    def test_short_platform(self, release, platform, answer):
+        o = short_platform(release, platform)
+        assert o == answer, (o, answer)

theano/__init__.py

@@ -199,3 +199,8 @@ def sparse_grad(var):
 
 import theano.tensor.random.var
 from theano.scan import checkpoints, clone, foldl, foldr, map, reduce, scan
+
+
+# Some config variables are registered by submodules. Only after all those imports
+# were executed, we can warn about remaining flags provided by the user through THEANO_FLAGS.
+config.warn_unused_flags()

theano/configparser.py

@@ -262,6 +262,10 @@ class TheanoConfigParser:
         """
         return _ChangeFlagsDecorator(*args, _root=self, **kwargs)
 
+    def warn_unused_flags(self):
+        for key in self._flags_dict.keys():
+            warnings.warn(f"Theano does not recognise this flag: {key}")
+
 
 class ConfigParam:
     """Base class of all kinds of configuration parameters.

theano/misc/elemwise_openmp_speedup.py

@@ -5,7 +5,6 @@ from locale import getpreferredencoding
 from optparse import OptionParser
 
 import theano
-from theano.utils import decode_with
 
 
 console_encoding = getpreferredencoding()
@@ -37,7 +36,7 @@ def runScript(N):
     if err:
         print(err)
         sys.exit()
-    return list(map(float, decode_with(out, console_encoding).split(" ")))
+    return list(map(float, out.decode(console_encoding).split(" ")))
 
 
 if __name__ == "__main__":

theano/misc/windows.py

-import os
-import subprocess
-
-
-def subprocess_Popen(command, **params):
-    """
-    Utility function to work around windows behavior that open windows.
-
-    :see: call_subprocess_Popen and output_subprocess_Popen
-    """
-    startupinfo = None
-    if os.name == "nt":
-        startupinfo = subprocess.STARTUPINFO()
-        try:
-            startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
-        except AttributeError:
-            startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
-
-        # Anaconda for Windows does not always provide .exe files
-        # in the PATH, they also have .bat files that call the corresponding
-        # executable. For instance, "g++.bat" is in the PATH, not "g++.exe"
-        # Unless "shell=True", "g++.bat" is not executed when trying to
-        # execute "g++" without extensions.
-        # (Executing "g++.bat" explicitly would also work.)
-        params["shell"] = True
-        # "If shell is True, it is recommended to pass args as a string rather than as a sequence." (cite taken from https://docs.python.org/2/library/subprocess.html#frequently-used-arguments)
-        # In case when command arguments have spaces, passing a command as a list will result in incorrect arguments break down, and consequently
-        # in "The filename, directory name, or volume label syntax is incorrect" error message.
-        # Passing the command as a single string solves this problem.
-        if isinstance(command, list):
-            command = " ".join(command)
-
-    # Using the dummy file descriptors below is a workaround for a
-    # crash experienced in an unusual Python 2.4.4 Windows environment
-    # with the default None values.
-    stdin = None
-    if "stdin" not in params:
-        stdin = open(os.devnull)
-        params["stdin"] = stdin.fileno()
-
-    try:
-        proc = subprocess.Popen(command, startupinfo=startupinfo, **params)
-    finally:
-        if stdin is not None:
-            stdin.close()
-    return proc
-
-
-def call_subprocess_Popen(command, **params):
-    """
-    Calls subprocess_Popen and discards the output, returning only the
-    exit code.
-    """
-    if "stdout" in params or "stderr" in params:
-        raise TypeError("don't use stderr or stdout with call_subprocess_Popen")
-    with open(os.devnull, "wb") as null:
-        # stdin to devnull is a workaround for a crash in a weird Windows
-        # environment where sys.stdin was None
-        params.setdefault("stdin", null)
-        params["stdout"] = null
-        params["stderr"] = null
-        p = subprocess_Popen(command, **params)
-        returncode = p.wait()
-    return returncode
-
-
-def output_subprocess_Popen(command, **params):
-    """
-    Calls subprocess_Popen, returning the output, error and exit code
-    in a tuple.
-    """
-    if "stdout" in params or "stderr" in params:
-        raise TypeError("don't use stderr or stdout with output_subprocess_Popen")
-    params["stdout"] = subprocess.PIPE
-    params["stderr"] = subprocess.PIPE
-    p = subprocess_Popen(command, **params)
-    # we need to use communicate to make sure we don't deadlock around
-    # the stdout/stderr pipe.
-    out = p.communicate()
-    return out + (p.returncode,)

theano/scan/__init__.py

@@ -47,6 +47,11 @@ __authors__ = (
 __copyright__ = "(c) 2010, Universite de Montreal"
 __contact__ = "Razvan Pascanu <r.pascanu@gmail>"
 
+from theano import configdefaults
+
+
+configdefaults.add_scan_configvars()
+
 from theano.scan import opt
 from theano.scan.basic import scan
 from theano.scan.checkpoints import scan_checkpoints

theano/utils.py

-"""Utility functions for Theano."""
+"""Utility functions that only depend on the standard library."""
 
 
 import inspect
+import os
+import subprocess
 import traceback
 import warnings
 from collections import OrderedDict
@@ -11,19 +13,22 @@ from functools import wraps
 
 __all__ = [
     "cmp",
-    "decode",
-    "decode_with",
-    "decode_iter",
     "get_unbound_function",
     "maybe_add_to_os_environ_pathlist",
     "DefaultOrderedDict",
+    "deprecated",
+    "subprocess_Popen",
+    "call_subprocess_Popen",
+    "output_subprocess_Popen",
 ]
 
 
-# In python 3.x, when an exception is reraised it saves original
-# exception in its args, therefore in order to find the actual
-# message, we need to unpack arguments recursively.
 def exc_message(e):
+    """
+    In python 3.x, when an exception is reraised it saves original
+    exception in its args, therefore in order to find the actual
+    message, we need to unpack arguments recursively.
+    """
     msg = e.args[0]
     if isinstance(msg, Exception):
         return exc_message(msg)
@@ -44,19 +49,6 @@ def get_unbound_function(unbound):
     return unbound
 
 
-def decode(x):
-    return x.decode()
-
-
-def decode_iter(itr):
-    for x in itr:
-        yield x.decode()
-
-
-def decode_with(x, encoding):
-    return x.decode(encoding)
-
-
 class DefaultOrderedDict(OrderedDict):
     def __init__(self, default_factory=None, *a, **kw):
         if default_factory is not None and not isinstance(default_factory, Callable):
@@ -146,3 +138,81 @@ def deprecated(message: str = ""):
         return function_wrapper
 
     return decorator_wrapper
+
+
+def subprocess_Popen(command, **params):
+    """
+    Utility function to work around windows behavior that open windows.
+
+    :see: call_subprocess_Popen and output_subprocess_Popen
+    """
+    startupinfo = None
+    if os.name == "nt":
+        startupinfo = subprocess.STARTUPINFO()
+        try:
+            startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
+        except AttributeError:
+            startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
+
+        # Anaconda for Windows does not always provide .exe files
+        # in the PATH, they also have .bat files that call the corresponding
+        # executable. For instance, "g++.bat" is in the PATH, not "g++.exe"
+        # Unless "shell=True", "g++.bat" is not executed when trying to
+        # execute "g++" without extensions.
+        # (Executing "g++.bat" explicitly would also work.)
+        params["shell"] = True
+        # "If shell is True, it is recommended to pass args as a string rather than as a sequence." (cite taken from https://docs.python.org/2/library/subprocess.html#frequently-used-arguments)
+        # In case when command arguments have spaces, passing a command as a list will result in incorrect arguments break down, and consequently
+        # in "The filename, directory name, or volume label syntax is incorrect" error message.
+        # Passing the command as a single string solves this problem.
+        if isinstance(command, list):
+            command = " ".join(command)
+
+    # Using the dummy file descriptors below is a workaround for a
+    # crash experienced in an unusual Python 2.4.4 Windows environment
+    # with the default None values.
+    stdin = None
+    if "stdin" not in params:
+        stdin = open(os.devnull)
+        params["stdin"] = stdin.fileno()
+
+    try:
+        proc = subprocess.Popen(command, startupinfo=startupinfo, **params)
+    finally:
+        if stdin is not None:
+            stdin.close()
+    return proc
+
+
+def call_subprocess_Popen(command, **params):
+    """
+    Calls subprocess_Popen and discards the output, returning only the
+    exit code.
+    """
+    if "stdout" in params or "stderr" in params:
+        raise TypeError("don't use stderr or stdout with call_subprocess_Popen")
+    with open(os.devnull, "wb") as null:
+        # stdin to devnull is a workaround for a crash in a weird Windows
+        # environment where sys.stdin was None
+        params.setdefault("stdin", null)
+        params["stdout"] = null
+        params["stderr"] = null
+        p = subprocess_Popen(command, **params)
+        returncode = p.wait()
+    return returncode
+
+
+def output_subprocess_Popen(command, **params):
+    """
+    Calls subprocess_Popen, returning the output, error and exit code
+    in a tuple.
+    """
+    if "stdout" in params or "stderr" in params:
+        raise TypeError("don't use stderr or stdout with output_subprocess_Popen")
+    params["stdout"] = subprocess.PIPE
+    params["stderr"] = subprocess.PIPE
+    p = subprocess_Popen(command, **params)
+    # we need to use communicate to make sure we don't deadlock around
+    # the stdout/stderr pipe.
+    out = p.communicate()
+    return out + (p.returncode,)