Skip to content

P
pytensor
Unverified 提交 9f7cf7d0 authored 作者: Brandon T. Willard's avatar Brandon T. Willard 提交者： GitHub
浏览文件
操作

Merge pull request #221 from michaelosthege/clean-config-importing

Clean config importing
上级 819122e6 57864f52
显示空白字符变更
内嵌 并排
正在显示 包含 506 行增加 和 509 行删除
tests/gof/test_cmodule.py
浏览文件 @ 9f7cf7d0
...@@ -5,11 +5,13 @@ But this one tests a current behavior that isn't good: the c_code isn't ...@@ -5,11 +5,13 @@ But this one tests a current behavior that isn't good: the c_code isn't
deterministic based on the input type and the op. deterministic based on the input type and the op.
""" """
import logging
from unittest.mock import patch
import numpy as np import numpy as np
import theano import theano
from theano.gof.cmodule import GCC_compiler from theano.gof.cmodule import GCC_compiler, default_blas_ldflags
class MyOp(theano.compile.ops.DeepCopyOp): class MyOp(theano.compile.ops.DeepCopyOp):
...@@ -68,3 +70,16 @@ def test_flag_detection(): ...@@ -68,3 +70,16 @@ def test_flag_detection():
# but was not detected because that path is not usually taken, # but was not detected because that path is not usually taken,
# so we test it here directly. # so we test it here directly.
GCC_compiler.try_flags(["-lblas"]) GCC_compiler.try_flags(["-lblas"])
@patch("theano.gof.cmodule.try_blas_flag", return_value=None)
@patch("theano.gof.cmodule.sys")
def test_default_blas_ldflags(sys_mock, try_blas_flag_mock, caplog):
sys_mock.version = "3.8.0 | packaged by conda-forge | (default, Nov 22 2019, 19:11:38) \n[GCC 7.3.0]"
with patch.dict("sys.modules", {"mkl": None}):
with caplog.at_level(logging.WARNING):
default_blas_ldflags()
assert "install mkl with" in caplog.text
tests/gof/test_compiledir.py deleted 100644 → 0
浏览文件 @ 819122e6
from theano.configdefaults import short_platform
def test_short_platform():
for r, p, a in [ # (release, platform, answer)
(
"3.2.0-70-generic",
"Linux-3.2.0-70-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"3.2.0-70.1-generic",
"Linux-3.2.0-70.1-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"3.2.0-70.1.2-generic",
"Linux-3.2.0-70.1.2-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"2.6.35.14-106.fc14.x86_64",
"Linux-2.6.35.14-106.fc14.x86_64-x86_64-with-fedora-14-Laughlin",
"Linux-2.6-fc14.x86_64-x86_64-with-fedora-14-Laughlin",
),
]:
o = short_platform(r, p)
assert o == a, (o, a)
tests/tensor/test_mpi.py
浏览文件 @ 9f7cf7d0
...@@ -68,8 +68,8 @@ def test_mpi_roundtrip(): ...@@ -68,8 +68,8 @@ def test_mpi_roundtrip():
) )
(stdout, stderr) = p.communicate() (stdout, stderr) = p.communicate()
result = theano.utils.decode(stdout) result = stdout.decode()
assert "True" in result, theano.utils.decode(stderr) assert "True" in result, stderr.decode()
def test_mpi_send_wait_cmp(): def test_mpi_send_wait_cmp():
......
tests/test_config.py
浏览文件 @ 9f7cf7d0
"""Test config options.""" """Test config options."""
import configparser as stdlib_configparser import configparser as stdlib_configparser
import logging
from unittest.mock import patch
import pytest import pytest
from theano import configdefaults, configparser from theano import configdefaults, configparser
from theano.configdefaults import default_blas_ldflags from theano.configdefaults import short_platform
from theano.configparser import ConfigParam from theano.configparser import ConfigParam
...@@ -98,19 +96,6 @@ def test_invalid_default(): ...@@ -98,19 +96,6 @@ def test_invalid_default():
assert "test__test_invalid_default_b" not in root._flags_dict assert "test__test_invalid_default_b" not in root._flags_dict
@patch("theano.configdefaults.try_blas_flag", return_value=None)
@patch("theano.configdefaults.sys")
def test_default_blas_ldflags(sys_mock, try_blas_flag_mock, caplog):
sys_mock.version = "3.8.0 | packaged by conda-forge | (default, Nov 22 2019, 19:11:38) \n[GCC 7.3.0]"
with patch.dict("sys.modules", {"mkl": None}):
with caplog.at_level(logging.WARNING):
default_blas_ldflags()
assert "install mkl with" in caplog.text
def test_config_param_apply_and_validation(): def test_config_param_apply_and_validation():
cp = ConfigParam( cp = ConfigParam(
"TheDeFauLt", "TheDeFauLt",
...@@ -269,3 +254,34 @@ def test_mode_apply(): ...@@ -269,3 +254,34 @@ def test_mode_apply():
configdefaults.filter_mode(theano.compile.mode.FAST_COMPILE) configdefaults.filter_mode(theano.compile.mode.FAST_COMPILE)
== theano.compile.mode.FAST_COMPILE == theano.compile.mode.FAST_COMPILE
) )
class TestConfigHelperFunctions:
@pytest.mark.parametrize(
"release,platform,answer",
[
(
"3.2.0-70-generic",
"Linux-3.2.0-70-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"3.2.0-70.1-generic",
"Linux-3.2.0-70.1-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"3.2.0-70.1.2-generic",
"Linux-3.2.0-70.1.2-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"2.6.35.14-106.fc14.x86_64",
"Linux-2.6.35.14-106.fc14.x86_64-x86_64-with-fedora-14-Laughlin",
"Linux-2.6-fc14.x86_64-x86_64-with-fedora-14-Laughlin",
),
],
)
def test_short_platform(self, release, platform, answer):
o = short_platform(release, platform)
assert o == answer, (o, answer)
theano/__init__.py
浏览文件 @ 9f7cf7d0
...@@ -199,3 +199,8 @@ def sparse_grad(var): ...@@ -199,3 +199,8 @@ def sparse_grad(var):
import theano.tensor.random.var import theano.tensor.random.var
from theano.scan import checkpoints, clone, foldl, foldr, map, reduce, scan from theano.scan import checkpoints, clone, foldl, foldr, map, reduce, scan
# Some config variables are registered by submodules. Only after all those imports
# were executed, we can warn about remaining flags provided by the user through THEANO_FLAGS.
config.warn_unused_flags()
theano/configdefaults.py
浏览文件 @ 9f7cf7d0
...@@ -7,7 +7,6 @@ import socket ...@@ -7,7 +7,6 @@ import socket
import struct import struct
import sys import sys
import textwrap import textwrap
import warnings
import numpy as np import numpy as np
...@@ -23,8 +22,11 @@ from theano.configparser import ( ...@@ -23,8 +22,11 @@ from theano.configparser import (
IntParam, IntParam,
StrParam, StrParam,
) )
from theano.misc.windows import call_subprocess_Popen, output_subprocess_Popen from theano.utils import (
from theano.utils import maybe_add_to_os_environ_pathlist call_subprocess_Popen,
maybe_add_to_os_environ_pathlist,
output_subprocess_Popen,
)
_logger = logging.getLogger("theano.configdefaults") _logger = logging.getLogger("theano.configdefaults")
...@@ -178,48 +180,6 @@ def is_valid_check_preallocated_output_param(param): ...@@ -178,48 +180,6 @@ def is_valid_check_preallocated_output_param(param):
return True return True
def check_mkl_openmp():
if not theano.config.blas__check_openmp:
return
if sys.platform == "darwin":
return
if (
"MKL_THREADING_LAYER" in os.environ
and os.environ["MKL_THREADING_LAYER"] == "GNU"
):
return
try:
import numpy._mklinit # noqa
return
except ImportError:
pass
try:
import mkl
if "2018" in mkl.get_version_string():
raise RuntimeError(
"""
To use MKL 2018 with Theano either update the numpy conda packages to
their latest build or set "MKL_THREADING_LAYER=GNU" in your
environment.
"""
)
except ImportError:
raise RuntimeError(
"""
Could not import 'mkl'. If you are using conda, update the numpy
packages to the latest build otherwise, set MKL_THREADING_LAYER=GNU in
your environment for MKL 2018.
If you have MKL 2017 install and are not in a conda environment you
can set the Theano flag blas__check_openmp to False. Be warned that if
you set this flag and don't set the appropriate environment or make
sure you have the right version you *will* get wrong results.
"""
)
def _timeout_default(): def _timeout_default():
return theano.config.compile__wait * 24 return theano.config.compile__wait * 24
...@@ -250,279 +210,6 @@ def python_int_bitwidth(): ...@@ -250,279 +210,6 @@ def python_int_bitwidth():
return struct.calcsize("l") * 8 return struct.calcsize("l") * 8
def default_blas_ldflags():
"""Read local NumPy and MKL build settings and construct `ld` flags from them.
Returns
-------
str
"""
import numpy.distutils # noqa
warn_record = []
try:
if hasattr(numpy.distutils, "__config__") and numpy.distutils.__config__:
# If the old private interface is available use it as it
# don't print information to the user.
blas_info = numpy.distutils.__config__.blas_opt_info
else:
# We do this import only here, as in some setup, if we
# just import theano and exit, with the import at global
# scope, we get this error at exit: "Exception TypeError:
# "'NoneType' object is not callable" in <bound method
# Popen.__del__ of <subprocess.Popen object at 0x21359d0>>
# ignored"
# This happen with Python 2.7.3 |EPD 7.3-1 and numpy 1.8.1
# isort: off
import numpy.distutils.system_info # noqa
# We need to catch warnings as in some cases NumPy print
# stuff that we don't want the user to see.
# I'm not able to remove all printed stuff
with warnings.catch_warnings(record=True):
numpy.distutils.system_info.system_info.verbosity = 0
blas_info = numpy.distutils.system_info.get_info("blas_opt")
# If we are in a EPD installation, mkl is available
if "EPD" in sys.version:
use_unix_epd = True
if sys.platform == "win32":
return " ".join(
['-L"%s"' % os.path.join(sys.prefix, "Scripts")]
+
# Why on Windows, the library used are not the
# same as what is in
# blas_info['libraries']?
[f"-l{l}" for l in ["mk2_core", "mk2_intel_thread", "mk2_rt"]]
)
elif sys.platform == "darwin":
# The env variable is needed to link with mkl
new_path = os.path.join(sys.prefix, "lib")
v = os.getenv("DYLD_FALLBACK_LIBRARY_PATH", None)
if v is not None:
# Explicit version could be replaced by a symbolic
# link called 'Current' created by EPD installer
# This will resolve symbolic links
v = os.path.realpath(v)
# The python __import__ don't seam to take into account
# the new env variable "DYLD_FALLBACK_LIBRARY_PATH"
# when we set with os.environ['...'] = X or os.putenv()
# So we warn the user and tell him what todo.
if v is None or new_path not in v.split(":"):
_logger.warning(
"The environment variable "
"'DYLD_FALLBACK_LIBRARY_PATH' does not contain "
"the '{new_path}' path in its value. This will make "
"Theano use a slow version of BLAS. Update "
"'DYLD_FALLBACK_LIBRARY_PATH' to contain the "
"said value, this will disable this warning."
)
use_unix_epd = False
if use_unix_epd:
return " ".join(
["-L%s" % os.path.join(sys.prefix, "lib")]
+ ["-l%s" % l for l in blas_info["libraries"]]
)
# Canopy
if "Canopy" in sys.prefix:
subsub = "lib"
if sys.platform == "win32":
subsub = "Scripts"
lib_path = os.path.join(sys.base_prefix, subsub)
if not os.path.exists(lib_path):
# Old logic to find the path. I don't think we still
# need it, but I don't have the time to test all
# installation configuration. So I keep this as a fall
# back in case the current expectation don't work.
# This old logic don't work when multiple version of
# Canopy is installed.
p = os.path.join(sys.base_prefix, "..", "..", "appdata")
assert os.path.exists(p), "Canopy changed the location of MKL"
lib_paths = os.listdir(p)
# Try to remove subdir that can't contain MKL
for sub in lib_paths:
if not os.path.exists(os.path.join(p, sub, subsub)):
lib_paths.remove(sub)
assert len(lib_paths) == 1, (
"Unexpected case when looking for Canopy MKL libraries",
p,
lib_paths,
[os.listdir(os.path.join(p, sub)) for sub in lib_paths],
)
lib_path = os.path.join(p, lib_paths[0], subsub)
assert os.path.exists(lib_path), "Canopy changed the location of MKL"
if sys.platform == "linux2" or sys.platform == "darwin":
return " ".join(
["-L%s" % lib_path] + ["-l%s" % l for l in blas_info["libraries"]]
)
elif sys.platform == "win32":
return " ".join(
['-L"%s"' % lib_path]
+
# Why on Windows, the library used are not the
# same as what is in blas_info['libraries']?
[f"-l{l}" for l in ["mk2_core", "mk2_intel_thread", "mk2_rt"]]
)
# MKL
# If mkl can be imported then use it. On conda:
# "conda install mkl-service" installs the Python wrapper and
# the low-level C libraries as well as optimised version of
# numpy and scipy.
try:
import mkl # noqa
except ImportError as e:
if any([m for m in ("conda", "Continuum") if m in sys.version]):
warn_record.append(f"install mkl with `conda install mkl-service`: {e}")
else:
# This branch is executed if no exception was raised
if sys.platform == "win32":
lib_path = os.path.join(sys.prefix, "Library", "bin")
flags = [f'-L"{lib_path}"']
else:
lib_path = blas_info.get("library_dirs", [])
flags = []
if lib_path:
flags = [f"-L{lib_path[0]}"]
if "2018" in mkl.get_version_string():
thr = "mkl_gnu_thread"
else:
thr = "mkl_intel_thread"
base_flags = list(flags)
flags += [f"-l{l}" for l in ["mkl_core", thr, "mkl_rt"]]
res = try_blas_flag(flags)
if not res and sys.platform == "win32" and thr == "mkl_gnu_thread":
# Check if it would work for intel OpenMP on windows
flags = base_flags + [
f"-l{l}" for l in ["mkl_core", "mkl_intel_thread", "mkl_rt"]
]
res = try_blas_flag(flags)
if res:
check_mkl_openmp()
return res
flags.extend(["-Wl,-rpath," + l for l in blas_info.get("library_dirs", [])])
res = try_blas_flag(flags)
if res:
check_mkl_openmp()
maybe_add_to_os_environ_pathlist("PATH", lib_path[0])
return res
# to support path that includes spaces, we need to wrap it with double quotes on Windows
path_wrapper = '"' if os.name == "nt" else ""
ret = (
# TODO: the Gemm op below should separate the
# -L and -l arguments into the two callbacks
# that CLinker uses for that stuff. for now,
# we just pass the whole ldflags as the -l
# options part.
[
f"-L{path_wrapper}{l}{path_wrapper}"
for l in blas_info.get("library_dirs", [])
]
+ [f"-l{l}" for l in blas_info.get("libraries", [])]
+ blas_info.get("extra_link_args", [])
)
# For some very strange reason, we need to specify -lm twice
# to get mkl to link correctly. I have no idea why.
if any("mkl" in fl for fl in ret):
ret.extend(["-lm", "-lm"])
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
# If we are using conda and can't reuse numpy blas, then doing
# the fallback and test -lblas could give slow computation, so
# warn about this.
for warn in warn_record:
_logger.warning(warn)
del warn_record
# Some environment don't have the lib dir in LD_LIBRARY_PATH.
# So add it.
ret.extend(["-Wl,-rpath," + l for l in blas_info.get("library_dirs", [])])
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
# Add sys.prefix/lib to the runtime search path. On
# non-system installations of Python that use the
# system linker, this is generally necessary.
if sys.platform in ("linux", "darwin"):
lib_path = os.path.join(sys.prefix, "lib")
ret.append("-Wl,-rpath," + lib_path)
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
except KeyError:
pass
# Even if we could not detect what was used for numpy, or if these
# libraries are not found, most Linux systems have a libblas.so
# readily available. We try to see if that's the case, rather
# than disable blas. To test it correctly, we must load a program.
# Otherwise, there could be problem in the LD_LIBRARY_PATH.
return try_blas_flag(["-lblas"])
def try_blas_flag(flags):
from theano.gof.cmodule import GCC_compiler
test_code = textwrap.dedent(
"""\
extern "C" double ddot_(int*, double*, int*, double*, int*);
int main(int argc, char** argv)
{
int Nx = 5;
int Sx = 1;
double x[5] = {0, 1, 2, 3, 4};
double r = ddot_(&Nx, x, &Sx, x, &Sx);
if ((r - 30.) > 1e-6 || (r - 30.) < -1e-6)
{
return -1;
}
return 0;
}
"""
)
cflags = list(flags)
# to support path that includes spaces, we need to wrap it with double quotes on Windows
path_wrapper = '"' if os.name == "nt" else ""
cflags.extend(
[
f"-L{path_wrapper}{d}{path_wrapper}"
for d in theano.gof.cmodule.std_lib_dirs()
]
)
res = GCC_compiler.try_compile_tmp(
test_code, tmp_prefix="try_blas_", flags=cflags, try_run=True
)
# res[0]: shows successful compilation
# res[1]: shows successful execution
if res and res[0] and res[1]:
return " ".join(flags)
else:
return ""
def filter_vm_lazy(val): def filter_vm_lazy(val):
if val == "False" or val is False: if val == "False" or val is False:
return False return False
...@@ -1743,23 +1430,6 @@ def add_optimizer_configvars(): ...@@ -1743,23 +1430,6 @@ def add_optimizer_configvars():
) )
def add_blas_configvars():
config.add(
"blas__ldflags",
"lib[s] to include for [Fortran] level-3 blas implementation",
StrParam(default_blas_ldflags),
# Added elsewhere in the c key only when needed.
in_c_key=False,
)
config.add(
"blas__check_openmp",
"Check for openmp library conflict.\nWARNING: Setting this to False leaves you open to wrong results in blas-related operations.",
BoolParam(True),
in_c_key=False,
)
def add_metaopt_configvars(): def add_metaopt_configvars():
config.add( config.add(
"metaopt__verbose", "metaopt__verbose",
...@@ -2243,12 +1913,9 @@ add_optimizer_configvars() ...@@ -2243,12 +1913,9 @@ add_optimizer_configvars()
# This would mean either calling the function from there, or even moving all the related code there. # This would mean either calling the function from there, or even moving all the related code there.
# Blas-related config are a special pain-point, because their addition depends on a lot of stuff from # Blas-related config are a special pain-point, because their addition depends on a lot of stuff from
# that module, which introduces a circular dependency! # that module, which introduces a circular dependency!
add_blas_configvars()
add_metaopt_configvars() add_metaopt_configvars()
add_vm_configvars() add_vm_configvars()
add_deprecated_configvars() add_deprecated_configvars()
# TODO: another example for config that should be added by the module itself:
add_scan_configvars()
# TODO: `gcc_version_str` is used by other modules.. Should it become an immutable config var? # TODO: `gcc_version_str` is used by other modules.. Should it become an immutable config var?
try: try:
...@@ -2260,7 +1927,3 @@ except OSError: ...@@ -2260,7 +1927,3 @@ except OSError:
# TODO: The caching dir resolution is a procedural mess of helper functions, local variables # TODO: The caching dir resolution is a procedural mess of helper functions, local variables
# and config definitions. And the result is also not particularly pretty.. # and config definitions. And the result is also not particularly pretty..
add_caching_dir_configvars() add_caching_dir_configvars()
# Check if there are remaining flags provided by the user through THEANO_FLAGS.
for key in config._flags_dict.keys():
warnings.warn(f"Theano does not recognise this flag: {key}")
theano/configparser.py
浏览文件 @ 9f7cf7d0
...@@ -262,6 +262,10 @@ class TheanoConfigParser: ...@@ -262,6 +262,10 @@ class TheanoConfigParser:
""" """
return _ChangeFlagsDecorator(*args, _root=self, **kwargs) return _ChangeFlagsDecorator(*args, _root=self, **kwargs)
def warn_unused_flags(self):
for key in self._flags_dict.keys():
warnings.warn(f"Theano does not recognise this flag: {key}")
class ConfigParam: class ConfigParam:
"""Base class of all kinds of configuration parameters. """Base class of all kinds of configuration parameters.
......
theano/gof/cmodule.py
浏览文件 @ 9f7cf7d0
...@@ -28,8 +28,7 @@ from theano.configdefaults import gcc_version_str, local_bitwidth ...@@ -28,8 +28,7 @@ from theano.configdefaults import gcc_version_str, local_bitwidth
# we will abuse the lockfile mechanism when reading and writing the registry # we will abuse the lockfile mechanism when reading and writing the registry
from theano.gof import compilelock from theano.gof import compilelock
from theano.gof.utils import flatten, hash_from_code from theano.gof.utils import flatten, hash_from_code
from theano.misc.windows import output_subprocess_Popen, subprocess_Popen from theano.utils import output_subprocess_Popen, subprocess_Popen
from theano.utils import decode, decode_iter
importlib = None importlib = None
...@@ -1786,7 +1785,7 @@ def gcc_llvm(): ...@@ -1786,7 +1785,7 @@ def gcc_llvm():
""" """
if gcc_llvm.is_llvm is None: if gcc_llvm.is_llvm is None:
try: try:
p_out = output_subprocess_Popen([theano.config.cxx, "--version"]) p_out = output_subprocess_Popen([config.cxx, "--version"])
output = p_out[0] + p_out[1] output = p_out[0] + p_out[1]
except OSError: except OSError:
# Typically means g++ cannot be found. # Typically means g++ cannot be found.
...@@ -1942,6 +1941,41 @@ class Compiler: ...@@ -1942,6 +1941,41 @@ class Compiler:
) )
def try_blas_flag(flags):
test_code = textwrap.dedent(
"""\
extern "C" double ddot_(int*, double*, int*, double*, int*);
int main(int argc, char** argv)
{
int Nx = 5;
int Sx = 1;
double x[5] = {0, 1, 2, 3, 4};
double r = ddot_(&Nx, x, &Sx, x, &Sx);
if ((r - 30.) > 1e-6 || (r - 30.) < -1e-6)
{
return -1;
}
return 0;
}
"""
)
cflags = list(flags)
# to support path that includes spaces, we need to wrap it with double quotes on Windows
path_wrapper = '"' if os.name == "nt" else ""
cflags.extend([f"-L{path_wrapper}{d}{path_wrapper}" for d in std_lib_dirs()])
res = GCC_compiler.try_compile_tmp(
test_code, tmp_prefix="try_blas_", flags=cflags, try_run=True
)
# res[0]: shows successful compilation
# res[1]: shows successful execution
if res and res[0] and res[1]:
return " ".join(flags)
else:
return ""
def try_march_flag(flags): def try_march_flag(flags):
""" """
Try to compile and run a simple C snippet using current flags. Try to compile and run a simple C snippet using current flags.
...@@ -1980,7 +2014,7 @@ class GCC_compiler(Compiler): ...@@ -1980,7 +2014,7 @@ class GCC_compiler(Compiler):
@staticmethod @staticmethod
def version_str(): def version_str():
return theano.config.cxx + " " + gcc_version_str return config.cxx + " " + gcc_version_str
@staticmethod @staticmethod
def compile_args(march_flags=True): def compile_args(march_flags=True):
...@@ -2008,10 +2042,10 @@ class GCC_compiler(Compiler): ...@@ -2008,10 +2042,10 @@ class GCC_compiler(Compiler):
break break
if ( if (
"g++" not in theano.config.cxx "g++" not in config.cxx
and "clang++" not in theano.config.cxx and "clang++" not in config.cxx
and "clang-omp++" not in theano.config.cxx and "clang-omp++" not in config.cxx
and "icpc" not in theano.config.cxx and "icpc" not in config.cxx
): ):
_logger.warning( _logger.warning(
"Your Theano flag `cxx` seems not to be" "Your Theano flag `cxx` seems not to be"
...@@ -2043,7 +2077,7 @@ class GCC_compiler(Compiler): ...@@ -2043,7 +2077,7 @@ class GCC_compiler(Compiler):
return None return None
lines = BytesIO(stdout + stderr).readlines() lines = BytesIO(stdout + stderr).readlines()
lines = decode_iter(lines) lines = (l.decode() for l in lines)
if parse: if parse:
selected_lines = [] selected_lines = []
for line in lines: for line in lines:
...@@ -2063,7 +2097,7 @@ class GCC_compiler(Compiler): ...@@ -2063,7 +2097,7 @@ class GCC_compiler(Compiler):
# The '-' at the end is needed. Otherwise, g++ do not output # The '-' at the end is needed. Otherwise, g++ do not output
# enough information. # enough information.
native_lines = get_lines(f"{theano.config.cxx} -march=native -E -v -") native_lines = get_lines(f"{config.cxx} -march=native -E -v -")
if native_lines is None: if native_lines is None:
_logger.info( _logger.info(
"Call to 'g++ -march=native' failed," "not setting -march flag" "Call to 'g++ -march=native' failed," "not setting -march flag"
...@@ -2078,7 +2112,7 @@ class GCC_compiler(Compiler): ...@@ -2078,7 +2112,7 @@ class GCC_compiler(Compiler):
# That means we did not select the right lines, so # That means we did not select the right lines, so
# we have to report all the lines instead # we have to report all the lines instead
reported_lines = get_lines( reported_lines = get_lines(
f"{theano.config.cxx} -march=native -E -v -", parse=False f"{config.cxx} -march=native -E -v -", parse=False
) )
else: else:
reported_lines = native_lines reported_lines = native_lines
...@@ -2091,7 +2125,7 @@ class GCC_compiler(Compiler): ...@@ -2091,7 +2125,7 @@ class GCC_compiler(Compiler):
f" problem:\n {reported_lines}" f" problem:\n {reported_lines}"
) )
else: else:
default_lines = get_lines(f"{theano.config.cxx} -E -v -") default_lines = get_lines(f"{config.cxx} -E -v -")
_logger.info(f"g++ default lines: {default_lines}") _logger.info(f"g++ default lines: {default_lines}")
if len(default_lines) < 1: if len(default_lines) < 1:
_logger.warning( _logger.warning(
...@@ -2102,7 +2136,7 @@ class GCC_compiler(Compiler): ...@@ -2102,7 +2136,7 @@ class GCC_compiler(Compiler):
" functions. Please submit the following lines to" " functions. Please submit the following lines to"
" Theano's mailing list so that we can fix this" " Theano's mailing list so that we can fix this"
" problem:\n %s", " problem:\n %s",
get_lines(f"{theano.config.cxx} -E -v -", parse=False), get_lines(f"{config.cxx} -E -v -", parse=False),
) )
else: else:
# Some options are actually given as "-option value", # Some options are actually given as "-option value",
...@@ -2330,7 +2364,7 @@ class GCC_compiler(Compiler): ...@@ -2330,7 +2364,7 @@ class GCC_compiler(Compiler):
comp_args=True, comp_args=True,
): ):
return cls._try_compile_tmp( return cls._try_compile_tmp(
src_code, tmp_prefix, flags, try_run, output, theano.config.cxx, comp_args src_code, tmp_prefix, flags, try_run, output, config.cxx, comp_args
) )
@classmethod @classmethod
...@@ -2344,7 +2378,7 @@ class GCC_compiler(Compiler): ...@@ -2344,7 +2378,7 @@ class GCC_compiler(Compiler):
comp_args=True, comp_args=True,
): ):
return cls._try_flags( return cls._try_flags(
flag_list, preambule, body, try_run, output, theano.config.cxx, comp_args flag_list, preambule, body, try_run, output, config.cxx, comp_args
) )
@staticmethod @staticmethod
...@@ -2394,7 +2428,7 @@ class GCC_compiler(Compiler): ...@@ -2394,7 +2428,7 @@ class GCC_compiler(Compiler):
""" """
# TODO: Do not do the dlimport in this function # TODO: Do not do the dlimport in this function
if not theano.config.cxx: if not config.cxx:
raise MissingGXX("g++ not available! We can't compile c code.") raise MissingGXX("g++ not available! We can't compile c code.")
if include_dirs is None: if include_dirs is None:
...@@ -2438,7 +2472,7 @@ class GCC_compiler(Compiler): ...@@ -2438,7 +2472,7 @@ class GCC_compiler(Compiler):
lib_filename = os.path.join(location, filepath) lib_filename = os.path.join(location, filepath)
_logger.debug(f"Generating shared lib {lib_filename}") _logger.debug(f"Generating shared lib {lib_filename}")
cmd = [theano.config.cxx, get_gcc_shared_library_arg(), "-g"] cmd = [config.cxx, get_gcc_shared_library_arg(), "-g"]
if config.cmodule__remove_gxx_opt: if config.cmodule__remove_gxx_opt:
cmd.extend(p for p in preargs if not p.startswith("-O")) cmd.extend(p for p in preargs if not p.startswith("-O"))
...@@ -2472,7 +2506,7 @@ class GCC_compiler(Compiler): ...@@ -2472,7 +2506,7 @@ class GCC_compiler(Compiler):
try: try:
p_out = output_subprocess_Popen(cmd) p_out = output_subprocess_Popen(cmd)
compile_stderr = decode(p_out[1]) compile_stderr = p_out[1].decode()
except Exception: except Exception:
# An exception can occur e.g. if `g++` is not found. # An exception can occur e.g. if `g++` is not found.
print_command_line_error() print_command_line_error()
...@@ -2537,3 +2571,297 @@ class GCC_compiler(Compiler): ...@@ -2537,3 +2571,297 @@ class GCC_compiler(Compiler):
def icc_module_compile_str(*args): def icc_module_compile_str(*args):
raise NotImplementedError() raise NotImplementedError()
def check_mkl_openmp():
if not config.blas__check_openmp:
return
if sys.platform == "darwin":
return
if (
"MKL_THREADING_LAYER" in os.environ
and os.environ["MKL_THREADING_LAYER"] == "GNU"
):
return
try:
import numpy._mklinit # noqa
return
except ImportError:
pass
try:
import mkl
if "2018" in mkl.get_version_string():
raise RuntimeError(
"""
To use MKL 2018 with Theano either update the numpy conda packages to
their latest build or set "MKL_THREADING_LAYER=GNU" in your
environment.
"""
)
except ImportError:
raise RuntimeError(
"""
Could not import 'mkl'. If you are using conda, update the numpy
packages to the latest build otherwise, set MKL_THREADING_LAYER=GNU in
your environment for MKL 2018.
If you have MKL 2017 install and are not in a conda environment you
can set the Theano flag blas__check_openmp to False. Be warned that if
you set this flag and don't set the appropriate environment or make
sure you have the right version you *will* get wrong results.
"""
)
def default_blas_ldflags():
"""Read local NumPy and MKL build settings and construct `ld` flags from them.
Returns
-------
str
"""
import numpy.distutils # noqa
warn_record = []
try:
if hasattr(numpy.distutils, "__config__") and numpy.distutils.__config__:
# If the old private interface is available use it as it
# don't print information to the user.
blas_info = numpy.distutils.__config__.blas_opt_info
else:
# We do this import only here, as in some setup, if we
# just import theano and exit, with the import at global
# scope, we get this error at exit: "Exception TypeError:
# "'NoneType' object is not callable" in <bound method
# Popen.__del__ of <subprocess.Popen object at 0x21359d0>>
# ignored"
# This happen with Python 2.7.3 |EPD 7.3-1 and numpy 1.8.1
# isort: off
import numpy.distutils.system_info # noqa
# We need to catch warnings as in some cases NumPy print
# stuff that we don't want the user to see.
# I'm not able to remove all printed stuff
with warnings.catch_warnings(record=True):
numpy.distutils.system_info.system_info.verbosity = 0
blas_info = numpy.distutils.system_info.get_info("blas_opt")
# If we are in a EPD installation, mkl is available
if "EPD" in sys.version:
use_unix_epd = True
if sys.platform == "win32":
return " ".join(
['-L"%s"' % os.path.join(sys.prefix, "Scripts")]
+
# Why on Windows, the library used are not the
# same as what is in
# blas_info['libraries']?
[f"-l{l}" for l in ["mk2_core", "mk2_intel_thread", "mk2_rt"]]
)
elif sys.platform == "darwin":
# The env variable is needed to link with mkl
new_path = os.path.join(sys.prefix, "lib")
v = os.getenv("DYLD_FALLBACK_LIBRARY_PATH", None)
if v is not None:
# Explicit version could be replaced by a symbolic
# link called 'Current' created by EPD installer
# This will resolve symbolic links
v = os.path.realpath(v)
# The python __import__ don't seam to take into account
# the new env variable "DYLD_FALLBACK_LIBRARY_PATH"
# when we set with os.environ['...'] = X or os.putenv()
# So we warn the user and tell him what todo.
if v is None or new_path not in v.split(":"):
_logger.warning(
"The environment variable "
"'DYLD_FALLBACK_LIBRARY_PATH' does not contain "
"the '{new_path}' path in its value. This will make "
"Theano use a slow version of BLAS. Update "
"'DYLD_FALLBACK_LIBRARY_PATH' to contain the "
"said value, this will disable this warning."
)
use_unix_epd = False
if use_unix_epd:
return " ".join(
["-L%s" % os.path.join(sys.prefix, "lib")]
+ ["-l%s" % l for l in blas_info["libraries"]]
)
# Canopy
if "Canopy" in sys.prefix:
subsub = "lib"
if sys.platform == "win32":
subsub = "Scripts"
lib_path = os.path.join(sys.base_prefix, subsub)
if not os.path.exists(lib_path):
# Old logic to find the path. I don't think we still
# need it, but I don't have the time to test all
# installation configuration. So I keep this as a fall
# back in case the current expectation don't work.
# This old logic don't work when multiple version of
# Canopy is installed.
p = os.path.join(sys.base_prefix, "..", "..", "appdata")
assert os.path.exists(p), "Canopy changed the location of MKL"
lib_paths = os.listdir(p)
# Try to remove subdir that can't contain MKL
for sub in lib_paths:
if not os.path.exists(os.path.join(p, sub, subsub)):
lib_paths.remove(sub)
assert len(lib_paths) == 1, (
"Unexpected case when looking for Canopy MKL libraries",
p,
lib_paths,
[os.listdir(os.path.join(p, sub)) for sub in lib_paths],
)
lib_path = os.path.join(p, lib_paths[0], subsub)
assert os.path.exists(lib_path), "Canopy changed the location of MKL"
if sys.platform == "linux2" or sys.platform == "darwin":
return " ".join(
["-L%s" % lib_path] + ["-l%s" % l for l in blas_info["libraries"]]
)
elif sys.platform == "win32":
return " ".join(
['-L"%s"' % lib_path]
+
# Why on Windows, the library used are not the
# same as what is in blas_info['libraries']?
[f"-l{l}" for l in ["mk2_core", "mk2_intel_thread", "mk2_rt"]]
)
# MKL
# If mkl can be imported then use it. On conda:
# "conda install mkl-service" installs the Python wrapper and
# the low-level C libraries as well as optimised version of
# numpy and scipy.
try:
import mkl # noqa
except ImportError as e:
if any([m for m in ("conda", "Continuum") if m in sys.version]):
warn_record.append(f"install mkl with `conda install mkl-service`: {e}")
else:
# This branch is executed if no exception was raised
if sys.platform == "win32":
lib_path = os.path.join(sys.prefix, "Library", "bin")
flags = [f'-L"{lib_path}"']
else:
lib_path = blas_info.get("library_dirs", [])
flags = []
if lib_path:
flags = [f"-L{lib_path[0]}"]
if "2018" in mkl.get_version_string():
thr = "mkl_gnu_thread"
else:
thr = "mkl_intel_thread"
base_flags = list(flags)
flags += [f"-l{l}" for l in ["mkl_core", thr, "mkl_rt"]]
res = try_blas_flag(flags)
if not res and sys.platform == "win32" and thr == "mkl_gnu_thread":
# Check if it would work for intel OpenMP on windows
flags = base_flags + [
f"-l{l}" for l in ["mkl_core", "mkl_intel_thread", "mkl_rt"]
]
res = try_blas_flag(flags)
if res:
check_mkl_openmp()
return res
flags.extend(["-Wl,-rpath," + l for l in blas_info.get("library_dirs", [])])
res = try_blas_flag(flags)
if res:
check_mkl_openmp()
theano.utils.maybe_add_to_os_environ_pathlist("PATH", lib_path[0])
return res
# to support path that includes spaces, we need to wrap it with double quotes on Windows
path_wrapper = '"' if os.name == "nt" else ""
ret = (
# TODO: the Gemm op below should separate the
# -L and -l arguments into the two callbacks
# that CLinker uses for that stuff. for now,
# we just pass the whole ldflags as the -l
# options part.
[
f"-L{path_wrapper}{l}{path_wrapper}"
for l in blas_info.get("library_dirs", [])
]
+ [f"-l{l}" for l in blas_info.get("libraries", [])]
+ blas_info.get("extra_link_args", [])
)
# For some very strange reason, we need to specify -lm twice
# to get mkl to link correctly. I have no idea why.
if any("mkl" in fl for fl in ret):
ret.extend(["-lm", "-lm"])
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
# If we are using conda and can't reuse numpy blas, then doing
# the fallback and test -lblas could give slow computation, so
# warn about this.
for warn in warn_record:
_logger.warning(warn)
del warn_record
# Some environment don't have the lib dir in LD_LIBRARY_PATH.
# So add it.
ret.extend(["-Wl,-rpath," + l for l in blas_info.get("library_dirs", [])])
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
# Add sys.prefix/lib to the runtime search path. On
# non-system installations of Python that use the
# system linker, this is generally necessary.
if sys.platform in ("linux", "darwin"):
lib_path = os.path.join(sys.prefix, "lib")
ret.append("-Wl,-rpath," + lib_path)
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
except KeyError:
pass
# Even if we could not detect what was used for numpy, or if these
# libraries are not found, most Linux systems have a libblas.so
# readily available. We try to see if that's the case, rather
# than disable blas. To test it correctly, we must load a program.
# Otherwise, there could be problem in the LD_LIBRARY_PATH.
return try_blas_flag(["-lblas"])
def add_blas_configvars():
config.add(
"blas__ldflags",
"lib[s] to include for [Fortran] level-3 blas implementation",
theano.configparser.StrParam(default_blas_ldflags),
# Added elsewhere in the c key only when needed.
in_c_key=False,
)
config.add(
"blas__check_openmp",
"Check for openmp library conflict.\nWARNING: Setting this to False leaves you open to wrong results in blas-related operations.",
theano.configparser.BoolParam(True),
in_c_key=False,
)
# Register config parameters that are specific to this module:
add_blas_configvars()
theano/misc/elemwise_openmp_speedup.py
浏览文件 @ 9f7cf7d0
...@@ -5,7 +5,6 @@ from locale import getpreferredencoding ...@@ -5,7 +5,6 @@ from locale import getpreferredencoding
from optparse import OptionParser from optparse import OptionParser
import theano import theano
from theano.utils import decode_with
console_encoding = getpreferredencoding() console_encoding = getpreferredencoding()
...@@ -37,7 +36,7 @@ def runScript(N): ...@@ -37,7 +36,7 @@ def runScript(N):
if err: if err:
print(err) print(err)
sys.exit() sys.exit()
return list(map(float, decode_with(out, console_encoding).split(" "))) return list(map(float, out.decode(console_encoding).split(" ")))
if __name__ == "__main__": if __name__ == "__main__":
......
theano/misc/windows.py deleted 100644 → 0
浏览文件 @ 819122e6
import os
import subprocess
def subprocess_Popen(command, **params):
"""
Utility function to work around windows behavior that open windows.
:see: call_subprocess_Popen and output_subprocess_Popen
"""
startupinfo = None
if os.name == "nt":
startupinfo = subprocess.STARTUPINFO()
try:
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
except AttributeError:
startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
# Anaconda for Windows does not always provide .exe files
# in the PATH, they also have .bat files that call the corresponding
# executable. For instance, "g++.bat" is in the PATH, not "g++.exe"
# Unless "shell=True", "g++.bat" is not executed when trying to
# execute "g++" without extensions.
# (Executing "g++.bat" explicitly would also work.)
params["shell"] = True
# "If shell is True, it is recommended to pass args as a string rather than as a sequence." (cite taken from https://docs.python.org/2/library/subprocess.html#frequently-used-arguments)
# In case when command arguments have spaces, passing a command as a list will result in incorrect arguments break down, and consequently
# in "The filename, directory name, or volume label syntax is incorrect" error message.
# Passing the command as a single string solves this problem.
if isinstance(command, list):
command = " ".join(command)
# Using the dummy file descriptors below is a workaround for a
# crash experienced in an unusual Python 2.4.4 Windows environment
# with the default None values.
stdin = None
if "stdin" not in params:
stdin = open(os.devnull)
params["stdin"] = stdin.fileno()
try:
proc = subprocess.Popen(command, startupinfo=startupinfo, **params)
finally:
if stdin is not None:
stdin.close()
return proc
def call_subprocess_Popen(command, **params):
"""
Calls subprocess_Popen and discards the output, returning only the
exit code.
"""
if "stdout" in params or "stderr" in params:
raise TypeError("don't use stderr or stdout with call_subprocess_Popen")
with open(os.devnull, "wb") as null:
# stdin to devnull is a workaround for a crash in a weird Windows
# environment where sys.stdin was None
params.setdefault("stdin", null)
params["stdout"] = null
params["stderr"] = null
p = subprocess_Popen(command, **params)
returncode = p.wait()
return returncode
def output_subprocess_Popen(command, **params):
"""
Calls subprocess_Popen, returning the output, error and exit code
in a tuple.
"""
if "stdout" in params or "stderr" in params:
raise TypeError("don't use stderr or stdout with output_subprocess_Popen")
params["stdout"] = subprocess.PIPE
params["stderr"] = subprocess.PIPE
p = subprocess_Popen(command, **params)
# we need to use communicate to make sure we don't deadlock around
# the stdout/stderr pipe.
out = p.communicate()
return out + (p.returncode,)
theano/scan/__init__.py
浏览文件 @ 9f7cf7d0
...@@ -47,6 +47,11 @@ __authors__ = ( ...@@ -47,6 +47,11 @@ __authors__ = (
__copyright__ = "(c) 2010, Universite de Montreal" __copyright__ = "(c) 2010, Universite de Montreal"
__contact__ = "Razvan Pascanu <r.pascanu@gmail>" __contact__ = "Razvan Pascanu <r.pascanu@gmail>"
from theano import configdefaults
configdefaults.add_scan_configvars()
from theano.scan import opt from theano.scan import opt
from theano.scan.basic import scan from theano.scan.basic import scan
from theano.scan.checkpoints import scan_checkpoints from theano.scan.checkpoints import scan_checkpoints
......
theano/utils.py
浏览文件 @ 9f7cf7d0
"""Utility functions for Theano.""" """Utility functions that only depend on the standard library."""
import inspect import inspect
import os
import subprocess
import traceback import traceback
import warnings import warnings
from collections import OrderedDict from collections import OrderedDict
...@@ -11,19 +13,22 @@ from functools import wraps ...@@ -11,19 +13,22 @@ from functools import wraps
__all__ = [ __all__ = [
"cmp", "cmp",
"decode",
"decode_with",
"decode_iter",
"get_unbound_function", "get_unbound_function",
"maybe_add_to_os_environ_pathlist", "maybe_add_to_os_environ_pathlist",
"DefaultOrderedDict", "DefaultOrderedDict",
"deprecated",
"subprocess_Popen",
"call_subprocess_Popen",
"output_subprocess_Popen",
] ]
# In python 3.x, when an exception is reraised it saves original
# exception in its args, therefore in order to find the actual
# message, we need to unpack arguments recursively.
def exc_message(e): def exc_message(e):
"""
In python 3.x, when an exception is reraised it saves original
exception in its args, therefore in order to find the actual
message, we need to unpack arguments recursively.
"""
msg = e.args[0] msg = e.args[0]
if isinstance(msg, Exception): if isinstance(msg, Exception):
return exc_message(msg) return exc_message(msg)
...@@ -44,19 +49,6 @@ def get_unbound_function(unbound): ...@@ -44,19 +49,6 @@ def get_unbound_function(unbound):
return unbound return unbound
def decode(x):
return x.decode()
def decode_iter(itr):
for x in itr:
yield x.decode()
def decode_with(x, encoding):
return x.decode(encoding)
class DefaultOrderedDict(OrderedDict): class DefaultOrderedDict(OrderedDict):
def __init__(self, default_factory=None, *a, **kw): def __init__(self, default_factory=None, *a, **kw):
if default_factory is not None and not isinstance(default_factory, Callable): if default_factory is not None and not isinstance(default_factory, Callable):
...@@ -146,3 +138,81 @@ def deprecated(message: str = ""): ...@@ -146,3 +138,81 @@ def deprecated(message: str = ""):
return function_wrapper return function_wrapper
return decorator_wrapper return decorator_wrapper
def subprocess_Popen(command, **params):
"""
Utility function to work around windows behavior that open windows.
:see: call_subprocess_Popen and output_subprocess_Popen
"""
startupinfo = None
if os.name == "nt":
startupinfo = subprocess.STARTUPINFO()
try:
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
except AttributeError:
startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
# Anaconda for Windows does not always provide .exe files
# in the PATH, they also have .bat files that call the corresponding
# executable. For instance, "g++.bat" is in the PATH, not "g++.exe"
# Unless "shell=True", "g++.bat" is not executed when trying to
# execute "g++" without extensions.
# (Executing "g++.bat" explicitly would also work.)
params["shell"] = True
# "If shell is True, it is recommended to pass args as a string rather than as a sequence." (cite taken from https://docs.python.org/2/library/subprocess.html#frequently-used-arguments)
# In case when command arguments have spaces, passing a command as a list will result in incorrect arguments break down, and consequently
# in "The filename, directory name, or volume label syntax is incorrect" error message.
# Passing the command as a single string solves this problem.
if isinstance(command, list):
command = " ".join(command)
# Using the dummy file descriptors below is a workaround for a
# crash experienced in an unusual Python 2.4.4 Windows environment
# with the default None values.
stdin = None
if "stdin" not in params:
stdin = open(os.devnull)
params["stdin"] = stdin.fileno()
try:
proc = subprocess.Popen(command, startupinfo=startupinfo, **params)
finally:
if stdin is not None:
stdin.close()
return proc
def call_subprocess_Popen(command, **params):
"""
Calls subprocess_Popen and discards the output, returning only the
exit code.
"""
if "stdout" in params or "stderr" in params:
raise TypeError("don't use stderr or stdout with call_subprocess_Popen")
with open(os.devnull, "wb") as null:
# stdin to devnull is a workaround for a crash in a weird Windows
# environment where sys.stdin was None
params.setdefault("stdin", null)
params["stdout"] = null
params["stderr"] = null
p = subprocess_Popen(command, **params)
returncode = p.wait()
return returncode
def output_subprocess_Popen(command, **params):
"""
Calls subprocess_Popen, returning the output, error and exit code
in a tuple.
"""
if "stdout" in params or "stderr" in params:
raise TypeError("don't use stderr or stdout with output_subprocess_Popen")
params["stdout"] = subprocess.PIPE
params["stderr"] = subprocess.PIPE
p = subprocess_Popen(command, **params)
# we need to use communicate to make sure we don't deadlock around
# the stdout/stderr pipe.
out = p.communicate()
return out + (p.returncode,)
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论