Skip to content

P
pytensor
Unverified 提交 9f7cf7d0 authored 作者: Brandon T. Willard's avatar Brandon T. Willard 提交者： GitHub
浏览文件
操作

Merge pull request #221 from michaelosthege/clean-config-importing

Clean config importing
上级 819122e6 57864f52
隐藏空白字符变更
内嵌 并排
正在显示 包含 506 行增加 和 509 行删除
tests/gof/test_cmodule.py
浏览文件 @ 9f7cf7d0
......@@ -5,11 +5,13 @@ But this one tests a current behavior that isn't good: the c_code isn't
deterministic based on the input type and the op.
"""
import logging
from unittest.mock import patch
import numpy as np
import theano
from theano.gof.cmodule import GCC_compiler
from theano.gof.cmodule import GCC_compiler, default_blas_ldflags
class MyOp(theano.compile.ops.DeepCopyOp):
......@@ -68,3 +70,16 @@ def test_flag_detection():
# but was not detected because that path is not usually taken,
# so we test it here directly.
GCC_compiler.try_flags(["-lblas"])
@patch("theano.gof.cmodule.try_blas_flag", return_value=None)
@patch("theano.gof.cmodule.sys")
def test_default_blas_ldflags(sys_mock, try_blas_flag_mock, caplog):
sys_mock.version = "3.8.0 | packaged by conda-forge | (default, Nov 22 2019, 19:11:38) \n[GCC 7.3.0]"
with patch.dict("sys.modules", {"mkl": None}):
with caplog.at_level(logging.WARNING):
default_blas_ldflags()
assert "install mkl with" in caplog.text
tests/gof/test_compiledir.py deleted 100644 → 0
浏览文件 @ 819122e6
from theano.configdefaults import short_platform
def test_short_platform():
for r, p, a in [ # (release, platform, answer)
(
"3.2.0-70-generic",
"Linux-3.2.0-70-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"3.2.0-70.1-generic",
"Linux-3.2.0-70.1-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"3.2.0-70.1.2-generic",
"Linux-3.2.0-70.1.2-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"2.6.35.14-106.fc14.x86_64",
"Linux-2.6.35.14-106.fc14.x86_64-x86_64-with-fedora-14-Laughlin",
"Linux-2.6-fc14.x86_64-x86_64-with-fedora-14-Laughlin",
),
]:
o = short_platform(r, p)
assert o == a, (o, a)
tests/tensor/test_mpi.py
浏览文件 @ 9f7cf7d0
......@@ -68,8 +68,8 @@ def test_mpi_roundtrip():
)
(stdout, stderr) = p.communicate()
result = theano.utils.decode(stdout)
assert "True" in result, theano.utils.decode(stderr)
result = stdout.decode()
assert "True" in result, stderr.decode()
def test_mpi_send_wait_cmp():
......
tests/test_config.py
浏览文件 @ 9f7cf7d0
"""Test config options."""
import configparser as stdlib_configparser
import logging
from unittest.mock import patch
import pytest
from theano import configdefaults, configparser
from theano.configdefaults import default_blas_ldflags
from theano.configdefaults import short_platform
from theano.configparser import ConfigParam
......@@ -98,19 +96,6 @@ def test_invalid_default():
assert "test__test_invalid_default_b" not in root._flags_dict
@patch("theano.configdefaults.try_blas_flag", return_value=None)
@patch("theano.configdefaults.sys")
def test_default_blas_ldflags(sys_mock, try_blas_flag_mock, caplog):
sys_mock.version = "3.8.0 | packaged by conda-forge | (default, Nov 22 2019, 19:11:38) \n[GCC 7.3.0]"
with patch.dict("sys.modules", {"mkl": None}):
with caplog.at_level(logging.WARNING):
default_blas_ldflags()
assert "install mkl with" in caplog.text
def test_config_param_apply_and_validation():
cp = ConfigParam(
"TheDeFauLt",
......@@ -269,3 +254,34 @@ def test_mode_apply():
configdefaults.filter_mode(theano.compile.mode.FAST_COMPILE)
== theano.compile.mode.FAST_COMPILE
)
class TestConfigHelperFunctions:
@pytest.mark.parametrize(
"release,platform,answer",
[
(
"3.2.0-70-generic",
"Linux-3.2.0-70-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"3.2.0-70.1-generic",
"Linux-3.2.0-70.1-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"3.2.0-70.1.2-generic",
"Linux-3.2.0-70.1.2-generic-x86_64-with-debian-wheezy-sid",
"Linux-3.2--generic-x86_64-with-debian-wheezy-sid",
),
(
"2.6.35.14-106.fc14.x86_64",
"Linux-2.6.35.14-106.fc14.x86_64-x86_64-with-fedora-14-Laughlin",
"Linux-2.6-fc14.x86_64-x86_64-with-fedora-14-Laughlin",
),
],
)
def test_short_platform(self, release, platform, answer):
o = short_platform(release, platform)
assert o == answer, (o, answer)
theano/__init__.py
浏览文件 @ 9f7cf7d0
......@@ -199,3 +199,8 @@ def sparse_grad(var):
import theano.tensor.random.var
from theano.scan import checkpoints, clone, foldl, foldr, map, reduce, scan
# Some config variables are registered by submodules. Only after all those imports
# were executed, we can warn about remaining flags provided by the user through THEANO_FLAGS.
config.warn_unused_flags()
theano/configdefaults.py
浏览文件 @ 9f7cf7d0
......@@ -7,7 +7,6 @@ import socket
import struct
import sys
import textwrap
import warnings
import numpy as np
......@@ -23,8 +22,11 @@ from theano.configparser import (
IntParam,
StrParam,
)
from theano.misc.windows import call_subprocess_Popen, output_subprocess_Popen
from theano.utils import maybe_add_to_os_environ_pathlist
from theano.utils import (
call_subprocess_Popen,
maybe_add_to_os_environ_pathlist,
output_subprocess_Popen,
)
_logger = logging.getLogger("theano.configdefaults")
......@@ -178,48 +180,6 @@ def is_valid_check_preallocated_output_param(param):
return True
def check_mkl_openmp():
if not theano.config.blas__check_openmp:
return
if sys.platform == "darwin":
return
if (
"MKL_THREADING_LAYER" in os.environ
and os.environ["MKL_THREADING_LAYER"] == "GNU"
):
return
try:
import numpy._mklinit # noqa
return
except ImportError:
pass
try:
import mkl
if "2018" in mkl.get_version_string():
raise RuntimeError(
"""
To use MKL 2018 with Theano either update the numpy conda packages to
their latest build or set "MKL_THREADING_LAYER=GNU" in your
environment.
"""
)
except ImportError:
raise RuntimeError(
"""
Could not import 'mkl'. If you are using conda, update the numpy
packages to the latest build otherwise, set MKL_THREADING_LAYER=GNU in
your environment for MKL 2018.
If you have MKL 2017 install and are not in a conda environment you
can set the Theano flag blas__check_openmp to False. Be warned that if
you set this flag and don't set the appropriate environment or make
sure you have the right version you *will* get wrong results.
"""
)
def _timeout_default():
return theano.config.compile__wait * 24
......@@ -250,279 +210,6 @@ def python_int_bitwidth():
return struct.calcsize("l") * 8
def default_blas_ldflags():
"""Read local NumPy and MKL build settings and construct `ld` flags from them.
Returns
-------
str
"""
import numpy.distutils # noqa
warn_record = []
try:
if hasattr(numpy.distutils, "__config__") and numpy.distutils.__config__:
# If the old private interface is available use it as it
# don't print information to the user.
blas_info = numpy.distutils.__config__.blas_opt_info
else:
# We do this import only here, as in some setup, if we
# just import theano and exit, with the import at global
# scope, we get this error at exit: "Exception TypeError:
# "'NoneType' object is not callable" in <bound method
# Popen.__del__ of <subprocess.Popen object at 0x21359d0>>
# ignored"
# This happen with Python 2.7.3 |EPD 7.3-1 and numpy 1.8.1
# isort: off
import numpy.distutils.system_info # noqa
# We need to catch warnings as in some cases NumPy print
# stuff that we don't want the user to see.
# I'm not able to remove all printed stuff
with warnings.catch_warnings(record=True):
numpy.distutils.system_info.system_info.verbosity = 0
blas_info = numpy.distutils.system_info.get_info("blas_opt")
# If we are in a EPD installation, mkl is available
if "EPD" in sys.version:
use_unix_epd = True
if sys.platform == "win32":
return " ".join(
['-L"%s"' % os.path.join(sys.prefix, "Scripts")]
+
# Why on Windows, the library used are not the
# same as what is in
# blas_info['libraries']?
[f"-l{l}" for l in ["mk2_core", "mk2_intel_thread", "mk2_rt"]]
)
elif sys.platform == "darwin":
# The env variable is needed to link with mkl
new_path = os.path.join(sys.prefix, "lib")
v = os.getenv("DYLD_FALLBACK_LIBRARY_PATH", None)
if v is not None:
# Explicit version could be replaced by a symbolic
# link called 'Current' created by EPD installer
# This will resolve symbolic links
v = os.path.realpath(v)
# The python __import__ don't seam to take into account
# the new env variable "DYLD_FALLBACK_LIBRARY_PATH"
# when we set with os.environ['...'] = X or os.putenv()
# So we warn the user and tell him what todo.
if v is None or new_path not in v.split(":"):
_logger.warning(
"The environment variable "
"'DYLD_FALLBACK_LIBRARY_PATH' does not contain "
"the '{new_path}' path in its value. This will make "
"Theano use a slow version of BLAS. Update "
"'DYLD_FALLBACK_LIBRARY_PATH' to contain the "
"said value, this will disable this warning."
)
use_unix_epd = False
if use_unix_epd:
return " ".join(
["-L%s" % os.path.join(sys.prefix, "lib")]
+ ["-l%s" % l for l in blas_info["libraries"]]
)
# Canopy
if "Canopy" in sys.prefix:
subsub = "lib"
if sys.platform == "win32":
subsub = "Scripts"
lib_path = os.path.join(sys.base_prefix, subsub)
if not os.path.exists(lib_path):
# Old logic to find the path. I don't think we still
# need it, but I don't have the time to test all
# installation configuration. So I keep this as a fall
# back in case the current expectation don't work.
# This old logic don't work when multiple version of
# Canopy is installed.
p = os.path.join(sys.base_prefix, "..", "..", "appdata")
assert os.path.exists(p), "Canopy changed the location of MKL"
lib_paths = os.listdir(p)
# Try to remove subdir that can't contain MKL
for sub in lib_paths:
if not os.path.exists(os.path.join(p, sub, subsub)):
lib_paths.remove(sub)
assert len(lib_paths) == 1, (
"Unexpected case when looking for Canopy MKL libraries",
p,
lib_paths,
[os.listdir(os.path.join(p, sub)) for sub in lib_paths],
)
lib_path = os.path.join(p, lib_paths[0], subsub)
assert os.path.exists(lib_path), "Canopy changed the location of MKL"
if sys.platform == "linux2" or sys.platform == "darwin":
return " ".join(
["-L%s" % lib_path] + ["-l%s" % l for l in blas_info["libraries"]]
)
elif sys.platform == "win32":
return " ".join(
['-L"%s"' % lib_path]
+
# Why on Windows, the library used are not the
# same as what is in blas_info['libraries']?
[f"-l{l}" for l in ["mk2_core", "mk2_intel_thread", "mk2_rt"]]
)
# MKL
# If mkl can be imported then use it. On conda:
# "conda install mkl-service" installs the Python wrapper and
# the low-level C libraries as well as optimised version of
# numpy and scipy.
try:
import mkl # noqa
except ImportError as e:
if any([m for m in ("conda", "Continuum") if m in sys.version]):
warn_record.append(f"install mkl with `conda install mkl-service`: {e}")
else:
# This branch is executed if no exception was raised
if sys.platform == "win32":
lib_path = os.path.join(sys.prefix, "Library", "bin")
flags = [f'-L"{lib_path}"']
else:
lib_path = blas_info.get("library_dirs", [])
flags = []
if lib_path:
flags = [f"-L{lib_path[0]}"]
if "2018" in mkl.get_version_string():
thr = "mkl_gnu_thread"
else:
thr = "mkl_intel_thread"
base_flags = list(flags)
flags += [f"-l{l}" for l in ["mkl_core", thr, "mkl_rt"]]
res = try_blas_flag(flags)
if not res and sys.platform == "win32" and thr == "mkl_gnu_thread":
# Check if it would work for intel OpenMP on windows
flags = base_flags + [
f"-l{l}" for l in ["mkl_core", "mkl_intel_thread", "mkl_rt"]
]
res = try_blas_flag(flags)
if res:
check_mkl_openmp()
return res
flags.extend(["-Wl,-rpath," + l for l in blas_info.get("library_dirs", [])])
res = try_blas_flag(flags)
if res:
check_mkl_openmp()
maybe_add_to_os_environ_pathlist("PATH", lib_path[0])
return res
# to support path that includes spaces, we need to wrap it with double quotes on Windows
path_wrapper = '"' if os.name == "nt" else ""
ret = (
# TODO: the Gemm op below should separate the
# -L and -l arguments into the two callbacks
# that CLinker uses for that stuff. for now,
# we just pass the whole ldflags as the -l
# options part.
[
f"-L{path_wrapper}{l}{path_wrapper}"
for l in blas_info.get("library_dirs", [])
]
+ [f"-l{l}" for l in blas_info.get("libraries", [])]
+ blas_info.get("extra_link_args", [])
)
# For some very strange reason, we need to specify -lm twice
# to get mkl to link correctly. I have no idea why.
if any("mkl" in fl for fl in ret):
ret.extend(["-lm", "-lm"])
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
# If we are using conda and can't reuse numpy blas, then doing
# the fallback and test -lblas could give slow computation, so
# warn about this.
for warn in warn_record:
_logger.warning(warn)
del warn_record
# Some environment don't have the lib dir in LD_LIBRARY_PATH.
# So add it.
ret.extend(["-Wl,-rpath," + l for l in blas_info.get("library_dirs", [])])
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
# Add sys.prefix/lib to the runtime search path. On
# non-system installations of Python that use the
# system linker, this is generally necessary.
if sys.platform in ("linux", "darwin"):
lib_path = os.path.join(sys.prefix, "lib")
ret.append("-Wl,-rpath," + lib_path)
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
except KeyError:
pass
# Even if we could not detect what was used for numpy, or if these
# libraries are not found, most Linux systems have a libblas.so
# readily available. We try to see if that's the case, rather
# than disable blas. To test it correctly, we must load a program.
# Otherwise, there could be problem in the LD_LIBRARY_PATH.
return try_blas_flag(["-lblas"])
def try_blas_flag(flags):
from theano.gof.cmodule import GCC_compiler
test_code = textwrap.dedent(
"""\
extern "C" double ddot_(int*, double*, int*, double*, int*);
int main(int argc, char** argv)
{
int Nx = 5;
int Sx = 1;
double x[5] = {0, 1, 2, 3, 4};
double r = ddot_(&Nx, x, &Sx, x, &Sx);
if ((r - 30.) > 1e-6 || (r - 30.) < -1e-6)
{
return -1;
}
return 0;
}
"""
)
cflags = list(flags)
# to support path that includes spaces, we need to wrap it with double quotes on Windows
path_wrapper = '"' if os.name == "nt" else ""
cflags.extend(
[
f"-L{path_wrapper}{d}{path_wrapper}"
for d in theano.gof.cmodule.std_lib_dirs()
]
)
res = GCC_compiler.try_compile_tmp(
test_code, tmp_prefix="try_blas_", flags=cflags, try_run=True
)
# res[0]: shows successful compilation
# res[1]: shows successful execution
if res and res[0] and res[1]:
return " ".join(flags)
else:
return ""
def filter_vm_lazy(val):
if val == "False" or val is False:
return False
......@@ -1743,23 +1430,6 @@ def add_optimizer_configvars():
)
def add_blas_configvars():
config.add(
"blas__ldflags",
"lib[s] to include for [Fortran] level-3 blas implementation",
StrParam(default_blas_ldflags),
# Added elsewhere in the c key only when needed.
in_c_key=False,
)
config.add(
"blas__check_openmp",
"Check for openmp library conflict.\nWARNING: Setting this to False leaves you open to wrong results in blas-related operations.",
BoolParam(True),
in_c_key=False,
)
def add_metaopt_configvars():
config.add(
"metaopt__verbose",
......@@ -2243,12 +1913,9 @@ add_optimizer_configvars()
# This would mean either calling the function from there, or even moving all the related code there.
# Blas-related config are a special pain-point, because their addition depends on a lot of stuff from
# that module, which introduces a circular dependency!
add_blas_configvars()
add_metaopt_configvars()
add_vm_configvars()
add_deprecated_configvars()
# TODO: another example for config that should be added by the module itself:
add_scan_configvars()
# TODO: `gcc_version_str` is used by other modules.. Should it become an immutable config var?
try:
......@@ -2260,7 +1927,3 @@ except OSError:
# TODO: The caching dir resolution is a procedural mess of helper functions, local variables
# and config definitions. And the result is also not particularly pretty..
add_caching_dir_configvars()
# Check if there are remaining flags provided by the user through THEANO_FLAGS.
for key in config._flags_dict.keys():
warnings.warn(f"Theano does not recognise this flag: {key}")
theano/configparser.py
浏览文件 @ 9f7cf7d0
......@@ -262,6 +262,10 @@ class TheanoConfigParser:
"""
return _ChangeFlagsDecorator(*args, _root=self, **kwargs)
def warn_unused_flags(self):
for key in self._flags_dict.keys():
warnings.warn(f"Theano does not recognise this flag: {key}")
class ConfigParam:
"""Base class of all kinds of configuration parameters.
......
theano/gof/cmodule.py
浏览文件 @ 9f7cf7d0
......@@ -28,8 +28,7 @@ from theano.configdefaults import gcc_version_str, local_bitwidth
# we will abuse the lockfile mechanism when reading and writing the registry
from theano.gof import compilelock
from theano.gof.utils import flatten, hash_from_code
from theano.misc.windows import output_subprocess_Popen, subprocess_Popen
from theano.utils import decode, decode_iter
from theano.utils import output_subprocess_Popen, subprocess_Popen
importlib = None
......@@ -1786,7 +1785,7 @@ def gcc_llvm():
"""
if gcc_llvm.is_llvm is None:
try:
p_out = output_subprocess_Popen([theano.config.cxx, "--version"])
p_out = output_subprocess_Popen([config.cxx, "--version"])
output = p_out[0] + p_out[1]
except OSError:
# Typically means g++ cannot be found.
......@@ -1942,6 +1941,41 @@ class Compiler:
)
def try_blas_flag(flags):
test_code = textwrap.dedent(
"""\
extern "C" double ddot_(int*, double*, int*, double*, int*);
int main(int argc, char** argv)
{
int Nx = 5;
int Sx = 1;
double x[5] = {0, 1, 2, 3, 4};
double r = ddot_(&Nx, x, &Sx, x, &Sx);
if ((r - 30.) > 1e-6 || (r - 30.) < -1e-6)
{
return -1;
}
return 0;
}
"""
)
cflags = list(flags)
# to support path that includes spaces, we need to wrap it with double quotes on Windows
path_wrapper = '"' if os.name == "nt" else ""
cflags.extend([f"-L{path_wrapper}{d}{path_wrapper}" for d in std_lib_dirs()])
res = GCC_compiler.try_compile_tmp(
test_code, tmp_prefix="try_blas_", flags=cflags, try_run=True
)
# res[0]: shows successful compilation
# res[1]: shows successful execution
if res and res[0] and res[1]:
return " ".join(flags)
else:
return ""
def try_march_flag(flags):
"""
Try to compile and run a simple C snippet using current flags.
......@@ -1980,7 +2014,7 @@ class GCC_compiler(Compiler):
@staticmethod
def version_str():
return theano.config.cxx + " " + gcc_version_str
return config.cxx + " " + gcc_version_str
@staticmethod
def compile_args(march_flags=True):
......@@ -2008,10 +2042,10 @@ class GCC_compiler(Compiler):
break
if (
"g++" not in theano.config.cxx
and "clang++" not in theano.config.cxx
and "clang-omp++" not in theano.config.cxx
and "icpc" not in theano.config.cxx
"g++" not in config.cxx
and "clang++" not in config.cxx
and "clang-omp++" not in config.cxx
and "icpc" not in config.cxx
):
_logger.warning(
"Your Theano flag `cxx` seems not to be"
......@@ -2043,7 +2077,7 @@ class GCC_compiler(Compiler):
return None
lines = BytesIO(stdout + stderr).readlines()
lines = decode_iter(lines)
lines = (l.decode() for l in lines)
if parse:
selected_lines = []
for line in lines:
......@@ -2063,7 +2097,7 @@ class GCC_compiler(Compiler):
# The '-' at the end is needed. Otherwise, g++ do not output
# enough information.
native_lines = get_lines(f"{theano.config.cxx} -march=native -E -v -")
native_lines = get_lines(f"{config.cxx} -march=native -E -v -")
if native_lines is None:
_logger.info(
"Call to 'g++ -march=native' failed," "not setting -march flag"
......@@ -2078,7 +2112,7 @@ class GCC_compiler(Compiler):
# That means we did not select the right lines, so
# we have to report all the lines instead
reported_lines = get_lines(
f"{theano.config.cxx} -march=native -E -v -", parse=False
f"{config.cxx} -march=native -E -v -", parse=False
)
else:
reported_lines = native_lines
......@@ -2091,7 +2125,7 @@ class GCC_compiler(Compiler):
f" problem:\n {reported_lines}"
)
else:
default_lines = get_lines(f"{theano.config.cxx} -E -v -")
default_lines = get_lines(f"{config.cxx} -E -v -")
_logger.info(f"g++ default lines: {default_lines}")
if len(default_lines) < 1:
_logger.warning(
......@@ -2102,7 +2136,7 @@ class GCC_compiler(Compiler):
" functions. Please submit the following lines to"
" Theano's mailing list so that we can fix this"
" problem:\n %s",
get_lines(f"{theano.config.cxx} -E -v -", parse=False),
get_lines(f"{config.cxx} -E -v -", parse=False),
)
else:
# Some options are actually given as "-option value",
......@@ -2330,7 +2364,7 @@ class GCC_compiler(Compiler):
comp_args=True,
):
return cls._try_compile_tmp(
src_code, tmp_prefix, flags, try_run, output, theano.config.cxx, comp_args
src_code, tmp_prefix, flags, try_run, output, config.cxx, comp_args
)
@classmethod
......@@ -2344,7 +2378,7 @@ class GCC_compiler(Compiler):
comp_args=True,
):
return cls._try_flags(
flag_list, preambule, body, try_run, output, theano.config.cxx, comp_args
flag_list, preambule, body, try_run, output, config.cxx, comp_args
)
@staticmethod
......@@ -2394,7 +2428,7 @@ class GCC_compiler(Compiler):
"""
# TODO: Do not do the dlimport in this function
if not theano.config.cxx:
if not config.cxx:
raise MissingGXX("g++ not available! We can't compile c code.")
if include_dirs is None:
......@@ -2438,7 +2472,7 @@ class GCC_compiler(Compiler):
lib_filename = os.path.join(location, filepath)
_logger.debug(f"Generating shared lib {lib_filename}")
cmd = [theano.config.cxx, get_gcc_shared_library_arg(), "-g"]
cmd = [config.cxx, get_gcc_shared_library_arg(), "-g"]
if config.cmodule__remove_gxx_opt:
cmd.extend(p for p in preargs if not p.startswith("-O"))
......@@ -2472,7 +2506,7 @@ class GCC_compiler(Compiler):
try:
p_out = output_subprocess_Popen(cmd)
compile_stderr = decode(p_out[1])
compile_stderr = p_out[1].decode()
except Exception:
# An exception can occur e.g. if `g++` is not found.
print_command_line_error()
......@@ -2537,3 +2571,297 @@ class GCC_compiler(Compiler):
def icc_module_compile_str(*args):
raise NotImplementedError()
def check_mkl_openmp():
if not config.blas__check_openmp:
return
if sys.platform == "darwin":
return
if (
"MKL_THREADING_LAYER" in os.environ
and os.environ["MKL_THREADING_LAYER"] == "GNU"
):
return
try:
import numpy._mklinit # noqa
return
except ImportError:
pass
try:
import mkl
if "2018" in mkl.get_version_string():
raise RuntimeError(
"""
To use MKL 2018 with Theano either update the numpy conda packages to
their latest build or set "MKL_THREADING_LAYER=GNU" in your
environment.
"""
)
except ImportError:
raise RuntimeError(
"""
Could not import 'mkl'. If you are using conda, update the numpy
packages to the latest build otherwise, set MKL_THREADING_LAYER=GNU in
your environment for MKL 2018.
If you have MKL 2017 install and are not in a conda environment you
can set the Theano flag blas__check_openmp to False. Be warned that if
you set this flag and don't set the appropriate environment or make
sure you have the right version you *will* get wrong results.
"""
)
def default_blas_ldflags():
"""Read local NumPy and MKL build settings and construct `ld` flags from them.
Returns
-------
str
"""
import numpy.distutils # noqa
warn_record = []
try:
if hasattr(numpy.distutils, "__config__") and numpy.distutils.__config__:
# If the old private interface is available use it as it
# don't print information to the user.
blas_info = numpy.distutils.__config__.blas_opt_info
else:
# We do this import only here, as in some setup, if we
# just import theano and exit, with the import at global
# scope, we get this error at exit: "Exception TypeError:
# "'NoneType' object is not callable" in <bound method
# Popen.__del__ of <subprocess.Popen object at 0x21359d0>>
# ignored"
# This happen with Python 2.7.3 |EPD 7.3-1 and numpy 1.8.1
# isort: off
import numpy.distutils.system_info # noqa
# We need to catch warnings as in some cases NumPy print
# stuff that we don't want the user to see.
# I'm not able to remove all printed stuff
with warnings.catch_warnings(record=True):
numpy.distutils.system_info.system_info.verbosity = 0
blas_info = numpy.distutils.system_info.get_info("blas_opt")
# If we are in a EPD installation, mkl is available
if "EPD" in sys.version:
use_unix_epd = True
if sys.platform == "win32":
return " ".join(
['-L"%s"' % os.path.join(sys.prefix, "Scripts")]
+
# Why on Windows, the library used are not the
# same as what is in
# blas_info['libraries']?
[f"-l{l}" for l in ["mk2_core", "mk2_intel_thread", "mk2_rt"]]
)
elif sys.platform == "darwin":
# The env variable is needed to link with mkl
new_path = os.path.join(sys.prefix, "lib")
v = os.getenv("DYLD_FALLBACK_LIBRARY_PATH", None)
if v is not None:
# Explicit version could be replaced by a symbolic
# link called 'Current' created by EPD installer
# This will resolve symbolic links
v = os.path.realpath(v)
# The python __import__ don't seam to take into account
# the new env variable "DYLD_FALLBACK_LIBRARY_PATH"
# when we set with os.environ['...'] = X or os.putenv()
# So we warn the user and tell him what todo.
if v is None or new_path not in v.split(":"):
_logger.warning(
"The environment variable "
"'DYLD_FALLBACK_LIBRARY_PATH' does not contain "
"the '{new_path}' path in its value. This will make "
"Theano use a slow version of BLAS. Update "
"'DYLD_FALLBACK_LIBRARY_PATH' to contain the "
"said value, this will disable this warning."
)
use_unix_epd = False
if use_unix_epd:
return " ".join(
["-L%s" % os.path.join(sys.prefix, "lib")]
+ ["-l%s" % l for l in blas_info["libraries"]]
)
# Canopy
if "Canopy" in sys.prefix:
subsub = "lib"
if sys.platform == "win32":
subsub = "Scripts"
lib_path = os.path.join(sys.base_prefix, subsub)
if not os.path.exists(lib_path):
# Old logic to find the path. I don't think we still
# need it, but I don't have the time to test all
# installation configuration. So I keep this as a fall
# back in case the current expectation don't work.
# This old logic don't work when multiple version of
# Canopy is installed.
p = os.path.join(sys.base_prefix, "..", "..", "appdata")
assert os.path.exists(p), "Canopy changed the location of MKL"
lib_paths = os.listdir(p)
# Try to remove subdir that can't contain MKL
for sub in lib_paths:
if not os.path.exists(os.path.join(p, sub, subsub)):
lib_paths.remove(sub)
assert len(lib_paths) == 1, (
"Unexpected case when looking for Canopy MKL libraries",
p,
lib_paths,
[os.listdir(os.path.join(p, sub)) for sub in lib_paths],
)
lib_path = os.path.join(p, lib_paths[0], subsub)
assert os.path.exists(lib_path), "Canopy changed the location of MKL"
if sys.platform == "linux2" or sys.platform == "darwin":
return " ".join(
["-L%s" % lib_path] + ["-l%s" % l for l in blas_info["libraries"]]
)
elif sys.platform == "win32":
return " ".join(
['-L"%s"' % lib_path]
+
# Why on Windows, the library used are not the
# same as what is in blas_info['libraries']?
[f"-l{l}" for l in ["mk2_core", "mk2_intel_thread", "mk2_rt"]]
)
# MKL
# If mkl can be imported then use it. On conda:
# "conda install mkl-service" installs the Python wrapper and
# the low-level C libraries as well as optimised version of
# numpy and scipy.
try:
import mkl # noqa
except ImportError as e:
if any([m for m in ("conda", "Continuum") if m in sys.version]):
warn_record.append(f"install mkl with `conda install mkl-service`: {e}")
else:
# This branch is executed if no exception was raised
if sys.platform == "win32":
lib_path = os.path.join(sys.prefix, "Library", "bin")
flags = [f'-L"{lib_path}"']
else:
lib_path = blas_info.get("library_dirs", [])
flags = []
if lib_path:
flags = [f"-L{lib_path[0]}"]
if "2018" in mkl.get_version_string():
thr = "mkl_gnu_thread"
else:
thr = "mkl_intel_thread"
base_flags = list(flags)
flags += [f"-l{l}" for l in ["mkl_core", thr, "mkl_rt"]]
res = try_blas_flag(flags)
if not res and sys.platform == "win32" and thr == "mkl_gnu_thread":
# Check if it would work for intel OpenMP on windows
flags = base_flags + [
f"-l{l}" for l in ["mkl_core", "mkl_intel_thread", "mkl_rt"]
]
res = try_blas_flag(flags)
if res:
check_mkl_openmp()
return res
flags.extend(["-Wl,-rpath," + l for l in blas_info.get("library_dirs", [])])
res = try_blas_flag(flags)
if res:
check_mkl_openmp()
theano.utils.maybe_add_to_os_environ_pathlist("PATH", lib_path[0])
return res
# to support path that includes spaces, we need to wrap it with double quotes on Windows
path_wrapper = '"' if os.name == "nt" else ""
ret = (
# TODO: the Gemm op below should separate the
# -L and -l arguments into the two callbacks
# that CLinker uses for that stuff. for now,
# we just pass the whole ldflags as the -l
# options part.
[
f"-L{path_wrapper}{l}{path_wrapper}"
for l in blas_info.get("library_dirs", [])
]
+ [f"-l{l}" for l in blas_info.get("libraries", [])]
+ blas_info.get("extra_link_args", [])
)
# For some very strange reason, we need to specify -lm twice
# to get mkl to link correctly. I have no idea why.
if any("mkl" in fl for fl in ret):
ret.extend(["-lm", "-lm"])
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
# If we are using conda and can't reuse numpy blas, then doing
# the fallback and test -lblas could give slow computation, so
# warn about this.
for warn in warn_record:
_logger.warning(warn)
del warn_record
# Some environment don't have the lib dir in LD_LIBRARY_PATH.
# So add it.
ret.extend(["-Wl,-rpath," + l for l in blas_info.get("library_dirs", [])])
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
# Add sys.prefix/lib to the runtime search path. On
# non-system installations of Python that use the
# system linker, this is generally necessary.
if sys.platform in ("linux", "darwin"):
lib_path = os.path.join(sys.prefix, "lib")
ret.append("-Wl,-rpath," + lib_path)
res = try_blas_flag(ret)
if res:
if "mkl" in res:
check_mkl_openmp()
return res
except KeyError:
pass
# Even if we could not detect what was used for numpy, or if these
# libraries are not found, most Linux systems have a libblas.so
# readily available. We try to see if that's the case, rather
# than disable blas. To test it correctly, we must load a program.
# Otherwise, there could be problem in the LD_LIBRARY_PATH.
return try_blas_flag(["-lblas"])
def add_blas_configvars():
config.add(
"blas__ldflags",
"lib[s] to include for [Fortran] level-3 blas implementation",
theano.configparser.StrParam(default_blas_ldflags),
# Added elsewhere in the c key only when needed.
in_c_key=False,
)
config.add(
"blas__check_openmp",
"Check for openmp library conflict.\nWARNING: Setting this to False leaves you open to wrong results in blas-related operations.",
theano.configparser.BoolParam(True),
in_c_key=False,
)
# Register config parameters that are specific to this module:
add_blas_configvars()
theano/misc/elemwise_openmp_speedup.py
浏览文件 @ 9f7cf7d0
......@@ -5,7 +5,6 @@ from locale import getpreferredencoding
from optparse import OptionParser
import theano
from theano.utils import decode_with
console_encoding = getpreferredencoding()
......@@ -37,7 +36,7 @@ def runScript(N):
if err:
print(err)
sys.exit()
return list(map(float, decode_with(out, console_encoding).split(" ")))
return list(map(float, out.decode(console_encoding).split(" ")))
if __name__ == "__main__":
......
theano/misc/windows.py deleted 100644 → 0
浏览文件 @ 819122e6
import os
import subprocess
def subprocess_Popen(command, **params):
"""
Utility function to work around windows behavior that open windows.
:see: call_subprocess_Popen and output_subprocess_Popen
"""
startupinfo = None
if os.name == "nt":
startupinfo = subprocess.STARTUPINFO()
try:
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
except AttributeError:
startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
# Anaconda for Windows does not always provide .exe files
# in the PATH, they also have .bat files that call the corresponding
# executable. For instance, "g++.bat" is in the PATH, not "g++.exe"
# Unless "shell=True", "g++.bat" is not executed when trying to
# execute "g++" without extensions.
# (Executing "g++.bat" explicitly would also work.)
params["shell"] = True
# "If shell is True, it is recommended to pass args as a string rather than as a sequence." (cite taken from https://docs.python.org/2/library/subprocess.html#frequently-used-arguments)
# In case when command arguments have spaces, passing a command as a list will result in incorrect arguments break down, and consequently
# in "The filename, directory name, or volume label syntax is incorrect" error message.
# Passing the command as a single string solves this problem.
if isinstance(command, list):
command = " ".join(command)
# Using the dummy file descriptors below is a workaround for a
# crash experienced in an unusual Python 2.4.4 Windows environment
# with the default None values.
stdin = None
if "stdin" not in params:
stdin = open(os.devnull)
params["stdin"] = stdin.fileno()
try:
proc = subprocess.Popen(command, startupinfo=startupinfo, **params)
finally:
if stdin is not None:
stdin.close()
return proc
def call_subprocess_Popen(command, **params):
"""
Calls subprocess_Popen and discards the output, returning only the
exit code.
"""
if "stdout" in params or "stderr" in params:
raise TypeError("don't use stderr or stdout with call_subprocess_Popen")
with open(os.devnull, "wb") as null:
# stdin to devnull is a workaround for a crash in a weird Windows
# environment where sys.stdin was None
params.setdefault("stdin", null)
params["stdout"] = null
params["stderr"] = null
p = subprocess_Popen(command, **params)
returncode = p.wait()
return returncode
def output_subprocess_Popen(command, **params):
"""
Calls subprocess_Popen, returning the output, error and exit code
in a tuple.
"""
if "stdout" in params or "stderr" in params:
raise TypeError("don't use stderr or stdout with output_subprocess_Popen")
params["stdout"] = subprocess.PIPE
params["stderr"] = subprocess.PIPE
p = subprocess_Popen(command, **params)
# we need to use communicate to make sure we don't deadlock around
# the stdout/stderr pipe.
out = p.communicate()
return out + (p.returncode,)
theano/scan/__init__.py
浏览文件 @ 9f7cf7d0
......@@ -47,6 +47,11 @@ __authors__ = (
__copyright__ = "(c) 2010, Universite de Montreal"
__contact__ = "Razvan Pascanu <r.pascanu@gmail>"
from theano import configdefaults
configdefaults.add_scan_configvars()
from theano.scan import opt
from theano.scan.basic import scan
from theano.scan.checkpoints import scan_checkpoints
......
theano/utils.py
浏览文件 @ 9f7cf7d0
"""Utility functions for Theano."""
"""Utility functions that only depend on the standard library."""
import inspect
import os
import subprocess
import traceback
import warnings
from collections import OrderedDict
......@@ -11,19 +13,22 @@ from functools import wraps
__all__ = [
"cmp",
"decode",
"decode_with",
"decode_iter",
"get_unbound_function",
"maybe_add_to_os_environ_pathlist",
"DefaultOrderedDict",
"deprecated",
"subprocess_Popen",
"call_subprocess_Popen",
"output_subprocess_Popen",
]
# In python 3.x, when an exception is reraised it saves original
# exception in its args, therefore in order to find the actual
# message, we need to unpack arguments recursively.
def exc_message(e):
"""
In python 3.x, when an exception is reraised it saves original
exception in its args, therefore in order to find the actual
message, we need to unpack arguments recursively.
"""
msg = e.args[0]
if isinstance(msg, Exception):
return exc_message(msg)
......@@ -44,19 +49,6 @@ def get_unbound_function(unbound):
return unbound
def decode(x):
return x.decode()
def decode_iter(itr):
for x in itr:
yield x.decode()
def decode_with(x, encoding):
return x.decode(encoding)
class DefaultOrderedDict(OrderedDict):
def __init__(self, default_factory=None, *a, **kw):
if default_factory is not None and not isinstance(default_factory, Callable):
......@@ -146,3 +138,81 @@ def deprecated(message: str = ""):
return function_wrapper
return decorator_wrapper
def subprocess_Popen(command, **params):
"""
Utility function to work around windows behavior that open windows.
:see: call_subprocess_Popen and output_subprocess_Popen
"""
startupinfo = None
if os.name == "nt":
startupinfo = subprocess.STARTUPINFO()
try:
startupinfo.dwFlags |= subprocess.STARTF_USESHOWWINDOW
except AttributeError:
startupinfo.dwFlags |= subprocess._subprocess.STARTF_USESHOWWINDOW
# Anaconda for Windows does not always provide .exe files
# in the PATH, they also have .bat files that call the corresponding
# executable. For instance, "g++.bat" is in the PATH, not "g++.exe"
# Unless "shell=True", "g++.bat" is not executed when trying to
# execute "g++" without extensions.
# (Executing "g++.bat" explicitly would also work.)
params["shell"] = True
# "If shell is True, it is recommended to pass args as a string rather than as a sequence." (cite taken from https://docs.python.org/2/library/subprocess.html#frequently-used-arguments)
# In case when command arguments have spaces, passing a command as a list will result in incorrect arguments break down, and consequently
# in "The filename, directory name, or volume label syntax is incorrect" error message.
# Passing the command as a single string solves this problem.
if isinstance(command, list):
command = " ".join(command)
# Using the dummy file descriptors below is a workaround for a
# crash experienced in an unusual Python 2.4.4 Windows environment
# with the default None values.
stdin = None
if "stdin" not in params:
stdin = open(os.devnull)
params["stdin"] = stdin.fileno()
try:
proc = subprocess.Popen(command, startupinfo=startupinfo, **params)
finally:
if stdin is not None:
stdin.close()
return proc
def call_subprocess_Popen(command, **params):
"""
Calls subprocess_Popen and discards the output, returning only the
exit code.
"""
if "stdout" in params or "stderr" in params:
raise TypeError("don't use stderr or stdout with call_subprocess_Popen")
with open(os.devnull, "wb") as null:
# stdin to devnull is a workaround for a crash in a weird Windows
# environment where sys.stdin was None
params.setdefault("stdin", null)
params["stdout"] = null
params["stderr"] = null
p = subprocess_Popen(command, **params)
returncode = p.wait()
return returncode
def output_subprocess_Popen(command, **params):
"""
Calls subprocess_Popen, returning the output, error and exit code
in a tuple.
"""
if "stdout" in params or "stderr" in params:
raise TypeError("don't use stderr or stdout with output_subprocess_Popen")
params["stdout"] = subprocess.PIPE
params["stderr"] = subprocess.PIPE
p = subprocess_Popen(command, **params)
# we need to use communicate to make sure we don't deadlock around
# the stdout/stderr pipe.
out = p.communicate()
return out + (p.returncode,)
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论