Merge pull request #1908 from Tanjay94/linalg

doc/library/tensor/index.txt

@@ -25,3 +25,5 @@ They are grouped into the following sections:
     utils
     extra_ops
     io
+    slinalg
+    nlinalg

doc/library/tensor/nlinalg.txt

+..  ../../../../theano/sandbox/nlinalg.py
+
+.. _libdoc_linalg:
+
+===================================================================
+:mod:`tensor.nlinalg` --  Linear Algebra Ops Using Numpy
+===================================================================
+
+.. module:: tensor.nlinalg
+   :platform: Unix, Windows
+   :synopsis: Linear Algebra Ops Using Numpy
+.. moduleauthor:: LISA
+
+API
+===
+
+.. automodule:: theano.tensor.nlinalg
+    :members:

doc/library/tensor/slinalg.txt

+..  ../../../../theano/sandbox/slinalg.py
+
+.. _libdoc_linalg:
+
+===================================================================
+:mod:`tensor.slinalg` --  Linear Algebra Ops Using Scipy
+===================================================================
+
+.. module:: tensor.slinalg
+   :platform: Unix, Windows
+   :synopsis: Linear Algebra Ops Using Scipy
+.. moduleauthor:: LISA
+
+API
+===
+
+.. automodule:: theano.tensor.slinalg
+    :members:

theano/__init__.py

@@ -197,7 +197,6 @@ else:
 # This cannot be done in tensor/__init__.py due to a circular dependency -- randomstreams
 # depends on raw_random which depends on tensor.  As a work-around, we import RandomStreams
 # here and inject an instance in tensor.
-from theano import tensor
 from theano.tensor.randomstreams import RandomStreams
 # Imitate the numpy.random symbol with a tensor.random one
 tensor.random = RandomStreams(seed=0xBAD5EED, no_warn=True)

theano/sandbox/cuda/opt.py

@@ -42,7 +42,7 @@ from theano.sandbox.cuda.elemwise import erfinv_gpu
 from theano.sandbox.cuda.var import CudaNdarrayConstant
 from theano.scan_module import scan_utils, scan_op, scan_opt
 from theano.tensor.blas import _is_real_vector, _is_real_matrix
-linalg = None
+from theano.tensor import nlinalg
 
 #optdb.print_summary()  # shows what is currently registered
 
@@ -1643,31 +1643,26 @@ def tensor_to_cuda(x):
 
 
 @register_opt()
-@local_optimizer(None) # XXX: linalg is in sandbox, so don't import it globally
+@local_optimizer([nlinalg.ExtractDiag])
 def local_gpu_extract_diagonal(node):
     """
     extract_diagonal(host_from_gpu()) -> host_from_gpu(extract_diagonal)
     gpu_from_host(extract_diagonal) -> extract_diagonal(gpu_from_host)
     """
-    global linalg
-    if linalg is None:
-        from theano.sandbox import linalg
-        linalg = theano.sandbox.linalg
-
-    if (isinstance(node.op, linalg.ops.ExtractDiag) and
+    if (isinstance(node.op, nlinalg.ExtractDiag) and
         isinstance(node.inputs[0].type,
                    theano.tensor.TensorType)):
         inp = node.inputs[0]
         if inp.owner and isinstance(inp.owner.op, HostFromGpu):
-            return [host_from_gpu(linalg.extract_diag(gpu_from_host(inp)))]
+            return [host_from_gpu(nlinalg.extract_diag(gpu_from_host(inp)))]
     if isinstance(node.op, GpuFromHost):
         host_input = node.inputs[0]
         if (host_input.owner and
-            isinstance(host_input.owner.op, linalg.ops.ExtractDiag) and
+            isinstance(host_input.owner.op, nlinalg.ExtractDiag) and
             isinstance(host_input.owner.inputs[0].type,
                        theano.tensor.TensorType)):
             diag_node = host_input.owner
-            return [linalg.extract_diag(
+            return [nlinalg.extract_diag(
                 gpu_from_host(diag_node.inputs[0]))]
     return False
 

theano/sandbox/linalg/kron.py

-from theano import tensor
+from theano.tensor.slinalg import kron
+import warnings
 
-
-def kron(a, b):
-    """ Kronecker product
-
+warnings.warn(
+        "theano modules are deprecated and will be removed in release 0.7",
+        stacklevel=3)
\ No newline at end of file
-    Same as scipy.linalg.kron(a, b).
-
-    :note: numpy.kron(a, b) != scipy.linalg.kron(a, b)!
-        They don't have the same shape and order when
-        a.ndim != b.ndim != 2.
-
-    :param a: array_like
-    :param b: array_like
-    :return: array_like with a.ndim + b.ndim - 2 dimensions.
-
-    """
-    a = tensor.as_tensor_variable(a)
-    b = tensor.as_tensor_variable(b)
-    if (a.ndim + b.ndim <= 2):
-        raise TypeError('kron: inputs dimensions must sum to 3 or more. '
-                        'You passed %d and %d.' % (a.ndim, b.ndim))
-    o = tensor.outer(a, b)
-    o = o.reshape(tensor.concatenate((a.shape, b.shape)),
-                  a.ndim + b.ndim)
-    shf = o.dimshuffle(0, 2, 1, * range(3, o.ndim))
-    if shf.ndim == 3:
-        shf = o.dimshuffle(1, 0, 2)
-        o = shf.flatten()
-    else:
-        o = shf.reshape((o.shape[0] * o.shape[2],
-                         o.shape[1] * o.shape[3]) +
-                        tuple([o.shape[i] for i in range(4, o.ndim)]))
-    return o

theano/tensor/__init__.py

@@ -38,6 +38,7 @@ from theano.tensor.sharedvar import tensor_constructor as _shared
 
 from theano.tensor.io import *
 
+
 def shared(*args, **kw):
     """
     Backward-compatibility wrapper around `tensor._shared`.

theano/tensor/basic.py

@@ -4945,8 +4945,7 @@ class Diagonal(Op):
 
 def diagonal(a, offset=0, axis1=0, axis2=1):
     if (offset, axis1, axis2) == (0, 0, 1):
-        from theano.sandbox.linalg import extract_diag
-        return extract_diag(a)
+        return theano.tensor.nlinalg.extract_diag(a)
     return Diagonal(offset, axis1, axis2)(a)
 
 

theano/tensor/extra_ops.py

@@ -4,11 +4,11 @@ import numpy
 import theano
 
 from theano.tensor import basic
+from theano.tensor import nlinalg
 
 from theano import gof, scalar
-tensor = basic
 from theano.gradient import DisconnectedType
-
+tensor = basic
 
 class CumsumOp(theano.Op):
     # See function cumsum for docstring
@@ -727,8 +727,7 @@ class FillDiagonal(gof.Op):
                                       self.__class__.__name__)
         wr_a = fill_diagonal(grad, 0)  # valid for any number of dimensions
         # diag is only valid for matrices
-        import theano.sandbox.linalg
-        wr_val = theano.sandbox.linalg.ops.diag(grad).sum()
+        wr_val = theano.tensor.nlinalg.diag(grad).sum()
         return [wr_a, wr_val]
 fill_diagonal_ = FillDiagonal()
 

theano/tensor/tests/test_slinalg.py

+import unittest
+
+import numpy
+import numpy.linalg
+from numpy.testing import assert_array_almost_equal
+from numpy.testing import dec, assert_array_equal, assert_allclose
+from numpy import inf
+
+import theano
+from theano import tensor, function
+from theano.tensor.basic import _allclose
+from theano.tests.test_rop import break_op
+from theano.tests import unittest_tools as utt
+from theano import config
+
+from theano.tensor.slinalg import ( Cholesky,
+                                    cholesky,
+                                    CholeskyGrad,
+                                    Solve,
+                                    solve,
+                                    Eigvalsh,
+                                    EigvalshGrad,
+                                    eigvalsh
+                                    )
+
+from nose.plugins.skip import SkipTest
+from nose.plugins.attrib import attr
+from nose.tools import assert_raises
+
+try:
+    import scipy.linalg
+    imported_scipy = True
+except ImportError:
+    # some ops (e.g. Cholesky, Solve, A_Xinv_b) won't work
+    imported_scipy = False
+
+def check_lower_triangular(pd, ch_f):
+    ch = ch_f(pd)
+    assert ch[0, pd.shape[1] - 1] == 0
+    assert ch[pd.shape[0] - 1, 0] != 0
+    assert numpy.allclose(numpy.dot(ch, ch.T), pd)
+    assert not numpy.allclose(numpy.dot(ch.T, ch), pd)
+
+
+def check_upper_triangular(pd, ch_f):
+    ch = ch_f(pd)
+    assert ch[4, 0] == 0
+    assert ch[0, 4] != 0
+    assert numpy.allclose(numpy.dot(ch.T, ch), pd)
+    assert not numpy.allclose(numpy.dot(ch, ch.T), pd)
+
+
+def test_cholesky():
+    if not imported_scipy:
+        raise SkipTest("Scipy needed for the Cholesky op.")
+
+    rng = numpy.random.RandomState(utt.fetch_seed())
+    r = rng.randn(5, 5).astype(config.floatX)
+    pd = numpy.dot(r, r.T)
+    x = tensor.matrix()
+    chol = cholesky(x)
+    # Check the default.
+    ch_f = function([x], chol)
+    yield check_lower_triangular, pd, ch_f
+    # Explicit lower-triangular.
+    chol = Cholesky(lower=True)(x)
+    ch_f = function([x], chol)
+    yield check_lower_triangular, pd, ch_f
+    # Explicit upper-triangular.
+    chol = Cholesky(lower=False)(x)
+    ch_f = function([x], chol)
+    yield check_upper_triangular, pd, ch_f
+
+
+def test_cholesky_grad():
+    if not imported_scipy:
+        raise SkipTest("Scipy needed for the Cholesky op.")
+    rng = numpy.random.RandomState(utt.fetch_seed())
+    r = rng.randn(5, 5).astype(config.floatX)
+    pd = numpy.dot(r, r.T)
+    eps = None
+    if config.floatX == "float64":
+        eps = 2e-8
+    # Check the default.
+    yield (lambda: utt.verify_grad(cholesky, [pd], 3, rng, eps=eps))
+    # Explicit lower-triangular.
+    yield (lambda: utt.verify_grad(Cholesky(lower=True), [pd], 3,
+                                   rng, eps=eps))
+    # Explicit upper-triangular.
+    yield (lambda: utt.verify_grad(Cholesky(lower=False), [pd], 3,
+                                   rng, eps=eps))
+
+
+@attr('slow')
+def test_cholesky_and_cholesky_grad_shape():
+    if not imported_scipy:
+        raise SkipTest("Scipy needed for the Cholesky op.")
+
+    rng = numpy.random.RandomState(utt.fetch_seed())
+    x = tensor.matrix()
+    for l in (cholesky(x), Cholesky(lower=True)(x), Cholesky(lower=False)(x)):
+        f_chol = theano.function([x], l.shape)
+        g = tensor.grad(l.sum(), x)
+        f_cholgrad = theano.function([x], g.shape)
+        topo_chol = f_chol.maker.fgraph.toposort()
+        topo_cholgrad = f_cholgrad.maker.fgraph.toposort()
+        if config.mode != 'FAST_COMPILE':
+            assert sum([node.op.__class__ == Cholesky
+                        for node in topo_chol]) == 0
+            assert sum([node.op.__class__ == CholeskyGrad
+                        for node in topo_cholgrad]) == 0
+        for shp in [2, 3, 5]:
+            m = numpy.cov(rng.randn(shp, shp + 10)).astype(config.floatX)
+            yield numpy.testing.assert_equal, f_chol(m), (shp, shp)
+            yield numpy.testing.assert_equal, f_cholgrad(m), (shp, shp)
+
+
+
+def test_eigvalsh():
+    if not imported_scipy:
+        raise SkipTest("Scipy needed for the geigvalsh op.")
+    import scipy.linalg
+
+    A = theano.tensor.dmatrix('a')
+    B = theano.tensor.dmatrix('b')
+    f = function([A, B], eigvalsh(A, B))
+
+    rng = numpy.random.RandomState(utt.fetch_seed())
+    a = rng.randn(5, 5)
+    a = a + a.T
+    for b in [10 * numpy.eye(5, 5) + rng.randn(5, 5)]:
+        w = f(a, b)
+        refw = scipy.linalg.eigvalsh(a, b)
+        numpy.testing.assert_array_almost_equal(w, refw)
+
+    # We need to test None separatly, as otherwise DebugMode will
+    # complain, as this isn't a valid ndarray.
+    b = None
+    B = theano.tensor.NoneConst
+    f = function([A], eigvalsh(A, B))
+    w = f(a)
+    refw = scipy.linalg.eigvalsh(a, b)
+    numpy.testing.assert_array_almost_equal(w, refw)
+
+
+def test_eigvalsh_grad():
+    if not imported_scipy:
+        raise SkipTest("Scipy needed for the geigvalsh op.")
+    import scipy.linalg
+
+    rng = numpy.random.RandomState(utt.fetch_seed())
+    a = rng.randn(5, 5)
+    a = a + a.T
+    b = 10 * numpy.eye(5, 5) + rng.randn(5, 5)
+    tensor.verify_grad(lambda a, b: eigvalsh(a, b).dot([1, 2, 3, 4, 5]),
+                       [a, b], rng=numpy.random)
+
+
+class test_Solve(utt.InferShapeTester):
+    def setUp(self):
+        super(test_Solve, self).setUp()
+        self.op_class = Solve
+        self.op = Solve()
+
+    def test_infer_shape(self):
+        if not imported_scipy:
+            raise SkipTest("Scipy needed for the Cholesky op.")
+        rng = numpy.random.RandomState(utt.fetch_seed())
+        A = theano.tensor.matrix()
+        b = theano.tensor.matrix()
+        self._compile_and_check([A, b],  # theano.function inputs
+                                [self.op(A, b)],  # theano.function outputs
+                                # A must be square
+                                [numpy.asarray(rng.rand(5, 5),
+                                               dtype=config.floatX),
+                                 numpy.asarray(rng.rand(5, 1),
+                                               dtype=config.floatX)],
+                                self.op_class,
+                                warn=False)
+        rng = numpy.random.RandomState(utt.fetch_seed())
+        A = theano.tensor.matrix()
+        b = theano.tensor.vector()
+        self._compile_and_check([A, b],  # theano.function inputs
+                                [self.op(A, b)],  # theano.function outputs
+                                # A must be square
+                                [numpy.asarray(rng.rand(5, 5),
+                                               dtype=config.floatX),
+                                 numpy.asarray(rng.rand(5),
+                                               dtype=config.floatX)],
+                                self.op_class,
+                                warn=False)

theano/tensor/var.py

@@ -535,7 +535,7 @@ class _tensor_py_operators:
         return theano.tensor.basic.round(self, mode)
 
     def trace(self):
-        return theano.sandbox.linalg.trace(self)
+        return theano.tensor.nlinalg.trace(self)
 
     # TO TRUMP NUMPY OPERATORS
     __array_priority__ = 1000