Merge pull request #639 from nouiz/ifelse

NEWS.txt

@@ -88,6 +88,8 @@ New Features
  * Garbage collection of intermediate results during Theano function calls
    for Ops with C code (Pascal L.)
  * Theano flags compiledir_format now support the parameter numpy_version.
+ * Theano GPU variables, shared variable and constant now support <, <=,
+   > and >= as as those not on the GPU.
 
 Sparse
  * Implement theano.sparse.mul(sparse1, sparse2) when both inputs don't

theano/ifelse.py

@@ -340,7 +340,7 @@ def ifelse(condition, then_branch, else_branch, name=None):
 
             if then_branch_elem.type != else_branch_elem.type:
                 # If the types still don't match, there is a problem.
-                raise ValueError(
+                raise TypeError(
                         'The two branches should have identical types, but '
                         'they are %s and %s respectively. This error could be '
                         'raised if for example you provided a one element '

theano/sandbox/cuda/opt.py

@@ -11,6 +11,7 @@ import theano
 from theano.scan_module import scan_utils, scan_op, scan_opt
 from theano import scalar as scal
 from theano import tensor, compile, gof
+import theano.ifelse
 
 from theano.compile import optdb
 from theano.gof import (local_optimizer, EquilibriumDB, SequenceDB, ProxyDB,
@@ -366,36 +367,47 @@ def local_gpu_lazy_ifelse(node):
 
     ifelse(host_from_gpu) -> host_from_gpu(ifelse)
     """
-    if hasattr(theano, "lazycond"):
-        gpu_ifelse = theano.lazycond.IfElse(gpu=True)
-
-        if node.op == gpu_from_host:
-            host_input = node.inputs[0]
-            if (host_input.owner
-                    and host_input.owner.op == theano.lazycond.ifelse):
-                c, t, f = host_input.owner.inputs
-                if not isinstance(f.type, CudaNdarrayType):
-                    f = gpu_from_host(f)
-                if not isinstance(t.type, CudaNdarrayType):
-                    t = gpu_from_host(t)
-                if isinstance(c.type, CudaNdarrayType):
-                    c = host_from_gpu(c)
-
-                return [gpu_ifelse(c, t, f)]
-
-        if node.op == theano.lazycond.ifelse:
-            if numpy.any([(i.owner and i.owner.op == host_from_gpu)
-                          for i in node.inputs]):
-                c, t, f = node.inputs
-
-                if not isinstance(f.type, CudaNdarrayType):
-                    f = gpu_from_host(f)
-                if not isinstance(t.type, CudaNdarrayType):
-                    t = gpu_from_host(t)
-                if isinstance(c.type, CudaNdarrayType):
-                    c = host_from_gpu(c)
-
-                return [host_from_gpu(gpu_ifelse(c, t, f))]
+    if isinstance(node.op, theano.ifelse.IfElse) and not node.op.gpu:
+        gpu_ifelse = theano.ifelse.IfElse(node.op.n_outs, gpu=True)
+        outs_clients = reduce(list.__add__,
+                              [out.clients for out in node.outputs])
+        if numpy.any([(i.owner and i.owner.op == host_from_gpu)
+                      for i in node.inputs]) or numpy.any(
+                      [c != 'output' and c.op == gpu_from_host for c, idx
+                       in outs_clients]):
+
+            c = node.inputs[0]
+            outs = node.inputs[1:]
+            # Should not happen, but just in case
+            if isinstance(c.type, CudaNdarrayType):
+                c = host_from_gpu(c)
+
+            for i in range(len(outs)):
+                if not isinstance(outs[i], CudaNdarrayType):
+                    outs[i] = gpu_from_host(outs[i])
+            return [host_from_gpu(out) for out in
+                    gpu_ifelse.make_node(c, *outs).outputs]
+
+    if node.op == gpu_from_host:
+        host_input = node.inputs[0]
+        if (host_input.owner and
+            isinstance(host_input.owner.op, theano.ifelse.IfElse) and
+            not host_input.owner.op.gpu):
+            gpu_ifelse = theano.ifelse.IfElse(host_input.owner.op.n_outs,
+                                                  gpu=True)
+
+            c = host_input.owner.inputs[0]
+            outs = host_input.owner.inputs[1:]
+            # Should not happen, but just in case
+            if isinstance(c.type, CudaNdarrayType):
+                c = host_from_gpu(c)
+
+            for i in range(len(outs)):
+                if not isinstance(outs[i], CudaNdarrayType):
+                    outs[i] = gpu_from_host(outs[i])
+
+            outs = gpu_ifelse.make_node(c, *outs).outputs
+            return outs
 
     return False
 
@@ -1306,11 +1318,11 @@ def local_gpualloc(node):
         if node.inputs[0].owner and \
            node.inputs[0].owner.op == host_from_gpu:
             replace = True
-        if all([c != 'output' and c.op == gpu_from_host
+        elif all([c != 'output' and c.op == gpu_from_host
                 for c, idx in node.outputs[0].clients]):
             # if all clients are on gpu
             replace = True
-        if all([c != 'output' and
+        elif all([c != 'output' and
                 c.op == tensor.join and
                 all([i.owner and
                      i.owner.op in [host_from_gpu, tensor.alloc]

theano/sandbox/cuda/tests/test_memory.py

@@ -5,6 +5,7 @@ import numpy as np
 import theano
 from theano import tensor
 from theano.sandbox import cuda
+from theano import ifelse
 
 # Skip test if cuda_ndarray is not available.
 from nose.plugins.skip import SkipTest
@@ -112,3 +113,65 @@ def test_memory():
         del derp, variables, grad_derp
         print "After deleting shared variable and ref to it", freemem()
         assert mem1 == freemem(), (mem1, freemem())
+
+
+def test_memory_lazy():
+    """As test_memory, but with the ifelse op.
+
+    We need to test it as the ifelse op with the [c]vm create op not
+    executed in the graph. This mess with [c]vm gc implementation.
+    """
+    shapes = (200, 100)
+    # more_alloc1 and more_alloc2 is not the same for both dtype.
+    # when dtype is float32, the computation is done on the gpu.
+    # This insert constant on the gpu during compilation
+    # that raise the number of alloc.
+    # When dtype is float64, only the shared is on the gpu and it is transferd
+    # to the cpu for computation. So no extra alloc after compilation.
+    # more_alloc1 if after the first compilation, more_alloc2 after the second.
+    for dtype, more_alloc1 in [("float32", 3),
+                               ("float64", 0)]:
+        print dtype
+        test_params = np.asarray(np.random.randn(np.prod(shapes)), dtype)
+
+        some_vector = tensor.vector('some_vector', dtype=dtype)
+        some_matrix = some_vector.reshape(shapes)
+        branch_select = tensor.iscalar()
+
+        mem1 = freemem()
+        print "Before shared variable", mem1
+        variables = cuda.shared_constructor(np.ones((shapes[1],),
+                                                    dtype='float32'))
+        derp = tensor.sum(tensor.dot(some_matrix[:shapes[0]], variables))
+        derp = ifelse.IfElse(1)(branch_select,
+                                derp, some_matrix[:shapes[0]].sum())
+        derp += 1
+        print "Shared took ", np.prod(variables.get_value(
+                borrow=True,
+                return_internal_type=True).shape) * 4 / 1024, "kB"
+
+        mem2 = freemem()
+        print "Before compilation", mem2
+        mem2_1 = freemem(extra_alloc=more_alloc1)
+        obj = theano.function([some_vector, branch_select], derp,
+                              mode=mode_with_gpu)
+        #theano.printing.debugprint(obj, print_type=True)
+        mem3 = freemem()
+        print "After function compilation 1", mem3
+        assert mem2_1 == mem3, (mem2_1, mem3)
+
+        for i in range(3):
+            obj(test_params, 1)
+            print "After function evaluation branch true", freemem()
+            assert mem2_1 == freemem(), (mem2_1, freemem())
+            obj(test_params, 0)
+            print "After function evaluation branch false", freemem()
+            assert mem2_1 == freemem(), (mem2_1, freemem())
+
+        del obj
+        print "After deleting function 1", freemem()
+        assert mem2 == freemem(), (mem2, freemem())
+
+        del derp, variables
+        print "After deleting shared variable and ref to it", freemem()
+        assert mem1 == freemem(), (mem1, freemem())

theano/sandbox/cuda/tests/test_opt.py

@@ -11,6 +11,8 @@ import theano
 from theano.tests import unittest_tools as utt
 
 import theano.sandbox.cuda as cuda
+from theano.sandbox.cuda import basic_ops
+
 if cuda.cuda_available == False:
     raise SkipTest('Optional package cuda disabled')
 
@@ -330,6 +332,18 @@ class test_local_gpu_tensordot(unittest.TestCase):
         f3(tensor1, tensor3)
 
 
+import theano.tests.test_ifelse
+
+
+class TestIfElse(theano.tests.test_ifelse.test_ifelse):
+    dtype = "float32"
+    mode = mode_with_gpu
+    cast_output = staticmethod(basic_ops.as_cuda_ndarray_variable)
+    shared = staticmethod(cuda.shared_constructor)
+
+    def get_ifelse(self, n):
+        return theano.ifelse.IfElse(n, gpu=True, as_view=True)
+
 if __name__ == '__main__':
     test_gpualloc()
     test_opt_gpujoin_onlyajoin()

theano/sandbox/cuda/tests/test_var.py

@@ -5,7 +5,6 @@ from nose.plugins.skip import SkipTest
 import theano
 from theano import tensor
 
-from theano.ifelse import ifelse
 from theano import sparse
 from theano.tensor import TensorType
 from theano.tests import unittest_tools as utt
@@ -91,77 +90,3 @@ class T_updates(unittest.TestCase):
         output_func = theano.function(inputs=[], outputs=[],
                 updates={output_var: output_var.sum().dimshuffle('x', 'x')})
         output_func()
-
-
-class T_ifelse(unittest.TestCase):
-    def setUp(self):
-        utt.seed_rng()
-        self.rng = numpy.random.RandomState(seed=utt.fetch_seed())
-
-    def test_cuda_tensor(self):
-        data = self.rng.rand(4).astype('float32')
-        x = f32sc(data)
-        y = x + 1
-        cond = theano.tensor.iscalar('cond')
-
-        assert isinstance(x.type, CudaNdarrayType)
-        assert isinstance(y.type, TensorType)
-
-        out1 = ifelse(cond, x, y)
-        out2 = ifelse(cond, y, x)
-
-        assert isinstance(out1.type, TensorType)
-        assert isinstance(out2.type, TensorType)
-
-        f = theano.function([cond], out1)
-        g = theano.function([cond], out2)
-
-        assert numpy.all(f(0) == data + 1)
-        assert numpy.all(f(1) == data)
-        assert numpy.all(g(0) == data)
-        assert numpy.all(g(1) == data + 1)
-
-    def test_dtype_mismatch(self):
-        data = self.rng.rand(5).astype('float32')
-        x = f32sc(data)
-        y = tensor.cast(x, 'float64')
-        cond = theano.tensor.iscalar('cond')
-
-        self.assertRaises(TypeError, ifelse, cond, x, y)
-        self.assertRaises(TypeError, ifelse, cond, y, x)
-
-    def test_ndim_mismatch(self):
-        data = self.rng.rand(5).astype('float32')
-        x = f32sc(data)
-        y = tensor.fcol('y')
-        cond = theano.tensor.iscalar('cond')
-
-        self.assertRaises(TypeError, ifelse, cond, x, y)
-        self.assertRaises(TypeError, ifelse, cond, y, x)
-
-    def test_broadcast_mismatch(self):
-        data = self.rng.rand(2, 3).astype('float32')
-        x = f32sc(data)
-        print x.broadcastable
-        y = tensor.frow('y')
-        print y.broadcastable
-        cond = theano.tensor.iscalar('cond')
-
-        self.assertRaises(TypeError, ifelse, cond, x, y)
-        self.assertRaises(TypeError, ifelse, cond, y, x)
-
-    def test_sparse_tensor_error(self):
-        data = self.rng.rand(2, 3).astype('float32')
-        x = f32sc(data)
-        y = sparse.matrix('csc', dtype='float32', name='y')
-        z = sparse.matrix('csr', dtype='float32', name='z')
-        cond = theano.tensor.iscalar('cond')
-
-        # Right now (2012-01-19), a ValueError gets raised, but I thing
-        # a TypeError (like in the other cases) would be fine.
-        self.assertRaises((TypeError, ValueError), ifelse, cond, x, y)
-        self.assertRaises((TypeError, ValueError), ifelse, cond, y, x)
-        self.assertRaises((TypeError, ValueError), ifelse, cond, x, z)
-        self.assertRaises((TypeError, ValueError), ifelse, cond, z, x)
-        self.assertRaises((TypeError, ValueError), ifelse, cond, y, z)
-        self.assertRaises((TypeError, ValueError), ifelse, cond, z, y)

theano/sandbox/cuda/var.py

@@ -36,14 +36,14 @@ class _operators(tensor.basic._tensor_py_operators):
     ndim = property(lambda s:s.type.ndim)
 
 
-class CudaNdarrayVariable(Variable, _operators):
+class CudaNdarrayVariable(_operators, Variable):
     pass
 CudaNdarrayType.Variable = CudaNdarrayVariable
 
 class CudaNdarrayConstantSignature(tensor.TensorConstantSignature):
     pass
 
-class CudaNdarrayConstant(Constant, _operators):
+class CudaNdarrayConstant(_operators, Constant):
     def signature(self):
         return CudaNdarrayConstantSignature((self.type, numpy.asarray(self.data)))
     def __str__(self):
@@ -52,7 +52,7 @@ class CudaNdarrayConstant(Constant, _operators):
         return "CudaNdarrayConstant{"+str(numpy.asarray(self.data))+"}"
 CudaNdarrayType.Constant = CudaNdarrayConstant
 
-class CudaNdarraySharedVariable(SharedVariable, _operators):
+class CudaNdarraySharedVariable(_operators, SharedVariable):
     """
     Shared Variable interface to CUDA-allocated arrays
     """

theano/tensor/tests/test_sharedvar.py

@@ -432,7 +432,7 @@ def makeSharedTester(shared_constructor_,
             else:
                 shape_grad = tensor.grad(x1_specify_shape.sum(), x1_shared)
                 shape_constant_fct_grad = theano.function([], shape_grad)
-                theano.printing.debugprint(shape_constant_fct_grad)
+                #theano.printing.debugprint(shape_constant_fct_grad)
                 shape_constant_fct_grad()
 
             #Test that we can replace with values of the different shape

theano/tests/test_ifelse.py

@@ -13,48 +13,40 @@ from nose.plugins.skip import SkipTest
 
 import theano
 from theano import tensor
+import theano.ifelse
 from theano.ifelse import IfElse, ifelse
 from theano.tests  import unittest_tools as utt
 
 
-class test_ifelse(unittest.TestCase):
+class test_ifelse(unittest.TestCase, utt.TestOptimizationMixin):
+    mode = None
+    dtype = theano.config.floatX
+    cast_output = staticmethod(tensor.as_tensor_variable)
+    shared = staticmethod(theano.shared)
+
+    def get_ifelse(self, n):
+        if theano.config.mode == "FAST_COMPILE":
+            return IfElse(n)
+        else:
+            return IfElse(n, as_view=True)
+
     def test_lazy_if(self):
         # Tests that lazy if works .. even if the two results have different
         # shapes but the same type (i.e. both vectors, or matrices or
         # whatnot of same dtype)
-        x = tensor.vector('x')
-        y = tensor.vector('y')
+        x = tensor.vector('x', dtype=self.dtype)
+        y = tensor.vector('y', dtype=self.dtype)
         c = tensor.iscalar('c')
-        f = theano.function([c, x, y], ifelse(c, x, y))
+        f = theano.function([c, x, y], ifelse(c, x, y), mode=self.mode)
+        self.assertFunctionContains1(f, self.get_ifelse(1))
         rng = numpy.random.RandomState(utt.fetch_seed())
 
         xlen = rng.randint(200)
         ylen = rng.randint(200)
 
-        vx = numpy.asarray(rng.uniform(size=(xlen,)), theano.config.floatX)
-        vy = numpy.asarray(rng.uniform(size=(ylen,)), theano.config.floatX)
-
-        assert numpy.allclose(vx, f(1, vx, vy))
-        assert numpy.allclose(vy, f(0, vx, vy))
-
-    def test_lazy_if_inplace(self):
-        # Tests that lazy if works inplace
-        x = tensor.vector('x')
-        y = tensor.vector('y')
-        c = tensor.iscalar('c')
-        f = theano.function([c, x, y], ifelse(c, x, y))
-        rng = numpy.random.RandomState(utt.fetch_seed())
-
-        xlen = rng.randint(200)
-        ylen = rng.randint(200)
+        vx = numpy.asarray(rng.uniform(size=(xlen,)), self.dtype)
+        vy = numpy.asarray(rng.uniform(size=(ylen,)), self.dtype)
 
-        vx = numpy.asarray(rng.uniform(size=(xlen,)), theano.config.floatX)
-        vy = numpy.asarray(rng.uniform(size=(ylen,)), theano.config.floatX)
-        if theano.config.mode != "FAST_COMPILE":
-            assert numpy.all([x.op.as_view for x in f.maker.env.toposort() if
-                              isinstance(x.op, IfElse)])
-        assert len([x.op for x in f.maker.env.toposort()
-                   if isinstance(x.op, IfElse)]) > 0
         assert numpy.allclose(vx, f(1, vx, vy))
         assert numpy.allclose(vy, f(0, vx, vy))
 
@@ -71,31 +63,150 @@ class test_ifelse(unittest.TestCase):
 
     def test_grad_lazy_if(self):
         # Tests that we can compute the gradients through lazy if
-        x = tensor.vector('x')
-        y = tensor.vector('y')
+        x = tensor.vector('x', dtype=self.dtype)
+        y = tensor.vector('y', dtype=self.dtype)
         c = tensor.iscalar('c')
         z = ifelse(c, x, y)
         gx, gy = tensor.grad(z.sum(), [x, y])
 
-        f = theano.function([c, x, y], [gx, gy])
+        f = theano.function([c, x, y], [self.cast_output(gx),
+                                        self.cast_output(gy)],
+                            mode=self.mode)
+        # There is only 2 of the 3 ifelse that are moved on the GPU.
+        # The one that stay on the CPU is for the shape.
+        self.assertFunctionContains(f, self.get_ifelse(1), min=2, max=3)
         rng = numpy.random.RandomState(utt.fetch_seed())
 
         xlen = rng.randint(200)
         ylen = rng.randint(200)
 
-        vx = numpy.asarray(rng.uniform(size=(xlen,)), theano.config.floatX)
-        vy = numpy.asarray(rng.uniform(size=(ylen,)), theano.config.floatX)
+        vx = numpy.asarray(rng.uniform(size=(xlen,)), self.dtype)
+        vy = numpy.asarray(rng.uniform(size=(ylen,)), self.dtype)
         gx0, gy0 = f(1, vx, vy)
         assert numpy.allclose(gx0.shape, vx.shape)
         assert numpy.allclose(gy0.shape, vy.shape)
-        assert numpy.all(gx0 == 1.)
-        assert numpy.all(gy0 == 0.)
+        assert numpy.all(numpy.asarray(gx0) == 1.)
+        assert numpy.all(numpy.asarray(gy0) == 0.)
 
         gx0, gy0 = f(0, vx, vy)
         assert numpy.allclose(gx0.shape, vx.shape)
         assert numpy.allclose(gy0.shape, vy.shape)
-        assert numpy.all(gx0 == 0.)
-        assert numpy.all(gy0 == 1.)
+        assert numpy.all(numpy.asarray(gx0) == 0.)
+        assert numpy.all(numpy.asarray(gy0) == 1.)
+
+    def test_multiple_out(self):
+        x1 = tensor.vector('x1', dtype=self.dtype)
+        x2 = tensor.vector('x2', dtype=self.dtype)
+        y1 = tensor.vector('y1', dtype=self.dtype)
+        y2 = tensor.vector('y2', dtype=self.dtype)
+        c = tensor.iscalar('c')
+        z = ifelse(c, (x1, x2), (y1, y2))
+        f = theano.function([c, x1, x2, y1, y2], z, mode=self.mode)
+        self.assertFunctionContains1(f, self.get_ifelse(2))
+
+        ifnode = [x for x in f.maker.env.toposort()
+                  if isinstance(x.op, IfElse)][0]
+        assert len(ifnode.outputs) == 2
+
+        rng = numpy.random.RandomState(utt.fetch_seed())
+
+        x1len = rng.randint(200)
+        x2len = rng.randint(200)
+        y1len = rng.randint(200)
+        y2len = rng.randint(200)
+
+        vx1 = numpy.asarray(rng.uniform(size=(x1len,)), self.dtype)
+        vx2 = numpy.asarray(rng.uniform(size=(x2len,)), self.dtype)
+        vy1 = numpy.asarray(rng.uniform(size=(y1len,)), self.dtype)
+        vy2 = numpy.asarray(rng.uniform(size=(y2len,)), self.dtype)
+
+        ovx1, ovx2 = f(1, vx1, vx2, vy1, vy2)
+        ovy1, ovy2 = f(0, vx1, vx2, vy1, vy2)
+        assert numpy.allclose(vx1, ovx1)
+        assert numpy.allclose(vy1, ovy1)
+        assert numpy.allclose(vx2, ovx2)
+        assert numpy.allclose(vy2, ovy2)
+
+    def test_multiple_out_grad(self):
+        # Tests that we can compute the gradients through lazy if
+        x1 = tensor.vector('x1')
+        x2 = tensor.vector('x2')
+        y1 = tensor.vector('y1')
+        y2 = tensor.vector('y2')
+        c = tensor.iscalar('c')
+        z = ifelse(c, (x1, x2), (y1, y2))
+        grads = tensor.grad(z[0].sum() + z[1].sum(),
+                            [x1, x2, y1, y2])
+
+        f = theano.function([c, x1, x2, y1, y2], grads)
+        rng = numpy.random.RandomState(utt.fetch_seed())
+
+        lens = [rng.randint(200) for i in range(4)]
+        values = [numpy.asarray(rng.uniform(size=(l,)), theano.config.floatX)
+                  for l in lens]
+        outs_1 = f(1, *values)
+        assert all([x.shape[0] == y for x, y in zip(outs_1, lens)])
+        assert numpy.all(outs_1[0] == 1.)
+        assert numpy.all(outs_1[1] == 1.)
+        assert numpy.all(outs_1[2] == 0.)
+        assert numpy.all(outs_1[3] == 0.)
+
+        outs_0 = f(0, *values)
+        assert all([x.shape[0] == y for x, y in zip(outs_1, lens)])
+        assert numpy.all(outs_0[0] == 0.)
+        assert numpy.all(outs_0[1] == 0.)
+        assert numpy.all(outs_0[2] == 1.)
+        assert numpy.all(outs_0[3] == 1.)
+
+    def test_dtype_mismatch(self):
+        rng = numpy.random.RandomState(utt.fetch_seed())
+        data = rng.rand(5).astype(self.dtype)
+        x = self.shared(data)
+        y = tensor.cast(x * 10, 'int8')
+        cond = theano.tensor.iscalar('cond')
+
+        self.assertRaises(TypeError, ifelse, cond, x, y)
+        self.assertRaises(TypeError, ifelse, cond, y, x)
+
+    def test_ndim_mismatch(self):
+        rng = numpy.random.RandomState(utt.fetch_seed())
+        data = rng.rand(5).astype(self.dtype)
+        x = self.shared(data)
+        y = tensor.col('y', self.dtype)
+        cond = theano.tensor.iscalar('cond')
+
+        self.assertRaises(TypeError, ifelse, cond, x, y)
+        self.assertRaises(TypeError, ifelse, cond, y, x)
+
+    def test_broadcast_mismatch(self):
+        rng = numpy.random.RandomState(utt.fetch_seed())
+        data = rng.rand(5).astype(self.dtype)
+        x = self.shared(data)
+        #print x.broadcastable
+        y = tensor.row('y', self.dtype)
+        #print y.broadcastable
+        cond = theano.tensor.iscalar('cond')
+
+        self.assertRaises(TypeError, ifelse, cond, x, y)
+        self.assertRaises(TypeError, ifelse, cond, y, x)
+
+    def test_sparse_tensor_error(self):
+        import theano.sparse
+        if not theano.sparse.enable_sparse:
+            raise SkipTest("Optimization temporarily disabled")
+        rng = numpy.random.RandomState(utt.fetch_seed())
+        data = rng.rand(2, 3).astype(self.dtype)
+        x = self.shared(data)
+        y = theano.sparse.matrix('csc', dtype=self.dtype, name='y')
+        z = theano.sparse.matrix('csr', dtype=self.dtype, name='z')
+        cond = theano.tensor.iscalar('cond')
+
+        self.assertRaises(TypeError, ifelse, cond, x, y)
+        self.assertRaises(TypeError, ifelse, cond, y, x)
+        self.assertRaises(TypeError, ifelse, cond, x, z)
+        self.assertRaises(TypeError, ifelse, cond, z, x)
+        self.assertRaises(TypeError, ifelse, cond, y, z)
+        self.assertRaises(TypeError, ifelse, cond, z, y)
 
     def test_merge(self):
         raise SkipTest("Optimization temporarily disabled")