fixed orphan order bug in Function, continuing to bring Ops back

_test_gradient.py

@@ -273,39 +273,5 @@ def matrices(n):
     return [matrix() for i in xrange(n)]
 
 
-#TODO: move this to the _test_tensor_ops.py
-
-class _testCase_matinv:# (unittest.TestCase):
-
-    def setUp(self):
-        numpy.random.seed(1)
-
-    def matinv(self,dim):
-        # symbolic program
-        a,b = matrices(2)
-        ab = T.dot(a,b)
-        diff = ab - tensor.tensor(numpy.identity(dim))
-        ssdiff = T.sum((diff**2.0))
-        g = grad(ssdiff,None, tensor.tensor(numpy.ones(1)))
-
-        # compilation to function
-        fn = compile.Function([a,b], [ssdiff,g(b)])
-
-        # use the function
-        w = numpy.random.rand(dim,dim)
-        wi = numpy.random.rand(dim,dim)
-        for i in xrange(300):
-            ssd, gw = fn(w,wi)
-            #print ssdiff
-            if i == 0:
-                str0 = str(ssd)
-            wi -= 0.4 * gw
-
-        return str0, str(ssd)
-
-    def test_matinv(self):
-        """Matrix inversion by gradient descent (eval mode)"""
-        self.assertEqual(('2.67327580893', '0.000438649434819'), self.matinv(3))
-
 if __name__ == '__main__':
     unittest.main()

_test_tensor.py

 from tensor import *
-import tensor as T
+import tensor # for hidden symbols
 
 import unittest
 from copy import copy
 from compile import Function
 import gradient
-import gof
+import gof, gof.graph
 
 
 #TODO: consider moving this function / functionality to gradient.py
 #      rationale: it's tricky, and necessary everytime you want to verify
 #      gradient numerically
 
-def verify_grad(testcase, op_cls, pt_list, n_tests=1, rng=numpy.random, eps=0.0000001, tol=0.0001):
+def verify_grad(testcase, op_cls, pt, n_tests=1, rng=numpy.random, eps=0.0000001, tol=0.0001):
     """testcase.failUnless( analytic gradient matches finite-diff gradient) """
 
     for test_num in xrange(n_tests):
-        for pt in pt_list:
-            tensor_pt = [tensor(p) for p in pt]
+        tensor_pt = [tinit(p,name='input %i'%i) for i,p in enumerate(pt)]
         o = op_cls(*tensor_pt)
         if len(o.outputs) > 1:
             raise NotImplementedError('cant (yet) autotest gradient of op with multiple outputs')
             # we could make loop over outputs making random projections R for each,
             # but this doesn't handle the case where not all the outputs are
-                # differentiable... so I leave this as TODO for now -jsb.
+            # differentiable... so I leave this as TODO for now -JB.
         o_fn = Function(tensor_pt, o.outputs)
         o_fn_out = o_fn(*pt)
         random_projection = rng.rand(*o_fn_out.shape)
-            t_r = tensor(random_projection)
+        t_r = tinit(random_projection)
 
         #random projection of o onto t_r
         cost = sum(t_r * o.outputs[0])
@@ -35,9 +34,15 @@ def verify_grad(testcase, op_cls, pt_list, n_tests=1, rng=numpy.random, eps=0.00
 
         num_grad = gradient.numeric_grad(cost_fn, pt)
 
-            grad_fn = Function(tensor_pt, gradient.grad(cost, tensor_pt))
+        symbolic_grad = gradient.grad(cost, tensor_pt,tinit(1.0,name='g_cost'))
+        if 0:
+            print '-------'
+            print '----------'
+            for op in gof.graph.io_toposort(tensor_pt, symbolic_grad):
+                print op
+        grad_fn = Function(tensor_pt, symbolic_grad)
         
-            analytic_grad = grad_fn()
+        analytic_grad = grad_fn(*pt)
         if not isinstance(analytic_grad, (list, tuple)):
             analytic_grad = [analytic_grad]
 
@@ -56,7 +61,7 @@ def check_eq2(self, inputs, output, args_in, arg_out):
     val = fn(*args_in)
     self.failUnless( numpy.all(val == arg_out), (val, arg_out))
 
-def check_eq2(self, inputs, output, args_in, arg_out):
+def check_eq2_c(self, inputs, output, args_in, arg_out):
     fn = Function(inputs, [output], linker_cls = gof.CLinker)
     val = fn(*args_in)
     self.failUnless( numpy.all(val == arg_out), (val, arg_out))
@@ -64,20 +69,21 @@ def check_eq2(self, inputs, output, args_in, arg_out):
 
 class T_abs(unittest.TestCase):
     def test_impl(self):
-        t = tensor(1.0)
+        t = tinit(1.0)
         check_eq(self, t, abs(t), 1.0, 1.0)
         check_eq(self, t, abs(t), -1.0, 1.0)
 
         for shape in (2,), (3,4):
-            t = tensor(numpy.ones(shape))
+            t = tinit(numpy.ones(shape))
             d = numpy.random.rand(*shape)*2-1.0
             check_eq(self, t, abs(t), d, abs(d))
             check_eq(self, t, abs(t), -d, abs(-d))
 
     def test_grad(self):
-        verify_grad(self, Abs, [[numpy.ones(())], [numpy.ones(3)]])
+        verify_grad(self, Abs, [numpy.ones(())])
+        verify_grad(self, Abs, [numpy.ones(3)])
 
-    class AbsBadGrad(T._Elemwise):
+    class AbsBadGrad(tensor._Elemwise):
         def impl(self, x):
             return numpy.abs(x)
         def grad(self, x, gz):
@@ -87,52 +93,137 @@ class T_abs(unittest.TestCase):
 
     def test_badgrad(self):
         try:
-            verify_grad(self, T_abs.AbsBadGrad, [[numpy.ones(())], [numpy.ones(3)]])
+            verify_grad(self, T_abs.AbsBadGrad, [numpy.ones(())])
             self.fail()
         except Exception, e:
             self.failUnless(str(e) == verify_grad.E_grad, str(e))
 
-
+class T_fill(unittest.TestCase):
+    def test0(self):
+        t = fill(numpy.asarray([1,2,3]), 9.0)
+        self.failUnless(t.owner.__class__ == Fill)
+        o = t.owner
+        self.failUnless(o.inputs[0].broadcastable == (0,))
+        self.failUnless(o.inputs[0].dtype[0:3] == 'int')
+        self.failUnless(o.inputs[1].broadcastable == ())
+        self.failUnless(o.inputs[1].dtype[0:3] == 'flo')
+        self.failUnless(o.outputs[0].broadcastable == (0,))
+        self.failUnless(o.outputs[0].dtype[0:3] == 'flo')
 
 class T_sum(unittest.TestCase):
     def test_impl(self):
-        t = tensor(0.0)
+        t = tinit(0.0)
         check_eq(self, t, Sum(t).out, 1.0, 1.0)
         check_eq(self, t, Sum(t).out, -1.0, -1.0)
 
-        t = tensor([0.0, 0.0])
+        t = tinit([0.0, 0.0])
         d = numpy.asarray([-0.4, 1.2])
         check_eq(self, t, Sum(t).out, d, numpy.sum(d))
         check_eq(self, t, Sum(t).out, -d, -numpy.sum(d))
 
 class T_mul(unittest.TestCase):
+    def setUp(self):
+        numpy.random.seed([1,2,3,4])
+
     def test_elemwise(self):
-        a = tensor(0.0)
-        b = tensor(0.0)
+        a = tinit(0.0)
+        b = tinit(0.0)
         check_eq2(self, [a,b], mul_elemwise(a,b), [3.0, 4.0], 12.0)
-        check_eq2(self, [a,b], mul_elemwise(a,a), [-1.0,2.0], 1.0)
-        check_eq2(self, [a,b], mul(a,b), [3.0, 4.0], 12.0)
-        check_eq2(self, [a,b], mul(a,a), [-1.0,2.0], 1.0)
+        check_eq2(self, [a,b], mul_elemwise(b,a), [-1.0,2.0], -2.0)
+        self.failUnless(isinstance(mul(a,b).owner, Scale))
 
-        a = tensor(numpy.ones(2))
-        b = tensor(numpy.ones(2))
+        a = tinit(numpy.ones(2))
+        b = tinit(numpy.ones(2))
         aa = numpy.asarray([-0.5, 4.0])
         bb = numpy.asarray([-0.5, 2.0])
         check_eq2(self, [a,b], mul_elemwise(a,b), [aa,bb], numpy.asarray([0.25, 8.0]))
-        check_eq2(self, [a,b], mul_elemwise(a,b), [aa,aa], numpy.asarray([0.25, 16.0]))
-        check_eq2(self, [a,b], mul(a,b), [aa,bb], numpy.asarray([0.25, 8.0]))
-        check_eq2(self, [a,b], mul(a,b), [aa,aa], numpy.asarray([0.25, 16.0]))
+        check_eq2(self, [a,b], mul_elemwise(a,b), [bb,aa], numpy.asarray([0.25, 8.0]))
+        self.failUnless(isinstance(mul(a,b).owner, MulElemwise))
+
+    def test_scalar(self):
+        r = numpy.random.rand(2,3)
+        a = tinit(r)
+        b = tinit(2.0)
+        check_eq2(self, [a,b], scale(a,b), [r, 2.0], r*2.0)
+        check_eq2(self, [a,b], scale(a,b), [r, 4.0], r*4.0)
+        self.failUnless(b.data == 2.0)
+
+    def test_operator(self):
+        a = tinit([1,1])
+        aa = tinit([1,1])
+        b = tinit(4.0)
+        self.failUnless(isinstance((a*b).owner, Scale))
+        self.failUnless(isinstance((b*a).owner, Scale))
+        self.failUnless(isinstance((a*aa).owner, MulElemwise))
+        self.failUnless(isinstance((aa*a).owner, MulElemwise))
 
     def test_wrong_shapes(self):
-        a = tensor(numpy.ones(3))
-        b = tensor(numpy.ones(4))
+        a = tinit(numpy.ones(3))
+        b = tinit(numpy.ones(4))
         try:
             check_eq2(self, [a,b], MulElemwise(a,b).out,
                     [numpy.ones(3), numpy.ones(4)], 1.0)
             self.fail()
         except ValueError, e:
-            self.failUnless(e is T._assert_same_shapes.E_shape)
+            self.failUnless(e is tensor._assert_same_shapes.E_shape)
+
+class T_div(unittest.TestCase):
+    def setUp(self):
+        numpy.random.seed(9999)
+    def test_grad_e(self):
+        verify_grad(self, DivElemwise, [numpy.ones(()), numpy.ones(())])
+        verify_grad(self, DivElemwise, [numpy.random.rand(3), numpy.ones(3)])
+        verify_grad(self, DivElemwise, [numpy.random.rand(3,5), numpy.random.rand(3,5)+0.1])
 
+    def test_grad_sl(self):
+        verify_grad(self, DivElemwise, [numpy.ones(()), numpy.ones(())])
+        verify_grad(self, DivElemwise, [numpy.random.rand(3), numpy.ones(3)])
+        verify_grad(self, DivElemwise, [numpy.random.rand(3,5), numpy.random.rand(3,5)+0.1])
+
+
+class T_pow(unittest.TestCase):
+    def setUp(self):
+        numpy.random.seed(9999)
+    def test_elemwise(self):
+        verify_grad(self, DivElemwise, [numpy.random.rand(3,4), numpy.random.rand(3,4)+0.1])
+        verify_grad(self, PowElemwise, [numpy.random.rand(3,4), numpy.random.rand(3,4)])
+    def test_scalar_l(self):
+        verify_grad(self, PowScalarL, [numpy.random.rand(3), 3.0])
+    def test_scalar_r(self):
+        verify_grad(self, PowScalarR, [numpy.random.rand(3), 3.0])
+
+class _testCase_matinv:#(unittest.TestCase):
+
+    def setUp(self):
+        numpy.random.seed(1)
+
+    def mat_recip(self,dim):
+        # symbolic program
+        a = Tensor('float64', [0,0], name='a')
+        b = Tensor('float64', [0,0], name='b')
+        ab = a*b
+        diff = ab - tinit(numpy.ones((dim,dim)))
+        ssdiff = sum((diff**2.0))
+        g_b = gradient.grad(ssdiff, b, tinit(numpy.ones(1),name='g_cost'))
+
+        # compilation to function
+        fn = Function([a,b], [ssdiff,g_b])
+
+        # use the function
+        w = numpy.random.rand(dim,dim)
+        wi = numpy.random.rand(dim,dim)
+        for i in xrange(300):
+            ssd, gw = fn(w,wi)
+            #print ssd
+            if i == 0:
+                str0 = str(ssd)
+            wi -= 0.4 * gw
+
+        return str0, str(ssd)
+
+    def test_recip(self):
+        """Matrix reciprocal by gradient descent"""
+        self.assertEqual(('2.67327580893', '0.000438649434819'), self.mat_recip(3))
 
 if __name__ == '__main__':
     unittest.main()

base_tensor.py

@@ -55,7 +55,10 @@ class BaseTensor(ResultBase):
         if not isinstance(arr, numpy.ndarray):
             arr = numpy.asarray(arr, dtype = self.dtype)
         if len(self.broadcastable) != len(arr.shape):
-            raise ValueError(BaseTensor.filter.E_rank)
+            raise ValueError(BaseTensor.filter.E_rank,
+                    self.broadcastable,
+                    arr.shape,
+                    self.owner)
         for b, s in zip(self.broadcastable, arr.shape):
             if b and (s != 1):
                 raise ValueError(BaseTensor.filter.E_shape)

compile.py

@@ -71,7 +71,7 @@ class Function:
         #print 'orphans', orphans
 
         #print 'ops', gof.graph.ops(inputs, outputs)
-        env = gof.env.Env(inputs, outputs, features, consistency_check = True)
+        env = gof.env.Env(inputs, outputs, features + [gof.EquivTool], consistency_check = True)
 
         #print 'orphans in env', env.orphans()
 
@@ -79,7 +79,7 @@ class Function:
 
         #print 'orphans after clone', env.orphans()
 
-        for d, o in zip(orphan_data, env.orphans()):
+        for d, o in zip(orphan_data, [env.equiv(orphan) for orphan in orphans]):
             #print 'assigning orphan value', d
             o.data = d
 

gradient.py

@@ -95,13 +95,13 @@ def grad_sources_inputs(sources, graph_inputs):
                     gmap[r] = g_r
     return gmap
 
-def grad(cost, param):
+def grad(cost, param, g_cost=1.0):
     """Return symbolic expression of gradient of <cost> wrt <param>.
     If <param> is a list, then return a list containing the gradient of cost wrt
     each element of the list.
     """
     inputs = gof.graph.inputs([cost])
-    gmap = grad_sources_inputs([(cost, 1.0)], inputs)
+    gmap = grad_sources_inputs([(cost, g_cost)], inputs)
     if isinstance(param, list):
         return [gmap.get(p, None) for p in param]
     else:
@@ -136,9 +136,9 @@ class numeric_grad:
                     f_eps = f(*pt)
                     gf[idx][i] = numpy.asarray((f_eps - f_pt)/eps)
                     pt[idx][i] = orig
-            elif len(args[idx].shape) == 2:
+            elif len(pt[idx].shape) == 2:
                 for i in xrange(pt[idx].shape[0]):
-                    for j in xrange(args[idx].shape[1]):
+                    for j in xrange(pt[idx].shape[1]):
                         orig = pt[idx][i,j]
                         pt[idx][i,j] = pt[idx][i,j] + eps
                         f_eps = f(*pt)

tensor_ops.py

@@ -83,15 +83,6 @@ class InvElemwiseInplace(InvElemwise.inplace_version()):
         return x
 
 
-class Exp(Elemwise):
-    def impl(self, x): return numpy.exp(x)
-    def grad(self, x, gz): return gz * exp(x)
-    def c_foreach(self, (x_i, ), (z_i, )): return "z_i = exp(x_i);"
-    
-class Log(Elemwise):
-    def impl(self, x): return numpy.log(x)
-    def grad(self, x, gz): return gz / x
-    def c_foreach(self, (x_i, ), (z_i, )): return "z_i = log(x_i);"
     
 class Log2(Elemwise):
     def impl(self, x): return numpy.log2(x)