moving from specs to refresh, passing tests in compile.py

compile.py

@@ -18,6 +18,7 @@ def experimental_linker(env, target = None):
         
     py_ops = set()
     thunks = []
+    computed_results = []
 
     for op in order:
         try:
@@ -34,6 +35,7 @@ def experimental_linker(env, target = None):
             result = op._perform
             py_ops.add(op)
         thunks.append((result, op._perform_inplace))
+        computed_results.extend(op.outputs)
     
     def ret():
         for thunk, fallback in thunks:
@@ -41,6 +43,8 @@ def experimental_linker(env, target = None):
                 thunk()
             except NotImplementedError:
                 fallback()
+        for r in computed_results:
+            r.state = gof.result.Computed
 
     if not target:
         return ret
@@ -48,38 +52,6 @@ def experimental_linker(env, target = None):
         raise NotImplementedError("Cannot write thunk representation to a file.")
 
 
-# def experimental_linker(env, target = None):
-#     def fetch(op):
-#         try:
-#             factory = op.c_thunk_factory()
-# #            print "yea %s" % op
-#             thunk = factory()
-#             return lambda: cutils.run_cthunk(thunk)
-#         except NotImplementedError:
-# #            print "nope %s" % op
-#             return op._perform
-#     order = env.toposort()
-#     for op in order:
-#         op.refresh()
-# #     for op in order:
-# #         print op
-# #         print 'ispecs: ', [input.spec for input in op.inputs]
-# #         print 'ospecs: ', [output.spec for output in op.outputs]
-#     thunks = [fetch(op) for op in order]
-#     def ret():
-# #         print "=================================================="
-# #         for thunk, op in zip(thunks, order):
-# #             print op
-# #             print 'in: ', [id(input.data) for input in op.inputs]
-# #             print 'out:', [id(output.data) for output in op.outputs]
-# #             thunk()
-#         for thunk in thunks:
-#             thunk()
-#     if not target:
-#         return ret
-#     else:
-#         raise NotImplementedError("Cannot write thunk representation to a file.")
-
 class profile_linker:
     def __init__(self, env):
         self.order = env.toposort()
@@ -201,10 +173,9 @@ def to_func(inputs, outputs):
 def single(*outputs, **kwargs):
     return prog(gof.graph.inputs(outputs), outputs, **kwargs)
 
-
-class _test_single(unittest.TestCase):
+class _test_single_build_mode(unittest.TestCase):
     def setUp(self):
-        core.build_eval_mode()
+        core.build_mode()
         numpy.random.seed(44)
     def tearDown(self):
         core.pop_mode()
@@ -215,27 +186,44 @@ class _test_single(unittest.TestCase):
         c = core.add(a,b)
 
         self.failUnless(c.data is None)
-        self.failUnless(c.state is Empty)
+        self.failUnless(c.state is gof.result.Empty)
+        p = single(c)
+        self.failUnless(c.data is not None)
+        self.failUnless(c.state is gof.result.Allocated)
+        self.failUnless(not core._approx_eq(c, a.data + b.data))
+        p()
+        self.failUnless(c.state is gof.result.Computed)
+        self.failUnless(core._approx_eq(c, a.data + b.data))
 
         new_a = numpy.random.rand(2,2)
         new_b = numpy.random.rand(2,2)
-        a.data = new_a
-        b.data = new_b
-        p = single(c)
+        a.data[:] = new_a
+        b.data[:] = new_b
         p()
         self.failUnless(core._approx_eq(c, new_a + new_b))
 
     def test_get_element(self):
+        core.build_eval_mode()
         a_data = numpy.random.rand(2,2)
         a = core.Numpy2(data=a_data)
-        a_i = a[0,0]
+        pos = core.input((0,0))
+        a_i = core.get_slice(a, pos)
         p = single(a_i)
+        #p()
+        #print 'aaaa', a_i.owner.out, a_i.owner, a_i.data, pos.data
+
+        #print 'pre p()'
 
         for i in 0,1:
             for j in 0,1:
+                pos.data = (i,j)
                 p()
+                #print 'asdf', i,j,a_i.data
+                #print a_i.owner.inputs[1].data
+                #a_i.owner.inputs[1].data = [i,j]
                 self.failUnless(a_data[i,j] == a_i.data)
 
+        core.pop_mode()
 
 
 if __name__ == '__main__':

core.py

@@ -86,12 +86,13 @@ def _compile_dir():
 
 class Numpy2(ResultBase):
     """Result storing a numpy ndarray"""
-    __slots__ = ['_dtype', '_shape', ]
+    __slots__ = ['_dtype', '_shape', '_order']
 
     class ShapeUnknown: pass # TODO: use this as the shape of uncomputed ndarrays of unknown shape
     class StateError(Exception): pass
 
     def __init__(self, role=None, data=None, constant=False):
+        self._order = 'C'
         if isinstance(data, (tuple, list)): # unallocated setup
             shape, dtype = data
             ResultBase.__init__(self, role, data=None, constant=constant)
@@ -104,30 +105,23 @@ class Numpy2(ResultBase):
     # ResultBase
     # 
     def data_filter(self, data):
-        #TODO: decide which of these implementations is better
-        if 0:
-            if isinstance(data, numpy.ndarray):
-                return data
-            raise TypeError('failed to filter data to ndarray', data)
-        else:
-            return numpy.asarray(data)
+        return numpy.asarray(data)
         
 
     ################################
     # Numpy2 specific functionality
     #
-    __array__ = property(lambda self: self.data.__array__ )
-    __array_struct__ = property(lambda self: self.data.__array_struct__ )
+    __array__ = property(lambda self: self.data.__array__)
+    __array_struct__ = property(lambda self: self.data.__array_struct__)
 
     def data_alloc(self):
-        return numpy.ndarray(self.shape, self.dtype)
+        return numpy.ndarray(shape=self.shape, dtype=self.dtype, order=self._order)
 
     # self._dtype is used when self.data hasn't been set yet
     def __dtype_get(self):
-        if self.data is None:
-            return self._dtype
-        else:
-            return self.data.dtype
+        if self.data is not None:
+            self._dtype = self.data.dtype
+        return self._dtype
     def __dtype_set(self, dtype):
         if self.data is None:
             self._dtype = dtype
@@ -137,10 +131,9 @@ class Numpy2(ResultBase):
 
     # self._shape is used when self.data hasn't been set yet
     def __shape_get(self):
-        if self.data is None:
-            return self._shape
-        else:
-            return self.data.shape
+        if self.data is not None:
+            self._shape = self.data.shape
+        return self._shape
     def __shape_set(self, shape):
         if self.data is None:
             self._shape = shape
@@ -187,6 +180,7 @@ class Numpy2(ResultBase):
             self.data.itemset(value) # for scalars
         else:
             self.data[:] = value     # for matrices
+        self.state = gof.result.Computed
 
 class _test_Numpy2(unittest.TestCase):
     def setUp(self):
@@ -355,6 +349,7 @@ def cgen(name, behavior, names, vals, converters = None):
     
     def cgetspecs(names, vals, converters):
         d = {}
+        assert len(names) == len(vals)
         for name, value in zip(names, vals):
             d[name] = value.data
         specs = weave.ext_tools.assign_variable_types(names, d, type_converters = converters) #, auto_downcast = 0)
@@ -365,6 +360,7 @@ def cgen(name, behavior, names, vals, converters = None):
     for converter in converters:
         assert isinstance(converter, type_spec.omega_type_converter_extension)
 
+
     d, specs = cgetspecs(names, vals, converters)
     
     template = {}
@@ -420,22 +416,38 @@ def cgen(name, behavior, names, vals, converters = None):
     return d, names, code, struct + static, converters    
 
 
-class omega_op(gof.PythonOp):
+class Numpy2Op(gof.lib.PythonOp):
+    """What can we do given we are interacting with Numpy2 inputs and outputs"""
+    def refresh(self, alloc = True):
+        shape = self.refresh_shape()
+        dtype = self.refresh_dtype()
+        out = self.out
+
+        if out.data is not None \
+                and out.shape == shape \
+                and out.dtype == dtype:
+                    return
+
+        alloc |= out.data is not None
+
+        if alloc: out.data = None
+        out.shape = shape
+        out.dtype = dtype
+        if alloc: out.alloc()
+
+class omega_op(Numpy2Op):
 
     forbid_broadcast = False
 
     @staticmethod
     def __clsinit__(cls, name, bases, dct):
-        for fname in ['grad', 'c_impl']:
+        for fname in ['grad', 'c_impl', 'impl']:
             if hasattr(cls, fname):
                 gof.make_static(cls, fname)
 
-        # make impl a static method
-        gof.PythonOp.__clsinit__(cls, name, bases, dct)
-    
     def __new__(cls, *inputs):
         inputs = [wrap(input) for input in inputs]
-        return gof.PythonOp.__new__(cls, *inputs)
+        return Numpy2Op.__new__(cls, *inputs)
 
     def gen_outputs(self):
         return [Numpy2() for i in xrange(self.nout)]
@@ -662,7 +674,7 @@ class elemwise(omega_op):
         # make impl, grad, etc. static methods
         omega_op.__clsinit__(cls, name, bases, dct)
 
-    def _specs(self):
+    def TOGO_specs(self):
         try:
             return self.specs(*[input.spec for input in self.inputs])
         except NotImplementedError:
@@ -706,14 +718,55 @@ class elemwise(omega_op):
             else:
                 return res
         
-    def alloc(self, except_list = []):
+    def TOGO_alloc(self, except_list = []):
         dmap = self.destroy_map()
+        vmap = self.view_map()
 
         gof.PythonOp.alloc(self, except_list = except_list + dmap.keys())
         for output, (input, ) in dmap.items():
             if output not in except_list:
                 output.set_value(input.data)
 
+    def refresh_shape(self):
+        """Make the output have the right stuff"""
+        if len(self.outputs) > 1:
+            raise NotImplementedError('multiple outputs')
+
+        dmap = self.destroy_map()
+        vmap = self.view_map()
+        if dmap != {} or vmap != {}:
+            raise NotImplementedError('destroys or views confuse things',
+                    self.__class__, dmap, vmap)
+
+        # take the shape of the leftmost loop_variable input
+        inames, onames = self.variable_names()
+        linames, lonames = self.loop_variables()
+
+        unknown_output_names = [n for n in onames if n not in lonames]
+        if len(unknown_output_names):
+            raise Exception("cannot infer a specification automatically for variables " \
+                            "%s.{%s} because it is not part of the elementwise loop - "\
+                            "please override the specs method" % 
+                            (self.__class__.__name__, str(unknown_output_names)))
+
+        # shape is leftmost loop-variable input
+        input_loop_shapes = [i.shape for n,i in zip(inames, self.inputs) if n in linames]
+        if len(input_loop_shapes) == 0:
+            raise Exception("cannot infer a specification automatically for output variables " \
+                            "because there is no input loop variable ")
+        for i in xrange(1,len(input_loop_shapes)):
+            if  input_loop_shapes[i] != input_loop_shapes[0]:
+                raise Exception("Input loop variables have different shapes", self.__class__)
+
+        return input_loop_shapes[0]
+
+    def refresh_dtype(self):
+        return upcast(*[i.dtype for i in self.inputs if hasattr(i, 'dtype')])
+
+    @classmethod
+    def set_impl(cls, impl):
+        gof.lib.make_static(cls, 'impl')
+
     @staticmethod
     def is_loop_var(name):
         return name.endswith("_i")
@@ -1134,9 +1187,22 @@ class dot(omega_op):
     impl = numpy.dot
     def grad(x, y, gz):
         return dot(gz, transpose(y)), dot(transpose(x), gz)
-    def specs(x, y):
-        shape = dot._output_shape(x[2], y[2])
-        return (numpy.ndarray, upcast(x[1], y[1]), shape)
+    def refresh(self, alloc=False):
+        x,y = self.inputs
+        shape = self._output_shape(x.shape, y.shape)
+        dtype = upcast(x.dtype, y.dtype)
+        if self.out.data is not None \
+                and self.out.shape == shape \
+                and self.out.dtype == dtype:
+                    return  #everything is ok
+        if alloc or self.out.data is not None: #data should be allocated
+            self.out.data = None
+            self.out.shape = shape
+            self.out.dtype = dtype
+            self.out.alloc()
+        else:
+            self.out.shape = shape
+            self.out.dtype = dtype
     def c_support_code(self):
         return blas.cblas_header_text()
     def c_libs(self):
@@ -1297,8 +1363,8 @@ class _testCase_dot(unittest.TestCase):
         self.fail()
 
 class gemm(omega_op):
-    def destroy_map(self): return {self.out:[self.inputs[0]]}
-
+    def destroy_map(self):
+        return {self.out:[self.inputs[0]]}
     def impl(z, a, x, y, b):
         if b == 0.0:
             if a == 1.0:
@@ -1318,15 +1384,14 @@ class gemm(omega_op):
             z *= b
             z += a * numpy.dot(x,y)
         return z[:]
-
     def grad(z, a, x, y, b, gz):
         raise NotImplemented
-
-    def specs(z, a, x, y, b):
-        assert z[2] == dot._output_shape(x[2], y[2])
-        return z
-    def alloc(self, except_list):
-        self.outputs[0].data = self.inputs[0].data
+    def refresh(self, alloc = False):
+        z,a,x,y,b = self.inputs
+        self.out.shape = z.shape
+        self.out.dtype = z.dtype
+        if alloc:
+            self.out.data = z.data
     def c_support_code(self):
         return blas.cblas_header_text()
     def c_libs(self):
@@ -1355,9 +1420,12 @@ class transpose(omega_op):
     impl = numpy.transpose
     def grad(x, gz):
         return transpose_copy(gz)
-    def specs(x):
-        # todo: handle all tensors!
-        return (numpy.ndarray, x[1], (x[2][1], x[2][0]))
+    def refresh_shape(self):
+        rval = list(self.inputs[0].shape)
+        rval.reverse()
+        return rval
+    def refresh_dtype(self):
+        return  self.inputs[0].dtype
     def c_impl((x, ), (xt, )):
         return """
         const int l = x->nd;
@@ -1635,8 +1703,8 @@ class sum(elemwise):
     impl = numpy.sum
     def grad(x, gz):
         return fill(x, gz)
-    def specs(x):
-        return (numpy.ndarray, x[1], ())
+    def refresh_shape(self):
+        return ()
     def c_init((x, ), (sum, )):
         return "sum_dtype* sump = ((sum_dtype*)PyArray_DATA(sum)); sump[0] = 0;"
     def c_foreach((x_i, ), (sum, )):
@@ -1654,8 +1722,18 @@ class zeros_like(elemwise):
 
 class get_slice(omega_op):
     def view_map(self): return {self.out: [self.inputs[0]]}
-    def impl(x, item): return x.__getitem__(item)
+    def impl(x, item): 
+        rval = x.__getitem__(item)
+        #print 'get_slice running', rval
+        return rval
     def grad(x, gz): raise NotImplemented
+    def refresh_shape(self): 
+        x,item = self.inputs
+        rval = x.data.__getitem__(item.data).shape 
+        #print 'refresh_shape', rval
+        return rval
+    def refresh_dtype(self):
+        return self.inputs[0].data.dtype
 
 class _testCase_slicing(unittest.TestCase):
     def setUp(self):

gof/lib.py

@@ -350,7 +350,7 @@ class DestroyHandler(features.Listener, features.Constraint, features.Orderings)
 
 class NewPythonOp(Op):
 
-    __env_require__ = DestroyHandler
+    __env_require__ = DestroyHandler, ForbidConstantOverwrite
 
     def view_map(self):
         return {}
@@ -358,7 +358,6 @@ class NewPythonOp(Op):
     def destroy_map(self):
         return {}
 
-
 class PythonOp(NewPythonOp):
     
     __metaclass__ = ClsInit
@@ -369,10 +368,9 @@ class PythonOp(NewPythonOp):
     def __clsinit__(cls, name, bases, dct):
         # make impl a static method
         cls.set_impl(cls.impl)
-        make_static(cls, 'specs')
     
     def __new__(cls, *inputs, **kwargs):
-        op = Op.__new__(cls)
+        op = NewPythonOp.__new__(cls)
         op.__init__(*inputs)
         mode = kwargs.get('mode', None) or current_mode()
         if mode == 'eval':
@@ -471,40 +469,6 @@ class PythonOp(NewPythonOp):
     def impl(*args):
         raise NotImplementedError("This op has no implementation.")
 
-    def _specs(self):
-        try:
-            return self.specs(*[input.spec for input in self.inputs])
-        except NotImplementedError:
-            raise NotImplementedError("%s cannot infer the specs of its outputs" % self.__class__.__name__)
-
-    def specs(*inputs):
-        raise NotImplementedError
-
-    def refresh(self, except_list = []):
-        for input in self.inputs:
-            input.refresh()
-        change = self._propagate_specs()
-        if change:
-            self.alloc(except_list)
-        return change
-    
-    def _propagate_specs(self):
-        specs = self._specs()
-        if self.nout == 1:
-            specs = [specs]
-        change = False
-        for output, spec in zip(self.outputs, specs):
-            if output.spec != spec:
-                output.spec = spec
-                change = True
-        return change
-    
-    def alloc(self, except_list = []):
-        for output in self.outputs:
-            if output not in except_list:
-                output.alloc()
-    
-    __env_require__ = ForbidConstantOverwrite
 
     def __copy__(self):
         """
@@ -577,3 +541,41 @@ class DummyOp(NewPythonOp):
 
 DummyRemover = opt.OpRemover(DummyOp)
 
+
+if 0:
+    class RefreshableOp(NewPythonOp):
+
+        def _specs(self):
+            try:
+                return self.specs(*[input.spec for input in self.inputs])
+            except NotImplementedError:
+                raise NotImplementedError("%s cannot infer the specs of its outputs" % self.__class__.__name__)
+
+        def specs(*inputs):
+            raise NotImplementedError
+
+        def refresh(self):
+            """Update and allocate outputs if necessary"""
+
+            for input in self.inputs:
+                input.refresh()
+            change = self._propagate_specs()
+            if change:
+                self.alloc(except_list)
+            return change
+        
+        def _propagate_specs(self):
+            specs = self._specs()
+            if self.nout == 1:
+                specs = [specs]
+            change = False
+            for output, spec in zip(self.outputs, specs):
+                if output.spec != spec:
+                    output.spec = spec
+                    change = True
+            return change
+        
+        def alloc(self, except_list = []):
+            for output in self.outputs:
+                if output not in except_list:
+                    output.alloc()

gof/result.py

@@ -93,11 +93,11 @@ class ResultBase(object):
         def __init__(self, role): self.old_role = role
         def __nonzero__(self): return False
 
-    class BrokenLinkError(Exception):
-        """Exception thrown when an owner is a BrokenLink"""
+    class BrokenLinkError(Exception): 
+        """The owner is a BrokenLink"""
 
-    class AbstractFunction(Exception):
-        """Exception thrown when an abstract function is called"""
+    class StateError(Exception):
+        """The state of the Result is a problem"""
 
     __slots__ = ['_role', 'constant', '_data', 'state']
 
@@ -111,7 +111,7 @@ class ResultBase(object):
         else:
             try:
                 self._data[0] = self.data_filter(data)
-            except ResultBase.AbstractFunction:
+            except AbstractFunctionError:
                 self._data[0] = data
             self.state = Computed
 
@@ -175,10 +175,13 @@ class ResultBase(object):
             self._data[0] = None
             self.state = Empty
             return
+        if data is self or data is self._data[0]: return
         try:
             self._data[0] = self.data_filter(data)
-        except ResultBase.AbstractFunction: #use default behaviour
+        except AbstractFunctionError: #use default behaviour
             self._data[0] = data
+        if isinstance(data, ResultBase):
+            raise Exception()
         self.state = Computed
 
     data = property(__get_data, __set_data,
@@ -193,14 +196,19 @@ class ResultBase(object):
         the contents of self._data remain sensible.
         
         """
-        raise ResultBase.AbstractFunction()
+        raise AbstractFunctionError()
 
     #
     # alloc
     #
 
     def alloc(self):
-        """Create self.data from data_alloc, and set state to Allocated"""
+        """Create self.data from data_alloc, and set state to Allocated
+
+        Graph routines like the linker will ask Ops to allocate outputs.  The
+        Ops, in turn, usually call this function.  Results that are involved in
+        destroy maps and view maps are exceptions to the usual case.
+        """
         self.data = self.data_alloc()  #might raise exception
         self.state = Allocated
 
@@ -211,7 +219,7 @@ class ResultBase(object):
         implementation will be used in alloc() to produce a data object.
         
         """
-        raise ResultBase.AbstractFunction()
+        raise AbstractFunctionError()
 
 
     #