merged; updated compile.py

_test_compile.py

+import unittest
+import gof, gof.modes, gof.opt
+
+from compile import *
+
+class Double(gof.result.ResultBase):
+
+    def __init__(self, data, name = "oignon"):
+        assert isinstance(data, float)
+        gof.result.ResultBase.__init__(self, role = None, data = data, name = name)
+
+    def __str__(self):
+        return self.name
+
+    def __repr__(self):
+        return self.name
+
+class MyOp(gof.op.Op):
+
+    nin = -1
+
+    def __init__(self, *inputs):
+        assert len(inputs) == self.nin
+        for input in inputs:
+            if not isinstance(input, Double):
+                raise Exception("Error 1")
+        self.inputs = inputs
+        self.outputs = [Double(0.0, self.__class__.__name__ + "_R")]
+    
+    def perform(self):
+        self.outputs[0].data = self.impl(*[input.data for input in self.inputs])
+
+
+class Unary(MyOp):
+    nin = 1
+
+class Binary(MyOp):
+    nin = 2
+
+        
+class Add(Binary):
+    def impl(self, x, y):
+        return x + y
+        
+class Sub(Binary):
+    def impl(self, x, y):
+        return x - y
+        
+class Mul(Binary):
+    def impl(self, x, y):
+        return x * y
+        
+class Div(Binary):
+    def impl(self, x, y):
+        return x / y
+
+
+def env(inputs, outputs, validate = True, features = []):
+    return gof.env.Env(inputs, outputs, features = features, consistency_check = validate)
+def perform_linker(env):
+    lnk = gof.link.PerformLinker(env)
+    return lnk
+
+def graph1():
+    x = gof.modes.build(Double(1.0, 'x'))
+    y = gof.modes.build(Double(2.0, 'y'))
+    z = gof.modes.build(Double(3.0, 'z'))
+
+    o = Mul(Add(x, y).out, Div(x, y).out).out
+    return [x,y,z], [o]
+
+def graph2():
+    x = gof.modes.build(Double(1.0, 'x'))
+    y = gof.modes.build(Double(2.0, 'y'))
+    z = gof.modes.build(Double(3.0, 'z'))
+
+    o = Mul(Add(x, y).out, Div(x, y).out).out
+    return [x,y,z], [o, o, o.owner.inputs[1]]
+
+
+
+class _test_compile(unittest.TestCase):
+
+    def test_link_noopt(self):
+        gi, go = graph1()
+        fn, i, o = perform_linker(env(gi, go)).make_thunk(True)
+        fn()
+        self.failUnless(go[0].data == 1.5)
+
+    def test_link_opt(self):
+        opt = gof.opt.PatternOptimizer((Div, '1', '2'), (Div, '2', '1'))
+        gi, go = graph1()
+        e = env(gi, go)
+        opt.optimize(e)
+        fn, i, o = perform_linker(e).make_thunk(True)
+        fn()
+        self.failUnless(go[0].data == 6.0)
+
+    def test_prog_noopt(self):
+        gi, go = graph1()
+        p = Prog(gi,go)
+        self.failUnless(p() == 1.5)
+
+    def test_prog_opt(self):
+        opt = gof.opt.PatternOptimizer((Div, '1', '2'), (Div, '2', '1'))
+        gi, go = graph1()
+        p = Prog(gi,go, optimizer=opt)
+        self.failUnless(p() == 6.0)
+
+    def test_prog_multiout(self):
+        opt = gof.opt.PatternOptimizer((Div, '1', '2'), (Div, '2', '1'))
+        gi, go = graph2()
+        p = Prog(gi,go, optimizer=opt)
+        a,b,c = p()
+        self.failUnless(a == 6.0)
+        self.failUnless(b == 6.0)
+        self.failUnless(a is b)
+        self.failUnless(c == 2.0)
+
+
+if __name__ == '__main__':
+
+    unittest.main()
+

compile.py

-import time, unittest
-import numpy
+"""Convenient driver of graph construction, optimization, and linking."""
 
 import gof
-import gof.lib
-import cutils
-
-import core
-import opt
-from copy import copy
-
-
-def experimental_linker(env, target = None):
-    order = env.toposort()
-    
-    for op in order:
-        op.refresh()
-        
-    py_ops = set()
-    thunks = []
-    computed_results = []
-
-    for op in order:
-        try:
-            factory = op.c_thunk_factory()
-            for input in op.inputs:
-                producer = input.owner
-                if producer in py_ops:
-                    result = lambda factory = factory: cutils.run_cthunk(factory())
-                    break
-            else:
-                thunk = factory()
-                result = lambda thunk = thunk: cutils.run_cthunk(thunk)
-        except NotImplementedError:
-            result = op._perform
-            py_ops.add(op)
-        thunks.append((result, op._perform_inplace))
-        computed_results.extend(op.outputs)
-    
-    def ret():
-        for thunk, fallback in thunks:
-            try:
-                thunk()
-            except NotImplementedError:
-                fallback()
-        for r in computed_results:
-            r.state = gof.result.Computed
-
-    if not target:
-        return ret
-    else:
-        raise NotImplementedError("Cannot write thunk representation to a file.")
-
-
-class profile_linker:
-    def __init__(self, env):
-        self.order = env.toposort()
-        self.thunks = [op._perform for op in self.order]
-        self.n_calls = 0
-        self.n_thunks = 0
-        self.times = [0.0 for op in self.order]
-
-    def print_for_dot(self):
-        #TODO: popen2("dot -Tpng | display") and actually make the graph window pop up
-         print "digraph unix { size = '6,6'; node [color = lightblue2; style = filled];"
-         for op in self.order:
-             for input in op.inputs:
-                 if input.owner:
-                     print input.owner.__class__.__name__ + str(abs(id(input.owner))), " -> ", op.__class__.__name__ + str(abs(id(op))), ";"
-    
-    def slow_call(self):
-        """Run the program, timing each thunk.  """
-        for i, thunk in enumerate(self.thunks):
-            start_time = time.time()
-            thunk()
-            self.times[i] += time.time() - start_time
-            self.n_thunks += 1
-        self.n_calls += 1
-
-    def fast_call(self):
-        """Run the program, but only time the entire loop."""
-        start_time = time.time()
-        for th in self.thunks:
-            th()
-        self.n_thunks += len(self.thunks)
-        self.n_calls += 1
-        self.times[0] += time.time() - start_time
-
-    __call__ = slow_call
-
-    def dump(self, proportion=True):
-        """Print statistics accumulated so far."""
-        total_time = sum(self.times)
-        print self.n_calls, 'calls took', total_time, 'seconds to evaluate',
-        print self.n_thunks, 'thunks'
-
-        if 0:
-            print 'Proportion of CPU per op'
-            for op, t in zip(self.order, self.times):
-                s_op = str(op).split()[0][1:]
-                print "  %-35s %4.5f"% (s_op, t/total_time)
-
-        print 'Proportion of CPU per op class'
-        dct = {}
-        for op, t in zip(self.order, self.times):
-            s_op = str(op).split()[0][1:]
-            dct[s_op] = dct.get(s_op, 0.0) + t
-        for t, s_op in reversed(sorted([(t,op) for op, t in dct.items()])):
-            if proportion:
-                print "  %-35s %4.5f"% (s_op, t/total_time)
-            else:
-                print "  %-35s %4.5f"% (s_op, t)
-
-
-
-
-class prog(gof.Prog):
-    def __init__(self, inputs, outputs, optimizer = opt.optimizer([]),
-            linker = experimental_linker):
-        """Compile a subgraph.
-
-        N.B. This triggers computation of the subgraph leading to the outputs
-        that is not fed by the inputs (the orphans).
-
-        TODO: think about whether orphan computation should be in this function,
-        or in self.__call__()
+
+#TODO: put together some default optimizations
+
+_optimizations = None
+
+def prog_py_opt(inputs, outputs, features=[]):
+    """Return an optimized graph running purely python implementations"""
+    return Prog(intputs, outputs, features, _optimizations, gof.link.PerformLinker, False)
+
+def prog_opt(inputs, outputs, features=[]):
+    """Return a fast implementation"""
+    return Prog(intputs, outputs, features, _optimizations, gof.link.PerformLinker, False)
+
+class Prog:
+    """An 'executable' compiled from a graph
+
+    This class is meant to be used as a function: the idea is to use
+    __call__(*args) and it will compute your graph's function on the args and
+    return the value(s) corresponding to the output(s).
+    
+    Attributes
+    fn - the return value of linker.make_function(False)
+
+    Additional Attributes if keep_locals == True
+    inputs - inputs in the env
+    outputs - outputs in the env
+    features - features to add to the env
+    linker_cls - the linker class
+    linker - the linker allocated from env
+    env - The env passed to the linker
     """
-        new_outputs = gof.mark_outputs_as_destroyed(outputs)
-        gof.Prog.__init__(self,
+    def __init__(self, 
             inputs,
-                          new_outputs,
-                          optimizer,
-                          linker,
-                          [])
-        self.outputs = outputs
-        self.compute_orphans()
-
-    def __call__(self, check_uncomputed = True):
-        """Recompute the graph.
-
-        If the inputs are uncomputed (and check_uncomputed is True) then an
-        Exception is raised.
-        """
-        if check_uncomputed:
-            for input in self.env.inputs:
-                if input.data is None:
-                    raise Exception("You must provide a value for input %s!" % input)
-        return gof.Prog.__call__(self)
-
-    def compute_orphans(self):
-        for orphan in self.env.orphans():
-            if orphan.data is None:
-                if orphan.owner:
-                    gof.lib.compute(orphan.owner)
-                else:
-                    raise Exception("Orphan %s is uncomputed but needed to calculate the function." % orphan)
-            
-
-def to_func(inputs, outputs):
-#    print gof.Env(inputs, outputs).io_toposort()
-##    p = prog([copy(input) for input in inputs], gof.graph.clone(inputs, outputs))
-    p = prog(inputs, outputs)
-    def f(*args):
-        for input, value in zip(inputs, args):
-            p[input] = value
-        outputs = p()
-        if len(outputs) == 1:
-            return outputs[0]
-        else:
-            return outputs
-    return f
-
-def single(*outputs, **kwargs):
-    return prog(gof.graph.inputs(outputs), outputs, **kwargs)
-
-class _test_single_build_mode(unittest.TestCase):
-    def setUp(self):
-        core.build_mode()
-        numpy.random.seed(44)
-    def tearDown(self):
-        core.pop_mode()
-
-    def test_3(self):
-        a = core.Numpy2(data=numpy.random.rand(2,2))
-        b = core.Numpy2(data=numpy.random.rand(2,2))
-        c = core.add(a,b)
-
-        self.failUnless(c.data is None)
-        self.failUnless(c.state is gof.result.Empty)
-        p = single(c)
-        self.failUnless(c.data is not None)
-        self.failUnless(c.state is gof.result.Allocated)
-        self.failUnless(not core._approx_eq(c, a.data + b.data))
-        p()
-        self.failUnless(c.state is gof.result.Computed)
-        self.failUnless(core._approx_eq(c, a.data + b.data))
-
-        new_a = numpy.random.rand(2,2)
-        new_b = numpy.random.rand(2,2)
-        a.data[:] = new_a
-        b.data[:] = new_b
-        p()
-        self.failUnless(core._approx_eq(c, new_a + new_b))
-
-    def test_get_element(self):
-        core.build_eval_mode()
-        a_data = numpy.random.rand(2,2)
-        a = core.Numpy2(data=a_data)
-        pos = core.input((0,0))
-        a_i = core.get_slice(a, pos)
-        p = single(a_i)
-        #p()
-        #print 'aaaa', a_i.owner.out, a_i.owner, a_i.data, pos.data
-
-        #print 'pre p()'
-
-        for i in 0,1:
-            for j in 0,1:
-                pos.data = (i,j)
-                p()
-                #print 'asdf', i,j,a_i.data
-                #print a_i.owner.inputs[1].data
-                #a_i.owner.inputs[1].data = [i,j]
-                self.failUnless(a_data[i,j] == a_i.data)
-
-        core.pop_mode()
-
-
-if __name__ == '__main__':
-    unittest.main()
+            outputs, 
+            features=[],
+            optimizer=None,
+            linker_cls=gof.link.PerformLinker,
+            keep_locals=True):
+
+        env = gof.env.Env(inputs, outputs, features, consistency_check = True)
+
+        if None is not optimizer:
+            optimizer.optimize(env)
+
+        linker = linker_cls(env)
+
+        if keep_locals: # useful flag for debugging
+            self.__dict__.update(locals())
+
+        self.fn  = linker.make_function(False)
+
+    def __call__(self, *args):
+        return self.fn(*args)
+
 

gof/op.py

@@ -65,6 +65,9 @@ class Op(object):
     def get_outputs(self):
         return self._outputs
     def set_outputs(self, new):
+        # the point of this function is 
+        # 1. to save the subclass's __init__ function always having to set the role of the outputs
+        # 2. to prevent accidentally re-setting outputs, which would probably be a bug
         if not hasattr(self, '_outputs') or self._outputs is None:
             for i, output in enumerate(new):
                 output.role = (self, i)
@@ -86,6 +89,8 @@ class Op(object):
         Shallow copy of this Op. The inputs are the exact same, but
         the outputs are recreated because of the one-owner-per-result
         policy.
+
+        This implementation permits a bottom-up copy of an entire graph.
         """
         return self.__class__(*self.inputs)
 
@@ -186,6 +191,7 @@ class Op(object):
 
 
 class GuardedOp(Op):
+    """An Op that disallows input properties to change after construction"""
 
     def set_input(self, i, new):
         old = self._inputs[i]

gof/result.py

@@ -263,5 +263,6 @@ class ResultBase(object):
     #
 
     def same_properties(self, other):
+        """Return bool; True iff all properties are equal (ignores contents, role)"""
         raise AbstractFunction()
 

gof/utils.py

@@ -121,6 +121,14 @@ def toposort(prereqs_d):
     return seq
 
 
+def print_for_dot(self):
+    #TODO: popen2("dot -Tpng | display") and actually make the graph window pop up
+     print "digraph unix { size = '6,6'; node [color = lightblue2; style = filled];"
+     for op in self.order:
+         for input in op.inputs:
+             if input.owner:
+                 print input.owner.__class__.__name__ + str(abs(id(input.owner))), " -> ", op.__class__.__name__ + str(abs(id(op))), ";"
+
 # def schedule(**kwargs):
     
 #     after = kwargs.get('after', [])