Merge pull request #939 from nouiz/c_code_gpusub

theano/gof/tests/test_opt.py

@@ -403,7 +403,7 @@ class TestEquilibrium(object):
                  PatternSub((op4, 'x', 'y'), (op1, 'x', 'y')),
                  PatternSub((op3, (op2, 'x', 'y')), (op4, 'x', 'y'))
                  ],
-                max_use_ratio = 1. / len(g.nodes)) # each opt can only be applied once
+                max_use_ratio = 1. / len(g.apply_nodes)) # each opt can only be applied once
             opt.optimize(g)
         finally:
             _logger.setLevel(oldlevel)

theano/printing.py

@@ -544,12 +544,14 @@ def pydotprint(fct, outfile=None,
 
     if isinstance(fct, Function):
         mode = fct.maker.mode
-        fct_fgraph = fct.maker.fgraph
+        profile = getattr(fct, "profile", None)
         if (not isinstance(mode, ProfileMode)
             or not fct in mode.profile_stats):
             mode = None
+        fct_fgraph = fct.maker.fgraph
     elif isinstance(fct, gof.FunctionGraph):
         mode = None
+        profile = None
         fct_fgraph = fct
     else:
         raise ValueError(('pydotprint expects as input a theano.function or '
@@ -660,6 +662,14 @@ def pydotprint(fct, outfile=None,
             else:
                 pf = time * 100 / mode.profile_stats[fct].fct_call_time
             prof_str = '   (%.3fs,%.3f%%,%.3f%%)' % (time, pt, pf)
+        elif profile:
+            time = profile.apply_time.get(node, 0)
+            #second, %fct time in profiler
+            if profile.fct_callcount == 0:
+                pf = 0
+            else:
+                pf = time * 100 / profile.fct_call_time
+            prof_str = '   (%.3fs,%.3f%%)' % (time, pf)
         applystr = str(node.op).replace(':', '_')
         applystr += prof_str
         if (applystr in all_strings) or with_ids:

theano/sandbox/cuda/basic_ops.py

@@ -1911,7 +1911,7 @@ class GpuReshape(tensor.Reshape, GpuOp):
         out[0] = x.reshape(tuple(shp))
 
 
-class GpuSubtensor(tensor.Subtensor, GpuOp):
+class GpuSubtensor(GpuOp, tensor.Subtensor):
     """
     Implement subtensor on the gpu.
     """
@@ -1920,19 +1920,16 @@ class GpuSubtensor(tensor.Subtensor, GpuOp):
         assert isinstance(x.type, CudaNdarrayType)
         rval = tensor.Subtensor.make_node(self, x, *inputs)
         otype = CudaNdarrayType(rval.outputs[0].type.broadcastable)
-        #We reverse the index here as a speed optimization
-        #this opt was saving 0.40e-05s of 3.49e05s
-        return Apply(self, [x] + list(reversed(rval.inputs[1:])), [otype()])
+        return Apply(self, [x] + rval.inputs[1:], [otype()])
 
     def perform(self, node, inputs, out_):
         out, = out_
         x = inputs[0]
-        indices = inputs[1:]
+        indices = list(reversed(inputs[1:]))
 
         def convert(entry):
             if isinstance(entry, Type):
                 rval = indices.pop()
-                #the if take about .25e-05s
                 if sys.version_info < (2, 5):
                     # Before Python 2.5, PySlice_GetIndicesEx requires
                     # Python int to be passed.
@@ -1955,6 +1952,59 @@ class GpuSubtensor(tensor.Subtensor, GpuOp):
             cdata = cdata[0]
         out[0] = x.__getitem__(cdata)
 
+    def c_code(self, node, name, inputs, outputs, sub):
+        x = inputs[0]
+        z, = outputs
+        view_ndim = node.outputs[0].ndim
+        fail = sub['fail']
+
+        build_view = """
+        //TODO: give this Op a second output so that this view can be cached
+        //TODO: alternatively, fix the memory leak on failure
+        CudaNdarray* xview = (CudaNdarray*) CudaNdarray_New(%(view_ndim)s);
+        if (!xview)
+        {
+            %(fail)s;
+        }
+        if (CudaNdarray_set_device_data(xview, CudaNdarray_DEV_DATA(%(x)s),
+                                       (PyObject*) NULL))
+        {
+            PyErr_Format(PyExc_RuntimeError,
+                         "GpuSubtensor is not able to set the"
+                         " devdata field of the view");
+            Py_XDECREF(xview);
+            %(fail)s;
+        }
+        cnda_mark_dev_structure_dirty(xview);
+        #define CudaNdarray_set_device_data2(obj, ptr, base) \
+                CudaNdarray_set_device_data(obj, (float *)ptr, base)
+""" % locals()
+        get_xview = self.helper_c_code(node, name, inputs, outputs, sub,
+                                       self.idx_list,
+                                       c_prefix='CudaNdarray',
+                                       set_data='CudaNdarray_set_device_data2',
+                                       set_dim='CudaNdarray_set_dim',
+                                       set_stride='CudaNdarray_set_stride',
+                                       update_flags="", strides_mul=4)
+
+        finish_view = """
+        //Set the base only now
+
+        if(CudaNdarray_set_device_data(xview, CudaNdarray_DEV_DATA(xview),
+                                    %(x)s)){
+            PyErr_Format(PyExc_RuntimeError,
+                         "GpuSubtensor is not able to set"
+                         " the base of the view array");
+            Py_XDECREF(xview);
+            %(fail)s;
+        }
+
+        Py_XDECREF(%(z)s);
+        %(z)s = xview;
+        """ % locals()
+
+        return build_view + "{" + get_xview + "}" + finish_view
+
 
 class GpuAdvancedSubtensor1(tensor.AdvancedSubtensor1, GpuOp):
     """

theano/sandbox/cuda/cuda_ndarray.cuh

@@ -161,6 +161,14 @@ DllExport const int *CudaNdarray_DEV_STRIDES(const CudaNdarray * self);
 DllExport const int *CudaNdarray_DEV_LOG2DIMS(const CudaNdarray * self);
 DllExport float *CudaNdarray_DEV_DATA(const CudaNdarray * self);
 
+// The following 4 macro are here to help make c code generator that work on
+// both PyArray and CudaNdarray.  This is at least used for Subtensor and
+// GpuSubtensor
+#define CudaNdarray_DIMS CudaNdarray_HOST_DIMS
+#define CudaNdarray_NDIM(self) self->nd
+#define CudaNdarray_STRIDES CudaNdarray_HOST_STRIDES
+#define CudaNdarray_BYTES CudaNdarray_DEV_DATA
+
 /**
  * Return the number of elements in the ndarray (product of the dimensions)
  */

theano/sandbox/cuda/nvcc_compiler.py

@@ -7,6 +7,8 @@ import subprocess
 import sys
 import warnings
 
+import numpy
+
 import theano
 from theano.gof.cc import hash_from_file
 from theano.gof.cmodule import (std_libs, std_lib_dirs,
@@ -121,6 +123,17 @@ class NVCC_compiler(object):
             os.path.join(os.path.split(__file__)[0], 'cuda_ndarray.cuh'))
         flags.append('-DCUDA_NDARRAY_CUH=' + cuda_ndarray_cuh_hash)
 
+        # numpy 1.7 deprecated the following macro but the didn't
+        # existed in the past
+        numpy_ver = [int(n) for n in numpy.__version__.split('.')[:2]]
+        if bool(numpy_ver < [1, 7]):
+            flags.append("-D NPY_ARRAY_ENSURECOPY=NPY_ENSURECOPY")
+            flags.append("-D NPY_ARRAY_ALIGNED=NPY_ALIGNED")
+            flags.append("-D NPY_ARRAY_WRITEABLE=NPY_WRITEABLE")
+            flags.append("-D NPY_ARRAY_UPDATE_ALL=NPY_UPDATE_ALL")
+            flags.append("-D NPY_ARRAY_C_CONTIGUOUS=NPY_C_CONTIGUOUS")
+            flags.append("-D NPY_ARRAY_F_CONTIGUOUS=NPY_F_CONTIGUOUS")
+
         # We compile cuda_ndarray.cu during import.
         # We should not add device properties at that time.
         # As the device is not selected yet!

theano/tensor/nnet/nnet.py

@@ -159,9 +159,9 @@ class SoftmaxWithBias(gof.Op):
             double sum = 0.0;
             bool  discount_max = false;
 
-            const dtype_%(x)s* __restrict__ x_i = (dtype_%(x)s*)(PyArray_DATA(%(x)s) + PyArray_STRIDES(%(x)s)[0] * i);
-            const dtype_%(b)s* __restrict__ b_i = (dtype_%(b)s*)(PyArray_DATA(%(b)s));
-            dtype_%(sm) s* __restrict__ sm_i = (dtype_%(sm)s*)(PyArray_DATA(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i);
+            const dtype_%(x)s* __restrict__ x_i = (dtype_%(x)s*)(PyArray_BYTES(%(x)s) + PyArray_STRIDES(%(x)s)[0] * i);
+            const dtype_%(b)s* __restrict__ b_i = (dtype_%(b)s*)(PyArray_BYTES(%(b)s));
+            dtype_%(sm) s* __restrict__ sm_i = (dtype_%(sm)s*)(PyArray_BYTES(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i);
         """
 
         inside_row_loop = """
@@ -306,11 +306,11 @@ class SoftmaxGrad(gof.Op):
 
         for (size_t i = 0; i < PyArray_DIMS(%(dx)s)[0]; ++i)
         {
-            const dtype_%(dy)s* __restrict__ dy_i = (dtype_%(dy)s*) (PyArray_DATA(%(dy)s) + PyArray_STRIDES(%(dy)s)[0] * i);
+            const dtype_%(dy)s* __restrict__ dy_i = (dtype_%(dy)s*) (PyArray_BYTES(%(dy)s) + PyArray_STRIDES(%(dy)s)[0] * i);
             npy_intp Sdy = PyArray_STRIDES(%(dy)s)[1]/sizeof(dtype_%(dy)s);
-            const dtype_%(sm)s* __restrict__ sm_i = (dtype_%(sm)s*) (PyArray_DATA(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i);
+            const dtype_%(sm)s* __restrict__ sm_i = (dtype_%(sm)s*) (PyArray_BYTES(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i);
             npy_intp Ssm = PyArray_STRIDES(%(sm)s)[1]/sizeof(dtype_%(sm)s);
-            dtype_%(dx) s* __restrict__ dx_i = (dtype_%(dx)s*) (PyArray_DATA(%(dx)s) + PyArray_STRIDES(%(dx)s)[0] * i);
+            dtype_%(dx) s* __restrict__ dx_i = (dtype_%(dx)s*) (PyArray_BYTES(%(dx)s) + PyArray_STRIDES(%(dx)s)[0] * i);
             npy_intp Sdx = PyArray_STRIDES(%(dx)s)[1]/sizeof(dtype_%(dx)s);
 
             double sum_dy_times_sm = 0.;
@@ -825,9 +825,9 @@ class CrossentropySoftmaxArgmax1HotWithBias(gof.Op):
                 """,
                 begin_row_loop,
                 """
-            const %(y_idx_type) s y_i = ((%(y_idx_type)s*)(PyArray_DATA(%(y_idx)s) + PyArray_STRIDES(%(y_idx)s)[0] * i))[0];
-            dtype_%(nll) s* __restrict__ nll_i = (dtype_%(nll)s*)(PyArray_DATA(%(nll)s) + PyArray_STRIDES(%(nll)s)[0] * i);
-            %(am_type)s* __restrict__ am_i = (%(am_type)s*) (PyArray_DATA(%(am)s) + PyArray_STRIDES(%(am)s)[0] * i);
+            const %(y_idx_type) s y_i = ((%(y_idx_type)s*)(PyArray_BYTES(%(y_idx)s) + PyArray_STRIDES(%(y_idx)s)[0] * i))[0];
+            dtype_%(nll) s* __restrict__ nll_i = (dtype_%(nll)s*)(PyArray_BYTES(%(nll)s) + PyArray_STRIDES(%(nll)s)[0] * i);
+            %(am_type)s* __restrict__ am_i = (%(am_type)s*) (PyArray_BYTES(%(am)s) + PyArray_STRIDES(%(am)s)[0] * i);
                 """,
                 inside_row_loop,
                 """
@@ -977,14 +977,14 @@ class CrossentropySoftmax1HotWithBiasDx (gof.Op):
 
         for (size_t i = 0; i < PyArray_DIMS(%(dx)s)[0]; ++i)
         {
-            const dtype_%(dnll)s dnll_i = ((dtype_%(dnll)s*)(PyArray_DATA(%(dnll)s) + PyArray_STRIDES(%(dnll)s)[0] * i))[0];
+            const dtype_%(dnll)s dnll_i = ((dtype_%(dnll)s*)(PyArray_BYTES(%(dnll)s) + PyArray_STRIDES(%(dnll)s)[0] * i))[0];
 
-            const %(y_idx_type) s y_i = ((%(y_idx_type)s*)(PyArray_DATA(%(y_idx)s) + PyArray_STRIDES(%(y_idx)s)[0] * i))[0];
+            const %(y_idx_type) s y_i = ((%(y_idx_type)s*)(PyArray_BYTES(%(y_idx)s) + PyArray_STRIDES(%(y_idx)s)[0] * i))[0];
 
-            const dtype_%(sm)s* __restrict__ sm_i = (dtype_%(sm)s*)(PyArray_DATA(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i);
+            const dtype_%(sm)s* __restrict__ sm_i = (dtype_%(sm)s*)(PyArray_BYTES(%(sm)s) + PyArray_STRIDES(%(sm)s)[0] * i);
             npy_intp Ssm = PyArray_STRIDES(%(sm)s)[1]/sizeof(dtype_%(sm)s);
 
-            dtype_%(dx) s* __restrict__ dx_i = (dtype_%(dx)s*)(PyArray_DATA(%(dx)s) + PyArray_STRIDES(%(dx)s)[0] * i);
+            dtype_%(dx) s* __restrict__ dx_i = (dtype_%(dx)s*)(PyArray_BYTES(%(dx)s) + PyArray_STRIDES(%(dx)s)[0] * i);
             npy_intp Sdx = PyArray_STRIDES(%(dx)s)[1]/sizeof(dtype_%(dx)s);
 
             for (size_t j = 0; j < PyArray_DIMS(%(dx)s)[1]; ++j)

theano/tensor/tests/test_basic.py

@@ -2565,15 +2565,15 @@ class T_subtensor(unittest.TestCase, utt.TestOptimizationMixin):
         self.fail()
 
     def test1_ok_range_finite(self):
-        n = self.shared(numpy.ones(3, dtype=self.dtype) * 5)
+        n = self.shared(numpy.arange(3, dtype=self.dtype))
         t = n[0:2]
         self.assertTrue(isinstance(t.owner.op, Subtensor))
         tval = self.eval_output_and_check(t)
         self.assertTrue(tval.shape == (2,))
-        self.assertTrue(tval[1] == 5.0)
+        self.assertTrue((tval == [0, 1]).all())
 
     def test2_ok_range_finite(self):
-        n = self.shared(numpy.ones((3, 4), dtype=self.dtype) * 5)
+        n = self.shared(numpy.arange(12, dtype=self.dtype).reshape((3, 4)))
         # Also check negative index
         for idx in [(slice(0, 2), 3), ((slice(0, 2), -1)), (slice(0, 2), -4)]:
             t = n[idx]  # l]#0:2,3]
@@ -2612,25 +2612,25 @@ class T_subtensor(unittest.TestCase, utt.TestOptimizationMixin):
 
     def test1_ok_range_infinite(self):
         #Subtensor.debug = True
-        n = self.shared(numpy.ones(3, dtype=self.dtype) * 5)
+        n = self.shared(numpy.arange(3, dtype=self.dtype))
         t = n[1:]
         self.assertTrue(isinstance(t.owner.op, Subtensor))
         tval = self.eval_output_and_check(t)
         self.assertTrue(tval.shape == (2,))
-        self.assertTrue(tval[1] == 5.0)
+        self.assertTrue((tval == [1.0, 2.0]).all())
 
     def test1_ok_strided(self):
-        n = self.shared(numpy.ones(5, dtype=self.dtype) * 5)
+        n = self.shared(numpy.arange(5, dtype=self.dtype))
         t = n[1::2]
         self.assertTrue(isinstance(t.owner.op, Subtensor))
         tval = self.eval_output_and_check(t)
         self.assertTrue(tval.shape == (2,))
-        self.assertTrue(tval[1] == 5.0)
+        self.assertTrue((tval == [1.0, 3.0]).all())
 
         t = n[0:-1:2]  # 0 to 1 from the end stepping by 2
         tval = self.eval_output_and_check(t)
         self.assertTrue(tval.shape == (2,))
-        self.assertTrue(tval[1] == 5.0)
+        self.assertTrue((tval == [0.0, 2.0]).all())
 
     def test2_err_bounds0(self):
         n = self.shared(numpy.ones((2, 3), dtype=self.dtype) * 5)
@@ -2671,8 +2671,7 @@ class T_subtensor(unittest.TestCase, utt.TestOptimizationMixin):
             sys.stderr = old_stderr
 
     def test2_ok_elem(self):
-        n = self.shared(numpy.asarray(range(6), dtype=self.dtype).
-            reshape((2, 3)))
+        n = self.shared(numpy.arange(6, dtype=self.dtype).reshape((2, 3)))
         t = n[0, 2]
         self.assertTrue(isinstance(t.owner.op, Subtensor))
         tval = self.eval_output_and_check(t)
@@ -2680,8 +2679,7 @@ class T_subtensor(unittest.TestCase, utt.TestOptimizationMixin):
         self.assertTrue(numpy.all(tval == 2))
 
     def test2_ok_row(self):
-        n = self.shared(numpy.asarray(range(6), dtype=self.dtype).
-            reshape((2, 3)))
+        n = self.shared(numpy.arange(6, dtype=self.dtype).reshape((2, 3)))
         t = n[1]
         self.assertFalse(any(n.type.broadcastable))
         self.assertTrue(isinstance(t.owner.op, Subtensor))
@@ -2690,25 +2688,24 @@ class T_subtensor(unittest.TestCase, utt.TestOptimizationMixin):
         self.assertTrue(numpy.all(tval == [3, 4, 5]))
 
     def test2_ok_col(self):
-        n = self.shared(numpy.ones((2, 3), dtype=self.dtype) * 5)
+        n = self.shared(numpy.arange(6, dtype=self.dtype).reshape((2, 3)))
         t = n[:, 0]
         self.assertTrue(isinstance(t.owner.op, Subtensor))
         self.assertFalse(any(n.type.broadcastable))
         tval = self.eval_output_and_check(t)
         self.assertTrue(tval.shape == (2,))
-        self.assertTrue(numpy.all(tval == 5.0))
+        self.assertTrue(numpy.all(tval == [0, 3]))
 
     def test2_ok_rows_finite(self):
-        n = self.shared(numpy.ones((4, 3), dtype=self.dtype) * 5)
+        n = self.shared(numpy.arange(12, dtype=self.dtype).reshape((4, 3)))
         t = n[1:3, 0]
         self.assertTrue(isinstance(t.owner.op, Subtensor))
         tval = self.eval_output_and_check(t)
         self.assertTrue(tval.shape == (2,))
-        self.assertTrue(numpy.all(tval == 5.0))
+        self.assertTrue(numpy.all(tval == [3, 6]))
 
     def test2_ok_cols_infinite(self):
-        n = self.shared(numpy.asarray(range(12), dtype=self.dtype).
-            reshape((4, 3)))
+        n = self.shared(numpy.arange(12, dtype=self.dtype).reshape((4, 3)))
         t = n[1, 2:]
         self.assertTrue(isinstance(t.owner.op, Subtensor))
         tval = self.eval_output_and_check(t)
@@ -2716,8 +2713,7 @@ class T_subtensor(unittest.TestCase, utt.TestOptimizationMixin):
         self.assertTrue(numpy.all(tval == 5))
 
     def test2_ok_strided(self):
-        n = self.shared(numpy.asarray(range(20), dtype=self.dtype).
-            reshape((4, 5)))
+        n = self.shared(numpy.arange(20, dtype=self.dtype).reshape((4, 5)))
         t = n[1:4:2, 1:5:2]
         self.assertTrue(isinstance(t.owner.op, Subtensor))
         tval = self.eval_output_and_check(t)
@@ -2725,8 +2721,7 @@ class T_subtensor(unittest.TestCase, utt.TestOptimizationMixin):
         self.assertTrue(numpy.all(tval == [[6, 8], [16, 18]]))
 
     def test3_ok_mat(self):
-        n = self.shared(numpy.asarray(range(24), dtype=self.dtype).
-            reshape((2, 3, 4)))
+        n = self.shared(numpy.arange(24, dtype=self.dtype).reshape((2, 3, 4)))
         t = n[0, 0, 0]
         self.assertTrue(isinstance(t.owner.op, Subtensor))
         tval = self.eval_output_and_check(t)
@@ -2745,8 +2740,7 @@ class T_subtensor(unittest.TestCase, utt.TestOptimizationMixin):
         """
         newaxis = numpy.newaxis
 
-        n = self.shared(numpy.asarray(range(24), dtype=self.dtype).
-            reshape((2, 3, 4)))
+        n = self.shared(numpy.arange(24, dtype=self.dtype).reshape((2, 3, 4)))
         assert n.ndim == 3
 
         n4 = n[newaxis, :, :, :]