Merge pull request #2356 from abergeron/cudnn_r2

theano/sandbox/cuda/cudnn_helper.h

@@ -3,6 +3,12 @@
 
 #include <cudnn.h>
 
+#ifndef CUDNN_VERSION
+#include <assert.h>
+
+// Here we define the R2 API in terms of functions in the R1 interface
+// This is only for what we use
+
 static inline const char *cudnnGetErrorString(cudnnStatus_t err) {
   switch (err) {
   case CUDNN_STATUS_SUCCESS:
@@ -28,4 +34,118 @@ static inline const char *cudnnGetErrorString(cudnnStatus_t err) {
   }
 }
 
+// some macros to help support cudnn R1 while using R2 code.
+#define cudnnCreateTensorDescriptor cudnnCreateTensor4dDescriptor
+#define cudnnDestroyTensorDescriptor cudnnDestroyTensor4dDescriptor
+#define cudnnSetFilter4dDescriptor cudnnSetFilterDescriptor
+
+typedef cudnnTensor4dDescriptor_t cudnnTensorDescriptor_t;
+
+static inline cudnnStatus_t
+cudnnGetConvolution2dForwardOutputDim(
+  const cudnnConvolutionDescriptor_t convDesc,
+  const cudnnTensorDescriptor_t inputTensorDesc,
+  const cudnnFilterDescriptor_t filterDesc,
+  int *n,
+  int *c,
+  int *h,
+  int *w) {
+  return cudnnGetOutputTensor4dDim(convDesc, CUDNN_CONVOLUTION_FWD,
+				   n, c, h, w);
+}
+
+typedef int cudnnConvolutionFwdAlgo_t;
+typedef int cudnnConvolutionFwdPreference_t;
+
+#define CUDNN_CONVOLUTION_FWD_NO_WORKSPACE 0
+
+static inline cudnnStatus_t
+cudnnGetConvolutionForwardAlgorithm(
+  cudnnHandle_t handle,
+  const cudnnTensorDescriptor_t srcDesc,
+  const cudnnFilterDescriptor_t filterDesc,
+  const cudnnConvolutionDescriptor_t convDesc,
+  const cudnnTensorDescriptor_t destDesc,
+  cudnnConvolutionFwdPreference_t preference,
+  size_t memoryLimitInbytes,
+  cudnnConvolutionFwdAlgo_t *algo) {
+  *algo = 0;
+  return CUDNN_STATUS_SUCCESS;
+}
+
+static inline cudnnStatus_t
+cudnnConvolutionForward_v2(
+  cudnnHandle_t handle,
+  const void *alpha,
+  const cudnnTensorDescriptor_t srcDesc,
+  const void *srcData,
+  const cudnnFilterDescriptor_t filterDesc,
+  const void *filterData,
+  const cudnnConvolutionDescriptor_t convDesc,
+  cudnnConvolutionFwdAlgo_t algo,
+  void *workSpace,
+  size_t workSpaceSizeInBytes,
+  const void *beta,
+  const cudnnTensorDescriptor_t destDesc,
+  void *destData) {
+  assert(*(float *)alpha == 1.0);
+  assert(*(float *)beta == 0.0);
+  return cudnnConvolutionForward(handle, srcDesc, srcData,
+				 filterDesc, filterData,
+				 convDesc, destDesc, destData,
+				 CUDNN_RESULT_NO_ACCUMULATE);
+}
+#define cudnnConvolutionForward cudnnConvolutionForward_v2
+
+static inline cudnnStatus_t
+cudnnConvolutionBackwardFilter_v2(
+  cudnnHandle_t	handle,
+  const void *alpha,
+  const cudnnTensorDescriptor_t srcDesc,
+  const void *srcData,
+  const cudnnTensorDescriptor_t diffDesc,
+  const void *diffData,
+  const cudnnConvolutionDescriptor_t convDesc,
+  const void *beta,
+  const cudnnFilterDescriptor_t gradDesc,
+  void *gradData) {
+  assert(*(float *)alpha == 1.0);
+  assert(*(float *)beta == 0.0);
+  return cudnnConvolutionBackwardFilter(handle, srcDesc, srcData,
+					diffDesc, diffData,
+					convDesc, gradDesc, gradData,
+					CUDNN_RESULT_NO_ACCUMULATE);
+}
+
+#define cudnnConvolutionBackwardFilter cudnnConvolutionBackwardFilter_v2
+
+static inline cudnnStatus_t
+cudnnConvolutionBackwardData_v2(
+  cudnnHandle_t	handle,
+  const void *alpha,
+  const cudnnFilterDescriptor_t filterDesc,
+  const void *filterData,
+  const cudnnTensorDescriptor_t diffDesc,
+  const void *diffData,
+  const cudnnConvolutionDescriptor_t convDesc,
+  const void *beta,
+  const cudnnTensorDescriptor_t gradDesc,
+  void *gradData) {
+  assert(*(float *)alpha == 1.0);
+  assert(*(float *)beta == 0.0);
+  return cudnnConvolutionBackwardData(handle,
+				      (cudnnFilterDescriptor_t)filterDesc,
+				      filterData,
+				      (cudnnTensorDescriptor_t)diffDesc,
+				      diffData,
+				      (cudnnConvolutionDescriptor_t)convDesc,
+				      (cudnnTensorDescriptor_t)gradDesc,
+				      gradData,
+				      CUDNN_RESULT_NO_ACCUMULATE);
+}
+
+#define cudnnConvolutionBackwardData cudnnConvolutionBackwardData_v2
+
+#endif
+
 #endif

theano/sandbox/cuda/dnn_base.c

+#section support_code
+static cudnnHandle_t _handle = NULL;
+
+static int
+c_set_tensor4d(CudaNdarray *var, cudnnTensorDescriptor_t desc) {
+  cudnnStatus_t err = cudnnSetTensor4dDescriptorEx(
+    desc, CUDNN_DATA_FLOAT,
+    CudaNdarray_HOST_DIMS(var)[0],
+    CudaNdarray_HOST_DIMS(var)[1],
+    CudaNdarray_HOST_DIMS(var)[2],
+    CudaNdarray_HOST_DIMS(var)[3],
+    CudaNdarray_HOST_STRIDES(var)[0]?CudaNdarray_HOST_STRIDES(var)[0]:CudaNdarray_HOST_DIMS(var)[2]*CudaNdarray_HOST_DIMS(var)[3]*CudaNdarray_HOST_DIMS(var)[1],
+    CudaNdarray_HOST_STRIDES(var)[1]?CudaNdarray_HOST_STRIDES(var)[1]:CudaNdarray_HOST_DIMS(var)[2]*CudaNdarray_HOST_DIMS(var)[3],
+    CudaNdarray_HOST_STRIDES(var)[2]?CudaNdarray_HOST_STRIDES(var)[2]:CudaNdarray_HOST_DIMS(var)[3],
+    CudaNdarray_HOST_STRIDES(var)[3]?CudaNdarray_HOST_STRIDES(var)[3]:1
+    );
+  if (err != CUDNN_STATUS_SUCCESS) {
+    PyErr_Format(PyExc_RuntimeError,
+		 "Could not set tensor4d descriptor: %s"
+		 "shapes=%d %d %d %d strides=%d %d %d %d",
+		 cudnnGetErrorString(err),
+		 CudaNdarray_HOST_DIMS(var)[0],
+		 CudaNdarray_HOST_DIMS(var)[1],
+		 CudaNdarray_HOST_DIMS(var)[2],
+		 CudaNdarray_HOST_DIMS(var)[3],
+		 CudaNdarray_HOST_STRIDES(var)[0]?CudaNdarray_HOST_STRIDES(var)[0]:CudaNdarray_HOST_DIMS(var)[2]*CudaNdarray_HOST_DIMS(var)[3]*CudaNdarray_HOST_DIMS(var)[1],
+		 CudaNdarray_HOST_STRIDES(var)[1]?CudaNdarray_HOST_STRIDES(var)[1]:CudaNdarray_HOST_DIMS(var)[2]*CudaNdarray_HOST_DIMS(var)[3],
+		 CudaNdarray_HOST_STRIDES(var)[2]?CudaNdarray_HOST_STRIDES(var)[2]:CudaNdarray_HOST_DIMS(var)[3],
+		 CudaNdarray_HOST_STRIDES(var)[3]?CudaNdarray_HOST_STRIDES(var)[3]:1
+      );
+    return -1;
+  }
+  return 0;
+}
+
+static int
+c_set_filter(CudaNdarray *var, cudnnFilterDescriptor_t desc) {
+  if (!CudaNdarray_is_c_contiguous(var)) {
+    PyErr_SetString(PyExc_ValueError,
+		    "Only contiguous filters (kernels) are supported.");
+    return -1;
+  }
+  cudnnStatus_t err = cudnnSetFilter4dDescriptor(
+    desc, CUDNN_DATA_FLOAT,
+    CudaNdarray_HOST_DIMS(var)[0],
+    CudaNdarray_HOST_DIMS(var)[1],
+    CudaNdarray_HOST_DIMS(var)[2],
+    CudaNdarray_HOST_DIMS(var)[3]
+    );
+  if (err != CUDNN_STATUS_SUCCESS) {
+    PyErr_Format(PyExc_RuntimeError,
+		 "Could not set filter descriptor: %s."
+		 " dims= %d %d %d %d",
+		 cudnnGetErrorString(err),
+		 CudaNdarray_HOST_DIMS(var)[0],
+		 CudaNdarray_HOST_DIMS(var)[1],
+		 CudaNdarray_HOST_DIMS(var)[2],
+		 CudaNdarray_HOST_DIMS(var)[3]);
+    return -1;
+  }
+  return 0;
+}
+
+#section init_code
+
+{
+  cudnnStatus_t err;
+  if ((err = cudnnCreate(&_handle)) != CUDNN_STATUS_SUCCESS) {
+    PyErr_Format(PyExc_RuntimeError, "could not create cuDNN handle: %s",
+		 cudnnGetErrorString(err));
+#if PYTHON_MAJOR_VERSION >= 3
+    return NULL;
+#else
+    return;
+#endif
+  }
+}

theano/sandbox/cuda/dnn_conv_base.c

+#section support_code_struct
+cudnnTensorDescriptor_t APPLY_SPECIFIC(input);
+cudnnTensorDescriptor_t APPLY_SPECIFIC(output);
+cudnnFilterDescriptor_t APPLY_SPECIFIC(kerns);
+
+#section init_code_struct
+
+cudnnStatus_t APPLY_SPECIFIC(err);
+APPLY_SPECIFIC(input) = NULL;
+APPLY_SPECIFIC(output) = NULL;
+APPLY_SPECIFIC(kerns) = NULL;
+if ((APPLY_SPECIFIC(err) = cudnnCreateTensorDescriptor(&APPLY_SPECIFIC(input))) != CUDNN_STATUS_SUCCESS) {
+  PyErr_Format(PyExc_MemoryError, "could not allocate tensor4d descriptor "
+	       "(inp): %s", cudnnGetErrorString(APPLY_SPECIFIC(err)));
+  FAIL;
+}
+if ((APPLY_SPECIFIC(err) = cudnnCreateTensorDescriptor(&APPLY_SPECIFIC(output))) != CUDNN_STATUS_SUCCESS) {
+  PyErr_Format(PyExc_MemoryError, "could not allocate tensor4d descriptor "
+               "(out): %s", cudnnGetErrorString(APPLY_SPECIFIC(err)));
+  FAIL;
+}
+if ((APPLY_SPECIFIC(err) = cudnnCreateFilterDescriptor(&APPLY_SPECIFIC(kerns))) != CUDNN_STATUS_SUCCESS) {
+  PyErr_Format(PyExc_MemoryError, "could not allocate filter descriptor: %s", 
+	       cudnnGetErrorString(APPLY_SPECIFIC(err)));
+  FAIL;
+}
+
+#section cleanup_code_struct
+
+if (APPLY_SPECIFIC(input) != NULL)
+  cudnnDestroyTensorDescriptor(APPLY_SPECIFIC(input));
+if (APPLY_SPECIFIC(output) != NULL)
+  cudnnDestroyTensorDescriptor(APPLY_SPECIFIC(output));
+if (APPLY_SPECIFIC(kerns) != NULL)
+  cudnnDestroyFilterDescriptor(APPLY_SPECIFIC(kerns));

theano/sandbox/cuda/dnn_fwd.c

+#section support_code_struct
+
+int
+APPLY_SPECIFIC(conv_fwd)(CudaNdarray *input, CudaNdarray *kerns,
+			 cudnnConvolutionDescriptor_t desc,
+			 CudaNdarray **output) {
+  cudnnStatus_t err = CUDNN_STATUS_SUCCESS;
+
+  if (c_set_tensor4d(input, APPLY_SPECIFIC(input)) == -1)
+    return 1;
+  if (c_set_filter(kerns, APPLY_SPECIFIC(kerns)) == -1)
+    return 1;
+
+  {
+    int out_dims[4];
+    err = cudnnGetConvolution2dForwardOutputDim(
+      desc,
+      APPLY_SPECIFIC(input),
+      APPLY_SPECIFIC(kerns),
+      &out_dims[0], &out_dims[1], &out_dims[2], &out_dims[3]);
+    if (err != CUDNN_STATUS_SUCCESS) {
+      PyErr_Format(PyExc_RuntimeError,
+		   "GpuDnnConv: error while computing the output shape: %s",
+		   cudnnGetErrorString(err));
+      return 1;
+    }
+    if (CudaNdarray_prep_output(output, 4, out_dims) != 0) {
+      return 1;
+    }
+  }
+
+  if (c_set_tensor4d(*output, APPLY_SPECIFIC(output)) == -1)
+    return 1;
+
+  {
+    cudnnConvolutionFwdAlgo_t algo;
+    err = cudnnGetConvolutionForwardAlgorithm(
+      _handle,
+      APPLY_SPECIFIC(input),
+      APPLY_SPECIFIC(kerns),
+      desc,
+      APPLY_SPECIFIC(output),
+      CUDNN_CONVOLUTION_FWD_NO_WORKSPACE, // TODO: add op param
+      0,
+      &algo);
+
+    if (err != CUDNN_STATUS_SUCCESS) {
+      PyErr_Format(PyExc_RuntimeError,
+		   "GpuDnnConv: Couldn't select convolution algorithm: %s",
+		   cudnnGetErrorString(err));
+      return 1;
+    }
+
+    const float alpha = 1;
+    const float beta = 0;
+
+    err = cudnnConvolutionForward(
+      _handle,
+      (void *)&alpha,
+      APPLY_SPECIFIC(input), CudaNdarray_DEV_DATA(input),
+      APPLY_SPECIFIC(kerns), CudaNdarray_DEV_DATA(kerns),
+      desc,
+      algo,
+      NULL, 0,
+      (void *)&beta,
+      APPLY_SPECIFIC(output), CudaNdarray_DEV_DATA(*output));
+  }
+  if (err != CUDNN_STATUS_SUCCESS) {
+    PyErr_Format(PyExc_RuntimeError, "GpuDnnConv: error doing operation: %s",
+		 cudnnGetErrorString(err));
+    return 1;
+  }
+  return 0;
+}

theano/sandbox/cuda/dnn_gi.c

+#section support_code_struct
+
+int
+APPLY_SPECIFIC(conv_gi)(CudaNdarray *kerns, CudaNdarray *output,
+			cudnnConvolutionDescriptor_t desc,
+			int h, int w,
+			CudaNdarray **input) {
+  cudnnStatus_t err = CUDNN_STATUS_SUCCESS;
+
+  if (c_set_tensor4d(output, APPLY_SPECIFIC(output)) == -1)
+    return 1;
+  if (c_set_filter(kerns, APPLY_SPECIFIC(kerns)) == -1)
+    return 1;
+
+  {
+    int out_dims[4];
+    out_dims[0] = CudaNdarray_HOST_DIMS(output)[0];
+    out_dims[1] = CudaNdarray_HOST_DIMS(kerns)[1];
+    out_dims[2] = h;
+    out_dims[3] = w;
+    if (CudaNdarray_prep_output(input, 4, out_dims) != 0) {
+      return 1;
+    }
+  }
+
+  if (c_set_tensor4d(*input, APPLY_SPECIFIC(input)) == -1)
+    return 1;
+
+  {
+    const float alpha = 1;
+    const float beta = 0;
+
+    err = cudnnConvolutionBackwardData(
+      _handle,
+      (void *)&alpha,
+      APPLY_SPECIFIC(kerns), CudaNdarray_DEV_DATA(kerns),
+      APPLY_SPECIFIC(output), CudaNdarray_DEV_DATA(output),
+      desc,
+      (void *)&beta,
+      APPLY_SPECIFIC(input), CudaNdarray_DEV_DATA(*input));
+  }
+  if (err != CUDNN_STATUS_SUCCESS) {
+    PyErr_Format(PyExc_RuntimeError, "GpuDnnConvGradI: error doing operation: %s",
+		 cudnnGetErrorString(err));
+    return 1;
+  }
+  return 0;
+}

theano/sandbox/cuda/dnn_gw.c

+#section support_code_struct
+
+int 
+APPLY_SPECIFIC(conv_gw)(CudaNdarray *input, CudaNdarray *output,
+			cudnnConvolutionDescriptor_t desc,
+			int h, int w,
+			CudaNdarray **kerns) {
+  cudnnStatus_t err = CUDNN_STATUS_SUCCESS;
+
+  if (c_set_tensor4d(input, APPLY_SPECIFIC(input)) == -1)
+    return 1;
+  if (c_set_tensor4d(output, APPLY_SPECIFIC(output)) == -1)
+    return 1;
+
+  {
+    int out_dims[4];
+    out_dims[0] = CudaNdarray_HOST_DIMS(output)[1];
+    out_dims[1] = CudaNdarray_HOST_DIMS(input)[1];
+    out_dims[2] = h;
+    out_dims[3] = w;
+    if (CudaNdarray_prep_output(kerns, 4, out_dims) != 0) {
+      return 1;
+    }
+  }
+
+  if (c_set_filter(*kerns, APPLY_SPECIFIC(kerns)) == -1)
+    return 1;
+
+  {
+    const float alpha = 1;
+    const float beta = 0;
+
+    err = cudnnConvolutionBackwardFilter(
+      _handle,
+      (void *)&alpha,
+      APPLY_SPECIFIC(input), CudaNdarray_DEV_DATA(input),
+      APPLY_SPECIFIC(output), CudaNdarray_DEV_DATA(output),
+      desc,
+      (void *)&beta,
+      APPLY_SPECIFIC(kerns), CudaNdarray_DEV_DATA(*kerns));
+  }
+  if (err != CUDNN_STATUS_SUCCESS) {
+    PyErr_Format(PyExc_RuntimeError, "GpuDnnConvGradW: error doing operation: %s",
+		 cudnnGetErrorString(err));
+    return 1;
+  }
+  return 0;
+}

theano/sandbox/cuda/tests/test_conv_cuda_ndarray.py

@@ -27,7 +27,7 @@ from theano.sandbox import cuda
 if cuda.cuda_available == False:
     raise SkipTest('Optional package cuda disabled')
 
-from theano.sandbox.cuda.dnn import GpuDnnConv, GpuDnnConvBase, dnn_conv
+from theano.sandbox.cuda.dnn import GpuDnnConv, DnnBase, dnn_conv
 
 #needed as the gpu conv don't have a perform implementation.
 if theano.config.mode == 'FAST_COMPILE':
@@ -596,7 +596,7 @@ def test_gemm_valid():
 def test_dnn_valid():
     if not cuda.dnn.dnn_available():
         raise SkipTest(cuda.dnn.dnn_available.msg)
-    for t in _test_valid(GpuDnnConvBase, mode=theano_mode.including("cudnn")):
+    for t in _test_valid(DnnBase, mode=theano_mode.including("cudnn")):
         yield t
 
 
@@ -710,7 +710,7 @@ def test_gemm_full():
 def test_dnn_full():
     if not cuda.dnn.dnn_available():
         raise SkipTest(cuda.dnn.dnn_available.msg)
-    for t in _test_full(GpuDnnConvBase, mode=theano_mode.including("cudnn")):
+    for t in _test_full(DnnBase, mode=theano_mode.including("cudnn")):
         yield t
 
 
@@ -762,13 +762,13 @@ def test_gemm_subsample():
 def test_dnn_subsample():
     if not cuda.dnn.dnn_available():
         raise SkipTest(cuda.dnn.dnn_available.msg)
-    for t in _test_subsample(GpuDnnConvBase, theano_mode.including('cudnn')):
+    for t in _test_subsample(DnnBase, theano_mode.including('cudnn')):
         yield t
 
 
 class TestConv2DGPU(unittest.TestCase):
     conv_ops = (cuda.blas.GpuConv,
-                cuda.dnn.GpuDnnConvBase,
+                cuda.dnn.DnnBase,
                 cuda.blas.BaseGpuCorrMM)
 
     def test_logical_shapes(self):

theano/sandbox/cuda/tests/test_dnn.py

@@ -192,3 +192,9 @@ def test_dnn_tag():
         assert cuda.dnn.dnn_available()
         assert any([isinstance(n.op, cuda.dnn.GpuDnnPool)
                     for n in f.maker.fgraph.toposort()])
+
+
+def test_version():
+    if not cuda.dnn.dnn_available():
+        raise SkipTest(cuda.dnn.dnn_available.msg)
+    assert isinstance(cuda.dnn.version(), (int, tuple))

theano/scalar/basic.py

@@ -82,6 +82,7 @@ get_scalar_type.cache = {}
 
 
 def as_scalar(x, name=None):
+    from ..tensor import TensorType, scalar_from_tensor
     if isinstance(x, gof.Apply):
         if len(x.outputs) != 1:
             raise ValueError("It is ambiguous which output of a multi-output"
@@ -89,9 +90,12 @@ def as_scalar(x, name=None):
         else:
             x = x.outputs[0]
     if isinstance(x, Variable):
-        if not isinstance(x.type, Scalar):
+        if isinstance(x.type, Scalar):
+            return x
+        elif isinstance(x.type, TensorType) and x.ndim == 0:
+            return scalar_from_tensor(x)
+        else:
             raise TypeError("Variable type field must be a Scalar.", x, x.type)
-        return x
     try:
         return constant(x)
     except TypeError: