Fix the init/deinit of the cublas handle.

theano/sandbox/cuda/cuda_ndarray.cu

@@ -3090,6 +3090,10 @@ CudaNdarray_gpu_init(PyObject* _unused, PyObject* args)
                             "There is no device that supports CUDA");
     }
 
+    // Initialize cublas
+    if (handle != NULL)
+        cublas_shutdown();
+
     if(card_number_provided) {
         err = cudaSetDevice(card_nb);
         if(cudaSuccess != err) {
@@ -3100,9 +3104,6 @@ CudaNdarray_gpu_init(PyObject* _unused, PyObject* args)
         }
     }
 
-    // Initialize cublas
-    if (handle != NULL)
-        cublas_shutdown();
     cublas_init();
 
     Py_INCREF(Py_None);
@@ -3552,19 +3553,6 @@ static int
 cublas_init()
 {
     cublasStatus_t err;
-    //The following is causing problems so I comment it.
-    // if (handle != NULL)
-    // {
-    //     err = cublasDestroy(handle);
-    //     if (CUBLAS_STATUS_SUCCESS != err)
-    //     {
-    //         PyErr_SetString(PyExc_RuntimeError,
-    //                         "cublas_init tried to destroy the old cublas"
-    //                         " context, cublasDestroy() returned an error.");
-    //         return -1;
-    //     }
-    //     handle = NULL;
-    // }
     err = cublasCreate(&handle);
     if (CUBLAS_STATUS_SUCCESS != err)
     {
@@ -3594,11 +3582,15 @@ cublas_init()
 static int
 cublas_shutdown()
 {
-    if (CUBLAS_STATUS_SUCCESS != cublasDestroy(handle))
+    err = cublasDestroy(handle);
+    if (CUBLAS_STATUS_SUCCESS != err)
     {
-        PyErr_SetString(PyExc_RuntimeError, "error shutting down device");
+        PyErr_SetString(PyExc_RuntimeError,
+                        "cublas_init tried to destroy the old cublas"
+                        " context, cublasDestroy() returned an error.");
         return -1;
     }
+    handle = NULL;
     return 0;
 }