bugfix for automatic GPU selection when 'exclusive mode' is used

- fixing an issue introduced with CUDA-TK 7.0, when automatic selection of free gpu by os / library no longer works. Library always selects '0', which leads to crash in case of 'exclusive mode'. - this fix is inspired by Dan Povey's fix for Kaldi: https://github.com/kaldi-asr/kaldi/commit/6548565445167e00125848f91d7da5f3f949b2a2 - it does a loop over gpus until a free gpu is taken.

bugfix for automatic GPU selection when 'exclusive mode' is used
eed9d97d · vesis84 · 975e0d2b · eed9d97d
--- a/theano/sandbox/cuda/cuda_ndarray.cu
+++ b/theano/sandbox/cuda/cuda_ndarray.cu
@@ -2948,21 +2948,31 @@ PyObject *
 CudaNdarray_select_a_gpu(PyObject* _unused, PyObject* dummy)
 {
    void * rval = NULL;
+    cudaError_t err;
+    int num_gpus = 0;

-    cudaError_t err = cudaMalloc(&rval, 4);
+    err = cudaGetDeviceCount(&num_gpus);
    if (cudaSuccess != err){
        printf("ERR!\\n");
            PyErr_Format(PyExc_RuntimeError,
-                         "Not able to do basic stuff on the GPU (alloc of 4 bytes) (%s).",
+                         "Not able to get number of GPUs (%s).",
                         cudaGetErrorString(err));
            return NULL;
    }
-    err = cudaFree(rval);
+
+    for (int device = 0; device < num_gpus; device++) {
+        cudaSetDevice(device);
+        err = cudaDeviceSynchronize(); // << CUDA context gets created here.
+        cudaGetLastError(); // reset the error state     
+        if (cudaSuccess == err)
+            break;
+    }
+        
    if (cudaSuccess != err){
            printf("ERR!\\n");
                PyErr_Format(PyExc_RuntimeError,
-                         "Not able to do basic stuff on the GPU (cudaFree failed) (%s).",
-                         cudaGetErrorString(err));
+                             "Not able to select available GPU from %d cards (%s).",
+                             num_gpus, cudaGetErrorString(err));
                return NULL;
    }