Don't apply InputToGpuOptimizer to input that are already moved to the gpu.

theano/sandbox/cuda/opt.py

@@ -82,7 +82,14 @@ class InputToGpuOptimizer(Optimizer):
 
     def apply(self, env):
         for input in env.inputs:
-            if not isinstance(input.type, CudaNdarrayType):
+            if isinstance(input.type, CudaNdarrayType):
+                return
+
+            # This happen frequently as we do 2 pass of the gpu optimizations
+            if (len(input.clients) == 1 and
+                input.clients[0][0].op == gpu_from_host):
+                return
+
             try:
                 new_input = host_from_gpu(gpu_from_host(input))