Replace dy with zeros if disconnected instead of dhy[-1].

theano/gpuarray/dnn.py

@@ -2241,13 +2241,14 @@ class GpuDnnRNNOp(DnnBase):
         # of them. This leads to the situation that the other is
         # considered "disconnected" by theano in the gradient.
         # However we know that this isn't really the case so we fix it
-        # up here.
+        # here.
 
-        # If both dy and dhy are disconnected the fixup will fail, but
-        # that's ok as in that case we really are disconnected.
+        # If all the ys are disconnected, then you get a boring
+        # gradient instead of an error.  But in that case you
+        # shouldn't call this method anyway.
         if isinstance(dy.type, DisconnectedType):
-            dy = as_gpuarray_variable(dhy[-1],
-                                      context_name=dhy.type.context_name)
+            dy = as_gpuarray_variable(y.zeros_like(),
+                                      context_name=y.type.context_name)
         if isinstance(dhy.type, DisconnectedType):
             dhy = None
         if dcy and isinstance(dcy.type, DisconnectedType):