Merge pull request #287 from nouiz/shape_opt_disabled

NEWS.txt

@@ -162,8 +162,9 @@ Others:
  * Don't request to load the GPU module by default in scan module. (Razvan)
  * Fixed some import problems.
  * Filtering update. (James)
- * The buidbot now raises optimization errors instead of just printing a warning. (Frederic)
  * On Windows, the default compiledir changed to be local to the computer/user and not transferred with roaming profile. (Sebastian Urban)
+ * New theano flag "on_shape_error". Default to "warn" (same as previous behavior): it print a warning when an error occur when infering the shape of some apply node. The other accepted value is "raise" to raise an error when this happen.
+ * The buidbot now raises optimization/shape errors instead of just printing a warning. (Frederic)
 
 Reviewers (alphabetical order):
  * David, Frederic, Ian, James, Olivier, Razvan

doc/NEWS.txt

@@ -162,8 +162,8 @@ Others:
  * Don't request to load the GPU module by default in scan module. (Razvan)
  * Fixed some import problems.
  * Filtering update. (James)
- * The buidbot now raises optimization errors instead of just printing a warning. (Frederic)
  * On Windows, the default compiledir changed to be local to the computer/user and not transferred with roaming profile. (Sebastian Urban)
+ * The buidbot now raises optimization/shape errors instead of just printing a warning. (Frederic)
 
 Reviewers (alphabetical order):
  * David, Frederic, Ian, James, Olivier, Razvan

doc/library/config.txt

@@ -243,6 +243,16 @@ import theano and print the config variable, as in:
     the user and skip this optimization ('warn'), or raise the exception
     ('raise').
 
+.. attribute:: on_shape_error
+
+    String value: 'warn' or 'raise'
+
+    Default: 'warn'
+
+    When an exception is raised when infering the shape of some apply
+    node, either warn the user and use a default value ('warn'), or
+    raise the exception ('raise').
+
 .. attribute:: config.warn.ignore_bug_before
 
     String value: 'None', 'all', '0.3', '0.4', '0.4.1', '0.5'

theano/compile/debugmode.py

@@ -1333,7 +1333,8 @@ class _Linker(gof.link.LocalLinker):
                         if (storage_map[r][0] is None):
                             raise Exception('Missing input', r)
                         if not r.type.is_valid_value(storage_map[r][0]):
-                            raise InvalidValueError(r, storage_map[r][0])
+                            raise InvalidValueError(r, storage_map[r][0],
+                                   hint="Graph Input '%s' is missing" % str(r))
                         r_vals[r] = storage_map[r][0]
                         storage_map[r][0] = None
                         r_vals_initialized.append(r)

theano/misc/do_nightly_build

@@ -25,9 +25,10 @@ echo "Number of elements in the compiledir:"
 ls ${COMPILEDIR}|wc -l
 # We don't want warnings in the buildbod for errors already fixed.
 FLAGS=${THEANO_FLAGS},warn.argmax_pushdown_bug=False,warn.gpusum_01_011_0111_bug=False,warn.sum_sum_bug=False,warn.sum_div_dimshuffle_bug=False,$FLAGS
-# We want to see correctly optimization errors, so make make them raise an
+# We want to see correctly optimization/shape errors, so make make them raise an
 # error.
 FLAGS=on_opt_error=raise,$FLAGS
+FLAGS=on_shape_error=raise,$FLAGS
 # Ignore user device and floatX config, because:
 #   1. Tests are intended to be run with device=cpu.
 #   2. We explicitly add 'floatX=float32' in one run of the test suite below,

theano/scan_module/scan_op.py

@@ -166,7 +166,13 @@ class Scan(PureOp):
                     'could happen if the inner graph of scan results in '
                     'an upcast or downcast. Please make sure that you use'
                     'dtypes consistently')
-
+        # TODO make the assert exact
+        # TODO assert the type(dtype, nbdim of self.inputs and inputs correspond)
+        #assert len(inputs) >= len(self.inputs)
+#        if self.info['as_while']:
+#            assert len(inputs) == len(self.inputs) + 2 + self.info["n_nit_sot"]
+#        else:
+#            assert len(inputs) == len(self.inputs) + 1 + self.info["n_nit_sot"]
         # Flags that indicate which inputs are vectors
 
         self.vector_seqs = [seq.ndim == 1 for seq in
@@ -903,7 +909,12 @@ class Scan(PureOp):
         # Here, we build a list inner_ins_shape, such that inner_ins_shape[i]
         # is the shape of self.inputs[i]
 
+        for inp, inp_shp in zip(node.inputs, input_shapes):
+            assert inp_shp is None or len(inp_shp) == inp.ndim
+
         # sequences
+        # We skip iputs_shapes[0] as it is the total or current number
+        # of iteration
         seqs_shape = [x[1:] for x in input_shapes[1:1 + self.n_seqs]]
 
         # mit_mot, mit_sot, sit_sot

theano/scan_module/scan_opt.py

@@ -374,7 +374,13 @@ class ScanSaveMem(gof.Optimizer):
             else:
                 return tensor.as_tensor_variable(x)
 
-        shape_of = node.env.shape_feature.shape_of
+        if hasattr(env, 'shape_feature'):
+            shape_of = node.env.shape_feature.shape_of
+        else:
+            # Each call site of shape_of is in a try..except
+            # That use a default version when the variable is not
+            # in the dictionary
+            shape_of = {}
         # 1. Initialization of variables
         # Note 1) We do not actually care about outputs representing shared
         # variables (those have no intermediate values) so it is safer to
@@ -472,12 +478,12 @@ class ScanSaveMem(gof.Optimizer):
                     if i > op.n_mit_mot:
                         try:
                             length = shape_of[out][0]
-                        except Exception:
+                        except KeyError:
                             length = node.inputs[0] + init_l[i]
                     else:
                         try:
                             length = shape_of[out][0]
-                        except Exception:
+                        except KeyError:
                             length = out.shape[0]
                     cf_slice = tensor.basic.get_canonical_form_slice(
                                                     this_slice[0], length)
@@ -575,7 +581,7 @@ class ScanSaveMem(gof.Optimizer):
                     else:
                         try:
                             length = shape_of[out][0]
-                        except Exception:
+                        except KeyError:
                             length = out.shape[0]
                     cf_slice = tensor.basic.get_canonical_form_slice(
                                                     this_slice[0], length)

theano/scan_module/scan_utils.py

@@ -372,6 +372,10 @@ def infer_shape(outs, inputs, input_shapes):
     # inside.  We don't use the full ShapeFeature interface, but we
     # let it initialize itself with an empty env, otherwise we will
     # need to do it manually
+    for inp, inp_shp in zip(inputs, input_shapes):
+        if inp_shp is not None and len(inp_shp) != inp.ndim:
+            assert len(inp_shp) == inp.ndim
+
     shape_feature = tensor.opt.ShapeFeature()
     shape_feature.on_attach(theano.gof.Env([], []))
 

theano/tensor/blas.py

@@ -423,9 +423,12 @@ class GemmRelated(Op):
     #setup_z_Nz_Sz = None
 
     check_xyz_rank2 = """
-        if (%(_x)s->nd != 2) {PyErr_SetString(PyExc_NotImplementedError, "rank(x) != 2"); %(fail)s;}
-        if (%(_y)s->nd != 2) {PyErr_SetString(PyExc_NotImplementedError, "rank(y) != 2"); %(fail)s;}
-        if (%(_zout)s->nd != 2) {PyErr_SetString(PyExc_NotImplementedError, "rank(z) != 2"); %(fail)s;}
+        if (%(_x)s->nd != 2) {
+            PyErr_Format(PyExc_NotImplementedError, "rank(x) != 2. rank(x) is %%d.", %(_x)s->nd); %(fail)s;}
+        if (%(_y)s->nd != 2) {
+            PyErr_Format(PyExc_NotImplementedError, "rank(y) != 2. rank(y) is %%d.", %(_y)s->nd); %(fail)s;}
+        if (%(_zout)s && %(_zout)s->nd != 2) {
+            PyErr_Format(PyExc_NotImplementedError, "rank(z) != 2. rank(z) is %%d.", %(_zout)s->nd); %(fail)s;}
         """
     check_xyz_double_or_float = """
         if ((%(_x)s->descr->type_num != PyArray_DOUBLE)
@@ -442,7 +445,7 @@ class GemmRelated(Op):
 
         if ((%(_x)s->descr->type_num != %(_y)s->descr->type_num)
             ||(%(_x)s->descr->type_num != %(_zout)s->descr->type_num))
-        { PyErr_SetString(PyExc_NotImplementedError, "type(z), type(y), type(z) are not all the same"); %(fail)s; }
+        { PyErr_SetString(PyExc_NotImplementedError, "type(x), type(y), type(z) are not all the same"); %(fail)s; }
         """
 
     #it is not necessary that a or b have the same type as x,y,z
@@ -626,8 +629,8 @@ class GemmRelated(Op):
 
         return reduce(str.__add__, (
             self.declare_NS,
-            self.setup_z_Nz_Sz,
             self.check_xyz_rank2,
+            self.setup_z_Nz_Sz,
             self.check_xyz_double_or_float,
             self.check_ab_double_or_float,
             self.check_dims,
@@ -644,7 +647,7 @@ class GemmRelated(Op):
             self.end_switch_typenum), '')
 
     def build_gemm_version(self):
-        return (9,)
+        return (10,)
 
 class Gemm(GemmRelated):
     """In-place version of matrix-matrix multiplication (with accumulation):

theano/tensor/opt.py

@@ -34,6 +34,12 @@ from theano.gof import toolbox, DestroyHandler
 from basic import get_constant_value, ShapeError
 
 
+theano.configparser.AddConfigVar('on_shape_error',
+                                 "warn: print a warning and use the default"
+                                 " value. raise: raise an error",
+                                 theano.configparser.EnumStr("warn", "raise"),
+                                 in_c_key=False)
+
 # Utilities
 
 
@@ -101,13 +107,16 @@ def broadcast_like(value, template, env, dtype=None):
     value = T.as_tensor_variable(value)
     if value.type == template.type:
         return value
-    shape_of = env.shape_feature.shape_of
-    if template not in shape_of:
+    if template not in env.variables:
         raise NotImplementedError('broadcast_like currently requires the '
                                   'template Variable to be in the env already')
+    if hasattr(env, 'shape_feature'):
+        new_shape = env.shape_feature.shape_of[template]
+    else:
+        new_shape = template.shape
     if dtype is None:
         dtype = template.dtype
-    rval = T.alloc(T.cast(value, dtype), *shape_of[template])
+    rval = T.alloc(T.cast(value, dtype), *new_shape)
     # the template may have 1s in its shape without being broadcastable
     if rval.broadcastable != template.broadcastable:
         rval = T.unbroadcast(rval, *[i for i in xrange(rval.ndim)
@@ -776,6 +785,11 @@ class ShapeFeature(object):
         if s is None:
             self.shape_of[r] = s
         else:
+            if  r.ndim != len(s):
+                raise ShapeError("Something infered a shape with %d dimensions"
+                                 " for a variable with %d dimensions." % (
+                                 len(s), r.ndim))
+
             shape_vars = [self.unpack(s_i) for s_i in s]
             self.shape_of[r] = tuple(shape_vars)
             for sv in shape_vars:
@@ -893,6 +907,8 @@ class ShapeFeature(object):
             o_shapes = shape_infer(node,
                                    [self.shape_of[r] for r in node.inputs])
         except ShapeError:
+            if config.on_shape_error == "raise":
+                raise
             o_shapes = self.default_infer_shape(node, [self.shape_of[r] for
                                                        r in node.inputs])
         except NotImplementedError, e:
@@ -903,11 +919,15 @@ class ShapeFeature(object):
                     'supported, and one should now use tensor.ShapeError '
                     'instead. The original exception message is: %s' % e)
         except Exception, e:
-            _logger.error(('Failed to infer_shape from Op %s.\nInput shapes:'
-                           '%s\nException encountered during infer_shape: '
-                           '%s\nException message: %s\nTraceback: %s') %
-                          (node.op, [self.shape_of[r] for r in node.inputs],
-                           type(e), str(e), traceback.format_exc()))
+            msg = ('Failed to infer_shape from Op %s.\nInput shapes:'
+                   '%s\nException encountered during infer_shape: '
+                   '%s\nException message: %s\nTraceback: %s') % (
+                node.op, [self.shape_of[r] for r in node.inputs],
+                type(e), str(e), traceback.format_exc())
+            if config.on_shape_error == "raise":
+                raise Exception(msg)
+            else:
+                _logger.error(msg)
             o_shapes = self.default_infer_shape(
                 node, [self.shape_of[r] for r in node.inputs])