Skip to content

P
pytensor
提交 9d55e60f authored 作者: James Bergstra's avatar James Bergstra
浏览文件
操作

Various modifs to make Xent tests pass with new ShapeFeature.

上级 c6fc7c59
隐藏空白字符变更
内嵌 并排
正在显示 包含 177 行增加 和 86 行删除
theano/tensor/nnet/nnet.py
浏览文件 @ 9d55e60f
...@@ -6,7 +6,7 @@ ...@@ -6,7 +6,7 @@
from theano import gof from theano import gof
from theano import printing from theano import printing
from theano.tensor import basic as tensor from theano.tensor import basic as tensor
from theano.tensor import elemwise from theano.tensor import elemwise, dmatrix, fmatrix, dvector, fvector
from theano.tensor import opt from theano.tensor import opt
from theano.compile import optdb from theano.compile import optdb
import numpy import numpy
...@@ -919,6 +919,15 @@ def _check_rows_is_arange_len_labels(rows, labels): ...@@ -919,6 +919,15 @@ def _check_rows_is_arange_len_labels(rows, labels):
shape_of = stop.owner.env.shape_feature.shape_of shape_of = stop.owner.env.shape_feature.shape_of
return shape_of[labels][0] is stop return shape_of[labels][0] is stop
def _is_const(z, val, approx=False):
try:
maybe = opt.get_constant_value(z)
except TypeError:
return False
if approx:
return numpy.allclose(maybe,val)
else:
return numpy.all(maybe == val)
@opt.register_specialize @opt.register_specialize
@gof.local_optimizer([]) @gof.local_optimizer([])
def local_advanced_indexing_crossentropy_onehot(node): def local_advanced_indexing_crossentropy_onehot(node):
...@@ -969,7 +978,7 @@ def local_advanced_indexing_crossentropy_onehot_grad(node): ...@@ -969,7 +978,7 @@ def local_advanced_indexing_crossentropy_onehot_grad(node):
except: except:
return return
if sm is not None and sm.owner and sm.owner.op in (softmax, softmax_with_bias): if (sm is not None) and sm.owner and (sm.owner.op in (softmax, softmax_with_bias)):
sm_w_bias = local_softmax_with_bias.transform(sm.owner) sm_w_bias = local_softmax_with_bias.transform(sm.owner)
if sm_w_bias: if sm_w_bias:
assert sm_w_bias[0].owner.op == softmax_with_bias assert sm_w_bias[0].owner.op == softmax_with_bias
...@@ -1023,13 +1032,7 @@ def local_advanced_indexing_crossentropy_onehot_grad(node): ...@@ -1023,13 +1032,7 @@ def local_advanced_indexing_crossentropy_onehot_grad(node):
return return
# Check that z == zeros_like(softmax(x)) # Check that z == zeros_like(softmax(x))
if z.owner and z.owner.op == tensor.fill: if not _is_const(z, 0):
model, value = z.owner.inputs
if not (model is sm and hasattr(value, 'data') and numpy.all(value.data == 0)):
return
#else: OK
else:
return return
# In the base case (output gradient = 1), incr is -1./sm[arange(len(y)), y] # In the base case (output gradient = 1), incr is -1./sm[arange(len(y)), y]
...@@ -1112,11 +1115,17 @@ def local_advanced_indexing_crossentropy_onehot_grad(node): ...@@ -1112,11 +1115,17 @@ def local_advanced_indexing_crossentropy_onehot_grad(node):
# Second case # Second case
elif out_grad.owner and out_grad.owner.op == tensor.true_div: elif out_grad.owner and out_grad.owner.op == tensor.true_div:
# we know
# we're looking for
# AdvIncSubtensor(zeros, grad_nll, arange(len(y)), y) / softmax
try: try:
num, denom = out_grad.owner.inputs num, denom = out_grad.owner.inputs
except: except:
return return
if denom != sm:
return
# Check the numerator (AdvancedIncSubtensor) # Check the numerator (AdvancedIncSubtensor)
if num.owner and isinstance(num.owner.op, tensor.AdvancedIncSubtensor): if num.owner and isinstance(num.owner.op, tensor.AdvancedIncSubtensor):
try: try:
...@@ -1125,74 +1134,94 @@ def local_advanced_indexing_crossentropy_onehot_grad(node): ...@@ -1125,74 +1134,94 @@ def local_advanced_indexing_crossentropy_onehot_grad(node):
return return
# Check z is zeros_like(log(sm)) # Check z is zeros_like(log(sm))
if z.owner and z.owner.op == tensor.fill: # JB - do we really care if this is zeros?
model, value = z.owner.inputs if not _is_const(z, 0):
return
if z.type not in (dmatrix, fmatrix):
return
# here we know that we are incrementing a matrix of zeros
if model.owner and model.owner.op == tensor.log: if 0:
if sm is model.owner.inputs[0]: if z.owner and z.owner.op == tensor.fill:
log_sm = model model, value = z.owner.inputs
else:
return if model.owner and model.owner.op == tensor.log:
if sm is model.owner.inputs[0]:
log_sm = model
else:
return
if not (hasattr(value, 'data') and numpy.all(value.data == 0)): if not (hasattr(value, 'data') and numpy.all(value.data == 0)):
return
#else: OK
else:
return return
#else: OK
else: else:
return return
else:
if incr.type not in (dvector, fvector):
return return
# Check incr is ((-1.) like log(softmax(x))[arange(len(y)), y]) # here we know that we are incrementing some part of matrix z by a vector
if incr.owner and incr.owner.op == tensor.fill:
model, value = incr.owner.inputs # unless the user has taken care to mark that the data and labels have the
adv_subtensor = None # same number of rows, we cannot be sure here that
outgrad_factor = None # len(y) == len(z)
if model.owner and isinstance(model.owner.op, tensor.AdvancedSubtensor): # However, in the common case that these are predictions and labels it is true.
adv_subtensor = model # We leave it to the Op to crash (and the user to complain) if this assumption is
else: # ever not true.
if model.owner and isinstance(model.owner.op, tensor.Elemwise):
for input in model.owner.inputs: outgrad_factor = None
if input.owner and isinstance(input.owner.op, tensor.AdvancedSubtensor):
adv_subtensor = input if 0:
break # Check incr is ((-1.) like log(softmax(x))[arange(len(y)), y])
#TODO: try them all, not just the first one if incr.owner and incr.owner.op == tensor.fill:
model, value = incr.owner.inputs
adv_subtensor = None
outgrad_factor = None
if model.owner and isinstance(model.owner.op, tensor.AdvancedSubtensor):
adv_subtensor = model
else: else:
return if model.owner and isinstance(model.owner.op, tensor.Elemwise):
for input in model.owner.inputs:
if input.owner and isinstance(input.owner.op, tensor.AdvancedSubtensor):
adv_subtensor = input
break
#TODO: try them all, not just the first one
else:
return
if adv_subtensor is not None:
try:
maybe_log_sm, maybe_rows, maybe_labels = adv_subtensor.owner.inputs
except:
return
if adv_subtensor is not None: if not (maybe_log_sm is log_sm and maybe_rows is rows and maybe_labels is labels):
try: return
maybe_log_sm, maybe_rows, maybe_labels = adv_subtensor.owner.inputs #else: OK
except: else:
return return
if not (maybe_log_sm is log_sm and maybe_rows is rows and maybe_labels is labels): # In the base case, value is the constant '-1'
if hasattr(value, 'data') and numpy.all(value.data == -1):
outgrad_factor = 1.
# Otherwise, it should be a scalar, and the output gradient
# would be -value
elif numpy.all(value.broadcastable):
outgrad_factor = -value
else:
return return
#else: OK
else:
return
# In the base case, value is the constant '-1'
if hasattr(value, 'data') and numpy.all(value.data == -1):
outgrad_factor = 1.
# Otherwise, it should be a scalar, and the output gradient
# would be -value
elif numpy.all(value.broadcastable):
outgrad_factor = -value
else: else:
return return
else:
return
# Check that rows is arange(labels.shape[0]) # Check that rows is arange(labels.shape[0])
if not _check_rows_is_arange_len_labels(rows, labels): if not _check_rows_is_arange_len_labels(rows, labels):
return return
# else, arguments of AdvancedIncSubtensor are OK # else, arguments of AdvancedIncSubtensor are OK
return [crossentropy_softmax_1hot_with_bias_dx(-incr, sm, labels)]
# Check the denominator (sm)
if not denom is sm:
return
# else, numerator and denominator are OK, # else, numerator and denominator are OK,
# it was really case 2. # it was really case 2.
......
theano/tensor/nnet/tests/test_nnet.py
浏览文件 @ 9d55e60f
...@@ -306,14 +306,22 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase): ...@@ -306,14 +306,22 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase):
# Verify the optimizer worked on the expressions # Verify the optimizer worked on the expressions
f = theano.function([x,y], expr, mode=mode) f = theano.function([x,y], expr, mode=mode)
if verbose: print_graph(f) if verbose: print_graph(f)
assert len(f.maker.env.toposort()) == 4 try:
f(x_val, y_val) assert len(f.maker.env.toposort()) == 4
f(x_val, y_val)
except:
theano.printing.debugprint(f)
raise
# Also verify the gradient wrt x # Also verify the gradient wrt x
g = theano.function([x,y], T.grad(expr, x), mode=mode) g = theano.function([x,y], T.grad(expr, x), mode=mode)
if verbose: print_graph(g) if verbose: print_graph(g)
assert len(g.maker.env.toposort()) == 4 try:
g(x_val, y_val) assert len(g.maker.env.toposort()) == 4
g(x_val, y_val)
except:
theano.printing.debugprint(g)
raise
## Test that a biased softmax is optimized correctly ## Test that a biased softmax is optimized correctly
...@@ -326,13 +334,21 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase): ...@@ -326,13 +334,21 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase):
for expr in bias_expressions: for expr in bias_expressions:
f = theano.function([x,b,y], expr, mode=mode) f = theano.function([x,b,y], expr, mode=mode)
if verbose: print_graph(f) if verbose: print_graph(f)
assert len(f.maker.env.toposort()) == 2 # [big_op, sum] try:
f(x_val, b_val, y_val) assert len(f.maker.env.toposort()) == 2 # [big_op, sum]
f(x_val, b_val, y_val)
except:
theano.printing.debugprint(f)
raise
g = theano.function([x,b,y], T.grad(expr, x), mode=mode) g = theano.function([x,b,y], T.grad(expr, x), mode=mode)
if verbose: print_graph(g) if verbose: print_graph(g)
assert len(g.maker.env.toposort()) == 4 try:
g(x_val, b_val, y_val) assert len(g.maker.env.toposort()) == 4
g(x_val, b_val, y_val)
except:
theano.printing.debugprint(g)
raise
## Test that using "mean" instead of sum works, too ## Test that using "mean" instead of sum works, too
mean_expressions = [ mean_expressions = [
...@@ -344,13 +360,22 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase): ...@@ -344,13 +360,22 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase):
for expr in mean_expressions: for expr in mean_expressions:
f = theano.function([x,y], expr, mode=mode) f = theano.function([x,y], expr, mode=mode)
if verbose: print_graph(f) if verbose: print_graph(f)
assert len(f.maker.env.toposort()) == 7 try:
f(x_val, y_val) assert len(f.maker.env.toposort()) == 6
f(x_val, y_val)
except:
theano.printing.debugprint(f)
raise
g = theano.function([x,y], T.grad(expr, x), mode=mode) g = theano.function([x,y], T.grad(expr, x), mode=mode)
if verbose: print_graph(g) if verbose: print_graph(g)
assert len(g.maker.env.toposort()) == 8 try:
g(x_val, y_val) assert len(g.maker.env.toposort()) in (6,7) #there's an extra dimshuffle in there
# but I can't think of a good rule to get rid of it
g(x_val, y_val)
except:
theano.printing.debugprint(g)
raise
mean_bias_expressions = [ mean_bias_expressions = [
T.mean(-T.log(softmax(x+b)[T.arange(y.shape[0]), y])), T.mean(-T.log(softmax(x+b)[T.arange(y.shape[0]), y])),
...@@ -361,12 +386,20 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase): ...@@ -361,12 +386,20 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase):
for expr in mean_bias_expressions: for expr in mean_bias_expressions:
f = theano.function([x,b,y], expr, mode=mode) f = theano.function([x,b,y], expr, mode=mode)
if verbose: print_graph(f) if verbose: print_graph(f)
assert len(f.maker.env.toposort()) == 5 try:
assert len(f.maker.env.toposort()) == 4
except:
theano.printing.debugprint(f)
raise
g = theano.function([x,b,y], T.grad(expr, x), mode=mode) g = theano.function([x,b,y], T.grad(expr, x), mode=mode)
if verbose: print_graph(g) if verbose: print_graph(g)
assert len(g.maker.env.toposort()) == 8 try:
g(x_val, b_val, y_val) assert len(g.maker.env.toposort()) in (6,7)
g(x_val, b_val, y_val)
except:
theano.printing.debugprint(g)
raise
def test_scale_cost(self): def test_scale_cost(self):
...@@ -450,21 +483,33 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase): ...@@ -450,21 +483,33 @@ class T_CrossentropyCategorical1Hot(unittest.TestCase):
for expr in expressions: for expr in expressions:
# Verify the optimizer worked on the expressions # Verify the optimizer worked on the expressions
f = theano.function([x,y,a], expr, mode=mode) f = theano.function([x,y,a], expr, mode=mode)
assert 5 <= len(f.maker.env.toposort()) <= 10 try:
validate_fn_graph(f) assert 5 <= len(f.maker.env.toposort()) <= 10
f(x_val, y_val, 0.1) validate_fn_graph(f)
f(x_val, y_val, 0.1)
except:
theano.printing.debugprint(f)
raise
# Verify the gradient wrt x # Verify the gradient wrt x
g = theano.function([x,y,a], T.grad(expr, x), mode=mode) g = theano.function([x,y,a], T.grad(expr, x), mode=mode)
assert 5 <= len(g.maker.env.toposort()) <= 12 try:
validate_grad_graph(g) assert 5 <= len(g.maker.env.toposort()) <= 12
g(x_val, y_val, 0.1) validate_grad_graph(g)
g(x_val, y_val, 0.1)
except:
theano.printing.debugprint(g)
raise
# Verify the gradient when providing output gradient # Verify the gradient when providing output gradient
h = theano.function([x,y,a], T.grad(expr, x, g_cost=a*x.sum()), mode=mode) h = theano.function([x,y,a], T.grad(expr, x, g_cost=a*x.sum()), mode=mode)
assert 8 <= len(h.maker.env.toposort()) <= 17 try:
validate_grad_graph(h) assert 8 <= len(h.maker.env.toposort()) <= 17
h(x_val, y_val, 0.1) validate_grad_graph(h)
h(x_val, y_val, 0.1)
except:
theano.printing.debugprint(h)
raise
def test_argmax_pushdown(): def test_argmax_pushdown():
......
theano/tensor/opt.py
浏览文件 @ 9d55e60f
...@@ -80,12 +80,15 @@ def get_constant_value(v): ...@@ -80,12 +80,15 @@ def get_constant_value(v):
return v.data return v.data
except: except:
raise TypeError(v) raise TypeError(v)
if v.owner and isinstance(v.owner.op, T.DimShuffle): if v.owner:
return get_constant_value(v.owner.inputs[0]) if isinstance(v.owner.op, T.Alloc):
if v.owner and v.owner.op == T.fill: return get_constant_value(v.owner.inputs[0])
shape, val = v.owner.inputs if isinstance(v.owner.op, T.DimShuffle):
# fill(a,b) fills the shape of 'a' filled with 'b' return get_constant_value(v.owner.inputs[0])
return get_constant_value(val) if v.owner.op == T.fill:
shape, val = v.owner.inputs
# fill(a,b) fills the shape of 'a' filled with 'b'
return get_constant_value(val)
raise TypeError(v) raise TypeError(v)
def scalarconsts_rest(inputs): def scalarconsts_rest(inputs):
...@@ -530,6 +533,20 @@ def local_subtensor_make_vector(node): ...@@ -530,6 +533,20 @@ def local_subtensor_make_vector(node):
_logger.error('failed to index with "%s"' % str(idx)) _logger.error('failed to index with "%s"' % str(idx))
raise raise
@register_specialize
@gof.local_optimizer([T.Alloc])
def local_alloc_unary(node):
"""unary(alloc(x, shp)) -> alloc(unary(x), shp)
"""
if isinstance(node.op, T.Elemwise) and len(node.inputs)==1:
x = node.inputs[0]
if x.owner and isinstance(x.owner.op, T.Alloc):
return [T.Alloc(node.outputs[0].dtype)(
node.op(T.cast(x.owner.inputs[0], x.dtype)),
*x.owner.inputs[1:]
)]
################## ##################
# Subtensor opts # # Subtensor opts #
################## ##################
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论