Skip to content

P
pytensor
提交 ce1b02e7 authored 作者: Brandon T. Willard's avatar Brandon T. Willard
浏览文件
操作

Remove irrelevant function evaluations and C compilation from TestCrossentropyCategorical1Hot

上级 d079273a
隐藏空白字符变更
内嵌 并排
正在显示 包含 206 行增加 和 500 行删除
tests/tensor/nnet/test_nnet.py
浏览文件 @ ce1b02e7
...@@ -10,7 +10,8 @@ from tests.tensor.utils import ( ...@@ -10,7 +10,8 @@ from tests.tensor.utils import (
makeBroadcastTester, makeBroadcastTester,
upcast_int8_nfunc, upcast_int8_nfunc,
) )
from theano import config, gof, printing from theano import config, gof
from theano.compile.mode import OPT_FAST_RUN, optdb
from theano.gof.opt import check_stack_trace from theano.gof.opt import check_stack_trace
from theano.tensor import lvector, matrix, scalar, vector from theano.tensor import lvector, matrix, scalar, vector
from theano.tensor.nnet import ( from theano.tensor.nnet import (
...@@ -307,9 +308,7 @@ class TestLogSoftmax(utt.InferShapeTester): ...@@ -307,9 +308,7 @@ class TestLogSoftmax(utt.InferShapeTester):
new_g = softmax_grad(tt.add(*true_div_node.inputs), softmax_grad_node.inputs[1]) new_g = softmax_grad(tt.add(*true_div_node.inputs), softmax_grad_node.inputs[1])
fgraph = gof.FunctionGraph([x], [new_g]) fgraph = gof.FunctionGraph([x], [new_g])
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize( optdb.query(OPT_FAST_RUN).optimize(fgraph)
fgraph
)
assert softmax_grad in [n.op for n in fgraph.toposort()] assert softmax_grad in [n.op for n in fgraph.toposort()]
...@@ -588,9 +587,7 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -588,9 +587,7 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
fgraph = gof.FunctionGraph([x, one_of_n], [op(softmax_op(x), one_of_n)]) fgraph = gof.FunctionGraph([x, one_of_n], [op(softmax_op(x), one_of_n)])
assert fgraph.outputs[0].owner.op == op assert fgraph.outputs[0].owner.op == op
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize( optdb.query(OPT_FAST_RUN).optimize(fgraph)
fgraph
)
assert fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias assert fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias
def test_softmax_optimizations_vector(self): def test_softmax_optimizations_vector(self):
...@@ -600,9 +597,7 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -600,9 +597,7 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
fgraph = gof.FunctionGraph([x, one_of_n], [op(softmax_op(x), one_of_n)]) fgraph = gof.FunctionGraph([x, one_of_n], [op(softmax_op(x), one_of_n)])
assert fgraph.outputs[0].owner.op == op assert fgraph.outputs[0].owner.op == op
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize( optdb.query(OPT_FAST_RUN).optimize(fgraph)
fgraph
)
assert fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias assert fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias
def test_softmax_optimizations_w_bias(self): def test_softmax_optimizations_w_bias(self):
...@@ -610,26 +605,12 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -610,26 +605,12 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
b = tt.vector("b") b = tt.vector("b")
one_of_n = tt.lvector("one_of_n") one_of_n = tt.lvector("one_of_n")
op = crossentropy_categorical_1hot op = crossentropy_categorical_1hot
# xe = op(x, one_of_n)
fgraph = gof.FunctionGraph([x, b, one_of_n], [op(softmax_op(x + b), one_of_n)]) fgraph = gof.FunctionGraph([x, b, one_of_n], [op(softmax_op(x + b), one_of_n)])
assert fgraph.outputs[0].owner.op == op assert fgraph.outputs[0].owner.op == op
# print 'BEFORE' optdb.query(OPT_FAST_RUN).optimize(fgraph)
# for node in fgraph.toposort():
# print node.op
# print printing.pprint(node.outputs[0])
# print '----'
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize(
fgraph
)
# print 'AFTER'
# for node in fgraph.toposort():
# print node.op
# print printing.pprint(node.outputs[0])
# print '===='
assert len(fgraph.toposort()) == 1 assert len(fgraph.toposort()) == 1
assert fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias assert fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias
...@@ -645,19 +626,8 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -645,19 +626,8 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
) )
assert fgraph.outputs[0].owner.op == op assert fgraph.outputs[0].owner.op == op
# print 'BEFORE' optdb.query(OPT_FAST_RUN).optimize(fgraph)
# for node in fgraph.toposort():
# print node.op
# print '----'
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize(
fgraph
)
# print 'AFTER'
# for node in fgraph.toposort():
# print node.op
# print '===='
assert len(fgraph.toposort()) == 2 assert len(fgraph.toposort()) == 2
assert fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias assert fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias
...@@ -668,19 +638,9 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -668,19 +638,9 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
op = crossentropy_categorical_1hot op = crossentropy_categorical_1hot
fgraph = gof.FunctionGraph([x, b, one_of_n], [op(softmax_op(x + b), one_of_n)]) fgraph = gof.FunctionGraph([x, b, one_of_n], [op(softmax_op(x + b), one_of_n)])
assert fgraph.outputs[0].owner.op == op assert fgraph.outputs[0].owner.op == op
# print 'BEFORE'
# for node in fgraph.toposort():
# print node.op
# print printing.pprint(node.outputs[0])
# print '----'
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize( optdb.query(OPT_FAST_RUN).optimize(fgraph)
fgraph
)
# print 'AFTER'
# for node in fgraph.toposort():
# print node.op
# print '===='
assert len(fgraph.toposort()) == 2 assert len(fgraph.toposort()) == 2
assert fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias assert fgraph.outputs[0].owner.op == crossentropy_softmax_argmax_1hot_with_bias
...@@ -696,35 +656,13 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -696,35 +656,13 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
fgraph, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, softmax_op] fgraph, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, softmax_op]
) )
# print 'BEFORE' optdb.query(OPT_FAST_RUN).optimize(fgraph)
# for node in fgraph.toposort():
# print node.op, node.inputs
# print '----'
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize(
fgraph
)
# print 'AFTER' ops = {node.op for node in fgraph.toposort()}
# for node in fgraph.toposort(): assert crossentropy_softmax_argmax_1hot_with_bias not in ops
# print node.op, node.inputs assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_op in ops
has_cx1hot = False assert softmax_grad not in ops
has_cx1hotdx = False
has_softmax = False
has_softmaxdx = False
for node in fgraph.toposort():
if node.op == crossentropy_softmax_argmax_1hot_with_bias:
has_cx1hot = True
if node.op == crossentropy_softmax_1hot_with_bias_dx:
has_cx1hotdx = True
if node.op == softmax_op:
has_softmax = True
if node.op == softmax_grad:
has_softmaxdx = True
assert not has_cx1hot
assert has_cx1hotdx
assert has_softmax
assert not has_softmaxdx
def test_softmax_grad_optimizations_vector(self): def test_softmax_grad_optimizations_vector(self):
x = tt.vector("x") x = tt.vector("x")
...@@ -735,47 +673,15 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -735,47 +673,15 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
g_x = tt.grad(sum_xe, x) g_x = tt.grad(sum_xe, x)
fgraph = gof.FunctionGraph([x, one_of_n], [g_x]) fgraph = gof.FunctionGraph([x, one_of_n], [g_x])
# print 'BEFORE' optdb.query(OPT_FAST_RUN).optimize(fgraph)
# for node in fgraph.toposort():
# print node.op, node.inputs
# print '----'
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize(
fgraph
)
# print 'AFTER' ops = {node.op for node in fgraph.toposort()}
# for node in fgraph.toposort(): assert crossentropy_softmax_argmax_1hot_with_bias not in ops
# print node.op, node.inputs assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_op in ops
has_cx1hot = False assert softmax_grad not in ops
has_cx1hotdx = False
has_softmax = False
has_softmaxdx = False
for node in fgraph.toposort():
if node.op == crossentropy_softmax_argmax_1hot_with_bias:
has_cx1hot = True
if node.op == crossentropy_softmax_1hot_with_bias_dx:
has_cx1hotdx = True
if node.op == softmax_op:
has_softmax = True
if node.op == softmax_grad:
has_softmaxdx = True
assert not has_cx1hot
assert has_cx1hotdx
assert has_softmax
assert not has_softmaxdx
def test_get_rid_of_advanced_indexing_version_of_xent(self): def test_get_rid_of_advanced_indexing_version_of_xent(self):
verbose = 0
# TODO: add the optimization in FAST_COMPILE?
# In the mean time, run it as 'FAST_RUN' instead
mode = theano.compile.mode.get_default_mode()
if mode == theano.compile.mode.get_mode("FAST_COMPILE"):
mode = "FAST_RUN"
rng = np.random.RandomState(utt.fetch_seed())
x_val = rng.randn(3, 5).astype(config.floatX)
b_val = rng.randn(5).astype(config.floatX)
y_val = np.asarray([2, 4, 1])
x = tt.matrix("x") x = tt.matrix("x")
b = tt.vector("b") b = tt.vector("b")
y = tt.lvector("y") y = tt.lvector("y")
...@@ -788,40 +694,24 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -788,40 +694,24 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
tt.sum(-tt.log(softmax(x))[tt.arange(y.shape[0]), y]), tt.sum(-tt.log(softmax(x))[tt.arange(y.shape[0]), y]),
] ]
for expr in expressions: for expr in expressions:
# Verify the optimizer worked on the expressions
f = theano.function([x, y], expr, mode=mode) fgraph = gof.FunctionGraph([x, y], [expr])
# todo: only the first output of the op has a stack trace optdb.query(OPT_FAST_RUN).optimize(fgraph)
# assert check_stack_trace(
# f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias) ops = [node.op for node in fgraph.toposort()]
if verbose: assert len(ops) == 4
theano.printing.debugprint(f) assert crossentropy_softmax_argmax_1hot_with_bias in ops
try: assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
ops = [node.op for node in f.maker.fgraph.toposort()]
assert len(ops) == 4
assert crossentropy_softmax_argmax_1hot_with_bias in ops
assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
f(x_val, y_val)
except Exception:
theano.printing.debugprint(f)
raise
# Also verify the gradient wrt x # Also verify the gradient wrt x
g = theano.function([x, y], tt.grad(expr, x), mode=mode) fgraph = gof.FunctionGraph([x, y], [tt.grad(expr, x)])
assert check_stack_trace( optdb.query(OPT_FAST_RUN).optimize(fgraph)
g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, softmax_op]
) ops = [node.op for node in fgraph.toposort()]
if verbose: assert len(ops) == 2
theano.printing.debugprint(g) assert crossentropy_softmax_1hot_with_bias_dx in ops
try: assert softmax_op in ops
ops = [node.op for node in g.maker.fgraph.toposort()] assert softmax_grad not in ops
assert len(ops) == 2
assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_op in ops
assert softmax_grad not in ops
g(x_val, y_val)
except Exception:
theano.printing.debugprint(g)
raise
# Test that a biased softmax is optimized correctly # Test that a biased softmax is optimized correctly
bias_expressions = [ bias_expressions = [
...@@ -832,40 +722,21 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -832,40 +722,21 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
] ]
for expr in bias_expressions: for expr in bias_expressions:
f = theano.function([x, b, y], expr, mode=mode) fgraph = gof.FunctionGraph([x, b, y], [expr, x])
# todo: only the first output of the op has a stack trace optdb.query(OPT_FAST_RUN).optimize(fgraph)
# assert check_stack_trace(
# f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias) ops = [node.op for node in fgraph.toposort()]
if verbose: assert len(ops) == 2 # [big_op, sum]
theano.printing.debugprint(f) assert crossentropy_softmax_argmax_1hot_with_bias in ops
try:
ops = [node.op for node in f.maker.fgraph.toposort()] fgraph = gof.FunctionGraph([x, b, y], [tt.grad(expr, x)])
assert len(ops) == 2 # [big_op, sum] optdb.query(OPT_FAST_RUN).optimize(fgraph)
assert crossentropy_softmax_argmax_1hot_with_bias in ops
f(x_val, b_val, y_val) ops = [node.op for node in fgraph.toposort()]
except Exception: assert len(ops) == 2
theano.printing.debugprint(f) assert crossentropy_softmax_1hot_with_bias_dx in ops
raise assert softmax_with_bias in ops
g = theano.function([x, b, y], tt.grad(expr, x), mode=mode) assert softmax_grad not in ops
assert check_stack_trace(
g,
ops_to_check=[
crossentropy_softmax_1hot_with_bias_dx,
softmax_with_bias,
],
)
if verbose:
theano.printing.debugprint(g)
try:
ops = [node.op for node in g.maker.fgraph.toposort()]
assert len(ops) == 2
assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_with_bias in ops
assert softmax_grad not in ops
g(x_val, b_val, y_val)
except Exception:
theano.printing.debugprint(g)
raise
# Test that using "mean" instead of sum works, too # Test that using "mean" instead of sum works, too
mean_expressions = [ mean_expressions = [
...@@ -876,40 +747,25 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -876,40 +747,25 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
] ]
for expr in mean_expressions: for expr in mean_expressions:
f = theano.function([x, y], expr, mode=mode)
# todo: only the first output of the op has a stack trace fgraph = gof.FunctionGraph([x, y], [expr])
# assert check_stack_trace( optdb.query(OPT_FAST_RUN).optimize(fgraph)
# f, ops_to_check=[crossentropy_softmax_argmax_1hot_with_bias])
if verbose: ops = [node.op for node in fgraph.toposort()]
theano.printing.debugprint(f) assert len(ops) == 6
try: assert crossentropy_softmax_argmax_1hot_with_bias in ops
ops = [node.op for node in f.maker.fgraph.toposort()] assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
assert len(ops) == 6
assert crossentropy_softmax_argmax_1hot_with_bias in ops fgraph = gof.FunctionGraph([x, y], [tt.grad(expr, x)])
assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)] optdb.query(OPT_FAST_RUN).optimize(fgraph)
f(x_val, y_val)
except Exception: ops = [node.op for node in fgraph.toposort()]
theano.printing.debugprint(f) assert len(ops) == 5
raise # there's an extra dimshuffle in there
# but I can't think of a good rule to get rid of it
g = theano.function([x, y], tt.grad(expr, x), mode=mode) assert crossentropy_softmax_1hot_with_bias_dx in ops
assert check_stack_trace( assert softmax_op in ops
g, ops_to_check=[crossentropy_softmax_1hot_with_bias_dx, softmax_op] assert softmax_grad not in ops
)
if verbose:
theano.printing.debugprint(g)
try:
ops = [node.op for node in g.maker.fgraph.toposort()]
assert len(ops) == 5
# there's an extra dimshuffle in there
# but I can't think of a good rule to get rid of it
assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_op in ops
assert softmax_grad not in ops
g(x_val, y_val)
except Exception:
theano.printing.debugprint(g)
raise
mean_bias_expressions = [ mean_bias_expressions = [
tt.mean(-tt.log(softmax(x + b)[tt.arange(y.shape[0]), y])), tt.mean(-tt.log(softmax(x + b)[tt.arange(y.shape[0]), y])),
...@@ -919,49 +775,25 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -919,49 +775,25 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
] ]
for expr in mean_bias_expressions: for expr in mean_bias_expressions:
f = theano.function([x, b, y], expr, mode=mode)
# todo: only the first output of the op has a stack trace fgraph = gof.FunctionGraph([x, b, y], [expr])
# assert check_stack_trace( optdb.query(OPT_FAST_RUN).optimize(fgraph)
# f, ops_to_check=crossentropy_softmax_argmax_1hot_with_bias)
if verbose: ops = [node.op for node in fgraph.toposort()]
theano.printing.debugprint(f) assert len(ops) == 4
try: assert crossentropy_softmax_argmax_1hot_with_bias in ops
ops = [node.op for node in f.maker.fgraph.toposort()] assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
assert len(ops) == 4
assert crossentropy_softmax_argmax_1hot_with_bias in ops fgraph = gof.FunctionGraph([x, b, y], [tt.grad(expr, x)])
assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)] optdb.query(OPT_FAST_RUN).optimize(fgraph)
except Exception:
theano.printing.debugprint(f) ops = [node.op for node in fgraph.toposort()]
raise assert len(ops) == 5
g = theano.function([x, b, y], tt.grad(expr, x), mode=mode) assert crossentropy_softmax_1hot_with_bias_dx in ops
assert check_stack_trace( assert softmax_with_bias in ops
g, assert softmax_grad not in ops
ops_to_check=[
crossentropy_softmax_1hot_with_bias_dx,
softmax_with_bias,
],
)
if verbose:
theano.printing.debugprint(g)
try:
ops = [node.op for node in g.maker.fgraph.toposort()]
assert len(ops) == 5
assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_with_bias in ops
assert softmax_grad not in ops
g(x_val, b_val, y_val)
except Exception:
theano.printing.debugprint(g)
raise
def test_xent_thing_int32(self): def test_xent_thing_int32(self):
verbose = 0
mode = theano.compile.mode.get_default_mode()
if mode == theano.compile.mode.get_mode("FAST_COMPILE"):
mode = "FAST_RUN"
rng = np.random.RandomState(utt.fetch_seed())
x_val = rng.randn(3, 5).astype(config.floatX)
y_val = np.asarray([2, 4, 1], dtype="int64")
x = tt.matrix("x") x = tt.matrix("x")
y = tt.lvector("y") y = tt.lvector("y")
yi = tt.cast(y, "int32") yi = tt.cast(y, "int32")
...@@ -973,44 +805,25 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -973,44 +805,25 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
] ]
for expr in expressions: for expr in expressions:
# Verify the optimizer worked on the expressions fgraph = gof.FunctionGraph([x, y], [expr])
f = theano.function([x, y], expr, mode=mode) optdb.query(OPT_FAST_RUN).optimize(fgraph)
if verbose:
theano.printing.debugprint(f) ops = [node.op for node in fgraph.toposort()]
try: assert len(ops) == 5
ops = [node.op for node in f.maker.fgraph.toposort()] assert crossentropy_softmax_argmax_1hot_with_bias in ops
assert len(ops) == 5 assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
assert crossentropy_softmax_argmax_1hot_with_bias in ops
assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
f(x_val, y_val)
except Exception:
theano.printing.debugprint(f)
raise
# Also verify the gradient wrt x # Also verify the gradient wrt x
g = theano.function([x, y], tt.grad(expr, x), mode=mode) fgraph = gof.FunctionGraph([x, y], [tt.grad(expr, x)])
if verbose: optdb.query(OPT_FAST_RUN).optimize(fgraph)
theano.printing.debugprint(g)
try:
ops = [node.op for node in g.maker.fgraph.toposort()]
assert len(ops) == 3
assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_op in ops
assert softmax_grad not in ops
g(x_val, y_val)
except Exception:
theano.printing.debugprint(g)
raise
def test_optimize_xent_vector(self): ops = [node.op for node in fgraph.toposort()]
verbose = 0 assert len(ops) == 3
mode = theano.compile.mode.get_default_mode() assert crossentropy_softmax_1hot_with_bias_dx in ops
if mode == theano.compile.mode.get_mode("FAST_COMPILE"): assert softmax_op in ops
mode = "FAST_RUN" assert softmax_grad not in ops
rng = np.random.RandomState(utt.fetch_seed())
x_val = rng.randn(5).astype(config.floatX)
y_val = np.asarray([2])
def test_optimize_xent_vector(self):
x = tt.vector("x") x = tt.vector("x")
y = tt.lvector("y") y = tt.lvector("y")
...@@ -1021,42 +834,24 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -1021,42 +834,24 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
] ]
for expr in bias_expressions: for expr in bias_expressions:
f = theano.function([x, y], expr, mode=mode) fgraph = gof.FunctionGraph([x, y], [expr])
if verbose: optdb.query(OPT_FAST_RUN).optimize(fgraph)
printing.debugprint(f)
try:
ops = [node.op for node in f.maker.fgraph.toposort()]
assert len(ops) == 5
assert crossentropy_softmax_argmax_1hot_with_bias in ops
assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
f(x_val, y_val)
except Exception:
theano.printing.debugprint(f)
raise
g = theano.function([x, y], tt.grad(expr, x), mode=mode)
if verbose:
printing.debugprint(g)
try:
ops = [node.op for node in g.maker.fgraph.toposort()]
assert len(ops) == 4
assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_op in ops
assert softmax_grad not in ops
g(x_val, y_val)
except Exception:
theano.printing.debugprint(g)
raise
def test_optimize_xent_vector2(self): ops = [node.op for node in fgraph.toposort()]
verbose = 0 assert len(ops) == 5
mode = theano.compile.mode.get_default_mode() assert crossentropy_softmax_argmax_1hot_with_bias in ops
if mode == theano.compile.mode.get_mode("FAST_COMPILE"): assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
mode = "FAST_RUN"
rng = np.random.RandomState(utt.fetch_seed())
x_val = rng.randn(5).astype(config.floatX)
b_val = rng.randn(5).astype(config.floatX)
y_val = np.asarray([2])
fgraph = gof.FunctionGraph([x, y], [tt.grad(expr, x)])
optdb.query(OPT_FAST_RUN).optimize(fgraph)
ops = [node.op for node in fgraph.toposort()]
assert len(ops) == 4
assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_op in ops
assert softmax_grad not in ops
def test_optimize_xent_vector2(self):
x = tt.vector("x") x = tt.vector("x")
b = tt.vector("b") b = tt.vector("b")
y = tt.lvector("y") y = tt.lvector("y")
...@@ -1070,53 +865,29 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -1070,53 +865,29 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
] ]
for expr in bias_expressions: for expr in bias_expressions:
f = theano.function([x, b, y], expr, mode=mode) fgraph = gof.FunctionGraph([x, b, y], [expr])
if verbose: optdb.query(OPT_FAST_RUN).optimize(fgraph)
printing.debugprint(f)
try: ops = [node.op for node in fgraph.toposort()]
ops = [node.op for node in f.maker.fgraph.toposort()] # [big_op, sum, dim_shuffle]
# [big_op, sum, dim_shuffle] assert len(ops) == 3
assert len(ops) == 3 assert crossentropy_softmax_argmax_1hot_with_bias in ops
assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
f(x_val, b_val, y_val) with theano.change_flags([("warn.sum_div_dimshuffle_bug", False)]):
except Exception: fgraph = gof.FunctionGraph([x, b, y], [tt.grad(expr, x)])
theano.printing.debugprint(f) optdb.query(OPT_FAST_RUN).optimize(fgraph)
raise
ops = [node.op for node in fgraph.toposort()]
backup = config.warn.sum_div_dimshuffle_bug assert len(ops) <= 6
config.warn.sum_div_dimshuffle_bug = False assert crossentropy_softmax_1hot_with_bias_dx in ops
try: assert softmax_with_bias in ops
g = theano.function([x, b, y], tt.grad(expr, x), mode=mode) assert softmax_grad not in ops
finally:
config.warn.sum_div_dimshuffle_bug = backup
if verbose:
printing.debugprint(g)
try:
ops = [node.op for node in g.maker.fgraph.toposort()]
assert len(ops) <= 6
assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_with_bias in ops
assert softmax_grad not in ops
g(x_val, b_val, y_val)
except Exception:
theano.printing.debugprint(g)
raise
def test_optimize_xent_vector3(self): def test_optimize_xent_vector3(self):
# Same as test_optimize_xent_vector2, but y is the result of # Same as test_optimize_xent_vector2, but y is the result of
# a "flatten", and it used to make the constant-folding # a "flatten", and it used to make the constant-folding
# of arange(y.shape[0]) happen before the xent optimization # of arange(y.shape[0]) happen before the xent optimization
verbose = 0
mode = theano.compile.mode.get_default_mode()
if mode == theano.compile.mode.get_mode("FAST_COMPILE"):
mode = "FAST_RUN"
rng = np.random.RandomState(utt.fetch_seed())
x_val = rng.randn(5).astype(config.floatX)
b_val = rng.randn(5).astype(config.floatX)
y_val = np.asarray([2])
x = tt.vector("x") x = tt.vector("x")
b = tt.vector("b") b = tt.vector("b")
y_ = tt.lvector("y_") y_ = tt.lvector("y_")
...@@ -1131,54 +902,30 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -1131,54 +902,30 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
] ]
for expr in bias_expressions: for expr in bias_expressions:
f = theano.function([x, b, y_], expr, mode=mode) fgraph = gof.FunctionGraph([x, b, y_], [expr])
if verbose: optdb.query(OPT_FAST_RUN).optimize(fgraph)
printing.debugprint(f)
try: ops = [node.op for node in fgraph.toposort()]
ops = [node.op for node in f.maker.fgraph.toposort()] # [big_op, sum, dim_shuffle, flatten]
# [big_op, sum, dim_shuffle, flatten] assert len(ops) <= 4
assert len(ops) <= 4 assert crossentropy_softmax_argmax_1hot_with_bias in ops
assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
f(x_val, b_val, y_val) with theano.change_flags([("warn.sum_div_dimshuffle_bug", False)]):
except Exception: fgraph = gof.FunctionGraph([x, b, y], [tt.grad(expr, x)])
theano.printing.debugprint(f) optdb.query(OPT_FAST_RUN).optimize(fgraph)
raise
ops = [node.op for node in fgraph.toposort()]
backup = config.warn.sum_div_dimshuffle_bug assert len(ops) <= 6
config.warn.sum_div_dimshuffle_bug = False assert crossentropy_softmax_1hot_with_bias_dx in ops
try: assert softmax_with_bias in ops
g = theano.function([x, b, y], tt.grad(expr, x), mode=mode) assert softmax_grad not in ops
finally:
config.warn.sum_div_dimshuffle_bug = backup
if verbose:
printing.debugprint(g)
try:
ops = [node.op for node in g.maker.fgraph.toposort()]
assert len(ops) <= 6
assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_with_bias in ops
assert softmax_grad not in ops
g(x_val, b_val, y_val)
except Exception:
theano.printing.debugprint(g)
raise
def test_optimize_xent_vector4(self): def test_optimize_xent_vector4(self):
# Same as test_optimize_xent_vector2, but y is the result of # Same as test_optimize_xent_vector2, but y is the result of a
# a "specify_shape" that indicates its length is 1, so the # "specify_shape" that indicates its length is 1, so the
# constant-folding of arange(y.shape[0]) happen before the xent # constant-folding of arange(y.shape[0]) happen before the xent
# optimization # optimization
verbose = 0
mode = theano.compile.mode.get_default_mode()
if mode == theano.compile.mode.get_mode("FAST_COMPILE"):
mode = "FAST_RUN"
rng = np.random.RandomState(utt.fetch_seed())
x_val = rng.randn(5).astype(config.floatX)
b_val = rng.randn(5).astype(config.floatX)
y_val = np.asarray([2])
x = tt.vector("x") x = tt.vector("x")
b = tt.vector("b") b = tt.vector("b")
y_ = tt.lvector("y_") y_ = tt.lvector("y_")
...@@ -1193,66 +940,30 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -1193,66 +940,30 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
] ]
for expr in bias_expressions: for expr in bias_expressions:
f = theano.function([x, b, y_], expr, mode=mode) fgraph = gof.FunctionGraph([x, b, y_], [expr])
if verbose: optdb.query(OPT_FAST_RUN).optimize(fgraph)
printing.debugprint(f)
try: ops = [node.op for node in fgraph.toposort()]
ops = [node.op for node in f.maker.fgraph.toposort()] # [big_op, sum, dim_shuffle, specify_shape]
# [big_op, sum, dim_shuffle, specify_shape] assert len(ops) <= 4
assert len(ops) <= 4 assert crossentropy_softmax_argmax_1hot_with_bias in ops
assert crossentropy_softmax_argmax_1hot_with_bias in ops assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
assert not [1 for o in ops if isinstance(o, tt.AdvancedSubtensor)]
f(x_val, b_val, y_val) with theano.change_flags([("warn.sum_div_dimshuffle_bug", False)]):
except Exception: fgraph = gof.FunctionGraph([x, b, y], [tt.grad(expr, x)])
theano.printing.debugprint(f) optdb.query(OPT_FAST_RUN).optimize(fgraph)
raise
ops = [node.op for node in fgraph.toposort()]
backup = config.warn.sum_div_dimshuffle_bug assert len(ops) <= 6
config.warn.sum_div_dimshuffle_bug = False assert crossentropy_softmax_1hot_with_bias_dx in ops
try: assert softmax_with_bias in ops
g = theano.function([x, b, y], tt.grad(expr, x), mode=mode) assert softmax_grad not in ops
finally:
config.warn.sum_div_dimshuffle_bug = backup
if verbose:
printing.debugprint(g)
try:
ops = [node.op for node in g.maker.fgraph.toposort()]
assert len(ops) <= 6
assert crossentropy_softmax_1hot_with_bias_dx in ops
assert softmax_with_bias in ops
assert softmax_grad not in ops
g(x_val, b_val, y_val)
except Exception:
theano.printing.debugprint(g)
raise
def test_crossentropy_softmax_1hot_with_bias_dxcale_cost(self): def test_crossentropy_softmax_1hot_with_bias_dxcale_cost(self):
# TODO: add the optimization in FAST_COMPILE?
# In the mean time, run it as 'FAST_RUN' instead
mode = theano.compile.mode.get_default_mode()
if mode == theano.compile.mode.get_mode("FAST_COMPILE"):
mode = "FAST_RUN"
rng = np.random.RandomState(utt.fetch_seed())
x_val = rng.randn(3, 5).astype(config.floatX)
y_val = np.asarray([2, 4, 1])
x = tt.matrix("x") x = tt.matrix("x")
y = tt.lvector("y") y = tt.lvector("y")
a = tt.scalar("a") a = tt.scalar("a")
def validate_fn_graph(func):
# The graph of the function should not have softmax anymore
has_cx1hot = False
has_softmax = False
for node in func.maker.fgraph.toposort():
if node.op == crossentropy_softmax_argmax_1hot_with_bias:
has_cx1hot = True
if node.op == softmax_op:
has_softmax = True
assert has_cx1hot
assert not has_softmax
def validate_grad_graph(func): def validate_grad_graph(func):
# The graph of the gradient should not have softmaxgrad anymore # The graph of the gradient should not have softmaxgrad anymore
has_cx1hotdx = False has_cx1hotdx = False
...@@ -1291,37 +1002,38 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester): ...@@ -1291,37 +1002,38 @@ class TestCrossentropyCategorical1Hot(utt.InferShapeTester):
] ]
for expr in expressions: for expr in expressions:
# Verify the optimizer worked on the expressions fgraph = gof.FunctionGraph([x, y, a], [expr])
f = theano.function([x, y, a], expr, mode=mode) optdb.query(OPT_FAST_RUN).optimize(fgraph)
try:
assert 5 <= len(f.maker.fgraph.toposort()) <= 10 assert 5 <= len(fgraph.toposort()) <= 10
validate_fn_graph(f)
f(x_val, y_val, 0.1) ops = {node.op for node in fgraph.toposort()}
except Exception: assert crossentropy_softmax_argmax_1hot_with_bias in ops
theano.printing.debugprint(f) assert softmax_op not in ops
raise
# Verify the gradient wrt x # Verify the gradient wrt x
g = theano.function([x, y, a], tt.grad(expr, x), mode=mode) fgraph = gof.FunctionGraph([x, y, a], [tt.grad(expr, x)])
try: optdb.query(OPT_FAST_RUN).optimize(fgraph)
assert 3 <= len(g.maker.fgraph.toposort()) <= 6
validate_grad_graph(g) assert 3 <= len(fgraph.toposort()) <= 6
g(x_val, y_val, 0.1)
except Exception: ops = {node.op for node in fgraph.toposort()}
theano.printing.debugprint(g) assert crossentropy_softmax_1hot_with_bias_dx in ops
raise assert softmax_op in ops
assert softmax_grad not in ops
# Verify the gradient when providing output gradient # Verify the gradient when providing output gradient
h = theano.function( fgraph = gof.FunctionGraph(
[x, y, a], tt.grad(expr, x, known_grads={expr: a * x.sum()}), mode=mode [x, y, a], [tt.grad(expr, x, known_grads={expr: a * x.sum()})]
) )
try: optdb.query(OPT_FAST_RUN).optimize(fgraph)
assert 6 <= len(h.maker.fgraph.toposort()) <= 8
validate_grad_graph(h) assert 6 <= len(fgraph.toposort()) <= 8
h(x_val, y_val, 0.1)
except Exception: ops = {node.op for node in fgraph.toposort()}
theano.printing.debugprint(h) assert crossentropy_softmax_1hot_with_bias_dx in ops
raise assert softmax_op in ops
assert softmax_grad not in ops
def test_argmax_pushdown(): def test_argmax_pushdown():
...@@ -1330,9 +1042,7 @@ def test_argmax_pushdown(): ...@@ -1330,9 +1042,7 @@ def test_argmax_pushdown():
# test that the max_and_argmax is pushed down if the max is not used # test that the max_and_argmax is pushed down if the max is not used
out = tt.max_and_argmax(sm(tt.exp(tt.tanh(sigmoid(x)))), axis=-1)[1] out = tt.max_and_argmax(sm(tt.exp(tt.tanh(sigmoid(x)))), axis=-1)[1]
fgraph = gof.FunctionGraph([x], [out]) fgraph = gof.FunctionGraph([x], [out])
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize( optdb.query(OPT_FAST_RUN).optimize(fgraph)
fgraph
)
# print 'AFTER' # print 'AFTER'
# for node in fgraph.toposort(): # for node in fgraph.toposort():
...@@ -1349,9 +1059,7 @@ def test_argmax_pushdown(): ...@@ -1349,9 +1059,7 @@ def test_argmax_pushdown():
backup = config.warn.argmax_pushdown_bug backup = config.warn.argmax_pushdown_bug
config.warn.argmax_pushdown_bug = False config.warn.argmax_pushdown_bug = False
try: try:
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize( optdb.query(OPT_FAST_RUN).optimize(fgraph)
fgraph
)
finally: finally:
config.warn.argmax_pushdown_bug = backup config.warn.argmax_pushdown_bug = backup
...@@ -1372,7 +1080,7 @@ def test_argmax_pushdown_bias(): ...@@ -1372,7 +1080,7 @@ def test_argmax_pushdown_bias():
out = tt.argmax(softmax_with_bias(x, b), axis=-1) out = tt.argmax(softmax_with_bias(x, b), axis=-1)
fgraph = gof.FunctionGraph([x, b], [out]) fgraph = gof.FunctionGraph([x, b], [out])
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize(fgraph) optdb.query(OPT_FAST_RUN).optimize(fgraph)
# print 'AFTER' # print 'AFTER'
# for node in fgraph.toposort(): # for node in fgraph.toposort():
...@@ -1392,9 +1100,7 @@ def test_argmax_pushdown_bias(): ...@@ -1392,9 +1100,7 @@ def test_argmax_pushdown_bias():
backup = config.warn.argmax_pushdown_bug backup = config.warn.argmax_pushdown_bug
config.warn.argmax_pushdown_bug = False config.warn.argmax_pushdown_bug = False
try: try:
theano.compile.mode.optdb.query(theano.compile.mode.OPT_FAST_RUN).optimize( optdb.query(OPT_FAST_RUN).optimize(fgraph)
fgraph
)
finally: finally:
config.warn.argmax_pushdown_bug = backup config.warn.argmax_pushdown_bug = backup
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论