Skip to content

P
pytensor
提交 7aa2d1a9 authored 作者: Arnaud Bergeron's avatar Arnaud Bergeron
浏览文件
操作

Move the softmax dnn test to test_dnn.py

上级 b85a130a
隐藏空白字符变更
内嵌 并排
正在显示 包含 134 行增加 和 216 行删除
theano/sandbox/cuda/tests/test_dnn.py
浏览文件 @ 7aa2d1a9
...@@ -451,60 +451,130 @@ def test_pooling_opt(): ...@@ -451,60 +451,130 @@ def test_pooling_opt():
for n in f.maker.fgraph.toposort()]) for n in f.maker.fgraph.toposort()])
def test_log_softmax(): class test_DnnSoftMax(test_.test_SoftMax):
# This is a test for an optimization that depends on CuDNN v3 or gpu_op = dnn.GpuDnnSoftmax
# more recent. Don't test if the CuDNN version is too old. gpu_grad_op = dnn.GpuDnnSoftmaxGrad
if not cuda.dnn.dnn_available() or cuda.dnn.version() < (3000, 3000): mode = mode_with_gpu
raise SkipTest(cuda.dnn.dnn_available.msg) topo_idx = -3
x = T.ftensor4()
softmax_out = dnn.GpuDnnSoftmax('bc01', 'accurate', 'channel')(x)
log_out = T.log(T.as_tensor_variable(softmax_out))
f = theano.function([x], log_out, mode=mode_with_gpu)
# Ensure that the optimization has been applied
dnn_softmax_nodes = [n for n in f.maker.fgraph.toposort() if
isinstance(n.op, cuda.dnn.GpuDnnSoftmax)]
assert len(dnn_softmax_nodes) == 1
assert dnn_softmax_nodes[0].op.algo == "log"
# Ensure that the output of the function is valid
input_shapes = [(3, 4, 5, 6),
(1025, 2, 3, 4),
(2, 1025, 3, 4),
(2, 3, 1025, 4),
(2, 3, 4, 1025),
(66000, 2, 3, 4),
(2, 66000, 3, 4),
(2, 3, 66000, 4),
(2, 3, 4, 66000)]
for inp_shape in input_shapes:
input_val = numpy.random.normal(0, 1, inp_shape).astype("float32")
out = f(input_val)
expected_out = numpy.log(numpy.exp(input_val) /
numpy.exp(input_val).sum(1)[:, None, :, :])
utt.assert_allclose(out, expected_out)
def test_dnn_softmax_grad():
utt.seed_rng()
softmax_op = dnn.GpuDnnSoftmax('bc01', 'accurate', 'channel')
x_val = numpy.random.normal(0, 1, (3, 4, 1, 1)).astype('float32') def setUp(self):
if not cuda.dnn.dnn_available():
raise SkipTest(cuda.dnn.dnn_available.msg)
utt.seed_rng()
utt.verify_grad(softmax_op, [x_val]) def test_dnn_softmax_grad(self):
softmax_op = dnn.GpuDnnSoftmax('bc01', 'accurate', 'channel')
x_val = numpy.random.normal(0, 1, (3, 4, 2, 5)).astype('float32')
x_val2 = numpy.random.normal(0, 1, (3, 4, 1, 1)).astype('float32')
def test_dnn_softmax_grad_opt(): utt.verify_grad(softmax_op, [x_val])
utt.seed_rng()
x_val = numpy.random.normal(0, 1, (3, 4)).astype('float32') utt.verify_grad(softmax_op, [x_val2])
utt.verify_grad(softmax_op, [x_val], mode=mode_with_gpu) def test_cudnn_softmax_grad_opt(self):
# Verify that the SoftmaxGrad -> GpuDnnSoftmaxGrad optimization is
# applied when cudnn is required
y = T.fvector('y')
f = theano.function(
[y],
T.grad(T.nnet.softmax(y).mean(), y),
mode=mode_with_gpu
)
sorted_f = f.maker.fgraph.toposort()
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.sandbox.cuda.dnn.GpuDnnSoftmaxGrad
)]) == 1)
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.tensor.nnet.SoftmaxGrad
)]) == 0)
# Verify that the SoftmaxGrad -> GpuDnnSoftmaxGrad optimization is not
# applied when cudnn is excluded or not available
mode_wo_cudnn = mode_with_gpu.excluding("cudnn")
y = T.fvector('y')
f = theano.function(
[y],
T.grad(T.nnet.softmax(y).mean(), y),
mode=mode_wo_cudnn
)
sorted_f = f.maker.fgraph.toposort()
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.sandbox.cuda.dnn.GpuDnnSoftmaxGrad
)]) == 0)
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.tensor.nnet.SoftmaxGrad
)]) == 1)
# Verify that the SoftmaxGrad -> GpuDnnSoftmaxGrad do not
# crash with manual graph
y = T.fvector('y')
o = theano.tensor.nnet.SoftmaxGrad()(y, y*2)
f = theano.function([y], o, mode=mode_with_gpu)
sorted_f = f.maker.fgraph.toposort()
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.sandbox.cuda.dnn.GpuDnnSoftmaxGrad
)]) == 1)
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.tensor.nnet.SoftmaxGrad
)]) == 0)
def test_log_softmax(self):
# This is a test for an optimization that depends on CuDNN v3 or
# more recent. Don't test if the CuDNN version is too old.
if cuda.dnn.version() < (3000, 3000):
raise SkipTest("Log-softmax is only in cudnn v3+")
x = T.ftensor4()
softmax_out = dnn.GpuDnnSoftmax('bc01', 'accurate', 'channel')(x)
log_out = T.log(T.as_tensor_variable(softmax_out))
f = theano.function([x], log_out, mode=mode_with_gpu)
# Ensure that the optimization has been applied
dnn_softmax_nodes = [n for n in f.maker.fgraph.toposort() if
isinstance(n.op, cuda.dnn.GpuDnnSoftmax)]
assert len(dnn_softmax_nodes) == 1
assert dnn_softmax_nodes[0].op.algo == "log"
# Ensure that the output of the function is valid
input_shapes = [(3, 4, 5, 6),
(1025, 2, 3, 4),
(2, 1025, 3, 4),
(2, 3, 1025, 4),
(2, 3, 4, 1025),
(66000, 2, 3, 4),
(2, 66000, 3, 4),
(2, 3, 66000, 4),
(2, 3, 4, 66000)]
for inp_shape in input_shapes:
input_val = numpy.random.normal(0, 1, inp_shape).astype("float32")
out = f(input_val)
expected_out = numpy.log(numpy.exp(input_val) /
numpy.exp(input_val).sum(1)[:, None, :, :])
utt.assert_allclose(out, expected_out)
def test_dnn_tag(): def test_dnn_tag():
......
theano/sandbox/cuda/tests/test_nnet.py
浏览文件 @ 7aa2d1a9
...@@ -212,6 +212,12 @@ def test_softmax_with_bias(): ...@@ -212,6 +212,12 @@ def test_softmax_with_bias():
class test_SoftMax(unittest.TestCase): class test_SoftMax(unittest.TestCase):
gpu_op = cuda.nnet.GpuSoftmax
mode = mode_with_gpu.excluding("cudnn")
do_big = True
do_0 = True
topo_idx = -2
def _test_softmax( def _test_softmax(
self, self,
x, x,
...@@ -219,7 +225,6 @@ class test_SoftMax(unittest.TestCase): ...@@ -219,7 +225,6 @@ class test_SoftMax(unittest.TestCase):
f_z, f_z,
f_gpu_z, f_gpu_z,
cmp, cmp,
gpu_mode,
check_types check_types
): ):
""" """
...@@ -232,7 +237,7 @@ class test_SoftMax(unittest.TestCase): ...@@ -232,7 +237,7 @@ class test_SoftMax(unittest.TestCase):
f_gpu_z_out = f_gpu_z(x_gpu) f_gpu_z_out = f_gpu_z(x_gpu)
f = theano.function([x], f_z_out, mode=mode_without_gpu) f = theano.function([x], f_z_out, mode=mode_without_gpu)
f_gpu = theano.function([x_gpu], f_gpu_z_out, mode=gpu_mode) f_gpu = theano.function([x_gpu], f_gpu_z_out, mode=self.mode)
check_types(f, f_gpu) check_types(f, f_gpu)
# we need to test n>32*1024 to check that we make the block loop. # we need to test n>32*1024 to check that we make the block loop.
...@@ -261,16 +266,15 @@ class test_SoftMax(unittest.TestCase): ...@@ -261,16 +266,15 @@ class test_SoftMax(unittest.TestCase):
return f, f_gpu return f, f_gpu
def _cmp(self, n, m, f, f_gpu): def _cmp(self, n, m, f, f_gpu):
# print "test_softmax",n,m
data = numpy.arange(n * m, dtype='float32').reshape(n, m) data = numpy.arange(n * m, dtype='float32').reshape(n, m)
out = f(data) out = f(data)
gout = f_gpu(data) gout = f_gpu(data)
assert numpy.allclose(out, gout), numpy.absolute(out - gout) utt.assert_allclose(out, gout)
def _check_types(self, graph, graph_gpu, topo_idx, f_type, f_gpu_type): def _check_types(self, graph, graph_gpu, f_type, f_gpu_type):
assert isinstance(graph.maker.fgraph.toposort()[-1].op, f_type) assert isinstance(graph.maker.fgraph.toposort()[-1].op, f_type)
assert isinstance( assert isinstance(
graph_gpu.maker.fgraph.toposort()[topo_idx].op, graph_gpu.maker.fgraph.toposort()[self.topo_idx].op,
f_gpu_type f_gpu_type
) )
...@@ -278,44 +282,12 @@ class test_SoftMax(unittest.TestCase): ...@@ -278,44 +282,12 @@ class test_SoftMax(unittest.TestCase):
x = T.fmatrix('x') x = T.fmatrix('x')
z = T.nnet.softmax_op z = T.nnet.softmax_op
def check_types_without_cudnn(graph, graph_gpu): def check_types(graph, graph_gpu):
self._check_types(
graph,
graph_gpu,
-2,
type(z),
cuda.nnet.GpuSoftmax
)
mode_wo_cudnn = mode_with_gpu.excluding("cudnn")
f, f_gpu = self._test_softmax(
x,
x,
z,
z,
self._cmp,
mode_wo_cudnn,
check_types_without_cudnn
)
# cuDNN R1 cannot handle these test cases but the Theano softmax can so
# we test them only for the Theano softmax.
self._cmp(2 << 15, 5, f, f_gpu)
self._cmp(0, 10, f, f_gpu)
def test_softmax_cudnn(self):
if not cuda.dnn.dnn_available():
raise SkipTest(cuda.dnn.dnn_available.msg)
x = T.fmatrix('x')
z = T.nnet.softmax_op
def check_types_with_cudnn(graph, graph_gpu):
self._check_types( self._check_types(
graph, graph,
graph_gpu, graph_gpu,
-3,
type(z), type(z),
theano.sandbox.cuda.dnn.GpuDnnSoftmax self.gpu_op
) )
f, f_gpu = self._test_softmax( f, f_gpu = self._test_softmax(
...@@ -324,134 +296,10 @@ class test_SoftMax(unittest.TestCase): ...@@ -324,134 +296,10 @@ class test_SoftMax(unittest.TestCase):
z, z,
z, z,
self._cmp, self._cmp,
mode_with_gpu,
check_types_with_cudnn
)
def test_cudnn_softmax_grad(self):
if not cuda.dnn.dnn_available():
raise SkipTest(cuda.dnn.dnn_available.msg)
def cmp(n, m, f, f_gpu):
data = numpy.arange(n * m, dtype='float32').reshape(n, m)
gdata = numpy.asarray(data)[:, :, None, None]
out = f(data)
gout = numpy.asarray(f_gpu(gdata))[:, :, 0, 0]
assert numpy.allclose(out, gout), numpy.absolute(out - gout)
x = T.matrix('x', 'float32')
x_gpu = T.tensor4('x_gpu', 'float32')
f_z = T.nnet.softmax_op
f_gpu = theano.sandbox.cuda.dnn.GpuDnnSoftmax(
'bc01',
'accurate',
'channel'
)
# Verify the grad operation
dims = (2, 3, 4, 5)
gdata = numpy.arange(
numpy.product(dims),
dtype='float32'
).reshape(dims)
T.verify_grad(f_gpu, [gdata], rng=numpy.random,
mode=mode_with_gpu)
def check_types(graph, graph_gpu):
self._check_types(
graph,
graph_gpu,
-1,
type(f_z),
theano.sandbox.cuda.dnn.GpuDnnSoftmax
)
def check_types_opt(graph, graph_gpu):
assert isinstance(graph.maker.fgraph.toposort()[-1].op, type(f_z))
assert len([n for n in graph_gpu.maker.fgraph.toposort()
if isinstance(
n.op,
theano.sandbox.cuda.dnn.GpuDnnSoftmax
)]) == 1
# Verify that the CPU and GPU implementations return the same results
# up to a tolerance.
self._test_softmax(
x,
x_gpu,
f_z,
f_gpu,
cmp,
mode_with_gpu,
check_types check_types
) )
mode_w_cudnn = mode_with_gpu.including("cudnn") if do_big:
self._test_softmax( self._cmp(2 << 15, 5, f, f_gpu)
x, x, f_z, f_z, self._cmp, if do_0:
mode_w_cudnn, check_types_opt self._cmp(0, 10, f, f_gpu)
)
# Verify that the SoftmaxGrad -> GpuDnnSoftmaxGrad optimization is
# applied when cudnn is required
y = T.fvector('y')
f = theano.function(
[y],
T.grad(T.nnet.softmax(y).mean(), y),
mode=mode_with_gpu
)
sorted_f = f.maker.fgraph.toposort()
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.sandbox.cuda.dnn.GpuDnnSoftmaxGrad
)]) == 1)
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.tensor.nnet.SoftmaxGrad
)]) == 0)
# Verify that the SoftmaxGrad -> GpuDnnSoftmaxGrad optimization is not
# applied when cudnn is excluded or not available
mode_wo_cudnn = mode_with_gpu.excluding("cudnn")
y = T.fvector('y')
f = theano.function(
[y],
T.grad(T.nnet.softmax(y).mean(), y),
mode=mode_wo_cudnn
)
sorted_f = f.maker.fgraph.toposort()
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.sandbox.cuda.dnn.GpuDnnSoftmaxGrad
)]) == 0)
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.tensor.nnet.SoftmaxGrad
)]) == 1)
# Verify that the SoftmaxGrad -> GpuDnnSoftmaxGrad do not
# crash with manual graph
y = T.fvector('y')
o = theano.tensor.nnet.SoftmaxGrad()(y, y*2)
f = theano.function([y], o, mode=mode_with_gpu)
sorted_f = f.maker.fgraph.toposort()
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.sandbox.cuda.dnn.GpuDnnSoftmaxGrad
)]) == 1)
assert(len([i
for i in sorted_f
if isinstance(
i.op,
theano.tensor.nnet.SoftmaxGrad
)]) == 0)
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论