Skip to content

P
pytensor
提交 c675144a authored 作者: notoraptor's avatar notoraptor
浏览文件
操作

Add special implementations for dnn_conv, dnn_gradinput, dnn_gradweight

that take algo, alpha and beta as parameters. Rewrite tests to handle both 2D and 3D conv correctly. Simplify gradinput and gradweight tests.
上级 546daeff
隐藏空白字符变更
内嵌 并排
正在显示 包含 222 行增加 和 121 行删除
theano/gpuarray/tests/check_dnn.py
浏览文件 @ c675144a
......@@ -6,38 +6,150 @@ import numpy as np
import theano
import theano.tests.unittest_tools as utt
from theano.configdefaults import (SUPPORTED_DNN_CONV_ALGO_FWD, SUPPORTED_DNN_CONV_ALGO_BWD_FILTER, SUPPORTED_DNN_CONV_ALGO_BWD_DATA)
from theano.compile.ops import shape_i_op
from theano.configdefaults import (SUPPORTED_DNN_CONV_ALGO_FWD, SUPPORTED_DNN_CONV3D_ALGO_FWD,
SUPPORTED_DNN_CONV_ALGO_BWD_FILTER, SUPPORTED_DNN_CONV3D_ALGO_BWD_FILTER,
SUPPORTED_DNN_CONV_ALGO_BWD_DATA, SUPPORTED_DNN_CONV3D_ALGO_BWD_DATA)
from theano.tensor.nnet.abstract_conv import get_conv_output_shape, assert_conv_shape
from theano.tensor.opt import Assert
from .config import mode_with_gpu, ref_cast
from .. import dnn
from ..basic_ops import infer_context_name, as_gpuarray_variable, gpu_contiguous, GpuAllocEmpty
from ..dnn import (GpuDnnConvDesc, GpuDnnConv, GpuDnnConvGradW, GpuDnnConvGradI, version, get_precision)
SUPPORTED_DNN_CONV_PRECISION = ('float16', 'float32', 'float64')
PRECISIONS = ('float16', 'float32', 'float64')
def get_available_precisions():
# Starting from floatX and up to max supported precision (float64).
return SUPPORTED_DNN_CONV_PRECISION[SUPPORTED_DNN_CONV_PRECISION.index(theano.config.floatX):]
# Starting from floatX up to max supported precision (float64).
return PRECISIONS[PRECISIONS.index(theano.config.floatX):]
def array_like_conv_output(inputs_shape, filters_shape, border_mode, subsample, dilation):
# Return an random array with inferred convolution output shape.
out_shp = get_conv_output_shape(inputs_shape, filters_shape,
border_mode,
subsample,
filter_dilation=dilation)
out_shp = assert_conv_shape(out_shp)
return np.random.random(out_shp).astype(theano.config.floatX)
# We provide a special implementation of dnn_conv, dnn_gradweight and dnn_gradinput
# that take algo, alpha, beta and out as parameters.
def dnn_conv(img, kerns, alpha=1, beta=0, out=None, border_mode='valid', subsample=(1, 1), dilation=(1, 1),
conv_mode='conv', algo=None, precision=None):
# Establish dtype in which to perform the computation of the convolution
precision = get_precision(precision, [img, kerns])
ctx_name = infer_context_name(img, kerns)
img = gpu_contiguous(img)
kerns = gpu_contiguous(kerns)
desc = GpuDnnConvDesc(border_mode=border_mode, subsample=subsample, dilation=dilation,
conv_mode=conv_mode, precision=precision)(kerns.shape)
desc_op = desc.owner.op
# We can use Shape_i and bypass the infer_shape here as this is on
# the input of node and it will always be present.
ishape = [shape_i_op(i)(img) for i in range(img.ndim)]
kshape = [shape_i_op(i)(kerns) for i in range(kerns.ndim)]
out_shp = get_conv_output_shape(ishape, kshape,
desc_op.border_mode,
desc_op.subsample,
filter_dilation=dilation)
out_shp = assert_conv_shape(out_shp)
if beta != 0:
assert out is not None
out = as_gpuarray_variable(out, ctx_name)
out = gpu_contiguous(out)
check = Assert('GpuDnnConv: qiven output (for beta not null) does not have expected shape')
real_out = check(out, theano.tensor.all(theano.tensor.eq(out.shape, out_shp)))
else:
real_out = GpuAllocEmpty(dtype=img.dtype, context_name=ctx_name)(*out_shp)
return GpuDnnConv(algo=algo)(img, kerns, real_out, desc, alpha, beta)
def dnn_gradweight(img, topgrad, kerns_shp, alpha=1, beta=0, out=None, border_mode='valid', subsample=(1, 1),
dilation=(1, 1), conv_mode='conv', algo=None, precision=None):
ctx_name = infer_context_name(img, topgrad)
img = as_gpuarray_variable(img, ctx_name)
topgrad = as_gpuarray_variable(topgrad, ctx_name)
img = gpu_contiguous(img)
topgrad = gpu_contiguous(topgrad)
kerns_shp = theano.tensor.as_tensor_variable(kerns_shp)
precision = get_precision(precision, [img, topgrad])
desc = GpuDnnConvDesc(border_mode=border_mode, subsample=subsample, dilation=dilation,
conv_mode=conv_mode, precision=precision)(kerns_shp)
if beta == 0:
real_out = GpuAllocEmpty(dtype=img.dtype, context_name=ctx_name)(*kerns_shp)
else:
assert out is not None
out = as_gpuarray_variable(out, ctx_name)
out = gpu_contiguous(out)
check = Assert('GpuDnnConvGradW: qiven output (for beta not null) does not have expected shape')
real_out = check(out, theano.tensor.all(theano.tensor.eq(out.shape, kerns_shp)))
return GpuDnnConvGradW(algo=algo)(img, topgrad, real_out, desc, alpha, beta)
def dnn_gradinput(kerns, topgrad, img_shp, alpha=1, beta=0, out=None, border_mode='valid', subsample=(1, 1),
dilation=(1, 1), conv_mode='conv', algo=None, precision=None):
ctx_name = infer_context_name(kerns, topgrad)
kerns = as_gpuarray_variable(kerns, ctx_name)
topgrad = as_gpuarray_variable(topgrad, ctx_name)
kerns = gpu_contiguous(kerns)
topgrad = gpu_contiguous(topgrad)
img_shp = theano.tensor.as_tensor_variable(img_shp)
precision = get_precision(precision, [kerns, topgrad])
desc = GpuDnnConvDesc(border_mode=border_mode, subsample=subsample, dilation=dilation,
conv_mode=conv_mode, precision=precision)(kerns.shape)
if beta == 0:
real_out = GpuAllocEmpty(dtype=kerns.dtype, context_name=ctx_name)(*img_shp)
else:
assert out is not None
out = as_gpuarray_variable(out, ctx_name)
out = gpu_contiguous(out)
check = Assert('GpuDnnConvGradI: qiven output (for beta not null) does not have expected shape')
real_out = check(out, theano.tensor.all(theano.tensor.eq(out.shape, img_shp)))
return GpuDnnConvGradI(algo=algo)(kerns, topgrad, real_out, desc, alpha, beta)
class BaseTestDnnConv(object):
functions_checked_for_fwd = False
functions_checked_for_gradinput = False
functions_checked_for_gradweights = False
_functions_checked_for_fwd = False
_functions_checked_for_gradinput = False
_functions_checked_for_gradweight = False
# Abstract functions.
# Abstract attributes.
def get_gpu_conv_function(self):
raise NotImplementedError
fwd_algorithms = None
bwd_filter_algorithms = None
bwd_data_algorithms = None
def get_cpu_conv_class(self):
raise NotImplementedError
cpu_conv_class = None
cpu_gradinput_class = None
cpu_gradweight_class = None
# Abstract methods.
def get_cases(self):
# Should return an iterable of test cases. Each test case is a tuple (or list) with following syntax:
# ( (input shape, filter shape, subsample, dilation), border mode, convolution mode )
# ( (input shape, filter shape, subsample, dilation), border mode, convolution mode, alpha, beta )
raise NotImplementedError
def run_conv3d_fwd(self, algo, precision, inputs_shape, filters_shape, subsample, dilation, border_mode, conv_mode):
# Run methods.
def run_conv_fwd(self, algo, precision, parameters):
(inputs_shape, filters_shape, subsample, dilation), border_mode, conv_mode, alpha, beta = parameters
inputs_val = np.random.random(inputs_shape).astype(theano.config.floatX)
filters_val = np.random.random(filters_shape).astype(theano.config.floatX)
......@@ -49,92 +161,91 @@ class BaseTestDnnConv(object):
inputs = theano.shared(inputs_val)
filters = theano.shared(filters_val)
out = None if beta == 0 else array_like_conv_output(inputs_shape, filters_shape, border_mode, subsample,
dilation)
# Compile a theano function for the cuDNN implementation
conv = self.get_gpu_conv_function()(img=inputs, kerns=filters,
border_mode=border_mode, subsample=subsample,
dilation=dilation,
conv_mode=conv_mode,
algo=algo,
precision=precision)
conv = dnn_conv(img=inputs, kerns=filters, alpha=alpha, beta=beta, out=out, border_mode=border_mode,
subsample=subsample, dilation=dilation, conv_mode=conv_mode, algo=algo, precision=precision)
f = theano.function([], conv, mode=mode_with_gpu)
# If conv_mode is 'conv' the reference implementation should use
# filters filpped according to the width, height and time axis
if conv_mode == 'conv':
flipped_filters = filters[:, :, ::-1, ::-1, ::-1]
if inputs.ndim == 5:
flipped_filters = filters[:, :, ::-1, ::-1, ::-1]
else:
flipped_filters = filters[:, :, ::-1, ::-1]
else:
flipped_filters = filters
# Compile a theano function for the reference implementation
conv_ref = self.get_cpu_conv_class()(border_mode=border_mode,
subsample=subsample,
filter_dilation=dilation,
)(ref_cast(inputs), flipped_filters)
conv_ref = self.cpu_conv_class(border_mode=border_mode,
subsample=subsample,
filter_dilation=dilation)(ref_cast(inputs), flipped_filters)
f_ref = theano.function([], conv_ref, mode="FAST_RUN")
if not self.functions_checked_for_fwd:
self.functions_checked_for_fwd = True
assert any(isinstance(node.op, dnn.GpuDnnConv) for node in f.maker.fgraph.apply_nodes)
assert not any(isinstance(node.op, (dnn.GpuDnnConvGradI, dnn.GpuDnnConvGradW))
if not self._functions_checked_for_fwd:
self._functions_checked_for_fwd = True
assert any(isinstance(node.op, GpuDnnConv) for node in f.maker.fgraph.apply_nodes)
assert not any(isinstance(node.op, (GpuDnnConvGradI, GpuDnnConvGradW))
for node in f.maker.fgraph.apply_nodes)
assert not any(isinstance(node.op, (dnn.GpuDnnConv, dnn.GpuDnnConvGradW, dnn.GpuDnnConvGradI))
assert not any(isinstance(node.op, (GpuDnnConv, GpuDnnConvGradW, GpuDnnConvGradI))
for node in f_ref.maker.fgraph.apply_nodes)
# Compare the results of the two implementations
res_ref = f_ref()
res = f()
# raise rtol to make the test pass with more seed.
rtol = None
# Raise tolerance for float16
if theano.config.floatX == 'float16':
rtol = 6e-2
utt.assert_allclose(res_ref, res, rtol=rtol)
def run_conv3d_gradinput(self, algo, precision, inputs_shape, filters_shape, subsample, dilation, border_mode, conv_mode):
# Raise tolerance for float16
rtol = 6e-2 if theano.config.floatX == 'float16' else None
if beta == 0:
utt.assert_allclose(alpha * res_ref, res, rtol=rtol)
else:
print('(conv: beta not null) ', end='')
utt.assert_allclose(alpha * res_ref + beta * out, res, rtol=rtol)
theano.config.dnn.conv.algo_bwd_data = algo
theano.config.dnn.conv.precision = precision
def run_conv_gradinput(self, algo, precision, parameters):
(inputs_shape, filters_shape, subsample, dilation), border_mode, conv_mode, alpha, beta = parameters
inputs_val = np.random.random(inputs_shape).astype(theano.config.floatX)
filters_val = np.random.random(filters_shape).astype(theano.config.floatX)
topgrad_val = array_like_conv_output(inputs_shape, filters_shape, border_mode, subsample, dilation)
inputs = theano.shared(inputs_val)
filters = theano.shared(filters_val)
topgrad = theano.shared(topgrad_val)
# Compile a theano function for the cuDNN implementation
conv = self.get_gpu_conv_function()(img=inputs, kerns=filters,
border_mode=border_mode,
subsample=subsample,
dilation=dilation,
conv_mode=conv_mode)
grad_i = theano.tensor.grad(conv.sum(), [inputs])
grad_i = dnn_gradinput(filters, topgrad, inputs_shape, alpha=alpha, beta=beta, out=inputs_val,
border_mode=border_mode, subsample=subsample, dilation=dilation, conv_mode=conv_mode,
algo=algo, precision=precision)
f = theano.function([], grad_i, mode=mode_with_gpu)
# If conv_mode is 'conv' the reference implementation should use
# filters filpped according to the width, height and time axis
if conv_mode == 'conv':
flipped_filters = filters[:, :, ::-1, ::-1, ::-1]
if filters.ndim == 5:
flipped_filters = filters[:, :, ::-1, ::-1, ::-1]
else:
flipped_filters = filters[:, :, ::-1, ::-1]
else:
flipped_filters = filters
# Compile a theano function for the reference implementation
conv_ref = self.get_cpu_conv_class()(border_mode=border_mode,
subsample=subsample,
filter_dilation=dilation,
)(ref_cast(inputs), flipped_filters)
grad_i_ref, = theano.tensor.grad(conv_ref.sum(), [inputs])
grad_i_ref = self.cpu_gradinput_class(border_mode=border_mode,
subsample=subsample,
filter_dilation=dilation
)(ref_cast(flipped_filters), ref_cast(topgrad), inputs_shape[2:])
f_ref = theano.function([], grad_i_ref, mode="FAST_RUN")
if not self.functions_checked_for_gradinput:
self.functions_checked_for_gradinput = True
assert any(isinstance(node.op, dnn.GpuDnnConvGradI) for node in f.maker.fgraph.apply_nodes)
assert not any(isinstance(node.op, (dnn.GpuDnnConv, dnn.GpuDnnConvGradW))
if not self._functions_checked_for_gradinput:
self._functions_checked_for_gradinput = True
assert any(isinstance(node.op, GpuDnnConvGradI) for node in f.maker.fgraph.apply_nodes)
assert not any(isinstance(node.op, (GpuDnnConv, GpuDnnConvGradW))
for node in f.maker.fgraph.apply_nodes)
assert not any(isinstance(node.op, (dnn.GpuDnnConv, dnn.GpuDnnConvGradW, dnn.GpuDnnConvGradI))
assert not any(isinstance(node.op, (GpuDnnConv, GpuDnnConvGradW, GpuDnnConvGradI))
for node in f_ref.maker.fgraph.apply_nodes)
# Compare the results of the two implementations
......@@ -142,56 +253,44 @@ class BaseTestDnnConv(object):
res = f()
# Needed for big size for some seed
# raise rtol to make the test pass with more seed.
rtol = None
# Raise tolerance for float16
if theano.config.floatX == 'float16':
rtol = 5e-2
utt.assert_allclose(res_ref, res, rtol=rtol)
def run_conv3d_gradweights(self, algo, precision, inputs_shape, filters_shape, subsample, dilation, border_mode, conv_mode):
# Raise tolerance for float16
rtol = 5e-2 if theano.config.floatX == 'float16' else None
utt.assert_allclose(alpha * res_ref + beta * inputs_val, res, rtol=rtol)
theano.config.dnn.conv.algo_bwd_filter = algo
theano.config.dnn.conv.precision = precision
def run_conv_gradweight(self, algo, precision, parameters):
(inputs_shape, filters_shape, subsample, dilation), border_mode, conv_mode, alpha, beta = parameters
inputs_val = np.random.random(inputs_shape).astype(theano.config.floatX)
filters_val = np.random.random(filters_shape).astype(theano.config.floatX)
topgrad_val = array_like_conv_output(inputs_shape, filters_shape, border_mode, subsample, dilation)
inputs = theano.shared(inputs_val)
filters = theano.shared(filters_val)
topgrad = theano.shared(topgrad_val)
# Compile a theano function for the cuDNN implementation
conv = self.get_gpu_conv_function()(img=inputs, kerns=filters,
border_mode=border_mode,
subsample=subsample,
dilation=dilation,
conv_mode=conv_mode)
grad_w, = theano.tensor.grad(conv.sum(), [filters])
grad_w = dnn_gradweight(inputs, topgrad, filters_shape, alpha=alpha, beta=beta, out=filters_val,
border_mode=border_mode, subsample=subsample, dilation=dilation, conv_mode=conv_mode,
algo=algo, precision=precision)
f = theano.function([], grad_w, mode=mode_with_gpu)
# If conv_mode is 'conv' the reference implementation should use
# filters filpped according to the width, height and time axis
if conv_mode == 'conv':
flipped_filters = filters[:, :, ::-1, ::-1, ::-1]
else:
flipped_filters = filters
# Compile a theano function for the reference implementation
conv_ref = self.get_cpu_conv_class()(border_mode=border_mode,
subsample=subsample,
filter_dilation=dilation,
)(ref_cast(inputs), flipped_filters)
grad_w_ref, = theano.tensor.grad(conv_ref.sum(), [filters])
grad_w_ref = self.cpu_gradweight_class(border_mode=border_mode,
subsample=subsample,
filter_dilation=dilation)(ref_cast(inputs), ref_cast(topgrad),
filters_shape[2:])
if conv_mode == 'conv':
grad_w_ref = grad_w_ref[:, :, ::-1, ::-1, ::-1]
f_ref = theano.function([], grad_w_ref, mode="FAST_RUN")
if not self.functions_checked_for_gradweights:
self.functions_checked_for_gradweights = True
assert any(isinstance(node.op, dnn.GpuDnnConvGradW) for node in f.maker.fgraph.apply_nodes)
assert not any(isinstance(node.op, (dnn.GpuDnnConv, dnn.GpuDnnConvGradI))
if not self._functions_checked_for_gradweight:
self._functions_checked_for_gradweight = True
assert any(isinstance(node.op, GpuDnnConvGradW) for node in f.maker.fgraph.apply_nodes)
assert not any(isinstance(node.op, (GpuDnnConv, GpuDnnConvGradI))
for node in f.maker.fgraph.apply_nodes)
assert not any(isinstance(node.op, (dnn.GpuDnnConv, dnn.GpuDnnConvGradW, dnn.GpuDnnConvGradI))
assert not any(isinstance(node.op, (GpuDnnConv, GpuDnnConvGradW, GpuDnnConvGradI))
for node in f_ref.maker.fgraph.apply_nodes)
# Compare the results of the two implementations
......@@ -199,40 +298,40 @@ class BaseTestDnnConv(object):
res = f()
# Needed for big size for some seed
# raise rtol to make the test pass with more seed.
rtol = None
# Raise tolerance for float16
if theano.config.floatX == 'float16':
rtol = 5e-2
utt.assert_allclose(res_ref, res, rtol=rtol)
rtol = 5e-2 if theano.config.floatX == 'float16' else None
utt.assert_allclose(alpha * res_ref + beta * filters_val, res, rtol=rtol)
# Iterable test methods.
def test_fwd(self):
for precision in get_available_precisions():
for algo in SUPPORTED_DNN_CONV_ALGO_FWD:
for (inputs_shape, filters_shape, subsample, dilation), border_mode, conv_mode in self.get_cases():
yield (self.run_conv3d_fwd, algo, precision, inputs_shape, filters_shape,
subsample, dilation, border_mode, conv_mode)
for algo in self.fwd_algorithms:
for parameters in self.get_cases():
yield (self.run_conv_fwd, algo, precision, parameters)
def test_gradinput(self):
for precision in get_available_precisions():
for algo in SUPPORTED_DNN_CONV_ALGO_BWD_DATA:
for (inputs_shape, filters_shape, subsample, dilation), border_mode, conv_mode in self.get_cases():
yield (self.run_conv3d_gradinput, algo, precision, inputs_shape, filters_shape,
subsample, dilation, border_mode, conv_mode)
for algo in self.bwd_data_algorithms:
for parameters in self.get_cases():
yield (self.run_conv_gradinput, algo, precision, parameters)
def test_gradweights(self):
def test_gradweight(self):
for precision in get_available_precisions():
for algo in SUPPORTED_DNN_CONV_ALGO_BWD_FILTER:
for (inputs_shape, filters_shape, subsample, dilation), border_mode, conv_mode in self.get_cases():
yield (self.run_conv3d_gradweights, algo, precision, inputs_shape, filters_shape,
subsample, dilation, border_mode, conv_mode)
for algo in self.bwd_filter_algorithms:
for parameters in self.get_cases():
yield (self.run_conv_gradweight, algo, precision, parameters)
class TestDnnConv2D(BaseTestDnnConv):
def get_gpu_conv_function(self):
return dnn.dnn_conv
fwd_algorithms = SUPPORTED_DNN_CONV_ALGO_FWD
bwd_filter_algorithms = SUPPORTED_DNN_CONV_ALGO_BWD_FILTER
bwd_data_algorithms = SUPPORTED_DNN_CONV_ALGO_BWD_DATA
def get_cpu_conv_class(self):
return theano.tensor.nnet.corr.CorrMM
cpu_conv_class = theano.tensor.nnet.corr.CorrMM
cpu_gradinput_class = theano.tensor.nnet.corr.CorrMM_gradInputs
cpu_gradweight_class = theano.tensor.nnet.corr.CorrMM_gradWeights
def get_cases(self):
# Inspired from:
......@@ -265,17 +364,19 @@ class TestDnnConv2D(BaseTestDnnConv):
local_dilations = [default_dilation]
iterables += [product(product([input_shape], [filter_shape], local_subsamples, local_dilations),
border_modes,
conv_modes)]
conv_modes, [1], [0])]
return chain(*iterables)
class TestDnnConv3D(BaseTestDnnConv):
def get_gpu_conv_function(self):
return dnn.dnn_conv3d
fwd_algorithms = SUPPORTED_DNN_CONV3D_ALGO_FWD
bwd_filter_algorithms = SUPPORTED_DNN_CONV3D_ALGO_BWD_FILTER
bwd_data_algorithms = SUPPORTED_DNN_CONV3D_ALGO_BWD_DATA
def get_cpu_conv_class(self):
return theano.tensor.nnet.corr3d.Corr3dMM
cpu_conv_class = theano.tensor.nnet.corr3d.Corr3dMM
cpu_gradinput_class = theano.tensor.nnet.corr3d.Corr3dMM_gradInputs
cpu_gradweight_class = theano.tensor.nnet.corr3d.Corr3dMM_gradWeights
def get_cases(self):
# small case for quick test.
......@@ -285,7 +386,7 @@ class TestDnnConv3D(BaseTestDnnConv):
dilation = (1, 1, 1)
border_mode = 'valid'
conv_mode = 'conv'
return (((input_shape, filter_shape, subsample, dilation), border_mode, conv_mode),)
return (((input_shape, filter_shape, subsample, dilation), border_mode, conv_mode, 2.1, -5.7),)
def get_cases_real(self):
# Copy of: theano.gpuarray.tests.test_dnn.get_conv3d_test_cases
......@@ -315,7 +416,7 @@ class TestDnnConv3D(BaseTestDnnConv):
[(6, 2, 2, 2, 2), (4, 2, 1, 1, 3), (1, 1, 1), (1, 1, 1)],
[(6, 2, 2, 2, 2), (4, 2, 5, 5, 5), (1, 1, 1), (1, 1, 1)]]
if dnn.version() >= 6000:
if version() >= 6000:
test_shapes.extend([
[(8, 1, 20, 12, 15), (5, 1, 6, 3, 4), (1, 1, 2), (3, 2, 1)],
[(8, 1, 20, 12, 15), (5, 1, 6, 3, 4), (2, 2, 1), (1, 2, 3)]])
......
Markdown 格式
0% 或
您添加了 0 人 到此讨论。请谨慎行事。
请先完成此评论的编辑！
请 注册 或者 后发表评论