Merge pull request #6430 from erakra/upsamp2

theano/tensor/nnet/abstract_conv.py

@@ -6,6 +6,7 @@ from __future__ import absolute_import, print_function, division
 import logging
 from six import reraise, integer_types
 import sys
+from fractions import gcd
 
 import theano
 
@@ -1712,8 +1713,16 @@ def bilinear_kernel_2D(ratio, normalize=True):
 
     """
 
-    hkern = bilinear_kernel_1D(ratio=ratio, normalize=normalize).dimshuffle('x', 0)
-    vkern = bilinear_kernel_1D(ratio=ratio, normalize=normalize).dimshuffle(0, 'x')
+    if isinstance(ratio, tuple):
+        ratio_h = ratio[1]
+        ratio_v = ratio[0]
+    else:
+        ratio_h = ratio
+        ratio_v = ratio
+    hkern = bilinear_kernel_1D(ratio=ratio_h,
+                               normalize=normalize).dimshuffle('x', 0)
+    vkern = bilinear_kernel_1D(ratio=ratio_v,
+                               normalize=normalize).dimshuffle(0, 'x')
     kern = hkern * vkern
     return kern
 
@@ -1751,13 +1760,114 @@ def bilinear_kernel_1D(ratio, normalize=True):
     return kern
 
 
+def frac_bilinear_upsampling(input,
+                             ratio=None,
+                             frac_ratio=None):
+    """Compute bilinear upsampling
+    This function will build the symbolic graph for upsampling
+    a tensor by the given ratio using bilinear interpolation.
+
+    Parameters
+    ----------
+    input: symbolic 4D tensor
+        mini-batch of feature map stacks, of shape (batch size,
+        input channels, input rows, input columns) that will be upsampled.
+    ratio: `int or Constant or Scalar Tensor of int* dtype`
+        the ratio by which the input is upsampled in the 2D space (row and
+        col size).
+    frac_ratio: None, tuple of int or tuple of tuples of int
+        The tuple defining the fractional ratio by which the input is
+        upsampled in the 2D space. One fractional ratio should be
+        represented as (numerator, denominator). If row and col ratios are
+        different frac_ratio should be a tuple of fractional ratios, i.e
+        a tuple of tuples.
+    Returns
+    -------
+    symbolic 4D tensor
+        set of feature maps generated by bilinear upsampling. Tensor
+        is of shape (batch size, num_input_channels, input row size * row ratio,
+        input column size * column ratio). Each of these ratios can be fractional.
+    Notes
+    -----
+    :note: The kernel used for bilinear interpolation is fixed (not learned).
+    :note: When the upsampling ratio is even, the last row and column is
+        repeated one extra time compared to the first row and column which makes
+        the upsampled tensor asymmetrical on both sides. This does not happen when
+        the upsampling ratio is odd.
+    :note: This function must get either ratio or frac_ratio as parameter and
+        never both at once.
+    """
+
+    T = theano.tensor
+    row, col = input.shape[2:]
+    up_input = input.reshape((-1, 1, row, col))
+
+    # redefince the ratio depending on the case
+    if frac_ratio is None:
+        if not isinstance(ratio, tuple):
+            ratio = (ratio, ratio)
+        subsample = (1, 1)
+    else:
+        if not isinstance(frac_ratio, tuple):
+            raise ValueError("frac_ratio must be a tuple")
+        else:
+            if isinstance(frac_ratio[0], tuple):
+                f_r = []
+                for i, fr in enumerate(frac_ratio):
+                    p, q = fr
+                    div = gcd(p, q)
+                    f_r.append(tuple(np.array(fr) // div))
+                frac_ratio = tuple(f_r)
+                ratio = (frac_ratio[0][0], frac_ratio[1][0])
+                subsample = (frac_ratio[0][1], frac_ratio[1][1])
+            else:
+                p, q = frac_ratio
+                div = gcd(p, q)
+                frac_ratio = tuple(np.array(frac_ratio) // div)
+                ratio = (frac_ratio[0], frac_ratio[0])
+                subsample = (frac_ratio[1], frac_ratio[1])
+
+    # duplicate borders of the input
+    concat_mat = T.concatenate((up_input[:, :, :1, :], up_input,
+                                up_input[:, :, -1:, :]), axis=2)
+    concat_mat = T.concatenate((concat_mat[:, :, :, :1], concat_mat,
+                                concat_mat[:, :, :, -1:]), axis=3)
+
+    # add padding for the pyramidal kernel
+    double_pad = (2 * T.as_tensor([row, col]) - 1) * np.array(ratio) + 1
+    pad = double_pad // 2
+
+    # build pyramidal kernel
+    kern = bilinear_kernel_2D(ratio=ratio)[np.newaxis, np.newaxis, :, :].astype(theano.config.floatX)
+
+    # add corresponding padding
+    pad_kern = T.concatenate((T.zeros(tuple(kern.shape[:2]) + (pad[0], kern.shape[-1]),
+                                      dtype=theano.config.floatX),
+                              kern,
+                              T.zeros(tuple(kern.shape[:2]) + (double_pad[0] - pad[0], kern.shape[-1]),
+                                      dtype=theano.config.floatX)),
+                             axis=2)
+    pad_kern = T.concatenate((T.zeros(tuple(pad_kern.shape[:3]) + (pad[1],), dtype=theano.config.floatX),
+                              pad_kern,
+                              T.zeros(tuple(pad_kern.shape[:3]) + (double_pad[1] - pad[1],),
+                                      dtype=theano.config.floatX)),
+                             axis=3)
+
+    # upsample the input by passing it as kernl of conv and using filter_dilation
+    upsamp = T.nnet.conv2d(pad_kern, concat_mat, border_mode='valid',
+                           filter_dilation=ratio, subsample=subsample)
+
+    up_img_sh = T.ceil(T.as_tensor([row, col]) * np.array(ratio) / np.array(subsample)).astype('int64')
+    return upsamp.reshape((input.shape[0], input.shape[1], up_img_sh[0], up_img_sh[1]))
+
+
 def bilinear_upsampling(input,
-                        ratio,
+                        ratio=None,
+                        frac_ratio=None,
                         batch_size=None,
                         num_input_channels=None,
                         use_1D_kernel=True):
     """Compute bilinear upsampling
-
     This function will build the symbolic graph for upsampling
     a tensor by the given ratio using bilinear interpolation.
 
@@ -1766,46 +1876,52 @@ def bilinear_upsampling(input,
     input: symbolic 4D tensor
         mini-batch of feature map stacks, of shape (batch size,
         input channels, input rows, input columns) that will be upsampled.
-
     ratio: `int or Constant or Scalar Tensor of int* dtype`
         the ratio by which the input is upsampled in the 2D space (row and
         col size).
-
-    batch_size: None, int or Constant variable
-        The size of the first dimension of the input variable.
-        Optional, possibly used to choose an optimal implementation.
-        batch_size will be used only if num_input_channels is not None.
-
-    num_input_channels: None, int or Constant variable
-        The size of the second dimension of the input variable.
-        Optional, possibly used to choose an optimal implementation.
-        num_input_channels will be used only if batch_size is not None.
-
+    frac_ratio: None, tuple of int or tuple of tuples of int
+        The tuple defining the fractional ratio by which the input is
+        upsampled in the 2D space. One fractional ratio should be
+        represented as (numerator, denominator). If row and col ratios are
+        different frac_ratio should be a tuple of fractional ratios, i.e
+        a tuple of tuples.
     use_1D_kernel: bool
         if set to true, row and column will be upsampled seperately by 1D
         kernels, otherwise they are upsampled together using a 2D kernel. The
         final result is the same, only the speed can differ, given factors such
         as upsampling ratio.
-
     Returns
     -------
     symbolic 4D tensor
         set of feature maps generated by bilinear upsampling. Tensor
-        is of shape (batch size, num_input_channels, input row size * ratio,
-        input column size * ratio)
-
+        is of shape (batch size, num_input_channels, input row size * row ratio,
+        input column size * column ratio). Each of these ratios can be fractional.
     Notes
     -----
-
     :note: The kernel used for bilinear interpolation is fixed (not learned).
-
     :note: When the upsampling ratio is even, the last row and column is
         repeated one extra time compared to the first row and column which makes
         the upsampled tensor asymmetrical on both sides. This does not happen when
         the upsampling ratio is odd.
-
+    :note: This function must get either ratio or frac_ratio as parameter and
+        never both at once.
     """
 
+    if ratio and frac_ratio:
+        raise ValueError("can't use ratio and frac_ratio together")
+    if not (ratio or frac_ratio):
+        raise ValueError("No ratio (or frac_ratio) provided")
+
+    if frac_ratio:
+        if use_1D_kernel:
+            raise ValueError('For fractional ratios 1D kernel'
+                             'method not implemented. You may want to pass '
+                             'use_1D_kernel as False')
+        return frac_bilinear_upsampling(input,
+                                        ratio=ratio,
+                                        frac_ratio=frac_ratio)
+
+    # the remaining case if integer ratio with use_1D_kernel
     T = theano.tensor
     try:
         up_bs = batch_size * num_input_channels

theano/tensor/nnet/opt.py

@@ -251,6 +251,8 @@ def local_conv2d_cpu(node):
         return None
     if node.op.num_groups > 1 or node.op.unshared:
         return None
+    if node.op.filter_dilation != (1, 1):
+        return None
 
     rval = conv2d(img, kern,
                   node.op.imshp, node.op.kshp,

theano/tensor/nnet/tests/test_abstract_conv.py

@@ -1292,6 +1292,34 @@ class TestBilinearUpsampling(unittest.TestCase):
         f_2D = theano.function([], mat_2D, mode=self.compile_mode)
         utt.assert_allclose(f_1D(), f_2D(), rtol=1e-06)
 
+    def test_fractional_bilinear_upsampling(self):
+        """Test bilinear upsampling with nonsimilar fractional
+        row and col ratios
+        """
+        input_x = np.array([[[1, 2], [3, 4]],
+                            [[5, 6], [7, 8]],
+                            [[9, 10], [11, 12]]],
+                           ndmin=4).astype(theano.config.floatX)
+        up_x = bilinear_upsampling(input=input_x,
+                                   frac_ratio=((7, 4), (5, 3)),
+                                   use_1D_kernel=False)
+        num_up_x = np.array(
+            [[[[1., 1.2, 1.8, 2.],
+              [1.28571429, 1.48571429, 2.08571429, 2.28571429],
+              [2.42857143, 2.62857143, 3.22857143, 3.42857143],
+              [3., 3.2, 3.8, 4.]],
+             [[5., 5.2, 5.8, 6.],
+              [5.28571429, 5.48571429, 6.08571429, 6.28571429],
+              [6.42857143, 6.62857143, 7.22857143, 7.42857143],
+              [7., 7.2, 7.8, 8.]],
+             [[9., 9.2, 9.8, 10.],
+              [9.28571429, 9.48571429, 10.08571429, 10.28571429],
+              [10.42857143, 10.62857143, 11.22857143, 11.42857143],
+              [11., 11.2, 11.8, 12.]]]]
+            ).astype(theano.config.floatX)
+        f_up_x = theano.function([], up_x, mode=self.compile_mode)
+        utt.assert_allclose(f_up_x(), num_up_x, rtol=1e-6)
+
 
 class TestConv2dTranspose(unittest.TestCase):
     mode = None