* moved tensor/nnet.py to tensor/nnet/nnet.py

doc/library/tensor/index.txt

@@ -18,8 +18,7 @@ They are grouped into the following sections:
     :maxdepth: 1
 
     basic
+    nnet/index
     raw_random
     shared_randomstreams
-    nnet
-    signal
-
+    signal/index

doc/library/tensor/signal.txt → doc/library/tensor/nnet/conv.txt

-.. _libdoc_tensor_signal:
+.. _libdoc_tensor_nnet_conv:
 
-======================================================
-:mod:`signal` -- Signal processing
-======================================================
+==========================================================
+:mod:`conv` -- Ops for convolutional neural nets
+==========================================================
 
-.. module:: signal
+.. module:: conv
    :platform: Unix, Windows
    :synopsis: ops for signal processing
 .. moduleauthor:: LISA
@@ -12,9 +12,7 @@
 
 TODO: Give examples for how to use these things! They are pretty complicated.
 
-.. function:: conv2D(*todo)
-
-.. function:: downsample2D(*todo)
+.. autofunction:: theano.tensor.nnet.conv.conv2d
 
 .. function:: fft(*todo)
 

doc/library/tensor/nnet/index.txt

+.. _libdoc_tensor_nnet:
+
+==================================================
+:mod:`nnet`  -- Ops related to neural networks
+==================================================
+
+.. module:: nnet
+   :platform: Unix, Windows
+   :synopsis: various ops relating to neural networks
+.. moduleauthor:: LISA
+
+Theano was originally developped for machine learning applications, particularly
+for the topic of deep learning. As such, our lab has developed many functions
+and ops which are particular to neural networks and deep learning.
+
+.. toctree::
+    :maxdepth: 1
+
+    conv
+    nnet

doc/library/tensor/nnet.txt → doc/library/tensor/nnet/nnet.txt

-.. _libdoc_tensor_nnet:
+.. _libdoc_tensor_nnet_nnet:
 
 ======================================================
 :mod:`nnet` -- Ops for neural networks

doc/library/tensor/signal/conv.txt

+.. _libdoc_tensor_signal_conv:
+
+======================================================
+:mod:`conv` -- Convolution
+======================================================
+
+.. module:: conv
+   :platform: Unix, Windows
+   :synopsis: ops for performing convolutions
+.. moduleauthor:: LISA
+
+.. autofunction:: theano.tensor.signal.conv.conv2d
+
+.. function:: fft(*todo)
+
+    [James has some code for this, but hasn't gotten it into the source tree yet.]
+

doc/library/tensor/signal/downsample.txt

+.. _libdoc_tensor_signal_downsample:
+
+======================================================
+:mod:`downsample` -- Down-Sampling
+======================================================
+
+.. module:: downsample
+   :platform: Unix, Windows
+   :synopsis: ops for performing various forms of downsampling
+.. moduleauthor:: LISA
+
+
+.. autofunction:: theano.tensor.signal.downsample.max_pool2D
+
+.. function:: fft(*todo)
+
+    [James has some code for this, but hasn't gotten it into the source tree yet.]
+

doc/library/tensor/signal/index.txt

+.. _libdoc_tensor:
+
+=====================================================
+:mod:`signal` -- Signal Processing
+=====================================================
+
+Signal Processing
+-----------------
+
+.. module:: signal
+   :platform: Unix, Windows
+   :synopsis: various ops for performing basic signal processing
+   (convolutions, subsampling, fft, etc.)
+.. moduleauthor:: LISA
+
+The signal subpackage contains ops which are useful for performing various
+forms of signal processing.
+
+.. toctree::
+    :maxdepth: 1
+
+    conv
+    downsample

theano/tensor/nnet/__init__.py

+from nnet import *

theano/tensor/nnet.py → theano/tensor/nnet/nnet.py

@@ -7,11 +7,11 @@ from theano import gof
 from theano import scalar
 from theano import printing
 from theano.printing import pprint
-import basic as tensor
-import elemwise
-import numpy
-import opt
+from theano.tensor import basic as tensor
+from theano.tensor import elemwise
+from theano.tensor import opt
 from theano.compile import optdb
+import numpy
 
 ############
 #

theano/tensor/signal/tests/speed_test_conv.py → theano/tensor/nnet/tests/speed_test_conv.py

@@ -7,7 +7,7 @@ from theano.tests import unittest_tools as utt
 
 from theano import function, Mode
 import theano.tensor as T
-from theano.tensor.signal.conv import ConvOp
+from theano.tensor.nnet.conv import ConvOp
 
 def flip(kern, kshp):
     "flip the kernel as scipy.convolv2d do it flipped."

theano/tensor/nnet/tests/test_conv.py

+import sys, time, unittest
+import numpy
+from scipy import signal
+
+import theano
+import theano.tensor as T
+from theano import function, Mode
+from theano.tests import unittest_tools as utt
+
+from theano.tensor.nnet import conv
+
+from theano.tensor.basic import _allclose
+
+class TestConv2D(unittest.TestCase):
+
+    def setUp(self):
+        utt.seed_rng()
+        self.input = T.dtensor4('input')
+        self.filters = T.dtensor4('filters')
+
+    def validate(self, image_shape, filter_shape,
+                 border_mode='valid', subsample=(1,1),
+                 N_image_shape=None, N_filter_shape=None,
+                 input=None, filters=None, 
+                 unroll_batch=0, unroll_kern=0, unroll_patch=True,
+                 verify_grad=True):
+
+        if N_image_shape is None:
+            N_image_shape = image_shape
+        if N_filter_shape is None:
+            N_filter_shape = filter_shape
+    
+        if not input:
+            input = self.input
+        if not filters:
+            filters = self.filters
+        
+        ############# THEANO IMPLEMENTATION ############
+        
+        # we create a symbolic function so that verify_grad can work
+        def sym_conv2d(input, filters):
+            # define theano graph and function
+            return conv.conv2d(input, filters, image_shape, filter_shape,
+                          border_mode, subsample, unroll_batch=unroll_batch,
+                          unroll_kern=unroll_kern, unroll_patch=unroll_patch)
+
+        output = sym_conv2d(input, filters)
+        theano_conv = theano.function([input, filters], output)
+          
+        # initialize input and compute result
+        image_data  = numpy.random.random(N_image_shape)
+        filter_data = numpy.random.random(N_filter_shape)
+        theano_output = theano_conv(image_data, filter_data)
+
+        ############# REFERENCE IMPLEMENTATION ############
+        s = 1. if border_mode is 'full' else -1.
+        out_shape2d = numpy.array(N_image_shape[-2:]) +\
+                      s*numpy.array(N_filter_shape[-2:]) - s
+        out_shape2d = numpy.ceil(out_shape2d / numpy.array(subsample))
+        out_shape = (N_image_shape[0],N_filter_shape[0]) + tuple(out_shape2d)
+        ref_output = numpy.zeros(out_shape)
+
+        # loop over output feature maps
+        for k in range(N_filter_shape[0]):
+            # loop over input feature maps
+            for l in range(N_filter_shape[1]):
+
+                filter2d = filter_data[k,l,:,:]
+
+                # loop over mini-batches
+                for b in range(N_image_shape[0]):
+                    image2d = image_data[b,l,:,:]
+                    output2d = signal.convolve2d(image2d, filter2d, border_mode)
+
+                    ref_output[b,k,:,:] +=\
+                       output2d[::subsample[0],::subsample[1]]
+
+        self.failUnless(_allclose(theano_output, ref_output))
+
+        ############# TEST GRADIENT ############
+        if verify_grad:
+            utt.verify_grad(sym_conv2d, [image_data, filter_data])
+
+
+    def test_basic(self):
+        """
+        Tests that basic convolutions work for odd and even dimensions of image and filter
+        shapes, as well as rectangular images and filters.
+        """
+        self.validate((3,2,8,8), (4,2,5,5), 'valid')
+        self.validate((3,2,7,5), (5,2,2,3), 'valid')
+        self.validate((3,2,7,5), (5,2,3,2), 'valid')
+        self.validate((3,2,8,8), (4,2,5,5), 'full')
+        self.validate((3,2,7,5), (5,2,2,3), 'full')
+        # test filter same size as input
+        self.validate((3,2,3,3), (4,2,3,3), 'valid')
+
+    def test_unroll_patch_false(self):
+        """
+        unroll_patch is True by default. Test basic convs with False.
+        """
+        self.validate((3,2,7,5), (5,2,2,3), 'valid', unroll_patch=False)
+        self.validate((3,2,7,5), (5,2,2,3), 'full', unroll_patch=False)
+        self.validate((3,2,3,3), (4,2,3,3), 'valid', unroll_patch=False)
+
+    def test_unroll_special(self):
+        """
+        (unroll_kern, unroll_batch) in (0,1),(1,0) is special case.
+        """
+        self.validate((6,2,3,3), (3,2,2,2), 'valid', unroll_batch=1)
+
+    def test_unroll_batch(self):
+        """
+        Test mini-batch unrolling for various legal values.
+        """
+        # mini-batch of size 6 is multiple of 2 and 3. Should work.
+        self.validate((6,2,3,3), (3,2,2,2), 'valid', unroll_batch=2, verify_grad=False)
+        self.validate((6,2,3,3), (3,2,2,2), 'valid', unroll_batch=3, verify_grad=False)
+
+    def test_unroll_kern(self):
+        """
+        Test kernel unrolling for various legal values.
+        """
+        # 6 filters is a multiple of 2 and 3. Should work.
+        self.validate((2,3,3,3), (6,3,2,2), 'valid', unroll_kern=2, verify_grad=False)
+        self.validate((2,3,3,3), (6,3,2,2), 'valid', unroll_kern=3, verify_grad=False)
+
+    def test_subsample(self):
+        """
+        Tests convolution where subsampling != (1,1)
+        """
+        self.validate((3,2,7,5), (5,2,2,3), 'valid', subsample=(2,2))
+        self.validate((3,2,7,5), (5,2,2,3), 'full', subsample=(2,2))
+        self.validate((3,2,7,5), (5,2,2,3), 'valid', subsample=(2,1))
+
+    def test_invalid_filter_shape(self):
+        """
+        Tests scenario where filter_shape[1] != input_shape[1]
+        """
+        def f():
+            self.validate((3,2,8,8), (4,3,5,5), 'valid')
+        self.failUnlessRaises(AssertionError, f)
+
+    def test_missing_info(self):
+        """
+        Test convolutions for various pieces of missing info.
+        """
+        self.validate(None, None, 
+                      N_image_shape=(3,2,8,8), 
+                      N_filter_shape=(4,2,5,5))
+        self.validate((3,2,None,None), None,
+                      N_image_shape=(3,2,8,8), 
+                      N_filter_shape=(4,2,5,5))
+        self.validate((None,2,None,None), (None,2,5,5),
+                      N_image_shape=(3,2,8,8), 
+                      N_filter_shape=(4,2,5,5))
+
+    def test_full_mode(self):
+        """
+        Tests basic convolution in full mode and case where filter 
+        is larger than the input image.
+        """
+        self.validate((3,2,5,5), (4,2,8,8), 'full')
+        def f():
+            self.validate((3,2,5,5), (4,2,8,8), 'valid')
+        self.failUnlessRaises(Exception, f)
+
+    def test_wrong_input(self):
+        """
+        Make sure errors are raised when image and kernel are not 4D tensors
+        """
+        try:
+            self.validate((3,2,8,8), (4,2,5,5), 'valid', input = T.dmatrix())
+            self.validate((3,2,8,8), (4,2,5,5), 'valid', filters = T.dvector())
+            self.validate((3,2,8,8), (4,2,5,5), 'valid', input = T.dtensor3())
+            # should never reach here
+            self.fail()
+        except: 
+            pass

theano/tensor/tests/test_nnet.py → theano/tensor/nnet/tests/test_nnet.py

@@ -3,9 +3,9 @@ import unittest
 import theano
 from theano import tensor as T
 from theano import gof
-import test_basic as TT
 import numpy
 from theano.tests import unittest_tools as utt
+from theano.tensor.tests import test_basic as TT
 
 from theano.tensor.nnet import *
 

theano/tensor/signal/downsample.py

@@ -15,13 +15,13 @@ def max_pool2D(input, ds, ignore_border=False):
     Takes as input a N-D tensor, where N >= 2. It downscales the input image by
     the specified factor, by keeping only the maximum value of non-overlapping
     patches of size (ds[0],ds[1])
+
     :type input: N-D theano tensor of input images. 
     :param input: input images. Max pooling will be done over the 2 last dimensions.
     :type ds: tuple of length 2
-    :param ds: factor by which to downscale. (2,2) will halve the image in each
-               dimension.
-    :param ignore_border: boolean value. When True, (5,5) input with ds=(2,2)
-                         will generate a (2,2) output. (3,3) otherwise.
+    :param ds: factor by which to downscale. (2,2) will halve the image in each dimension.
+    :param ignore_border: boolean value. When True, (5,5) input with ds=(2,2) will generate a
+      (2,2) output. (3,3) otherwise.
     """
     if input.ndim < 2:
         raise NotImplementedError('max_pool2D requires a dimension >= 2')

theano/tensor/signal/tests/test_conv.py

@@ -11,169 +11,78 @@ from theano.tensor.signal import conv
 
 from theano.tensor.basic import _allclose
 
-class TestConv2D(unittest.TestCase):
+class TestSignalConv2D(unittest.TestCase):
 
     def setUp(self):
         utt.seed_rng()
-        self.input = T.dtensor4('input')
-        self.filters = T.dtensor4('filters')
-
-    def validate(self, image_shape, filter_shape,
-                 border_mode='valid', subsample=(1,1),
-                 N_image_shape=None, N_filter_shape=None,
-                 input=None, filters=None, 
-                 unroll_batch=0, unroll_kern=0, unroll_patch=True,
-                 verify_grad=True):
-
-        if N_image_shape is None:
-            N_image_shape = image_shape
-        if N_filter_shape is None:
-            N_filter_shape = filter_shape
-    
-        if not input:
-            input = self.input
-        if not filters:
-            filters = self.filters
+
+    def validate(self, image_shape, filter_shape, verify_grad=True):
         
+        image_dim = len(image_shape)
+        filter_dim = len(filter_shape)
+        input = T.TensorType('float64', [False]*image_dim)()
+        filters = T.TensorType('float64', [False]*filter_dim)()
+
+        bsize = image_shape[0] if image_dim==3 else 1
+        nkern = filter_shape[0] if filter_dim==3 else 1
+
         ############# THEANO IMPLEMENTATION ############
-        
         # we create a symbolic function so that verify_grad can work
         def sym_conv2d(input, filters):
-            # define theano graph and function
-            return conv.conv2d(input, filters, image_shape, filter_shape,
-                          border_mode, subsample, unroll_batch=unroll_batch,
-                          unroll_kern=unroll_kern, unroll_patch=unroll_patch)
-
+            return conv.conv2d(input, filters)
         output = sym_conv2d(input, filters)
         theano_conv = theano.function([input, filters], output)
           
         # initialize input and compute result
-        image_data  = numpy.random.random(N_image_shape)
-        filter_data = numpy.random.random(N_filter_shape)
+        image_data  = numpy.random.random(image_shape)
+        filter_data = numpy.random.random(filter_shape)
         theano_output = theano_conv(image_data, filter_data)
-
+        
         ############# REFERENCE IMPLEMENTATION ############
-        s = 1. if border_mode is 'full' else -1.
-        out_shape2d = numpy.array(N_image_shape[-2:]) +\
-                      s*numpy.array(N_filter_shape[-2:]) - s
-        out_shape2d = numpy.ceil(out_shape2d / numpy.array(subsample))
-        out_shape = (N_image_shape[0],N_filter_shape[0]) + tuple(out_shape2d)
-        ref_output = numpy.zeros(out_shape)
+        out_shape2d = numpy.array(image_shape[-2:]) -\
+                      numpy.array(filter_shape[-2:]) + 1
+        ref_output = numpy.zeros(tuple(out_shape2d))
 
-        # loop over output feature maps
-        for k in range(N_filter_shape[0]):
-            # loop over input feature maps
-            for l in range(N_filter_shape[1]):
+        # reshape as 3D input tensors to make life easier
+        image_data3d  = image_data.reshape((bsize,)+image_shape[-2:])
+        filter_data3d = filter_data.reshape((nkern,)+filter_shape[-2:])
+        # reshape theano output as 4D to make life easier
+        theano_output4d = theano_output.reshape((bsize,nkern,)+theano_output.shape[-2:])
 
-                filter2d = filter_data[k,l,:,:]
+        # loop over mini-batches (if required)
+        for b in range(bsize):
 
-                # loop over mini-batches
-                for b in range(N_image_shape[0]):
-                    image2d = image_data[b,l,:,:]
-                    output2d = signal.convolve2d(image2d, filter2d, border_mode)
+            # loop over filters (if required)
+            for k in range(nkern):
 
-                    ref_output[b,k,:,:] +=\
-                       output2d[::subsample[0],::subsample[1]]
+                image2d = image_data3d[b,:,:]
+                filter2d = filter_data3d[k,:,:]
+                output2d = signal.convolve2d(image2d, filter2d, 'valid')
 
-        self.failUnless(_allclose(theano_output, ref_output))
+                self.failUnless(_allclose(theano_output4d[b,k,:,:], output2d))
 
         ############# TEST GRADIENT ############
         if verify_grad:
             utt.verify_grad(sym_conv2d, [image_data, filter_data])
 
-
     def test_basic(self):
         """
-        Tests that basic convolutions work for odd and even dimensions of image and filter
-        shapes, as well as rectangular images and filters.
-        """
-        self.validate((3,2,8,8), (4,2,5,5), 'valid')
-        self.validate((3,2,7,5), (5,2,2,3), 'valid')
-        self.validate((3,2,7,5), (5,2,3,2), 'valid')
-        self.validate((3,2,8,8), (4,2,5,5), 'full')
-        self.validate((3,2,7,5), (5,2,2,3), 'full')
-        # test filter same size as input
-        self.validate((3,2,3,3), (4,2,3,3), 'valid')
-
-    def test_unroll_patch_false(self):
-        """
-        unroll_patch is True by default. Test basic convs with False.
-        """
-        self.validate((3,2,7,5), (5,2,2,3), 'valid', unroll_patch=False)
-        self.validate((3,2,7,5), (5,2,2,3), 'full', unroll_patch=False)
-        self.validate((3,2,3,3), (4,2,3,3), 'valid', unroll_patch=False)
-
-    def test_unroll_special(self):
-        """
-        (unroll_kern, unroll_batch) in (0,1),(1,0) is special case.
-        """
-        self.validate((6,2,3,3), (3,2,2,2), 'valid', unroll_batch=1)
-
-    def test_unroll_batch(self):
-        """
-        Test mini-batch unrolling for various legal values.
-        """
-        # mini-batch of size 6 is multiple of 2 and 3. Should work.
-        self.validate((6,2,3,3), (3,2,2,2), 'valid', unroll_batch=2, verify_grad=False)
-        self.validate((6,2,3,3), (3,2,2,2), 'valid', unroll_batch=3, verify_grad=False)
-
-    def test_unroll_kern(self):
-        """
-        Test kernel unrolling for various legal values.
-        """
-        # 6 filters is a multiple of 2 and 3. Should work.
-        self.validate((2,3,3,3), (6,3,2,2), 'valid', unroll_kern=2, verify_grad=False)
-        self.validate((2,3,3,3), (6,3,2,2), 'valid', unroll_kern=3, verify_grad=False)
-
-    def test_subsample(self):
-        """
-        Tests convolution where subsampling != (1,1)
-        """
-        self.validate((3,2,7,5), (5,2,2,3), 'valid', subsample=(2,2))
-        self.validate((3,2,7,5), (5,2,2,3), 'full', subsample=(2,2))
-        self.validate((3,2,7,5), (5,2,2,3), 'valid', subsample=(2,1))
-
-    def test_invalid_filter_shape(self):
-        """
-        Tests scenario where filter_shape[1] != input_shape[1]
-        """
-        def f():
-            self.validate((3,2,8,8), (4,3,5,5), 'valid')
-        self.failUnlessRaises(AssertionError, f)
-
-    def test_missing_info(self):
-        """
-        Test convolutions for various pieces of missing info.
-        """
-        self.validate(None, None, 
-                      N_image_shape=(3,2,8,8), 
-                      N_filter_shape=(4,2,5,5))
-        self.validate((3,2,None,None), None,
-                      N_image_shape=(3,2,8,8), 
-                      N_filter_shape=(4,2,5,5))
-        self.validate((None,2,None,None), (None,2,5,5),
-                      N_image_shape=(3,2,8,8), 
-                      N_filter_shape=(4,2,5,5))
-
-    def test_full_mode(self):
-        """
-        Tests basic convolution in full mode and case where filter 
-        is larger than the input image.
+        Basic functionality of nnet.conv.ConvOp is already tested by its own test suite.  We
+        just have to test whether or not signal.conv.conv2d can support inputs and filters of
+        type matrix or tensor3.
         """
-        self.validate((3,2,5,5), (4,2,8,8), 'full')
-        def f():
-            self.validate((3,2,5,5), (4,2,8,8), 'valid')
-        self.failUnlessRaises(Exception, f)
+        self.validate((3,7,5), (5,2,3), 'valid')
+        self.validate((7,5), (5,2,3), 'valid')
+        self.validate((3,7,5), (2,3), 'valid')
+        self.validate((7,5), (2,3), 'valid')
 
-    def test_wrong_input(self):
+    def test_fail(self):
         """
-        Make sure errors are raised when image and kernel are not 4D tensors
+        Test that conv2d fails for dimensions other than 2 or 3.
         """
         try:
-            self.validate((3,2,8,8), (4,2,5,5), 'valid', input = T.dmatrix())
-            self.validate((3,2,8,8), (4,2,5,5), 'valid', filters = T.dvector())
-            self.validate((3,2,8,8), (4,2,5,5), 'valid', input = T.dtensor3())
-            # should never reach here
+            conv.conv2d(T.dtensor4(), T.dtensor3())
+            conv.conv2d(T.dtensor3(), T.dvector())
             self.fail()
-        except: 
+        except:
             pass