Move call_nose.py to bin/theano-nose, add --batch

MANIFEST.in

@@ -5,3 +5,4 @@ global-include *.sh
 recursive-include docs *.png *.gp
 include distribute_setup.py
 include bin/theano-cache
+include bin/theano-nose

theano/tests/call_nose.py → bin/theano-nose

-__authors__   = "Olivier Delalleau"
-__contact__   = "delallea@iro"
+#!/usr/bin/env python
 
+__authors__   = "Olivier Delalleau, Pascal Lamblin"
+__contact__   = "delallea@iro"
 
 """
 This script should behave the same as the `nosetests` command.
@@ -8,16 +9,30 @@ This script should behave the same as the `nosetests` command.
 The reason for its existence is that on some systems, it may not be obvious to
 find where nosetests is installed in order to run it in a different process.
 
-This script is called from `run_tests_in_batch.py`.
-
 It is also used to load the KnownFailure plugin, in order to hide
-KnownFailureTests error messages.
+KnownFailureTests error messages. Use --without-knownfailure to
+disable that plugin.
+
+If the --batch option is used, it will call `run_tests_in_batch.py`,
+in order to run the tests by batches, not all at the same time.
+`run_tests_in_batch.py` will in turn call back this script in another
+process.
 """
 
+import logging
+_logger = logging.getLogger('theano.bin.theano-nose')
+_logger.setLevel(logging.WARN)
+
 import nose
 import sys
 
-if __name__ == '__main__':
+
+def main():
+    if '--batch' in sys.argv:
+        from theano.tests import run_tests_in_batch
+        sys.argv.remove('--batch')
+        return run_tests_in_batch.main()
+
     addplugins = []
     # We include KnownFailure plugin by default, unless
     # it is disabled by the "--without-knownfailure" arg.
@@ -26,8 +41,14 @@ if __name__ == '__main__':
             from numpy.testing.noseclasses import KnownFailure
             addplugins.append(KnownFailure())
         except ImportError:
-            pass
+            _logger.warn(
+                'KnownFailure plugin from NumPy could not be imported')
     else:
         sys.argv.remove('--without-knownfailure')
 
-    sys.exit(nose.main(addplugins=addplugins))
+    return nose.main(addplugins=addplugins)
+
+
+if __name__ == '__main__':
+    result = main()
+    sys.exit(result)

setup.py

@@ -141,7 +141,7 @@ def do_setup():
                    'ChangeLog'],
               'theano.misc': ['*.sh']
           },
-          scripts=['bin/theano-cache'],
+          scripts=['bin/theano-cache', 'bin/theano-nose'],
           keywords=' '.join([
             'theano', 'math', 'numerical', 'symbolic', 'blas',
             'numpy', 'gpu', 'autodiff', 'differentiation'

theano/tests/run_tests_in_batch.py

@@ -36,7 +36,7 @@ import cPickle, os, subprocess, sys
 import theano
 
 
-def main(stdout=None, stderr=None, argv=None, call_nose=None):
+def main(stdout=None, stderr=None, argv=None, theano_nose=None):
     """
     Run tests with optional output redirection.
 
@@ -46,8 +46,8 @@ def main(stdout=None, stderr=None, argv=None, call_nose=None):
     If argv is None, then we use arguments from sys.argv, otherwise we use the
     provided arguments instead.
 
-    If call_nose is None, then we use the call_nose.py script found in
-    theano/tests to call nosetests. Otherwise we call the provided script.
+    If theano_nose is None, then we use the theano-nose script found in
+    Theano/bin to call nosetests. Otherwise we call the provided script.
     """
     if stdout is None:
         stdout = sys.stdout
@@ -55,24 +55,25 @@ def main(stdout=None, stderr=None, argv=None, call_nose=None):
         stderr = sys.stderr
     if argv is None:
         argv = sys.argv
-    if call_nose is None:
-        call_nose = os.path.join(theano.__path__[0], 'tests', 'call_nose.py')
+    if theano_nose is None:
+        theano_nose = os.path.join(theano.__path__[0], '..', 'bin', 'theano-nose')
     stdout_backup = sys.stdout
     stderr_backup = sys.stderr
     try:
         sys.stdout = stdout
         sys.stderr = stderr
-        run(stdout, stderr, argv, call_nose)
+        run(stdout, stderr, argv, theano_nose)
     finally:
         sys.stdout = stdout_backup
         sys.stderr = stderr_backup
 
-def run(stdout, stderr, argv, call_nose):
+def run(stdout, stderr, argv, theano_nose):
     if len(argv) == 1:
         tests_dir = theano.__path__[0]
     else:
-        assert len(argv) == 2
-        tests_dir = argv[1]
+        # tests_dir should be at the end of argv, there can be other arguments
+        tests_dir = argv[-1]
+        other_args = argv[1:-1]
         assert os.path.isdir(tests_dir)
     os.chdir(tests_dir)
     # It seems safer to fully regenerate the list of tests on each call.
@@ -86,9 +87,11 @@ def run(stdout, stderr, argv, call_nose):
     stdout.flush()
     stderr.flush()
     dummy_in = open(os.devnull)
-    rval = subprocess.call(['python', call_nose, '--collect-only', '--with-id'],
-                           stdin=dummy_in.fileno(), stdout=stdout.fileno(),
-                           stderr=stderr.fileno())
+    rval = subprocess.call(
+            [theano_nose, '--collect-only', '--with-id'] + other_args,
+            stdin=dummy_in.fileno(),
+            stdout=stdout.fileno(),
+            stderr=stderr.fileno())
     stdout.flush()
     stderr.flush()
     assert rval == 0
@@ -111,10 +114,13 @@ def run(stdout, stderr, argv, call_nose):
         # We suppress all output because we want the user to focus only on the
         # failed tests, which are re-run (with output) below.
         dummy_out = open(os.devnull, 'w')
-        rval = subprocess.call(['python', call_nose, '-q', '--with-id'] +
-                               map(str, test_range), stdout=dummy_out.fileno(),
-                               stderr=dummy_out.fileno(),
-                               stdin=dummy_in.fileno())
+        rval = subprocess.call(
+                ([theano_nose, '-q', '--with-id']
+                    + map(str, test_range)
+                    + other_args),
+                stdout=dummy_out.fileno(),
+                stderr=dummy_out.fileno(),
+                stdin=dummy_in.fileno())
         # Recover failed test indices from the 'failed' field of the '.noseids'
         # file. We need to do it after each batch because otherwise this field
         # may get erased. We use a set because it seems like it is not
@@ -132,9 +138,11 @@ def run(stdout, stderr, argv, call_nose):
 ################################"""
         stdout.flush()
         stderr.flush()
-        subprocess.call(['python', call_nose, '-v', '--with-id'] + failed,
-                        stdin=dummy_in.fileno(), stdout=stdout.fileno(),
-                        stderr=stderr.fileno())
+        subprocess.call(
+                [theano_nose, '-v', '--with-id'] + failed + other_args,
+                stdin=dummy_in.fileno(),
+                stdout=stdout.fileno(),
+                stderr=stderr.fileno())
         stdout.flush()
         stderr.flush()
         return 0