Merge pull request #489 from lamblin/better_nosetest_script

MANIFEST.in

@@ -5,3 +5,4 @@ global-include *.sh
 recursive-include docs *.png *.gp
 include distribute_setup.py
 include bin/theano-cache
+include bin/theano-nose

bin/theano-nose

+#!/usr/bin/env python
+
+__authors__   = "Olivier Delalleau, Pascal Lamblin"
+__contact__   = "delallea@iro"
+
+"""
+This script should behave the same as the `nosetests` command.
+
+The reason for its existence is that on some systems, it may not be obvious to
+find where nosetests is installed in order to run it in a different process.
+
+It is also used to load the KnownFailure plugin, in order to hide
+KnownFailureTests error messages. Use --without-knownfailure to
+disable that plugin.
+
+If the --batch option is used, it will call `run_tests_in_batch.py`,
+in order to run the tests by batches, not all at the same time.
+`run_tests_in_batch.py` will in turn call back this script in another
+process.
+"""
+
+import logging
+_logger = logging.getLogger('theano.bin.theano-nose')
+_logger.setLevel(logging.WARN)
+
+import nose
+import textwrap
+import sys
+
+
+def main():
+    # Handle --batch[=n] arguments
+    batch_args = [arg for arg in sys.argv if arg.startswith('--batch')]
+    for arg in batch_args:
+        sys.argv.remove(arg)
+
+    if len(batch_args):
+        if len(batch_args) > 1:
+            _logger.warn(
+                'Multiple --batch arguments detected, using the last one '
+                'and ignoring the first ones.')
+
+        batch_arg = batch_args[-1]
+        elems = batch_arg.split('=', 1)
+        if len(elems) == 2:
+            batch_size = int(elems[1])
+        else:
+            # Use run_tests_in_batch's default
+            batch_size = None
+
+        from theano.tests import run_tests_in_batch
+        return run_tests_in_batch.main(batch_size=batch_size)
+
+    # Non-batch mode.
+    addplugins = []
+    # We include KnownFailure plugin by default, unless
+    # it is disabled by the "--without-knownfailure" arg.
+    if '--without-knownfailure' not in sys.argv:
+        try:
+            from numpy.testing.noseclasses import KnownFailure
+            addplugins.append(KnownFailure())
+        except ImportError:
+            _logger.warn(
+                'KnownFailure plugin from NumPy could not be imported. '
+                'Use --without-knownfailure to disable this warning.')
+    else:
+        sys.argv.remove('--without-knownfailure')
+
+    return nose.main(addplugins=addplugins)
+
+def help():
+    help_msg = """
+        This script behaves mostly the same as the `nosetests` command.
+
+        The main difference is that it loads automatically the
+        KnownFailure plugin, in order to hide KnownFailureTests error
+        messages. It also supports executing tests by batches.
+
+        Options:
+
+            --batch[=n]: Do not run all the tests in one run, but split
+                         the execution in batches of `n` tests each.
+                         Default n is 100.
+
+            --help, -h: Displays this help.
+
+            --without-knownfailure: Do not load the KnownFailure plugin.
+
+        The other options will be passed to nosetests, see ``nosetests -h``.
+        """
+
+    print textwrap.dedent(help_msg)
+
+
+if __name__ == '__main__':
+    if '--help' in sys.argv or '-h' in sys.argv:
+        help()
+    else:
+        result = main()
+        sys.exit(result)

doc/LICENSE.txt

@@ -6,6 +6,9 @@ LICENSE
 Copyright (c) 2008--2012, Theano Development Team
 All rights reserved.
 
+Contains code from NumPy, Copyright (c) 2005-2011, NumPy Developers.
+All rights reserved.
+
 Redistribution and use in source and binary forms, with or without
 modification, are permitted provided that the following conditions are met:
 

setup.py

@@ -141,7 +141,7 @@ def do_setup():
                    'ChangeLog'],
               'theano.misc': ['*.sh']
           },
-          scripts=['bin/theano-cache'],
+          scripts=['bin/theano-cache', 'bin/theano-nose'],
           keywords=' '.join([
             'theano', 'math', 'numerical', 'symbolic', 'blas',
             'numpy', 'gpu', 'autodiff', 'differentiation'

theano/__init__.py

@@ -77,8 +77,8 @@ from compile import \
 
 from misc.safe_asarray import _asarray
 
-import numpy.testing
-test = numpy.testing.Tester().test
+import theano.tests
+test = theano.tests.TheanoNoseTester().test
 
 FancyModule = Module
 

theano/misc/do_nightly_build

 #!/bin/bash
-#we set the compiledir to the /Tmp dir to make the test faster by bypassing the nfs network.
 date
 START=`date +%s`
-NOSETESTS=nosetests
 ARGS=$@
 PROFILING=""
 RELEASE=""
+
 if [ "$1" == "--release" ]; then
     RELEASE="True"
     shift
     ARGS=$@
 fi
+
 if [ "$1" == "--buildbot" ]; then
+    #we set the compiledir to the /Tmp dir to make the test faster by bypassing the nfs network.
     COMPILEDIR=/Tmp/lisa_theano_compile_dir_theano
     ROOT_CWD=/Tmp/nightly_build
     FLAGS=compiledir=$COMPILEDIR
@@ -20,24 +21,30 @@ if [ "$1" == "--buildbot" ]; then
     cd ..
     ARGS="Theano"
     PROFILING="--with-coverage --cover-package=theano"
+    NOSETESTS=${ROOT_CWD}/bin/theano-nose
+    export PYTHONPATH=${ROOT_CWD}:$PYTHONPATH
 else
-    COMPILEDIR=`python -c "import theano;print theano.config.compiledir"`
+    COMPILEDIR=`python -c "import theano; print theano.config.compiledir"`
+    NOSETESTS=`python -c "import theano; print theano.__path__[0]"`/../bin/theano-nose
 fi
 
 echo "Number of elements in the compiledir:"
 ls ${COMPILEDIR}|wc -l
+
 # We don't want warnings in the buildbot for errors already fixed.
 FLAGS=${THEANO_FLAGS},warn.argmax_pushdown_bug=False,warn.gpusum_01_011_0111_bug=False,warn.sum_sum_bug=False,warn.sum_div_dimshuffle_bug=False,warn.subtensor_merge_bug=False,$FLAGS
+
 # We want to see correctly optimization/shape errors, so make make them raise an
 # error.
 FLAGS=on_opt_error=raise,$FLAGS
 FLAGS=on_shape_error=raise,$FLAGS
+
 # Ignore user device and floatX config, because:
 #   1. Tests are intended to be run with device=cpu.
 #   2. We explicitly add 'floatX=float32' in one run of the test suite below,
 #      while we want all other runs to run with 'floatX=float64'.
 FLAGS=${FLAGS},device=cpu,floatX=float64
-export PYTHONPATH=${ROOT_CWD}:$PYTHONPATH
+
 
 if [ "$RELEASE" ]; then
     echo "Executing nosetests with default mode and compute_test_value"

theano/misc/do_nightly_build_send

@@ -66,7 +66,12 @@ def mysend(subject, file):
         elif token.startswith("SKIP="):
             skip+=int(token[5:])
         elif token == "KnownFailureTest:":
+            # This means that KnownFailure plugin is off,
+            # so knownfails are also counted as errors
             knownfail+=1
+            errors-=1
+        elif token.startswith("KNOWNFAIL="):
+            knownfail += int(token.split('=')[1])
         elif token.startswith("speed_failure_"):
             speed_failure+=int(token.split('=')[1])
             show_speed_failure=True
@@ -98,11 +103,10 @@ def mysend(subject, file):
                     float64_time = start
                     start = ""
     
-    s="KnownFailure are removed from Error. \n Resume of the output:\n"+filter_output(open(file))+"Full output:\n"+s
+    s="Resume of the output:\n\n"+filter_output(open(file))+"\n\nFull output:\n\n"+s
     img = MIMEText(s)
     fp.close()
     msg.attach(img)
-    errors-=knownfail
     
 # Send the email via our own SMTP server.
     if show_speed_failure:

theano/tests/__init__.py

-from main import main
+from main import main, TheanoNoseTester
 
 import unittest_tools

theano/tests/call_nose.py

-__authors__   = "Olivier Delalleau"
-__contact__   = "delallea@iro"
-
-
-"""
-This script should behave the same as the `nosetests` command.
-
-The reason for its existence is that on some systems, it may not be obvious to
-find where nosetests is installed in order to run it in a different process.
-This script is called from `run_tests_in_batch.py`.
-"""
-
-
-import sys
-
-import nose
-
-
-if __name__ == '__main__':
-    sys.exit(nose.main())

theano/tests/main.py

-import unittest,sys
+import os, unittest, sys
+import nose.plugins.builtin
+
+from numpy.testing.nosetester import import_nose, NoseTester
+from numpy.testing.noseclasses import KnownFailure, NumpyTestProgram
+
+
+# This class contains code adapted from NumPy,
+# numpy/testing/nosetester.py,
+# Copyright (c) 2005-2011, NumPy Developers
+class TheanoNoseTester(NoseTester):
+    """
+    Nose test runner.
+
+    This class enables running nose tests from inside Theano,
+    by calling theano.test().
+    This version is more adapted to what we want than Numpy's one.
+    """
+
+    def _test_argv(self, verbose, extra_argv):
+        """
+        Generate argv for nosetest command
+
+        :type verbose: int
+        :param verbose: Verbosity value for test outputs, in the range 1-10.
+                        Default is 1.
+
+        :type extra_argv: list
+        :param extra_argv: List with any extra arguments to pass to nosetests.
+        """
+        #self.package_path = os.path.abspath(self.package_path)
+        argv = [__file__, self.package_path]
+        argv += ['--verbosity', str(verbose)]
+        if extra_argv:
+            argv += extra_argv
+        return argv
+
+    def _show_system_info(self):
+        nose = import_nose()
+
+        import theano
+        print "Theano version %s" % theano.__version__
+        theano_dir = os.path.dirname(theano.__file__)
+        print "theano is installed in %s" % theano_dir
+
+        super(TheanoNoseTester, self)._show_system_info()
+
+    def prepare_test_args(self, verbose=1, extra_argv=None, coverage=False,
+            capture=True, knownfailure=True):
+        """
+        Prepare arguments for the `test` method.
+
+        Takes the same arguments as `test`.
+        """
+        # fail with nice error message if nose is not present
+        nose = import_nose()
+
+        # compile argv
+        argv = self._test_argv(verbose, extra_argv)
+
+        # numpy way of doing coverage
+        if coverage:
+            argv += ['--cover-package=%s' % self.package_name, '--with-coverage',
+                    '--cover-tests', '--cover-inclusive', '--cover-erase']
+
+        # Capture output only if needed
+        if not capture:
+            argv += ['-s']
+
+        # construct list of plugins
+        plugins = []
+        if knownfailure:
+            plugins.append(KnownFailure())
+        plugins += [p() for p in nose.plugins.builtin.plugins]
+
+        return argv, plugins
+
+    def test(self, verbose=1, extra_argv=None, coverage=False, capture=True,
+            knownfailure=True):
+        """
+        Run tests for module using nose.
+
+        :type verbose: int
+        :param verbose: Verbosity value for test outputs, in the range 1-10.
+                        Default is 1.
+
+        :type extra_argv: list
+        :param extra_argv: List with any extra arguments to pass to nosetests.
+
+        :type coverage: bool
+        :param coverage: If True, report coverage of Theano code. Default is False.
+
+        :type capture: bool
+        :param capture: If True, capture the standard output of the tests, like
+                        nosetests does in command-line. The output of failing
+                        tests will be displayed at the end. Default is True.
+
+        :type knownfailure: bool
+        :param knownfailure: If True, tests raising KnownFailureTest will
+                not be considered Errors nor Failure, but reported as
+                "known failures" and treated quite like skipped tests.
+                Default is True.
+
+        :returns: Returns the result of running the tests as a
+                  ``nose.result.TextTestResult`` object.
+        """
+        # cap verbosity at 3 because nose becomes *very* verbose beyond that
+        verbose = min(verbose, 3)
+        self._show_system_info()
+
+        cwd = os.getcwd()
+        if self.package_path in os.listdir(cwd):
+            # The tests give weird errors if the package to test is
+            # in current directory.
+            raise RuntimeError((
+                "This function does not run correctly when, at the time "
+                "theano was imported, the working directory was theano's "
+                "parent directory. You should exit your Python prompt, change "
+                "directory, then launch Python again, import theano, then "
+                "launch theano.test()."))
+
+        argv, plugins = self.prepare_test_args(verbose, extra_argv, coverage,
+                capture, knownfailure)
+
+        t = NumpyTestProgram(argv=argv, exit=False, plugins=plugins)
+        return t.result
+
 
 def main(modulename):
     debug = False

theano/tests/run_tests_in_batch.py

@@ -36,7 +36,7 @@ import cPickle, os, subprocess, sys
 import theano
 
 
-def main(stdout=None, stderr=None, argv=None, call_nose=None):
+def main(stdout=None, stderr=None, argv=None, theano_nose=None, batch_size=None):
     """
     Run tests with optional output redirection.
 
@@ -46,8 +46,10 @@ def main(stdout=None, stderr=None, argv=None, call_nose=None):
     If argv is None, then we use arguments from sys.argv, otherwise we use the
     provided arguments instead.
 
-    If call_nose is None, then we use the call_nose.py script found in
-    theano/tests to call nosetests. Otherwise we call the provided script.
+    If theano_nose is None, then we use the theano-nose script found in
+    Theano/bin to call nosetests. Otherwise we call the provided script.
+
+    If batch_size is None, we use a default value of 100.
     """
     if stdout is None:
         stdout = sys.stdout
@@ -55,24 +57,28 @@ def main(stdout=None, stderr=None, argv=None, call_nose=None):
         stderr = sys.stderr
     if argv is None:
         argv = sys.argv
-    if call_nose is None:
-        call_nose = os.path.join(theano.__path__[0], 'tests', 'call_nose.py')
+    if theano_nose is None:
+        theano_nose = os.path.join(theano.__path__[0], '..', 'bin', 'theano-nose')
+    if batch_size is None:
+        batch_size = 100
     stdout_backup = sys.stdout
     stderr_backup = sys.stderr
     try:
         sys.stdout = stdout
         sys.stderr = stderr
-        run(stdout, stderr, argv, call_nose)
+        run(stdout, stderr, argv, theano_nose, batch_size)
     finally:
         sys.stdout = stdout_backup
         sys.stderr = stderr_backup
 
-def run(stdout, stderr, argv, call_nose):
+def run(stdout, stderr, argv, theano_nose, batch_size):
     if len(argv) == 1:
         tests_dir = theano.__path__[0]
+        other_args = []
     else:
-        assert len(argv) == 2
-        tests_dir = argv[1]
+        # tests_dir should be at the end of argv, there can be other arguments
+        tests_dir = argv[-1]
+        other_args = argv[1:-1]
         assert os.path.isdir(tests_dir)
     os.chdir(tests_dir)
     # It seems safer to fully regenerate the list of tests on each call.
@@ -86,8 +92,13 @@ def run(stdout, stderr, argv, call_nose):
     stdout.flush()
     stderr.flush()
     dummy_in = open(os.devnull)
-    rval = subprocess.call(['python', call_nose, '--collect-only', '--with-id'],
-                           stdin=dummy_in.fileno(), stdout=stdout.fileno(),
+    # We need to call 'python' on Windows, because theano-nose is not a
+    # native Windows app; and it does not hurt to call it on Unix.
+    rval = subprocess.call(
+            (['python', theano_nose, '--collect-only', '--with-id']
+                + other_args),
+            stdin=dummy_in.fileno(),
+            stdout=stdout.fileno(),
             stderr=stderr.fileno())
     stdout.flush()
     stderr.flush()
@@ -98,21 +109,23 @@ def run(stdout, stderr, argv, call_nose):
     n_tests = len(ids)
     assert n_tests == max(ids)
     # Run tests.
-    n_batch = 100
     failed = set()
     print """\
 ###################################
 # RUNNING TESTS IN BATCHES OF %s #
-###################################""" % n_batch
-    for test_id in xrange(1, n_tests + 1, n_batch):
+###################################""" % batch_size
+    for test_id in xrange(1, n_tests + 1, batch_size):
         stdout.flush()
         stderr.flush()
-        test_range = range(test_id, min(test_id + n_batch, n_tests + 1))
+        test_range = range(test_id, min(test_id + batch_size, n_tests + 1))
         # We suppress all output because we want the user to focus only on the
         # failed tests, which are re-run (with output) below.
         dummy_out = open(os.devnull, 'w')
-        rval = subprocess.call(['python', call_nose, '-q', '--with-id'] +
-                               map(str, test_range), stdout=dummy_out.fileno(),
+        rval = subprocess.call(
+                (['python', theano_nose, '-q', '--with-id']
+                    + map(str, test_range)
+                    + other_args),
+                stdout=dummy_out.fileno(),
                 stderr=dummy_out.fileno(),
                 stdin=dummy_in.fileno())
         # Recover failed test indices from the 'failed' field of the '.noseids'
@@ -132,8 +145,12 @@ def run(stdout, stderr, argv, call_nose):
 ################################"""
         stdout.flush()
         stderr.flush()
-        subprocess.call(['python', call_nose, '-v', '--with-id'] + failed,
-                        stdin=dummy_in.fileno(), stdout=stdout.fileno(),
+        subprocess.call(
+                (['python', theano_nose, '-v', '--with-id']
+                    + failed
+                    + other_args),
+                stdin=dummy_in.fileno(),
+                stdout=stdout.fileno(),
                 stderr=stderr.fileno())
         stdout.flush()
         stderr.flush()