Merge pull request #768 from larseeri/time_nose_tests

bin/theano-nose

 #!/usr/bin/env python
 
-__authors__ = "Olivier Delalleau, Pascal Lamblin"
+__authors__ = "Olivier Delalleau, Pascal Lamblin, Eric Larsen"
 __contact__ = "delallea@iro"
 
 """
@@ -13,10 +13,20 @@ It is also used to load the KnownFailure plugin, in order to hide
 KnownFailureTests error messages. Use --without-knownfailure to
 disable that plugin.
 
-If the --batch option is used, it will call `run_tests_in_batch.py`,
-in order to run the tests by batches, not all at the same time.
-`run_tests_in_batch.py` will in turn call back this script in another
-process.
+There are two additional local options: '--batch[=n]' and '--time-profile'.
+
+If '--batch[=n]' is used without '--time-profile', this script will call
+run_tests_in_batch.py` in order to run the tests by batches, not all at the
+same time. The batches will comprise 100 elements each by default and
+'n' elements if the option '=n' is specified.
+
+If the '--time-profile' option is used, it will call `run_tests_in_batch.py`
+with the option time_profile=True to conduct time-profiling of the tests.
+(See 'help' function below for details.) If also specified, '--batch[=n]'
+option will be interpreted as an indication of the number of tests to be run
+between notifications of progress to standard output.
+
+`run_tests_in_batch.py` will in turn call back this script in another process.
 """
 
 import logging
@@ -26,14 +36,14 @@ _logger.setLevel(logging.WARN)
 import nose
 import textwrap
 import sys
-
+from nose.plugins import Plugin
 
 def main():
     # Handle --batch[=n] arguments
     batch_args = [arg for arg in sys.argv if arg.startswith('--batch')]
     for arg in batch_args:
         sys.argv.remove(arg)
-
+    batch_size = None
     if len(batch_args):
         if len(batch_args) > 1:
             _logger.warn(
@@ -44,12 +54,17 @@ def main():
         elems = batch_arg.split('=', 1)
         if len(elems) == 2:
             batch_size = int(elems[1])
-        else:
-            # Use run_tests_in_batch's default
-            batch_size = None
 
+    # Handle --time_prof arguments
+    time_prof_args = [arg for arg in sys.argv if arg=='--time-profile']
+    for arg in time_prof_args:
+        sys.argv.remove(arg)
+            
+    # Time-profiling and batch modes
+    if len(time_prof_args) or len(batch_args):
         from theano.tests import run_tests_in_batch
-        return run_tests_in_batch.main(batch_size=batch_size)
+        return run_tests_in_batch.main(batch_size=batch_size,
+                            time_profile=len(time_prof_args) > 0)
 
     # Non-batch mode.
     addplugins = []
@@ -65,6 +80,13 @@ def main():
                 'Use --without-knownfailure to disable this warning.')
     else:
         sys.argv.remove('--without-knownfailure')
+        
+    # When 'theano-nose' is called-back under the time-profile option, an
+    # instance of the custom Nosetests plugin class 'DisabDocString' (see
+    # below) is loaded. The latter ensures that the test name will not be
+    # replaced in display by the first line of the documentation string.
+    if '--disabdocstring' in sys.argv:
+        addplugins.append(DisabDocString())
 
     return nose.main(addplugins=addplugins)
 
@@ -77,12 +99,40 @@ def help():
         KnownFailure plugin, in order to hide KnownFailureTests error
         messages. It also supports executing tests by batches.
 
-        Options:
-
-            --batch[=n]: Do not run all the tests in one run, but split
-                         the execution in batches of `n` tests each.
-                         Default n is 100.
-
+        Local options:
+
+            --batch[=n]:
+                If specified without option '--time-profile', do not run all
+                the tests in one run, but split the execution in batches of
+                `n` tests each. Default n is 100.
+
+            --time-profile:
+                Each test will be run and timed separately and the results will
+                be deposited in the files 'timeprof_sort', 'timeprof_nosort'
+                and 'timeprof_rawlog' in the current directory. If the
+                '--batch[=n]' option is also specified, notification of the
+                progresses will be made to standard output after every group of
+                n tests. Otherwise, notification will occur after every group
+                of 100 tests.
+    
+                The files 'timeprof_sort' and 'timeprof_nosort' both contain one
+                record for each test and comprise the following fields:
+                - test running-time
+                - nosetests sequential test number
+                - test name
+                - name of class to which test belongs (if any), otherwise full
+                  information is contained in test name
+                - test outcome ('OK', 'SKIPPED TEST', 'FAILED TEST' or
+                  'FAILED PARSING')
+    
+                In 'timeprof_sort', test records are sorted according to
+                running-time whereas in 'timeprof_nosort' records are reported
+                according to sequential number. The former classification is the
+                main information source for time-profiling. Since tests belonging
+                to same or close classes and files have close sequential, the
+                latter may be used to identify duration patterns among the tests
+                numbers. A full log is also saved as 'timeprof_rawlog'.
+    
             --help, -h: Displays this help.
 
             --without-knownfailure: Do not load the KnownFailure plugin.
@@ -93,6 +143,63 @@ def help():
     print textwrap.dedent(help_msg)
 
 
+class DisabDocString(Plugin):
+    
+    """
+    When activated, a custom Nosetests plugin created through this class
+    will preclude automatic replacement in display of the name of the test
+    by the first line in its documentation string.
+    
+    Sources:
+    http://nose.readthedocs.org/en/latest/developing.html
+    http://nose.readthedocs.org/en/latest/further_reading.html
+    http://www.siafoo.net/article/54
+    https://github.com/nose-devs/nose/issues/294
+    http://python-nose.googlecode.com/svn/trunk/nose/plugins/base.py
+    Nat Williams:
+    https://github.com/Merino/nose-description-fixer-plugin/commit/
+        df94596f29c04fea8001713dd9b04bf3720aebf4
+    """
+    
+    enabled = False # plugin disabled by default
+    score = 2000  # high score ensures priority over other plugins
+    
+    def __init__(self):
+        # 'super.__init__(self):' would have achieved exactly the same
+        if self.name is None:
+            self.name = self.__class__.__name__.lower()
+        if self.enableOpt is None:
+            self.enableOpt = ("enable_plugin_%s"
+                              % self.name.replace('-', '_'))
+
+    def options(self, parser, env):
+        env_opt = 'NOSE_WITH_%s' % self.name.upper()
+        # latter expression to be used if plugin called from the command line
+        parser.add_option("--%s" % self.name,
+                          # will be called with Nosetests 'main' or 'run'
+                          # function's' argument '--disabdocstring'
+                          action="store_true",
+                          dest=self.enableOpt,
+                          # the latter entails that the boolean self.enableOpt
+                          # is set to 'True' when plugin is called through a
+                          # function's argument
+                          default=env.get(env_opt),
+                          # entails that plugin will be enabled when command
+                          # line trigger 'env_opt' will be activated
+                          help="Enable plugin %s: %s [%s]" %
+                          (self.__class__.__name__,
+                           self.help(), env_opt))
+
+    def configure(self, options, conf):
+        self.conf = conf
+        # plugin will be enabled when called through argument
+        self.enabled = getattr(options, self.enableOpt)
+
+    def describeTest(self, test):
+        # 'describeTest' is also called when the test result in Nosetests calls
+        # 'test.shortDescription()' and can thus be used to alter the display.
+        return False
+    
 if __name__ == '__main__':
     if '--help' in sys.argv or '-h' in sys.argv:
         help()