Update Travis infrastructure to Python 3.6/3.7 and current numpy (#7)

.travis.yml

 # After changing this file, check it on:
 # http://lint.travis-ci.org/
-sudo: false
+os: linux
+dist: xenial
 cache:
   directories:
     - $HOME/.cache/pip
     - $HOME/.theano
-    - $HOME/download # Sufficient to add miniconda.sh to TRAVIS cache.
-    - $HOME/miniconda2 # Add the installation to TRAVIS cache.
-
+    - $HOME/download
+    - $HOME/miniconda
 
 language: python
-# For now, Python versions have to be listed in the "jobs" matrix
 
-# NB:
-# In before_install and install sections below,
-# some codes have been moved to separate files
-# to better handle if-else shell syntax
-# for multiple lines. New files are in
-# new folder ".travis".
+python:
+  - "3.7"
+  - "3.6"
 
-# command to install dependencies
-before_install:
-  - ./.travis/travis_before_install.sh
-  - export PATH=/home/travis/miniconda2/bin:$PATH
+stages:
+  - doc
+  - test
+
+env:
+  global:
+    - NUMPY_VERSION=1.18.1
+  jobs:
+    # - DOC=1 PART="theano/tests/test_flake8.py"
+    - PART="theano/compat theano/compile theano/d3viz theano/gof theano/misc theano/sandbox theano/scalar theano/scan_module theano/tests -e test_flake8.py theano/typed_list"
+    - PART="theano/sparse theano/tensor --exclude-test=theano.tensor.tests.test_basic --exclude-test=theano.tensor.tests.test_elemwise --exclude-test=theano.tensor.tests.test_opt --exclude-dir=theano/tensor/nnet"
+    - PART="theano/tensor/tests/test_basic.py"
+    - PART="theano/tensor/tests/test_elemwise.py theano/tensor/tests/test_opt.py"
+    - PART="theano/tensor/nnet -e test_abstract_conv.py"
+    - PART="theano/tensor/nnet/tests/test_abstract_conv.py"
 
 addons:
   apt_packages:
@@ -31,109 +38,53 @@ addons:
     - texlive-fonts-recommended
     - dvipng
 
+before_install:
+  - source .travis/travis_before_install.sh
+
 install:
-  - ./.travis/travis_install.sh
-  - source activate pyenv
-  - if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then pip install pydot; else pip install pydot-ng; fi
-  - pip install . --no-deps --upgrade
-  - pip install flake8-future-import parameterized
-  - if [[ $TRAVIS_PYTHON_VERSION != '3.4' ]]; then pip install sphinx_rtd_theme; fi
-  # nose-exclude plugin allow us to tell nosetests to exclude folder with --exclude-dir=path/to/directory.
-  - pip install nose-exclude nose-timer
-  - if [[ $NUMPY_VERSION == '1.13.1' ]]; then conda install --yes -q scipy=0.19.1; else conda install --yes -q scipy=0.14; fi  # Try to reinstall it to fix the problem
+  - source .travis/travis_install.sh
 
 jobs:
   include:
-    # define prototype for doctest
     - &doctest
       stage: doc
-      python: "2.7"
-      env: NUMPY_VERSION=1.9.1 DOC=1 PART="theano/tests/test_flake8.py"
-    # re-use prototype, changing the Python version
-    - <<: *doctest
-      python: "2.7"
-      env: NUMPY_VERSION=1.13.1 DOC=1 PART="theano/tests/test_flake8.py"
-#    - <<: *doctest
-#      python: "3.4"
-#      env: NUMPY_VERSION=1.9.1 DOC=1 PART="theano/tests/test_flake8.py"
-    - <<: *doctest
-      python: "3.6"
-      env: NUMPY_VERSION=1.13.1 DOC=1 PART="theano/tests/test_flake8.py"
+      env: DOC=1 PART="theano/tests/test_flake8.py"
     - &normaltest
       stage: test
-      python: "2.7"
-      env: NUMPY_VERSION=1.9.1 PART="theano/compat theano/compile theano/d3viz theano/gof theano/misc theano/sandbox theano/scalar theano/scan_module theano/tests -e test_flake8.py theano/typed_list"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/compat theano/compile theano/d3viz theano/gof theano/misc theano/sandbox theano/scalar theano/scan_module theano/tests -e test_flake8.py theano/typed_list"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 PART="theano/sparse theano/tensor --exclude-test=theano.tensor.tests.test_basic --exclude-test=theano.tensor.tests.test_elemwise --exclude-test=theano.tensor.tests.test_opt --exclude-dir=theano/tensor/nnet"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.13.1 PART="theano/sparse theano/tensor --exclude-test=theano.tensor.tests.test_basic --exclude-test=theano.tensor.tests.test_elemwise --exclude-test=theano.tensor.tests.test_opt --exclude-dir=theano/tensor/nnet"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/sparse theano/tensor --exclude-test=theano.tensor.tests.test_basic --exclude-test=theano.tensor.tests.test_elemwise --exclude-test=theano.tensor.tests.test_opt --exclude-dir=theano/tensor/nnet"
-    - <<: *normaltest
-      python: "3.6"
-      env: NUMPY_VERSION=1.13.1 PART="theano/sparse theano/tensor --exclude-test=theano.tensor.tests.test_basic --exclude-test=theano.tensor.tests.test_elemwise --exclude-test=theano.tensor.tests.test_opt --exclude-dir=theano/tensor/nnet"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/tests/test_basic.py"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/tests/test_basic.py"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/tests/test_elemwise.py theano/tensor/tests/test_opt.py"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/tests/test_elemwise.py theano/tensor/tests/test_opt.py"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/nnet -e test_abstract_conv.py"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/nnet -e test_abstract_conv.py"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/nnet/tests/test_abstract_conv.py"
-    - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 PART="theano/tensor/nnet/tests/test_abstract_conv.py"
-    - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 FLOAT32=1 PART="theano -e test_flake8.py --exclude-dir=theano/tensor/nnet --exclude-dir=theano/tensor/signal"
+      env: FAST_COMPILE=1 FLOAT32=1 PART="theano -e test_flake8.py --exclude-dir=theano/tensor/nnet --exclude-dir=theano/tensor/signal"
     - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 PART="theano -e test_flake8.py --exclude-dir=theano/tensor/nnet --exclude-dir=theano/tensor/signal"
+      env: FAST_COMPILE=1 PART="theano -e test_flake8.py --exclude-dir=theano/tensor/nnet --exclude-dir=theano/tensor/signal"
     - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 FLOAT32=1 PART="theano/tensor/nnet"
+      env: FAST_COMPILE=1 FLOAT32=1 PART="theano/tensor/nnet"
     - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 PART="theano/tensor/nnet"
+      env: FAST_COMPILE=1 PART="theano/tensor/nnet"
     - <<: *normaltest
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 FLOAT32=1 PART="theano/tensor/signal"
+      env: FAST_COMPILE=1 FLOAT32=1 PART="theano/tensor/signal"
     - <<: *normaltest
-      python: "3.4"
-      env: NUMPY_VERSION=1.9.1 FAST_COMPILE=1 PART="theano/tensor/signal"
+      env: FAST_COMPILE=1 PART="theano/tensor/signal"
 
 script:
+  - conda activate pyenv
   - if [[ $FAST_COMPILE == "1" ]]; then export THEANO_FLAGS=$THEANO_FLAGS,mode=FAST_COMPILE; fi
   - if [[ $FLOAT32 == "1" ]]; then export THEANO_FLAGS=$THEANO_FLAGS,floatX=float32; fi
   - export THEANO_FLAGS=$THEANO_FLAGS,warn.ignore_bug_before=all,on_opt_error=raise,on_shape_error=raise,gcc.cxxflags=-pipe
   - export MKL_THREADING_LAYER=GNU
   - export MKL_NUM_THREADS=1
   - export OMP_NUM_THREADS=1
+  - which python
   - python --version
   - uname -a
   - free -m
   - df -h
   - ulimit -a
   - echo "$PART"
-# Print information to help debug problems
+  # Print information to help debug problems
   - python -c 'import numpy; print(numpy.__version__)'
   - python -c 'import theano; print(theano.__version__)'
   - python -c 'import theano; print(theano.config.__str__(print_doc=False))'
   - python -c 'import theano; assert(theano.config.blas.ldflags != "")'
-# Run tests for the given part
+  # Run tests for the given part
   - theano-nose -v --with-timer --timer-top-n 10 $PART
-  - if [[ $DOC == "1" ]]; then python doc/scripts/docgen.py --nopdf --check; fi
-  - if [[ $DOC == "1" ]]; then python doc/scripts/docgen.py --test --check; fi
 
 after_failure:
-  - cat /home/travis/.pip/pip.log
+  - cat $HOME/.pip/pip.log

.travis/travis_before_install.sh

 #!/usr/bin/env bash
+
+set -e
+
 # Install miniconda to avoid compiling scipy
-if test -e $HOME/miniconda2/bin ; then
+if test -e $HOME/miniconda/bin ; then
     echo "miniconda already installed."
 else
     echo "Installing miniconda."
-    rm -rf $HOME/miniconda2
+    rm -rf $HOME/miniconda
     mkdir -p $HOME/download
     if [[ -d $HOME/download/miniconda.sh ]] ; then rm -rf $HOME/download/miniconda.sh ; fi
-    wget -c https://repo.continuum.io/miniconda/Miniconda2-4.5.11-Linux-x86_64.sh -O $HOME/download/miniconda.sh
-    chmod +x $HOME/download/miniconda.sh
-    $HOME/download/miniconda.sh -b
+    wget -c https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh -O $HOME/download/miniconda.sh
+
+    mkdir $HOME/.conda
+    bash $HOME/download/miniconda.sh -b -p $HOME/miniconda
 fi
+
+$HOME/miniconda/bin/conda init bash
+source ~/.bash_profile
+conda activate base
+
+hash -r
+conda config --set always_yes yes --set changeps1 no
+conda update -q conda
+conda info -a

.travis/travis_install.sh

 #!/usr/bin/env bash
-# In Python 3.4, we test the min version of NumPy and SciPy. In Python 2.7, we test more recent version.
-if test -e $HOME/miniconda2/envs/pyenv; then
-    echo "pyenv already exists."
+
+set -e
+
+if [[ $DOC == "1" ]]; then
+# this is a hack to deal with the fact that the docs and flake8 config are all set up
+# for old versions
+conda create --yes -q -n pyenv python=3.6 numpy=1.13.1
+conda activate pyenv
+conda install --yes -q mkl numpy=1.13.1 scipy=0.19.1 nose=1.3.7 pip flake8=3.5 six=1.11.0 pep8=1.7.1 pyflakes=1.6.0 mkl-service graphviz
+python -m pip install pydot-ng flake8-future-import parameterized nose-exclude nose-timer
 else
-    echo "Creating pyenv."
-    if [[ $TRAVIS_PYTHON_VERSION == '2.7' ]]; then conda create --yes -q -n pyenv python=2.7 ; fi
-    if [[ $TRAVIS_PYTHON_VERSION == '3.4' ]]; then conda create --yes -q -n pyenv python=3.4 ; fi
-    if [[ $TRAVIS_PYTHON_VERSION == '3.6' ]]; then conda create --yes -q -n pyenv python=3.6 ; fi
+conda create --yes -q -n pyenv python=$TRAVIS_PYTHON_VERSION
+conda activate pyenv
+conda install --yes -q mkl numpy scipy nose pip flake8 six pep8 pyflakes sphinx mkl-service graphviz  # libgfortran
+python -m pip install -q pydot-ng flake8-future-import parameterized sphinx_rtd_theme nose-exclude nose-timer
 fi
 
-source activate pyenv
-if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $NUMPY_VERSION == '1.9.1' ]]; then conda install --yes -q mkl numpy=1.9.1 scipy=0.14.0 nose=1.3.0 pip flake8=2.3 six=1.9.0 pep8=1.6.2 pyflakes=0.8.1 sphinx=1.5.1 mkl-service libgfortran=1 graphviz; fi
-if [[ $TRAVIS_PYTHON_VERSION == '2.7' && $NUMPY_VERSION == '1.13.1' ]]; then conda install --yes -q mkl numpy=1.13.1 scipy=0.19.1 nose=1.3.0 pip flake8=2.3 six=1.9.0 pep8=1.6.2 pyflakes=0.8.1 sphinx=1.5.1 mkl-service libgfortran=3 graphviz; fi
-if [[ $TRAVIS_PYTHON_VERSION == '3.4' && $NUMPY_VERSION == '1.9.1' ]]; then conda install --yes -q mkl numpy=1.9.1 scipy=0.14.0 nose=1.3.4 pip flake8=2.3 six=1.9.0 pep8=1.6.2 pyflakes=0.8.1 sphinx=1.5.1 mkl-service libgfortran=1 graphviz pygments=2.1.3; fi
-if [[ $TRAVIS_PYTHON_VERSION == '3.6' && $NUMPY_VERSION == '1.13.1' ]]; then conda install --yes -q mkl numpy=1.13.1 scipy=0.19.1 nose=1.3.7 pip flake8=3.5 six=1.11.0 pep8=1.7.1 pyflakes=1.6.0 sphinx=1.5.1 mkl-service libgfortran=3 graphviz; fi
-source deactivate
+python -m pip install --no-deps --upgrade -e .

bin/theano_nose.py

@@ -101,11 +101,17 @@ def main_function():
     if '--without-knownfailure' not in sys.argv:
         try:
             from numpy.testing.noseclasses import KnownFailure
+        except ImportError:
+            try:
+                from numpy.testing._private.noseclasses import KnownFailure
                 addplugins.append(KnownFailure())
             except ImportError:
                 _logger.warn(
                     'KnownFailure plugin from NumPy could not be imported. '
                     'Use --without-knownfailure to disable this warning.')
+        else:
+            addplugins.append(KnownFailure())
+
     else:
         sys.argv.remove('--without-knownfailure')
 

theano/tests/main.py

@@ -3,7 +3,11 @@ import os
 import unittest
 import sys
 
-from numpy.testing.nosetester import NoseTester
+try:
+    from numpy.testing.nosetester import NoseTester
+except ImportError:
+    # The tester has been moved in recent versions of numpy
+    from numpy.testing._private.nosetester import NoseTester
 
 
 # This class contains code adapted from NumPy,
@@ -31,51 +35,70 @@ class TheanoNoseTester(NoseTester):
         """
         # self.package_path = os.path.abspath(self.package_path)
         argv = [__file__, self.package_path]
-        argv += ['--verbosity', str(verbose)]
+        argv += ["--verbosity", str(verbose)]
         if extra_argv:
             argv += extra_argv
         return argv
 
     def _show_system_info(self):
         import theano
+
         print("Theano version %s" % theano.__version__)
         theano_dir = os.path.dirname(theano.__file__)
         print("theano is installed in %s" % theano_dir)
 
         super(TheanoNoseTester, self)._show_system_info()
 
-    def prepare_test_args(self, verbose=1, extra_argv=None, coverage=False,
-                          capture=True, knownfailure=True):
+    def prepare_test_args(
+        self,
+        verbose=1,
+        extra_argv=None,
+        coverage=False,
+        capture=True,
+        knownfailure=True,
+    ):
         """
         Prepare arguments for the `test` method.
 
         Takes the same arguments as `test`.
         """
         import nose.plugins.builtin
+
         # compile argv
         argv = self._test_argv(verbose, extra_argv)
 
         # numpy way of doing coverage
         if coverage:
-            argv += ['--cover-package=%s' % self.package_name,
-                     '--with-coverage', '--cover-tests',
-                     '--cover-inclusive', '--cover-erase']
+            argv += [
+                "--cover-package=%s" % self.package_name,
+                "--with-coverage",
+                "--cover-tests",
+                "--cover-inclusive",
+                "--cover-erase",
+            ]
 
         # Capture output only if needed
         if not capture:
-            argv += ['-s']
+            argv += ["-s"]
 
         # construct list of plugins
         plugins = []
         if knownfailure:
             from numpy.testing.noseclasses import KnownFailure
+
             plugins.append(KnownFailure())
         plugins += [p() for p in nose.plugins.builtin.plugins]
 
         return argv, plugins
 
-    def test(self, verbose=1, extra_argv=None, coverage=False, capture=True,
-             knownfailure=True):
+    def test(
+        self,
+        verbose=1,
+        extra_argv=None,
+        coverage=False,
+        capture=True,
+        knownfailure=True,
+    ):
         """
         Run tests for module using nose.
 
@@ -107,17 +130,21 @@ class TheanoNoseTester(NoseTester):
         from nose.config import Config
         from nose.plugins.manager import PluginManager
         from numpy.testing.noseclasses import NumpyTestProgram
+
         # Many Theano tests suppose device=cpu, so we need to raise an
         # error if device==gpu.
-        if not os.path.exists('theano/__init__.py'):
+        if not os.path.exists("theano/__init__.py"):
             try:
                 from theano import config
+
                 if config.device != "cpu":
-                    raise ValueError("Theano tests must be run with device=cpu."
+                    raise ValueError(
+                        "Theano tests must be run with device=cpu."
                         " This will also run GPU tests when possible.\n"
                         " If you want GPU-related tests to run on a"
                         " specific GPU device, and not the default one,"
-                                     " you should use the init_gpu_device theano flag.")
+                        " you should use the init_gpu_device theano flag."
+                    )
             except ImportError:
                 pass
 
@@ -129,15 +156,19 @@ class TheanoNoseTester(NoseTester):
         if self.package_path in os.listdir(cwd):
             # The tests give weird errors if the package to test is
             # in current directory.
-            raise RuntimeError((
+            raise RuntimeError(
+                (
                     "This function does not run correctly when, at the time "
                     "theano was imported, the working directory was theano's "
                     "parent directory. You should exit your Python prompt, change "
                     "directory, then launch Python again, import theano, then "
-                "launch theano.test()."))
+                    "launch theano.test()."
+                )
+            )
 
-        argv, plugins = self.prepare_test_args(verbose, extra_argv, coverage,
-                                               capture, knownfailure)
+        argv, plugins = self.prepare_test_args(
+            verbose, extra_argv, coverage, capture, knownfailure
+        )
 
         # The "plugins" keyword of NumpyTestProgram gets ignored if config is
         # specified. Moreover, using "addplugins" instead can lead to strange