Merge pull request #3478 from piotrfrankowski/with_statement

theano/compile/function_module.py

@@ -1214,22 +1214,17 @@ class FunctionMaker(object):
                 print('graph_db already exists')
             else:
                 # create graph_db
-                f = open(graph_db_file, 'wb')
-                print('create new graph_db in %s' % graph_db_file)
-                # file needs to be open and closed for every pickle
-                f.close()
+                with open(graph_db_file, 'wb') as f:
+                    print('create new graph_db in %s' % graph_db_file)
             # load the graph_db dictionary
             try:
-                f = open(graph_db_file, 'rb')
-                # Temporary hack to allow
-                # theano.scan_module.tests.test_scan.T_Scan to
-                # finish. Should be changed in definitive version.
-                tmp = theano.config.unpickle_function
-                theano.config.unpickle_function = False
-                graph_db = pickle.load(f)
-
-                # hack end
-                f.close()
+                with open(graph_db_file, 'rb') as f:
+                    # Temporary hack to allow
+                    # theano.scan_module.tests.test_scan.T_Scan to
+                    # finish. Should be changed in definitive version.
+                    tmp = theano.config.unpickle_function
+                    theano.config.unpickle_function = False
+                    graph_db = pickle.load(f)
                 print('graph_db loaded and it is not empty')
             except EOFError as e:
                 # the file has nothing in it
@@ -1358,9 +1353,8 @@ class FunctionMaker(object):
             before_opt = self.fgraph.clone(check_integrity=False)
             optimizer_profile = optimizer(self.fgraph)
             graph_db.update({before_opt: self.fgraph})
-            f = open(graph_db_file, 'wb')
-            pickle.dump(graph_db, f, -1)
-            f.close()
+            with open(graph_db_file, 'wb') as f:
+                pickle.dump(graph_db, f, -1)
             print('new graph saved into graph_db')
         release_lock()
         return optimizer_profile

theano/compile/tests/test_function.py

@@ -18,9 +18,8 @@ def test_function_dump():
         tmpdir = tempfile.mkdtemp()
         fname = os.path.join(tmpdir, 'test_function_dump.pkl')
         theano.function_dump(fname, [v], v + 1)
-        f = open(fname, 'rb')
-        l = pickle.load(f)
-        f.close()
+        with open(fname, 'rb') as f:
+            l = pickle.load(f)
     finally:
         if tmpdir is not None:
             shutil.rmtree(tmpdir)

theano/d3viz/d3viz.py

@@ -86,9 +86,8 @@ def d3viz(fct, outfile, copy_deps=True, *args, **kwargs):
 
     # Read template HTML file
     template_file = os.path.join(__path__, 'html', 'template.html')
-    f = open(template_file)
-    template = f.read()
-    f.close()
+    with open(template_file) as f:
+        template = f.read()
 
     # Copy dependencies to output directory
     src_deps = __path__

theano/gof/callcache.py

@@ -10,18 +10,16 @@ class CallCache(object):
         try:
             if filename is None:
                 raise IOError('bad filename')  # just goes to except
-            f = open(filename, 'r')
-            self.cache = pickle.load(f)
-            f.close()
+            with open(filename, 'r') as f:
+                self.cache = pickle.load(f)
         except IOError:
             self.cache = {}
 
     def persist(self, filename=None):
         if filename is None:
             filename = self.filename
-        f = open(filename, 'w')
-        pickle.dump(self.cache, f)
-        f.close()
+        with open(filename, 'w') as f:
+            pickle.dump(self.cache, f)
 
     def call(self, fn, args=(), key=None):
         if key is None:

theano/gof/cmodule.py

@@ -2143,15 +2143,14 @@ class GCC_compiler(Compiler):
         lib_dirs = std_lib_dirs() + lib_dirs
 
         cppfilename = os.path.join(location, 'mod.cpp')
-        cppfile = open(cppfilename, 'w')
+        with open(cppfilename, 'w') as cppfile:
 
-        _logger.debug('Writing module C++ code to %s', cppfilename)
+            _logger.debug('Writing module C++ code to %s', cppfilename)
 
-        cppfile.write(src_code)
-        # Avoid gcc warning "no newline at end of file".
-        if not src_code.endswith('\n'):
-            cppfile.write('\n')
-        cppfile.close()
+            cppfile.write(src_code)
+            # Avoid gcc warning "no newline at end of file".
+            if not src_code.endswith('\n'):
+                cppfile.write('\n')
 
         lib_filename = os.path.join(
             location,

theano/gof/compiledir.py

@@ -361,11 +361,9 @@ def print_compiledir_content():
     total_key_sizes = 0
     nb_keys = {}
     for dir in os.listdir(compiledir):
-        file = None
-        try:
+        filename = os.path.join(compiledir, dir, "key.pkl")
+        with open(filename, 'rb') as file:
             try:
-                filename = os.path.join(compiledir, dir, "key.pkl")
-                file = open(filename, 'rb')
                 keydata = pickle.load(file)
                 ops = list(set([x for x in flatten(keydata.keys)
                                 if isinstance(x, theano.gof.Op)]))
@@ -387,9 +385,6 @@ def print_compiledir_content():
                 nb_keys[len(keydata.keys)] += 1
             except IOError:
                 pass
-        finally:
-            if file is not None:
-                file.close()
 
     print("List of %d compiled individual ops in this theano cache %s:" % (
         len(table), compiledir))

theano/gof/compilelock.py

@@ -339,9 +339,8 @@ def refresh_lock(lock_file):
         ''.join([str(random.randint(0, 9)) for i in range(10)]),
         hostname)
     try:
-        lock_write = open(lock_file, 'w')
-        lock_write.write(unique_id + '\n')
-        lock_write.close()
+        with open(lock_file, 'w') as lock_write:
+            lock_write.write(unique_id + '\n')
     except Exception:
         # In some strange case, this happen.  To prevent all tests
         # from failing, we release the lock, but as there is a

theano/misc/buildbot_filter.py

@@ -24,6 +24,7 @@ if __name__ == "__main__":
     import pdb
     pdb.set_trace()
     if len(sys.argv) > 1:
-        print(filter_output(open(sys.argv[1])))
+        with open(sys.argv[1]) as f:
+            print(filter_output(f))
     else:
         print(filter_output(sys.stdin))

theano/misc/check_duplicate_key.py

@@ -23,9 +23,8 @@ for dir in dirs:
 
     key = None
     try:
-        f = open(os.path.join(dir, "key.pkl"))
-        key = f.read()
-        f.close()
+        with open(os.path.join(dir, "key.pkl")) as f:
+            key = f.read()
         keys.setdefault(key, 0)
         keys[key] += 1
         del f
@@ -36,9 +35,8 @@ for dir in dirs:
         path = os.path.join(dir, "mod.cpp")
         if not os.path.exists(path):
             path = os.path.join(dir, "mod.cu")
-        f = open(path)
-        mod = f.read()
-        f.close()
+        with open(path) as f:
+            mod = f.read()
         mods.setdefault(mod, ())
         mods[mod] += (key,)
         del mod

theano/misc/hooks/check_whitespace.py

@@ -145,31 +145,27 @@ def get_file_contents(filename, revision="tip"):
 
 def save_commit_message(filename):
     commit_message = run_mercurial_command("tip --template '{desc}'")
-    save_file = open(filename, "w")
-    save_file.write(commit_message)
-    save_file.close()
+    with open(filename, "w") as save_file:
+        save_file.write(commit_message)
 
 
 def save_diffs(diffs, filename):
     diff = "\n\n".join(diffs)
-    diff_file = open(filename, "w")
-    diff_file.write(diff)
-    diff_file.close()
+    with open(filename, "w") as diff_file:
+        diff_file.write(diff)
 
 
 def should_skip_commit():
     if not os.path.exists(SKIP_WHITESPACE_CHECK_FILENAME):
         return False
-    whitespace_check_file = open(SKIP_WHITESPACE_CHECK_FILENAME, "r")
-    whitespace_check_changeset = whitespace_check_file.read()
-    whitespace_check_file.close()
+    with open(SKIP_WHITESPACE_CHECK_FILENAME, "r") as whitespace_check_file:
+        whitespace_check_changeset = whitespace_check_file.read()
     return whitespace_check_changeset == parent_commit()
 
 
 def save_skip_next_commit():
-    whitespace_check_file = open(SKIP_WHITESPACE_CHECK_FILENAME, "w")
-    whitespace_check_file.write(parent_commit())
-    whitespace_check_file.close()
+    with open(SKIP_WHITESPACE_CHECK_FILENAME, "w") as whitespace_check_file:
+        whitespace_check_file.write(parent_commit())
 
 
 def main(argv=None):

theano/misc/hooks/reindent.py

@@ -132,9 +132,8 @@ def check(file):
                 shutil.copyfile(file, bak)
                 if verbose:
                     print("backed up", file, "to", bak)
-            f = open(file, "w")
-            r.write(f)
-            f.close()
+            with open(file, "w") as f:
+                r.write(f)
             if verbose:
                 print("wrote new", file)
         return True

theano/sandbox/cuda/nvcc_compiler.py

@@ -233,12 +233,11 @@ class NVCC_compiler(Compiler):
                 lib_dirs.append(python_lib)
 
         cppfilename = os.path.join(location, 'mod.cu')
-        cppfile = open(cppfilename, 'w')
+        with open(cppfilename, 'w') as cppfile:
 
-        _logger.debug('Writing module C++ code to %s', cppfilename)
+            _logger.debug('Writing module C++ code to %s', cppfilename)
+            cppfile.write(src_code)
 
-        cppfile.write(src_code)
-        cppfile.close()
         lib_filename = os.path.join(location, '%s.%s' %
                 (module_name, get_lib_extension()))
 

theano/scan_module/scan_perform_ext.py

@@ -75,7 +75,8 @@ except ImportError:
                 )
                 raise ImportError("The file lazylinker_c.c is not available.")
 
-            code = open(cfile).read()
+            with open(cfile) as f:
+                code = f.read()
             loc = os.path.join(config.compiledir, dirname)
             if not os.path.exists(loc):
                 try:
@@ -113,7 +114,8 @@ except ImportError:
                                              hide_symbols=False)
             # Save version into the __init__.py file.
             init_py = os.path.join(loc, '__init__.py')
-            open(init_py, 'w').write('_version = %s\n' % version)
+            with open(init_py, 'w') as f:
+                f.write('_version = %s\n' % version)
             # If we just compiled the module for the first time, then it was
             # imported at the same time: we need to make sure we do not
             # reload the now outdated __init__.pyc below.

theano/scan_module/tests/test_scan.py

@@ -250,16 +250,10 @@ class T_Scan(unittest.TestCase):
             tmpdir = mkdtemp()
             os.chdir(tmpdir)
 
-            f_out = open('tmp_scan_test_pickle.pkl', 'wb')
-            try:
+            with open('tmp_scan_test_pickle.pkl', 'wb') as f_out:
                 pickle.dump(_my_f, f_out, protocol=-1)
-            finally:
-                f_out.close()
-            f_in = open('tmp_scan_test_pickle.pkl', 'rb')
-            try:
+            with open('tmp_scan_test_pickle.pkl', 'rb') as f_in:
                 my_f = pickle.load(f_in)
-            finally:
-                f_in.close()
         finally:
             # Get back to the original dir, and delete the temporary one.
             os.chdir(origdir)

theano/tensor/tests/test_opt.py

@@ -3852,16 +3852,13 @@ class test_shapeoptimizer(unittest.TestCase):
         # Due to incompatibilities between python 2 and 3 in the format
         # of pickled numpy ndarray, we have to force an encoding
         from theano.misc.pkl_utils import CompatUnpickler
-        pkl_file = open(pkl_filename, "rb")
-        try:
+        with open(pkl_filename, "rb") as pkl_file:
             if PY3:
                 u = CompatUnpickler(pkl_file, encoding="latin1")
             else:
                 u = CompatUnpickler(pkl_file)
             fn_args = u.load()
             theano.function(**fn_args)
-        finally:
-            pkl_file.close()
 
 
 class test_assert(utt.InferShapeTester):

theano/tests/run_tests_in_batch.py

@@ -265,99 +265,101 @@ def run(stdout, stderr, argv, theano_nose, batch_size, time_profile,
         fields = ('Fields: computation time; nosetests sequential id;'
                   ' test name; parent class (if any); outcome\n\n')
         path_nosort = os.path.join(sav_dir, 'timeprof_nosort')
-        f_nosort = open(path_nosort, 'w')
-        f_nosort.write('TIME-PROFILING OF THEANO\'S NOSETESTS'
-                       ' (by sequential id)\n\n' + stamp + fields)
-        f_nosort.flush()
-        for test_floor in xrange(1, n_tests + 1, batch_size):
-            for test_id in xrange(test_floor, min(test_floor + batch_size,
-                                                 n_tests + 1)):
-                # Print the test we will start in the raw log to help
-                # debug tests that are too long.
-                f_rawlog.write("\n%s Will run test #%d %s\n" % (
-                    time.ctime(), test_id, data["ids"][test_id]))
-                f_rawlog.flush()
-
-                p_out = output_subprocess_Popen(
-                    ([python, theano_nose, '-v', '--with-id']
-                     + [str(test_id)] + argv +
-                     ['--disabdocstring']))
-                    # the previous option calls a custom Nosetests plugin
-                    # precluding automatic sustitution of doc. string for
-                    # test name in display
-                    # (see class 'DisabDocString' in file theano-nose)
-
-                # recovering and processing data from pipe
-                err = p_out[1]
-                # print the raw log
-                f_rawlog.write(err)
-                f_rawlog.flush()
-
-                # parsing the output
-                l_err = err.split()
-                try:
-                    pos_id = getIndexOfFirst(l_err, '#')
-                    prof_id = l_err[pos_id]
-                    pos_dot = getIndexOfFirst(l_err, '...')
-                    prof_test = ''
-                    for s in l_err[pos_id + 1: pos_dot]:
-                        prof_test += s + ' '
-                    if 'OK' in err:
-                        pos_ok = getIndexOfLast(l_err, 'OK')
-                        if len(l_err) == pos_ok + 1:
-                            prof_time = float(l_err[pos_ok - 1][0:-1])
-                            prof_pass = 'OK'
-                        elif 'SKIP' in l_err[pos_ok + 1]:
-                            prof_time = 0.
-                            prof_pass = 'SKIPPED TEST'
-                        elif 'KNOWNFAIL' in l_err[pos_ok + 1]:
-                            prof_time = float(l_err[pos_ok - 1][0:-1])
-                            prof_pass = 'OK'
+        # probably this part can be extracted for function with many args
+        with open(path_nosort, 'w') as f_nosort:
+            # begin of saving nosort
+            f_nosort.write('TIME-PROFILING OF THEANO\'S NOSETESTS'
+                           ' (by sequential id)\n\n' + stamp + fields)
+            f_nosort.flush()
+            for test_floor in xrange(1, n_tests + 1, batch_size):
+                for test_id in xrange(test_floor, min(test_floor + batch_size,
+                                                     n_tests + 1)):
+                    # Print the test we will start in the raw log to help
+                    # debug tests that are too long.
+                    f_rawlog.write("\n%s Will run test #%d %s\n" % (
+                        time.ctime(), test_id, data["ids"][test_id]))
+                    f_rawlog.flush()
+
+                    p_out = output_subprocess_Popen(
+                        ([python, theano_nose, '-v', '--with-id']
+                         + [str(test_id)] + argv +
+                         ['--disabdocstring']))
+                        # the previous option calls a custom Nosetests plugin
+                        # precluding automatic sustitution of doc. string for
+                        # test name in display
+                        # (see class 'DisabDocString' in file theano-nose)
+
+                    # recovering and processing data from pipe
+                    err = p_out[1]
+                    # print the raw log
+                    f_rawlog.write(err)
+                    f_rawlog.flush()
+
+                    # parsing the output
+                    l_err = err.split()
+                    try:
+                        pos_id = getIndexOfFirst(l_err, '#')
+                        prof_id = l_err[pos_id]
+                        pos_dot = getIndexOfFirst(l_err, '...')
+                        prof_test = ''
+                        for s in l_err[pos_id + 1: pos_dot]:
+                            prof_test += s + ' '
+                        if 'OK' in err:
+                            pos_ok = getIndexOfLast(l_err, 'OK')
+                            if len(l_err) == pos_ok + 1:
+                                prof_time = float(l_err[pos_ok - 1][0:-1])
+                                prof_pass = 'OK'
+                            elif 'SKIP' in l_err[pos_ok + 1]:
+                                prof_time = 0.
+                                prof_pass = 'SKIPPED TEST'
+                            elif 'KNOWNFAIL' in l_err[pos_ok + 1]:
+                                prof_time = float(l_err[pos_ok - 1][0:-1])
+                                prof_pass = 'OK'
+                            else:
+                                prof_time = 0.
+                                prof_pass = 'FAILED TEST'
                         else:
                             prof_time = 0.
                             prof_pass = 'FAILED TEST'
-                    else:
-                        prof_time = 0.
-                        prof_pass = 'FAILED TEST'
-                except Exception:
-                    prof_time = 0
-                    prof_id = '#' + str(test_id)
-                    prof_test = ('FAILED PARSING, see raw log for details'
-                                 ' on test')
-                    prof_pass = ''
-                prof_tuple = (prof_time, prof_id, prof_test, prof_pass)
-
-                # appending tuple to master list
-                prof_master_nosort.append(prof_tuple)
-
-                # write the no sort file
-                s_nosort = ((str(prof_tuple[0]) + 's').ljust(10) +
-                 " " + prof_tuple[1].ljust(7) + " " +
-                 prof_tuple[2] + prof_tuple[3] +
-                 "\n")
-                f_nosort.write(s_nosort)
-                f_nosort.flush()
-
-            print('%s%% time-profiled' % ((test_id * 100) // n_tests))
-        f_rawlog.close()
-
-        # sorting tests according to running-time
-        prof_master_sort = sorted(prof_master_nosort,
-                                  key=lambda test: test[0], reverse=True)
-
-        # saving results to readable files
-        path_sort = os.path.join(sav_dir, 'timeprof_sort')
-        f_sort = open(path_sort, 'w')
-        f_sort.write('TIME-PROFILING OF THEANO\'S NOSETESTS'
-                     ' (sorted by computation time)\n\n' + stamp + fields)
-        for i in xrange(len(prof_master_nosort)):
-            s_sort = ((str(prof_master_sort[i][0]) + 's').ljust(10) +
-                 " " + prof_master_sort[i][1].ljust(7) + " " +
-                 prof_master_sort[i][2] + prof_master_sort[i][3] +
-                 "\n")
-            f_sort.write(s_sort)
-        f_nosort.close()
-        f_sort.close()
+                    except Exception:
+                        prof_time = 0
+                        prof_id = '#' + str(test_id)
+                        prof_test = ('FAILED PARSING, see raw log for details'
+                                     ' on test')
+                        prof_pass = ''
+                    prof_tuple = (prof_time, prof_id, prof_test, prof_pass)
+
+                    # appending tuple to master list
+                    prof_master_nosort.append(prof_tuple)
+
+                    # write the no sort file
+                    s_nosort = ((str(prof_tuple[0]) + 's').ljust(10) +
+                     " " + prof_tuple[1].ljust(7) + " " +
+                     prof_tuple[2] + prof_tuple[3] +
+                     "\n")
+                    f_nosort.write(s_nosort)
+                    f_nosort.flush()
+
+                print('%s%% time-profiled' % ((test_id * 100) // n_tests))
+            f_rawlog.close()
+
+            # sorting tests according to running-time
+            prof_master_sort = sorted(prof_master_nosort,
+                                      key=lambda test: test[0], reverse=True)
+
+            # saving results to readable files
+            path_sort = os.path.join(sav_dir, 'timeprof_sort')
+            with open(path_sort, 'w') as f_sort:
+                f_sort.write('TIME-PROFILING OF THEANO\'S NOSETESTS'
+                             ' (sorted by computation time)\n\n' + stamp + fields)
+                for i in xrange(len(prof_master_nosort)):
+                    s_sort = ((str(prof_master_sort[i][0]) + 's').ljust(10) +
+                         " " + prof_master_sort[i][1].ljust(7) + " " +
+                         prof_master_sort[i][2] + prof_master_sort[i][3] +
+                         "\n")
+                    f_sort.write(s_sort)
+
+            # end of saving nosort
 
 if __name__ == '__main__':
     sys.exit(main())

theano/tests/test_flake8.py

@@ -249,17 +249,16 @@ def check_all_files(dir_path=theano.__path__[0], pattern='*.py'):
     the "whitelist_flake8" in this file.
     """
 
-    f_txt = open('theano_filelist.txt', 'a')
-    for (dir, _, files) in os.walk(dir_path):
-        for f in files:
-            if fnmatch(f, pattern):
-                error_num = flake8.main.check_file(os.path.join(dir, f),
-                                                   ignore=ignore)
-                if error_num > 0:
-                    path = os.path.relpath(os.path.join(dir, f),
-                                           theano.__path__[0])
-                    f_txt.write('"' + path + '",\n')
-    f_txt.close()
+    with open('theano_filelist.txt', 'a') as f_txt:
+        for (dir, _, files) in os.walk(dir_path):
+            for f in files:
+                if fnmatch(f, pattern):
+                    error_num = flake8.main.check_file(os.path.join(dir, f),
+                                                       ignore=ignore)
+                    if error_num > 0:
+                        path = os.path.relpath(os.path.join(dir, f),
+                                               theano.__path__[0])
+                        f_txt.write('"' + path + '",\n')
 
 
 if __name__ == "__main__":