提交 19d905a8 authored 作者: Olivier Delalleau's avatar Olivier Delalleau

Make sure that 'theano-cache' clear actually deletes all modules

上级 01a13aad
...@@ -6,7 +6,7 @@ from theano.gof.cc import get_module_cache ...@@ -6,7 +6,7 @@ from theano.gof.cc import get_module_cache
if len(sys.argv) == 1: if len(sys.argv) == 1:
print config.compiledir print config.compiledir
elif sys.argv[1] in ('clear'): elif sys.argv[1] in ('clear'):
get_module_cache().clear() get_module_cache().clear(unversioned_min_age=-1)
else: else:
print 'command "%s" not recognized' % sys.argv[1] print 'command "%s" not recognized' % sys.argv[1]
print 'Type "theano-cache" to print the cache location' print 'Type "theano-cache" to print the cache location'
......
...@@ -471,7 +471,7 @@ class ModuleCache(object): ...@@ -471,7 +471,7 @@ class ModuleCache(object):
"key not equal to unpickled version (Hint:" "key not equal to unpickled version (Hint:"
" verify the __eq__ and __hash__ functions" " verify the __eq__ and __hash__ functions"
" for your Ops", (key, key_from_file)) " for your Ops", (key, key_from_file))
# Adding the key file to this set measn it is a # Adding the key file to this set means it is a
# versioned key. # versioned key.
self.loaded_key_pkl.add(key_pkl) self.loaded_key_pkl.add(key_pkl)
except cPickle.UnpicklingError: except cPickle.UnpicklingError:
...@@ -491,7 +491,7 @@ class ModuleCache(object): ...@@ -491,7 +491,7 @@ class ModuleCache(object):
return rval return rval
age_thresh_del = 60*60*24*31#31 days age_thresh_del = 60*60*24*31#31 days
age_thresh_del_unversionned = 60*60*24*7#7 days age_thresh_del_unversioned = 60*60*24*7#7 days
"""The default age threshold for `clear_old` (in seconds) """The default age threshold for `clear_old` (in seconds)
""" """
...@@ -510,13 +510,13 @@ class ModuleCache(object): ...@@ -510,13 +510,13 @@ class ModuleCache(object):
# update the age of modules that have been accessed by other processes # update the age of modules that have been accessed by other processes
# and get all module that are too old to use.(not loaded in self.entry_from_key) # and get all module that are too old to use.(not loaded in self.entry_from_key)
too_old_to_use = self.refresh() too_old_to_use = self.refresh()
too_old_to_use = [(None,entry) for entry in too_old_to_use] too_old_to_use = [(None, entry) for entry in too_old_to_use]
time_now = time.time() time_now = time.time()
# the .items() is important here: # the .items() is important here:
# we need to get a copy of the whole list of keys and entries # we need to get a copy of the whole list of keys and entries
items_copy = list(self.entry_from_key.iteritems()) items_copy = list(self.entry_from_key.iteritems())
for key, entry in items_copy+too_old_to_use: for key, entry in items_copy + too_old_to_use:
age = time_now - last_access_time(entry) age = time_now - last_access_time(entry)
if age > age_thresh_del: if age > age_thresh_del:
# TODO: we are assuming that modules that haven't been accessed in over # TODO: we are assuming that modules that haven't been accessed in over
...@@ -533,17 +533,19 @@ class ModuleCache(object): ...@@ -533,17 +533,19 @@ class ModuleCache(object):
finally: finally:
compilelock.release_lock() compilelock.release_lock()
def clear(self): def clear(self, unversioned_min_age=None):
""" """
Clear all the elements of the cache Clear all the elements of the cache
""" """
self.clear_old(-1.0) self.clear_old(-1.0)
self.clear_unversioned() self.clear_unversioned(min_age=unversioned_min_age)
def clear_unversioned(self): def clear_unversioned(self, min_age=None):
"""Delete unversioned dynamic modules from the internal dictionaries and from the """Delete unversioned dynamic modules from the internal dictionaries and from the
filesystem. filesystem.
""" """
if min_age is None:
min_age = self.age_thresh_del_unversioned
items_copy = list(self.entry_from_key.iteritems()) items_copy = list(self.entry_from_key.iteritems())
for key, entry in items_copy: for key, entry in items_copy:
version, rest = key version, rest = key
...@@ -571,12 +573,13 @@ class ModuleCache(object): ...@@ -571,12 +573,13 @@ class ModuleCache(object):
has_key = False has_key = False
if not has_key: if not has_key:
age = time_now - last_access_time(os.path.join(self.dirname, filename)) age = time_now - last_access_time(os.path.join(self.dirname, filename))
#In normal case, the processus that created this directory will delete it # In normal case, the processus that created this directory
#In case this processus crash, it won't be cleaned up. # will delete it. However, if this processus crashes, it
#As we don't know how to know if this directory is still used # will not be cleaned up.
#we wait 1 weak and suppose that the processus crashed # As we don't know if this directory is still used, we wait
#and we do the clean up for it. # one week and suppose that the processus crashed, and we
if age > self.age_thresh_del_unversionned: # take care of the clean-up.
if age > min_age:
info("clear_unversioned removing cache dir", filename) info("clear_unversioned removing cache dir", filename)
_rmtree(os.path.join(self.dirname, filename)) _rmtree(os.path.join(self.dirname, filename))
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论