提交 353cd469 authored 作者: Frederic Bastien's avatar Frederic Bastien

now cmodule don't use file that are 3 weeks old and delete them after 31 day.

This is to remove most of error about deleted file in the buildbot. It don't solve the problem if a process run for more then 1 week.
上级 7014437a
...@@ -266,6 +266,9 @@ class ModuleCache(object): ...@@ -266,6 +266,9 @@ class ModuleCache(object):
warning("Entry 1:", self.entry_from_key[k1]) warning("Entry 1:", self.entry_from_key[k1])
warning("hash 1:", hash(k1)) warning("hash 1:", hash(k1))
age_thresh_use = 60*60*24*24
"""The default age threshold for `clear_old` (in seconds)
"""
def refresh(self): def refresh(self):
"""Update self.entry_from_key by walking the cache directory structure. """Update self.entry_from_key by walking the cache directory structure.
...@@ -278,6 +281,7 @@ class ModuleCache(object): ...@@ -278,6 +281,7 @@ class ModuleCache(object):
compilelock.get_lock() compilelock.get_lock()
try: try:
# add entries that are not in the entry_from_key dictionary # add entries that are not in the entry_from_key dictionary
time_now = time.time()
for root, dirs, files in os.walk(self.dirname): for root, dirs, files in os.walk(self.dirname):
if os.path.join(root, 'key.pkl') in self.loaded_key_pkl: if os.path.join(root, 'key.pkl') in self.loaded_key_pkl:
continue continue
...@@ -291,7 +295,7 @@ class ModuleCache(object): ...@@ -291,7 +295,7 @@ class ModuleCache(object):
except: except:
# the directory is still in use?? We just leave it for future removal. # the directory is still in use?? We just leave it for future removal.
pass pass
elif 'key.pkl' in files: elif 'key.pkl' in files and (time_now - last_access_time(module_name_from_dir(root)))<self.age_thresh_use:
key_pkl = os.path.join(root, 'key.pkl') key_pkl = os.path.join(root, 'key.pkl')
debug('refresh adding', key_pkl) debug('refresh adding', key_pkl)
try: try:
...@@ -432,18 +436,18 @@ class ModuleCache(object): ...@@ -432,18 +436,18 @@ class ModuleCache(object):
#debug('stats', self.stats, sum(self.stats)) #debug('stats', self.stats, sum(self.stats))
return rval return rval
age_thresh = 60*60*24*31 age_thresh_del = 60*60*24*31
"""The default age threshold for `clear_old` (in seconds) """The default age threshold for `clear_old` (in seconds)
""" """
def clear_old(self, age_thresh=None): #default to a 31-day age_threshold def clear_old(self, age_thresh_del=None): #default to a 31-day age_thresh_delold
""" """
Delete entries from the filesystem for cache entries that are too old. Delete entries from the filesystem for cache entries that are too old.
:param age_thresh: dynamic modules whose last access time is more than ``age_thresh`` :param age_thresh_del: dynamic modules whose last access time is more than ``age_thresh_del``
seconds ago will be erased. seconds ago will be erased.
""" """
if age_thresh is None: if age_thresh_del is None:
age_thresh = self.age_thresh age_thresh_del = self.age_thresh_del
compilelock.get_lock() compilelock.get_lock()
try: try:
...@@ -455,9 +459,9 @@ class ModuleCache(object): ...@@ -455,9 +459,9 @@ class ModuleCache(object):
items_copy = list(self.entry_from_key.iteritems()) items_copy = list(self.entry_from_key.iteritems())
for key, entry in items_copy: for key, entry in items_copy:
age = time_now - last_access_time(entry) age = time_now - last_access_time(entry)
if age > age_thresh: if age > age_thresh_del:
# TODO: we are assuming that modules that haven't been accessed in over # TODO: we are assuming that modules that haven't been accessed in over
# age_thresh are not currently in use by other processes, but that could be # age_thresh_del are not currently in use by other processes, but that could be
# false for long-running jobs... # false for long-running jobs...
assert entry not in self.module_from_name assert entry not in self.module_from_name
del self.entry_from_key[key] del self.entry_from_key[key]
......
Markdown 格式
0%
您添加了 0 到此讨论。请谨慎行事。
请先完成此评论的编辑!
注册 或者 后发表评论