diff options
| author | Marcus Müller <mmueller@gnuradio.org> | 2021-05-26 22:08:44 +0200 |
|---|---|---|
| committer | mormj <34754695+mormj@users.noreply.github.com> | 2021-06-04 19:08:50 -0400 |
| commit | 9d92ffbc091ce34bba6ea5624c7fe5fe130d5136 (patch) | |
| tree | a79df78b6301d998d861eff73841aed8b12fa332 /grc/core | |
| parent | e2f152dd4c6d6bd9b3bc5986f7c13655fd2f6a61 (diff) | |
GRC: cache now time-dependent, version-sensitive
Complete Cache will be discarded if current cache file
* has no version info
* version older than version in cacheversion.in (i.e. after an update)
Cached entry will be reloaded from yaml if
* missing
* cached-at property smaller than modification time of yaml file (half
second tolerance, due to inelegant floating point modification time
handling in python)
Also, a bit of debug logging.
Signed-off-by: Marcus Müller <mmueller@gnuradio.org>
Diffstat (limited to 'grc/core')
| -rw-r--r-- | grc/core/cache.py | 42 | ||||
| -rw-r--r-- | grc/core/platform.py | 2 |
2 files changed, 35 insertions, 9 deletions
diff --git a/grc/core/cache.py b/grc/core/cache.py index 4b41d1494d..ab2cc4465b 100644 --- a/grc/core/cache.py +++ b/grc/core/cache.py @@ -7,6 +7,7 @@ import json import logging import os +import time from .io import yaml @@ -15,9 +16,11 @@ logger = logging.getLogger(__name__) class Cache(object): - def __init__(self, filename): + def __init__(self, filename, version = None): self.cache_file = filename + self.version = version self.cache = {} + self._cachetime = None self.need_cache_write = True self._accessed_items = set() try: @@ -31,24 +34,41 @@ class Cache(object): def load(self): try: - logger.debug("Loading block cache from: {}".format(self.cache_file)) - with open(self.cache_file, encoding='utf-8') as cache_file: - self.cache = json.load(cache_file) self.need_cache_write = False + logger.debug(f"Loading block cache from: {self.cache_file}") + with open(self.cache_file, encoding='utf-8') as cache_file: + cache = json.load(cache_file) + cacheversion = cache.get("version", None) + logger.debug(f"Cache version {cacheversion}") + self._cachetime = cache.get("cached-at", 0) + if cacheversion == self.version: + logger.debug("Loaded block cache") + self.cache = cache["cache"] + else: + logger.info(f"Outdated cache {self.cache_file} found, " + "will be overwritten.") + raise ValueError() except (IOError, ValueError): self.need_cache_write = True def get_or_load(self, filename): self._accessed_items.add(filename) - if os.path.getmtime(filename) <= self._converter_mtime: + modtime = os.path.getmtime(filename) + if modtime <= self._converter_mtime: try: - return self.cache[filename] + cached = self.cache[filename] + if int(cached["cached-at"]+0.5) >= modtime: + return cached["data"] + logger.info(f"Cache for {filename} outdated, loading yaml") except KeyError: pass with open(filename, encoding='utf-8') as fp: data = yaml.safe_load(fp) - self.cache[filename] = data + self.cache[filename] = { + "cached-at": int(time.time()), + "data": data + } self.need_cache_write = True return data @@ -59,7 +79,13 @@ class Cache(object): logger.debug('Saving %d entries to json cache', len(self.cache)) # Dumping to binary file is only supported for Python3 >= 3.6 with open(self.cache_file, 'w', encoding='utf8') as cache_file: - cache_file.write(json.dumps(self.cache, ensure_ascii=False)) + cache_content = { + "version": self.version, + "cached-at": self._cachetime, + "cache": self.cache + } + cache_file.write( + json.dumps(cache_content, ensure_ascii=False)) def prune(self): for filename in (set(self.cache) - self._accessed_items): diff --git a/grc/core/platform.py b/grc/core/platform.py index 02cb385e57..42d5b660c7 100644 --- a/grc/core/platform.py +++ b/grc/core/platform.py @@ -134,7 +134,7 @@ class Platform(Element): # converter.run() # logging.info('XML converter done.') - with Cache(Constants.CACHE_FILE) as cache: + with Cache(Constants.CACHE_FILE, version = self.config.version) as cache: for file_path in self._iter_files_in_block_path(path): if file_path.endswith('.block.yml'): |