grc: disable auto-conversion and implement json cache

author: Sebastian Koslowski <sebastian.koslowski@gmail.com> 2017-10-26 20:15:22 +0200
committer: Sebastian Koslowski <sebastian.koslowski@gmail.com> 2017-11-08 19:30:41 +0100
commit: 1fa89b3704d7f476e4395eb9358d5a6d7642251b (patch)
tree: bb553275eff3b791a5ac4482ed0e8355d5fce0b1
parent: 865e2586b4f34fce101d8aa4a240431273009b8c (diff)
10 files changed, 178 insertions, 73 deletions
diff --git a/grc/core/Config.py b/grc/core/Config.py
index eb53e1751d..4accb74c63 100644
--- a/grc/core/Config.py
+++ b/grc/core/Config.py
@@ -31,8 +31,6 @@ class Config(object):
 
     hier_block_lib_dir = os.environ.get('GRC_HIER_PATH', Constants.DEFAULT_HIER_BLOCK_LIB_DIR)
 
-    yml_block_cache = os.path.expanduser('~/.cache/grc_gnuradio')  # FIXME: remove this as soon as converter is stable
-
     def __init__(self, version, version_parts=None, name=None, prefs=None):
         self._gr_prefs = prefs if prefs else DummyPrefs()
         self.version = version
@@ -40,9 +38,6 @@ class Config(object):
         if name:
             self.name = name
 
-        if not os.path.exists(self.yml_block_cache):
-            os.mkdir(self.yml_block_cache)
-
     @property
     def block_paths(self):
         path_list_sep = {'/': ':', '\\': ';'}[os.path.sep]
@@ -50,7 +45,6 @@ class Config(object):
         paths_sources = (
             self.hier_block_lib_dir,
             os.environ.get('GRC_BLOCKS_PATH', ''),
-            self.yml_block_cache,
             self._gr_prefs.get_string('grc', 'local_blocks_path', ''),
             self._gr_prefs.get_string('grc', 'global_blocks_path', ''),
         )
diff --git a/grc/core/Constants.py b/grc/core/Constants.py
index fc5383378c..59c5dff35e 100644
--- a/grc/core/Constants.py
+++ b/grc/core/Constants.py
@@ -31,6 +31,8 @@ BLOCK_DTD = os.path.join(DATA_DIR, 'block.dtd')
 DEFAULT_FLOW_GRAPH = os.path.join(DATA_DIR, 'default_flow_graph.grc')
 DEFAULT_HIER_BLOCK_LIB_DIR = os.path.expanduser('~/.grc_gnuradio')
 
+CACHE_FILE = os.path.expanduser('~/.cache/grc_gnuradio/cache.json')
+
 BLOCK_DESCRIPTION_FILE_FORMAT_VERSION = 1
 # File format versions:
 #  0: undefined / legacy
diff --git a/grc/core/blocks/__init__.py b/grc/core/blocks/__init__.py
index e4a085d477..4ca0d5d2bc 100644
--- a/grc/core/blocks/__init__.py
+++ b/grc/core/blocks/__init__.py
@@ -29,6 +29,7 @@ build_ins = {}
 
 
 def register_build_in(cls):
+    cls.loaded_from = '(build-in)'
     build_ins[cls.key] = cls
     return cls
 
diff --git a/grc/core/blocks/_build.py b/grc/core/blocks/_build.py
index 9221433387..ce3496d9c4 100644
--- a/grc/core/blocks/_build.py
+++ b/grc/core/blocks/_build.py
@@ -29,7 +29,7 @@ def build(id, label='', category='', flags='', documentation='',
           parameters=None, inputs=None, outputs=None, templates=None, **kwargs):
     block_id = id
 
-    cls = type(block_id, (Block,), {})
+    cls = type(str(block_id), (Block,), {})
     cls.key = block_id
 
     cls.label = label or block_id.title()
@@ -63,7 +63,9 @@ def build(id, label='', category='', flags='', documentation='',
 
 
 def _single_mako_expr(value, block_id):
-    match = re.match(r'\s*\$\{\s*(.*?)\s*\}\s*', str(value))
-    if value and not match:
+    if not value:
+        return None
+    value = value.strip()
+    if not (value.startswith('${') and value.endswith('}')):
         raise ValueError('{} is not a mako substitution in {}'.format(value, block_id))
-    return match.group(1) if match else None
+    return value[2:-1].strip()
diff --git a/grc/core/cache.py b/grc/core/cache.py
new file mode 100644
index 0000000000..b72255ce1f
--- /dev/null
+++ b/grc/core/cache.py
@@ -0,0 +1,99 @@
+# Copyright 2017 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+from __future__ import absolute_import, print_function
+
+from io import open
+import json
+import logging
+import os
+
+import six
+
+from .io import yaml
+
+logger = logging.getLogger(__name__)
+
+
+class Cache(object):
+
+    def __init__(self, filename):
+        self.cache_file = filename
+        self.cache = {}
+        self.need_cache_write = True
+        self._accessed_items = set()
+        try:
+            os.makedirs(os.path.dirname(filename))
+        except OSError:
+            pass
+        try:
+            self._converter_mtime = os.path.getmtime(filename)
+        except OSError:
+            self._converter_mtime = -1
+
+    def load(self):
+        try:
+            logger.debug("Loading block cache from: {}".format(self.cache_file))
+            with open(self.cache_file, encoding='utf-8') as cache_file:
+                self.cache = json.load(cache_file)
+            self.need_cache_write = False
+        except (IOError, ValueError):
+            self.need_cache_write = True
+
+    def get_or_load(self, filename):
+        self._accessed_items.add(filename)
+        if os.path.getmtime(filename) <= self._converter_mtime:
+            try:
+                return self.cache[filename]
+            except KeyError:
+                pass
+
+        with open(filename, encoding='utf-8') as fp:
+            data = yaml.safe_load(fp)
+        self.cache[filename] = data
+        self.need_cache_write = True
+        return data
+
+    def save(self):
+        if not self.need_cache_write:
+            return
+
+        logger.info('Saving %d entries to json cache', len(self.cache))
+        with open(self.cache_file, 'wb') as cache_file:
+            json.dump(self.cache, cache_file, encoding='utf-8')
+
+    def prune(self):
+        for filename in (set(self.cache) - self._accessed_items):
+            del self.cache[filename]
+
+    def __enter__(self):
+        self.load()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.save()
+
+
+def byteify(data):
+    if isinstance(data, dict):
+        return {byteify(key): byteify(value) for key, value in six.iteritems(data)}
+    elif isinstance(data, list):
+        return [byteify(element) for element in data]
+    elif isinstance(data, six.text_type) and six.PY2:
+        return data.encode('utf-8')
+    else:
+        return data
diff --git a/grc/core/platform.py b/grc/core/platform.py
index 538bacade2..54deef3455 100644
--- a/grc/core/platform.py
+++ b/grc/core/platform.py
@@ -19,7 +19,6 @@ from __future__ import absolute_import, print_function
 
 from codecs import open
 from collections import namedtuple
-import glob
 import os
 import logging
 from itertools import chain
@@ -33,6 +32,7 @@ from . import (
 )
 
 from .Config import Config
+from .cache import Cache
 from .base import Element
 from .io import yaml
 from .generator import Generator
@@ -141,44 +141,41 @@ class Platform(Element):
         self.connection_templates.clear()
         self._block_categories.clear()
 
-        # FIXME: remove this as soon as converter is stable
-        from ..converter import Converter
-        converter = Converter(self.config.block_paths, self.config.yml_block_cache)
-        converter.run()
-        logging.info('XML converter done.')
-
-        for file_path in self._iter_files_in_block_path(path):
-            try:
-                data = converter.cache[file_path]
-            except KeyError:
-                with open(file_path, encoding='utf-8') as fp:
-                    data = yaml.safe_load(fp)
-
-            if file_path.endswith('.block.yml'):
-                loader = self.load_block_description
-                scheme = schema_checker.BLOCK_SCHEME
-            elif file_path.endswith('.domain.yml'):
-                loader = self.load_domain_description
-                scheme = schema_checker.DOMAIN_SCHEME
-            elif file_path.endswith('.tree.yml'):
-                loader = self.load_category_tree_description
-                scheme = None
-            else:
-                continue
-
-            try:
-                checker = schema_checker.Validator(scheme)
-                passed = checker.run(data)
-                for msg in checker.messages:
-                    logger.warning('{:<40s} {}'.format(os.path.basename(file_path), msg))
-                if not passed:
-                    logger.info('YAML schema check failed for: ' + file_path)
-
-                loader(data, file_path)
-            except Exception as error:
-                logger.exception('Error while loading %s', file_path)
-                logger.exception(error)
-                raise
+        # # FIXME: remove this as soon as converter is stable
+        # from ..converter import Converter
+        # converter = Converter(self.config.block_paths, self.config.yml_block_cache)
+        # converter.run()
+        # logging.info('XML converter done.')
+
+        with Cache(Constants.CACHE_FILE) as cache:
+            for file_path in self._iter_files_in_block_path(path):
+                data = cache.get_or_load(file_path)
+
+                if file_path.endswith('.block.yml'):
+                    loader = self.load_block_description
+                    scheme = schema_checker.BLOCK_SCHEME
+                elif file_path.endswith('.domain.yml'):
+                    loader = self.load_domain_description
+                    scheme = schema_checker.DOMAIN_SCHEME
+                elif file_path.endswith('.tree.yml'):
+                    loader = self.load_category_tree_description
+                    scheme = None
+                else:
+                    continue
+
+                try:
+                    checker = schema_checker.Validator(scheme)
+                    passed = checker.run(data)
+                    for msg in checker.messages:
+                        logger.warning('{:<40s} {}'.format(os.path.basename(file_path), msg))
+                    if not passed:
+                        logger.info('YAML schema check failed for: ' + file_path)
+
+                    loader(data, file_path)
+                except Exception as error:
+                    logger.exception('Error while loading %s', file_path)
+                    logger.exception(error)
+                    raise
 
         for key, block in six.iteritems(self.blocks):
             category = self._block_categories.get(key, block.category)
@@ -201,10 +198,9 @@ class Platform(Element):
             if os.path.isfile(entry):
                 yield entry
             elif os.path.isdir(entry):
-                pattern = os.path.join(entry, '**.' + ext)
-                yield_from = glob.iglob(pattern)
-                for file_path in yield_from:
-                    yield file_path
+                for dirpath, dirnames, filenames in os.walk(entry):
+                    for filename in sorted(filter(lambda f: f.endswith('.' + ext), filenames)):
+                        yield os.path.join(dirpath, filename)
             else:
                 logger.debug('Ignoring invalid path entry %r', entry)
 
@@ -232,16 +228,18 @@ class Platform(Element):
             log.error('Unknown format version %d in %s', file_format, file_path)
             return
 
-        block_id = data.pop('id').rstrip('_')
+        block_id = data['id'] = data['id'].rstrip('_')
 
         if block_id in self.block_classes_build_in:
             log.warning('Not overwriting build-in block %s with %s', block_id, file_path)
             return
         if block_id in self.blocks:
-            log.warning('Block with id "%s" overwritten by %s', block_id, file_path)
+            log.warning('Block with id "%s" loaded from\n  %s\noverwritten by\n  %s',
+                        block_id, self.blocks[block_id].loaded_from, file_path)
 
         try:
-            block_cls = self.blocks[block_id] = self.new_block_class(block_id, **data)
+            block_cls = self.blocks[block_id] = self.new_block_class(**data)
+            block_cls.loaded_from = file_path
         except errors.BlockLoadError as error:
             log.error('Unable to load block %s', block_id)
             log.exception(error)
@@ -288,19 +286,12 @@ class Platform(Element):
         path = []
 
         def load_category(name, elements):
-            if not isinstance(name, str):
-                log.debug('invalid name %r', name)
-                return
-            if isinstance(elements, list):
-                pass
-            elif isinstance(elements, str):
-                elements = [elements]
-            else:
-                log.debug('Ignoring elements of %s', name)
+            if not isinstance(name, six.string_types):
+                log.debug('Invalid name %r', name)
                 return
             path.append(name)
-            for element in elements:
-                if isinstance(element, str):
+            for element in utils.to_list(elements):
+                if isinstance(element, six.string_types):
                     block_id = element
                     self._block_categories[block_id] = list(path)
                 elif isinstance(element, dict):
@@ -415,8 +406,8 @@ class Platform(Element):
             fg.import_data(data)
         return fg
 
-    def new_block_class(self, block_id, **data):
-        return blocks.build(block_id, **data)
+    def new_block_class(self, **data):
+        return blocks.build(**data)
 
     def make_block(self, parent, block_id, **kwargs):
         cls = self.block_classes[block_id]
diff --git a/grc/core/schema_checker/block.py b/grc/core/schema_checker/block.py
index ea079b4276..d511e36887 100644
--- a/grc/core/schema_checker/block.py
+++ b/grc/core/schema_checker/block.py
@@ -37,7 +37,7 @@ TEMPLATES_SCHEME = expand(
 BLOCK_SCHEME = expand(
     id=Spec(types=str_, required=True, item_scheme=None),
     label=str_,
-    category=(list, str_),
+    category=str_,
     flags=(list, str_),
 
     parameters=Spec(types=list, required=False, item_scheme=PARAM_SCHEME),
diff --git a/grc/core/utils/__init__.py b/grc/core/utils/__init__.py
index 660eb594a5..f2ac986fb4 100644
--- a/grc/core/utils/__init__.py
+++ b/grc/core/utils/__init__.py
@@ -17,5 +17,17 @@
 
 from __future__ import absolute_import
 
+import six
+
 from . import epy_block_io, expr_utils, extract_docs, flow_graph_complexity
 from .hide_bokeh_gui_options_if_not_installed import hide_bokeh_gui_options_if_not_installed
+
+
+def to_list(value):
+    if not value:
+        return []
+    elif isinstance(value, six.string_types):
+        return [value]
+    else:
+        return list(value)
+
diff --git a/grc/core/utils/descriptors/evaluated.py b/grc/core/utils/descriptors/evaluated.py
index 313cee5b96..e8bce6e6ed 100644
--- a/grc/core/utils/descriptors/evaluated.py
+++ b/grc/core/utils/descriptors/evaluated.py
@@ -15,6 +15,10 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 
+from __future__ import absolute_import
+
+import six
+
 
 class Evaluated(object):
     def __init__(self, expected_type, default, name=None):
@@ -62,7 +66,7 @@ class Evaluated(object):
     def __set__(self, instance, value):
         attribs = instance.__dict__
         value = value or self.default
-        if isinstance(value, str) and value.startswith('${') and value.endswith('}'):
+        if isinstance(value, six.text_type) and value.startswith('${') and value.endswith('}'):
             attribs[self.name_raw] = value[2:-1].strip()
         else:
             attribs[self.name] = type(self.default)(value)
diff --git a/grc/gui/Platform.py b/grc/gui/Platform.py
index 2a38bc619e..8eb79f3459 100644
--- a/grc/gui/Platform.py
+++ b/grc/gui/Platform.py
@@ -63,8 +63,8 @@ class Platform(CorePlatform):
     FlowGraph = canvas.FlowGraph
     Connection = canvas.Connection
 
-    def new_block_class(self, block_id, **data):
-        cls = CorePlatform.new_block_class(self, block_id, **data)
+    def new_block_class(self, **data):
+        cls = CorePlatform.new_block_class(self, **data)
         return canvas.Block.make_cls_with_base(cls)
 
     block_classes_build_in = {key: canvas.Block.make_cls_with_base(cls)
author	Sebastian Koslowski <sebastian.koslowski@gmail.com>	2017-10-26 20:15:22 +0200
committer	Sebastian Koslowski <sebastian.koslowski@gmail.com>	2017-11-08 19:30:41 +0100
commit	1fa89b3704d7f476e4395eb9358d5a6d7642251b (patch)
tree	bb553275eff3b791a5ac4482ed0e8355d5fce0b1
parent	865e2586b4f34fce101d8aa4a240431273009b8c (diff)