grc: added yaml/mako support

Includes basic converter from XML/Cheetah to YAML/Mako based block format.
author: Sebastian Koslowski <sebastian.koslowski@gmail.com> 2016-05-03 17:13:08 +0200
committer: Johnathan Corgan <johnathan@corganlabs.com> 2017-06-29 09:16:49 -0700
commit: 7f7fa2f91467fdb2b11312be8562e7b51fdeb199 (patch)
tree: 24268bac15b9920d2a15ddbb45eaf3b9b03718a1 /grc/converter
parent: 44cae388881821942e691a4d69a923bbd8d347db (diff)
11 files changed, 1102 insertions, 0 deletions
diff --git a/grc/converter/__init__.py b/grc/converter/__init__.py
new file mode 100644
index 0000000000..224f2e9afc
--- /dev/null
+++ b/grc/converter/__init__.py
@@ -0,0 +1,20 @@
+# Copyright 2016 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+from __future__ import absolute_import
+
+from .main import Converter
diff --git a/grc/converter/__main__.py b/grc/converter/__main__.py
new file mode 100644
index 0000000000..6efc2d7c59
--- /dev/null
+++ b/grc/converter/__main__.py
@@ -0,0 +1,21 @@
+# Copyright 2016 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+from __future__ import absolute_import
+
+# TODO: implement cli
+
diff --git a/grc/converter/block.dtd b/grc/converter/block.dtd
new file mode 100644
index 0000000000..145f4d8610
--- /dev/null
+++ b/grc/converter/block.dtd
@@ -0,0 +1,69 @@
+<!--
+Copyright 2008 Free Software Foundation, Inc.
+This file is part of GNU Radio
+
+GNU Radio Companion is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+GNU Radio Companion is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+-->
+<!--
+    gnuradio_python.blocks.dtd
+    Josh Blum
+    The document type definition for blocks.
+ -->
+<!--
+    Top level element.
+    A block contains a name, ...parameters list, and list of IO ports.
+ -->
+<!ELEMENT block (name, key, category?, throttle?, flags?, import*, var_make?, var_value?,
+        make, callback*, param_tab_order?, param*, bus_sink?, bus_source?, check*,
+        sink*, source*, bus_structure_sink?, bus_structure_source?, doc?,  grc_source?)>
+<!--
+    Sub level elements.
+ -->
+<!ELEMENT param_tab_order (tab+)>
+<!ELEMENT param (base_key?, name, key, value?, type?, hide?, option*, tab?)>
+<!ELEMENT option (name, key, opt*)>
+<!ELEMENT sink (name, type, vlen?, domain?, nports?, optional?, hide?)>
+<!ELEMENT source (name, type, vlen?, domain?, nports?, optional?, hide?)>
+<!--
+    Bottom level elements.
+    Character data only.
+ -->
+<!ELEMENT category (#PCDATA)>
+<!ELEMENT import (#PCDATA)>
+<!ELEMENT doc (#PCDATA)>
+<!ELEMENT grc_source (#PCDATA)>
+<!ELEMENT tab (#PCDATA)>
+<!ELEMENT name (#PCDATA)>
+<!ELEMENT base_key (#PCDATA)>
+<!ELEMENT key (#PCDATA)>
+<!ELEMENT check (#PCDATA)>
+<!ELEMENT bus_sink (#PCDATA)>
+<!ELEMENT bus_source (#PCDATA)>
+<!ELEMENT opt (#PCDATA)>
+<!ELEMENT type (#PCDATA)>
+<!ELEMENT hide (#PCDATA)>
+<!ELEMENT vlen (#PCDATA)>
+<!ELEMENT domain (#PCDATA)>
+<!ELEMENT nports (#PCDATA)>
+<!ELEMENT bus_structure_sink (#PCDATA)>
+<!ELEMENT bus_structure_source (#PCDATA)>
+<!ELEMENT var_make (#PCDATA)>
+<!ELEMENT var_value (#PCDATA)>
+<!ELEMENT make (#PCDATA)>
+<!ELEMENT value (#PCDATA)>
+<!ELEMENT callback (#PCDATA)>
+<!ELEMENT optional (#PCDATA)>
+<!ELEMENT throttle (#PCDATA)>
+<!ELEMENT flags (#PCDATA)>
diff --git a/grc/converter/block.py b/grc/converter/block.py
new file mode 100644
index 0000000000..04e5c905a0
--- /dev/null
+++ b/grc/converter/block.py
@@ -0,0 +1,219 @@
+# Copyright 2016 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+"""
+Converter for legacy block definitions in XML format
+
+- Cheetah expressions that can not be converted are passed to Cheetah for now
+- Instead of generating a Block subclass directly a string representation is
+  used and evaluated. This is slower / lamer but allows us to show the user
+  how a converted definition would look like
+"""
+
+from __future__ import absolute_import, division, print_function
+
+from collections import OrderedDict, defaultdict
+from itertools import chain
+
+from ..core.io import yaml
+from . import cheetah_converter, xml
+
+current_file_format = 1
+reserved_block_keys = ('import', )  # todo: add more keys
+
+
+def from_xml(filename):
+    """Load block description from xml file"""
+    element, version_info = xml.load(filename, 'block.dtd')
+
+    try:
+        data = convert_block_xml(element)
+    except NameError:
+        raise ValueError('Conversion failed', filename)
+
+    return data
+
+
+def dump(data, stream):
+    out = yaml.dump(data)
+
+    replace = [
+        ('parameters:', '\nparameters:'),
+        ('inputs:', '\ninputs:'),
+        ('outputs:', '\noutputs:'),
+        ('templates:', '\ntemplates:'),
+        ('documentation:', '\ndocumentation:'),
+        ('file_format:', '\nfile_format:'),
+    ]
+    for r in replace:
+        out = out.replace(*r)
+    prefix = '# auto-generated by grc.converter\n\n'
+    stream.write(prefix + out)
+
+
+no_value = object()
+dummy = cheetah_converter.DummyConverter()
+
+
+def convert_block_xml(node):
+    converter = cheetah_converter.Converter(names={
+        param_node.findtext('key'): {
+            opt_node.text.split(':')[0]
+            for opt_node in next(param_node.iterfind('option'), param_node).iterfind('opt')
+        } for param_node in node.iterfind('param')
+    })
+
+    block_id = node.findtext('key')
+    if block_id in reserved_block_keys:
+        block_id += '_'
+
+    data = OrderedDict()
+    data['id'] = block_id
+    data['label'] = node.findtext('name') or no_value
+    data['category'] = node.findtext('category') or no_value
+    data['flags'] = node.findtext('flags') or no_value
+
+    data['parameters'] = [convert_param_xml(param_node, converter.to_python_dec)
+                          for param_node in node.iterfind('param')] or no_value
+    # data['params'] = {p.pop('key'): p for p in data['params']}
+
+    data['inputs'] = [convert_port_xml(port_node, converter.to_python_dec)
+                      for port_node in node.iterfind('sink')] or no_value
+
+    data['outputs'] = [convert_port_xml(port_node, converter.to_python_dec)
+                       for port_node in node.iterfind('source')] or no_value
+
+    data['checks'] = [converter.to_python_dec(check_node.text)
+                      for check_node in node.iterfind('checks')] or no_value
+    data['value'] = (
+        converter.to_python_dec(node.findtext('var_value')) or
+        ('${ value }' if block_id.startswith('variable') else no_value)
+    )
+
+    data['templates'] = convert_templates(node, converter.to_mako, block_id) or no_value
+
+    docs = node.findtext('doc')
+    if docs:
+        docs = docs.strip().replace('\\\n', '')
+        data['documentation'] = yaml.MultiLineString(docs)
+
+    data['file_format'] = current_file_format
+
+    data = OrderedDict((key, value) for key, value in data.items() if value is not no_value)
+    auto_hide_params_for_item_sizes(data)
+
+    return data
+
+
+def auto_hide_params_for_item_sizes(data):
+    item_size_templates = []
+    vlen_templates = []
+    for port in chain(*[data.get(direction, []) for direction in ['inputs', 'outputs']]):
+        for key in ['dtype', 'multiplicity']:
+            item_size_templates.append(str(port.get(key, '')))
+        vlen_templates.append(str(port.get('vlen', '')))
+    item_size_templates = ' '.join(value for value in item_size_templates if '${' in value)
+    vlen_templates = ' '.join(value for value in vlen_templates if '${' in value)
+
+    for param in data.get('parameters', []):
+        if param['id'] in item_size_templates:
+            param.setdefault('hide', 'part')
+        if param['id'] in vlen_templates:
+            param.setdefault('hide', "${ 'part' if vlen == 1 else 'none' }")
+
+
+def convert_templates(node, convert, block_id=''):
+    templates = OrderedDict()
+
+    imports = '\n'.join(convert(import_node.text)
+                        for import_node in node.iterfind('import'))
+    if '\n' in imports:
+        imports = yaml.MultiLineString(imports)
+    templates['imports'] = imports or no_value
+
+    templates['var_make'] = convert(node.findtext('var_make') or '') or no_value
+
+    make = convert(node.findtext('make') or '')
+    if make:
+        check_mako_template(block_id, make)
+    if '\n' in make:
+        make = yaml.MultiLineString(make)
+    templates['make'] = make or no_value
+
+    templates['callbacks'] = [
+         convert(cb_node.text) for cb_node in node.iterfind('callback')
+    ] or no_value
+
+    return OrderedDict((key, value) for key, value in templates.items() if value is not no_value)
+
+
+def convert_param_xml(node, convert):
+    param = OrderedDict()
+    param['id'] = node.findtext('key').strip()
+    param['label'] = node.findtext('name').strip()
+    param['category'] = node.findtext('tab') or no_value
+
+    param['dtype'] = convert(node.findtext('type') or '')
+    param['default'] = node.findtext('value') or no_value
+
+    options = yaml.ListFlowing(on.findtext('key') for on in node.iterfind('option'))
+    option_labels = yaml.ListFlowing(on.findtext('name') for on in node.iterfind('option'))
+    param['options'] = options or no_value
+    if not all(str(o).title() == l for o, l in zip(options, option_labels)):
+        param['option_labels'] = option_labels
+
+    attributes = defaultdict(yaml.ListFlowing)
+    for option_n in node.iterfind('option'):
+        for opt_n in option_n.iterfind('opt'):
+            key, value = opt_n.text.split(':', 2)
+            attributes[key].append(value)
+    param['option_attributes'] = dict(attributes) or no_value
+
+    param['hide'] = convert(node.findtext('hide')) or no_value
+
+    return OrderedDict((key, value) for key, value in param.items() if value is not no_value)
+
+
+def convert_port_xml(node, convert):
+    port = OrderedDict()
+    label = node.findtext('name')
+    # default values:
+    port['label'] = label if label not in ('in', 'out') else no_value
+
+    dtype = convert(node.findtext('type'))
+    # TODO: detect dyn message ports
+    port['domain'] = domain = 'message' if dtype == 'message' else 'stream'
+    if domain == 'message':
+        port['id'], port['label'] = label, no_value
+    else:
+        port['dtype'] = dtype
+        vlen = node.findtext('vlen')
+        port['vlen'] = int(vlen) if vlen and vlen.isdigit() else convert(vlen) or no_value
+
+    port['multiplicity'] = convert(node.findtext('nports')) or no_value
+    port['optional'] = bool(node.findtext('optional')) or no_value
+    port['hide'] = convert(node.findtext('hide')) or no_value
+
+    return OrderedDict((key, value) for key, value in port.items() if value is not no_value)
+
+
+def check_mako_template(block_id, expr):
+    import sys
+    from mako.template import Template
+    try:
+        Template(expr)
+    except Exception as error:
+        print(block_id, expr, type(error), error, '', sep='\n', file=sys.stderr)
diff --git a/grc/converter/block_tree.dtd b/grc/converter/block_tree.dtd
new file mode 100644
index 0000000000..9e23576477
--- /dev/null
+++ b/grc/converter/block_tree.dtd
@@ -0,0 +1,26 @@
+<!--
+Copyright 2008 Free Software Foundation, Inc.
+This file is part of GNU Radio
+
+GNU Radio Companion is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+GNU Radio Companion is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+-->
+<!--
+    block_tree.dtd
+    Josh Blum
+    The document type definition for a block tree category listing.
+ -->
+<!ELEMENT cat (name, cat*, block*)>
+<!ELEMENT name (#PCDATA)>
+<!ELEMENT block (#PCDATA)>
diff --git a/grc/converter/block_tree.py b/grc/converter/block_tree.py
new file mode 100644
index 0000000000..dee9adba49
--- /dev/null
+++ b/grc/converter/block_tree.py
@@ -0,0 +1,56 @@
+# Copyright 2016 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+"""
+Converter for legacy block tree definitions in XML format
+"""
+
+from __future__ import absolute_import, print_function
+
+from ..core.io import yaml
+from . import xml
+
+
+def from_xml(filename):
+    """Load block tree description from xml file"""
+    element, version_info = xml.load(filename, 'block_tree.dtd')
+
+    try:
+        data = convert_category_node(element)
+    except NameError:
+        raise ValueError('Conversion failed', filename)
+
+    return data
+
+
+def dump(data, stream):
+    out = yaml.dump(data, indent=2)
+    prefix = '# auto-generated by grc.converter\n\n'
+    stream.write(prefix + out)
+
+
+def convert_category_node(node):
+    """convert nested <cat> tags to nested lists dicts"""
+    assert node.tag == 'cat'
+    name, elements = '', []
+    for child in node:
+        if child.tag == 'name':
+            name = child.text.strip()
+        elif child.tag == 'block':
+            elements.append(child.text.strip())
+        elif child.tag == 'cat':
+            elements.append(convert_category_node(child))
+    return {name: elements}
diff --git a/grc/converter/cheetah_converter.py b/grc/converter/cheetah_converter.py
new file mode 100644
index 0000000000..16fea32c99
--- /dev/null
+++ b/grc/converter/cheetah_converter.py
@@ -0,0 +1,277 @@
+# Copyright 2016 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+from __future__ import absolute_import, print_function
+
+import collections
+import re
+import string
+
+delims = {'(': ')', '[': ']', '{': '}', '': ', #\\*:'}
+identifier_start = '_' + string.ascii_letters + ''.join(delims.keys())
+string_delims = '"\''
+
+cheetah_substitution = re.compile(
+    r'^\$((?P<d1>\()|(?P<d2>\{)|(?P<d3>\[)|)'
+    r'(?P<arg>[_a-zA-Z][_a-zA-Z0-9]*(?:\.[_a-zA-Z][_a-zA-Z0-9]*)?)(?P<eval>\(\))?'
+    r'(?(d1)\)|(?(d2)\}|(?(d3)\]|)))$'
+)
+cheetah_inline_if = re.compile(r'#if (?P<cond>.*) then (?P<then>.*?) ?else (?P<else>.*?) ?(#|$)')
+
+
+class Python(object):
+    start = ''
+    end = ''
+    nested_start = ''
+    nested_end = ''
+    eval = ''
+    type = str  # yaml_output.Eval
+
+
+class FormatString(Python):
+    start = '{'
+    end = '}'
+    nested_start = '{'
+    nested_end = '}'
+    eval = ':eval'
+    type = str
+
+
+class Mako(Python):
+    start = '${'
+    end = '}'
+    nested_start = ''
+    nested_end = ''
+    type = str
+
+
+class Converter(object):
+
+    def __init__(self, names):
+        self.stats = collections.defaultdict(int)
+        self.names = set(names)
+        self.extended = set(self._iter_identifiers(names))
+
+    @staticmethod
+    def _iter_identifiers(names):
+        if not isinstance(names, dict):
+            names = {name: {} for name in names}
+        for key, sub_keys in names.items():
+            yield key
+            for sub_key in sub_keys:
+                yield '{}.{}'.format(key, sub_key)
+
+    def to_python(self, expr):
+        return self.convert(expr=expr, spec=Python)
+
+    def to_python_dec(self, expr):
+        converted = self.convert(expr=expr, spec=Python)
+        if converted and converted != expr:
+            converted = '${ ' + converted.strip() + ' }'
+        return converted
+
+    def to_format_string(self, expr):
+        return self.convert(expr=expr, spec=FormatString)
+
+    def to_mako(self, expr):
+        return self.convert(expr=expr, spec=Mako)
+
+    def convert(self, expr, spec=Python):
+        if not expr:
+            return ''
+
+        elif '$' not in expr:
+            return expr
+
+        try:
+            return self.convert_simple(expr, spec)
+        except ValueError:
+            pass
+
+        try:
+            if '#if' in expr and '\n' not in expr:
+                expr = self.convert_inline_conditional(expr, spec)
+            return self.convert_hard(expr, spec)
+        except ValueError:
+            return 'Cheetah! ' + expr
+
+    def convert_simple(self, expr, spec=Python):
+        match = cheetah_substitution.match(expr)
+        if not match:
+            raise ValueError('Not a simple substitution: ' + expr)
+
+        identifier = match.group('arg')
+        if identifier not in self.extended:
+            raise NameError('Unknown substitution {!r}'.format(identifier))
+        if match.group('eval'):
+            identifier += spec.eval
+
+        out = spec.start + identifier + spec.end
+        if '$' in out or '#' in out:
+            raise ValueError('Failed to convert: ' + expr)
+
+        self.stats['simple'] += 1
+        return spec.type(out)
+
+    def convert_hard(self, expr, spec=Python):
+        lines = '\n'.join(self.convert_hard_line(line, spec) for line in expr.split('\n'))
+        if spec == Mako:
+            # no line-continuation before a mako control structure
+            lines = re.sub(r'\\\n(\s*%)', r'\n\1', lines)
+        return lines
+
+    def convert_hard_line(self, expr, spec=Python):
+        if spec == Mako:
+            if '#set' in expr:
+                ws, set_, statement = expr.partition('#set ')
+                return ws + '<% ' + self.to_python(statement) + ' %>'
+
+            if '#if' in expr:
+                ws, if_, condition = expr.partition('#if ')
+                return ws + '% if ' + self.to_python(condition) + ':'
+            if '#else if' in expr:
+                ws, elif_, condition = expr.partition('#else if ')
+                return ws + '% elif ' + self.to_python(condition) + ':'
+            if '#else' in expr:
+                return expr.replace('#else', '% else:')
+            if '#end if' in expr:
+                return expr.replace('#end if', '% endif')
+
+            if '#slurp' in expr:
+                expr = expr.split('#slurp', 1)[0] + '\\'
+        return self.convert_hard_replace(expr, spec)
+
+    def convert_hard_replace(self, expr, spec=Python):
+        counts = collections.Counter()
+
+        def all_delims_closed():
+            for opener_, closer_ in delims.items():
+                if counts[opener_] != counts[closer_]:
+                    return False
+            return True
+
+        def extra_close():
+            for opener_, closer_ in delims.items():
+                if counts[opener_] < counts[closer_]:
+                    return True
+            return False
+
+        out = []
+        delim_to_find = False
+
+        pos = 0
+        char = ''
+        in_string = None
+        while pos < len(expr):
+            prev, char = char, expr[pos]
+            counts.update(char)
+
+            if char in string_delims:
+                if not in_string:
+                    in_string = char
+                elif char == in_string:
+                    in_string = None
+                    out.append(char)
+                    pos += 1
+                    continue
+            if in_string:
+                out.append(char)
+                pos += 1
+                continue
+
+            if char == '$':
+                pass  # no output
+
+            elif prev == '$':
+                if char not in identifier_start:  # not a substitution
+                    out.append('$' + char)  # now print the $ we skipped over
+
+                elif not delim_to_find:  # start of a substitution
+                    try:
+                        delim_to_find = delims[char]
+                        out.append(spec.start)
+                    except KeyError:
+                        if char in identifier_start:
+                            delim_to_find = delims['']
+                            out.append(spec.start)
+                            out.append(char)
+
+                    counts.clear()
+                    counts.update(char)
+
+                else:  # nested substitution: simply match known variable names
+                    found = False
+                    for known_identifier in self.names:
+                        if expr[pos:].startswith(known_identifier):
+                            found = True
+                            break
+                    if found:
+                        out.append(spec.nested_start)
+                        out.append(known_identifier)
+                        out.append(spec.nested_end)
+                        pos += len(known_identifier)
+                        continue
+
+            elif delim_to_find and char in delim_to_find and all_delims_closed():  # end of substitution
+                out.append(spec.end)
+                if char in delims['']:
+                    out.append(char)
+                delim_to_find = False
+
+            elif delim_to_find and char in ')]}' and extra_close():  # end of substitution
+                out.append(spec.end)
+                out.append(char)
+                delim_to_find = False
+
+            else:
+                out.append(char)
+
+            pos += 1
+
+        if delim_to_find == delims['']:
+            out.append(spec.end)
+
+        out = ''.join(out)
+        # fix: eval stuff
+        out = re.sub(r'(?P<arg>' + r'|'.join(self.extended) + r')\(\)', '\g<arg>', out)
+
+        self.stats['hard'] += 1
+        return spec.type(out)
+
+    def convert_inline_conditional(self, expr, spec=Python):
+        if spec == FormatString:
+            raise ValueError('No conditionals in format strings: ' + expr)
+        matcher = r'\g<then> if \g<cond> else \g<else>'
+        if spec == Python:
+            matcher = '(' + matcher + ')'
+        expr = cheetah_inline_if.sub(matcher, expr)
+        return spec.type(self.convert_hard(expr, spec))
+
+
+class DummyConverter(object):
+
+    def __init__(self, names={}):
+        pass
+
+    def to_python(self, expr):
+        return expr
+
+    def to_format_string(self, expr):
+        return expr
+
+    def to_mako(self, expr):
+        return expr
diff --git a/grc/converter/flow_graph.dtd b/grc/converter/flow_graph.dtd
new file mode 100644
index 0000000000..bdfe1dc059
--- /dev/null
+++ b/grc/converter/flow_graph.dtd
@@ -0,0 +1,38 @@
+<!--
+Copyright 2008 Free Software Foundation, Inc.
+This file is part of GNU Radio
+
+GNU Radio Companion is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+GNU Radio Companion is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+-->
+<!--
+    flow_graph.dtd
+    Josh Blum
+    The document type definition for flow graph xml files.
+ -->
+<!ELEMENT flow_graph (timestamp?, block*, connection*)> <!-- optional timestamp -->
+<!ELEMENT timestamp (#PCDATA)>
+<!-- Block -->
+<!ELEMENT block (key, param*, bus_sink?, bus_source?)>
+<!ELEMENT param (key, value)>
+<!ELEMENT key (#PCDATA)>
+<!ELEMENT value (#PCDATA)>
+<!ELEMENT bus_sink (#PCDATA)>
+<!ELEMENT bus_source (#PCDATA)>
+<!-- Connection -->
+<!ELEMENT connection (source_block_id, sink_block_id, source_key, sink_key)>
+<!ELEMENT source_block_id (#PCDATA)>
+<!ELEMENT sink_block_id (#PCDATA)>
+<!ELEMENT source_key (#PCDATA)>
+<!ELEMENT sink_key (#PCDATA)>
diff --git a/grc/converter/flow_graph.py b/grc/converter/flow_graph.py
new file mode 100644
index 0000000000..d20c67703c
--- /dev/null
+++ b/grc/converter/flow_graph.py
@@ -0,0 +1,131 @@
+# Copyright 2017 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+from __future__ import absolute_import, division
+
+import ast
+from collections import OrderedDict
+
+from ..core.io import yaml
+from . import xml
+
+
+def from_xml(filename):
+    """Load flow graph from xml file"""
+    element, version_info = xml.load(filename, 'flow_graph.dtd')
+
+    data = convert_flow_graph_xml(element)
+    try:
+        file_format = int(version_info['format'])
+    except KeyError:
+        file_format = _guess_file_format_1(data)
+
+    data['metadata'] = {'file_format': file_format}
+
+    return data
+
+
+def dump(data, stream):
+    out = yaml.dump(data, indent=2)
+
+    replace = [
+        ('blocks:', '\nblocks:'),
+        ('connections:', '\nconnections:'),
+        ('metadata:', '\nmetadata:'),
+    ]
+    for r in replace:
+        out = out.replace(*r)
+    prefix = '# auto-generated by grc.converter\n\n'
+    stream.write(prefix + out)
+
+
+def convert_flow_graph_xml(node):
+    blocks = [
+        convert_block(block_data)
+        for block_data in node.findall('block')
+    ]
+
+    options = next(b for b in blocks if b['id'] == 'options')
+    blocks.remove(options)
+    options.pop('id')
+
+    connections = [
+        convert_connection(connection)
+        for connection in node.findall('connection')
+        ]
+
+    flow_graph = OrderedDict()
+    flow_graph['options'] = options
+    flow_graph['blocks'] = blocks
+    flow_graph['connections'] = connections
+    return flow_graph
+
+
+def convert_block(data):
+    block_id = data.findtext('key')
+
+    params = OrderedDict(sorted(
+        (param.findtext('key'), param.findtext('value'))
+        for param in data.findall('param')
+    ))
+    states = OrderedDict()
+    x, y = ast.literal_eval(params.pop('_coordinate', '(10, 10)'))
+    states['coordinate'] = yaml.ListFlowing([x, y])
+    states['rotation'] = int(params.pop('_rotation', '0'))
+    enabled = params.pop('_enabled', 'True')
+    states['state'] = (
+        'enabled' if enabled in ('1', 'True') else
+        'bypassed' if enabled == '2' else
+        'disabled'
+    )
+
+    block = OrderedDict()
+    if block_id != 'options':
+        block['name'] = params.pop('id')
+    block['id'] = block_id
+    block['parameters'] = params
+    block['states'] = states
+
+    return block
+
+
+def convert_connection(data):
+    src_blk_id = data.findtext('source_block_id')
+    src_port_id = data.findtext('source_key')
+    snk_blk_id = data.findtext('sink_block_id')
+    snk_port_id = data.findtext('sink_key')
+
+    if src_port_id.isdigit():
+        src_port_id = 'out' + src_port_id
+    if snk_port_id.isdigit():
+        snk_port_id = 'in' + snk_port_id
+
+    return yaml.ListFlowing([src_blk_id, src_port_id, snk_blk_id, snk_port_id])
+
+
+def _guess_file_format_1(data):
+    """Try to guess the file format for flow-graph files without version tag"""
+
+    def has_numeric_port_ids(src_id, src_port_id, snk_id, snk_port_id):
+        return src_port_id.isdigit() and snk_port_id.is_digit()
+
+    try:
+        if any(not has_numeric_port_ids(*con) for con in data['connections']):
+            return 1
+    except:
+        pass
+    return 0
diff --git a/grc/converter/main.py b/grc/converter/main.py
new file mode 100644
index 0000000000..f979cc0281
--- /dev/null
+++ b/grc/converter/main.py
@@ -0,0 +1,163 @@
+# Copyright 2016 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by the
+# Free Software Foundation; either version 2 of the License, or (at your
+# option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+# details.
+#
+# You should have received a copy of the GNU General Public License along with
+# this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+from __future__ import absolute_import
+
+from codecs import open
+import json
+import logging
+import os
+
+import six
+
+from . import block_tree, block
+
+path = os.path
+logger = logging.getLogger(__name__)
+
+excludes = [
+    'qtgui_',
+    '.grc_gnuradio/',
+    'blks2',
+    'wxgui',
+    'epy_block.xml',
+    'virtual_sink.xml',
+    'virtual_source.xml',
+    'dummy.xml',
+    'variable_struct.xml',  # todo: re-implement as class
+    'digital_constellation',  # todo: fix template
+]
+
+
+class Converter(object):
+
+    def __init__(self, search_path, output_dir='~/.cache/grc_gnuradio'):
+        self.search_path = search_path
+        self.output_dir = output_dir
+
+        self._force = False
+
+        converter_module_path = path.dirname(__file__)
+        self._converter_mtime = max(path.getmtime(path.join(converter_module_path, module))
+                                    for module in os.listdir(converter_module_path)
+                                    if not module.endswith('flow_graph.py'))
+
+        self.cache_file = os.path.join(self.output_dir, '_cache.json')
+        self.cache = {}
+
+    def run(self, force=False):
+        self._force = force
+
+        try:
+            with open(self.cache_file, encoding='utf-8') as cache_file:
+                self.cache = byteify(json.load(cache_file))
+        except (IOError, ValueError):
+            self.cache = {}
+            self._force = True
+        need_cache_write = False
+
+        if not path.isdir(self.output_dir):
+            os.makedirs(self.output_dir)
+        if self._force:
+            for name in os.listdir(self.output_dir):
+                os.remove(os.path.join(self.output_dir, name))
+
+        for xml_file in self.iter_files_in_block_path():
+            if xml_file.endswith("block_tree.xml"):
+                changed = self.load_category_tree_xml(xml_file)
+            elif xml_file.endswith('domain.xml'):
+                continue
+            else:
+                changed = self.load_block_xml(xml_file)
+
+            if changed:
+                need_cache_write = True
+
+        if need_cache_write:
+            logger.info('Saving %d entries to json cache', len(self.cache))
+            with open(self.cache_file, 'w', encoding='utf-8') as cache_file:
+                json.dump(self.cache, cache_file)
+
+    def load_block_xml(self, xml_file):
+        """Load block description from xml file"""
+        if any(part in xml_file for part in excludes):
+            return
+
+        block_id_from_xml = path.basename(xml_file)[:-4]
+        yml_file = path.join(self.output_dir, block_id_from_xml + '.block.yml')
+
+        if not self.needs_conversion(xml_file, yml_file):
+            return  # yml file up-to-date
+
+        logger.info('Converting block %s', path.basename(xml_file))
+        data = block.from_xml(xml_file)
+        if block_id_from_xml != data['id']:
+            logger.warning('block_id and filename differ')
+        self.cache[yml_file] = data
+
+        with open(yml_file, 'w', encoding='utf-8') as yml_file:
+            block.dump(data, yml_file)
+        return True
+
+    def load_category_tree_xml(self, xml_file):
+        """Validate and parse category tree file and add it to list"""
+        module_name = path.basename(xml_file)[:-len('block_tree.xml')].rstrip('._-')
+        yml_file = path.join(self.output_dir, module_name + '.tree.yml')
+
+        if not self.needs_conversion(xml_file, yml_file):
+            return  # yml file up-to-date
+
+        logger.info('Converting module %s', path.basename(xml_file))
+        data = block_tree.from_xml(xml_file)
+        self.cache[yml_file] = data
+
+        with open(yml_file, 'w', encoding='utf-8') as yml_file:
+            block_tree.dump(data, yml_file)
+        return True
+
+    def needs_conversion(self, source, destination):
+        """Check if source has already been converted and destination is up-to-date"""
+        if self._force or not path.exists(destination):
+            return True
+        xml_time = path.getmtime(source)
+        yml_time = path.getmtime(destination)
+
+        return yml_time < xml_time or yml_time < self._converter_mtime
+
+    def iter_files_in_block_path(self, suffix='.xml'):
+        """Iterator for block descriptions and category trees"""
+        for block_path in self.search_path:
+            if path.isfile(block_path):
+                yield block_path
+            elif path.isdir(block_path):
+                for root, _, files in os.walk(block_path, followlinks=True):
+                    for name in files:
+                        if name.endswith(suffix):
+                            yield path.join(root, name)
+            else:
+                logger.warning('Invalid entry in search path: {}'.format(block_path))
+
+
+def byteify(data):
+    if isinstance(data, dict):
+        return {byteify(key): byteify(value) for key, value in six.iteritems(data)}
+    elif isinstance(data, list):
+        return [byteify(element) for element in data]
+    elif isinstance(data, unicode):
+        return data.encode('utf-8')
+    else:
+        return data
diff --git a/grc/converter/xml.py b/grc/converter/xml.py
new file mode 100644
index 0000000000..2eda786c0f
--- /dev/null
+++ b/grc/converter/xml.py
@@ -0,0 +1,82 @@
+# Copyright 2017 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+from __future__ import absolute_import, division
+
+import re
+from os import path
+
+try:
+    # raise ImportError()
+    from lxml import etree
+    HAVE_LXML = True
+except ImportError:
+    import xml.etree.ElementTree as etree
+    HAVE_LXML = False
+
+
+_validator_cache = {None: lambda xml: True}
+
+
+def _get_validator(dtd=None):
+    validator = _validator_cache.get(dtd)
+    if not validator:
+        if not path.isabs(dtd):
+            dtd = path.join(path.dirname(__file__), dtd)
+        validator = _validator_cache[dtd] = etree.DTD(dtd).validate
+    return validator
+
+
+def load_lxml(filename, document_type_def=None):
+    """Load block description from xml file"""
+
+    try:
+        xml_tree = etree.parse(filename)
+        _get_validator(document_type_def)
+        element = xml_tree.getroot()
+    except etree.LxmlError:
+        raise ValueError("Failed to parse or validate {}".format(filename))
+
+    version_info = {}
+    for inst in xml_tree.xpath('/processing-instruction()'):
+        if inst.target == 'grc':
+            version_info.update(inst.attrib)
+
+    return element, version_info
+
+
+def load_stdlib(filename, document_type_def=None):
+    """Load block description from xml file"""
+
+    with open(filename, 'rb') as xml_file:
+        data = xml_file.read().decode('utf-8')
+
+    try:
+        element = etree.fromstring(data)
+    except etree.ParseError:
+        raise ValueError("Failed to parse {}".format(filename))
+
+    version_info = {}
+    for body in re.findall(r'<\?(.*?)\?>', data):
+        element = etree.fromstring('<' + body + '/>')
+        if element.tag == 'grc':
+            version_info.update(element.attrib)
+
+    return element, version_info
+
+
+load = load_lxml if HAVE_LXML else load_stdlib
author	Sebastian Koslowski <sebastian.koslowski@gmail.com>	2016-05-03 17:13:08 +0200
committer	Johnathan Corgan <johnathan@corganlabs.com>	2017-06-29 09:16:49 -0700
commit	7f7fa2f91467fdb2b11312be8562e7b51fdeb199 (patch)
tree	24268bac15b9920d2a15ddbb45eaf3b9b03718a1 /grc/converter
parent	44cae388881821942e691a4d69a923bbd8d347db (diff)