From 7f7fa2f91467fdb2b11312be8562e7b51fdeb199 Mon Sep 17 00:00:00 2001
From: Sebastian Koslowski <sebastian.koslowski@gmail.com>
Date: Tue, 3 May 2016 17:13:08 +0200
Subject: grc: added yaml/mako support
Includes basic converter from XML/Cheetah to YAML/Mako based block format.
---
grc/core/FlowGraph.py | 289 +++++++++++++++++++-------------------------------
1 file changed, 112 insertions(+), 177 deletions(-)
(limited to 'grc/core/FlowGraph.py')
diff --git a/grc/core/FlowGraph.py b/grc/core/FlowGraph.py
index bf5bf6d93e..3ad95eb207 100644
--- a/grc/core/FlowGraph.py
+++ b/grc/core/FlowGraph.py
@@ -17,24 +17,17 @@
from __future__ import absolute_import, print_function
-import imp
-import time
-import re
-from operator import methodcaller
import collections
+import imp
+import itertools
import sys
+from operator import methodcaller, attrgetter
-from . import Messages
+from . import Messages, blocks
from .Constants import FLOW_GRAPH_FILE_FORMAT_VERSION
-from .Element import Element
-from .utils import expr_utils, shlex
-
-_parameter_matcher = re.compile('^(parameter)$')
-_monitors_searcher = re.compile('(ctrlport_monitor)')
-_bussink_searcher = re.compile('^(bus_sink)$')
-_bussrc_searcher = re.compile('^(bus_source)$')
-_bus_struct_sink_searcher = re.compile('^(bus_structure_sink)$')
-_bus_struct_src_searcher = re.compile('^(bus_structure_source)$')
+from .base import Element
+from .utils import expr_utils
+from .utils.backports import shlex
class FlowGraph(Element):
@@ -52,11 +45,10 @@ class FlowGraph(Element):
the flow graph object
"""
Element.__init__(self, parent)
- self._timestamp = time.ctime()
- self._options_block = self.parent_platform.get_new_block(self, 'options')
+ self._options_block = self.parent_platform.make_block(self, 'options')
self.blocks = [self._options_block]
- self.connections = []
+ self.connections = set()
self._eval_cache = {}
self.namespace = {}
@@ -66,15 +58,14 @@ class FlowGraph(Element):
def __str__(self):
return 'FlowGraph - {}({})'.format(self.get_option('title'), self.get_option('id'))
- def get_imports(self):
+ def imports(self):
"""
Get a set of all import statements in this flow graph namespace.
Returns:
- a set of import statements
+ a list of import statements
"""
- imports = sum([block.get_imports() for block in self.iter_enabled_blocks()], [])
- return sorted(set(imports))
+ return [block.templates.render('imports') for block in self.iter_enabled_blocks()]
def get_variables(self):
"""
@@ -85,7 +76,7 @@ class FlowGraph(Element):
a sorted list of variable blocks in order of dependency (indep -> dep)
"""
variables = [block for block in self.iter_enabled_blocks() if block.is_variable]
- return expr_utils.sort_objects(variables, methodcaller('get_id'), methodcaller('get_var_make'))
+ return expr_utils.sort_objects(variables, attrgetter('name'), methodcaller('get_var_make'))
def get_parameters(self):
"""
@@ -94,47 +85,21 @@ class FlowGraph(Element):
Returns:
a list of parameterized variables
"""
- parameters = [b for b in self.iter_enabled_blocks() if _parameter_matcher.match(b.key)]
+ parameters = [b for b in self.iter_enabled_blocks() if b.key == 'parameter']
return parameters
def get_monitors(self):
"""
Get a list of all ControlPort monitors
"""
- monitors = [b for b in self.iter_enabled_blocks() if _monitors_searcher.search(b.key)]
+ monitors = [b for b in self.iter_enabled_blocks() if 'ctrlport_monitor' in b.key]
return monitors
def get_python_modules(self):
"""Iterate over custom code block ID and Source"""
for block in self.iter_enabled_blocks():
if block.key == 'epy_module':
- yield block.get_id(), block.get_param('source_code').get_value()
-
- def get_bussink(self):
- bussink = [b for b in self.get_enabled_blocks() if _bussink_searcher.search(b.key)]
-
- for i in bussink:
- for j in i.params.values():
- if j.name == 'On/Off' and j.get_value() == 'on':
- return True
- return False
-
- def get_bussrc(self):
- bussrc = [b for b in self.get_enabled_blocks() if _bussrc_searcher.search(b.key)]
-
- for i in bussrc:
- for j in i.params.values():
- if j.name == 'On/Off' and j.get_value() == 'on':
- return True
- return False
-
- def get_bus_structure_sink(self):
- bussink = [b for b in self.get_enabled_blocks() if _bus_struct_sink_searcher.search(b.key)]
- return bussink
-
- def get_bus_structure_src(self):
- bussrc = [b for b in self.get_enabled_blocks() if _bus_struct_src_searcher.search(b.key)]
- return bussrc
+ yield block.name, block.params[1].get_value()
def iter_enabled_blocks(self):
"""
@@ -180,7 +145,7 @@ class FlowGraph(Element):
Returns:
the value held by that param
"""
- return self._options_block.get_param(key).get_evaluated()
+ return self._options_block.params[key].get_evaluated()
def get_run_command(self, file_path, split=False):
run_command = self.get_option('run_command')
@@ -195,16 +160,19 @@ class FlowGraph(Element):
##############################################
# Access Elements
##############################################
- def get_block(self, id):
+ def get_block(self, name):
for block in self.blocks:
- if block.get_id() == id:
+ if block.name == name:
return block
- raise KeyError('No block with ID {!r}'.format(id))
+ raise KeyError('No block with name {!r}'.format(name))
def get_elements(self):
- return self.blocks + self.connections
+ elements = list(self.blocks)
+ elements.extend(self.connections)
+ return elements
- get_children = get_elements
+ def children(self):
+ return itertools.chain(self.blocks, self.connections)
def rewrite(self):
"""
@@ -216,7 +184,7 @@ class FlowGraph(Element):
def renew_namespace(self):
namespace = {}
# Load imports
- for expr in self.get_imports():
+ for expr in self.imports():
try:
exec(expr, namespace)
except:
@@ -232,19 +200,19 @@ class FlowGraph(Element):
# Load parameters
np = {} # params don't know each other
- for parameter in self.get_parameters():
+ for parameter_block in self.get_parameters():
try:
- value = eval(parameter.get_param('value').to_code(), namespace)
- np[parameter.get_id()] = value
+ value = eval(parameter_block.params['value'].to_code(), namespace)
+ np[parameter_block.name] = value
except:
pass
namespace.update(np) # Merge param namespace
# Load variables
- for variable in self.get_variables():
+ for variable_block in self.get_variables():
try:
- value = eval(variable.get_var_value(), namespace)
- namespace[variable.get_id()] = value
+ value = eval(variable_block.value, namespace, variable_block.namespace)
+ namespace[variable_block.name] = value
except:
pass
@@ -252,41 +220,37 @@ class FlowGraph(Element):
self._eval_cache.clear()
self.namespace.update(namespace)
- def evaluate(self, expr):
+ def evaluate(self, expr, namespace=None, local_namespace=None):
"""
Evaluate the expression.
-
- Args:
- expr: the string expression
- @throw Exception bad expression
-
- Returns:
- the evaluated data
"""
# Evaluate
if not expr:
raise Exception('Cannot evaluate empty statement.')
- return self._eval_cache.setdefault(expr, eval(expr, self.namespace))
+ if namespace is not None:
+ return eval(expr, namespace, local_namespace)
+ else:
+ return self._eval_cache.setdefault(expr, eval(expr, self.namespace))
##############################################
# Add/remove stuff
##############################################
- def new_block(self, key, **kwargs):
+ def new_block(self, block_id, **kwargs):
"""
Get a new block of the specified key.
Add the block to the list of elements.
Args:
- key: the block key
+ block_id: the block key
Returns:
the new block or None if not found
"""
- if key == 'options':
+ if block_id == 'options':
return self._options_block
try:
- block = self.parent_platform.get_new_block(self, key, **kwargs)
+ block = self.parent_platform.make_block(self, block_id, **kwargs)
self.blocks.append(block)
except KeyError:
block = None
@@ -304,12 +268,17 @@ class FlowGraph(Element):
Returns:
the new connection
"""
-
connection = self.parent_platform.Connection(
- parent=self, porta=porta, portb=portb)
- self.connections.append(connection)
+ parent=self, source=porta, sink=portb)
+ self.connections.add(connection)
return connection
+ def disconnect(self, *ports):
+ to_be_removed = [con for con in self.connections
+ if any(port in con for port in ports)]
+ for con in to_be_removed:
+ self.remove_element(con)
+
def remove_element(self, element):
"""
Remove the element from the list of elements.
@@ -321,21 +290,14 @@ class FlowGraph(Element):
return
if element.is_port:
- # Found a port, set to parent signal block
- element = element.parent
+ element = element.parent_block # remove parent block
if element in self.blocks:
# Remove block, remove all involved connections
- for port in element.get_ports():
- for connection in port.get_connections():
- self.remove_element(connection)
+ self.disconnect(*element.ports())
self.blocks.remove(element)
elif element in self.connections:
- if element.is_bus():
- for port in element.source_port.get_associated_ports():
- for connection in port.get_connections():
- self.remove_element(connection)
self.connections.remove(element)
##############################################
@@ -349,70 +311,61 @@ class FlowGraph(Element):
Returns:
a nested data odict
"""
- # sort blocks and connections for nicer diffs
- blocks = sorted(self.blocks, key=lambda b: (
- b.key != 'options', # options to the front
- not b.key.startswith('variable'), # then vars
- str(b)
- ))
- connections = sorted(self.connections, key=str)
- n = collections.OrderedDict()
- n['timestamp'] = self._timestamp
- n['block'] = [b.export_data() for b in blocks]
- n['connection'] = [c.export_data() for c in connections]
- instructions = collections.OrderedDict()
- instructions['created'] = '.'.join(self.parent.config.version_parts)
- instructions['format'] = FLOW_GRAPH_FILE_FORMAT_VERSION
- return {'flow_graph': n, '_instructions': instructions}
-
- def import_data(self, n):
+ def block_order(b):
+ return not b.key.startswith('variable'), b.name # todo: vars still first ?!?
+
+ data = collections.OrderedDict()
+ data['options'] = self._options_block.export_data()
+ data['blocks'] = [b.export_data() for b in sorted(self.blocks, key=block_order)
+ if b is not self._options_block]
+ data['connections'] = sorted(c.export_data() for c in self.connections)
+ data['metadata'] = {'file_format': FLOW_GRAPH_FILE_FORMAT_VERSION}
+ return data
+
+ def _build_depending_hier_block(self, block_id):
+ # we're before the initial fg update(), so no evaluated values!
+ # --> use raw value instead
+ path_param = self._options_block.params['hier_block_src_path']
+ file_path = self.parent_platform.find_file_in_paths(
+ filename=block_id + '.grc',
+ paths=path_param.get_value(),
+ cwd=self.grc_file_path
+ )
+ if file_path: # grc file found. load and get block
+ self.parent_platform.load_and_generate_flow_graph(file_path, hier_only=True)
+ return self.new_block(block_id) # can be None
+
+ def import_data(self, data):
"""
Import blocks and connections into this flow graph.
Clear this flow graph of all previous blocks and connections.
Any blocks or connections in error will be ignored.
Args:
- n: the nested data odict
+ data: the nested data odict
"""
# Remove previous elements
del self.blocks[:]
- del self.connections[:]
- # set file format
- try:
- instructions = n.get('_instructions', {})
- file_format = int(instructions.get('format', '0')) or _guess_file_format_1(n)
- except:
- file_format = 0
+ self.connections.clear()
- fg_n = n and n.get('flow_graph', {}) # use blank data if none provided
- self._timestamp = fg_n.get('timestamp', time.ctime())
+ file_format = data['metadata']['file_format']
# build the blocks
+ self._options_block.import_data(name='', **data.get('options', {}))
self.blocks.append(self._options_block)
- for block_n in fg_n.get('block', []):
- key = block_n['key']
- block = self.new_block(key)
-
- if not block:
- # we're before the initial fg update(), so no evaluated values!
- # --> use raw value instead
- path_param = self._options_block.get_param('hier_block_src_path')
- file_path = self.parent_platform.find_file_in_paths(
- filename=key + '.grc',
- paths=path_param.get_value(),
- cwd=self.grc_file_path
- )
- if file_path: # grc file found. load and get block
- self.parent_platform.load_and_generate_flow_graph(file_path, hier_only=True)
- block = self.new_block(key) # can be None
-
- if not block: # looks like this block key cannot be found
- # create a dummy block instead
- block = self.new_block('_dummy', missing_key=key,
- params_n=block_n.get('param', []))
- print('Block key "%s" not found' % key)
-
- block.import_data(block_n)
+
+ for block_data in data.get('blocks', []):
+ block_id = block_data['id']
+ block = (
+ self.new_block(block_id) or
+ self._build_depending_hier_block(block_id) or
+ self.new_block(block_id='_dummy', missing_block_id=block_id, **block_data)
+ )
+
+ if isinstance(block, blocks.DummyBlock):
+ print('Block id "%s" not found' % block_id)
+
+ block.import_data(**block_data)
self.rewrite() # evaluate stuff like nports before adding connections
@@ -420,9 +373,9 @@ class FlowGraph(Element):
def verify_and_get_port(key, block, dir):
ports = block.sinks if dir == 'sink' else block.sources
for port in ports:
- if key == port.key:
+ if key == port.key or key + '0' == port.key:
break
- if not key.isdigit() and port.get_type() == '' and key == port.name:
+ if not key.isdigit() and port.dtype == '' and key == port.name:
break
else:
if block.is_dummy_block:
@@ -431,34 +384,32 @@ class FlowGraph(Element):
raise LookupError('%s key %r not in %s block keys' % (dir, key, dir))
return port
- errors = False
- for connection_n in fg_n.get('connection', []):
- # get the block ids and port keys
- source_block_id = connection_n.get('source_block_id')
- sink_block_id = connection_n.get('sink_block_id')
- source_key = connection_n.get('source_key')
- sink_key = connection_n.get('sink_key')
+ had_connect_errors = False
+ _blocks = {block.name: block for block in self.blocks}
+ for src_blk_id, src_port_id, snk_blk_id, snk_port_id in data.get('connections', []):
try:
- source_block = self.get_block(source_block_id)
- sink_block = self.get_block(sink_block_id)
+ source_block = _blocks[src_blk_id]
+ sink_block = _blocks[snk_blk_id]
# fix old, numeric message ports keys
if file_format < 1:
- source_key, sink_key = _update_old_message_port_keys(
- source_key, sink_key, source_block, sink_block)
+ src_port_id, snk_port_id = _update_old_message_port_keys(
+ src_port_id, snk_port_id, source_block, sink_block)
# build the connection
- source_port = verify_and_get_port(source_key, source_block, 'source')
- sink_port = verify_and_get_port(sink_key, sink_block, 'sink')
+ source_port = verify_and_get_port(src_port_id, source_block, 'source')
+ sink_port = verify_and_get_port(snk_port_id, sink_block, 'sink')
+
self.connect(source_port, sink_port)
- except LookupError as e:
+
+ except (KeyError, LookupError) as e:
Messages.send_error_load(
'Connection between {}({}) and {}({}) could not be made.\n\t{}'.format(
- source_block_id, source_key, sink_block_id, sink_key, e))
- errors = True
+ src_blk_id, src_port_id, snk_blk_id, snk_port_id, e))
+ had_connect_errors = True
self.rewrite() # global rewrite
- return errors
+ return had_connect_errors
def _update_old_message_port_keys(source_key, sink_key, source_block, sink_block):
@@ -475,27 +426,11 @@ def _update_old_message_port_keys(source_key, sink_key, source_block, sink_block
message port.
"""
try:
- # get ports using the "old way" (assuming liner indexed keys)
+ # get ports using the "old way" (assuming linear indexed keys)
source_port = source_block.sources[int(source_key)]
sink_port = sink_block.sinks[int(sink_key)]
- if source_port.get_type() == "message" and sink_port.get_type() == "message":
+ if source_port.dtype == "message" and sink_port.dtype == "message":
source_key, sink_key = source_port.key, sink_port.key
except (ValueError, IndexError):
pass
return source_key, sink_key # do nothing
-
-
-def _guess_file_format_1(n):
- """
- Try to guess the file format for flow-graph files without version tag
- """
- try:
- has_non_numeric_message_keys = any(not (
- connection_n.get('source_key', '').isdigit() and
- connection_n.get('sink_key', '').isdigit()
- ) for connection_n in n.get('flow_graph', []).get('connection', []))
- if has_non_numeric_message_keys:
- return 1
- except:
- pass
- return 0
--
cgit v1.2.3