grc: add embedded python block definition and support in GRC

author: Sebastian Koslowski <koslowski@kit.edu> 2015-07-20 22:24:53 +0200
committer: Sebastian Koslowski <koslowski@kit.edu> 2015-11-11 21:39:32 +0100
commit: c9a5fabe17efe8af1c9435f746ed55c5f5790917 (patch)
tree: 77242850bfc2ebb159cb9c9d53eff9ea5459f8cb /grc/python
parent: 8b14a84ef8b8a73db002153cb398f79f1343ebc2 (diff)
5 files changed, 224 insertions, 16 deletions
diff --git a/grc/python/Block.py b/grc/python/Block.py
index 5289d5765e..accfd21913 100644
--- a/grc/python/Block.py
+++ b/grc/python/Block.py
@@ -21,11 +21,13 @@ import itertools
 import collections
 
 from .. base.Constants import BLOCK_FLAG_NEED_QT_GUI, BLOCK_FLAG_NEED_WX_GUI
+from .. base.odict import odict
+
 from .. base.Block import Block as _Block
 from .. gui.Block import Block as _GUIBlock
 
 from . FlowGraph import _variable_matcher
-import extract_docs
+from . import epy_block_io, extract_docs
 
 
 class Block(_Block, _GUIBlock):
@@ -59,6 +61,9 @@ class Block(_Block, _GUIBlock):
         )
         _GUIBlock.__init__(self)
 
+        self._epy_source_hash = -1  # for epy blocks
+        self._epy_reload_error = None
+
     def get_bus_structure(self, direction):
         if direction == 'source':
             bus_structure = self._bus_structure_source;
@@ -111,12 +116,16 @@ class Block(_Block, _GUIBlock):
 
         check_generate_mode('WX GUI', BLOCK_FLAG_NEED_WX_GUI, ('wx_gui',))
         check_generate_mode('QT GUI', BLOCK_FLAG_NEED_QT_GUI, ('qt_gui', 'hb_qt_gui'))
+        if self._epy_reload_error:
+            self.add_error_message(str(self._epy_reload_error))
 
     def rewrite(self):
         """
         Add and remove ports to adjust for the nports.
         """
         _Block.rewrite(self)
+        # Check and run any custom rewrite function for this block
+        getattr(self, 'rewrite_' + self._key, lambda: None)()
 
         # adjust nports, disconnect hidden ports
         for ports in (self.get_sources(), self.get_sinks()):
@@ -216,3 +225,88 @@ class Block(_Block, _GUIBlock):
 
     def is_virtual_source(self):
         return self.get_key() == 'virtual_source'
+
+    ###########################################################################
+    # Custom rewrite functions
+    ###########################################################################
+
+    def rewrite_epy_block(self):
+        flowgraph = self.get_parent()
+        platform = flowgraph.get_parent()
+        param_blk = self.get_param('_io_cache')
+        param_src = self.get_param('_source_code')
+
+        src = param_src.get_value()
+        src_hash = hash(src)
+        if src_hash == self._epy_source_hash:
+            return
+
+        try:
+            blk_io = epy_block_io.extract(src)
+
+        except Exception as e:
+            self._epy_reload_error = ValueError('Source code eval:\n' + str(e))
+            try:  # load last working block io
+                blk_io = epy_block_io.BlockIO(*eval(param_blk.get_value()))
+            except:
+                return
+        else:
+            self._epy_reload_error = None  # clear previous errors
+            param_blk.set_value(repr(tuple(blk_io)))
+
+        # print "Rewriting embedded python block {!r}".format(self.get_id())
+
+        self._epy_source_hash = src_hash
+        self._name = blk_io.name or blk_io.cls
+        self._doc = blk_io.doc
+        self._imports[0] = 'from {} import {}'.format(self.get_id(), blk_io.cls)
+        self._make = '{}({})'.format(blk_io.cls, ', '.join(
+            '{0}=${0}'.format(key) for key, _ in blk_io.params))
+
+        params = dict()
+        for param in list(self._params):
+            if hasattr(param, '__epy_param__'):
+                params[param.get_key()] = param
+                self._params.remove(param)
+
+        for key, value in blk_io.params:
+            if key in params:
+                param = params[key]
+                if not param.value_is_default():
+                    param.set_value(value)
+            else:
+                name = key.replace('_', ' ').title()
+                n = odict(dict(name=name, key=key, type='raw', value=value))
+                param = platform.Param(block=self, n=n)
+                setattr(param, '__epy_param__', True)
+            self._params.append(param)
+
+        def update_ports(label, ports, port_specs, direction):
+            ports_to_remove = list(ports)
+            iter_ports = iter(ports)
+            ports_new = list()
+            port_current = next(iter_ports, None)
+            for key, port_type in port_specs:
+                reuse_port = (
+                    port_current is not None and
+                    port_current.get_type() == port_type and
+                    (key.isdigit() or port_current.get_key() == key)
+                )
+                if reuse_port:
+                    ports_to_remove.remove(port_current)
+                    port, port_current = port_current, next(iter_ports, None)
+                else:
+                    n = odict(dict(name=label + str(key), type=port_type, key=key))
+                    port = platform.Port(block=self, n=n, dir=direction)
+                ports_new.append(port)
+            # replace old port list with new one
+            del ports[:]
+            ports.extend(ports_new)
+            # remove excess port connections
+            for port in ports_to_remove:
+                for connection in port.get_connections():
+                    flowgraph.remove_element(connection)
+
+        update_ports('in', self.get_sinks(), blk_io.sinks, 'sink')
+        update_ports('out', self.get_sources(), blk_io.sources, 'source')
+        _Block.rewrite(self)
diff --git a/grc/python/CMakeLists.txt b/grc/python/CMakeLists.txt
index 41d965e89c..3f9e273146 100644
--- a/grc/python/CMakeLists.txt
+++ b/grc/python/CMakeLists.txt
@@ -21,6 +21,7 @@
 GR_PYTHON_INSTALL(FILES
     expr_utils.py
     extract_docs.py
+    epy_block_io.py
     Block.py
     Connection.py
     Constants.py
diff --git a/grc/python/Generator.py b/grc/python/Generator.py
index d60befe3fa..0b9469b588 100644
--- a/grc/python/Generator.py
+++ b/grc/python/Generator.py
@@ -79,7 +79,7 @@ class TopBlockGenerator(object):
         self._flow_graph = flow_graph
         self._generate_options = self._flow_graph.get_option('generate_options')
         self._mode = TOP_BLOCK_FILE_MODE
-        dirname = os.path.dirname(file_path)
+        dirname = self._dirname = os.path.dirname(file_path)
         # handle the case where the directory is read-only
         # in this case, use the system's temp directory
         if not os.access(dirname, os.W_OK):
@@ -108,12 +108,14 @@ class TopBlockGenerator(object):
                                       "This is usually undesired. Consider "
                                       "removing the throttle block.")
         # generate
-        with codecs.open(self.get_file_path(), 'w', encoding = 'utf-8') as fp:
-            fp.write(self._build_python_code_from_template())
-        try:
-            os.chmod(self.get_file_path(), self._mode)
-        except:
-            pass
+        for filename, data in self._build_python_code_from_template():
+            with open(filename, 'w', encoding='utf-8') as fp:
+                fp.write(data)
+            if filename == self.get_file_path():
+                try:
+                    os.chmod(filename, self._mode)
+                except:
+                    pass
 
     def get_popen(self):
         """
@@ -148,6 +150,8 @@ class TopBlockGenerator(object):
         Returns:
             a string of python code
         """
+        output = list()
+
         title = self._flow_graph.get_option('title') or self._flow_graph.get_option('id').replace('_', ' ').title()
         imports = self._flow_graph.get_imports()
         variables = self._flow_graph.get_variables()
@@ -174,6 +178,12 @@ class TopBlockGenerator(object):
         # List of regular blocks (all blocks minus the special ones)
         blocks = filter(lambda b: b not in (imports + parameters), blocks)
 
+        for block in blocks:
+            if block.get_key() == 'epy_block':
+                file_path = os.path.join(self._dirname, block.get_id() + '.py')
+                src = block.get_param('_source_code').get_value()
+                output.append((file_path, src))
+
         # Filter out virtual sink connections
         cf = lambda c: not (c.is_bus() or c.is_msg() or c.get_sink().get_parent().is_virtual_sink())
         connections = filter(cf, self._flow_graph.get_enabled_connections())
@@ -258,7 +268,8 @@ class TopBlockGenerator(object):
         }
         # build the template
         t = Template(open(FLOW_GRAPH_TEMPLATE, 'r').read(), namespace)
-        return str(t)
+        output.append((self.get_file_path(), str(t)))
+        return output
 
 
 class HierBlockGenerator(TopBlockGenerator):
diff --git a/grc/python/Param.py b/grc/python/Param.py
index 50723ed2cc..746f677e46 100644
--- a/grc/python/Param.py
+++ b/grc/python/Param.py
@@ -17,6 +17,11 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 """
 
+import ast
+import re
+
+from gnuradio import gr
+
 from .. base.Param import Param as _Param
 from .. gui.Param import Param as _GUIParam
 
@@ -24,8 +29,6 @@ import Constants
 from Constants import VECTOR_TYPES, COMPLEX_TYPES, REAL_TYPES, INT_TYPES
 
 from gnuradio import eng_notation
-import re
-from gnuradio import gr
 
 _check_id_matcher = re.compile('^[a-z|A-Z]\w*$')
 _show_id_matcher = re.compile('^(variable\w*|parameter|options|notebook)$')
@@ -62,7 +65,7 @@ class Param(_Param, _GUIParam):
         'complex', 'real', 'float', 'int',
         'complex_vector', 'real_vector', 'float_vector', 'int_vector',
         'hex', 'string', 'bool',
-        'file_open', 'file_save', '_multiline',
+        'file_open', 'file_save', '_multiline', '_multiline_python_external',
         'id', 'stream_id',
         'grid_pos', 'notebook', 'gui_hint',
         'import',
@@ -266,7 +269,7 @@ class Param(_Param, _GUIParam):
         #########################
         # String Types
         #########################
-        elif t in ('string', 'file_open', 'file_save', '_multiline'):
+        elif t in ('string', 'file_open', 'file_save', '_multiline', '_multiline_python_external'):
             #do not check if file/directory exists, that is a runtime issue
             try:
                 e = self.get_parent().get_parent().evaluate(v)
@@ -274,8 +277,10 @@ class Param(_Param, _GUIParam):
                     raise Exception()
             except:
                 self._stringify_flag = True
-                e = v
-            return str(e)
+                e = str(v)
+            if t == '_multiline_python_external':
+                ast.parse(e)  # raises SyntaxError
+            return e
         #########################
         # Unique ID Type
         #########################
@@ -405,7 +410,7 @@ class Param(_Param, _GUIParam):
         """
         v = self.get_value()
         t = self.get_type()
-        if t in ('string', 'file_open', 'file_save', '_multiline'):  # string types
+        if t in ('string', 'file_open', 'file_save', '_multiline', '_multiline_python_external'):  # string types
             if not self._init: self.evaluate()
             if self._stringify_flag: return '"%s"'%v.replace('"', '\"')
             else: return v
diff --git a/grc/python/epy_block_io.py b/grc/python/epy_block_io.py
new file mode 100644
index 0000000000..8d3ce1caa1
--- /dev/null
+++ b/grc/python/epy_block_io.py
@@ -0,0 +1,97 @@
+
+import inspect
+import collections
+
+from gnuradio import gr
+import pmt
+
+
+TYPE_MAP = {
+    'complex64': 'complex', 'complex': 'complex',
+    'float32': 'float', 'float': 'float',
+    'int32': 'int', 'uint32': 'int',
+    'int16': 'short', 'uint16': 'short',
+    'int8': 'byte', 'uint8': 'byte',
+}
+
+BlockIO = collections.namedtuple('BlockIO', 'name cls params sinks sources doc')
+
+
+def _ports(sigs, msgs):
+    ports = list()
+    for i, dtype in enumerate(sigs):
+        port_type = TYPE_MAP.get(dtype.name, None)
+        if not port_type:
+            raise ValueError("Can't map {0:!r} to GRC port type".format(dtype))
+        ports.append((str(i), port_type))
+    for msg_key in msgs:
+        if msg_key == 'system':
+            continue
+        ports.append((msg_key, 'message'))
+    return ports
+
+
+def _blk_class(source_code):
+    ns = {}
+    try:
+        exec source_code in ns
+    except Exception as e:
+        raise ValueError("Can't interpret source code: " + str(e))
+    for var in ns.itervalues():
+        if inspect.isclass(var)and issubclass(var, gr.gateway.gateway_block):
+            break
+    else:
+        raise ValueError('No python block class found in code')
+    return var
+
+
+def extract(cls):
+    if not inspect.isclass(cls):
+        cls = _blk_class(cls)
+
+    spec = inspect.getargspec(cls.__init__)
+    defaults = map(repr, spec.defaults or ())
+    doc = cls.__doc__ or cls.__init__.__doc__ or ''
+    cls_name = cls.__name__
+
+    if len(defaults) + 1 != len(spec.args):
+        raise ValueError("Need all default values")
+
+    try:
+        instance = cls()
+    except Exception as e:
+        raise RuntimeError("Can't create an instance of your block: " + str(e))
+
+    name = instance.name()
+    params = list(zip(spec.args[1:], defaults))
+
+    sinks = _ports(instance.in_sig(),
+                      pmt.to_python(instance.message_ports_in()))
+    sources = _ports(instance.out_sig(),
+                        pmt.to_python(instance.message_ports_out()))
+
+    return BlockIO(name, cls_name, params, sinks, sources, doc)
+
+
+if __name__ == '__main__':
+    blk_code = """
+import numpy as np
+from gnuradio import gr
+import pmt
+
+class blk(gr.sync_block):
+    def __init__(self, param1=None, param2=None):
+        "Test Docu"
+        gr.sync_block.__init__(
+            self,
+            name='Embedded Python Block',
+            in_sig = (np.float32,),
+            out_sig = (np.float32,np.complex64,),
+        )
+        self.message_port_register_in(pmt.intern('msg_in'))
+        self.message_port_register_out(pmt.intern('msg_out'))
+
+    def work(self, inputs_items, output_items):
+        return 10
+    """
+    print extract(blk_code)
author	Sebastian Koslowski <koslowski@kit.edu>	2015-07-20 22:24:53 +0200
committer	Sebastian Koslowski <koslowski@kit.edu>	2015-11-11 21:39:32 +0100
commit	c9a5fabe17efe8af1c9435f746ed55c5f5790917 (patch)
tree	77242850bfc2ebb159cb9c9d53eff9ea5459f8cb /grc/python
parent	8b14a84ef8b8a73db002153cb398f79f1343ebc2 (diff)