pybind: add hash check to binding file creation (#3472)

author: mormj <34754695+mormj@users.noreply.github.com> 2020-05-22 14:21:09 -0400
committer: Josh Morman <mormjb@gmail.com> 2020-06-04 10:05:48 -0400
commit: 3165f73d7c6224523957fa69beade6069efea6ef (patch)
tree: bcc09005ab8cc6b2f6bef4589f5b612eeafb29be /gr-utils
parent: cb1185bc2c12c0c7bd76ab93040a63c39815e847 (diff)
11 files changed, 323 insertions, 15 deletions
diff --git a/gr-utils/CMakeLists.txt b/gr-utils/CMakeLists.txt
index 992b247832..42c6deda1a 100644
--- a/gr-utils/CMakeLists.txt
+++ b/gr-utils/CMakeLists.txt
@@ -24,12 +24,6 @@ GR_PYTHON_CHECK_MODULE_RAW(
   CLICK_PLUGINS_FOUND
   )
 
- GR_PYTHON_CHECK_MODULE_RAW(
-   "pygccxml"
-   "import pygccxml"
-   PYGCCXML_FOUND
-   )
-
 ########################################################################
 # Register component
 ########################################################################
diff --git a/gr-utils/bindtool/core/generator.py b/gr-utils/bindtool/core/generator.py
index 2112342408..702c1e4238 100644
--- a/gr-utils/bindtool/core/generator.py
+++ b/gr-utils/bindtool/core/generator.py
@@ -15,11 +15,14 @@ import pathlib
 import json
 from mako.template import Template
 from datetime import datetime
+import hashlib
 
 
 class BindingGenerator:
 
-    def __init__(self, prefix, namespace, prefix_include_root, output_dir="", addl_includes="", match_include_structure=False):
+    def __init__(self, prefix, namespace, prefix_include_root, output_dir="", addl_includes="", 
+                    match_include_structure=False, catch_exceptions=True, write_json_output=False, status_output=None,
+                    flag_automatic=False, flag_pygccxml=False):
         """Initialize BindingGenerator
         prefix -- path to installed gnuradio prefix (use gr.prefix() if unsure)
         namespace -- desired namespace to parse e.g. ['gr','module_name']
@@ -42,6 +45,11 @@ class BindingGenerator:
         self.prefix_include_root = prefix_include_root
         self.output_dir = output_dir
         self.match_include_structure = match_include_structure
+        self.catch_exceptions = catch_exceptions
+        self.write_json_output = write_json_output
+        self.status_output = status_output
+        self.flag_automatic = flag_automatic
+        self.flag_pygccxml = flag_pygccxml
 
         pass
 
@@ -60,7 +68,6 @@ class BindingGenerator:
             prefix_include_root=self.prefix_include_root,
         )
 
-
     def gen_pybind_cc(self, header_info, base_name):
         current_path = os.path.dirname(pathlib.Path(__file__).absolute())
         tpl = Template(filename=os.path.join(
@@ -69,11 +76,16 @@ class BindingGenerator:
 
         tpl = Template(filename=os.path.join(current_path, '..',
                                              'templates', 'generic_python_cc.mako'))
+
+
         return tpl.render(
             license=license,
             header_info=header_info,
             basename=base_name,
+            flag_automatic=self.flag_automatic,
+            flag_pygccxml=self.flag_pygccxml,
             prefix_include_root=self.prefix_include_root,
+
         )
 
     def write_pydoc_h(self, header_info, base_name, output_dir):
@@ -92,7 +104,6 @@ class BindingGenerator:
             print(e)
             return None
 
-
     def write_pybind_cc(self, header_info, base_name, output_dir):
 
         binding_pathname_cc = os.path.join(
@@ -143,12 +154,19 @@ class BindingGenerator:
             include_paths=include_paths, file_path=file_to_process)
         try:
             header_info = parser.get_header_info(self.namespace)
-            # TODO: Scrape the docstrings
-            self.write_json(header_info, base_name, output_dir)
+            
+            if self.write_json_output:
+                self.write_json(header_info, base_name, output_dir)
             self.write_pybind_cc(header_info, base_name, output_dir)
             self.write_pydoc_h(header_info, base_name, output_dir)
 
+            if (self.status_output):
+                with open(self.status_output, 'w') as outfile:
+                    outfile.write("OK: " + str(datetime.now()))
+
         except Exception as e:
+            if not self.catch_exceptions:
+                raise(e)
             print(e)
             failure_pathname = os.path.join(
                 output_dir, 'failed_conversions.txt')
@@ -170,7 +188,7 @@ class BindingGenerator:
 
         output_dir = os.path.join(
             self.output_dir, rel_path_after_include, 'bindings')
-        doc_dir = os.path.join(output_dir,'docstrings')
+        doc_dir = os.path.join(output_dir, 'docstrings')
         if output_dir and not os.path.exists(output_dir) and not os.path.exists(doc_dir):
             output_dir = os.path.abspath(output_dir)
             print('creating output directory {}'.format(output_dir))
@@ -222,6 +240,7 @@ class BindingGenerator:
         license = tpl.render(year=datetime.now().year)
 
         binding_pathname = os.path.join(output_dir, 'CMakeLists.txt')
+
         file_list = [os.path.split(f)[-1] for f in file_list]
         tpl = Template(filename=os.path.join(current_path, '..',
                                              'templates', 'CMakeLists.txt.mako'))
diff --git a/gr-utils/bindtool/scripts/bind_intree_file.py b/gr-utils/bindtool/scripts/bind_intree_file.py
new file mode 100644
index 0000000000..79a9c3da1a
--- /dev/null
+++ b/gr-utils/bindtool/scripts/bind_intree_file.py
@@ -0,0 +1,58 @@
+import warnings
+import argparse
+import os
+from gnuradio.bindtool import BindingGenerator
+import pathlib
+import sys
+
+parser = argparse.ArgumentParser(description='Bind a GR In-Tree Module')
+parser.add_argument('--module', type=str,
+                    help='Name of gr module containing file to bind (e.g. fft digital analog)')
+
+parser.add_argument('--output_dir', default='/tmp',
+                    help='Output directory of generated bindings')
+parser.add_argument('--prefix', help='Prefix of Installed GNU Radio')
+parser.add_argument('--src', help='Directory of gnuradio source tree',
+                    default=os.path.dirname(os.path.abspath(__file__))+'/../../..')
+
+parser.add_argument(
+    '--filename', help="File to be parsed")
+
+parser.add_argument(
+    '--include', help='Additional Include Dirs, separated', default=(), nargs='+')
+
+parser.add_argument(
+    '--status', help='Location of output file for general status (used during cmake)', default=None
+)
+parser.add_argument(
+    '--flag_automatic', default='0'
+)
+parser.add_argument(
+    '--flag_pygccxml', default='0'
+)
+
+args = parser.parse_args()
+
+prefix = args.prefix
+output_dir = args.output_dir
+includes = args.include
+name = args.module
+
+if name not in ['gr', 'pmt']:
+    namespace = ['gr', name]
+    prefix_include_root = 'gnuradio/'+name  # pmt, gnuradio/digital, etc.
+else:
+    namespace = [name]
+    if name == 'gr':
+        prefix_include_root = 'gnuradio'
+    elif name == 'pmt':
+        prefix_include_root = 'pmt'
+
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+    bg = BindingGenerator(args.prefix, namespace,
+                          prefix_include_root, output_dir, addl_includes=','.join(args.include), catch_exceptions=False, write_json_output=False, status_output=args.status,
+                            flag_automatic=True if args.flag_automatic.lower() in ['1','true'] else False,
+                            flag_pygccxml=True if args.flag_pygccxml.lower() in ['1','true'] else False)
+    bg.gen_file_binding(args.filename)
diff --git a/gr-utils/bindtool/scripts/header_utils.py b/gr-utils/bindtool/scripts/header_utils.py
new file mode 100644
index 0000000000..165124e969
--- /dev/null
+++ b/gr-utils/bindtool/scripts/header_utils.py
@@ -0,0 +1,78 @@
+# Utilities for reading values in header files
+
+from argparse import ArgumentParser
+import re
+
+
+class PybindHeaderParser:
+    def __init__(self, pathname):
+        with open(pathname,'r') as f:
+            self.file_txt = f.read()
+
+    def get_flag_automatic(self):
+        # p = re.compile(r'BINDTOOL_GEN_AUTOMATIC\(([^\s])\)')
+        # m = p.search(self.file_txt)
+        m = re.search(r'BINDTOOL_GEN_AUTOMATIC\(([^\s])\)', self.file_txt)
+        if (m and m.group(1) == '1'):
+            return True
+        else:
+            return False
+        
+    def get_flag_pygccxml(self):    
+        # p = re.compile(r'BINDTOOL_USE_PYGCCXML\(([^\s])\)')
+        # m = p.search(self.file_txt)
+        m = re.search(r'BINDTOOL_USE_PYGCCXML\(([^\s])\)', self.file_txt)
+        if (m and m.group(1) == '1'):
+            return True
+        else:
+            return False
+
+    def get_header_filename(self):
+        # p = re.compile(r'BINDTOOL_HEADER_FILE\(([^\s]*)\)')
+        # m = p.search(self.file_txt)
+        m = re.search(r'BINDTOOL_HEADER_FILE\(([^\s]*)\)', self.file_txt)
+        if (m):
+            return m.group(1)
+        else:
+            return None
+
+    def get_header_file_hash(self):
+        # p = re.compile(r'BINDTOOL_HEADER_FILE_HASH\(([^\s]*)\)')
+        # m = p.search(self.file_txt)
+        m = re.search(r'BINDTOOL_HEADER_FILE_HASH\(([^\s]*)\)', self.file_txt)
+        if (m):
+            return m.group(1)
+        else:
+            return None
+
+    def get_flags(self):
+        return f'{self.get_flag_automatic()};{self.get_flag_pygccxml()};{self.get_header_filename()};{self.get_header_file_hash()};'
+
+
+
+def argParse():
+    """Parses commandline args."""
+    desc='Reads the parameters from the comment block in the pybind files'
+    parser = ArgumentParser(description=desc)
+    
+    parser.add_argument("function", help="Operation to perform on comment block of pybind file", choices=["flag_auto","flag_pygccxml","header_filename","header_file_hash","all"])
+    parser.add_argument("pathname", help="Pathname of pybind c++ file to read, e.g. blockname_python.cc")
+
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    # Parse command line options and set up doxyxml.
+    args = argParse()
+
+    pbhp = PybindHeaderParser(args.pathname)
+
+    if args.function == "flag_auto":
+        print(pbhp.get_flag_automatic())
+    elif args.function == "flag_pygccxml":
+        print(pbhp.get_flag_pygccxml())
+    elif args.function == "header_filename":
+        print(pbhp.get_header_filename())
+    elif args.function == "header_file_hash":
+        print(pbhp.get_header_file_hash()) 
+    elif args.function == "all":
+        print(pbhp.get_flags())
+\ No newline at end of file
diff --git a/gr-utils/bindtool/templates/CMakeLists.txt.mako b/gr-utils/bindtool/templates/CMakeLists.txt.mako
index 7638c1620b..e151f36c47 100644
--- a/gr-utils/bindtool/templates/CMakeLists.txt.mako
+++ b/gr-utils/bindtool/templates/CMakeLists.txt.mako
@@ -17,7 +17,7 @@ basename = os.path.splitext(f)[0]
 % endfor
     python_bindings.cc)
 
-GR_PYBIND_MAKE(${module_name} 
+GR_PYBIND_MAKE_CHECK_HASH(${module_name} 
    ../../.. 
    gr::${module_name}
    "${'${'+file_list+'}'}")
diff --git a/gr-utils/bindtool/templates/generic_python_cc.mako b/gr-utils/bindtool/templates/generic_python_cc.mako
index cf16f2ee7a..a4743fb8fb 100644
--- a/gr-utils/bindtool/templates/generic_python_cc.mako
+++ b/gr-utils/bindtool/templates/generic_python_cc.mako
@@ -9,13 +9,23 @@
 <%
     namespace = header_info['namespace']
     modname = header_info['module_name']
+    header_filename = header_info['filename']
+    header_file_hash = header_info['md5hash']
 %>\
 ${license}
 
-/* This file is automatically generated using bindtool */
+/***********************************************************************************/
+/* This file is automatically generated using bindtool and can be manually edited  */
+/* The following lines can be configured to regenerate this file during cmake      */
+/* If manual edits are made, the following tags should be modified accordingly.    */
+/* BINDTOOL_GEN_AUTOMATIC(${'1' if flag_automatic else '0'})                                                       */
+/* BINDTOOL_USE_PYGCCXML(${'1' if flag_pygccxml else '0'})                                                        */
+/* BINDTOOL_HEADER_FILE(${header_filename})                                        */
+/* BINDTOOL_HEADER_FILE_HASH(${header_file_hash})                     */
+/***********************************************************************************/
 
-#include <pybind11/pybind11.h>
 #include <pybind11/complex.h>
+#include <pybind11/pybind11.h>
 #include <pybind11/stl.h>
 
 namespace py = pybind11;
diff --git a/gr-utils/blocktool/core/parseheader_generic.py b/gr-utils/blocktool/core/parseheader_generic.py
index 31a065a61e..ef6a1398ca 100644
--- a/gr-utils/blocktool/core/parseheader_generic.py
+++ b/gr-utils/blocktool/core/parseheader_generic.py
@@ -265,10 +265,19 @@ class GenericHeaderParser(BlockTool):
         """
         module = self.modname.split('-')[-1]
         self.parsed_data['module_name'] = module
+        self.parsed_data['filename'] = self.filename
+        
+        import hashlib
+        hasher = hashlib.md5()
+        with open(self.target_file, 'rb') as file_in:
+            buf = file_in.read()
+            hasher.update(buf)
+        self.parsed_data['md5hash'] = hasher.hexdigest()
 
         # Right now if pygccxml is not installed, it will only handle the make function
         # TODO: extend this to other publicly declared functions in the h file
         if not PYGCCXML_AVAILABLE:
+            self.parsed_data['parser'] = 'simple'
             (params, iosig, blockname) = self._parse_cc_h(self.target_file)
             self.parsed_data['target_namespace'] = namespace_to_parse
 
@@ -301,6 +310,7 @@ class GenericHeaderParser(BlockTool):
 
             return self.parsed_data
         else:
+            self.parsed_data['parser'] = 'pygccxml'
             generator_path, generator_name = utils.find_xml_generator()
             xml_generator_config = parser.xml_generator_configuration_t(
                 xml_generator_path=generator_path,
diff --git a/gr-utils/modtool/core/add.py b/gr-utils/modtool/core/add.py
index f8cb8329fe..712d44de48 100644
--- a/gr-utils/modtool/core/add.py
+++ b/gr-utils/modtool/core/add.py
@@ -256,6 +256,8 @@ class ModToolAdd(ModTool):
 
         header_info = {
             "module_name": self.info['modname'],
+            "filename": self.info['blockname'] + '.h',
+            "md5hash": "0",
             "namespace": {
                 "name": "::".join(['gr', self.info['modname']]),
                 "enums": [],
diff --git a/gr-utils/modtool/templates/gr-newmod/python/bindings/CMakeLists.txt b/gr-utils/modtool/templates/gr-newmod/python/bindings/CMakeLists.txt
index e0d1544973..b7aa881783 100644
--- a/gr-utils/modtool/templates/gr-newmod/python/bindings/CMakeLists.txt
+++ b/gr-utils/modtool/templates/gr-newmod/python/bindings/CMakeLists.txt
@@ -5,6 +5,12 @@
 # SPDX-License-Identifier: GPL-3.0-or-later
 #
 
+GR_PYTHON_CHECK_MODULE_RAW(
+    "pygccxml"
+    "import pygccxml"
+    PYGCCXML_FOUND
+    )
+
 include(GrPybind)
 
 ########################################################################
diff --git a/gr-utils/modtool/templates/gr-newmod/python/bindings/bind_oot_file.py b/gr-utils/modtool/templates/gr-newmod/python/bindings/bind_oot_file.py
new file mode 100644
index 0000000000..050f60fee0
--- /dev/null
+++ b/gr-utils/modtool/templates/gr-newmod/python/bindings/bind_oot_file.py
@@ -0,0 +1,53 @@
+import warnings
+import argparse
+import os
+from gnuradio.bindtool import BindingGenerator
+import pathlib
+import sys
+
+parser = argparse.ArgumentParser(description='Bind a GR Out of Tree Block')
+parser.add_argument('--module', type=str,
+                    help='Name of gr module containing file to bind (e.g. fft digital analog)')
+
+parser.add_argument('--output_dir', default='/tmp',
+                    help='Output directory of generated bindings')
+parser.add_argument('--prefix', help='Prefix of Installed GNU Radio')
+parser.add_argument('--src', help='Directory of gnuradio source tree',
+                    default=os.path.dirname(os.path.abspath(__file__))+'/../../..')
+
+parser.add_argument(
+    '--filename', help="File to be parsed")
+
+parser.add_argument(
+    '--include', help='Additional Include Dirs, separated', default=(), nargs='+')
+
+parser.add_argument(
+    '--status', help='Location of output file for general status (used during cmake)', default=None
+)
+parser.add_argument(
+    '--flag_automatic', default='0'
+)
+parser.add_argument(
+    '--flag_pygccxml', default='0'
+)
+
+args = parser.parse_args()
+
+prefix = args.prefix
+output_dir = args.output_dir
+includes = args.include
+name = args.module
+
+namespace = ['gr', name]
+prefix_include_root = name
+
+
+with warnings.catch_warnings():
+    warnings.filterwarnings("ignore", category=DeprecationWarning)
+
+    bg = BindingGenerator(prefix, namespace,
+                          prefix_include_root, output_dir, addl_includes=','.join(args.include), catch_exceptions=False, write_json_output=False, status_output=args.status,
+                          flag_automatic=True if args.flag_automatic.lower() in [
+                              '1', 'true'] else False,
+                          flag_pygccxml=True if args.flag_pygccxml.lower() in ['1', 'true'] else False)
+    bg.gen_file_binding(args.filename)
diff --git a/gr-utils/modtool/templates/gr-newmod/python/bindings/header_utils.py b/gr-utils/modtool/templates/gr-newmod/python/bindings/header_utils.py
new file mode 100644
index 0000000000..165124e969
--- /dev/null
+++ b/gr-utils/modtool/templates/gr-newmod/python/bindings/header_utils.py
@@ -0,0 +1,78 @@
+# Utilities for reading values in header files
+
+from argparse import ArgumentParser
+import re
+
+
+class PybindHeaderParser:
+    def __init__(self, pathname):
+        with open(pathname,'r') as f:
+            self.file_txt = f.read()
+
+    def get_flag_automatic(self):
+        # p = re.compile(r'BINDTOOL_GEN_AUTOMATIC\(([^\s])\)')
+        # m = p.search(self.file_txt)
+        m = re.search(r'BINDTOOL_GEN_AUTOMATIC\(([^\s])\)', self.file_txt)
+        if (m and m.group(1) == '1'):
+            return True
+        else:
+            return False
+        
+    def get_flag_pygccxml(self):    
+        # p = re.compile(r'BINDTOOL_USE_PYGCCXML\(([^\s])\)')
+        # m = p.search(self.file_txt)
+        m = re.search(r'BINDTOOL_USE_PYGCCXML\(([^\s])\)', self.file_txt)
+        if (m and m.group(1) == '1'):
+            return True
+        else:
+            return False
+
+    def get_header_filename(self):
+        # p = re.compile(r'BINDTOOL_HEADER_FILE\(([^\s]*)\)')
+        # m = p.search(self.file_txt)
+        m = re.search(r'BINDTOOL_HEADER_FILE\(([^\s]*)\)', self.file_txt)
+        if (m):
+            return m.group(1)
+        else:
+            return None
+
+    def get_header_file_hash(self):
+        # p = re.compile(r'BINDTOOL_HEADER_FILE_HASH\(([^\s]*)\)')
+        # m = p.search(self.file_txt)
+        m = re.search(r'BINDTOOL_HEADER_FILE_HASH\(([^\s]*)\)', self.file_txt)
+        if (m):
+            return m.group(1)
+        else:
+            return None
+
+    def get_flags(self):
+        return f'{self.get_flag_automatic()};{self.get_flag_pygccxml()};{self.get_header_filename()};{self.get_header_file_hash()};'
+
+
+
+def argParse():
+    """Parses commandline args."""
+    desc='Reads the parameters from the comment block in the pybind files'
+    parser = ArgumentParser(description=desc)
+    
+    parser.add_argument("function", help="Operation to perform on comment block of pybind file", choices=["flag_auto","flag_pygccxml","header_filename","header_file_hash","all"])
+    parser.add_argument("pathname", help="Pathname of pybind c++ file to read, e.g. blockname_python.cc")
+
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    # Parse command line options and set up doxyxml.
+    args = argParse()
+
+    pbhp = PybindHeaderParser(args.pathname)
+
+    if args.function == "flag_auto":
+        print(pbhp.get_flag_automatic())
+    elif args.function == "flag_pygccxml":
+        print(pbhp.get_flag_pygccxml())
+    elif args.function == "header_filename":
+        print(pbhp.get_header_filename())
+    elif args.function == "header_file_hash":
+        print(pbhp.get_header_file_hash()) 
+    elif args.function == "all":
+        print(pbhp.get_flags())
+\ No newline at end of file
author	mormj <34754695+mormj@users.noreply.github.com>	2020-05-22 14:21:09 -0400
committer	Josh Morman <mormjb@gmail.com>	2020-06-04 10:05:48 -0400
commit	3165f73d7c6224523957fa69beade6069efea6ef (patch)
tree	bcc09005ab8cc6b2f6bef4589f5b612eeafb29be /gr-utils
parent	cb1185bc2c12c0c7bd76ab93040a63c39815e847 (diff)