3 files changed, 137 insertions, 93 deletions
diff --git a/gr-utils/modtool/templates/gr-newmod/docs/doxygen/Doxyfile.in b/gr-utils/modtool/templates/gr-newmod/docs/doxygen/Doxyfile.in
index 55816e8252..307fed08c5 100644
--- a/gr-utils/modtool/templates/gr-newmod/docs/doxygen/Doxyfile.in
+++ b/gr-utils/modtool/templates/gr-newmod/docs/doxygen/Doxyfile.in
@@ -723,8 +723,6 @@ EXCLUDE_PATTERNS       = */.deps/* \
 
 EXCLUDE_SYMBOLS        = ad9862 \
                          numpy \
-                         *swig* \
-                         *Swig* \
                          *my_top_block* \
                          *my_graph* \
                          *app_top_block* \
diff --git a/gr-utils/modtool/templates/gr-newmod/docs/doxygen/pydoc_macros.h b/gr-utils/modtool/templates/gr-newmod/docs/doxygen/pydoc_macros.h
new file mode 100644
index 0000000000..98bf7cd639
--- /dev/null
+++ b/gr-utils/modtool/templates/gr-newmod/docs/doxygen/pydoc_macros.h
@@ -0,0 +1,19 @@
+#ifndef PYDOC_MACROS_H
+#define PYDOC_MACROS_H
+
+#define __EXPAND(x) x
+#define __COUNT(_1, _2, _3, _4, _5, _6, _7, COUNT, ...) COUNT
+#define __VA_SIZE(...) __EXPAND(__COUNT(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1))
+#define __CAT1(a, b) a##b
+#define __CAT2(a, b) __CAT1(a, b)
+#define __DOC1(n1) __doc_##n1
+#define __DOC2(n1, n2) __doc_##n1##_##n2
+#define __DOC3(n1, n2, n3) __doc_##n1##_##n2##_##n3
+#define __DOC4(n1, n2, n3, n4) __doc_##n1##_##n2##_##n3##_##n4
+#define __DOC5(n1, n2, n3, n4, n5) __doc_##n1##_##n2##_##n3##_##n4##_##n5
+#define __DOC6(n1, n2, n3, n4, n5, n6) __doc_##n1##_##n2##_##n3##_##n4##_##n5##_##n6
+#define __DOC7(n1, n2, n3, n4, n5, n6, n7) \
+    __doc_##n1##_##n2##_##n3##_##n4##_##n5##_##n6##_##n7
+#define DOC(...) __EXPAND(__EXPAND(__CAT2(__DOC, __VA_SIZE(__VA_ARGS__)))(__VA_ARGS__))
+
+#endif // PYDOC_MACROS_H
+\ No newline at end of file
diff --git a/gr-utils/modtool/templates/gr-newmod/docs/doxygen/swig_doc.py b/gr-utils/modtool/templates/gr-newmod/docs/doxygen/update_pydoc.py
index 288fed2aa7..44290e2fc6 100644
--- a/gr-utils/modtool/templates/gr-newmod/docs/doxygen/swig_doc.py
+++ b/gr-utils/modtool/templates/gr-newmod/docs/doxygen/update_pydoc.py
@@ -2,22 +2,23 @@
 # Copyright 2010-2012 Free Software Foundation, Inc.
 #
 # This file was generated by gr_modtool, a tool from the GNU Radio framework
-# This file is a part of gr-howto
+# This file is a part of gnuradio
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 #
 #
 """
-Creates the swig_doc.i SWIG interface file.
-Execute using: python swig_doc.py xml_path outputfilename
+Updates the *pydoc_h files for a module
+Execute using: python update_pydoc.py xml_path outputfilename
 
-The file instructs SWIG to transfer the doxygen comments into the
+The file instructs Pybind11 to transfer the doxygen comments into the
 python docstrings.
 
 """
 from __future__ import unicode_literals
 
-import sys, time
+import os, sys, time, glob, re, json
+from argparse import ArgumentParser
 
 from doxyxml import DoxyIndex, DoxyClass, DoxyFriend, DoxyFunction, DoxyFile
 from doxyxml import DoxyOther, base
@@ -75,6 +76,7 @@ def utoascii(text):
         return ''
     out = text.encode('ascii', 'replace')
     # swig will require us to replace blackslash with 4 backslashes
+    # TODO: evaluate what this should be for pybind11
     out = out.replace(b'\\', b'\\\\\\\\')
     out = out.replace(b'"', b'\\"').decode('ascii')
     return str(out)
@@ -103,7 +105,7 @@ def format_params(parameteritems):
 entry_templ = '%feature("docstring") {name} "{docstring}"'
 def make_entry(obj, name=None, templ="{description}", description=None, params=[]):
     """
-    Create a docstring entry for a swig interface file.
+    Create a docstring key/value pair, where the key is the object name.
 
     obj - a doxyxml object from which documentation will be extracted.
     name - the name of the C object (defaults to obj.name())
@@ -114,6 +116,8 @@ def make_entry(obj, name=None, templ="{description}", description=None, params=[
     """
     if name is None:
         name=obj.name()
+        if hasattr(obj,'_parse_data') and hasattr(obj._parse_data,'definition'):
+            name=obj._parse_data.definition.split(' ')[-1]
     if "operator " in name:
         return ''
     if description is None:
@@ -122,56 +126,28 @@ def make_entry(obj, name=None, templ="{description}", description=None, params=[
         description += '\n\n'
         description += utoascii(format_params(params))
     docstring = templ.format(description=description)
-    if not docstring:
-        return ''
-    return entry_templ.format(
-        name=name,
-        docstring=docstring,
-        )
-
 
-def make_func_entry(func, name=None, description=None, params=None):
-    """
-    Create a function docstring entry for a swig interface file.
-
-    func - a doxyxml object from which documentation will be extracted.
-    name - the name of the C object (defaults to func.name())
-    description - if this optional variable is set then it's value is
-            used as the description instead of extracting it from func.
-    params - a parameter list that overrides using func.params.
-    """
-    #if params is None:
-    #    params = func.params
-    #params = [prm.declname for prm in params]
-    #if params:
-    #    sig = "Params: (%s)" % ", ".join(params)
-    #else:
-    #    sig = "Params: (NONE)"
-    #templ = "{description}\n\n" + sig
-    #return make_entry(func, name=name, templ=utoascii(templ),
-    #                  description=description)
-    return make_entry(func, name=name, description=description, params=params)
+    return {name: docstring}
 
 
 def make_class_entry(klass, description=None, ignored_methods=[], params=None):
     """
-    Create a class docstring for a swig interface file.
+    Create a class docstring key/value pair.
     """
     if params is None:
         params = klass.params
-    output = []
-    output.append(make_entry(klass, description=description, params=params))
+    output = {}
+    output.update(make_entry(klass, description=description, params=params))
     for func in klass.in_category(DoxyFunction):
         if func.name() not in ignored_methods:
             name = klass.name() + '::' + func.name()
-            output.append(make_func_entry(func, name=name))
-    return "\n\n".join(output)
+            output.update(make_entry(func, name=name))
+    return output
 
 
 def make_block_entry(di, block):
     """
-    Create class and function docstrings of a gnuradio block for a
-    swig interface file.
+    Create class and function docstrings of a gnuradio block
     """
     descriptions = []
     # Get the documentation associated with the class.
@@ -196,18 +172,16 @@ def make_block_entry(di, block):
     super_description = "\n\n".join(descriptions)
     # Associate the combined description with the class and
     # the make function.
-    output = []
-    output.append(make_class_entry(block, description=super_description))
-    output.append(make_func_entry(make_func, description=super_description,
+    output = {}
+    output.update(make_class_entry(block, description=super_description))
+    output.update(make_entry(make_func, description=super_description,
                                   params=block.params))
-    return "\n\n".join(output)
+    return output
 
 def make_block2_entry(di, block):
     """
-    Create class and function docstrings of a new style gnuradio block for a
-    swig interface file.
+    Create class and function docstrings of a new style gnuradio block
     """
-    descriptions = []
     # For new style blocks all the relevant documentation should be
     # associated with the 'make' method.
     class_description = combine_descriptions(block)
@@ -216,28 +190,21 @@ def make_block2_entry(di, block):
     description = class_description + "\n\nConstructor Specific Documentation:\n\n" + make_description
     # Associate the combined description with the class and
     # the make function.
-    output = []
-    output.append(make_class_entry(
+    output = {}
+    output.update(make_class_entry(
             block, description=description,
             ignored_methods=['make'], params=make_func.params))
     makename = block.name() + '::make'
-    output.append(make_func_entry(
+    output.update(make_entry(
             make_func, name=makename, description=description,
             params=make_func.params))
-    return "\n\n".join(output)
+    return output
 
-def make_swig_interface_file(di, swigdocfilename, custom_output=None):
+def get_docstrings_dict(di, custom_output=None):
 
-    output = ["""
-/*
- * This file was automatically generated using swig_doc.py.
- *
- * Any changes to it will be lost next time it is regenerated.
- */
-"""]
-
-    if custom_output is not None:
-        output.append(custom_output)
+    output = {}
+    if custom_output:
+        output.update(custom_output)
 
     # Create docstrings for the blocks.
     blocks = di.in_category(Block)
@@ -250,7 +217,7 @@ def make_swig_interface_file(di, swigdocfilename, custom_output=None):
             # Don't want to risk writing to output twice.
             if make_func.name() not in make_funcs:
                 make_funcs.add(make_func.name())
-                output.append(make_block_entry(di, block))
+                output.update(make_block_entry(di, block))
         except block.ParsingError:
             sys.stderr.write('Parsing error for block {0}\n'.format(block.name()))
             raise
@@ -262,7 +229,7 @@ def make_swig_interface_file(di, swigdocfilename, custom_output=None):
             # Don't want to risk writing to output twice.
             if make_func_name not in make_funcs:
                 make_funcs.add(make_func_name)
-                output.append(make_block2_entry(di, block))
+                output.update(make_block2_entry(di, block))
         except block.ParsingError:
             sys.stderr.write('Parsing error for block {0}\n'.format(block.name()))
             raise
@@ -273,7 +240,7 @@ def make_swig_interface_file(di, swigdocfilename, custom_output=None):
              if f.name() not in make_funcs and not f.name().startswith('std::')]
     for f in funcs:
         try:
-            output.append(make_func_entry(f))
+            output.update(make_entry(f))
         except f.ParsingError:
             sys.stderr.write('Parsing error for function {0}\n'.format(f.name()))
 
@@ -284,37 +251,97 @@ def make_swig_interface_file(di, swigdocfilename, custom_output=None):
                if k.name() not in block_names and not k.name().startswith('std::')]
     for k in klasses:
         try:
-            output.append(make_class_entry(k))
+            output.update(make_class_entry(k))
         except k.ParsingError:
             sys.stderr.write('Parsing error for class {0}\n'.format(k.name()))
 
     # Docstrings are not created for anything that is not a function or a class.
     # If this excludes anything important please add it here.
 
-    output = "\n\n".join(output)
-
-    swig_doc = open(swigdocfilename, 'w')
-    swig_doc.write(output)
-    swig_doc.close()
+    return output
+
+def sub_docstring_in_pydoc_h(pydoc_files, docstrings_dict, output_dir, filter_str=None):
+    if filter_str:
+        docstrings_dict = {k: v for k, v in docstrings_dict.items() if k.startswith(filter_str)}
+
+    with open(os.path.join(output_dir,'docstring_status'),'w') as status_file:
+    
+        for pydoc_file in pydoc_files:
+            if filter_str:
+                filter_str2 = "::".join((filter_str,os.path.split(pydoc_file)[-1].split('_pydoc_template.h')[0]))
+                docstrings_dict2 = {k: v for k, v in docstrings_dict.items() if k.startswith(filter_str2)}
+            else:
+                docstrings_dict2 = docstrings_dict
+
+
+
+            file_in = open(pydoc_file,'r').read()
+            for key, value in docstrings_dict2.items(): 
+                file_in_tmp = file_in 
+                try:
+                    doc_key = key.split("::")
+                    # if 'gr' in doc_key:
+                    #     doc_key.remove('gr') 
+                    doc_key = '_'.join(doc_key)
+                    regexp = r'(__doc_{} =\sR\"doc\()[^)]*(\)doc\")'.format(doc_key)
+                    regexp = re.compile(regexp, re.MULTILINE)
+                
+                    (file_in, nsubs) = regexp.subn(r'\1'+value+r'\2', file_in, count=1)
+                    if nsubs == 1:
+                        status_file.write("PASS: " + pydoc_file + "\n")
+                except KeyboardInterrupt:
+                    raise KeyboardInterrupt
+                except: # be permissive, TODO log, but just leave the docstring blank
+                    status_file.write("FAIL: " + pydoc_file + "\n")
+                    file_in = file_in_tmp
+
+            output_pathname = os.path.join(output_dir, os.path.basename(pydoc_file).replace('_template.h','.h'))
+            # FIXME: Remove this debug print
+            print('output docstrings to {}'.format(output_pathname))
+            with open(output_pathname,'w') as file_out:
+                file_out.write(file_in)
+
+def copy_docstring_templates(pydoc_files, output_dir):
+    with open(os.path.join(output_dir,'docstring_status'),'w') as status_file:
+        for pydoc_file in pydoc_files:
+            file_in = open(pydoc_file,'r').read()
+            output_pathname = os.path.join(output_dir, os.path.basename(pydoc_file).replace('_template.h','.h'))
+            # FIXME: Remove this debug print
+            print('copy docstrings to {}'.format(output_pathname))
+            with open(output_pathname,'w') as file_out:
+                file_out.write(file_in)
+        status_file.write("DONE")
+
+def argParse():
+    """Parses commandline args."""
+    desc='Scrape the doxygen generated xml for docstrings to insert into python bindings'
+    parser = ArgumentParser(description=desc)
+    
+    parser.add_argument("function", help="Operation to perform on docstrings", choices=["scrape","sub","copy"])
+
+    parser.add_argument("--xml_path")
+    parser.add_argument("--bindings_dir")
+    parser.add_argument("--output_dir")
+    parser.add_argument("--json_path")
+    parser.add_argument("--filter", default=None)
+
+    return parser.parse_args()
 
 if __name__ == "__main__":
     # Parse command line options and set up doxyxml.
-    err_msg = "Execute using: python swig_doc.py xml_path outputfilename"
-    if len(sys.argv) != 3:
-        raise Exception(err_msg)
-    xml_path = sys.argv[1]
-    swigdocfilename = sys.argv[2]
-    di = DoxyIndex(xml_path)
-
-    # gnuradio.gr.msq_queue.insert_tail and delete_head create errors unless docstrings are defined!
-    # This is presumably a bug in SWIG.
-    #msg_q = di.get_member(u'gr_msg_queue', DoxyClass)
-    #insert_tail = msg_q.get_member(u'insert_tail', DoxyFunction)
-    #delete_head = msg_q.get_member(u'delete_head', DoxyFunction)
-    output = []
-    #output.append(make_func_entry(insert_tail, name='gr_py_msg_queue__insert_tail'))
-    #output.append(make_func_entry(delete_head, name='gr_py_msg_queue__delete_head'))
-    custom_output = "\n\n".join(output)
-
-    # Generate the docstrings interface file.
-    make_swig_interface_file(di, swigdocfilename, custom_output=custom_output)
+    args = argParse()
+    if args.function.lower() == 'scrape':
+        di = DoxyIndex(args.xml_path)
+        docstrings_dict = get_docstrings_dict(di)
+        with open(args.json_path, 'w') as fp:
+            json.dump(docstrings_dict, fp)
+    elif args.function.lower() == 'sub':
+        with open(args.json_path, 'r') as fp:
+            docstrings_dict = json.load(fp)
+        pydoc_files = glob.glob(os.path.join(args.bindings_dir,'*_pydoc_template.h'))
+        sub_docstring_in_pydoc_h(pydoc_files, docstrings_dict, args.output_dir, args.filter)
+    elif args.function.lower() == 'copy':
+        pydoc_files = glob.glob(os.path.join(args.bindings_dir,'*_pydoc_template.h'))
+        copy_docstring_templates(pydoc_files, args.output_dir)
+
+