pybind: add scripts for scraping docstrings

author: Josh Morman <mormjb@gmail.com> 2020-04-23 07:07:23 -0400
committer: Josh Morman <mormjb@gmail.com> 2020-06-04 10:05:47 -0400
commit: 3bf8aa1193a408d10e9a5cdbc2307a4eccd59e97 (patch)
tree: 46d030c3dcbe1781082b0afe64631d5dfa0ca4ad /docs
parent: 9f6086161f8693b0643215f7a3935ec13661c882 (diff)
3 files changed, 381 insertions, 0 deletions
diff --git a/docs/doxygen/CMakeLists.txt b/docs/doxygen/CMakeLists.txt
index af3ee28397..09d18b0dad 100644
--- a/docs/doxygen/CMakeLists.txt
+++ b/docs/doxygen/CMakeLists.txt
@@ -42,4 +42,20 @@ add_custom_command(
 
 add_custom_target(doxygen_target ALL DEPENDS ${BUILT_DIRS})
 
+if(ENABLE_DOXYGEN)
+    add_custom_command( 
+    OUTPUT ${CMAKE_CURRENT_BINARY_DIR}/gnuradio_docstrings.json
+    COMMAND python3 ${CMAKE_SOURCE_DIR}/docs/doxygen/update_pydoc.py "scrape"
+    "--xml_path" ${CMAKE_BINARY_DIR}/docs/doxygen/xml
+    "--json_path" ${CMAKE_CURRENT_BINARY_DIR}/gnuradio_docstrings.json
+    COMMENT "Scraping generated documentation for docstrings ..."
+    DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/update_pydoc.py doxygen_target)
+
+    add_custom_target(
+        gnuradio_docstrings ALL
+        DEPENDS ${CMAKE_CURRENT_BINARY_DIR}/gnuradio_docstrings.json
+        )
+endif(ENABLE_DOXYGEN)
+
+
 install(DIRECTORY ${BUILT_DIRS} DESTINATION ${GR_PKG_DOC_DIR})
diff --git a/docs/doxygen/pydoc_macros.h b/docs/doxygen/pydoc_macros.h
new file mode 100644
index 0000000000..746b3babcd
--- /dev/null
+++ b/docs/doxygen/pydoc_macros.h
@@ -0,0 +1,18 @@
+#ifndef PYDOC_MACROS_H
+#define PYDOC_MACROS_H
+
+#define __EXPAND(x)                                      x
+#define __COUNT(_1, _2, _3, _4, _5, _6, _7, COUNT, ...)  COUNT
+#define __VA_SIZE(...)                                   __EXPAND(__COUNT(__VA_ARGS__, 7, 6, 5, 4, 3, 2, 1))
+#define __CAT1(a, b)                                     a ## b
+#define __CAT2(a, b)                                     __CAT1(a, b)
+#define __DOC1(n1)                                       __doc_##n1
+#define __DOC2(n1, n2)                                   __doc_##n1##_##n2
+#define __DOC3(n1, n2, n3)                               __doc_##n1##_##n2##_##n3
+#define __DOC4(n1, n2, n3, n4)                           __doc_##n1##_##n2##_##n3##_##n4
+#define __DOC5(n1, n2, n3, n4, n5)                       __doc_##n1##_##n2##_##n3##_##n4##_##n5
+#define __DOC6(n1, n2, n3, n4, n5, n6)                   __doc_##n1##_##n2##_##n3##_##n4##_##n5##_##n6
+#define __DOC7(n1, n2, n3, n4, n5, n6, n7)               __doc_##n1##_##n2##_##n3##_##n4##_##n5##_##n6##_##n7
+#define DOC(...)                                         __EXPAND(__EXPAND(__CAT2(__DOC, __VA_SIZE(__VA_ARGS__)))(__VA_ARGS__))
+
+#endif //PYDOC_MACROS_H
+\ No newline at end of file
diff --git a/docs/doxygen/update_pydoc.py b/docs/doxygen/update_pydoc.py
new file mode 100644
index 0000000000..44290e2fc6
--- /dev/null
+++ b/docs/doxygen/update_pydoc.py
@@ -0,0 +1,347 @@
+#
+# Copyright 2010-2012 Free Software Foundation, Inc.
+#
+# This file was generated by gr_modtool, a tool from the GNU Radio framework
+# This file is a part of gnuradio
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+#
+"""
+Updates the *pydoc_h files for a module
+Execute using: python update_pydoc.py xml_path outputfilename
+
+The file instructs Pybind11 to transfer the doxygen comments into the
+python docstrings.
+
+"""
+from __future__ import unicode_literals
+
+import os, sys, time, glob, re, json
+from argparse import ArgumentParser
+
+from doxyxml import DoxyIndex, DoxyClass, DoxyFriend, DoxyFunction, DoxyFile
+from doxyxml import DoxyOther, base
+
+def py_name(name):
+    bits = name.split('_')
+    return '_'.join(bits[1:])
+
+def make_name(name):
+    bits = name.split('_')
+    return bits[0] + '_make_' + '_'.join(bits[1:])
+
+
+class Block(object):
+    """
+    Checks if doxyxml produced objects correspond to a gnuradio block.
+    """
+
+    @classmethod
+    def includes(cls, item):
+        if not isinstance(item, DoxyClass):
+            return False
+        # Check for a parsing error.
+        if item.error():
+            return False
+        friendname = make_name(item.name())
+        is_a_block = item.has_member(friendname, DoxyFriend)
+        # But now sometimes the make function isn't a friend so check again.
+        if not is_a_block:
+            is_a_block = di.has_member(friendname, DoxyFunction)
+        return is_a_block
+
+class Block2(object):
+    """
+    Checks if doxyxml produced objects correspond to a new style
+    gnuradio block.
+    """
+
+    @classmethod
+    def includes(cls, item):
+        if not isinstance(item, DoxyClass):
+            return False
+        # Check for a parsing error.
+        if item.error():
+            return False
+        is_a_block2 = item.has_member('make', DoxyFunction) and item.has_member('sptr', DoxyOther)
+        return is_a_block2
+
+
+def utoascii(text):
+    """
+    Convert unicode text into ascii and escape quotes and backslashes.
+    """
+    if text is None:
+        return ''
+    out = text.encode('ascii', 'replace')
+    # swig will require us to replace blackslash with 4 backslashes
+    # TODO: evaluate what this should be for pybind11
+    out = out.replace(b'\\', b'\\\\\\\\')
+    out = out.replace(b'"', b'\\"').decode('ascii')
+    return str(out)
+
+
+def combine_descriptions(obj):
+    """
+    Combines the brief and detailed descriptions of an object together.
+    """
+    description = []
+    bd = obj.brief_description.strip()
+    dd = obj.detailed_description.strip()
+    if bd:
+        description.append(bd)
+    if dd:
+        description.append(dd)
+    return utoascii('\n\n'.join(description)).strip()
+
+def format_params(parameteritems):
+    output = ['Args:']
+    template = '    {0} : {1}'
+    for pi in parameteritems:
+        output.append(template.format(pi.name, pi.description))
+    return '\n'.join(output)
+
+entry_templ = '%feature("docstring") {name} "{docstring}"'
+def make_entry(obj, name=None, templ="{description}", description=None, params=[]):
+    """
+    Create a docstring key/value pair, where the key is the object name.
+
+    obj - a doxyxml object from which documentation will be extracted.
+    name - the name of the C object (defaults to obj.name())
+    templ - an optional template for the docstring containing only one
+            variable named 'description'.
+    description - if this optional variable is set then it's value is
+            used as the description instead of extracting it from obj.
+    """
+    if name is None:
+        name=obj.name()
+        if hasattr(obj,'_parse_data') and hasattr(obj._parse_data,'definition'):
+            name=obj._parse_data.definition.split(' ')[-1]
+    if "operator " in name:
+        return ''
+    if description is None:
+        description = combine_descriptions(obj)
+    if params:
+        description += '\n\n'
+        description += utoascii(format_params(params))
+    docstring = templ.format(description=description)
+
+    return {name: docstring}
+
+
+def make_class_entry(klass, description=None, ignored_methods=[], params=None):
+    """
+    Create a class docstring key/value pair.
+    """
+    if params is None:
+        params = klass.params
+    output = {}
+    output.update(make_entry(klass, description=description, params=params))
+    for func in klass.in_category(DoxyFunction):
+        if func.name() not in ignored_methods:
+            name = klass.name() + '::' + func.name()
+            output.update(make_entry(func, name=name))
+    return output
+
+
+def make_block_entry(di, block):
+    """
+    Create class and function docstrings of a gnuradio block
+    """
+    descriptions = []
+    # Get the documentation associated with the class.
+    class_desc = combine_descriptions(block)
+    if class_desc:
+        descriptions.append(class_desc)
+    # Get the documentation associated with the make function
+    make_func = di.get_member(make_name(block.name()), DoxyFunction)
+    make_func_desc = combine_descriptions(make_func)
+    if make_func_desc:
+        descriptions.append(make_func_desc)
+    # Get the documentation associated with the file
+    try:
+        block_file = di.get_member(block.name() + ".h", DoxyFile)
+        file_desc = combine_descriptions(block_file)
+        if file_desc:
+            descriptions.append(file_desc)
+    except base.Base.NoSuchMember:
+        # Don't worry if we can't find a matching file.
+        pass
+    # And join them all together to make a super duper description.
+    super_description = "\n\n".join(descriptions)
+    # Associate the combined description with the class and
+    # the make function.
+    output = {}
+    output.update(make_class_entry(block, description=super_description))
+    output.update(make_entry(make_func, description=super_description,
+                                  params=block.params))
+    return output
+
+def make_block2_entry(di, block):
+    """
+    Create class and function docstrings of a new style gnuradio block
+    """
+    # For new style blocks all the relevant documentation should be
+    # associated with the 'make' method.
+    class_description = combine_descriptions(block)
+    make_func = block.get_member('make', DoxyFunction)
+    make_description = combine_descriptions(make_func)
+    description = class_description + "\n\nConstructor Specific Documentation:\n\n" + make_description
+    # Associate the combined description with the class and
+    # the make function.
+    output = {}
+    output.update(make_class_entry(
+            block, description=description,
+            ignored_methods=['make'], params=make_func.params))
+    makename = block.name() + '::make'
+    output.update(make_entry(
+            make_func, name=makename, description=description,
+            params=make_func.params))
+    return output
+
+def get_docstrings_dict(di, custom_output=None):
+
+    output = {}
+    if custom_output:
+        output.update(custom_output)
+
+    # Create docstrings for the blocks.
+    blocks = di.in_category(Block)
+    blocks2 = di.in_category(Block2)
+
+    make_funcs = set([])
+    for block in blocks:
+        try:
+            make_func = di.get_member(make_name(block.name()), DoxyFunction)
+            # Don't want to risk writing to output twice.
+            if make_func.name() not in make_funcs:
+                make_funcs.add(make_func.name())
+                output.update(make_block_entry(di, block))
+        except block.ParsingError:
+            sys.stderr.write('Parsing error for block {0}\n'.format(block.name()))
+            raise
+
+    for block in blocks2:
+        try:
+            make_func = block.get_member('make', DoxyFunction)
+            make_func_name = block.name() +'::make'
+            # Don't want to risk writing to output twice.
+            if make_func_name not in make_funcs:
+                make_funcs.add(make_func_name)
+                output.update(make_block2_entry(di, block))
+        except block.ParsingError:
+            sys.stderr.write('Parsing error for block {0}\n'.format(block.name()))
+            raise
+
+    # Create docstrings for functions
+    # Don't include the make functions since they have already been dealt with.
+    funcs = [f for f in di.in_category(DoxyFunction)
+             if f.name() not in make_funcs and not f.name().startswith('std::')]
+    for f in funcs:
+        try:
+            output.update(make_entry(f))
+        except f.ParsingError:
+            sys.stderr.write('Parsing error for function {0}\n'.format(f.name()))
+
+    # Create docstrings for classes
+    block_names = [block.name() for block in blocks]
+    block_names += [block.name() for block in blocks2]
+    klasses = [k for k in di.in_category(DoxyClass)
+               if k.name() not in block_names and not k.name().startswith('std::')]
+    for k in klasses:
+        try:
+            output.update(make_class_entry(k))
+        except k.ParsingError:
+            sys.stderr.write('Parsing error for class {0}\n'.format(k.name()))
+
+    # Docstrings are not created for anything that is not a function or a class.
+    # If this excludes anything important please add it here.
+
+    return output
+
+def sub_docstring_in_pydoc_h(pydoc_files, docstrings_dict, output_dir, filter_str=None):
+    if filter_str:
+        docstrings_dict = {k: v for k, v in docstrings_dict.items() if k.startswith(filter_str)}
+
+    with open(os.path.join(output_dir,'docstring_status'),'w') as status_file:
+    
+        for pydoc_file in pydoc_files:
+            if filter_str:
+                filter_str2 = "::".join((filter_str,os.path.split(pydoc_file)[-1].split('_pydoc_template.h')[0]))
+                docstrings_dict2 = {k: v for k, v in docstrings_dict.items() if k.startswith(filter_str2)}
+            else:
+                docstrings_dict2 = docstrings_dict
+
+
+
+            file_in = open(pydoc_file,'r').read()
+            for key, value in docstrings_dict2.items(): 
+                file_in_tmp = file_in 
+                try:
+                    doc_key = key.split("::")
+                    # if 'gr' in doc_key:
+                    #     doc_key.remove('gr') 
+                    doc_key = '_'.join(doc_key)
+                    regexp = r'(__doc_{} =\sR\"doc\()[^)]*(\)doc\")'.format(doc_key)
+                    regexp = re.compile(regexp, re.MULTILINE)
+                
+                    (file_in, nsubs) = regexp.subn(r'\1'+value+r'\2', file_in, count=1)
+                    if nsubs == 1:
+                        status_file.write("PASS: " + pydoc_file + "\n")
+                except KeyboardInterrupt:
+                    raise KeyboardInterrupt
+                except: # be permissive, TODO log, but just leave the docstring blank
+                    status_file.write("FAIL: " + pydoc_file + "\n")
+                    file_in = file_in_tmp
+
+            output_pathname = os.path.join(output_dir, os.path.basename(pydoc_file).replace('_template.h','.h'))
+            # FIXME: Remove this debug print
+            print('output docstrings to {}'.format(output_pathname))
+            with open(output_pathname,'w') as file_out:
+                file_out.write(file_in)
+
+def copy_docstring_templates(pydoc_files, output_dir):
+    with open(os.path.join(output_dir,'docstring_status'),'w') as status_file:
+        for pydoc_file in pydoc_files:
+            file_in = open(pydoc_file,'r').read()
+            output_pathname = os.path.join(output_dir, os.path.basename(pydoc_file).replace('_template.h','.h'))
+            # FIXME: Remove this debug print
+            print('copy docstrings to {}'.format(output_pathname))
+            with open(output_pathname,'w') as file_out:
+                file_out.write(file_in)
+        status_file.write("DONE")
+
+def argParse():
+    """Parses commandline args."""
+    desc='Scrape the doxygen generated xml for docstrings to insert into python bindings'
+    parser = ArgumentParser(description=desc)
+    
+    parser.add_argument("function", help="Operation to perform on docstrings", choices=["scrape","sub","copy"])
+
+    parser.add_argument("--xml_path")
+    parser.add_argument("--bindings_dir")
+    parser.add_argument("--output_dir")
+    parser.add_argument("--json_path")
+    parser.add_argument("--filter", default=None)
+
+    return parser.parse_args()
+
+if __name__ == "__main__":
+    # Parse command line options and set up doxyxml.
+    args = argParse()
+    if args.function.lower() == 'scrape':
+        di = DoxyIndex(args.xml_path)
+        docstrings_dict = get_docstrings_dict(di)
+        with open(args.json_path, 'w') as fp:
+            json.dump(docstrings_dict, fp)
+    elif args.function.lower() == 'sub':
+        with open(args.json_path, 'r') as fp:
+            docstrings_dict = json.load(fp)
+        pydoc_files = glob.glob(os.path.join(args.bindings_dir,'*_pydoc_template.h'))
+        sub_docstring_in_pydoc_h(pydoc_files, docstrings_dict, args.output_dir, args.filter)
+    elif args.function.lower() == 'copy':
+        pydoc_files = glob.glob(os.path.join(args.bindings_dir,'*_pydoc_template.h'))
+        copy_docstring_templates(pydoc_files, args.output_dir)
+
+
author	Josh Morman <mormjb@gmail.com>	2020-04-23 07:07:23 -0400
committer	Josh Morman <mormjb@gmail.com>	2020-06-04 10:05:47 -0400
commit	3bf8aa1193a408d10e9a5cdbc2307a4eccd59e97 (patch)
tree	46d030c3dcbe1781082b0afe64631d5dfa0ca4ad /docs
parent	9f6086161f8693b0643215f7a3935ec13661c882 (diff)