From 999c0e723240ee783bca17942f77a9d05bbfc168 Mon Sep 17 00:00:00 2001
From: Josh Morman <mormjb@gmail.com>
Date: Thu, 23 Apr 2020 15:03:55 -0400
Subject: utils: add functionality to generate bindings
This currently exists in two places
1) Bindtool (longevity TBD) which calls blocktool to parse the public
header file in the include directory
2) Modtool - binding of headers added to add and bind. rm, update,
info, etc still TODO
---
.../gr-newmod/docs/doxygen/update_pydoc.py | 347 +++++++++++++++++++++
1 file changed, 347 insertions(+)
create mode 100644 gr-utils/modtool/templates/gr-newmod/docs/doxygen/update_pydoc.py
(limited to 'gr-utils/modtool/templates/gr-newmod/docs/doxygen/update_pydoc.py')
diff --git a/gr-utils/modtool/templates/gr-newmod/docs/doxygen/update_pydoc.py b/gr-utils/modtool/templates/gr-newmod/docs/doxygen/update_pydoc.py
new file mode 100644
index 0000000000..44290e2fc6
--- /dev/null
+++ b/gr-utils/modtool/templates/gr-newmod/docs/doxygen/update_pydoc.py
@@ -0,0 +1,347 @@
+#
+# Copyright 2010-2012 Free Software Foundation, Inc.
+#
+# This file was generated by gr_modtool, a tool from the GNU Radio framework
+# This file is a part of gnuradio
+#
+# SPDX-License-Identifier: GPL-3.0-or-later
+#
+#
+"""
+Updates the *pydoc_h files for a module
+Execute using: python update_pydoc.py xml_path outputfilename
+
+The file instructs Pybind11 to transfer the doxygen comments into the
+python docstrings.
+
+"""
+from __future__ import unicode_literals
+
+import os, sys, time, glob, re, json
+from argparse import ArgumentParser
+
+from doxyxml import DoxyIndex, DoxyClass, DoxyFriend, DoxyFunction, DoxyFile
+from doxyxml import DoxyOther, base
+
+def py_name(name):
+ bits = name.split('_')
+ return '_'.join(bits[1:])
+
+def make_name(name):
+ bits = name.split('_')
+ return bits[0] + '_make_' + '_'.join(bits[1:])
+
+
+class Block(object):
+ """
+ Checks if doxyxml produced objects correspond to a gnuradio block.
+ """
+
+ @classmethod
+ def includes(cls, item):
+ if not isinstance(item, DoxyClass):
+ return False
+ # Check for a parsing error.
+ if item.error():
+ return False
+ friendname = make_name(item.name())
+ is_a_block = item.has_member(friendname, DoxyFriend)
+ # But now sometimes the make function isn't a friend so check again.
+ if not is_a_block:
+ is_a_block = di.has_member(friendname, DoxyFunction)
+ return is_a_block
+
+class Block2(object):
+ """
+ Checks if doxyxml produced objects correspond to a new style
+ gnuradio block.
+ """
+
+ @classmethod
+ def includes(cls, item):
+ if not isinstance(item, DoxyClass):
+ return False
+ # Check for a parsing error.
+ if item.error():
+ return False
+ is_a_block2 = item.has_member('make', DoxyFunction) and item.has_member('sptr', DoxyOther)
+ return is_a_block2
+
+
+def utoascii(text):
+ """
+ Convert unicode text into ascii and escape quotes and backslashes.
+ """
+ if text is None:
+ return ''
+ out = text.encode('ascii', 'replace')
+ # swig will require us to replace blackslash with 4 backslashes
+ # TODO: evaluate what this should be for pybind11
+ out = out.replace(b'\\', b'\\\\\\\\')
+ out = out.replace(b'"', b'\\"').decode('ascii')
+ return str(out)
+
+
+def combine_descriptions(obj):
+ """
+ Combines the brief and detailed descriptions of an object together.
+ """
+ description = []
+ bd = obj.brief_description.strip()
+ dd = obj.detailed_description.strip()
+ if bd:
+ description.append(bd)
+ if dd:
+ description.append(dd)
+ return utoascii('\n\n'.join(description)).strip()
+
+def format_params(parameteritems):
+ output = ['Args:']
+ template = ' {0} : {1}'
+ for pi in parameteritems:
+ output.append(template.format(pi.name, pi.description))
+ return '\n'.join(output)
+
+entry_templ = '%feature("docstring") {name} "{docstring}"'
+def make_entry(obj, name=None, templ="{description}", description=None, params=[]):
+ """
+ Create a docstring key/value pair, where the key is the object name.
+
+ obj - a doxyxml object from which documentation will be extracted.
+ name - the name of the C object (defaults to obj.name())
+ templ - an optional template for the docstring containing only one
+ variable named 'description'.
+ description - if this optional variable is set then it's value is
+ used as the description instead of extracting it from obj.
+ """
+ if name is None:
+ name=obj.name()
+ if hasattr(obj,'_parse_data') and hasattr(obj._parse_data,'definition'):
+ name=obj._parse_data.definition.split(' ')[-1]
+ if "operator " in name:
+ return ''
+ if description is None:
+ description = combine_descriptions(obj)
+ if params:
+ description += '\n\n'
+ description += utoascii(format_params(params))
+ docstring = templ.format(description=description)
+
+ return {name: docstring}
+
+
+def make_class_entry(klass, description=None, ignored_methods=[], params=None):
+ """
+ Create a class docstring key/value pair.
+ """
+ if params is None:
+ params = klass.params
+ output = {}
+ output.update(make_entry(klass, description=description, params=params))
+ for func in klass.in_category(DoxyFunction):
+ if func.name() not in ignored_methods:
+ name = klass.name() + '::' + func.name()
+ output.update(make_entry(func, name=name))
+ return output
+
+
+def make_block_entry(di, block):
+ """
+ Create class and function docstrings of a gnuradio block
+ """
+ descriptions = []
+ # Get the documentation associated with the class.
+ class_desc = combine_descriptions(block)
+ if class_desc:
+ descriptions.append(class_desc)
+ # Get the documentation associated with the make function
+ make_func = di.get_member(make_name(block.name()), DoxyFunction)
+ make_func_desc = combine_descriptions(make_func)
+ if make_func_desc:
+ descriptions.append(make_func_desc)
+ # Get the documentation associated with the file
+ try:
+ block_file = di.get_member(block.name() + ".h", DoxyFile)
+ file_desc = combine_descriptions(block_file)
+ if file_desc:
+ descriptions.append(file_desc)
+ except base.Base.NoSuchMember:
+ # Don't worry if we can't find a matching file.
+ pass
+ # And join them all together to make a super duper description.
+ super_description = "\n\n".join(descriptions)
+ # Associate the combined description with the class and
+ # the make function.
+ output = {}
+ output.update(make_class_entry(block, description=super_description))
+ output.update(make_entry(make_func, description=super_description,
+ params=block.params))
+ return output
+
+def make_block2_entry(di, block):
+ """
+ Create class and function docstrings of a new style gnuradio block
+ """
+ # For new style blocks all the relevant documentation should be
+ # associated with the 'make' method.
+ class_description = combine_descriptions(block)
+ make_func = block.get_member('make', DoxyFunction)
+ make_description = combine_descriptions(make_func)
+ description = class_description + "\n\nConstructor Specific Documentation:\n\n" + make_description
+ # Associate the combined description with the class and
+ # the make function.
+ output = {}
+ output.update(make_class_entry(
+ block, description=description,
+ ignored_methods=['make'], params=make_func.params))
+ makename = block.name() + '::make'
+ output.update(make_entry(
+ make_func, name=makename, description=description,
+ params=make_func.params))
+ return output
+
+def get_docstrings_dict(di, custom_output=None):
+
+ output = {}
+ if custom_output:
+ output.update(custom_output)
+
+ # Create docstrings for the blocks.
+ blocks = di.in_category(Block)
+ blocks2 = di.in_category(Block2)
+
+ make_funcs = set([])
+ for block in blocks:
+ try:
+ make_func = di.get_member(make_name(block.name()), DoxyFunction)
+ # Don't want to risk writing to output twice.
+ if make_func.name() not in make_funcs:
+ make_funcs.add(make_func.name())
+ output.update(make_block_entry(di, block))
+ except block.ParsingError:
+ sys.stderr.write('Parsing error for block {0}\n'.format(block.name()))
+ raise
+
+ for block in blocks2:
+ try:
+ make_func = block.get_member('make', DoxyFunction)
+ make_func_name = block.name() +'::make'
+ # Don't want to risk writing to output twice.
+ if make_func_name not in make_funcs:
+ make_funcs.add(make_func_name)
+ output.update(make_block2_entry(di, block))
+ except block.ParsingError:
+ sys.stderr.write('Parsing error for block {0}\n'.format(block.name()))
+ raise
+
+ # Create docstrings for functions
+ # Don't include the make functions since they have already been dealt with.
+ funcs = [f for f in di.in_category(DoxyFunction)
+ if f.name() not in make_funcs and not f.name().startswith('std::')]
+ for f in funcs:
+ try:
+ output.update(make_entry(f))
+ except f.ParsingError:
+ sys.stderr.write('Parsing error for function {0}\n'.format(f.name()))
+
+ # Create docstrings for classes
+ block_names = [block.name() for block in blocks]
+ block_names += [block.name() for block in blocks2]
+ klasses = [k for k in di.in_category(DoxyClass)
+ if k.name() not in block_names and not k.name().startswith('std::')]
+ for k in klasses:
+ try:
+ output.update(make_class_entry(k))
+ except k.ParsingError:
+ sys.stderr.write('Parsing error for class {0}\n'.format(k.name()))
+
+ # Docstrings are not created for anything that is not a function or a class.
+ # If this excludes anything important please add it here.
+
+ return output
+
+def sub_docstring_in_pydoc_h(pydoc_files, docstrings_dict, output_dir, filter_str=None):
+ if filter_str:
+ docstrings_dict = {k: v for k, v in docstrings_dict.items() if k.startswith(filter_str)}
+
+ with open(os.path.join(output_dir,'docstring_status'),'w') as status_file:
+
+ for pydoc_file in pydoc_files:
+ if filter_str:
+ filter_str2 = "::".join((filter_str,os.path.split(pydoc_file)[-1].split('_pydoc_template.h')[0]))
+ docstrings_dict2 = {k: v for k, v in docstrings_dict.items() if k.startswith(filter_str2)}
+ else:
+ docstrings_dict2 = docstrings_dict
+
+
+
+ file_in = open(pydoc_file,'r').read()
+ for key, value in docstrings_dict2.items():
+ file_in_tmp = file_in
+ try:
+ doc_key = key.split("::")
+ # if 'gr' in doc_key:
+ # doc_key.remove('gr')
+ doc_key = '_'.join(doc_key)
+ regexp = r'(__doc_{} =\sR\"doc\()[^)]*(\)doc\")'.format(doc_key)
+ regexp = re.compile(regexp, re.MULTILINE)
+
+ (file_in, nsubs) = regexp.subn(r'\1'+value+r'\2', file_in, count=1)
+ if nsubs == 1:
+ status_file.write("PASS: " + pydoc_file + "\n")
+ except KeyboardInterrupt:
+ raise KeyboardInterrupt
+ except: # be permissive, TODO log, but just leave the docstring blank
+ status_file.write("FAIL: " + pydoc_file + "\n")
+ file_in = file_in_tmp
+
+ output_pathname = os.path.join(output_dir, os.path.basename(pydoc_file).replace('_template.h','.h'))
+ # FIXME: Remove this debug print
+ print('output docstrings to {}'.format(output_pathname))
+ with open(output_pathname,'w') as file_out:
+ file_out.write(file_in)
+
+def copy_docstring_templates(pydoc_files, output_dir):
+ with open(os.path.join(output_dir,'docstring_status'),'w') as status_file:
+ for pydoc_file in pydoc_files:
+ file_in = open(pydoc_file,'r').read()
+ output_pathname = os.path.join(output_dir, os.path.basename(pydoc_file).replace('_template.h','.h'))
+ # FIXME: Remove this debug print
+ print('copy docstrings to {}'.format(output_pathname))
+ with open(output_pathname,'w') as file_out:
+ file_out.write(file_in)
+ status_file.write("DONE")
+
+def argParse():
+ """Parses commandline args."""
+ desc='Scrape the doxygen generated xml for docstrings to insert into python bindings'
+ parser = ArgumentParser(description=desc)
+
+ parser.add_argument("function", help="Operation to perform on docstrings", choices=["scrape","sub","copy"])
+
+ parser.add_argument("--xml_path")
+ parser.add_argument("--bindings_dir")
+ parser.add_argument("--output_dir")
+ parser.add_argument("--json_path")
+ parser.add_argument("--filter", default=None)
+
+ return parser.parse_args()
+
+if __name__ == "__main__":
+ # Parse command line options and set up doxyxml.
+ args = argParse()
+ if args.function.lower() == 'scrape':
+ di = DoxyIndex(args.xml_path)
+ docstrings_dict = get_docstrings_dict(di)
+ with open(args.json_path, 'w') as fp:
+ json.dump(docstrings_dict, fp)
+ elif args.function.lower() == 'sub':
+ with open(args.json_path, 'r') as fp:
+ docstrings_dict = json.load(fp)
+ pydoc_files = glob.glob(os.path.join(args.bindings_dir,'*_pydoc_template.h'))
+ sub_docstring_in_pydoc_h(pydoc_files, docstrings_dict, args.output_dir, args.filter)
+ elif args.function.lower() == 'copy':
+ pydoc_files = glob.glob(os.path.join(args.bindings_dir,'*_pydoc_template.h'))
+ copy_docstring_templates(pydoc_files, args.output_dir)
+
+
--
cgit v1.2.3