Merge remote-tracking branch 'gnuradio-wg-grc/master_grcwg'

author: Johnathan Corgan <johnathan@corganlabs.com> 2015-11-28 12:52:03 -0800
committer: Johnathan Corgan <johnathan@corganlabs.com> 2015-11-28 12:52:03 -0800
commit: ee06ad1eb9437d1db989076ca61ed0e4573244ec (patch)
tree: ebe51c26bf69183feefbd7cf919827be9274581d /grc/python/extract_docs.py
parent: 27e8311d8b6bf88c8b46461abe6de8d222e2ee25 (diff)
parent: dcc09deb064453f4add1ece3502ee99f12f3cb25 (diff)
1 files changed, 260 insertions, 38 deletions
diff --git a/grc/python/extract_docs.py b/grc/python/extract_docs.py
index 47b2820a1f..d8dc4f4e8f 100644
--- a/grc/python/extract_docs.py
+++ b/grc/python/extract_docs.py
@@ -17,58 +17,280 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 """
 
+import sys
 import re
+import subprocess
+import threading
+import json
+import Queue
+import random
+import itertools
 
-def _extract(key):
-    """
-    Extract the documentation from the python __doc__ strings.
-    If multiple modules match, combine the docs.
+
+###############################################################################
+# The docstring extraction
+###############################################################################
+
+def docstring_guess_from_key(key):
+    """Extract the documentation from the python __doc__ strings
+
+    By guessing module and constructor names from key
 
     Args:
         key: the block key
 
     Returns:
-        a string with documentation
+        a dict (block_name --> doc string)
     """
-    #extract matches
-    try:
-        module_name, constructor_name = key.split('_', 1)
-        module = __import__('gnuradio.'+module_name)
-        module = getattr(module, module_name)
-    except ImportError:
-            try:
-                    module_name, constructor_name = key.split('_', 1)
-                    module = __import__(module_name)
-            except: return ''
-    except:
-            return ''
-    pattern = constructor_name.replace('_', '_*').replace('x', '\w')
-    pattern_matcher = re.compile('^%s\w*$'%pattern)
-    matches = filter(lambda x: pattern_matcher.match(x), dir(module))
-    #combine all matches
-    doc_strs = list()
-    for match in matches:
+    doc_strings = dict()
+
+    in_tree = [key.partition('_')[::2] + (
+        lambda package: getattr(__import__('gnuradio.' + package), package),
+    )]
+
+    key_parts = key.split('_')
+    oot = [
+        ('_'.join(key_parts[:i]), '_'.join(key_parts[i:]), __import__)
+        for i in range(1, len(key_parts))
+    ]
+
+    for module_name, init_name, importer in itertools.chain(in_tree, oot):
+        if not module_name or not init_name:
+            continue
         try:
-            title = '   ---   ' + match + '   ---   '
-            doc_strs.append('\n\n'.join([title, getattr(module, match).__doc__]).strip())
-        except: pass
-    return '\n\n'.join(doc_strs)
+            module = importer(module_name)
+            break
+        except ImportError:
+            continue
+    else:
+        return doc_strings
 
-_docs_cache = dict()
-def extract(key):
-    """
-    Call the private extract and cache the result.
+    pattern = re.compile(
+        '^' + init_name.replace('_', '_*').replace('x', r'\w') + r'\w*$'
+    )
+    for match in filter(pattern.match, dir(module)):
+        try:
+            doc_strings[match] = getattr(module, match).__doc__.strip()
+        except AttributeError:
+            continue
+
+    return doc_strings
+
+
+def docstring_from_make(key, imports, make):
+    """Extract the documentation from the python __doc__ strings
+
+    By importing it and checking a truncated make
 
     Args:
         key: the block key
+        imports: a list of import statements (string) to execute
+        make: block constructor template
 
     Returns:
-        a string with documentation
+        a list of tuples (block_name, doc string)
     """
-    if not _docs_cache.has_key(key):
-        _docs_cache[key] = _extract(key)
-    return _docs_cache[key]
 
-if __name__ == '__main__':
-    import sys
-    print extract(sys.argv[1])
+    try:
+        blk_cls = make.partition('(')[0].strip()
+        if '$' in blk_cls:
+            raise ValueError('Not an identifier')
+
+        ns = dict()
+        for _import in imports:
+            exec(_import.strip(), ns)
+        blk = eval(blk_cls, ns)
+
+        doc_strings = {key: blk.__doc__}
+
+    except (ImportError, AttributeError, SyntaxError, ValueError):
+        doc_strings = docstring_guess_from_key(key)
+
+    return doc_strings
+
+
+###############################################################################
+# Manage docstring extraction in separate process
+###############################################################################
+
+class SubprocessLoader(object):
+    """Start and manage docstring extraction process
+
+    Manages subprocess and handles RPC.
+    """
+    BOOTSTRAP = "import runpy; runpy.run_path({!r}, run_name='__worker__')"
+    AUTH_CODE = random.random()  # sort out unwanted output of worker process
+    RESTART = 5  # number of worker restarts before giving up
+    DONE = object()  # sentinel value to signal end-of-queue
+
+    def __init__(self, callback_query_result, callback_finished=None):
+        self.callback_query_result = callback_query_result
+        self.callback_finished = callback_finished or (lambda: None)
+
+        self._queue = Queue.Queue()
+        self._thread = None
+        self._worker = None
+        self._shutdown = threading.Event()
+        self._last_cmd = None
+
+    def start(self):
+        """Start the worker process handler thread"""
+        if self._thread is not None:
+            return
+        self._shutdown.clear()
+        thread = self._thread = threading.Thread(target=self.run_worker)
+        thread.daemon = True
+        thread.start()
+
+    def run_worker(self):
+        """Read docstring back from worker stdout and execute callback."""
+        for _ in range(self.RESTART):
+            if self._shutdown.is_set():
+                break
+            try:
+                self._worker = subprocess.Popen(
+                    args=(sys.executable, '-uc', self.BOOTSTRAP.format(__file__)),
+                    stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE
+                )
+                self._handle_worker()
+
+            except (OSError, IOError):
+                msg = "Warning: restarting the docstring loader"
+                cmd, args = self._last_cmd
+                if cmd == 'query':
+                    msg += " (crashed while loading {0!r})".format(args[0])
+                print >> sys.stderr, msg
+                continue  # restart
+            else:
+                break  # normal termination, return
+            finally:
+                self._worker.terminate()
+        else:
+            print >> sys.stderr, "Warning: docstring loader crashed too often"
+        self._thread = None
+        self._worker = None
+        self.callback_finished()
+
+    def _handle_worker(self):
+        """Send commands and responses back from worker."""
+        assert '1' == self._worker.stdout.read(1)
+        for cmd, args in iter(self._queue.get, self.DONE):
+            self._last_cmd = cmd, args
+            self._send(cmd, args)
+            cmd, args = self._receive()
+            self._handle_response(cmd, args)
+
+    def _send(self, cmd, args):
+        """send a command to worker's stdin"""
+        fd = self._worker.stdin
+        json.dump((self.AUTH_CODE, cmd, args), fd)
+        fd.write('\n'.encode())
+
+    def _receive(self):
+        """receive response from worker's stdout"""
+        for line in iter(self._worker.stdout.readline, ''):
+            try:
+                key, cmd, args = json.loads(line, encoding='utf-8')
+                if key != self.AUTH_CODE:
+                    raise ValueError('Got wrong auth code')
+                return cmd, args
+            except ValueError:
+                continue  # ignore invalid output from worker
+        else:
+            raise IOError("Can't read worker response")
+
+    def _handle_response(self, cmd, args):
+        """Handle response from worker, call the callback"""
+        if cmd == 'result':
+            key, docs = args
+            self.callback_query_result(key, docs)
+        elif cmd == 'error':
+            print args
+        else:
+            print >> sys.stderr, "Unknown response:", cmd, args
+
+    def query(self, key, imports=None, make=None):
+        """request docstring extraction for a certain key"""
+        if self._thread is None:
+            self.start()
+        if imports and make:
+            self._queue.put(('query', (key, imports, make)))
+        else:
+            self._queue.put(('query_key_only', (key,)))
+
+    def finish(self):
+        """signal end of requests"""
+        self._queue.put(self.DONE)
+
+    def wait(self):
+        """Wait for the handler thread to die"""
+        if self._thread:
+            self._thread.join()
+
+    def terminate(self):
+        """Terminate the worker and wait"""
+        self._shutdown.set()
+        try:
+            self._worker.terminate()
+            self.wait()
+        except (OSError, AttributeError):
+            pass
+
+
+###############################################################################
+# Main worker entry point
+###############################################################################
+
+def worker_main():
+    """Main entry point for the docstring extraction process.
+
+    Manages RPC with main process through.
+    Runs a docstring extraction for each key it read on stdin.
+    """
+    def send(cmd, args):
+        json.dump((code, cmd, args), sys.stdout)
+        sys.stdout.write('\n'.encode())
+
+    sys.stdout.write('1')
+    for line in iter(sys.stdin.readline, ''):
+        code, cmd, args = json.loads(line, encoding='utf-8')
+        try:
+            if cmd == 'query':
+                key, imports, make = args
+                send('result', (key, docstring_from_make(key, imports, make)))
+            elif cmd == 'query_key_only':
+                key, = args
+                send('result', (key, docstring_guess_from_key(key)))
+            elif cmd == 'exit':
+                break
+        except Exception as e:
+            send('error', repr(e))
+
+
+if __name__ == '__worker__':
+    worker_main()
+
+elif __name__ == '__main__':
+    def callback(key, docs):
+        print key
+        for match, doc in docs.iteritems():
+            print '-->', match
+            print doc.strip()
+            print
+        print
+
+    r = SubprocessLoader(callback)
+
+    # r.query('analog_feedforward_agc_cc')
+    # r.query('uhd_source')
+    r.query('expr_utils_graph')
+    r.query('blocks_add_cc')
+    r.query('blocks_add_cc', ['import gnuradio.blocks'], 'gnuradio.blocks.add_cc(')
+    # r.query('analog_feedforward_agc_cc')
+    # r.query('uhd_source')
+    # r.query('uhd_source')
+    # r.query('analog_feedforward_agc_cc')
+    r.finish()
+    # r.terminate()
+    r.wait()
author	Johnathan Corgan <johnathan@corganlabs.com>	2015-11-28 12:52:03 -0800
committer	Johnathan Corgan <johnathan@corganlabs.com>	2015-11-28 12:52:03 -0800
commit	ee06ad1eb9437d1db989076ca61ed0e4573244ec (patch)
tree	ebe51c26bf69183feefbd7cf919827be9274581d /grc/python/extract_docs.py
parent	27e8311d8b6bf88c8b46461abe6de8d222e2ee25 (diff)
parent	dcc09deb064453f4add1ece3502ee99f12f3cb25 (diff)