grc: move docstring extraction into subprocess

author: Sebastian Koslowski <koslowski@kit.edu> 2015-08-18 16:47:41 +0200
committer: Sebastian Koslowski <koslowski@kit.edu> 2015-11-13 21:47:58 +0100
commit: d15065de35a535eae9448dff60270053f35885bd (patch)
tree: f44d59c1ba0875eb0ddafdb4e81f78d3afec77e1 /grc/python/extract_docs.py
parent: f7add89403edcfdf8baa59c80b3e1d2b5c4ee307 (diff)
1 files changed, 182 insertions, 19 deletions
diff --git a/grc/python/extract_docs.py b/grc/python/extract_docs.py
index 1124459282..837d26ca6f 100644
--- a/grc/python/extract_docs.py
+++ b/grc/python/extract_docs.py
@@ -17,10 +17,20 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 """
 
+import sys
 import re
+import subprocess
+import threading
+import json
+import Queue
+import random
 import itertools
 
 
+###############################################################################
+# The docstring extraction
+###############################################################################
+
 def docstring_guess_from_key(key):
     """Extract the documentation from the python __doc__ strings
 
@@ -67,28 +77,181 @@ def docstring_guess_from_key(key):
     return doc_strings
 
 
-_docs_cache = dict()
+###############################################################################
+# Manage docstring extraction in separate process
+###############################################################################
 
+class SubprocessLoader(object):
+    """Start and manage docstring extraction process
 
-def extract(key):
+    Manages subprocess and handles RPC.
     """
-    Call the private extract and cache the result.
+    BOOTSTRAP = "import runpy; runpy.run_path({!r}, run_name='__worker__')"
+    AUTH_CODE = random.random()  # sort out unwanted output of worker process
+    RESTART = 5  # number of worker restarts before giving up
+    DONE = object()  # sentinel value to signal end-of-queue
 
-    Args:
-        key: the block key
+    def __init__(self, callback_query_result, callback_finished=None):
+        self.callback_query_result = callback_query_result
+        self.callback_finished = callback_finished or (lambda: None)
 
-    Returns:
-        a string with documentation
+        self._queue = Queue.Queue()
+        self._thread = None
+        self._worker = None
+        self._shutdown = threading.Event()
+        self._last_cmd = None
+
+    def start(self):
+        """Start the worker process handler thread"""
+        if self._thread is not None:
+            return
+        self._shutdown.clear()
+        thread = self._thread = threading.Thread(target=self.run_worker)
+        thread.daemon = True
+        thread.start()
+
+    def run_worker(self):
+        """Read docstring back from worker stdout and execute callback."""
+        for _ in range(self.RESTART):
+            if self._shutdown.is_set():
+                break
+            try:
+                self._worker = subprocess.Popen(
+                    args=(sys.executable, '-uc', self.BOOTSTRAP.format(__file__)),
+                    stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE
+                )
+                self._handle_worker()
+
+            except (OSError, IOError):
+                msg = "Warning: restarting the docstring loader"
+                cmd, args = self._last_cmd
+                if cmd == 'query':
+                    msg += " (crashed while loading {0!r})".format(args[0])
+                print >> sys.stderr, msg
+                continue  # restart
+            else:
+                break  # normal termination, return
+            finally:
+                self._worker.terminate()
+        else:
+            print >> sys.stderr, "Warning: docstring loader crashed too often"
+        self._thread = None
+        self._worker = None
+        self.callback_finished()
+
+    def _handle_worker(self):
+        """Send commands and responses back from worker."""
+        assert '1' == self._worker.stdout.read(1)
+        for cmd, args in iter(self._queue.get, self.DONE):
+            self._last_cmd = cmd, args
+            self._send(cmd, args)
+            cmd, args = self._receive()
+            self._handle_response(cmd, args)
+
+    def _send(self, cmd, args):
+        """send a command to worker's stdin"""
+        fd = self._worker.stdin
+        json.dump((self.AUTH_CODE, cmd, args), fd)
+        fd.write('\n'.encode())
+
+    def _receive(self):
+        """receive response from worker's stdout"""
+        for line in iter(self._worker.stdout.readline, ''):
+            try:
+                key, cmd, args = json.loads(line, encoding='utf-8')
+                if key != self.AUTH_CODE:
+                    raise ValueError('Got wrong auth code')
+                return cmd, args
+            except ValueError:
+                continue  # ignore invalid output from worker
+        else:
+            raise IOError("Can't read worker response")
+
+    def _handle_response(self, cmd, args):
+        """Handle response from worker, call the callback"""
+        if cmd == 'result':
+            key, docs = args
+            self.callback_query_result(key, docs)
+        elif cmd == 'error':
+            print args
+        else:
+            print >> sys.stderr, "Unknown response:", cmd, args
+
+    def query(self, key):
+        """request docstring extraction for a certain key"""
+        if self._thread is None:
+            self.start()
+        self._queue.put(('query', (key,)))
+
+    def finish(self):
+        """signal end of requests"""
+        self._queue.put(self.DONE)
+
+    def wait(self):
+        """Wait for the handler thread to die"""
+        if self._thread:
+            self._thread.join()
+
+    def terminate(self):
+        """Terminate the worker and wait"""
+        self._shutdown.set()
+        try:
+            self._worker.terminate()
+            self.wait()
+        except (OSError, AttributeError):
+            pass
+
+
+###############################################################################
+# Main worker entry point
+###############################################################################
+
+def worker_main():
+    """Main entry point for the docstring extraction process.
+
+    Manages RPC with main process through.
+    Runs a docstring extraction for each key it read on stdin.
     """
-    if not _docs_cache.has_key(key):
-        docstrings = docstring_guess_from_key(key)
-        _docs_cache[key] = '\n\n'.join(
-            '   ---   {0}   ---   \n\n{1}'.format(match, docstring)
-            for match, docstring in docstrings.iteritems()
-        )
-    return _docs_cache[key]
-
-
-if __name__ == '__main__':
-    import sys
-    print extract(sys.argv[1])
+    def send(cmd, args):
+        json.dump((code, cmd, args), sys.stdout)
+        sys.stdout.write('\n'.encode())
+
+    sys.stdout.write('1')
+    for line in iter(sys.stdin.readline, ''):
+        code, cmd, args = json.loads(line, encoding='utf-8')
+        try:
+            if cmd == 'query':
+                key, = args
+                send('result', (key, docstring_guess_from_key(key)))
+            elif cmd == 'exit':
+                break
+        except Exception as e:
+            send('error', repr(e))
+
+
+if __name__ == '__worker__':
+    worker_main()
+
+elif __name__ == '__main__':
+    def callback(key, docs):
+        print key
+        for match, doc in docs.iteritems():
+            print '-->', match
+            print doc.strip()
+            print
+        print
+
+    r = SubprocessLoader(callback)
+
+    # r.query('analog_feedforward_agc_cc')
+    # r.query('uhd_source')
+    r.query('expr_utils_graph')
+    r.query('blocks_add_cc')
+    # r.query('analog_feedforward_agc_cc')
+    # r.query('uhd_source')
+    # r.query('uhd_source')
+    # r.query('analog_feedforward_agc_cc')
+    r.finish()
+    # r.terminate()
+    r.wait()
author	Sebastian Koslowski <koslowski@kit.edu>	2015-08-18 16:47:41 +0200
committer	Sebastian Koslowski <koslowski@kit.edu>	2015-11-13 21:47:58 +0100
commit	d15065de35a535eae9448dff60270053f35885bd (patch)
tree	f44d59c1ba0875eb0ddafdb4e81f78d3afec77e1 /grc/python/extract_docs.py
parent	f7add89403edcfdf8baa59c80b3e1d2b5c4ee307 (diff)