grc: move docstring extraction into subprocess

author: Sebastian Koslowski <koslowski@kit.edu> 2015-08-18 16:47:41 +0200
committer: Sebastian Koslowski <koslowski@kit.edu> 2015-11-13 21:47:58 +0100
commit: d15065de35a535eae9448dff60270053f35885bd (patch)
tree: f44d59c1ba0875eb0ddafdb4e81f78d3afec77e1 /grc
parent: f7add89403edcfdf8baa59c80b3e1d2b5c4ee307 (diff)
4 files changed, 233 insertions, 36 deletions
diff --git a/grc/base/Platform.py b/grc/base/Platform.py
index a72c21ad61..db2bb76888 100644
--- a/grc/base/Platform.py
+++ b/grc/base/Platform.py
@@ -127,6 +127,7 @@ class Platform(_Element):
         else:  # store the block
             self._blocks[key] = block
             self._blocks_n[key] = n
+        return block
 
     def load_category_tree_xml(self, xml_file):
         """Validate and parse category tree file and add it to list"""
diff --git a/grc/python/Block.py b/grc/python/Block.py
index f5f406449e..239352de1f 100644
--- a/grc/python/Block.py
+++ b/grc/python/Block.py
@@ -27,7 +27,7 @@ from .. base.Block import Block as _Block
 from .. gui.Block import Block as _GUIBlock
 
 from . FlowGraph import _variable_matcher
-from . import epy_block_io, extract_docs
+from . import epy_block_io
 
 
 class Block(_Block, _GUIBlock):
@@ -44,7 +44,7 @@ class Block(_Block, _GUIBlock):
             block a new block
         """
         #grab the data
-        self._doc = n.find('doc') or ''
+        self._doc = (n.find('doc') or '').strip('\n').replace('\\\n', '')
         self._imports = map(lambda i: i.strip(), n.findall('import'))
         self._make = n.find('make')
         self._var_make = n.find('var_make')
@@ -184,14 +184,14 @@ class Block(_Block, _GUIBlock):
         return changed
 
     def get_doc(self):
-        doc = self._doc.strip('\n').replace('\\\n', '')
-        #merge custom doc with doxygen docs
-        return '\n'.join([doc, extract_docs.extract(self.get_key())]).strip('\n')
+        platform = self.get_parent().get_parent()
+        extracted_docs = platform.block_docstrings.get(self._key, '')
+        return (self._doc + '\n\n' + extracted_docs).strip()
 
     def get_category(self):
         return _Block.get_category(self)
 
-    def get_imports(self):
+    def get_imports(self, raw=False):
         """
         Resolve all import statements.
         Split each import statement at newlines.
@@ -201,11 +201,20 @@ class Block(_Block, _GUIBlock):
         Returns:
             a list of import statements
         """
+        if raw:
+            return self._imports
         return filter(lambda i: i, sum(map(lambda i: self.resolve_dependencies(i).split('\n'), self._imports), []))
 
-    def get_make(self): return self.resolve_dependencies(self._make)
-    def get_var_make(self): return self.resolve_dependencies(self._var_make)
-    def get_var_value(self): return self.resolve_dependencies(self._var_value)
+    def get_make(self, raw=False):
+        if raw:
+            return self._make
+        return self.resolve_dependencies(self._make)
+
+    def get_var_make(self):
+        return self.resolve_dependencies(self._var_make)
+
+    def get_var_value(self):
+        return self.resolve_dependencies(self._var_value)
 
     def get_callbacks(self):
         """
diff --git a/grc/python/Platform.py b/grc/python/Platform.py
index 1497099f3f..56986773b5 100644
--- a/grc/python/Platform.py
+++ b/grc/python/Platform.py
@@ -24,22 +24,24 @@ from gnuradio import gr
 
 from .. base.Platform import Platform as _Platform
 from .. gui.Platform import Platform as _GUIPlatform
-from FlowGraph import FlowGraph as _FlowGraph
-from Connection import Connection as _Connection
-from Block import Block as _Block
-from Port import Port as _Port
-from Param import Param as _Param
-from Generator import Generator
-from Constants import (
+
+from . import extract_docs
+from .FlowGraph import FlowGraph as _FlowGraph
+from .Connection import Connection as _Connection
+from .Block import Block as _Block
+from .Port import Port as _Port
+from .Param import Param as _Param
+from .Generator import Generator
+from .Constants import (
     HIER_BLOCKS_LIB_DIR, BLOCK_DTD, DEFAULT_FLOW_GRAPH, BLOCKS_DIRS,
     PREFS_FILE, PREFS_FILE_OLD, CORE_TYPES
 )
 
-
 COLORS = [(name, color) for name, key, sizeof, color in CORE_TYPES]
 
 
 class Platform(_Platform, _GUIPlatform):
+
     def __init__(self):
         """
         Make a platform for gnuradio.
@@ -49,6 +51,17 @@ class Platform(_Platform, _GUIPlatform):
             os.mkdir(HIER_BLOCKS_LIB_DIR)
         if not os.path.exists(os.path.dirname(PREFS_FILE)):
             os.mkdir(os.path.dirname(PREFS_FILE))
+
+        self.block_docstrings = block_docstrings = dict()
+
+        def setter(key, docs):
+            block_docstrings[key] = '\n\n'.join(
+                '--- {0} ---\n{1}\n'.format(b, d.replace('\n\n', '\n'))
+                for b, d in docs.iteritems() if d is not None
+            )
+
+        self._docstring_extractor = extract_docs.SubprocessLoader(setter)
+
         # init
         _Platform.__init__(
             self,
@@ -80,6 +93,17 @@ class Platform(_Platform, _GUIPlatform):
             except Exception as e:
                 print >> sys.stderr, e
 
+    def load_blocks(self):
+        self._docstring_extractor.start()
+        _Platform.load_blocks(self)
+        self._docstring_extractor.finish()
+        self._docstring_extractor.wait()
+
+    def load_block_xml(self, xml_file):
+        block = _Platform.load_block_xml(self, xml_file)
+        self._docstring_extractor.query(block.get_key())
+        return block
+
     ##############################################
     # Constructors
     ##############################################
diff --git a/grc/python/extract_docs.py b/grc/python/extract_docs.py
index 1124459282..837d26ca6f 100644
--- a/grc/python/extract_docs.py
+++ b/grc/python/extract_docs.py
@@ -17,10 +17,20 @@ along with this program; if not, write to the Free Software
 Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
 """
 
+import sys
 import re
+import subprocess
+import threading
+import json
+import Queue
+import random
 import itertools
 
 
+###############################################################################
+# The docstring extraction
+###############################################################################
+
 def docstring_guess_from_key(key):
     """Extract the documentation from the python __doc__ strings
 
@@ -67,28 +77,181 @@ def docstring_guess_from_key(key):
     return doc_strings
 
 
-_docs_cache = dict()
+###############################################################################
+# Manage docstring extraction in separate process
+###############################################################################
 
+class SubprocessLoader(object):
+    """Start and manage docstring extraction process
 
-def extract(key):
+    Manages subprocess and handles RPC.
     """
-    Call the private extract and cache the result.
+    BOOTSTRAP = "import runpy; runpy.run_path({!r}, run_name='__worker__')"
+    AUTH_CODE = random.random()  # sort out unwanted output of worker process
+    RESTART = 5  # number of worker restarts before giving up
+    DONE = object()  # sentinel value to signal end-of-queue
 
-    Args:
-        key: the block key
+    def __init__(self, callback_query_result, callback_finished=None):
+        self.callback_query_result = callback_query_result
+        self.callback_finished = callback_finished or (lambda: None)
 
-    Returns:
-        a string with documentation
+        self._queue = Queue.Queue()
+        self._thread = None
+        self._worker = None
+        self._shutdown = threading.Event()
+        self._last_cmd = None
+
+    def start(self):
+        """Start the worker process handler thread"""
+        if self._thread is not None:
+            return
+        self._shutdown.clear()
+        thread = self._thread = threading.Thread(target=self.run_worker)
+        thread.daemon = True
+        thread.start()
+
+    def run_worker(self):
+        """Read docstring back from worker stdout and execute callback."""
+        for _ in range(self.RESTART):
+            if self._shutdown.is_set():
+                break
+            try:
+                self._worker = subprocess.Popen(
+                    args=(sys.executable, '-uc', self.BOOTSTRAP.format(__file__)),
+                    stdin=subprocess.PIPE, stdout=subprocess.PIPE,
+                    stderr=subprocess.PIPE
+                )
+                self._handle_worker()
+
+            except (OSError, IOError):
+                msg = "Warning: restarting the docstring loader"
+                cmd, args = self._last_cmd
+                if cmd == 'query':
+                    msg += " (crashed while loading {0!r})".format(args[0])
+                print >> sys.stderr, msg
+                continue  # restart
+            else:
+                break  # normal termination, return
+            finally:
+                self._worker.terminate()
+        else:
+            print >> sys.stderr, "Warning: docstring loader crashed too often"
+        self._thread = None
+        self._worker = None
+        self.callback_finished()
+
+    def _handle_worker(self):
+        """Send commands and responses back from worker."""
+        assert '1' == self._worker.stdout.read(1)
+        for cmd, args in iter(self._queue.get, self.DONE):
+            self._last_cmd = cmd, args
+            self._send(cmd, args)
+            cmd, args = self._receive()
+            self._handle_response(cmd, args)
+
+    def _send(self, cmd, args):
+        """send a command to worker's stdin"""
+        fd = self._worker.stdin
+        json.dump((self.AUTH_CODE, cmd, args), fd)
+        fd.write('\n'.encode())
+
+    def _receive(self):
+        """receive response from worker's stdout"""
+        for line in iter(self._worker.stdout.readline, ''):
+            try:
+                key, cmd, args = json.loads(line, encoding='utf-8')
+                if key != self.AUTH_CODE:
+                    raise ValueError('Got wrong auth code')
+                return cmd, args
+            except ValueError:
+                continue  # ignore invalid output from worker
+        else:
+            raise IOError("Can't read worker response")
+
+    def _handle_response(self, cmd, args):
+        """Handle response from worker, call the callback"""
+        if cmd == 'result':
+            key, docs = args
+            self.callback_query_result(key, docs)
+        elif cmd == 'error':
+            print args
+        else:
+            print >> sys.stderr, "Unknown response:", cmd, args
+
+    def query(self, key):
+        """request docstring extraction for a certain key"""
+        if self._thread is None:
+            self.start()
+        self._queue.put(('query', (key,)))
+
+    def finish(self):
+        """signal end of requests"""
+        self._queue.put(self.DONE)
+
+    def wait(self):
+        """Wait for the handler thread to die"""
+        if self._thread:
+            self._thread.join()
+
+    def terminate(self):
+        """Terminate the worker and wait"""
+        self._shutdown.set()
+        try:
+            self._worker.terminate()
+            self.wait()
+        except (OSError, AttributeError):
+            pass
+
+
+###############################################################################
+# Main worker entry point
+###############################################################################
+
+def worker_main():
+    """Main entry point for the docstring extraction process.
+
+    Manages RPC with main process through.
+    Runs a docstring extraction for each key it read on stdin.
     """
-    if not _docs_cache.has_key(key):
-        docstrings = docstring_guess_from_key(key)
-        _docs_cache[key] = '\n\n'.join(
-            '   ---   {0}   ---   \n\n{1}'.format(match, docstring)
-            for match, docstring in docstrings.iteritems()
-        )
-    return _docs_cache[key]
-
-
-if __name__ == '__main__':
-    import sys
-    print extract(sys.argv[1])
+    def send(cmd, args):
+        json.dump((code, cmd, args), sys.stdout)
+        sys.stdout.write('\n'.encode())
+
+    sys.stdout.write('1')
+    for line in iter(sys.stdin.readline, ''):
+        code, cmd, args = json.loads(line, encoding='utf-8')
+        try:
+            if cmd == 'query':
+                key, = args
+                send('result', (key, docstring_guess_from_key(key)))
+            elif cmd == 'exit':
+                break
+        except Exception as e:
+            send('error', repr(e))
+
+
+if __name__ == '__worker__':
+    worker_main()
+
+elif __name__ == '__main__':
+    def callback(key, docs):
+        print key
+        for match, doc in docs.iteritems():
+            print '-->', match
+            print doc.strip()
+            print
+        print
+
+    r = SubprocessLoader(callback)
+
+    # r.query('analog_feedforward_agc_cc')
+    # r.query('uhd_source')
+    r.query('expr_utils_graph')
+    r.query('blocks_add_cc')
+    # r.query('analog_feedforward_agc_cc')
+    # r.query('uhd_source')
+    # r.query('uhd_source')
+    # r.query('analog_feedforward_agc_cc')
+    r.finish()
+    # r.terminate()
+    r.wait()
author	Sebastian Koslowski <koslowski@kit.edu>	2015-08-18 16:47:41 +0200
committer	Sebastian Koslowski <koslowski@kit.edu>	2015-11-13 21:47:58 +0100
commit	d15065de35a535eae9448dff60270053f35885bd (patch)
tree	f44d59c1ba0875eb0ddafdb4e81f78d3afec77e1 /grc
parent	f7add89403edcfdf8baa59c80b3e1d2b5c4ee307 (diff)