10 files changed, 415 insertions, 77 deletions
diff --git a/grc/core/utils/__init__.py b/grc/core/utils/__init__.py
index d095179a10..2d12e280b5 100644
--- a/grc/core/utils/__init__.py
+++ b/grc/core/utils/__init__.py
@@ -1,4 +1,4 @@
-# Copyright 2008-2015 Free Software Foundation, Inc.
+# Copyright 2016 Free Software Foundation, Inc.
 # This file is part of GNU Radio
 #
 # GNU Radio Companion is free software; you can redistribute it and/or
@@ -17,8 +17,4 @@
 
 from __future__ import absolute_import
 
-from . import expr_utils
-from . import epy_block_io
-from . import extract_docs
-
-from ._complexity import calculate_flowgraph_complexity
+from . import epy_block_io, expr_utils, extract_docs, flow_graph_complexity
diff --git a/grc/core/utils/backports/__init__.py b/grc/core/utils/backports/__init__.py
new file mode 100644
index 0000000000..a24ee3ae01
--- /dev/null
+++ b/grc/core/utils/backports/__init__.py
@@ -0,0 +1,25 @@
+# Copyright 2016 Free Software Foundation, Inc.
+#
+# This file is part of GNU Radio
+#
+# GNU Radio is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; either version 3, or (at your option)
+# any later version.
+#
+# GNU Radio is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with GNU Radio; see the file COPYING.  If not, write to
+# the Free Software Foundation, Inc., 51 Franklin Street,
+# Boston, MA 02110-1301, USA.
+
+from __future__ import absolute_import
+
+try:
+    from collections import ChainMap
+except ImportError:
+    from .chainmap import ChainMap
diff --git a/grc/core/utils/backports/chainmap.py b/grc/core/utils/backports/chainmap.py
new file mode 100644
index 0000000000..1f4f4a96fb
--- /dev/null
+++ b/grc/core/utils/backports/chainmap.py
@@ -0,0 +1,106 @@
+# from https://hg.python.org/cpython/file/default/Lib/collections/__init__.py
+
+from collections import MutableMapping
+
+
+class ChainMap(MutableMapping):
+    """ A ChainMap groups multiple dicts (or other mappings) together
+    to create a single, updateable view.
+
+    The underlying mappings are stored in a list.  That list is public and can
+    be accessed or updated using the *maps* attribute.  There is no other
+    state.
+
+    Lookups search the underlying mappings successively until a key is found.
+    In contrast, writes, updates, and deletions only operate on the first
+    mapping.
+
+    """
+
+    def __init__(self, *maps):
+        """Initialize a ChainMap by setting *maps* to the given mappings.
+        If no mappings are provided, a single empty dictionary is used.
+
+        """
+        self.maps = list(maps) or [{}]          # always at least one map
+
+    def __missing__(self, key):
+        raise KeyError(key)
+
+    def __getitem__(self, key):
+        for mapping in self.maps:
+            try:
+                return mapping[key]             # can't use 'key in mapping' with defaultdict
+            except KeyError:
+                pass
+        return self.__missing__(key)            # support subclasses that define __missing__
+
+    def get(self, key, default=None):
+        return self[key] if key in self else default
+
+    def __len__(self):
+        return len(set().union(*self.maps))     # reuses stored hash values if possible
+
+    def __iter__(self):
+        return iter(set().union(*self.maps))
+
+    def __contains__(self, key):
+        return any(key in m for m in self.maps)
+
+    def __bool__(self):
+        return any(self.maps)
+
+    def __repr__(self):
+        return '{0.__class__.__name__}({1})'.format(
+            self, ', '.join(map(repr, self.maps)))
+
+    @classmethod
+    def fromkeys(cls, iterable, *args):
+        """Create a ChainMap with a single dict created from the iterable."""
+        return cls(dict.fromkeys(iterable, *args))
+
+    def copy(self):
+        """New ChainMap or subclass with a new copy of maps[0] and refs to maps[1:]"""
+        return self.__class__(self.maps[0].copy(), *self.maps[1:])
+
+    __copy__ = copy
+
+    def new_child(self, m=None):                # like Django's Context.push()
+        """New ChainMap with a new map followed by all previous maps.
+        If no map is provided, an empty dict is used.
+        """
+        if m is None:
+            m = {}
+        return self.__class__(m, *self.maps)
+
+    @property
+    def parents(self):                          # like Django's Context.pop()
+        """New ChainMap from maps[1:]."""
+        return self.__class__(*self.maps[1:])
+
+    def __setitem__(self, key, value):
+        self.maps[0][key] = value
+
+    def __delitem__(self, key):
+        try:
+            del self.maps[0][key]
+        except KeyError:
+            raise KeyError('Key not found in the first mapping: {!r}'.format(key))
+
+    def popitem(self):
+        """Remove and return an item pair from maps[0]. Raise KeyError is maps[0] is empty."""
+        try:
+            return self.maps[0].popitem()
+        except KeyError:
+            raise KeyError('No keys found in the first mapping.')
+
+    def pop(self, key, *args):
+        """Remove *key* from maps[0] and return its value. Raise KeyError if *key* not in maps[0]."""
+        try:
+            return self.maps[0].pop(key, *args)
+        except KeyError:
+            raise KeyError('Key not found in the first mapping: {!r}'.format(key))
+
+    def clear(self):
+        """Clear maps[0], leaving maps[1:] intact."""
+        self.maps[0].clear()
diff --git a/grc/core/utils/shlex.py b/grc/core/utils/backports/shlex.py
index 6b620fa396..6b620fa396 100644
--- a/grc/core/utils/shlex.py
+++ b/grc/core/utils/backports/shlex.py
diff --git a/grc/core/utils/descriptors/__init__.py b/grc/core/utils/descriptors/__init__.py
new file mode 100644
index 0000000000..80c5259230
--- /dev/null
+++ b/grc/core/utils/descriptors/__init__.py
@@ -0,0 +1,26 @@
+# Copyright 2016 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+from ._lazy import lazy_property, nop_write
+
+from .evaluated import (
+    Evaluated,
+    EvaluatedEnum,
+    EvaluatedPInt,
+    EvaluatedFlag,
+    setup_names,
+)
diff --git a/grc/core/utils/descriptors/_lazy.py b/grc/core/utils/descriptors/_lazy.py
new file mode 100644
index 0000000000..a0cb126932
--- /dev/null
+++ b/grc/core/utils/descriptors/_lazy.py
@@ -0,0 +1,39 @@
+# Copyright 2016 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+import functools
+
+
+class lazy_property(object):
+
+    def __init__(self, func):
+        self.func = func
+        functools.update_wrapper(self, func)
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+        value = self.func(instance)
+        setattr(instance, self.func.__name__, value)
+        return value
+
+
+def nop_write(prop):
+    """Make this a property with a nop setter"""
+    def nop(self, value):
+        pass
+    return prop.setter(nop)
diff --git a/grc/core/utils/descriptors/evaluated.py b/grc/core/utils/descriptors/evaluated.py
new file mode 100644
index 0000000000..313cee5b96
--- /dev/null
+++ b/grc/core/utils/descriptors/evaluated.py
@@ -0,0 +1,112 @@
+# Copyright 2016 Free Software Foundation, Inc.
+# This file is part of GNU Radio
+#
+# GNU Radio Companion is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# GNU Radio Companion is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA
+
+
+class Evaluated(object):
+    def __init__(self, expected_type, default, name=None):
+        self.expected_type = expected_type
+        self.default = default
+
+        self.name = name or 'evaled_property_{}'.format(id(self))
+        self.eval_function = self.default_eval_func
+
+    @property
+    def name_raw(self):
+        return '_' + self.name
+
+    def default_eval_func(self, instance):
+        raw = getattr(instance, self.name_raw)
+        try:
+            value = instance.parent_block.evaluate(raw)
+        except Exception as error:
+            if raw:
+                instance.add_error_message("Failed to eval '{}': {}".format(raw, error))
+            return self.default
+
+        if not isinstance(value, self.expected_type):
+            instance.add_error_message("Can not cast evaluated value '{}' to type {}"
+                                       "".format(value, self.expected_type))
+            return self.default
+        # print(instance, self.name, raw, value)
+        return value
+
+    def __call__(self, func):
+        self.name = func.__name__
+        self.eval_function = func
+        return self
+
+    def __get__(self, instance, owner):
+        if instance is None:
+            return self
+        attribs = instance.__dict__
+        try:
+            value = attribs[self.name]
+        except KeyError:
+            value = attribs[self.name] = self.eval_function(instance)
+        return value
+
+    def __set__(self, instance, value):
+        attribs = instance.__dict__
+        value = value or self.default
+        if isinstance(value, str) and value.startswith('${') and value.endswith('}'):
+            attribs[self.name_raw] = value[2:-1].strip()
+        else:
+            attribs[self.name] = type(self.default)(value)
+
+    def __delete__(self, instance):
+        attribs = instance.__dict__
+        if self.name_raw in attribs:
+            attribs.pop(self.name, None)
+
+
+class EvaluatedEnum(Evaluated):
+    def __init__(self, allowed_values, default=None, name=None):
+        self.allowed_values = allowed_values if isinstance(allowed_values, (list, tuple)) else \
+            allowed_values.split()
+        default = default if default is not None else self.allowed_values[0]
+        super(EvaluatedEnum, self).__init__(str, default, name)
+
+    def default_eval_func(self, instance):
+        value = super(EvaluatedEnum, self).default_eval_func(instance)
+        if value not in self.allowed_values:
+            instance.add_error_message("Value '{}' not in allowed values".format(value))
+            return self.default
+        return value
+
+
+class EvaluatedPInt(Evaluated):
+    def __init__(self, name=None):
+        super(EvaluatedPInt, self).__init__(int, 1, name)
+
+    def default_eval_func(self, instance):
+        value = super(EvaluatedPInt, self).default_eval_func(instance)
+        if value < 1:
+            # todo: log
+            return self.default
+        return value
+
+
+class EvaluatedFlag(Evaluated):
+    def __init__(self, name=None):
+        super(EvaluatedFlag, self).__init__((bool, int), False, name)
+
+
+def setup_names(cls):
+    for name, attrib in cls.__dict__.items():
+        if isinstance(attrib, Evaluated):
+            attrib.name = name
+    return cls
diff --git a/grc/core/utils/expr_utils.py b/grc/core/utils/expr_utils.py
index cc03e9cb1c..427585e93c 100644
--- a/grc/core/utils/expr_utils.py
+++ b/grc/core/utils/expr_utils.py
@@ -23,17 +23,105 @@ import string
 
 import six
 
+
+def expr_replace(expr, replace_dict):
+    """
+    Search for vars in the expression and add the prepend.
+
+    Args:
+        expr: an expression string
+        replace_dict: a dict of find:replace
+
+    Returns:
+        a new expression with the prepend
+    """
+    expr_splits = _expr_split(expr, var_chars=VAR_CHARS + '.')
+    for i, es in enumerate(expr_splits):
+        if es in list(replace_dict.keys()):
+            expr_splits[i] = replace_dict[es]
+    return ''.join(expr_splits)
+
+
+def get_variable_dependencies(expr, vars):
+    """
+    Return a set of variables used in this expression.
+
+    Args:
+        expr: an expression string
+        vars: a list of variable names
+
+    Returns:
+        a subset of vars used in the expression
+    """
+    expr_toks = _expr_split(expr)
+    return set(v for v in vars if v in expr_toks)
+
+
+def sort_objects(objects, get_id, get_expr):
+    """
+    Sort a list of objects according to their expressions.
+
+    Args:
+        objects: the list of objects to sort
+        get_id: the function to extract an id from the object
+        get_expr: the function to extract an expression from the object
+
+    Returns:
+        a list of sorted objects
+    """
+    id2obj = {get_id(obj): obj for obj in objects}
+    # Map obj id to expression code
+    id2expr = {get_id(obj): get_expr(obj) for obj in objects}
+    # Sort according to dependency
+    sorted_ids = _sort_variables(id2expr)
+    # Return list of sorted objects
+    return [id2obj[id] for id in sorted_ids]
+
+
+import ast
+
+
+def dependencies(expr, names=None):
+    node = ast.parse(expr, mode='eval')
+    used_ids = frozenset([n.id for n in ast.walk(node) if isinstance(n, ast.Name)])
+    return used_ids & names if names else used_ids
+
+
+def sort_objects2(objects, id_getter, expr_getter, check_circular=True):
+    known_ids = {id_getter(obj) for obj in objects}
+
+    def dependent_ids(obj):
+        deps = dependencies(expr_getter(obj))
+        return [id_ if id_ in deps else None for id_ in known_ids]
+
+    objects = sorted(objects, key=dependent_ids)
+
+    if check_circular:  # walk var defines step by step
+        defined_ids = set()  # variables defined so far
+        for obj in objects:
+            deps = dependencies(expr_getter(obj), known_ids)
+            if not defined_ids.issuperset(deps):  # can't have an undefined dep
+                raise RuntimeError(obj, deps, defined_ids)
+            defined_ids.add(id_getter(obj))  # define this one
+
+    return objects
+
+
+
+
 VAR_CHARS = string.ascii_letters + string.digits + '_'
 
 
-class graph(object):
+class _graph(object):
     """
     Simple graph structure held in a dictionary.
     """
 
-    def __init__(self): self._graph = dict()
+    def __init__(self):
+        self._graph = dict()
 
-    def __str__(self): return str(self._graph)
+    def __str__(self):
+        return str(self._graph)
 
     def add_node(self, node_key):
         if node_key in self._graph:
@@ -61,7 +149,7 @@ class graph(object):
         return self._graph[node_key]
 
 
-def expr_split(expr, var_chars=VAR_CHARS):
+def _expr_split(expr, var_chars=VAR_CHARS):
     """
     Split up an expression by non alphanumeric characters, including underscore.
     Leave strings in-tact.
@@ -93,40 +181,7 @@ def expr_split(expr, var_chars=VAR_CHARS):
     return [t for t in toks if t]
 
 
-def expr_replace(expr, replace_dict):
-    """
-    Search for vars in the expression and add the prepend.
-
-    Args:
-        expr: an expression string
-        replace_dict: a dict of find:replace
-
-    Returns:
-        a new expression with the prepend
-    """
-    expr_splits = expr_split(expr, var_chars=VAR_CHARS + '.')
-    for i, es in enumerate(expr_splits):
-        if es in list(replace_dict.keys()):
-            expr_splits[i] = replace_dict[es]
-    return ''.join(expr_splits)
-
-
-def get_variable_dependencies(expr, vars):
-    """
-    Return a set of variables used in this expression.
-
-    Args:
-        expr: an expression string
-        vars: a list of variable names
-
-    Returns:
-        a subset of vars used in the expression
-    """
-    expr_toks = expr_split(expr)
-    return set(v for v in vars if v in expr_toks)
-
-
-def get_graph(exprs):
+def _get_graph(exprs):
     """
     Get a graph representing the variable dependencies
 
@@ -138,7 +193,7 @@ def get_graph(exprs):
     """
     vars = list(exprs.keys())
     # Get dependencies for each expression, load into graph
-    var_graph = graph()
+    var_graph = _graph()
     for var in vars:
         var_graph.add_node(var)
     for var, expr in six.iteritems(exprs):
@@ -148,7 +203,7 @@ def get_graph(exprs):
     return var_graph
 
 
-def sort_variables(exprs):
+def _sort_variables(exprs):
     """
     Get a list of variables in order of dependencies.
 
@@ -159,7 +214,7 @@ def sort_variables(exprs):
         a list of variable names
     @throws Exception circular dependencies
     """
-    var_graph = get_graph(exprs)
+    var_graph = _get_graph(exprs)
     sorted_vars = list()
     # Determine dependency order
     while var_graph.get_nodes():
@@ -173,24 +228,3 @@ def sort_variables(exprs):
         for var in indep_vars:
             var_graph.remove_node(var)
     return reversed(sorted_vars)
-
-
-def sort_objects(objects, get_id, get_expr):
-    """
-    Sort a list of objects according to their expressions.
-
-    Args:
-        objects: the list of objects to sort
-        get_id: the function to extract an id from the object
-        get_expr: the function to extract an expression from the object
-
-    Returns:
-        a list of sorted objects
-    """
-    id2obj = dict([(get_id(obj), obj) for obj in objects])
-    # Map obj id to expression code
-    id2expr = dict([(get_id(obj), get_expr(obj)) for obj in objects])
-    # Sort according to dependency
-    sorted_ids = sort_variables(id2expr)
-    # Return list of sorted objects
-    return [id2obj[id] for id in sorted_ids]
diff --git a/grc/core/utils/extract_docs.py b/grc/core/utils/extract_docs.py
index cff8a81099..7688f98de5 100644
--- a/grc/core/utils/extract_docs.py
+++ b/grc/core/utils/extract_docs.py
@@ -98,8 +98,7 @@ def docstring_from_make(key, imports, make):
         if '$' in blk_cls:
             raise ValueError('Not an identifier')
         ns = dict()
-        for _import in imports:
-            exec(_import.strip(), ns)
+        exec(imports.strip(), ns)
         blk = eval(blk_cls, ns)
         doc_strings = {key: blk.__doc__}
 
@@ -166,7 +165,8 @@ class SubprocessLoader(object):
             else:
                 break  # normal termination, return
             finally:
-                self._worker.terminate()
+                if self._worker:
+                    self._worker.terminate()
         else:
             print("Warning: docstring loader crashed too often", file=sys.stderr)
         self._thread = None
@@ -277,7 +277,7 @@ elif __name__ == '__main__':
         print(key)
         for match, doc in six.iteritems(docs):
             print('-->', match)
-            print(doc.strip())
+            print(str(doc).strip())
             print()
         print()
 
diff --git a/grc/core/utils/_complexity.py b/grc/core/utils/flow_graph_complexity.py
index c0f3ae9de4..d06f04ab5f 100644
--- a/grc/core/utils/_complexity.py
+++ b/grc/core/utils/flow_graph_complexity.py
@@ -1,5 +1,5 @@
 
-def calculate_flowgraph_complexity(flowgraph):
+def calculate(flowgraph):
     """ Determines the complexity of a flowgraph """
     dbal = 0
     for block in flowgraph.blocks:
@@ -8,8 +8,8 @@ def calculate_flowgraph_complexity(flowgraph):
             continue
 
         # Don't worry about optional sinks?
-        sink_list = [c for c in block.sinks if not c.get_optional()]
-        source_list = [c for c in block.sources if not c.get_optional()]
+        sink_list = [c for c in block.sinks if not c.optional]
+        source_list = [c for c in block.sources if not c.optional]
         sinks = float(len(sink_list))
         sources = float(len(source_list))
         base = max(min(sinks, sources), 1)
@@ -22,8 +22,8 @@ def calculate_flowgraph_complexity(flowgraph):
             multi = 1
 
         # Connection ratio multiplier
-        sink_multi = max(float(sum(len(c.get_connections()) for c in sink_list) / max(sinks, 1.0)), 1.0)
-        source_multi = max(float(sum(len(c.get_connections()) for c in source_list) / max(sources, 1.0)), 1.0)
+        sink_multi = max(float(sum(len(c.connections()) for c in sink_list) / max(sinks, 1.0)), 1.0)
+        source_multi = max(float(sum(len(c.connections()) for c in source_list) / max(sources, 1.0)), 1.0)
         dbal += base * multi * sink_multi * source_multi
 
     blocks = float(len(flowgraph.blocks))