diff options
Diffstat (limited to 'grc/core')
-rw-r--r-- | grc/core/Config.py | 6 | ||||
-rw-r--r-- | grc/core/Constants.py | 22 | ||||
-rw-r--r-- | grc/core/FlowGraph.py | 2 | ||||
-rw-r--r-- | grc/core/Param.py | 413 | ||||
-rw-r--r-- | grc/core/blocks/__init__.py | 1 | ||||
-rw-r--r-- | grc/core/blocks/_build.py | 86 | ||||
-rw-r--r-- | grc/core/blocks/_flags.py | 16 | ||||
-rw-r--r-- | grc/core/blocks/block.py | 82 | ||||
-rw-r--r-- | grc/core/cache.py | 99 | ||||
-rw-r--r-- | grc/core/default_flow_graph.grc | 1 | ||||
-rw-r--r-- | grc/core/generator/hier_block.py | 4 | ||||
-rw-r--r-- | grc/core/params/__init__.py | 18 | ||||
-rw-r--r-- | grc/core/params/dtypes.py | 103 | ||||
-rw-r--r-- | grc/core/params/param.py | 407 | ||||
-rw-r--r-- | grc/core/params/template_arg.py | 50 | ||||
-rw-r--r-- | grc/core/platform.py | 114 | ||||
-rw-r--r-- | grc/core/schema_checker/block.py | 2 | ||||
-rw-r--r-- | grc/core/utils/__init__.py | 12 | ||||
-rw-r--r-- | grc/core/utils/descriptors/evaluated.py | 13 |
19 files changed, 865 insertions, 586 deletions
diff --git a/grc/core/Config.py b/grc/core/Config.py index eb53e1751d..4accb74c63 100644 --- a/grc/core/Config.py +++ b/grc/core/Config.py @@ -31,8 +31,6 @@ class Config(object): hier_block_lib_dir = os.environ.get('GRC_HIER_PATH', Constants.DEFAULT_HIER_BLOCK_LIB_DIR) - yml_block_cache = os.path.expanduser('~/.cache/grc_gnuradio') # FIXME: remove this as soon as converter is stable - def __init__(self, version, version_parts=None, name=None, prefs=None): self._gr_prefs = prefs if prefs else DummyPrefs() self.version = version @@ -40,9 +38,6 @@ class Config(object): if name: self.name = name - if not os.path.exists(self.yml_block_cache): - os.mkdir(self.yml_block_cache) - @property def block_paths(self): path_list_sep = {'/': ':', '\\': ';'}[os.path.sep] @@ -50,7 +45,6 @@ class Config(object): paths_sources = ( self.hier_block_lib_dir, os.environ.get('GRC_BLOCKS_PATH', ''), - self.yml_block_cache, self._gr_prefs.get_string('grc', 'local_blocks_path', ''), self._gr_prefs.get_string('grc', 'global_blocks_path', ''), ) diff --git a/grc/core/Constants.py b/grc/core/Constants.py index fc5383378c..8ed8899c70 100644 --- a/grc/core/Constants.py +++ b/grc/core/Constants.py @@ -20,6 +20,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA from __future__ import absolute_import import os +import numbers import stat import numpy @@ -31,6 +32,8 @@ BLOCK_DTD = os.path.join(DATA_DIR, 'block.dtd') DEFAULT_FLOW_GRAPH = os.path.join(DATA_DIR, 'default_flow_graph.grc') DEFAULT_HIER_BLOCK_LIB_DIR = os.path.expanduser('~/.grc_gnuradio') +CACHE_FILE = os.path.expanduser('~/.cache/grc_gnuradio/cache.json') + BLOCK_DESCRIPTION_FILE_FORMAT_VERSION = 1 # File format versions: # 0: undefined / legacy @@ -52,7 +55,7 @@ TOP_BLOCK_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP stat.S_IWGRP | stat.S_IXGRP | stat.S_IROTH HIER_BLOCK_FILE_MODE = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP | stat.S_IROTH -PARAM_TYPE_NAMES = ( +PARAM_TYPE_NAMES = { 'raw', 'enum', 'complex', 'real', 'float', 'int', 'complex_vector', 'real_vector', 'float_vector', 'int_vector', @@ -61,18 +64,17 @@ PARAM_TYPE_NAMES = ( 'id', 'stream_id', 'gui_hint', 'import', -) +} + +PARAM_TYPE_MAP = { + 'complex': numbers.Complex, + 'float': numbers.Real, + 'real': numbers.Real, + 'int': numbers.Integral, +} # Define types, native python + numpy VECTOR_TYPES = (tuple, list, set, numpy.ndarray) -COMPLEX_TYPES = [complex, numpy.complex, numpy.complex64, numpy.complex128] -REAL_TYPES = [float, numpy.float, numpy.float32, numpy.float64] -INT_TYPES = [int, numpy.int, numpy.int8, numpy.int16, numpy.int32, numpy.uint64, - numpy.uint, numpy.uint8, numpy.uint16, numpy.uint32, numpy.uint64] -# Cast to tuple for isinstance, concat subtypes -COMPLEX_TYPES = tuple(COMPLEX_TYPES + REAL_TYPES + INT_TYPES) -REAL_TYPES = tuple(REAL_TYPES + INT_TYPES) -INT_TYPES = tuple(INT_TYPES) # Updating colors. Using the standard color palette from: # http://www.google.com/design/spec/style/color.html#color-color-palette diff --git a/grc/core/FlowGraph.py b/grc/core/FlowGraph.py index 3f21ec6a9c..8c59ec0bea 100644 --- a/grc/core/FlowGraph.py +++ b/grc/core/FlowGraph.py @@ -172,7 +172,7 @@ class FlowGraph(Element): return elements def children(self): - return itertools.chain(self.blocks, self.connections) + return itertools.chain(self.iter_enabled_blocks(), self.connections) def rewrite(self): """ diff --git a/grc/core/Param.py b/grc/core/Param.py deleted file mode 100644 index 56855908ea..0000000000 --- a/grc/core/Param.py +++ /dev/null @@ -1,413 +0,0 @@ -""" -Copyright 2008-2015 Free Software Foundation, Inc. -This file is part of GNU Radio - -GNU Radio Companion is free software; you can redistribute it and/or -modify it under the terms of the GNU General Public License -as published by the Free Software Foundation; either version 2 -of the License, or (at your option) any later version. - -GNU Radio Companion is distributed in the hope that it will be useful, -but WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -GNU General Public License for more details. - -You should have received a copy of the GNU General Public License -along with this program; if not, write to the Free Software -Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA -""" - -from __future__ import absolute_import - -import ast -import numbers -import re -import collections - -import six -from six.moves import builtins, filter, map, range, zip - -from . import Constants, blocks -from .base import Element -from .utils.descriptors import Evaluated, EvaluatedEnum, setup_names - -# Blacklist certain ids, its not complete, but should help -ID_BLACKLIST = ['self', 'options', 'gr', 'math', 'firdes'] + dir(builtins) -try: - from gnuradio import gr - ID_BLACKLIST.extend(attr for attr in dir(gr.top_block()) if not attr.startswith('_')) -except (ImportError, AttributeError): - pass - - -class TemplateArg(str): - """ - A cheetah template argument created from a param. - The str of this class evaluates to the param's to code method. - The use of this class as a dictionary (enum only) will reveal the enum opts. - The __call__ or () method can return the param evaluated to a raw python data type. - """ - - def __new__(cls, param): - value = param.to_code() - instance = str.__new__(cls, value) - setattr(instance, '_param', param) - return instance - - def __getitem__(self, item): - return str(self._param.get_opt(item)) if self._param.is_enum() else NotImplemented - - def __getattr__(self, item): - if not self._param.is_enum(): - raise AttributeError() - try: - return str(self._param.get_opt(item)) - except KeyError: - raise AttributeError() - - def __str__(self): - return str(self._param.to_code()) - - def __call__(self): - return self._param.get_evaluated() - - -@setup_names -class Param(Element): - - is_param = True - - name = Evaluated(str, default='no name') - dtype = EvaluatedEnum(Constants.PARAM_TYPE_NAMES, default='raw') - hide = EvaluatedEnum('none all part') - - # region init - def __init__(self, parent, id, label='', dtype='raw', default='', - options=None, option_labels=None, option_attributes=None, - category='', hide='none', **_): - """Make a new param from nested data""" - super(Param, self).__init__(parent) - self.key = id - self.name = label.strip() or id.title() - self.category = category or Constants.DEFAULT_PARAM_TAB - - self.dtype = dtype - self.value = self.default = str(default) - - self.options = self._init_options(options or [], option_labels or [], - option_attributes or {}) - self.hide = hide or 'none' - # end of args ######################################################## - - self._evaluated = None - self._stringify_flag = False - self._lisitify_flag = False - self._init = False - - @property - def template_arg(self): - return TemplateArg(self) - - def _init_options(self, values, labels, attributes): - """parse option and option attributes""" - options = collections.OrderedDict() - options.attributes = collections.defaultdict(dict) - - padding = [''] * max(len(values), len(labels)) - attributes = {key: value + padding for key, value in six.iteritems(attributes)} - - for i, option in enumerate(values): - # Test against repeated keys - if option in options: - raise KeyError('Value "{}" already exists in options'.format(option)) - # get label - try: - label = str(labels[i]) - except IndexError: - label = str(option) - # Store the option - options[option] = label - options.attributes[option] = {attrib: values[i] for attrib, values in six.iteritems(attributes)} - - default = next(iter(options)) if options else '' - if not self.value: - self.value = self.default = default - - if self.is_enum() and self.value not in options: - self.value = self.default = default # TODO: warn - # raise ValueError('The value {!r} is not in the possible values of {}.' - # ''.format(self.get_value(), ', '.join(self.options))) - return options - # endregion - - def __str__(self): - return 'Param - {}({})'.format(self.name, self.key) - - def __repr__(self): - return '{!r}.param[{}]'.format(self.parent, self.key) - - def is_enum(self): - return self.get_raw('dtype') == 'enum' - - def get_value(self): - value = self.value - if self.is_enum() and value not in self.options: - value = self.default - self.set_value(value) - return value - - def set_value(self, value): - # Must be a string - self.value = str(value) - - def set_default(self, value): - if self.default == self.value: - self.set_value(value) - self.default = str(value) - - def rewrite(self): - Element.rewrite(self) - del self.name - del self.dtype - del self.hide - - self._evaluated = None - try: - self._evaluated = self.evaluate() - except Exception as e: - self.add_error_message(str(e)) - - def validate(self): - """ - Validate the param. - The value must be evaluated and type must a possible type. - """ - Element.validate(self) - if self.dtype not in Constants.PARAM_TYPE_NAMES: - self.add_error_message('Type "{}" is not a possible type.'.format(self.dtype)) - - def get_evaluated(self): - return self._evaluated - - def evaluate(self): - """ - Evaluate the value. - - Returns: - evaluated type - """ - self._init = True - self._lisitify_flag = False - self._stringify_flag = False - dtype = self.dtype - expr = self.get_value() - - ######################### - # Enum Type - ######################### - if self.is_enum(): - return expr - - ######################### - # Numeric Types - ######################### - elif dtype in ('raw', 'complex', 'real', 'float', 'int', 'hex', 'bool'): - # Raise exception if python cannot evaluate this value - try: - value = self.parent_flowgraph.evaluate(expr) - except Exception as value: - raise Exception('Value "{}" cannot be evaluated:\n{}'.format(expr, value)) - # Raise an exception if the data is invalid - if dtype == 'raw': - return value - elif dtype == 'complex': - if not isinstance(value, Constants.COMPLEX_TYPES): - raise Exception('Expression "{}" is invalid for type complex.'.format(str(value))) - return value - elif dtype in ('real', 'float'): - if not isinstance(value, Constants.REAL_TYPES): - raise Exception('Expression "{}" is invalid for type float.'.format(str(value))) - return value - elif dtype == 'int': - if not isinstance(value, Constants.INT_TYPES): - raise Exception('Expression "{}" is invalid for type integer.'.format(str(value))) - return value - elif dtype == 'hex': - return hex(value) - elif dtype == 'bool': - if not isinstance(value, bool): - raise Exception('Expression "{}" is invalid for type bool.'.format(str(value))) - return value - else: - raise TypeError('Type "{}" not handled'.format(dtype)) - ######################### - # Numeric Vector Types - ######################### - elif dtype in ('complex_vector', 'real_vector', 'float_vector', 'int_vector'): - default = [] - - if not expr: - return default # Turn a blank string into an empty list, so it will eval - - try: - value = self.parent.parent.evaluate(expr) - except Exception as value: - raise Exception('Value "{}" cannot be evaluated:\n{}'.format(expr, value)) - - if not isinstance(value, Constants.VECTOR_TYPES): - self._lisitify_flag = True - value = [value] - - # Raise an exception if the data is invalid - if dtype == 'complex_vector' and not all(isinstance(item, numbers.Complex) for item in value): - raise Exception('Expression "{}" is invalid for type complex vector.'.format(value)) - elif dtype in ('real_vector', 'float_vector') and not all(isinstance(item, numbers.Real) for item in value): - raise Exception('Expression "{}" is invalid for type float vector.'.format(value)) - elif dtype == 'int_vector' and not all(isinstance(item, Constants.INT_TYPES) for item in value): - raise Exception('Expression "{}" is invalid for type integer vector.'.format(str(value))) - return value - ######################### - # String Types - ######################### - elif dtype in ('string', 'file_open', 'file_save', '_multiline', '_multiline_python_external'): - # Do not check if file/directory exists, that is a runtime issue - try: - value = self.parent.parent.evaluate(expr) - if not isinstance(value, str): - raise Exception() - except: - self._stringify_flag = True - value = str(expr) - if dtype == '_multiline_python_external': - ast.parse(value) # Raises SyntaxError - return value - ######################### - # Unique ID Type - ######################### - elif dtype == 'id': - self.validate_block_id() - return expr - - ######################### - # Stream ID Type - ######################### - elif dtype == 'stream_id': - self.validate_stream_id() - return expr - - ######################### - # GUI Position/Hint - ######################### - elif dtype == 'gui_hint': - if ':' in expr: - tab, pos = expr.split(':') - elif '@' in expr: - tab, pos = expr, '' - else: - tab, pos = '', expr - - if '@' in tab: - tab, index = tab.split('@') - else: - index = '?' - - # TODO: Problem with this code. Produces bad tabs - widget_str = ({ - (True, True): 'self.%(tab)s_grid_layout_%(index)s.addWidget(%(widget)s, %(pos)s)', - (True, False): 'self.%(tab)s_layout_%(index)s.addWidget(%(widget)s)', - (False, True): 'self.top_grid_layout.addWidget(%(widget)s, %(pos)s)', - (False, False): 'self.top_layout.addWidget(%(widget)s)', - }[bool(tab), bool(pos)]) % {'tab': tab, 'index': index, 'widget': '%s', 'pos': pos} - - # FIXME: Move replace(...) into the make template of the qtgui blocks - # Return a string here - class GuiHint(object): - def __init__(self, ws): - self._ws = ws - - def __call__(self, w): - return (self._ws.replace('addWidget', 'addLayout') if 'layout' in w else self._ws) % w - - def __str__(self): - return self._ws - return GuiHint(widget_str) - ######################### - # Import Type - ######################### - elif dtype == 'import': - # New namespace - n = dict() - try: - exec(expr, n) - except ImportError: - raise Exception('Import "{}" failed.'.format(expr)) - except Exception: - raise Exception('Bad import syntax: "{}".'.format(expr)) - return [k for k in list(n.keys()) if str(k) != '__builtins__'] - - ######################### - else: - raise TypeError('Type "{}" not handled'.format(dtype)) - - def validate_block_id(self): - value = self.value - # Can python use this as a variable? - if not re.match(r'^[a-z|A-Z]\w*$', value): - raise Exception('ID "{}" must begin with a letter and may contain letters, numbers, ' - 'and underscores.'.format(value)) - if value in ID_BLACKLIST: - raise Exception('ID "{}" is blacklisted.'.format(value)) - block_names = [block.name for block in self.parent_flowgraph.iter_enabled_blocks()] - # Id should only appear once, or zero times if block is disabled - if self.key == 'id' and block_names.count(value) > 1: - raise Exception('ID "{}" is not unique.'.format(value)) - elif value not in block_names: - raise Exception('ID "{}" does not exist.'.format(value)) - return value - - def validate_stream_id(self): - value = self.value - stream_ids = [ - block.params['stream_id'].value - for block in self.parent_flowgraph.iter_enabled_blocks() - if isinstance(block, blocks.VirtualSink) - ] - # Check that the virtual sink's stream id is unique - if isinstance(self.parent_block, blocks.VirtualSink) and stream_ids.count(value) >= 2: - # Id should only appear once, or zero times if block is disabled - raise Exception('Stream ID "{}" is not unique.'.format(value)) - # Check that the virtual source's steam id is found - elif isinstance(self.parent_block, blocks.VirtualSource) and value not in stream_ids: - raise Exception('Stream ID "{}" is not found.'.format(value)) - - def to_code(self): - """ - Convert the value to code. - For string and list types, check the init flag, call evaluate(). - This ensures that evaluate() was called to set the xxxify_flags. - - Returns: - a string representing the code - """ - self._init = True - v = self.get_value() - t = self.dtype - # String types - if t in ('string', 'file_open', 'file_save', '_multiline', '_multiline_python_external'): - if not self._init: - self.evaluate() - return repr(v) if self._stringify_flag else v - - # Vector types - elif t in ('complex_vector', 'real_vector', 'float_vector', 'int_vector'): - if not self._init: - self.evaluate() - if self._lisitify_flag: - return '(%s, )' % v - else: - return '(%s)' % v - else: - return v - - def get_opt(self, item): - return self.options.attributes[self.get_value()][item] diff --git a/grc/core/blocks/__init__.py b/grc/core/blocks/__init__.py index e4a085d477..4ca0d5d2bc 100644 --- a/grc/core/blocks/__init__.py +++ b/grc/core/blocks/__init__.py @@ -29,6 +29,7 @@ build_ins = {} def register_build_in(cls): + cls.loaded_from = '(build-in)' build_ins[cls.key] = cls return cls diff --git a/grc/core/blocks/_build.py b/grc/core/blocks/_build.py index 9221433387..6db06040cf 100644 --- a/grc/core/blocks/_build.py +++ b/grc/core/blocks/_build.py @@ -17,8 +17,13 @@ from __future__ import absolute_import +import collections +import itertools import re +from ..Constants import ADVANCED_PARAM_TAB +from ..utils import to_list + from .block import Block from ._flags import Flags from ._templates import MakoTemplates @@ -29,23 +34,24 @@ def build(id, label='', category='', flags='', documentation='', parameters=None, inputs=None, outputs=None, templates=None, **kwargs): block_id = id - cls = type(block_id, (Block,), {}) + cls = type(str(block_id), (Block,), {}) cls.key = block_id cls.label = label or block_id.title() cls.category = [cat.strip() for cat in category.split('/') if cat.strip()] - cls.flags = Flags(flags) + cls.flags = Flags(to_list(flags)) if re.match(r'options$|variable|virtual', block_id): - cls.flags += Flags.NOT_DSP + Flags.DISABLE_BYPASS + cls.flags.set(Flags.NOT_DSP, Flags.DISABLE_BYPASS) cls.documentation = {'': documentation.strip('\n\t ').replace('\\\n', '')} - cls.asserts = [_single_mako_expr(a, block_id) for a in (asserts or [])] + cls.asserts = [_single_mako_expr(a, block_id) for a in to_list(asserts)] - cls.parameters_data = parameters or [] - cls.inputs_data = inputs or [] - cls.outputs_data = outputs or [] + cls.inputs_data = _build_ports(inputs, 'sink') if inputs else [] + cls.outputs_data = _build_ports(outputs, 'source') if outputs else [] + cls.parameters_data = _build_params(parameters or [], + bool(cls.inputs_data), bool(cls.outputs_data), cls.flags) cls.extra_data = kwargs templates = templates or {} @@ -62,8 +68,68 @@ def build(id, label='', category='', flags='', documentation='', return cls +def _build_ports(ports_raw, direction): + ports = [] + port_ids = set() + stream_port_ids = itertools.count() + + for i, port_params in enumerate(ports_raw): + port = port_params.copy() + port['direction'] = direction + + port_id = port.setdefault('id', str(next(stream_port_ids))) + if port_id in port_ids: + raise Exception('Port id "{}" already exists in {}s'.format(port_id, direction)) + port_ids.add(port_id) + + ports.append(port) + return ports + + +def _build_params(params_raw, have_inputs, have_outputs, flags): + params = [] + + def add_param(**data): + params.append(data) + + add_param(id='id', name='ID', dtype='id', hide='part') + + if not flags.not_dsp: + add_param(id='alias', name='Block Alias', dtype='string', + hide='part', category=ADVANCED_PARAM_TAB) + + if have_outputs or have_inputs: + add_param(id='affinity', name='Core Affinity', dtype='int_vector', + hide='part', category=ADVANCED_PARAM_TAB) + + if have_outputs: + add_param(id='minoutbuf', name='Min Output Buffer', dtype='int', + hide='part', value='0', category=ADVANCED_PARAM_TAB) + add_param(id='maxoutbuf', name='Max Output Buffer', dtype='int', + hide='part', value='0', category=ADVANCED_PARAM_TAB) + + base_params_n = {} + for param_data in params_raw: + param_id = param_data['id'] + if param_id in params: + raise Exception('Param id "{}" is not unique'.format(param_id)) + + base_key = param_data.get('base_key', None) + param_data_ext = base_params_n.get(base_key, {}).copy() + param_data_ext.update(param_data) + + add_param(**param_data_ext) + base_params_n[param_id] = param_data_ext + + add_param(id='comment', name='Comment', dtype='_multiline', hide='part', + value='', category=ADVANCED_PARAM_TAB) + return params + + def _single_mako_expr(value, block_id): - match = re.match(r'\s*\$\{\s*(.*?)\s*\}\s*', str(value)) - if value and not match: + if not value: + return None + value = value.strip() + if not (value.startswith('${') and value.endswith('}')): raise ValueError('{} is not a mako substitution in {}'.format(value, block_id)) - return match.group(1) if match else None + return value[2:-1].strip() diff --git a/grc/core/blocks/_flags.py b/grc/core/blocks/_flags.py index ffea2ad569..bbedd6a2d7 100644 --- a/grc/core/blocks/_flags.py +++ b/grc/core/blocks/_flags.py @@ -17,10 +17,8 @@ from __future__ import absolute_import -import six - -class Flags(six.text_type): +class Flags(object): THROTTLE = 'throttle' DISABLE_BYPASS = 'disable_bypass' @@ -28,12 +26,14 @@ class Flags(six.text_type): DEPRECATED = 'deprecated' NOT_DSP = 'not_dsp' + def __init__(self, flags): + self.data = set(flags) + def __getattr__(self, item): return item in self - def __add__(self, other): - if not isinstance(other, six.string_types): - return NotImplemented - return self.__class__(str(self) + other) + def __contains__(self, item): + return item in self.data - __iadd__ = __add__ + def set(self, *flags): + self.data.update(flags) diff --git a/grc/core/blocks/block.py b/grc/core/blocks/block.py index adc046936d..0cb3f61237 100644 --- a/grc/core/blocks/block.py +++ b/grc/core/blocks/block.py @@ -19,7 +19,6 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA from __future__ import absolute_import -import ast import collections import itertools @@ -29,7 +28,6 @@ from six.moves import range from ._templates import MakoTemplates from ._flags import Flags -from ..Constants import ADVANCED_PARAM_TAB from ..base import Element from ..utils.descriptors import lazy_property @@ -63,82 +61,28 @@ class Block(Element): outputs_data = [] extra_data = {} + loaded_from = '(unknown)' - # region Init def __init__(self, parent): """Make a new block from nested data.""" super(Block, self).__init__(parent) - self.params = self._init_params() - self.sinks = self._init_ports(self.inputs_data, direction='sink') - self.sources = self._init_ports(self.outputs_data, direction='source') - - self.active_sources = [] # on rewrite - self.active_sinks = [] # on rewrite - - self.states = {'state': True} - - def _init_params(self): - is_dsp_block = not self.flags.not_dsp - has_inputs = bool(self.inputs_data) - has_outputs = bool(self.outputs_data) - - params = collections.OrderedDict() param_factory = self.parent_platform.make_param - - def add_param(id, **kwargs): - params[id] = param_factory(self, id=id, **kwargs) - - add_param(id='id', name='ID', dtype='id', - hide='none' if (self.key == 'options' or self.is_variable) else 'part') - - if is_dsp_block: - add_param(id='alias', name='Block Alias', dtype='string', - hide='part', category=ADVANCED_PARAM_TAB) - - if has_outputs or has_inputs: - add_param(id='affinity', name='Core Affinity', dtype='int_vector', - hide='part', category=ADVANCED_PARAM_TAB) - - if has_outputs: - add_param(id='minoutbuf', name='Min Output Buffer', dtype='int', - hide='part', value='0', category=ADVANCED_PARAM_TAB) - add_param(id='maxoutbuf', name='Max Output Buffer', dtype='int', - hide='part', value='0', category=ADVANCED_PARAM_TAB) - - base_params_n = {} - for param_data in self.parameters_data: - param_id = param_data['id'] - if param_id in params: - raise Exception('Param id "{}" is not unique'.format(param_id)) - - base_key = param_data.get('base_key', None) - param_data_ext = base_params_n.get(base_key, {}).copy() - param_data_ext.update(param_data) - - add_param(**param_data_ext) - base_params_n[param_id] = param_data_ext - - add_param(id='comment', name='Comment', dtype='_multiline', hide='part', - value='', category=ADVANCED_PARAM_TAB) - return params - - def _init_ports(self, ports_n, direction): - ports = [] port_factory = self.parent_platform.make_port - port_ids = set() - stream_port_ids = itertools.count() + self.params = collections.OrderedDict( + (data['id'], param_factory(parent=self, **data)) + for data in self.parameters_data + ) + if self.key == 'options' or self.is_variable: + self.params['id'].hide = 'part' - for i, port_data in enumerate(ports_n): - port_id = port_data.setdefault('id', str(next(stream_port_ids))) - if port_id in port_ids: - raise Exception('Port id "{}" already exists in {}s'.format(port_id, direction)) - port_ids.add(port_id) + self.sinks = [port_factory(parent=self, **params) for params in self.inputs_data] + self.sources = [port_factory(parent=self, **params) for params in self.outputs_data] - port = port_factory(parent=self, direction=direction, **port_data) - ports.append(port) - return ports - # endregion + self.active_sources = [] # on rewrite + self.active_sinks = [] # on rewrite + + self.states = {'state': True} # region Rewrite_and_Validation def rewrite(self): diff --git a/grc/core/cache.py b/grc/core/cache.py new file mode 100644 index 0000000000..f438d58bd9 --- /dev/null +++ b/grc/core/cache.py @@ -0,0 +1,99 @@ +# Copyright 2017 Free Software Foundation, Inc. +# This file is part of GNU Radio +# +# GNU Radio Companion is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# GNU Radio Companion is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +from __future__ import absolute_import, print_function + +from io import open +import json +import logging +import os + +import six + +from .io import yaml + +logger = logging.getLogger(__name__) + + +class Cache(object): + + def __init__(self, filename): + self.cache_file = filename + self.cache = {} + self.need_cache_write = True + self._accessed_items = set() + try: + os.makedirs(os.path.dirname(filename)) + except OSError: + pass + try: + self._converter_mtime = os.path.getmtime(filename) + except OSError: + self._converter_mtime = -1 + + def load(self): + try: + logger.debug("Loading block cache from: {}".format(self.cache_file)) + with open(self.cache_file, encoding='utf-8') as cache_file: + self.cache = json.load(cache_file) + self.need_cache_write = False + except (IOError, ValueError): + self.need_cache_write = True + + def get_or_load(self, filename): + self._accessed_items.add(filename) + if os.path.getmtime(filename) <= self._converter_mtime: + try: + return self.cache[filename] + except KeyError: + pass + + with open(filename, encoding='utf-8') as fp: + data = yaml.safe_load(fp) + self.cache[filename] = data + self.need_cache_write = True + return data + + def save(self): + if not self.need_cache_write: + return + + logger.info('Saving %d entries to json cache', len(self.cache)) + with open(self.cache_file, 'w', encoding='utf8') as cache_file: + json.dump(self.cache, cache_file) + + def prune(self): + for filename in (set(self.cache) - self._accessed_items): + del self.cache[filename] + + def __enter__(self): + self.load() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.save() + + +def byteify(data): + if isinstance(data, dict): + return {byteify(key): byteify(value) for key, value in six.iteritems(data)} + elif isinstance(data, list): + return [byteify(element) for element in data] + elif isinstance(data, six.text_type) and six.PY2: + return data.encode('utf-8') + else: + return data diff --git a/grc/core/default_flow_graph.grc b/grc/core/default_flow_graph.grc index 9df289f327..d57ec75aea 100644 --- a/grc/core/default_flow_graph.grc +++ b/grc/core/default_flow_graph.grc @@ -5,6 +5,7 @@ options: parameters: + id: 'top_block' title: 'top_block' states: coordinate: diff --git a/grc/core/generator/hier_block.py b/grc/core/generator/hier_block.py index 237fd71377..31cd198c01 100644 --- a/grc/core/generator/hier_block.py +++ b/grc/core/generator/hier_block.py @@ -149,8 +149,8 @@ class QtHierBlockGenerator(HierBlockGenerator): block_n['param'].append(gui_hint_param) block_n['make'] += ( - "\n#set $win = 'self.%s' % $id" - "\n${gui_hint()($win)}" + "\n<% win = 'self.' + id %>" + "\n${ gui_hint % win }" ) return {'block': block_n} diff --git a/grc/core/params/__init__.py b/grc/core/params/__init__.py new file mode 100644 index 0000000000..93663bdada --- /dev/null +++ b/grc/core/params/__init__.py @@ -0,0 +1,18 @@ +# Copyright 2017 Free Software Foundation, Inc. +# This file is part of GNU Radio +# +# GNU Radio Companion is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# GNU Radio Companion is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +from .param import Param diff --git a/grc/core/params/dtypes.py b/grc/core/params/dtypes.py new file mode 100644 index 0000000000..f52868c080 --- /dev/null +++ b/grc/core/params/dtypes.py @@ -0,0 +1,103 @@ +# Copyright 2008-2017 Free Software Foundation, Inc. +# This file is part of GNU Radio +# +# GNU Radio Companion is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# GNU Radio Companion is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +from __future__ import absolute_import + +import re + +from six.moves import builtins + +from .. import blocks +from .. import Constants + + +# Blacklist certain ids, its not complete, but should help +ID_BLACKLIST = ['self', 'options', 'gr', 'math', 'firdes'] + dir(builtins) +try: + from gnuradio import gr + ID_BLACKLIST.extend(attr for attr in dir(gr.top_block()) if not attr.startswith('_')) +except (ImportError, AttributeError): + pass + + +validators = {} + + +def validates(*dtypes): + def decorator(func): + for dtype in dtypes: + assert dtype in Constants.PARAM_TYPE_NAMES + validators[dtype] = func + return func + return decorator + + +class ValidateError(Exception): + """Raised by validate functions""" + + +@validates('id') +def validate_block_id(param): + value = param.value + # Can python use this as a variable? + if not re.match(r'^[a-z|A-Z]\w*$', value): + raise ValidateError('ID "{}" must begin with a letter and may contain letters, numbers, ' + 'and underscores.'.format(value)) + if value in ID_BLACKLIST: + raise ValidateError('ID "{}" is blacklisted.'.format(value)) + block_names = [block.name for block in param.parent_flowgraph.iter_enabled_blocks()] + # Id should only appear once, or zero times if block is disabled + if param.key == 'id' and block_names.count(value) > 1: + raise ValidateError('ID "{}" is not unique.'.format(value)) + elif value not in block_names: + raise ValidateError('ID "{}" does not exist.'.format(value)) + return value + + +@validates('stream_id') +def validate_stream_id(param): + value = param.value + stream_ids = [ + block.params['stream_id'].value + for block in param.parent_flowgraph.iter_enabled_blocks() + if isinstance(block, blocks.VirtualSink) + ] + # Check that the virtual sink's stream id is unique + if isinstance(param.parent_block, blocks.VirtualSink) and stream_ids.count(value) >= 2: + # Id should only appear once, or zero times if block is disabled + raise ValidateError('Stream ID "{}" is not unique.'.format(value)) + # Check that the virtual source's steam id is found + elif isinstance(param.parent_block, blocks.VirtualSource) and value not in stream_ids: + raise ValidateError('Stream ID "{}" is not found.'.format(value)) + + +@validates('complex', 'real', 'float', 'int') +def validate_scalar(param): + valid_types = Constants.PARAM_TYPE_MAP[param.dtype] + if not isinstance(param.get_evaluated(), valid_types): + raise ValidateError('Expression {!r} is invalid for type {!r}.'.format( + param.get_evaluated(), param.dtype)) + + +@validates('complex_vector', 'real_vector', 'float_vector', 'int_vector') +def validate_vector(param): + # todo: check vector types + + valid_types = Constants.PARAM_TYPE_MAP[param.dtype.split('_', 1)[0]] + if not all(isinstance(item, valid_types) for item in param.get_evaluated()): + raise ValidateError('Expression {!r} is invalid for type {!r}.'.format( + param.get_evaluated(), param.dtype)) diff --git a/grc/core/params/param.py b/grc/core/params/param.py new file mode 100644 index 0000000000..30a48bb434 --- /dev/null +++ b/grc/core/params/param.py @@ -0,0 +1,407 @@ +# Copyright 2008-2017 Free Software Foundation, Inc. +# This file is part of GNU Radio +# +# GNU Radio Companion is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# GNU Radio Companion is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +from __future__ import absolute_import + +import ast +import collections +import textwrap + +import six +from six.moves import range + +from .. import Constants +from ..base import Element +from ..utils.descriptors import Evaluated, EvaluatedEnum, setup_names + +from . import dtypes +from .template_arg import TemplateArg + + +@setup_names +class Param(Element): + + is_param = True + + name = Evaluated(str, default='no name') + dtype = EvaluatedEnum(Constants.PARAM_TYPE_NAMES, default='raw') + hide = EvaluatedEnum('none all part') + + # region init + def __init__(self, parent, id, label='', dtype='raw', default='', + options=None, option_labels=None, option_attributes=None, + category='', hide='none', **_): + """Make a new param from nested data""" + super(Param, self).__init__(parent) + self.key = id + self.name = label.strip() or id.title() + self.category = category or Constants.DEFAULT_PARAM_TAB + + self.dtype = dtype + self.value = self.default = str(default) + + self.options = self._init_options(options or [], option_labels or [], + option_attributes or {}) + self.hide = hide or 'none' + # end of args ######################################################## + + self._evaluated = None + self._stringify_flag = False + self._lisitify_flag = False + self.hostage_cells = set() + self._init = False + + def _init_options(self, values, labels, attributes): + """parse option and option attributes""" + options = collections.OrderedDict() + options.attributes = collections.defaultdict(dict) + + padding = [''] * max(len(values), len(labels)) + attributes = {key: value + padding for key, value in six.iteritems(attributes)} + + for i, option in enumerate(values): + # Test against repeated keys + if option in options: + raise KeyError('Value "{}" already exists in options'.format(option)) + # get label + try: + label = str(labels[i]) + except IndexError: + label = str(option) + # Store the option + options[option] = label + options.attributes[option] = {attrib: values[i] for attrib, values in six.iteritems(attributes)} + + default = next(iter(options)) if options else '' + if not self.value: + self.value = self.default = default + + if self.is_enum() and self.value not in options: + self.value = self.default = default # TODO: warn + # raise ValueError('The value {!r} is not in the possible values of {}.' + # ''.format(self.get_value(), ', '.join(self.options))) + return options + # endregion + + @property + def template_arg(self): + return TemplateArg(self) + + def __str__(self): + return 'Param - {}({})'.format(self.name, self.key) + + def __repr__(self): + return '{!r}.param[{}]'.format(self.parent, self.key) + + def is_enum(self): + return self.get_raw('dtype') == 'enum' + + def get_value(self): + value = self.value + if self.is_enum() and value not in self.options: + value = self.default + self.set_value(value) + return value + + def set_value(self, value): + # Must be a string + self.value = str(value) + + def set_default(self, value): + if self.default == self.value: + self.set_value(value) + self.default = str(value) + + def rewrite(self): + Element.rewrite(self) + del self.name + del self.dtype + del self.hide + + self._evaluated = None + try: + self._evaluated = self.evaluate() + except Exception as e: + self.add_error_message(str(e)) + + rewriter = getattr(dtypes, 'rewrite_' + self.dtype, None) + if rewriter: + rewriter(self) + + def validate(self): + """ + Validate the param. + The value must be evaluated and type must a possible type. + """ + Element.validate(self) + if self.dtype not in Constants.PARAM_TYPE_NAMES: + self.add_error_message('Type "{}" is not a possible type.'.format(self.dtype)) + + validator = dtypes.validators.get(self.dtype, None) + if self._init and validator: + try: + validator(self) + except dtypes.ValidateError as e: + self.add_error_message(e.message) + + def get_evaluated(self): + return self._evaluated + + def evaluate(self): + """ + Evaluate the value. + + Returns: + evaluated type + """ + self._init = True + self._lisitify_flag = False + self._stringify_flag = False + dtype = self.dtype + expr = self.get_value() + + ######################### + # ID and Enum types (not evaled) + ######################### + if dtype in ('id', 'stream_id') or self.is_enum(): + return expr + + ######################### + # Numeric Types + ######################### + elif dtype in ('raw', 'complex', 'real', 'float', 'int', 'hex', 'bool'): + if expr: + try: + value = self.parent_flowgraph.evaluate(expr) + except Exception as e: + raise Exception('Value "{}" cannot be evaluated:\n{}'.format(expr, e)) + else: + value = 0 + if dtype == 'hex': + value = hex(value) + elif dtype == 'bool': + value = bool(value) + return value + + ######################### + # Numeric Vector Types + ######################### + elif dtype in ('complex_vector', 'real_vector', 'float_vector', 'int_vector'): + if not expr: + return [] # Turn a blank string into an empty list, so it will eval + try: + value = self.parent.parent.evaluate(expr) + except Exception as value: + raise Exception('Value "{}" cannot be evaluated:\n{}'.format(expr, value)) + if not isinstance(value, Constants.VECTOR_TYPES): + self._lisitify_flag = True + value = [value] + return value + ######################### + # String Types + ######################### + elif dtype in ('string', 'file_open', 'file_save', '_multiline', '_multiline_python_external'): + # Do not check if file/directory exists, that is a runtime issue + try: + value = self.parent_flowgraph.evaluate(expr) + if not isinstance(value, str): + raise Exception() + except: + self._stringify_flag = True + value = str(expr) + if dtype == '_multiline_python_external': + ast.parse(value) # Raises SyntaxError + return value + ######################### + # GUI Position/Hint + ######################### + elif dtype == 'gui_hint': + return self.parse_gui_hint(expr) if self.parent_block.state == 'enabled' else '' + ######################### + # Import Type + ######################### + elif dtype == 'import': + # New namespace + n = dict() + try: + exec(expr, n) + except ImportError: + raise Exception('Import "{}" failed.'.format(expr)) + except Exception: + raise Exception('Bad import syntax: "{}".'.format(expr)) + return [k for k in list(n.keys()) if str(k) != '__builtins__'] + + ######################### + else: + raise TypeError('Type "{}" not handled'.format(dtype)) + + def to_code(self): + """ + Convert the value to code. + For string and list types, check the init flag, call evaluate(). + This ensures that evaluate() was called to set the xxxify_flags. + + Returns: + a string representing the code + """ + self._init = True + value = self.get_value() + # String types + if self.dtype in ('string', 'file_open', 'file_save', '_multiline', '_multiline_python_external'): + if not self._init: + self.evaluate() + return repr(value) if self._stringify_flag else value + + # Vector types + elif self.dtype in ('complex_vector', 'real_vector', 'float_vector', 'int_vector'): + if not self._init: + self.evaluate() + return '[' + value + ']' if self._lisitify_flag else value + else: + return value + + def get_opt(self, item): + return self.options.attributes[self.get_value()][item] + + ############################################## + # GUI Hint + ############################################## + def parse_gui_hint(self, expr): + """ + Parse/validate gui hint value. + + Args: + expr: gui_hint string from a block's 'gui_hint' param + + Returns: + string of python code for positioning GUI elements in pyQT + """ + self.hostage_cells.clear() + + # Parsing + if ':' in expr: + tab, pos = expr.split(':') + elif ',' in expr: + tab, pos = '', expr + else: + tab, pos = expr, '' + + if '@' in tab: + tab, index = tab.split('@') + else: + index = '0' + index = int(index) + + # Validation + def parse_pos(): + e = self.parent_flowgraph.evaluate(pos) + + if not isinstance(e, (list, tuple)) or len(e) not in (2, 4) or not all(isinstance(ei, int) for ei in e): + raise Exception('Invalid GUI Hint entered: {e!r} (Must be a list of {{2,4}} non-negative integers).'.format(e=e)) + + if len(e) == 2: + row, col = e + row_span = col_span = 1 + else: + row, col, row_span, col_span = e + + if (row < 0) or (col < 0): + raise Exception('Invalid GUI Hint entered: {e!r} (non-negative integers only).'.format(e=e)) + + if (row_span < 1) or (col_span < 1): + raise Exception('Invalid GUI Hint entered: {e!r} (positive row/column span required).'.format(e=e)) + + return row, col, row_span, col_span + + def validate_tab(): + tabs = (block for block in self.parent_flowgraph.iter_enabled_blocks() + if block.key == 'qtgui_tab_widget' and block.name == tab) + tab_block = next(iter(tabs), None) + if not tab_block: + raise Exception('Invalid tab name entered: {tab} (Tab name not found).'.format(tab=tab)) + + tab_index_size = int(tab_block.params['num_tabs'].value) + if index >= tab_index_size: + raise Exception('Invalid tab index entered: {tab}@{index} (Index out of range).'.format( + tab=tab, index=index)) + + # Collision Detection + def collision_detection(row, col, row_span, col_span): + my_parent = '{tab}@{index}'.format(tab=tab, index=index) if tab else 'main' + # Calculate hostage cells + for r in range(row, row + row_span): + for c in range(col, col + col_span): + self.hostage_cells.add((my_parent, (r, c))) + + for other in self.get_all_params('gui_hint'): + if other is self: + continue + collision = next(iter(self.hostage_cells & other.hostage_cells), None) + if collision: + raise Exception('Block {block!r} is also using parent {parent!r}, cell {cell!r}.'.format( + block=other.parent_block.name, parent=collision[0], cell=collision[1] + )) + + # Code Generation + if tab: + validate_tab() + layout = '{tab}_grid_layout_{index}'.format(tab=tab, index=index) + else: + layout = 'top_grid_layout' + + widget = '%s' # to be fill-out in the mail template + + if pos: + row, col, row_span, col_span = parse_pos() + collision_detection(row, col, row_span, col_span) + + widget_str = textwrap.dedent(""" + self.{layout}.addWidget({widget}, {row}, {col}, {row_span}, {col_span}) + for r in range({row}, {row_end}): + self.{layout}.setRowStretch(r, 1) + for c in range({col}, {col_end}): + self.{layout}.setColumnStretch(c, 1) + """.strip('\n')).format( + layout=layout, widget=widget, + row=row, row_span=row_span, row_end=row+row_span, + col=col, col_span=col_span, col_end=col+col_span, + ) + + else: + widget_str = 'self.{layout}.addWidget({widget})'.format(layout=layout, widget=widget) + + return widget_str + + def get_all_params(self, dtype, key=None): + """ + Get all the params from the flowgraph that have the given type and + optionally a given key + + Args: + dtype: the specified type + key: the key to match against + + Returns: + a list of params + """ + params = [] + for block in self.parent_flowgraph.iter_enabled_blocks(): + params.extend( + param for param in block.params.values() + if param.dtype == dtype and (key is None or key == param.name) + ) + return params diff --git a/grc/core/params/template_arg.py b/grc/core/params/template_arg.py new file mode 100644 index 0000000000..5c8c610b4f --- /dev/null +++ b/grc/core/params/template_arg.py @@ -0,0 +1,50 @@ +# Copyright 2008-2017 Free Software Foundation, Inc. +# This file is part of GNU Radio +# +# GNU Radio Companion is free software; you can redistribute it and/or +# modify it under the terms of the GNU General Public License +# as published by the Free Software Foundation; either version 2 +# of the License, or (at your option) any later version. +# +# GNU Radio Companion is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA + +from __future__ import absolute_import + + +class TemplateArg(str): + """ + A cheetah template argument created from a param. + The str of this class evaluates to the param's to code method. + The use of this class as a dictionary (enum only) will reveal the enum opts. + The __call__ or () method can return the param evaluated to a raw python data type. + """ + + def __new__(cls, param): + value = param.to_code() + instance = str.__new__(cls, value) + setattr(instance, '_param', param) + return instance + + def __getitem__(self, item): + return str(self._param.get_opt(item)) if self._param.is_enum() else NotImplemented + + def __getattr__(self, item): + if not self._param.is_enum(): + raise AttributeError() + try: + return str(self._param.get_opt(item)) + except KeyError: + raise AttributeError() + + def __str__(self): + return str(self._param.to_code()) + + def __call__(self): + return self._param.get_evaluated() diff --git a/grc/core/platform.py b/grc/core/platform.py index 538bacade2..6d02cb6441 100644 --- a/grc/core/platform.py +++ b/grc/core/platform.py @@ -19,7 +19,6 @@ from __future__ import absolute_import, print_function from codecs import open from collections import namedtuple -import glob import os import logging from itertools import chain @@ -29,16 +28,16 @@ from six.moves import range from . import ( Messages, Constants, - blocks, ports, errors, utils, schema_checker + blocks, params, ports, errors, utils, schema_checker ) from .Config import Config +from .cache import Cache from .base import Element from .io import yaml from .generator import Generator from .FlowGraph import FlowGraph from .Connection import Connection -from .Param import Param logger = logging.getLogger(__name__) logging.basicConfig(level=logging.INFO) @@ -141,44 +140,41 @@ class Platform(Element): self.connection_templates.clear() self._block_categories.clear() - # FIXME: remove this as soon as converter is stable - from ..converter import Converter - converter = Converter(self.config.block_paths, self.config.yml_block_cache) - converter.run() - logging.info('XML converter done.') - - for file_path in self._iter_files_in_block_path(path): - try: - data = converter.cache[file_path] - except KeyError: - with open(file_path, encoding='utf-8') as fp: - data = yaml.safe_load(fp) - - if file_path.endswith('.block.yml'): - loader = self.load_block_description - scheme = schema_checker.BLOCK_SCHEME - elif file_path.endswith('.domain.yml'): - loader = self.load_domain_description - scheme = schema_checker.DOMAIN_SCHEME - elif file_path.endswith('.tree.yml'): - loader = self.load_category_tree_description - scheme = None - else: - continue - - try: - checker = schema_checker.Validator(scheme) - passed = checker.run(data) - for msg in checker.messages: - logger.warning('{:<40s} {}'.format(os.path.basename(file_path), msg)) - if not passed: - logger.info('YAML schema check failed for: ' + file_path) - - loader(data, file_path) - except Exception as error: - logger.exception('Error while loading %s', file_path) - logger.exception(error) - raise + # # FIXME: remove this as soon as converter is stable + # from ..converter import Converter + # converter = Converter(self.config.block_paths, self.config.yml_block_cache) + # converter.run() + # logging.info('XML converter done.') + + with Cache(Constants.CACHE_FILE) as cache: + for file_path in self._iter_files_in_block_path(path): + data = cache.get_or_load(file_path) + + if file_path.endswith('.block.yml'): + loader = self.load_block_description + scheme = schema_checker.BLOCK_SCHEME + elif file_path.endswith('.domain.yml'): + loader = self.load_domain_description + scheme = schema_checker.DOMAIN_SCHEME + elif file_path.endswith('.tree.yml'): + loader = self.load_category_tree_description + scheme = None + else: + continue + + try: + checker = schema_checker.Validator(scheme) + passed = checker.run(data) + for msg in checker.messages: + logger.warning('{:<40s} {}'.format(os.path.basename(file_path), msg)) + if not passed: + logger.info('YAML schema check failed for: ' + file_path) + + loader(data, file_path) + except Exception as error: + logger.exception('Error while loading %s', file_path) + logger.exception(error) + raise for key, block in six.iteritems(self.blocks): category = self._block_categories.get(key, block.category) @@ -201,10 +197,9 @@ class Platform(Element): if os.path.isfile(entry): yield entry elif os.path.isdir(entry): - pattern = os.path.join(entry, '**.' + ext) - yield_from = glob.iglob(pattern) - for file_path in yield_from: - yield file_path + for dirpath, dirnames, filenames in os.walk(entry): + for filename in sorted(filter(lambda f: f.endswith('.' + ext), filenames)): + yield os.path.join(dirpath, filename) else: logger.debug('Ignoring invalid path entry %r', entry) @@ -232,16 +227,18 @@ class Platform(Element): log.error('Unknown format version %d in %s', file_format, file_path) return - block_id = data.pop('id').rstrip('_') + block_id = data['id'] = data['id'].rstrip('_') if block_id in self.block_classes_build_in: log.warning('Not overwriting build-in block %s with %s', block_id, file_path) return if block_id in self.blocks: - log.warning('Block with id "%s" overwritten by %s', block_id, file_path) + log.warning('Block with id "%s" loaded from\n %s\noverwritten by\n %s', + block_id, self.blocks[block_id].loaded_from, file_path) try: - block_cls = self.blocks[block_id] = self.new_block_class(block_id, **data) + block_cls = self.blocks[block_id] = self.new_block_class(**data) + block_cls.loaded_from = file_path except errors.BlockLoadError as error: log.error('Unable to load block %s', block_id) log.exception(error) @@ -288,19 +285,12 @@ class Platform(Element): path = [] def load_category(name, elements): - if not isinstance(name, str): - log.debug('invalid name %r', name) - return - if isinstance(elements, list): - pass - elif isinstance(elements, str): - elements = [elements] - else: - log.debug('Ignoring elements of %s', name) + if not isinstance(name, six.string_types): + log.debug('Invalid name %r', name) return path.append(name) - for element in elements: - if isinstance(element, str): + for element in utils.to_list(elements): + if isinstance(element, six.string_types): block_id = element self._block_categories[block_id] = list(path) elif isinstance(element, dict): @@ -404,7 +394,7 @@ class Platform(Element): 'clone': ports.PortClone, # clone of ports with multiplicity > 1 } param_classes = { - None: Param, # default + None: params.Param, # default } def make_flow_graph(self, from_filename=None): @@ -415,8 +405,8 @@ class Platform(Element): fg.import_data(data) return fg - def new_block_class(self, block_id, **data): - return blocks.build(block_id, **data) + def new_block_class(self, **data): + return blocks.build(**data) def make_block(self, parent, block_id, **kwargs): cls = self.block_classes[block_id] diff --git a/grc/core/schema_checker/block.py b/grc/core/schema_checker/block.py index ea079b4276..d511e36887 100644 --- a/grc/core/schema_checker/block.py +++ b/grc/core/schema_checker/block.py @@ -37,7 +37,7 @@ TEMPLATES_SCHEME = expand( BLOCK_SCHEME = expand( id=Spec(types=str_, required=True, item_scheme=None), label=str_, - category=(list, str_), + category=str_, flags=(list, str_), parameters=Spec(types=list, required=False, item_scheme=PARAM_SCHEME), diff --git a/grc/core/utils/__init__.py b/grc/core/utils/__init__.py index 660eb594a5..f2ac986fb4 100644 --- a/grc/core/utils/__init__.py +++ b/grc/core/utils/__init__.py @@ -17,5 +17,17 @@ from __future__ import absolute_import +import six + from . import epy_block_io, expr_utils, extract_docs, flow_graph_complexity from .hide_bokeh_gui_options_if_not_installed import hide_bokeh_gui_options_if_not_installed + + +def to_list(value): + if not value: + return [] + elif isinstance(value, six.string_types): + return [value] + else: + return list(value) + diff --git a/grc/core/utils/descriptors/evaluated.py b/grc/core/utils/descriptors/evaluated.py index 313cee5b96..0e1b68761c 100644 --- a/grc/core/utils/descriptors/evaluated.py +++ b/grc/core/utils/descriptors/evaluated.py @@ -15,6 +15,10 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA +from __future__ import absolute_import + +import six + class Evaluated(object): def __init__(self, expected_type, default, name=None): @@ -62,7 +66,7 @@ class Evaluated(object): def __set__(self, instance, value): attribs = instance.__dict__ value = value or self.default - if isinstance(value, str) and value.startswith('${') and value.endswith('}'): + if isinstance(value, six.text_type) and value.startswith('${') and value.endswith('}'): attribs[self.name_raw] = value[2:-1].strip() else: attribs[self.name] = type(self.default)(value) @@ -75,9 +79,10 @@ class Evaluated(object): class EvaluatedEnum(Evaluated): def __init__(self, allowed_values, default=None, name=None): - self.allowed_values = allowed_values if isinstance(allowed_values, (list, tuple)) else \ - allowed_values.split() - default = default if default is not None else self.allowed_values[0] + if isinstance(allowed_values, six.string_types): + allowed_values = set(allowed_values.split()) + self.allowed_values = allowed_values + default = default if default is not None else next(iter(self.allowed_values)) super(EvaluatedEnum, self).__init__(str, default, name) def default_eval_func(self, instance): |