From 1be80fc62dbf69afbcc91bfd57f35c0a1d0709ad Mon Sep 17 00:00:00 2001 From: Tomas Gavenciak Date: Thu, 6 May 2010 20:15:19 -0400 Subject: [PATCH] New module for config evaluation. Basic testing. Implemented: * Configuration tree, * Variables, * Variable-setting operations ('SET' and 'APPEND') * Equality conditions (WIP, not tested yet), * Substitution expressions with text and variables, * Pretty-printing, * Caching for variables and conditions with invalidation along reverse-dependencies, * Add and remove operations from variables with correct reverse-dependency handling (add and remove) TODO: smarter conditions, variable lookup (and maybe transparent creation), possibly cleanup of unused variables and conditions (tests full coverage except conditions) --- t/moe/conf.py | 252 +++++++++++++++++++++++++++++++++++++++++++++ t/moe/conf.test.py | 39 +++++++ 2 files changed, 291 insertions(+) create mode 100644 t/moe/conf.py create mode 100644 t/moe/conf.test.py diff --git a/t/moe/conf.py b/t/moe/conf.py new file mode 100644 index 0000000..3f77e48 --- /dev/null +++ b/t/moe/conf.py @@ -0,0 +1,252 @@ +import types, itertools +import logging as log + +# ALL in unicode + +c_tree_sep = u'.' +c_comment = u'\n' +c_open = u'{' +c_close = u'}' +c_if = u'if' +c_maxdepth = 42 +c_pprint_indent = 2 + +class ConfigError(Exception): + pass + +# WARNING: Currently, only appending to a variable results in undefined + +class ConfigTree(object): + """ + Configuration subtree (namespace). Contains subtrees and variables. + """ + def __init__(self, name, tree): + """ + Create configuration subtree `name` under `tree`. + Use `name=None` and `tree=None` for the root tree. + """ + # Name of the tree + self.name = name + # Parent tree + self.tree = tree + # Elements of this tree (variables and subtrees) + self.elements = {} + # Full name with separators (None for root tree) + if self.tree and self.tree.fullname: + self.fullname = self.tree.fullname + c_tree_sep + self.name + else: + self.fullname = self.name + if self.tree: + self.tree.add(self.name, self) + def add(self, name, element): + "Add an element (subtree or a variable)." + if self.elements.has_key(name): + raise ConfigError('Element %s already present in tree %s'%(name, self.fullname)) + self.elements[name] = element + def remove(self, name): + "Remove an element (subtree or a variable)." + self.elements.pop(name) + def pprint(self, indent=0): + """ + Pretty printing of the tree. + Returns list of lines (strings). + """ + if self.name: + s = [u'%*sSubtree %s'%(indent, u'', self.name)] + else: + s = [u'%*sRoot tree'%(indent, u'')] + for e in sorted(self.elements.values()): + s.extend(e.pprint(indent + c_pprint_indent)) + return s + +class ConfigElem(object): + """ + Base class for cahed config elements - variables and conditions + """ + def __init__(self, name, tree): + # Name in the subtree, for conditions the defining text + self.name = name + # Parent tree + self.tree = tree + # Full name with separators (only informative for conditions) + if self.tree.fullname: + self.fullname = self.tree.fullname + c_tree_sep + self.name + else: + self.fullname = self.name + # Vars and conditions depending on value of this one + self.dependants = set([]) + # Cached value (may be None in case of evaluation error) + self.cached = False + self.cached_val = None + def depth_check(self, depth): + """ + Helper to check for recursion depth. + """ + if depth > c_maxdepth: + raise ConfigError('Too deep recursion in config evaluation (cyclic substitution?)') + def invalidate(self, depth=0): + """ + Invalidate cached data and invalidate all dependants. + Does nothing if not cached. + """ + self.depth_check(depth) + if self.cached: + log.debug('invalidating %r', self.fullname) + self.cached = False + for d in self.dependants: + d.invalidate(depth + 1) + def evaluate(self, depth=0): + """ + Cached interface for get_value(), returns a value or throws an exception. + """ + self.depth_check(depth) + if not self.cached: + self.cached_val = self.get_value(depth+1) + self.cached = True + if self.cached_val == None: + raise ConfigError("Unable to evaluate %r."%(self.fullname,)) + return self.cached_val + def get_value(self, depth=0): + raise ConfigError('get_value() not implemented') + +class ConfigEqCondition(ConfigElem): + "Simple (in)equality condition" + def __init__(self, name, tree, parent_cond, ex1, ex2, equality=True): + super(ConfigVar, self).__init__(name, tree) + # Tuple of two compared expressions + self.expressions = (ex1, ex2) + # If true, test for equality, otherwise inequality + self.equality = equality + # Parent condition in the configuration tree (or None) + self.parent_cond = parent_cond + # Setup dependencies on used variables (not on the parent condition) + for e in self.expressions: + for v in e.variables(): + v.dependants.add(self) + def remove_dependencies(self): + "Remove self as dependant from all dependencies" + for e in self.expressions: + for v in e.variables(): + v.dependants.remove(self) + def get_value(self, depth=0): + "Evaluate the condition (not cached)" + v = (ex1.evaluate(depth+1) == ex2.evaluate(depth+1)) + if self.equality: + return v + return not v + def __str__(self): + return self.name + +class ConfigVar(ConfigElem): + def __init__(self, name, tree): + super(ConfigVar, self).__init__(name, tree) + # Ordered list of operations + # (operation, [condition], expression ) + # operation is currently restricted to 'SET' and 'APPEND' + self.expressions = [] + # Fixed to value (may bi None) + self.fixed = False + self.fixed_val = None + # Register in a subtree + if self.tree: + self.tree.add(name, self) + def variables(self): + "Return a set of variables used in the expressions" + return set(sum([ list(e[2].variables()) for e in self.expressions ], [])) + def conditions(self): + "Return a set of conditions used in the expressions" + return set(sum([ e[1] for e in self.expressions ], [])) + def add_operation(self, operation, conditions, expression, index=None): + """ + Adds a new operation to the given index (None for appending). + Adds the variable as a dependant to the conditions and variables in the expressions. + """ + # First invalidate cached value + self.invalidate() + # Add the operation + expr = (operation, conditions, expression) + if index: + self.expressions.insert(index, expr) + else: + self.expressions.append(expr) + # Create dependencies + for v in expression.variables(): + v.dependants.add(self) + for c in conditions: + c.dependants.add(self) + def remove_operation(self, index): + """ + Remove the operation at given index. + Also removes the variable as dependant from all conditions and variables used in this + operation that are no longer used. NOTE: this may be slow. + """ + # First invalidate cached value + self.invalidate() + # Remove the operation + operation, conditions, expression = self.expressions[index] + self.expressions.pop(index) + # Remove obsolete dependencies on variables + vs = self.variables() + for v in expression.variables(): + if not v in vs: + v.dependants.remove(self) + # Remove obsolete dependencies on variables + cs = self.conditions() + for c in conditions: + if not c in cs(): + c.dependants.remove(self) + def get_value(self, depth=0): + log.debug('evaluating var %r', self.fullname) + # List of strings to be concatenated + val = [] + # Scan for last applicable expression - try each starting from the end, concatenate extensions + for i in range(len(self.expressions)-1, -1, -1): + operation, conditions, expr = self.expressions[i] + # Check all conditions guarding an expression + if all([c.evaluate(depth+1) for c in conditions]): + val.insert(0, expr.evaluate(depth+1)) + if operation == 'SET': + return u''.join(val) + return None + def pprint(self, indent=0): + """ + Pretty printing of the variable. Includes all operations. + Returns iterator of lines (unicode strings). + """ + yield u'%*s%s = \'%s\'' % (indent, u'', self.name, self.evaluate(0)) + for operation, conditions, expr in self.expressions: + yield u'%*s%s %s %s' % (indent + c_pprint_indent, '', operation, conditions, expr) + +class ConfigExpression(object): + """ + String expression with some unexpanded config variables. Used in variable operations and conditions. + Expression is given as a list of unicode strings and ConfigVar variables to be expanded. + """ + def __init__(self, exprlist, original = ''): + self.exprlist = exprlist + # Original defining string + self.original = original + # Replace strings with + for i in range(len(self.exprlist)): + e = self.exprlist[i] + if isinstance(e, types.StringTypes): + if not isinstance(e, unicode): + self.exprlist[i] = unicode(e, 'ascii') + def variables(self): + "Return an iterator of variables user in the expression" + return itertools.ifilter(lambda e: isinstance(e, ConfigVar), self.exprlist) + def __str__(self): + return self.original + def evaluate(self, depth): + "Return a unicode result of expansion of the variables." + s = [] + for e in self.exprlist: + if isinstance(e, ConfigVar): + s.append(e.evaluate(depth+1)) + elif isinstance(e, unicode): + s.append(e) + else: + raise ConfigError('Unsupported type %s in expression \'%s\'.'%(type(e), self)) + return u''.join(s) + + diff --git a/t/moe/conf.test.py b/t/moe/conf.test.py new file mode 100644 index 0000000..d43a129 --- /dev/null +++ b/t/moe/conf.test.py @@ -0,0 +1,39 @@ +import conf +import logging as log +log.getLogger().setLevel(log.DEBUG) + +vcnt = 3 + +def cs(s): + return conf.ConfigExpression([s], s) + +root = conf.ConfigTree(None, None) +t_a = conf.ConfigTree('a', root) +t_a_b = conf.ConfigTree('b', t_a) +t_c = conf.ConfigTree('c', root) + +v_r = conf.ConfigVar('r', root) +v_r.add_operation('SET', [], cs('ROOTVAR')) + +v_a = [] +v_b = [] +for i in range(vcnt): + v_a.append(conf.ConfigVar('va%d'%i, t_a)) + v_a[i].add_operation('SET', [], cs('VALUE-A%d'%i)) + + v_b.append(conf.ConfigVar('vb%d'%i, t_a_b)) + v_b[i].add_operation('APPEND', [], cs(' FOO')) + v_b[i].add_operation('SET', [], conf.ConfigExpression([v_a[i]], '{va%d}'%i)) + v_b[i].add_operation('APPEND', [], cs(' BAR')) + if i>0: + v_b[i].add_operation('APPEND', [], conf.ConfigExpression([' ', v_b[i-1]], ' {vb%d}'%(i-1))) +print '\n'.join(root.pprint()) + +v_b[0].remove_operation(1) +v_b[0].add_operation('SET', [], cs('NEW-VALUE')) +v_b[1].add_operation('APPEND', [], cs(' NEW-ADDED')) +v_a[1].add_operation('APPEND', [], cs(' NEW-ADDED-A')) +print '\n'.join(root.pprint()) + +v_a[0].add_operation('SET', [], cs('NEW-VALUE-A0')) +print '\n'.join(root.pprint()) -- 2.39.5