New module for config evaluation. Basic testing.

Implemented:
* Configuration tree,
* Variables,
* Variable-setting operations ('SET' and 'APPEND')
* Equality conditions (WIP, not tested yet),
* Substitution expressions with text and variables,
* Pretty-printing,
* Caching for variables and conditions with invalidation along reverse-dependencies,
* Add and remove operations from variables with correct reverse-dependency handling (add and remove)

TODO: smarter conditions, variable lookup (and maybe transparent creation),
possibly cleanup of unused variables and conditions

(tests full coverage except conditions)

diff --git a/t/moe/conf.py b/t/moe/conf.py
new file mode 100644 (file)
index 0000000..3f77e48
--- /dev/null
+++ b/t/moe/conf.py
@@ -0,0 +1,252 @@
+import types, itertools
+import logging as log
+
+# ALL in unicode
+
+c_tree_sep = u'.'
+c_comment = u'\n'
+c_open = u'{'
+c_close = u'}'
+c_if = u'if'
+c_maxdepth = 42
+c_pprint_indent = 2
+
+class ConfigError(Exception):
+  pass
+
+# WARNING: Currently, only appending to a variable results in undefined 
+
+class ConfigTree(object):
+  """
+  Configuration subtree (namespace). Contains subtrees and variables.
+  """
+  def __init__(self, name, tree):
+    """
+    Create configuration subtree `name` under `tree`.
+    Use `name=None` and `tree=None` for the root tree.
+    """
+    # Name of the tree
+    self.name = name   
+    # Parent tree 
+    self.tree = tree
+    # Elements of this tree (variables and subtrees)
+    self.elements = {}
+    # Full name with separators (None for root tree)
+    if self.tree and self.tree.fullname:
+      self.fullname = self.tree.fullname + c_tree_sep + self.name
+    else:
+      self.fullname = self.name
+    if self.tree:
+      self.tree.add(self.name, self)
+  def add(self, name, element):
+    "Add an element (subtree or a variable)."
+    if self.elements.has_key(name):
+      raise ConfigError('Element %s already present in tree %s'%(name, self.fullname))
+    self.elements[name] = element
+  def remove(self, name):
+    "Remove an element (subtree or a variable)."
+    self.elements.pop(name)
+  def pprint(self, indent=0):
+    """
+    Pretty printing of the tree.
+    Returns list of lines (strings).
+    """
+    if self.name:
+      s = [u'%*sSubtree %s'%(indent, u'', self.name)]
+    else:
+      s = [u'%*sRoot tree'%(indent, u'')]
+    for e in sorted(self.elements.values()):
+      s.extend(e.pprint(indent + c_pprint_indent))
+    return s
+
+class ConfigElem(object):
+  """
+  Base class for cahed config elements - variables and conditions
+  """
+  def __init__(self, name, tree):
+    # Name in the subtree, for conditions the defining text
+    self.name = name   
+    # Parent tree 
+    self.tree = tree
+    # Full name with separators (only informative for conditions)
+    if self.tree.fullname:
+      self.fullname = self.tree.fullname + c_tree_sep + self.name
+    else:
+      self.fullname = self.name
+    # Vars and conditions depending on value of this one
+    self.dependants = set([])
+    # Cached value (may be None in case of evaluation error)
+    self.cached = False
+    self.cached_val = None
+  def depth_check(self, depth):
+    """
+    Helper to check for recursion depth.
+    """
+    if depth > c_maxdepth:
+      raise ConfigError('Too deep recursion in config evaluation (cyclic substitution?)')
+  def invalidate(self, depth=0):
+    """
+    Invalidate cached data and invalidate all dependants. 
+    Does nothing if not cached.
+    """
+    self.depth_check(depth)
+    if self.cached:
+      log.debug('invalidating %r', self.fullname)
+      self.cached = False
+      for d in self.dependants:
+       d.invalidate(depth + 1)
+  def evaluate(self, depth=0):
+    """
+    Cached interface for get_value(), returns a value or throws an exception.
+    """
+    self.depth_check(depth)
+    if not self.cached:
+      self.cached_val = self.get_value(depth+1)
+      self.cached = True
+    if self.cached_val == None:
+      raise ConfigError("Unable to evaluate %r."%(self.fullname,))
+    return self.cached_val 
+  def get_value(self, depth=0):
+    raise ConfigError('get_value() not implemented')
+
+class ConfigEqCondition(ConfigElem):
+  "Simple (in)equality condition"
+  def __init__(self, name, tree, parent_cond, ex1, ex2, equality=True):
+    super(ConfigVar, self).__init__(name, tree)
+    # Tuple of two compared expressions
+    self.expressions = (ex1, ex2)
+    # If true, test for equality, otherwise inequality
+    self.equality = equality
+    # Parent condition in the configuration tree (or None)
+    self.parent_cond = parent_cond
+    # Setup dependencies on used variables (not on the parent condition)
+    for e in self.expressions:
+      for v in e.variables():
+       v.dependants.add(self)
+  def remove_dependencies(self):
+    "Remove self as dependant from all dependencies"
+    for e in self.expressions:
+      for v in e.variables():
+       v.dependants.remove(self)
+  def get_value(self, depth=0):
+    "Evaluate the condition (not cached)"
+    v = (ex1.evaluate(depth+1) == ex2.evaluate(depth+1))
+    if self.equality:
+      return v
+    return not v
+  def __str__(self):
+    return self.name
+
+class ConfigVar(ConfigElem):
+  def __init__(self, name, tree):
+    super(ConfigVar, self).__init__(name, tree)
+    # Ordered list of operations
+    # (operation, [condition], expression )
+    # operation is currently restricted to 'SET' and 'APPEND'
+    self.expressions = []
+    # Fixed to value (may bi None) 
+    self.fixed = False
+    self.fixed_val = None
+    # Register in a subtree 
+    if self.tree:
+      self.tree.add(name, self)
+  def variables(self):
+    "Return a set of variables used in the expressions"
+    return set(sum([ list(e[2].variables()) for e in self.expressions ], []))
+  def conditions(self):
+    "Return a set of conditions used in the expressions"
+    return set(sum([ e[1] for e in self.expressions ], []))
+  def add_operation(self, operation, conditions, expression, index=None):
+    """
+    Adds a new operation to the given index (None for appending).
+    Adds the variable as a dependant to the conditions and variables in the expressions. 
+    """
+    # First invalidate cached value
+    self.invalidate()
+    # Add the operation 
+    expr = (operation, conditions, expression)
+    if index:
+      self.expressions.insert(index, expr)
+    else:
+      self.expressions.append(expr)
+    # Create dependencies
+    for v in expression.variables():
+      v.dependants.add(self)
+    for c in conditions:
+      c.dependants.add(self)
+  def remove_operation(self, index):
+    """
+    Remove the operation at given index.
+    Also removes the variable as dependant from all conditions and variables used in this 
+    operation that are no longer used. NOTE: this may be slow.
+    """
+    # First invalidate cached value
+    self.invalidate()
+    # Remove the operation 
+    operation, conditions, expression =  self.expressions[index] 
+    self.expressions.pop(index)
+    # Remove obsolete dependencies on variables
+    vs = self.variables()
+    for v in expression.variables():
+      if not v in vs:
+       v.dependants.remove(self)
+    # Remove obsolete dependencies on variables
+    cs = self.conditions()
+    for c in conditions:
+      if not c in cs():
+       c.dependants.remove(self)
+  def get_value(self, depth=0):
+    log.debug('evaluating var %r', self.fullname)
+    # List of strings to be concatenated
+    val = []
+    # Scan for last applicable expression - try each starting from the end, concatenate extensions
+    for i in range(len(self.expressions)-1, -1, -1):
+      operation, conditions, expr = self.expressions[i]
+      # Check all conditions guarding an expression 
+      if all([c.evaluate(depth+1) for c in conditions]):
+       val.insert(0, expr.evaluate(depth+1))
+       if operation == 'SET':
+         return u''.join(val)
+    return None
+  def pprint(self, indent=0):
+    """
+    Pretty printing of the variable. Includes all operations.
+    Returns iterator of lines (unicode strings).
+    """
+    yield u'%*s%s = \'%s\'' % (indent, u'', self.name, self.evaluate(0))
+    for operation, conditions, expr in self.expressions:
+      yield u'%*s%s %s %s' % (indent + c_pprint_indent, '', operation, conditions, expr)
+
+class ConfigExpression(object):
+  """
+  String expression with some unexpanded config variables. Used in variable operations and conditions.
+  Expression is given as a list of unicode strings and ConfigVar variables to be expanded.
+  """
+  def __init__(self, exprlist, original = ''):
+    self.exprlist = exprlist
+    # Original defining string 
+    self.original = original
+    # Replace strings with 
+    for i in range(len(self.exprlist)):
+      e = self.exprlist[i]
+      if isinstance(e, types.StringTypes):
+       if not isinstance(e, unicode):
+         self.exprlist[i] = unicode(e, 'ascii')
+  def variables(self):
+    "Return an iterator of variables user in the expression"
+    return itertools.ifilter(lambda e: isinstance(e, ConfigVar), self.exprlist)
+  def __str__(self):
+    return self.original
+  def evaluate(self, depth):
+    "Return a unicode result of expansion of the variables."
+    s = []
+    for e in self.exprlist:
+      if isinstance(e, ConfigVar):
+       s.append(e.evaluate(depth+1))
+      elif isinstance(e, unicode):
+       s.append(e)
+      else:
+       raise ConfigError('Unsupported type %s in expression \'%s\'.'%(type(e), self))
+    return u''.join(s)
+
+

diff --git a/t/moe/conf.test.py b/t/moe/conf.test.py
new file mode 100644 (file)
index 0000000..d43a129
--- /dev/null
+++ b/t/moe/conf.test.py
@@ -0,0 +1,39 @@
+import conf
+import logging as log
+log.getLogger().setLevel(log.DEBUG)
+
+vcnt = 3
+
+def cs(s):
+  return conf.ConfigExpression([s], s)
+
+root = conf.ConfigTree(None, None)
+t_a = conf.ConfigTree('a', root)
+t_a_b = conf.ConfigTree('b', t_a)
+t_c = conf.ConfigTree('c', root)
+
+v_r = conf.ConfigVar('r', root)
+v_r.add_operation('SET', [], cs('ROOTVAR'))
+
+v_a = []
+v_b = []
+for i in range(vcnt):
+  v_a.append(conf.ConfigVar('va%d'%i, t_a))
+  v_a[i].add_operation('SET', [], cs('VALUE-A%d'%i))
+
+  v_b.append(conf.ConfigVar('vb%d'%i, t_a_b))
+  v_b[i].add_operation('APPEND', [], cs(' FOO'))
+  v_b[i].add_operation('SET', [], conf.ConfigExpression([v_a[i]], '{va%d}'%i))
+  v_b[i].add_operation('APPEND', [], cs(' BAR'))
+  if i>0: 
+    v_b[i].add_operation('APPEND', [], conf.ConfigExpression([' ', v_b[i-1]], ' {vb%d}'%(i-1)))
+print '\n'.join(root.pprint())
+
+v_b[0].remove_operation(1)
+v_b[0].add_operation('SET', [], cs('NEW-VALUE'))
+v_b[1].add_operation('APPEND', [], cs(' NEW-ADDED'))
+v_a[1].add_operation('APPEND', [], cs(' NEW-ADDED-A'))
+print '\n'.join(root.pprint())
+
+v_a[0].add_operation('SET', [], cs('NEW-VALUE-A0'))
+print '\n'.join(root.pprint())