X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;f=t%2Fmoe%2Fconfig.py;h=ec057e1cc3bb9edeb347a0ca9fe348fe684e3cfe;hb=9f4eaf4234e6a7074fca72a3a9229d75035daac8;hp=3075d8036989a461f47f822499bcec5a4c6a1fc2;hpb=e47d578a9cc92cbaf5291ee87d4e2a9da2d76fef;p=moe.git

diff --git a/t/moe/config.py b/t/moe/config.py
index 3075d80..ec057e1 100644
--- a/t/moe/config.py
+++ b/t/moe/config.py
@@ -1,13 +1,445 @@
-#!/usr/bin/env python
+"""
+Module for managing and evaluation of Moe configuration variables.
 
-class MoeConfig:
-    """Moe configuration file."""
+.. todo:: (OPT) Cleanup of unused undefined variables.
+.. todo:: (OPT) Better variable name checking (no name '.'-structural prefix of another)
+.. todo:: (OPT) Implemet "subtree" listing.
+"""
 
-    def __init__(self):
-        cfg = { 'XYZZY' : 'brum' }
+import types, itertools, re, bisect
+import logging as log
+
+from moe import MoeError
+
+
+"Allowed depth of recursion - includes ALL recursive calls, so should quite high."
+c_maxdepth = 256
+
+"Maximum attained depth of recursion - for debug/testing"
+debug_maxdepth = 0 
+
+"Variable name regexp, dots (separators) must be separated from edges and each other."
+re_VARNAME = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z')
+
+
+def check_depth(depth):
+  "Helper to check for recursion depth."
+  global debug_maxdepth
+  if depth > c_maxdepth:
+    raise CyclicConfigError('Too deep recursion in config evaluation (cyclic substitution?)')
+  if depth > debug_maxdepth:
+    debug_maxdepth = depth
+
+
+class ConfigError(MoeError):
+  "Base class for moe.config errors"
+  pass
+
+class UndefinedError(ConfigError):
+  "Raised when no **SET** operation applies to evaluated variable."
+  pass
+
+class VariableNameError(ConfigError):
+  "Raised on invalid config variable name."
+  pass
+
+class VariableFixedError(ConfigError):
+  "Raised when modifying a fixed variable"
+  pass
+
+class CyclicConfigError(ConfigError):
+  "Raised when evaluation recursion is too deep"
+  pass
+
+
+class ParseProxy(list):
+  """Proxy helper class around values returned by `parse` and `parse_file`, 
+  useful in "with" constructs."""
+  def __init__(self, config, parsed_ops):
+    super(ParseProxy, self).__init__(parsed_ops)  
+    self.config = config
+  def __enter__(self):
+    pass
+  def __exit__(self, etype, value, traceback):
+    self.config.remove(list(self))
+
+
+class ConfigTree(object):
+  """
+  Configuration environment containing the variables.
+  """
+
+  def __init__(self):
+    self.variables = {}
+
+  def lookup(self, key, create = True):
+    """
+    Lookup and return a variable. 
+    If not found and `create` set, check the name and transparently create a new one.
+    """
+    if key not in self.variables:
+      if not re_VARNAME.match(key):
+	raise VariableNameError('Invalid variable identifier %r in config', key)
+      if not create:
+	raise UndefinedError('Config variable %r undefined.', key)
+      self.variables[key] = ConfigVar(key)
+    return self.variables[key]
+
+  def __getitem__(self, key):
+    """
+    Return the value of an existing variable.
+    """
+    return self.lookup(key, create=False).value()
+
+  def dump(self, prefix=''):
+    """
+    Pretty printing of the tree.
+    Returns an iterator of lines (strings).
+    """
+    return itertools.chain(*[
+      self.variables[k].dump(prefix) for k in sorted(self.variables.keys())
+      ])
+
+  def fix(self, keys):
+    "Fix value of variable or list of variables. Fixing undefined variable raises `UndefinedError`."
+    if isinstance(keys, types.StringTypes):
+      keys = [keys]
+    for key in keys:
+      self.lookup(key, create=False).fix()
+  
+  def unfix(self, keys):
+    "Unfix value of variable or list of variables. Unfixing undefined variable raises `UndefinedError`."
+    if isinstance(keys, types.StringTypes):
+      keys = [keys]
+    for key in keys:
+      self.lookup(key, create=False).unfix()
+
+  def remove(self, parsed):
+    """Given a list [(varname, `Operation`)] as returned by `parse` or `parse_file`, 
+    removes the operations from the respective variables config tree.
+    Variables/operations not present int the tree raise ValueError.
+    """
+    for vname, o in parsed:
+      v = self.lookup(vname, create = True)
+      v.remove_operation(o)
+
+  def parse(self, s, source=None, level=0, proxy=True):
+    """Parse `s` (stream/string) into the tree, see `moe.config_parser.ConfigParser` for details.
+    Returns list of parset operations: [(varname, `Operation`)].
+    By default returns a proxy list-like object that can be used in "with" constructs:
+      
+      with config.parse("TEST='1'"):
+	print config['TEST']
+	raise StupidError
+    """
+    import moe.config_parser
+    p = moe.config_parser.ConfigParser(s, self, source=source, level=level)
+    l = p.parse()
+    if not proxy:
+      return l
+    return ParseProxy(self, l)
+
+  def parse_file(self, filename, desc=None, level=0, proxy=True):
+    """Parse an utf-8 file into the tree using func:`parse`. 
+    Names the source "`filename` <`desc`>". """
+    with open(filename, 'rt') as f:
+      if desc: 
+	filename += " <" + desc + ">" 
+      return self.parse(f, source=filename, level=level, proxy=proxy)
+
+
+class ConfigElem(object):
+  """
+  Base class for cached config elements - variables and conditions
+  """
+
+  def __init__(self, name):
+    # Full name with separators, definition for conditions
+    self.name = name
+    # Vars and conditions depending on value of this one
+    self.dependants = set([])
+    # Cached value (may be None in case of evaluation error)
+    self.cached = False
+    self.cached_val = None
+
+  def invalidate(self, depth=0):
+    """
+    Invalidate cached data and invalidate all dependants. 
+    Does nothing if not cached.
+    """
+    check_depth(depth)
+    if self.cached:
+      log.debug('invalidating %s', self)
+      self.cached = False
+      for d in self.dependants:
+	d.invalidate(depth + 1)
+
+  def value(self, depth=0):
+    "Caching helper calling self.evaluate(), returns a value or throws an exception."
+    check_depth(depth)
+    if not self.cached:
+      self.cached_val = self.evaluate(depth=depth+1)
+      self.cached = True
+    if self.cached_val == None:
+      raise UndefinedError("Unable to evaluate %r."%(self.name,))
+    return self.cached_val 
+
+  def __str__(self):
+    return self.name
+
+
+class ConfigCondition(ConfigElem):
+  """
+  Condition using equality and logic operators.
+  Formula is a tuple-tree in the following recursive form::
+    
+    ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), ('==', e1, e2), ('!=', e1, e2) 
+    
+  where ``e1``, ``e2`` are :class:`ConfigExpression`, ``c1``, ``c2``, :class:`ConfigCondition`.
+  """
+
+  def __init__(self, formula, text=None, parent=None):
+    """
+    Condition defined by `text` (informative), `formula` as in class definition, 
+    `parent` is the parent condition (if any).
+    """
+    if not text:
+      text = self.formula_string(formula)
+    super(ConfigCondition, self).__init__(text)
+    self.formula = formula
+    self.parent = parent
+    # Setup dependencies on used variables (not on the parent condition)
+    for v in self.variables():
+      v.dependants.add(self)
+    if self.parent:
+      self.parent.dependants.add(self)
+
+  def variables(self, cl=None):
+    "Return an iterator of variables used in formula `cl`"
+    if not cl: 
+      cl = self.formula
+    if cl[0] in ['==','!=']:
+      return itertools.chain(cl[1].variables(), cl[2].variables())
+    if cl[0] in ['AND','OR']:
+      return itertools.chain(self.variables(cl[1]), self.variables(cl[2]))
+    return self.variables(cl[1]) # only 'NOT' left
+
+  def remove_dependencies(self):
+    "Remove self as a dependant from all used variables"
+    for v in self.variables():
+      v.dependants.discard(self)
+    if self.parent:
+      self.parent.dependants.discard(self)
+
+  def evaluate(self, cl=None, depth=0):
+    """Evaluate formula `cl` (or the entire condition).
+    Partial evaluation for AND and OR. Tests the parent condition first."""
+    check_depth(depth)
+    if not cl: 
+      cl = self.formula
+    if self.parent and not self.parent.value():
+      return False
+    if cl[0] in ['==','!=']:
+      v = cl[1].evaluate(depth=depth+1) == cl[2].evaluate(depth=depth+1)
+      if cl[0] == '!=': v = not v
+      return v
+    v1 = self.evaluate(cl=cl[1], depth=depth+1)
+    if cl[0] == 'NOT':
+      return not v1
+    if cl[0] == 'OR' and v1: return True
+    if cl[0] == 'AND' and not v1: return False
+    return self.evaluate(cl=cl[2], depth=depth+1)
+
+  def formula_string(self, formula):
+    "Create a string representation of a formula."
+    if formula[0] == 'AND':
+      return itertools.chain(['('], self.formula_string(formula[1]), [' and '], self.formula_string(formula[2]),[')'])
+    elif formula[0] == 'OR':
+      return itertools.chain(['('], self.formula_string(formula[1]), [' or '], self.formula_string(formula[2]),[')'])
+    elif formula[0] == 'NOT':
+      return itertools.chain(['(not '], self.formula_string(formula[1]),[')'])
+    elif formula[0] in ['==', '!=']:
+      return itertools.chain(formula[1], formula[0], formula[2])
+    return iter(['<invalid formula>'])
+
+  def str(self, parents=False):
+    "Retur the defining expression, if `parents` set, then prefixed with parent conditions."
+    if parents and self.parent:
+      return self.parent.str(parents=True) + u' && ' + self.name
+    return self.name
+
+  def __str__(self):
+    return self.str(parents=False)
+
+
+class Operation(object):
+  """
+  Helper class for operation data. Must be present at most once in at most one variable.
+
+  ``operation`` is either ``"SET"`` or ``"APPEND"``, ``condition`` is a :class:`ConfigCondition` instance or ``None``,
+  ``expression`` is a :class:`ConfigExpression` instance, ``level`` is the priority of the operation and ``source``
+  is an informative string describing the operation origin.  
+  """
+
+  def __init__(self, operation, condition, expression, level=0, source='?'):
+    self.operation = operation
+    self.condition = condition
+    self.expression = expression
+    self.level = level
+    self.source = source
+
+  def __str__(self):
+    return "%s <%d, %s> [%s] %r" % ( {'SET':'=', 'APPEND':'+'}[self.operation], self.level, self.source, 
+      (self.condition and self.condition.str(parents=True)) or '', unicode(self.expression))
+
+
+class ConfigVar(ConfigElem):
+  "Class representing a single configuration variable"
+
+  def __init__(self, name):
+    super(ConfigVar, self).__init__(name)
+    # Ordered list of `Operations` (ascending by `level`)
+    self.operations = []
+    # Fixed to value (may be None) 
+    self.fixed = False
+    self.fixed_val = None
+
+  def variables(self):
+    "Return a set of variables used in the expressions of the operations"
+    if not self.operations:
+      return set([])
+    return set.union(*[ op.expression.variables() for op in self.operations ])
+
+  def fix(self):
+    """
+    Fixes the value of the variable. Exception is raised should the variable
+    evaluate to a different value while fixed. 
+    """
+    if self.fixed: 
+      return 
+    self.fixed_val = self.value()
+    self.fixed = True
+
+  def unfix(self):
+    "Make the variable modifiable again."
+    self.fixed = False
+
+  def value(self, depth=0):
+    "Handle the case when fixed, raise exc. on different evaluation"
+    val = super(ConfigVar,self).value(depth)
+    if self.fixed and self.fixed_val != val:
+      raise VariableFixedError("value of var %r was fixed to %r but evaluated to %r", self.name, self.fixed_val, val)
+    return val
+
+  def add_operation(self, operation):
+    """
+    Inserts an operation. The operations are sorted by `level` (ascending), new operation goes last among
+    these with the same level.
+    Adds the variable as a dependant of the conditions and variables used in the expressions. 
+    """
+    # Invalidate cached value
+    self.invalidate()
+    # Add the operation 
+    pos = bisect.bisect_right([o.level for o in self.operations], operation.level)
+    self.operations.insert(pos, operation)
+    # Create dependencies
+    for v in operation.expression.variables():
+      v.dependants.add(self)
+    if operation.condition:
+      operation.condition.dependants.add(self)
+
+  def remove_operation(self, operation):
+    """
+    Remove the Operation.
+    Also removes the variable as dependant from all conditions and variables used in this 
+    operation that are no longer used. 
+    """
+    # Invalidate cached value
+    self.invalidate()
+    # Remove the operation 
+    self.operations.remove(operation)
+    # Remove dependencies on variables unused in other defining operations
+    vs = self.variables()
+    for v in operation.expression.variables():
+      if v not in vs:
+	v.dependants.remove(self)
+    # Remove the dependency on the conditions (if not used in another operation)
+    if operation.condition and operation.condition not in [op.condition for op in self.operations]:
+      operation.condition.dependants.remove(self)
+
+  def evaluate(self, depth=0):
+    """
+    Find the last 'SET' operation that applies and return the result of concatenating with all
+    subsequent applicable 'APPEND' operations. The result is the same as performing the operations 
+    first-to-last.
+    NOTE: undefined if some 'APPEND' apply but no 'SET' applies.
+    """
+    check_depth(depth)
+    log.debug('evaluating var %r', self.name)
+    # List of strings to be concatenated
+    val = []
+    # Scan for last applicable expression - try each starting from the end, concatenate extensions
+    for i in range(len(self.operations)-1, -1, -1):
+      op = self.operations[i]
+      # Check the guarding condition
+      if (not op.condition) or op.condition.value(depth+1):
+	val.insert(0, op.expression.evaluate(depth=depth+1))
+	if op.operation == 'SET':
+	  return u''.join(val)
+    return None
+
+  def dump(self, prefix=''):
+    """
+    Pretty printing of the variable. Includes all operations.
+    Returns iterator of lines (unicode strings).
+    """
+    # Try to evaluate the variable, but avoid undefined exceptions 
+    v = None
+    try: 
+      v = self.value(depth=0)
+    except ConfigError: 
+      pass
+    yield prefix+u'%s = %r' % (self.name, v)
+    for op in self.operations:
+      #yield prefix+u'  %s [%s] %s' % (op.operation, op.condition and op.condition.str(parents=True), op.expression)
+      yield prefix + u'  ' + unicode(op)
+
+
+class ConfigExpression(object):
+  """
+  String expression with some unexpanded config variables. Used in variable operations and conditions.
+  Expression is given as a list of unicode strings and ConfigVar variables to be expanded.
+  """
+
+  def __init__(self, exprlist, original = u'<unknown>'):
+    self.exprlist = list(exprlist)
+    # Original defining string 
+    self.original = original
+    # Replace strings with unicode
+    for i in range(len(self.exprlist)):
+      e = self.exprlist[i]
+      if isinstance(e, types.StringTypes):
+	if not isinstance(e, unicode):
+	  self.exprlist[i] = unicode(e, 'ascii')
+
+  def variables(self):
+    "Return a set of variables used in the expression"
+    return set([e for e in self.exprlist if isinstance(e, ConfigVar)])
+
+  def __str__(self):
+    return self.original
+
+  def evaluate(self, depth):
+    check_depth(depth)
+    "Return unicode result of expansion of the variables."
+    s = []
+    for e in self.exprlist:
+      if isinstance(e, ConfigVar):
+	s.append(e.value(depth+1))
+      elif isinstance(e, unicode):
+	s.append(e)
+      else:
+	raise ConfigError('Invalid type %s in expression \'%s\'.'%(type(e), self))
+    return u''.join(s)
 
-class MoeConfigStack:
-    """Stack of configuration files."""
 
-    def __init__(self):
-        stk = []