X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;ds=sidebyside;f=t%2Fmoe%2Fconfig.py;h=bcef93be2cb841e394bc496abeb687078275b1aa;hb=750990d7dbbaf2740f290a26dab527ce4444cbfb;hp=2a8443f10db9e515c9ef42a96e686d858eec19ce;hpb=6b13ac50371c389d1e8196d2a7319b26185dd3db;p=eval.git

diff --git a/t/moe/config.py b/t/moe/config.py
index 2a8443f..bcef93b 100644
--- a/t/moe/config.py
+++ b/t/moe/config.py
@@ -1,197 +1,398 @@
-#!/usr/bin/env python
-
-import re
-import sys
-
-key_pattern = re.compile('^[A-Za-z0-9_-]+$')
-ref_pattern = re.compile('^[A-Za-z0-9_-]+')
-
-class MoeConfigInvalid(Exception):
-    pass
-
-class MoeConfigEvalErr(Exception):
-    pass
-
-class MoeConfig:
-    """Moe configuration file. Should be immutable once a part of a stack."""
-
-    def __init__(self, file=None, name=None):
-        self.vars = {}
-	if file is not None:
-	    self.load(file)
-	elif name is not None:
-	    self.name = name
-	    self.load(open(name, 'r'))
-
-    def parse_line(self, x):
-	x = x.rstrip("\n").lstrip(" \t")
-	if x=='' or x.startswith('#'):
-	    pass
-	else:
-	    sep = x.find('=')
-	    if sep >= 0:
-		k = x[:sep]
-		v = x[sep+1:]
-		if k.endswith("+"):
-		    k = k[:-1]
-		    if not self.vars.has_key(k):
-			self.vars[k] = [('a','')];
-		else:
-		    self.vars[k] = []
-		if not key_pattern.match(k):
-		    raise MoeConfigInvalid, "Malformed name of configuration variable"
-		if v.startswith("'"):
-		    v=v[1:]
-		    if not v.endswith("'"):
-			raise MoeConfigInvalid, "Misquoted string"
-		    self.vars[k].append(('s', v[:-1]))
-		elif v.startswith('"'):
-		    v=v[1:]
-		    if not v.endswith('"'):
-			raise MoeConfigInvalid, "Misquoted string"
-		    self.parse_interpolated(self.vars[k], v[:-1])
-		else:
-		    self.parse_interpolated(self.vars[k], v)
-	    else:
-		raise MoeConfigInvalid, "Parse error"
-
-    def load(self, file):
-        lino = 0
-	for x in file.readlines():
-	    lino += 1
-	    try:
-		self.parse_line(x)
-	    except MoeConfigInvalid, x:
-		msg = x.message + ' at line ' + str(lino)
-		if hasattr(self, 'name'):
-		    msg += ' of ' + self.name
-		raise MoeConfigInvalid, msg
-
-    def parse_interpolated(self, list, s):
-        while s<>'':
-	    if s.startswith('$'):
-	        s = s[1:]
-		if s.startswith('{'):
-		    p = s.find('}')
-		    if not p:
-		    	raise MoeConfigInvalid, "Unbalanced braces"
-		    k, s = s[1:p], s[p+1:]
-		    if not key_pattern.match(k):
-		        raise MoeConfigInvalid, "Invalid variable name"
-		else:
-		    m = ref_pattern.match(s)
-		    if m:
-		        k, s = s[:m.end()], s[m.end():]
-		    else:
-			raise MoeConfigInvalid, "Invalid variable reference"
-		list.append(('i', k))
-	    else:
-	        p = s.find('$')
-		if p < 0:
-		    p = len(s)
-		list.append(('s', s[:p]))
-		s = s[p:]
-
-    def dump(self, file=sys.stdout):
-        for k,v in self.vars.items():
-	    file.write(k)
-	    if len(v) > 0 and v[0][0] == 'a':
-	        file.write('+')
-		v = v[1:]
-	    file.write('=')
-	    for t,w in v:
-	        if t == 's':
-		    file.write("'" + w + "'")
-		elif t == 'i':
-		    file.write('"$' + w + '"')
-	    file.write("\n")
-
-class MoeConfigStack:
-    """Stack of configuration files."""
-
-    def __init__(self, base=None):
-    	## FIXME: Do we need to duplicate the config files themselves?
-        if base:
-	    self.stk = base.stk[:]
-	else:
-	    self.stk = []
-	self.in_progress = {}
-	self.reset_cache()
-
-    def reset_cache(self):
-	self.cache = {}
-
-    def push(self, cfg):
-    	self.stk.append(cfg)
-	self.reset_cache()
-
-    def __getitem__(self, k):
-	if self.cache.has_key(k):
-	    return self.cache[k]
-        if self.in_progress.has_key(k):
-	    raise MoeConfigEvalErr, "Definition of $%s is recursive" % k;
-	self.in_progress[k] = 1;
-        v = self.do_get(k, len(self.stk)-1)
-	del self.in_progress[k]
-	self.cache[k] = v
-	return v
-
-    def do_get(self, k, pos):
-        while pos >= 0:
-	    cfg = self.stk[pos]
-	    if cfg.vars.has_key(k):
-		new = cfg.vars[k]
-		if new[0][0] == 'a':
-		    v = self.do_get(k, pos-1)
-		else:
-		    v = ''
-		for op,arg in new:
-		    if op == 's':
-			v = v + arg
-		    elif op == 'i':
-			v = v + self[arg]
-		return v
-	    pos -= 1
-	return ''
-
-    def keys(self):
-        seen = {}
-	for cfg in self.stk:
-	    for k in cfg.vars.keys():
-		seen[k] = None
-	return seen.keys()
-
-    def dump(self, file=sys.stdout):
-	for k in self.keys():
-	    v = self[k]
-	    file.write("%s=%s\n" % (k,v))
-
-    def dump_defs(self, file=sys.stdout):
-	file.write("Configuration stack:\n")
-	level = 0
-	for cfg in self.stk:
-	    level += 1
-	    file.write("(level %d)\n" % level)
-	    cfg.dump(file)
-	file.write("(end)\n")
-
-    def apply_overrides(self, prefix):
-        newstk = []
-	for cfg in self.stk:
-	    over = MoeConfig()
-	    changed = False
-	    for k in cfg.vars.keys():
-		if k.startswith(prefix):
-		    over.vars[k[len(prefix):]] = cfg.vars[k]
-		    changed = True
-	    if changed:
-		clean = MoeConfig()
-		for k in cfg.vars.keys():
-		    if not k.startswith(prefix):
-			clean.vars[k] = cfg.vars[k]
-		newstk.append(clean)
-		newstk.append(over)
-	    else:
-		newstk.append(cfg)
-	self.stk = newstk
-	self.reset_cache()
+"""
+Lazy conditional string evaluation module for Moe configuration variables.
+
+* Each variable has ordered list of operations (definitions), each defining operation either 
+  assigns (SET) or appends (APPEND) value of an expression to the variable. Each operation may be guarded by condition(s). 
+
+* Each condition is a formula (tree consisting of 'AND', 'OR', 'NOT' and '==', '!=' between two expressions.
+
+* Expression is a list of strings and variables to be expanded.
+
+.. note:: If no 'SET' applies, a variable is still undefined even if some 'APPEND' applies. This might change.
+.. note:: All expanded data should be (or is converted to) unicode 
+.. todo:: (OPT) Cleanup of unused undefined variables.
+.. todo:: (OPT) Better variable name checking (no name '.'-structural prefix of another)
+.. todo:: (OPT) Implemet "subtree" listing.
+"""
+
+import types, itertools, re, bisect
+import logging as log
+
+from moe import MoeError
+
+
+"Allowed depth of recursion - includes ALL recursive calls, so should quite high."
+c_maxdepth = 256
+
+"Maximum attained depth of recursion - for debug/testing"
+debug_maxdepth = 0 
+
+"Variable name regexp, dots (separators) must be separated from edges and each other."
+re_VARNAME = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z')
+
+
+def check_depth(depth):
+  "Helper to check for recursion depth."
+  global debug_maxdepth
+  if depth > c_maxdepth:
+    raise CyclicConfigError('Too deep recursion in config evaluation (cyclic substitution?)')
+  if depth > debug_maxdepth:
+    debug_maxdepth = depth
+
+
+class ConfigError(MoeError):
+  pass
+
+class UndefinedError(ConfigError):
+  pass
+
+class VariableNameError(ConfigError):
+  pass
+
+class VariableFixedError(ConfigError):
+  pass
+
+class CyclicConfigError(ConfigError):
+  pass
+
+
+class ConfigTree(object):
+  """
+  Configuration tree containing all the variables.
+
+  The variables in `self.variables` are referenced directly by the full name.
+  """
+
+  def __init__(self):
+    self.variables = {}
+
+  def lookup(self, key, create = True):
+    """
+    Lookup and return a variable. 
+    If not found and `create` set, check the name and transparently create a new one.
+    """
+    if key not in self.variables:
+      if not re_VARNAME.match(key):
+	raise VariableNameError('Invalid variable identifier %r in config', key)
+      if not create:
+	raise UndefinedError('Config variable %r undefined.', key)
+      self.variables[key] = ConfigVar(key)
+    return self.variables[key]
+
+  def __getitem__(self, key):
+    """
+    Return the value of an existing variable.
+    """
+    return self.lookup(key, create=False).value()
+
+  def dump(self, prefix=''):
+    """
+    Pretty printing of the tree.
+    Returns an iterator of lines (strings).
+    """
+    return itertools.chain(*[
+      self.variables[k].dump(prefix) for k in sorted(self.variables.keys())
+      ])
+
+  def parse(self, s, source=None, level=0):
+    """Parse `s` (stream/string) into the tree, see `moe.confparser.ConfigParser` for details."""
+    import moe.confparser
+    p = moe.confparser.ConfigParser(text, self, source=source, level=level)
+    p.parse()
+
+  def parse_file(self, filename, desc=None, level=0):
+    """Parse an utf-8 file into the tree, see `moe.confparser.ConfigParser` for details. 
+    Names the source "`filename` <`desc`>". """
+    f = open(filename, 'rt')
+    if desc: 
+      filename += " <" + desc + ">" 
+    self.parse(f, source=filename, level=level)
+
+
+class ConfigElem(object):
+  """
+  Base class for cahed config elements - variables and conditions
+  """
+
+  def __init__(self, name):
+    # Full name with separators, definition for conditions
+    self.name = name
+    # Vars and conditions depending on value of this one
+    self.dependants = set([])
+    # Cached value (may be None in case of evaluation error)
+    self.cached = False
+    self.cached_val = None
+
+  def invalidate(self, depth=0):
+    """
+    Invalidate cached data and invalidate all dependants. 
+    Does nothing if not cached.
+    """
+    check_depth(depth)
+    if self.cached:
+      log.debug('invalidating %s', self)
+      self.cached = False
+      for d in self.dependants:
+	d.invalidate(depth + 1)
+
+  def value(self, depth=0):
+    "Caching helper calling self.evaluate(), returns a value or throws an exception."
+    check_depth(depth)
+    if not self.cached:
+      self.cached_val = self.evaluate(depth=depth+1)
+      self.cached = True
+    if self.cached_val == None:
+      raise UndefinedError("Unable to evaluate %r."%(self.name,))
+    return self.cached_val 
+
+  def __str__(self):
+    return self.name
+
+
+class ConfigCondition(ConfigElem):
+  """
+  Condition using equality and logic operators.
+  Clause is a tuple-tree in the following recursive form::
+    
+    ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), ('==', e1, e2), ('!=', e1, e2) 
+    
+  where e1, e2 are `ConfigExpression`, c1, c2, `ConfigCondition`.
+  """
+
+  def __init__(self, formula, text=None, parent=None):
+    """
+    Condition defined by `text` (informative), `formula` as in class definition, 
+    `parent` is the parent condition (if any).
+    """
+    if not text:
+      text = self.formula_string(formula)
+    super(ConfigCondition, self).__init__(text)
+    self.formula = formula
+    self.parent = parent
+    # Setup dependencies on used variables (not on the parent condition)
+    for v in self.variables():
+      v.dependants.add(self)
+    if self.parent:
+      self.parent.dependants.add(self)
+
+  def variables(self, cl=None):
+    "Return an iterator of variables used in formula `cl`"
+    if not cl: 
+      cl = self.formula
+    if cl[0] in ['==','!=']:
+      return itertools.chain(cl[1].variables(), cl[2].variables())
+    if cl[0] in ['AND','OR']:
+      return itertools.chain(self.variables(cl[1]), self.variables(cl[2]))
+    return self.variables(cl[1]) # only 'NOT' left
+
+  def remove_dependencies(self):
+    "Remove self as a dependant from all used variables"
+    for v in self.variables():
+      v.dependants.discard(self)
+    if self.parent:
+      self.parent.dependants.discard(self)
+
+  def evaluate(self, cl=None, depth=0):
+    """Evaluate formula `cl` (or the entire condition).
+    Partial evaluation for AND and OR. Tests the parent condition first."""
+    check_depth(depth)
+    if not cl: 
+      cl = self.formula
+    if self.parent and not self.parent.value():
+      return False
+    if cl[0] in ['==','!=']:
+      v = cl[1].evaluate(depth=depth+1) == cl[2].evaluate(depth=depth+1)
+      if cl[0] == '!=': v = not v
+      return v
+    v1 = self.evaluate(cl=cl[1], depth=depth+1)
+    if cl[0] == 'NOT':
+      return not v1
+    if cl[0] == 'OR' and v1: return True
+    if cl[0] == 'AND' and not v1: return False
+    return self.evaluate(cl=cl[2], depth=depth+1)
+
+  def formula_string(self, formula):
+    "Create a string representation of a formula."
+    if formula[0] == 'AND':
+      return itertools.chain(['('], self.formula_string(formula[1]), [' and '], self.formula_string(formula[2]),[')'])
+    elif formula[0] == 'OR':
+      return itertools.chain(['('], self.formula_string(formula[1]), [' or '], self.formula_string(formula[2]),[')'])
+    elif formula[0] == 'NOT':
+      return itertools.chain(['(not '], self.formula_string(formula[1]),[')'])
+    elif formula[0] in ['==', '!=']:
+      return itertools.chain(formula[1], formula[0], formula[2])
+    return iter(['<invalid formula>'])
+
+  def str(self, parents=False):
+    "Retur the defining expression, if `parents` set, then prefixed with parent conditions."
+    if parents and self.parent:
+      return self.parent.str(parents=True) + u' && ' + self.name
+    return self.name
+
+  def __str__(self):
+    return self.str(parents=False)
+
+
+class Operation(object):
+  "Helper class for operation data. Must not be present in more variables or present multiple times."
+
+  def __init__(self, operation, condition, expression, level=0, source='?'):
+    # operation is currently 'SET' and 'APPEND'
+    self.operation = operation
+    self.condition = condition
+    self.expression = expression
+    self.level = level
+    self.source = source
+
+  def __str__(self):
+    return "%s <%d, %s> [%s] %r" % ( {'SET':'=', 'APPEND':'+'}[self.operation], self.level, self.source, 
+      (self.condition and self.condition.str(parents=True)) or '', unicode(self.expression))
+
+
+class ConfigVar(ConfigElem):
+
+  def __init__(self, name):
+    super(ConfigVar, self).__init__(name)
+    # Ordered list of `Operations` (ascending by `level`)
+    self.operations = []
+    # Fixed to value (may be None) 
+    self.fixed = False
+    self.fixed_val = None
+
+  def variables(self):
+    "Return a set of variables used in the expressions"
+    return set(sum([ list(op.expression.variables()) for op in self.operations ], []))
+
+  def fix(self):
+    """
+    Fixes the value of the variable. Exception is raised should the variable
+    evaluate to a different value while fixed. 
+    """
+    if self.fixed: 
+      return 
+    self.fixed_val = self.value()
+    self.fixed = True
+
+  def unfix(self):
+    "Set the variable to be modifiable again."
+    self.fixed = False
+
+  def value(self, depth=0):
+    "Handle the case when fixed, raise exc. on different evaluation"
+    val = super(ConfigVar,self).value(depth)
+    if self.fixed and self.fixed_val != val:
+      raise VariableFixedError("value of var %s was fixed to %r but evaluated to %r", self.name, self.fixed_val, val)
+    return val
+
+  def add_operation(self, operation):
+    """
+    Inserts an operation. The operations are sorted by `level` (ascending), new operation goes last among
+    these with the same level.
+    Adds the variable as a dependant of the conditions and variables used in the expressions. 
+    """
+    # Invalidate cached value
+    self.invalidate()
+    # Add the operation 
+    pos = bisect.bisect_right([o.level for o in self.operations], operation.level)
+    self.operations.insert(pos, operation)
+    # Create dependencies
+    for v in operation.expression.variables():
+      v.dependants.add(self)
+    if operation.condition:
+      operation.condition.dependants.add(self)
+
+  def remove_operation(self, operation):
+    """
+    Remove the Operation.
+    Also removes the variable as dependant from all conditions and variables used in this 
+    operation that are no longer used. 
+    """
+    # Invalidate cached value
+    self.invalidate()
+    # Remove the operation 
+    self.operations.remove(operation)
+    # Remove dependencies on variables unused in other operations
+    vs = self.variables()
+    for v in operation.expression.variables():
+      if v not in vs:
+	v.dependants.remove(self)
+    # Remove the dependency on the conditions (if not used in another operation)
+    if operation.condition and operation.condition not in [op.condition for op in self.operations]:
+      operation.condition.dependants.remove(self)
+
+  def evaluate(self, depth=0):
+    """
+    Find the last 'SET' operation that applies and return the result of concatenating with all
+    subsequent applicable 'APPEND' operations. The result is the same as performing the operations 
+    first-to-last.
+    NOTE: undefined if some 'APPEND' apply but no 'SET' applies.
+    """
+    check_depth(depth)
+    log.debug('evaluating var %r', self.name)
+    # List of strings to be concatenated
+    val = []
+    # Scan for last applicable expression - try each starting from the end, concatenate extensions
+    for i in range(len(self.operations)-1, -1, -1):
+      op = self.operations[i]
+      # Check the guarding condition
+      if (not op.condition) or op.condition.value(depth+1):
+	val.insert(0, op.expression.evaluate(depth=depth+1))
+	if op.operation == 'SET':
+	  return u''.join(val)
+    return None
+
+  def dump(self, prefix=''):
+    """
+    Pretty printing of the variable. Includes all operations.
+    Returns iterator of lines (unicode strings).
+    """
+    # Try to evaluate the variable, but avoid undefined exceptions 
+    v = None
+    try: 
+      v = self.value(depth=0)
+    except ConfigError: 
+      pass
+    yield prefix+u'%s = %r' % (self.name, v)
+    for op in self.operations:
+      #yield prefix+u'  %s [%s] %s' % (op.operation, op.condition and op.condition.str(parents=True), op.expression)
+      yield prefix + u'  ' + unicode(op)
+
+
+class ConfigExpression(object):
+  """
+  String expression with some unexpanded config variables. Used in variable operations and conditions.
+  Expression is given as a list of unicode strings and ConfigVar variables to be expanded.
+  """
+
+  def __init__(self, exprlist, original = u'<unknown>'):
+    self.exprlist = list(exprlist)
+    # Original defining string 
+    self.original = original
+    # Replace strings with unicode
+    for i in range(len(self.exprlist)):
+      e = self.exprlist[i]
+      if isinstance(e, types.StringTypes):
+	if not isinstance(e, unicode):
+	  self.exprlist[i] = unicode(e, 'ascii')
+
+  def variables(self):
+    "Return an iterator of variables user in the expression"
+    return itertools.ifilter(lambda e: isinstance(e, ConfigVar), self.exprlist)
+
+  def __str__(self):
+    return self.original
+
+  def evaluate(self, depth):
+    check_depth(depth)
+    "Return unicode result of expansion of the variables."
+    s = []
+    for e in self.exprlist:
+      if isinstance(e, ConfigVar):
+	s.append(e.value(depth+1))
+      elif isinstance(e, unicode):
+	s.append(e)
+      else:
+	raise ConfigError('Invalid type %s in expression \'%s\'.'%(type(e), self))
+    return u''.join(s)
+
+