-#!/usr/bin/env python
-
-import re
-import sys
-
-key_pattern = re.compile("^[A-Za-z0-9_-]+$")
-ref_pattern = re.compile("^[A-Za-z0-9_-]+")
-
-class MoeConfigInvalid(Exception):
- pass
-
-class MoeConfigEvalErr(Exception):
- pass
-
-class MoeConfig:
- """Moe configuration file. Should be immutable once a part of a stack."""
-
- def __init__(self, file=None, name=None):
- self.vars = {}
- if file is not None:
- self.load(file)
- elif name is not None:
- self.name = name
- self.load(open(name, "r"))
-
- def parse_line(self, x):
- x = x.rstrip("\n").lstrip(" \t")
- if x=="" or x.startswith("#"):
- pass
- else:
- sep = x.find("=")
- if sep >= 0:
- k = x[:sep]
- v = x[sep+1:]
- if k.endswith("+"):
- k = k[:-1]
- if not self.vars.has_key(k):
- self.vars[k] = [("a","")];
- else:
- self.vars[k] = []
- if not key_pattern.match(k):
- raise MoeConfigInvalid, "Malformed name of configuration variable"
- if v.startswith("'"):
- v=v[1:]
- if not v.endswith("'"):
- raise MoeConfigInvalid, "Misquoted string"
- self.vars[k].append(("s", v[:-1]))
- elif v.startswith('"'):
- v=v[1:]
- if not v.endswith('"'):
- raise MoeConfigInvalid, "Misquoted string"
- self.parse_interpolated(self.vars[k], v[:-1])
- else:
- self.parse_interpolated(self.vars[k], v)
- else:
- raise MoeConfigInvalid, "Parse error"
-
- def load(self, file):
- lino = 0
- for x in file.readlines():
- lino += 1
- try:
- self.parse_line(x)
- except MoeConfigInvalid, x:
- msg = x.message + " at line " + str(lino)
- if hasattr(self, "name"):
- msg += " of " + self.name
- raise MoeConfigInvalid, msg
-
- def parse_interpolated(self, list, s):
- while s<>"":
- if s.startswith("$"):
- s = s[1:]
- if s.startswith("{"):
- p = s.find("}")
- if not p:
- raise MoeConfigInvalid, "Unbalanced braces"
- k, s = s[1:p], s[p+1:]
- if not key_pattern.match(k):
- raise MoeConfigInvalid, "Invalid variable name"
- else:
- m = ref_pattern.match(s)
- if m:
- k, s = s[:m.end()], s[m.end():]
- else:
- raise MoeConfigInvalid, "Invalid variable reference"
- list.append(("i", k))
- else:
- p = s.find("$")
- if p < 0:
- p = len(s)
- list.append(("s", s[:p]))
- s = s[p:]
-
- def dump(self, file=sys.stdout):
- for k,v in self.vars.items():
- file.write(k)
- if len(v) > 0 and v[0][0] == "a":
- file.write("+")
- v = v[1:]
- file.write("=")
- for t,w in v:
- if t == "s":
- file.write("'" + w + "'")
- elif t == "i":
- file.write('"$' + w + '"')
- file.write("\n")
-
-class MoeConfigStack:
- """Stack of configuration files."""
-
- def __init__(self, base=None):
- ## FIXME: Do we need to duplicate the config files themselves?
- if base:
- self.stk = base.stk[:]
- else:
- self.stk = []
- self.in_progress = {}
- self.reset_cache()
-
- def reset_cache(self):
- self.cache = {}
-
- def push(self, cfg):
- self.stk.append(cfg)
- self.reset_cache()
-
- def __getitem__(self, k):
- if self.cache.has_key(k):
- return self.cache[k]
- if self.in_progress.has_key(k):
- raise MoeConfigEvalErr, "Definition of $%s is recursive" % k;
- self.in_progress[k] = 1;
- v = self.do_get(k, len(self.stk)-1)
- del self.in_progress[k]
- self.cache[k] = v
- return v
-
- def do_get(self, k, pos):
- while pos >= 0:
- cfg = self.stk[pos]
- if cfg.vars.has_key(k):
- new = cfg.vars[k]
- if new[0][0] == "a":
- v = self.do_get(k, pos-1)
- else:
- v = ""
- for op,arg in new:
- if op == "s":
- v = v + arg
- elif op == "i":
- v = v + self[arg]
- return v
- pos -= 1
- return ""
-
- def keys(self):
- seen = {}
- for cfg in self.stk:
- for k in cfg.vars.keys():
- seen[k] = None
- return seen.keys()
-
- def dump(self, file=sys.stdout):
- for k in self.keys():
- v = self[k]
- file.write("%s=%s\n" % (k,v))
-
- def dump_defs(self, file=sys.stdout):
- file.write("Configuration stack:\n")
- level = 0
- for cfg in self.stk:
- level += 1
- file.write("(level %d)\n" % level)
- cfg.dump(file)
- file.write("(end)\n")
-
- def apply_overrides(self, prefix):
- newstk = []
- for cfg in self.stk:
- over = MoeConfig()
- changed = False
- for k in cfg.vars.keys():
- if k.startswith(prefix):
- over.vars[k[len(prefix):]] = cfg.vars[k]
- changed = True
- if changed:
- clean = MoeConfig()
- for k in cfg.vars.keys():
- if not k.startswith(prefix):
- clean.vars[k] = cfg.vars[k]
- newstk.append(clean)
- newstk.append(over)
- else:
- newstk.append(cfg)
- self.stk = newstk
- self.reset_cache()
+"""
+Lazy conditional string evaluation module for Moe configuration variables.
+
+* Each variable has ordered list of operations (definitions), each defining operation either
+ assigns (SET) or appends (APPEND) value of an expression to the variable. Each operation may be guarded by condition(s).
+
+* Each condition is a formula (tree consisting of 'AND', 'OR', 'NOT' and '==', '!=' between two expressions.
+
+* Expression is a list of strings and variables to be expanded.
+
+.. note:: If no 'SET' applies, a variable is still undefined even if some 'APPEND' applies. This might change.
+.. note:: All expanded data should be (or is converted to) unicode
+.. todo:: (OPT) Cleanup of unused undefined variables.
+.. todo:: (OPT) Better variable name checking (no name '.'-structural prefix of another)
+.. todo:: (OPT) Implemet "subtree" listing.
+"""
+
+import types, itertools, re, bisect
+import logging as log
+
+from moe import MoeError
+
+
+"Allowed depth of recursion - includes ALL recursive calls, so should quite high."
+c_maxdepth = 256
+
+"Maximum attained depth of recursion - for debug/testing"
+debug_maxdepth = 0
+
+"Variable name regexp, dots (separators) must be separated from edges and each other."
+re_VARNAME = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z')
+
+
+def check_depth(depth):
+ "Helper to check for recursion depth."
+ global debug_maxdepth
+ if depth > c_maxdepth:
+ raise CyclicConfigError('Too deep recursion in config evaluation (cyclic substitution?)')
+ if depth > debug_maxdepth:
+ debug_maxdepth = depth
+
+
+class ConfigError(MoeError):
+ pass
+
+class UndefinedError(ConfigError):
+ pass
+
+class VariableNameError(ConfigError):
+ pass
+
+class VariableFixedError(ConfigError):
+ pass
+
+class CyclicConfigError(ConfigError):
+ pass
+
+
+class ConfigTree(object):
+ """
+ Configuration tree containing all the variables.
+
+ The variables in `self.variables` are referenced directly by the full name.
+ """
+
+ def __init__(self):
+ self.variables = {}
+
+ def lookup(self, key, create = True):
+ """
+ Lookup and return a variable.
+ If not found and `create` set, check the name and transparently create a new one.
+ """
+ if key not in self.variables:
+ if not re_VARNAME.match(key):
+ raise VariableNameError('Invalid variable identifier %r in config', key)
+ if not create:
+ raise UndefinedError('Config variable %r undefined.', key)
+ self.variables[key] = ConfigVar(key)
+ return self.variables[key]
+
+ def __getitem__(self, key):
+ """
+ Return the value of an existing variable.
+ """
+ return self.lookup(key, create=False).value()
+
+ def dump(self, prefix=''):
+ """
+ Pretty printing of the tree.
+ Returns an iterator of lines (strings).
+ """
+ return itertools.chain(*[
+ self.variables[k].dump(prefix) for k in sorted(self.variables.keys())
+ ])
+
+ def fix(self, key):
+ "Fix variable value. Fixing undefined variable raises `UndefinedError`."
+ self.lookup(key, create=True).fix()
+
+ def parse(self, s, source=None, level=0):
+ """Parse `s` (stream/string) into the tree, see `moe.confparser.ConfigParser` for details."""
+ import moe.confparser
+ p = moe.confparser.ConfigParser(text, self, source=source, level=level)
+ p.parse()
+
+ def parse_file(self, filename, desc=None, level=0):
+ """Parse an utf-8 file into the tree, see `moe.confparser.ConfigParser` for details.
+ Names the source "`filename` <`desc`>". """
+ with open(filename, 'rt') as f:
+ if desc:
+ filename += " <" + desc + ">"
+ self.parse(f, source=filename, level=level)
+
+
+class ConfigElem(object):
+ """
+ Base class for cahed config elements - variables and conditions
+ """
+
+ def __init__(self, name):
+ # Full name with separators, definition for conditions
+ self.name = name
+ # Vars and conditions depending on value of this one
+ self.dependants = set([])
+ # Cached value (may be None in case of evaluation error)
+ self.cached = False
+ self.cached_val = None
+
+ def invalidate(self, depth=0):
+ """
+ Invalidate cached data and invalidate all dependants.
+ Does nothing if not cached.
+ """
+ check_depth(depth)
+ if self.cached:
+ log.debug('invalidating %s', self)
+ self.cached = False
+ for d in self.dependants:
+ d.invalidate(depth + 1)
+
+ def value(self, depth=0):
+ "Caching helper calling self.evaluate(), returns a value or throws an exception."
+ check_depth(depth)
+ if not self.cached:
+ self.cached_val = self.evaluate(depth=depth+1)
+ self.cached = True
+ if self.cached_val == None:
+ raise UndefinedError("Unable to evaluate %r."%(self.name,))
+ return self.cached_val
+
+ def __str__(self):
+ return self.name
+
+
+class ConfigCondition(ConfigElem):
+ """
+ Condition using equality and logic operators.
+ Clause is a tuple-tree in the following recursive form::
+
+ ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), ('==', e1, e2), ('!=', e1, e2)
+
+ where e1, e2 are `ConfigExpression`, c1, c2, `ConfigCondition`.
+ """
+
+ def __init__(self, formula, text=None, parent=None):
+ """
+ Condition defined by `text` (informative), `formula` as in class definition,
+ `parent` is the parent condition (if any).
+ """
+ if not text:
+ text = self.formula_string(formula)
+ super(ConfigCondition, self).__init__(text)
+ self.formula = formula
+ self.parent = parent
+ # Setup dependencies on used variables (not on the parent condition)
+ for v in self.variables():
+ v.dependants.add(self)
+ if self.parent:
+ self.parent.dependants.add(self)
+
+ def variables(self, cl=None):
+ "Return an iterator of variables used in formula `cl`"
+ if not cl:
+ cl = self.formula
+ if cl[0] in ['==','!=']:
+ return itertools.chain(cl[1].variables(), cl[2].variables())
+ if cl[0] in ['AND','OR']:
+ return itertools.chain(self.variables(cl[1]), self.variables(cl[2]))
+ return self.variables(cl[1]) # only 'NOT' left
+
+ def remove_dependencies(self):
+ "Remove self as a dependant from all used variables"
+ for v in self.variables():
+ v.dependants.discard(self)
+ if self.parent:
+ self.parent.dependants.discard(self)
+
+ def evaluate(self, cl=None, depth=0):
+ """Evaluate formula `cl` (or the entire condition).
+ Partial evaluation for AND and OR. Tests the parent condition first."""
+ check_depth(depth)
+ if not cl:
+ cl = self.formula
+ if self.parent and not self.parent.value():
+ return False
+ if cl[0] in ['==','!=']:
+ v = cl[1].evaluate(depth=depth+1) == cl[2].evaluate(depth=depth+1)
+ if cl[0] == '!=': v = not v
+ return v
+ v1 = self.evaluate(cl=cl[1], depth=depth+1)
+ if cl[0] == 'NOT':
+ return not v1
+ if cl[0] == 'OR' and v1: return True
+ if cl[0] == 'AND' and not v1: return False
+ return self.evaluate(cl=cl[2], depth=depth+1)
+
+ def formula_string(self, formula):
+ "Create a string representation of a formula."
+ if formula[0] == 'AND':
+ return itertools.chain(['('], self.formula_string(formula[1]), [' and '], self.formula_string(formula[2]),[')'])
+ elif formula[0] == 'OR':
+ return itertools.chain(['('], self.formula_string(formula[1]), [' or '], self.formula_string(formula[2]),[')'])
+ elif formula[0] == 'NOT':
+ return itertools.chain(['(not '], self.formula_string(formula[1]),[')'])
+ elif formula[0] in ['==', '!=']:
+ return itertools.chain(formula[1], formula[0], formula[2])
+ return iter(['<invalid formula>'])
+
+ def str(self, parents=False):
+ "Retur the defining expression, if `parents` set, then prefixed with parent conditions."
+ if parents and self.parent:
+ return self.parent.str(parents=True) + u' && ' + self.name
+ return self.name
+
+ def __str__(self):
+ return self.str(parents=False)
+
+
+class Operation(object):
+ "Helper class for operation data. Must not be present in more variables or present multiple times."
+
+ def __init__(self, operation, condition, expression, level=0, source='?'):
+ # operation is currently 'SET' and 'APPEND'
+ self.operation = operation
+ self.condition = condition
+ self.expression = expression
+ self.level = level
+ self.source = source
+
+ def __str__(self):
+ return "%s <%d, %s> [%s] %r" % ( {'SET':'=', 'APPEND':'+'}[self.operation], self.level, self.source,
+ (self.condition and self.condition.str(parents=True)) or '', unicode(self.expression))
+
+
+class ConfigVar(ConfigElem):
+
+ def __init__(self, name):
+ super(ConfigVar, self).__init__(name)
+ # Ordered list of `Operations` (ascending by `level`)
+ self.operations = []
+ # Fixed to value (may be None)
+ self.fixed = False
+ self.fixed_val = None
+
+ def variables(self):
+ "Return a set of variables used in the expressions"
+ return set(sum([ list(op.expression.variables()) for op in self.operations ], []))
+
+ def fix(self):
+ """
+ Fixes the value of the variable. Exception is raised should the variable
+ evaluate to a different value while fixed.
+ """
+ if self.fixed:
+ return
+ self.fixed_val = self.value()
+ self.fixed = True
+
+ def unfix(self):
+ "Make the variable modifiable again."
+ self.fixed = False
+
+ def value(self, depth=0):
+ "Handle the case when fixed, raise exc. on different evaluation"
+ val = super(ConfigVar,self).value(depth)
+ if self.fixed and self.fixed_val != val:
+ raise VariableFixedError("value of var %s was fixed to %r but evaluated to %r", self.name, self.fixed_val, val)
+ return val
+
+ def add_operation(self, operation):
+ """
+ Inserts an operation. The operations are sorted by `level` (ascending), new operation goes last among
+ these with the same level.
+ Adds the variable as a dependant of the conditions and variables used in the expressions.
+ """
+ # Invalidate cached value
+ self.invalidate()
+ # Add the operation
+ pos = bisect.bisect_right([o.level for o in self.operations], operation.level)
+ self.operations.insert(pos, operation)
+ # Create dependencies
+ for v in operation.expression.variables():
+ v.dependants.add(self)
+ if operation.condition:
+ operation.condition.dependants.add(self)
+
+ def remove_operation(self, operation):
+ """
+ Remove the Operation.
+ Also removes the variable as dependant from all conditions and variables used in this
+ operation that are no longer used.
+ """
+ # Invalidate cached value
+ self.invalidate()
+ # Remove the operation
+ self.operations.remove(operation)
+ # Remove dependencies on variables unused in other operations
+ vs = self.variables()
+ for v in operation.expression.variables():
+ if v not in vs:
+ v.dependants.remove(self)
+ # Remove the dependency on the conditions (if not used in another operation)
+ if operation.condition and operation.condition not in [op.condition for op in self.operations]:
+ operation.condition.dependants.remove(self)
+
+ def evaluate(self, depth=0):
+ """
+ Find the last 'SET' operation that applies and return the result of concatenating with all
+ subsequent applicable 'APPEND' operations. The result is the same as performing the operations
+ first-to-last.
+ NOTE: undefined if some 'APPEND' apply but no 'SET' applies.
+ """
+ check_depth(depth)
+ log.debug('evaluating var %r', self.name)
+ # List of strings to be concatenated
+ val = []
+ # Scan for last applicable expression - try each starting from the end, concatenate extensions
+ for i in range(len(self.operations)-1, -1, -1):
+ op = self.operations[i]
+ # Check the guarding condition
+ if (not op.condition) or op.condition.value(depth+1):
+ val.insert(0, op.expression.evaluate(depth=depth+1))
+ if op.operation == 'SET':
+ return u''.join(val)
+ return None
+
+ def dump(self, prefix=''):
+ """
+ Pretty printing of the variable. Includes all operations.
+ Returns iterator of lines (unicode strings).
+ """
+ # Try to evaluate the variable, but avoid undefined exceptions
+ v = None
+ try:
+ v = self.value(depth=0)
+ except ConfigError:
+ pass
+ yield prefix+u'%s = %r' % (self.name, v)
+ for op in self.operations:
+ #yield prefix+u' %s [%s] %s' % (op.operation, op.condition and op.condition.str(parents=True), op.expression)
+ yield prefix + u' ' + unicode(op)
+
+
+class ConfigExpression(object):
+ """
+ String expression with some unexpanded config variables. Used in variable operations and conditions.
+ Expression is given as a list of unicode strings and ConfigVar variables to be expanded.
+ """
+
+ def __init__(self, exprlist, original = u'<unknown>'):
+ self.exprlist = list(exprlist)
+ # Original defining string
+ self.original = original
+ # Replace strings with unicode
+ for i in range(len(self.exprlist)):
+ e = self.exprlist[i]
+ if isinstance(e, types.StringTypes):
+ if not isinstance(e, unicode):
+ self.exprlist[i] = unicode(e, 'ascii')
+
+ def variables(self):
+ "Return an iterator of variables user in the expression"
+ return itertools.ifilter(lambda e: isinstance(e, ConfigVar), self.exprlist)
+
+ def __str__(self):
+ return self.original
+
+ def evaluate(self, depth):
+ check_depth(depth)
+ "Return unicode result of expansion of the variables."
+ s = []
+ for e in self.exprlist:
+ if isinstance(e, ConfigVar):
+ s.append(e.value(depth+1))
+ elif isinstance(e, unicode):
+ s.append(e)
+ else:
+ raise ConfigError('Invalid type %s in expression \'%s\'.'%(type(e), self))
+ return u''.join(s)
+
+
projects / moe.git / blobdiff