X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;ds=sidebyside;f=t%2Fmoe%2Fconfig.py;h=ec057e1cc3bb9edeb347a0ca9fe348fe684e3cfe;hb=9f4eaf4234e6a7074fca72a3a9229d75035daac8;hp=3280283e0e3c63add243ee591e463ae935e94185;hpb=4574d9c6b1e2e9425f1919233a03a123b38a5e70;p=moe.git diff --git a/t/moe/config.py b/t/moe/config.py index 3280283..ec057e1 100644 --- a/t/moe/config.py +++ b/t/moe/config.py @@ -1,136 +1,445 @@ -#!/usr/bin/env python +""" +Module for managing and evaluation of Moe configuration variables. -import re -import sys +.. todo:: (OPT) Cleanup of unused undefined variables. +.. todo:: (OPT) Better variable name checking (no name '.'-structural prefix of another) +.. todo:: (OPT) Implemet "subtree" listing. +""" -key_pattern = re.compile('^[A-Za-z0-9_-]+$') -ref_pattern = re.compile('^[A-Za-z0-9_-]+') +import types, itertools, re, bisect +import logging as log + +from moe import MoeError + + +"Allowed depth of recursion - includes ALL recursive calls, so should quite high." +c_maxdepth = 256 + +"Maximum attained depth of recursion - for debug/testing" +debug_maxdepth = 0 + +"Variable name regexp, dots (separators) must be separated from edges and each other." +re_VARNAME = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z') + + +def check_depth(depth): + "Helper to check for recursion depth." + global debug_maxdepth + if depth > c_maxdepth: + raise CyclicConfigError('Too deep recursion in config evaluation (cyclic substitution?)') + if depth > debug_maxdepth: + debug_maxdepth = depth -class MoeConfigInvalid(Exception): - pass -class MoeConfigEvalErr(Exception): +class ConfigError(MoeError): + "Base class for moe.config errors" + pass + +class UndefinedError(ConfigError): + "Raised when no **SET** operation applies to evaluated variable." + pass + +class VariableNameError(ConfigError): + "Raised on invalid config variable name." + pass + +class VariableFixedError(ConfigError): + "Raised when modifying a fixed variable" + pass + +class CyclicConfigError(ConfigError): + "Raised when evaluation recursion is too deep" + pass + + +class ParseProxy(list): + """Proxy helper class around values returned by `parse` and `parse_file`, + useful in "with" constructs.""" + def __init__(self, config, parsed_ops): + super(ParseProxy, self).__init__(parsed_ops) + self.config = config + def __enter__(self): pass + def __exit__(self, etype, value, traceback): + self.config.remove(list(self)) + + +class ConfigTree(object): + """ + Configuration environment containing the variables. + """ + + def __init__(self): + self.variables = {} + + def lookup(self, key, create = True): + """ + Lookup and return a variable. + If not found and `create` set, check the name and transparently create a new one. + """ + if key not in self.variables: + if not re_VARNAME.match(key): + raise VariableNameError('Invalid variable identifier %r in config', key) + if not create: + raise UndefinedError('Config variable %r undefined.', key) + self.variables[key] = ConfigVar(key) + return self.variables[key] + + def __getitem__(self, key): + """ + Return the value of an existing variable. + """ + return self.lookup(key, create=False).value() + + def dump(self, prefix=''): + """ + Pretty printing of the tree. + Returns an iterator of lines (strings). + """ + return itertools.chain(*[ + self.variables[k].dump(prefix) for k in sorted(self.variables.keys()) + ]) + + def fix(self, keys): + "Fix value of variable or list of variables. Fixing undefined variable raises `UndefinedError`." + if isinstance(keys, types.StringTypes): + keys = [keys] + for key in keys: + self.lookup(key, create=False).fix() + + def unfix(self, keys): + "Unfix value of variable or list of variables. Unfixing undefined variable raises `UndefinedError`." + if isinstance(keys, types.StringTypes): + keys = [keys] + for key in keys: + self.lookup(key, create=False).unfix() + + def remove(self, parsed): + """Given a list [(varname, `Operation`)] as returned by `parse` or `parse_file`, + removes the operations from the respective variables config tree. + Variables/operations not present int the tree raise ValueError. + """ + for vname, o in parsed: + v = self.lookup(vname, create = True) + v.remove_operation(o) + + def parse(self, s, source=None, level=0, proxy=True): + """Parse `s` (stream/string) into the tree, see `moe.config_parser.ConfigParser` for details. + Returns list of parset operations: [(varname, `Operation`)]. + By default returns a proxy list-like object that can be used in "with" constructs: + + with config.parse("TEST='1'"): + print config['TEST'] + raise StupidError + """ + import moe.config_parser + p = moe.config_parser.ConfigParser(s, self, source=source, level=level) + l = p.parse() + if not proxy: + return l + return ParseProxy(self, l) + + def parse_file(self, filename, desc=None, level=0, proxy=True): + """Parse an utf-8 file into the tree using func:`parse`. + Names the source "`filename` <`desc`>". """ + with open(filename, 'rt') as f: + if desc: + filename += " <" + desc + ">" + return self.parse(f, source=filename, level=level, proxy=proxy) + + +class ConfigElem(object): + """ + Base class for cached config elements - variables and conditions + """ + + def __init__(self, name): + # Full name with separators, definition for conditions + self.name = name + # Vars and conditions depending on value of this one + self.dependants = set([]) + # Cached value (may be None in case of evaluation error) + self.cached = False + self.cached_val = None + + def invalidate(self, depth=0): + """ + Invalidate cached data and invalidate all dependants. + Does nothing if not cached. + """ + check_depth(depth) + if self.cached: + log.debug('invalidating %s', self) + self.cached = False + for d in self.dependants: + d.invalidate(depth + 1) + + def value(self, depth=0): + "Caching helper calling self.evaluate(), returns a value or throws an exception." + check_depth(depth) + if not self.cached: + self.cached_val = self.evaluate(depth=depth+1) + self.cached = True + if self.cached_val == None: + raise UndefinedError("Unable to evaluate %r."%(self.name,)) + return self.cached_val + + def __str__(self): + return self.name + + +class ConfigCondition(ConfigElem): + """ + Condition using equality and logic operators. + Formula is a tuple-tree in the following recursive form:: + + ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), ('==', e1, e2), ('!=', e1, e2) + + where ``e1``, ``e2`` are :class:`ConfigExpression`, ``c1``, ``c2``, :class:`ConfigCondition`. + """ + + def __init__(self, formula, text=None, parent=None): + """ + Condition defined by `text` (informative), `formula` as in class definition, + `parent` is the parent condition (if any). + """ + if not text: + text = self.formula_string(formula) + super(ConfigCondition, self).__init__(text) + self.formula = formula + self.parent = parent + # Setup dependencies on used variables (not on the parent condition) + for v in self.variables(): + v.dependants.add(self) + if self.parent: + self.parent.dependants.add(self) + + def variables(self, cl=None): + "Return an iterator of variables used in formula `cl`" + if not cl: + cl = self.formula + if cl[0] in ['==','!=']: + return itertools.chain(cl[1].variables(), cl[2].variables()) + if cl[0] in ['AND','OR']: + return itertools.chain(self.variables(cl[1]), self.variables(cl[2])) + return self.variables(cl[1]) # only 'NOT' left + + def remove_dependencies(self): + "Remove self as a dependant from all used variables" + for v in self.variables(): + v.dependants.discard(self) + if self.parent: + self.parent.dependants.discard(self) + + def evaluate(self, cl=None, depth=0): + """Evaluate formula `cl` (or the entire condition). + Partial evaluation for AND and OR. Tests the parent condition first.""" + check_depth(depth) + if not cl: + cl = self.formula + if self.parent and not self.parent.value(): + return False + if cl[0] in ['==','!=']: + v = cl[1].evaluate(depth=depth+1) == cl[2].evaluate(depth=depth+1) + if cl[0] == '!=': v = not v + return v + v1 = self.evaluate(cl=cl[1], depth=depth+1) + if cl[0] == 'NOT': + return not v1 + if cl[0] == 'OR' and v1: return True + if cl[0] == 'AND' and not v1: return False + return self.evaluate(cl=cl[2], depth=depth+1) + + def formula_string(self, formula): + "Create a string representation of a formula." + if formula[0] == 'AND': + return itertools.chain(['('], self.formula_string(formula[1]), [' and '], self.formula_string(formula[2]),[')']) + elif formula[0] == 'OR': + return itertools.chain(['('], self.formula_string(formula[1]), [' or '], self.formula_string(formula[2]),[')']) + elif formula[0] == 'NOT': + return itertools.chain(['(not '], self.formula_string(formula[1]),[')']) + elif formula[0] in ['==', '!=']: + return itertools.chain(formula[1], formula[0], formula[2]) + return iter(['']) + + def str(self, parents=False): + "Retur the defining expression, if `parents` set, then prefixed with parent conditions." + if parents and self.parent: + return self.parent.str(parents=True) + u' && ' + self.name + return self.name + + def __str__(self): + return self.str(parents=False) + + +class Operation(object): + """ + Helper class for operation data. Must be present at most once in at most one variable. + + ``operation`` is either ``"SET"`` or ``"APPEND"``, ``condition`` is a :class:`ConfigCondition` instance or ``None``, + ``expression`` is a :class:`ConfigExpression` instance, ``level`` is the priority of the operation and ``source`` + is an informative string describing the operation origin. + """ + + def __init__(self, operation, condition, expression, level=0, source='?'): + self.operation = operation + self.condition = condition + self.expression = expression + self.level = level + self.source = source + + def __str__(self): + return "%s <%d, %s> [%s] %r" % ( {'SET':'=', 'APPEND':'+'}[self.operation], self.level, self.source, + (self.condition and self.condition.str(parents=True)) or '', unicode(self.expression)) + + +class ConfigVar(ConfigElem): + "Class representing a single configuration variable" + + def __init__(self, name): + super(ConfigVar, self).__init__(name) + # Ordered list of `Operations` (ascending by `level`) + self.operations = [] + # Fixed to value (may be None) + self.fixed = False + self.fixed_val = None + + def variables(self): + "Return a set of variables used in the expressions of the operations" + if not self.operations: + return set([]) + return set.union(*[ op.expression.variables() for op in self.operations ]) + + def fix(self): + """ + Fixes the value of the variable. Exception is raised should the variable + evaluate to a different value while fixed. + """ + if self.fixed: + return + self.fixed_val = self.value() + self.fixed = True + + def unfix(self): + "Make the variable modifiable again." + self.fixed = False + + def value(self, depth=0): + "Handle the case when fixed, raise exc. on different evaluation" + val = super(ConfigVar,self).value(depth) + if self.fixed and self.fixed_val != val: + raise VariableFixedError("value of var %r was fixed to %r but evaluated to %r", self.name, self.fixed_val, val) + return val + + def add_operation(self, operation): + """ + Inserts an operation. The operations are sorted by `level` (ascending), new operation goes last among + these with the same level. + Adds the variable as a dependant of the conditions and variables used in the expressions. + """ + # Invalidate cached value + self.invalidate() + # Add the operation + pos = bisect.bisect_right([o.level for o in self.operations], operation.level) + self.operations.insert(pos, operation) + # Create dependencies + for v in operation.expression.variables(): + v.dependants.add(self) + if operation.condition: + operation.condition.dependants.add(self) + + def remove_operation(self, operation): + """ + Remove the Operation. + Also removes the variable as dependant from all conditions and variables used in this + operation that are no longer used. + """ + # Invalidate cached value + self.invalidate() + # Remove the operation + self.operations.remove(operation) + # Remove dependencies on variables unused in other defining operations + vs = self.variables() + for v in operation.expression.variables(): + if v not in vs: + v.dependants.remove(self) + # Remove the dependency on the conditions (if not used in another operation) + if operation.condition and operation.condition not in [op.condition for op in self.operations]: + operation.condition.dependants.remove(self) + + def evaluate(self, depth=0): + """ + Find the last 'SET' operation that applies and return the result of concatenating with all + subsequent applicable 'APPEND' operations. The result is the same as performing the operations + first-to-last. + NOTE: undefined if some 'APPEND' apply but no 'SET' applies. + """ + check_depth(depth) + log.debug('evaluating var %r', self.name) + # List of strings to be concatenated + val = [] + # Scan for last applicable expression - try each starting from the end, concatenate extensions + for i in range(len(self.operations)-1, -1, -1): + op = self.operations[i] + # Check the guarding condition + if (not op.condition) or op.condition.value(depth+1): + val.insert(0, op.expression.evaluate(depth=depth+1)) + if op.operation == 'SET': + return u''.join(val) + return None + + def dump(self, prefix=''): + """ + Pretty printing of the variable. Includes all operations. + Returns iterator of lines (unicode strings). + """ + # Try to evaluate the variable, but avoid undefined exceptions + v = None + try: + v = self.value(depth=0) + except ConfigError: + pass + yield prefix+u'%s = %r' % (self.name, v) + for op in self.operations: + #yield prefix+u' %s [%s] %s' % (op.operation, op.condition and op.condition.str(parents=True), op.expression) + yield prefix + u' ' + unicode(op) + + +class ConfigExpression(object): + """ + String expression with some unexpanded config variables. Used in variable operations and conditions. + Expression is given as a list of unicode strings and ConfigVar variables to be expanded. + """ + + def __init__(self, exprlist, original = u''): + self.exprlist = list(exprlist) + # Original defining string + self.original = original + # Replace strings with unicode + for i in range(len(self.exprlist)): + e = self.exprlist[i] + if isinstance(e, types.StringTypes): + if not isinstance(e, unicode): + self.exprlist[i] = unicode(e, 'ascii') + + def variables(self): + "Return a set of variables used in the expression" + return set([e for e in self.exprlist if isinstance(e, ConfigVar)]) + + def __str__(self): + return self.original + + def evaluate(self, depth): + check_depth(depth) + "Return unicode result of expansion of the variables." + s = [] + for e in self.exprlist: + if isinstance(e, ConfigVar): + s.append(e.value(depth+1)) + elif isinstance(e, unicode): + s.append(e) + else: + raise ConfigError('Invalid type %s in expression \'%s\'.'%(type(e), self)) + return u''.join(s) + -class MoeConfig: - """Moe configuration file.""" - - def __init__(self, file=None, name=None): - self.cfg = {} - if file is not None: - self.load(file) - elif name is not None: - self.load(open(name, 'r')) - - def load(self, file): - for x in file.readlines(): - x = x.rstrip("\n").lstrip(" \t") - if x=='' or x.startswith('#'): - pass - else: - sep = x.find('=') - if sep >= 0: - k = x[:sep] - v = x[sep+1:] - if k.endswith("+"): - k = k[:-1] - if not self.cfg.has_key(k): - self.cfg[k] = [('a','')]; - else: - self.cfg[k] = [] - if not key_pattern.match(k): - raise MoeConfigInvalid, "Malformed name of configuration variable" - if v.startswith("'"): - v=v[1:] - if not v.endswith("'"): - raise MoeConfigInvalid, "Misquoted string" - self.cfg[k].append(('s', v[:-1])) - elif v.startswith('"'): - v=v[1:] - if not v.endswith('"'): - raise MoeConfigInvalid, "Misquoted string" - self.parse_interpolated(self.cfg[k], v[:-1]) - else: - self.cfg[k].append(('s', v)) - else: - ## FIXME: Report line numbers - raise MoeConfigInvalid, "Parse error" - - def parse_interpolated(self, list, s): - while s<>'': - if s.startswith('$'): - s = s[1:] - if s.startswith('{'): - p = s.find('}') - if not p: - raise MoeConfigInvalid, "Unbalanced braces" - k, s = s[1:p], s[p+1:] - if not key_pattern.match(k): - raise MoeConfigInvalid, "Invalid variable name" - else: - m = ref_pattern.match(s) - if m: - k, s = s[:m.end()], s[m.end():] - else: - raise MoeConfigInvalid, "Invalid variable reference" - list.append(('i', k)) - else: - p = s.find('$') - if p < 0: - p = len(s) - list.append(('s', s[:p])) - s = s[p:] - - def dump(self, file=sys.stdout): - for k,v in self.cfg.items(): - file.write(k) - if len(v) > 0 and v[0][0] == 'a': - file.write('+') - v = v[1:] - file.write('=') - for t,w in v: - if t == 's': - file.write("'" + w + "'") - elif t == 'i': - file.write('"$' + w + '"') - file.write("\n") - -class MoeConfigStack: - """Stack of configuration files.""" - - def __init__(self, base=None): - ## FIXME: Do we need to duplicate the config files themselves? - if base: - self.stk = base.stk[:] - else: - self.stk = [] - self.in_progress = {} - - def push(self, cfg): - self.stk.append(cfg) - - def __getitem__(self, k): - if self.in_progress.has_key(k): - raise MoeConfigEvalErr, "Definition of $%s is recursive" % k; - self.in_progress[k] = 1; - v = self.do_get(k, len(self.stk)-1) - del self.in_progress[k] - return v - - def do_get(self, k, pos): - while pos >= 0: - cfg = self.stk[pos] - if cfg.cfg.has_key(k): - new = cfg.cfg[k] - if new[0][0] == 'a': - v = self.do_get(k, pos-1) - else: - v = '' - for op,arg in new: - if op == 's': - v = v + arg - elif op == 'i': - v = v + self[arg] - return v - pos -= 1 - return ''