2 Lazy conditional string evaluation module for Moe configuration variables.
4 * Each variable has ordered list of operations (definitions), each defining operation either
5 assigns (SET) or appends (APPEND) value of an expression to the variable. Each operation may be guarded by condition(s).
7 * Each condition is a formula (tree consisting of 'AND', 'OR', 'NOT' and '==', '!=' between two expressions.
9 * Expression is a list of strings and variables to be expanded.
11 .. note:: If no 'SET' applies, a variable is still undefined even if some 'APPEND' applies. This might change.
12 .. note:: All expanded data should be (or is converted to) unicode
13 .. todo:: (OPT) Cleanup of unused undefined variables.
14 .. todo:: (OPT) Better variable name checking (no name '.'-structural prefix of another)
15 .. todo:: (OPT) Implemet "subtree" listing.
18 import types, itertools, re, bisect
21 from moe import MoeError
24 "Allowed depth of recursion - includes ALL recursive calls, so should quite high."
27 "Maximum attained depth of recursion - for debug/testing"
30 "Variable name regexp, dots (separators) must be separated from edges and each other."
31 re_VARNAME = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z')
34 def check_depth(depth):
35 "Helper to check for recursion depth."
37 if depth > c_maxdepth:
38 raise CyclicConfigError('Too deep recursion in config evaluation (cyclic substitution?)')
39 if depth > debug_maxdepth:
40 debug_maxdepth = depth
43 class ConfigError(MoeError):
46 class UndefinedError(ConfigError):
49 class VariableNameError(ConfigError):
52 class VariableFixedError(ConfigError):
55 class CyclicConfigError(ConfigError):
58 class ParseProxy(list):
59 """Proxy helper class around values returned by `parse` and `parse_file`,
60 useful in "with" constructs."""
61 def __init__(self, config, parsed_ops):
62 super(ParseProxy, self).__init__(parsed_ops)
66 def __exit__(self, etype, value, traceback):
67 self.config.remove(list(self))
70 class ConfigTree(object):
72 Configuration tree containing all the variables.
74 The variables in `self.variables` are referenced directly by the full name.
80 def lookup(self, key, create = True):
82 Lookup and return a variable.
83 If not found and `create` set, check the name and transparently create a new one.
85 if key not in self.variables:
86 if not re_VARNAME.match(key):
87 raise VariableNameError('Invalid variable identifier %r in config', key)
89 raise UndefinedError('Config variable %r undefined.', key)
90 self.variables[key] = ConfigVar(key)
91 return self.variables[key]
93 def __getitem__(self, key):
95 Return the value of an existing variable.
97 return self.lookup(key, create=False).value()
99 def dump(self, prefix=''):
101 Pretty printing of the tree.
102 Returns an iterator of lines (strings).
104 return itertools.chain(*[
105 self.variables[k].dump(prefix) for k in sorted(self.variables.keys())
109 "Fix value of variable or list of variables. Fixing undefined variable raises `UndefinedError`."
110 if isinstance(keys, types.StringTypes):
113 self.lookup(key, create=False).fix()
115 def unfix(self, keys):
116 "Unfix value of variable or list of variables. Unfixing undefined variable raises `UndefinedError`."
117 if isinstance(keys, types.StringTypes):
120 self.lookup(key, create=False).unfix()
122 def remove(self, parsed):
123 """Given a list [(varname, `Operation`)] as returned by `parse` or `parse_file`,
124 removes the operations from the respective variables config tree.
125 Variables/operations not present int the tree raise ValueError.
127 for vname, o in parsed:
128 v = self.lookup(vname, create = True)
129 v.remove_operation(o)
131 def parse(self, s, source=None, level=0, proxy=True):
132 """Parse `s` (stream/string) into the tree, see `moe.config_parser.ConfigParser` for details.
133 Returns list of parset operations: [(varname, `Operation`)].
134 By default returns a proxy list-like object that can be used in "with" constructs:
136 with config.parse("TEST='1'"):
140 import moe.config_parser
141 p = moe.config_parser.ConfigParser(s, self, source=source, level=level)
145 return ParseProxy(self, l)
147 def parse_file(self, filename, desc=None, level=0, proxy=True):
148 """Parse an utf-8 file into the tree using func:`parse`.
149 Names the source "`filename` <`desc`>". """
150 with open(filename, 'rt') as f:
152 filename += " <" + desc + ">"
153 return self.parse(f, source=filename, level=level, proxy=proxy)
156 class ConfigElem(object):
158 Base class for cahed config elements - variables and conditions
161 def __init__(self, name):
162 # Full name with separators, definition for conditions
164 # Vars and conditions depending on value of this one
165 self.dependants = set([])
166 # Cached value (may be None in case of evaluation error)
168 self.cached_val = None
170 def invalidate(self, depth=0):
172 Invalidate cached data and invalidate all dependants.
173 Does nothing if not cached.
177 log.debug('invalidating %s', self)
179 for d in self.dependants:
180 d.invalidate(depth + 1)
182 def value(self, depth=0):
183 "Caching helper calling self.evaluate(), returns a value or throws an exception."
186 self.cached_val = self.evaluate(depth=depth+1)
188 if self.cached_val == None:
189 raise UndefinedError("Unable to evaluate %r."%(self.name,))
190 return self.cached_val
196 class ConfigCondition(ConfigElem):
198 Condition using equality and logic operators.
199 Clause is a tuple-tree in the following recursive form::
201 ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), ('==', e1, e2), ('!=', e1, e2)
203 where e1, e2 are `ConfigExpression`, c1, c2, `ConfigCondition`.
206 def __init__(self, formula, text=None, parent=None):
208 Condition defined by `text` (informative), `formula` as in class definition,
209 `parent` is the parent condition (if any).
212 text = self.formula_string(formula)
213 super(ConfigCondition, self).__init__(text)
214 self.formula = formula
216 # Setup dependencies on used variables (not on the parent condition)
217 for v in self.variables():
218 v.dependants.add(self)
220 self.parent.dependants.add(self)
222 def variables(self, cl=None):
223 "Return an iterator of variables used in formula `cl`"
226 if cl[0] in ['==','!=']:
227 return itertools.chain(cl[1].variables(), cl[2].variables())
228 if cl[0] in ['AND','OR']:
229 return itertools.chain(self.variables(cl[1]), self.variables(cl[2]))
230 return self.variables(cl[1]) # only 'NOT' left
232 def remove_dependencies(self):
233 "Remove self as a dependant from all used variables"
234 for v in self.variables():
235 v.dependants.discard(self)
237 self.parent.dependants.discard(self)
239 def evaluate(self, cl=None, depth=0):
240 """Evaluate formula `cl` (or the entire condition).
241 Partial evaluation for AND and OR. Tests the parent condition first."""
245 if self.parent and not self.parent.value():
247 if cl[0] in ['==','!=']:
248 v = cl[1].evaluate(depth=depth+1) == cl[2].evaluate(depth=depth+1)
249 if cl[0] == '!=': v = not v
251 v1 = self.evaluate(cl=cl[1], depth=depth+1)
254 if cl[0] == 'OR' and v1: return True
255 if cl[0] == 'AND' and not v1: return False
256 return self.evaluate(cl=cl[2], depth=depth+1)
258 def formula_string(self, formula):
259 "Create a string representation of a formula."
260 if formula[0] == 'AND':
261 return itertools.chain(['('], self.formula_string(formula[1]), [' and '], self.formula_string(formula[2]),[')'])
262 elif formula[0] == 'OR':
263 return itertools.chain(['('], self.formula_string(formula[1]), [' or '], self.formula_string(formula[2]),[')'])
264 elif formula[0] == 'NOT':
265 return itertools.chain(['(not '], self.formula_string(formula[1]),[')'])
266 elif formula[0] in ['==', '!=']:
267 return itertools.chain(formula[1], formula[0], formula[2])
268 return iter(['<invalid formula>'])
270 def str(self, parents=False):
271 "Retur the defining expression, if `parents` set, then prefixed with parent conditions."
272 if parents and self.parent:
273 return self.parent.str(parents=True) + u' && ' + self.name
277 return self.str(parents=False)
280 class Operation(object):
281 "Helper class for operation data. Must not be present in more variables or present multiple times."
283 def __init__(self, operation, condition, expression, level=0, source='?'):
284 # operation is currently 'SET' and 'APPEND'
285 self.operation = operation
286 self.condition = condition
287 self.expression = expression
292 return "%s <%d, %s> [%s] %r" % ( {'SET':'=', 'APPEND':'+'}[self.operation], self.level, self.source,
293 (self.condition and self.condition.str(parents=True)) or '', unicode(self.expression))
296 class ConfigVar(ConfigElem):
298 def __init__(self, name):
299 super(ConfigVar, self).__init__(name)
300 # Ordered list of `Operations` (ascending by `level`)
302 # Fixed to value (may be None)
304 self.fixed_val = None
307 "Return a set of variables used in the expressions"
308 if not self.operations:
310 return set.union(*[ op.expression.variables() for op in self.operations ])
314 Fixes the value of the variable. Exception is raised should the variable
315 evaluate to a different value while fixed.
319 self.fixed_val = self.value()
323 "Make the variable modifiable again."
326 def value(self, depth=0):
327 "Handle the case when fixed, raise exc. on different evaluation"
328 val = super(ConfigVar,self).value(depth)
329 if self.fixed and self.fixed_val != val:
330 raise VariableFixedError("value of var %r was fixed to %r but evaluated to %r", self.name, self.fixed_val, val)
333 def add_operation(self, operation):
335 Inserts an operation. The operations are sorted by `level` (ascending), new operation goes last among
336 these with the same level.
337 Adds the variable as a dependant of the conditions and variables used in the expressions.
339 # Invalidate cached value
342 pos = bisect.bisect_right([o.level for o in self.operations], operation.level)
343 self.operations.insert(pos, operation)
344 # Create dependencies
345 for v in operation.expression.variables():
346 v.dependants.add(self)
347 if operation.condition:
348 operation.condition.dependants.add(self)
350 def remove_operation(self, operation):
352 Remove the Operation.
353 Also removes the variable as dependant from all conditions and variables used in this
354 operation that are no longer used.
356 # Invalidate cached value
358 # Remove the operation
359 self.operations.remove(operation)
360 # Remove dependencies on variables unused in other defining operations
361 vs = self.variables()
362 for v in operation.expression.variables():
364 v.dependants.remove(self)
365 # Remove the dependency on the conditions (if not used in another operation)
366 if operation.condition and operation.condition not in [op.condition for op in self.operations]:
367 operation.condition.dependants.remove(self)
369 def evaluate(self, depth=0):
371 Find the last 'SET' operation that applies and return the result of concatenating with all
372 subsequent applicable 'APPEND' operations. The result is the same as performing the operations
374 NOTE: undefined if some 'APPEND' apply but no 'SET' applies.
377 log.debug('evaluating var %r', self.name)
378 # List of strings to be concatenated
380 # Scan for last applicable expression - try each starting from the end, concatenate extensions
381 for i in range(len(self.operations)-1, -1, -1):
382 op = self.operations[i]
383 # Check the guarding condition
384 if (not op.condition) or op.condition.value(depth+1):
385 val.insert(0, op.expression.evaluate(depth=depth+1))
386 if op.operation == 'SET':
390 def dump(self, prefix=''):
392 Pretty printing of the variable. Includes all operations.
393 Returns iterator of lines (unicode strings).
395 # Try to evaluate the variable, but avoid undefined exceptions
398 v = self.value(depth=0)
401 yield prefix+u'%s = %r' % (self.name, v)
402 for op in self.operations:
403 #yield prefix+u' %s [%s] %s' % (op.operation, op.condition and op.condition.str(parents=True), op.expression)
404 yield prefix + u' ' + unicode(op)
407 class ConfigExpression(object):
409 String expression with some unexpanded config variables. Used in variable operations and conditions.
410 Expression is given as a list of unicode strings and ConfigVar variables to be expanded.
413 def __init__(self, exprlist, original = u'<unknown>'):
414 self.exprlist = list(exprlist)
415 # Original defining string
416 self.original = original
417 # Replace strings with unicode
418 for i in range(len(self.exprlist)):
420 if isinstance(e, types.StringTypes):
421 if not isinstance(e, unicode):
422 self.exprlist[i] = unicode(e, 'ascii')
425 "Return a set of variables used in the expression"
426 return set([e for e in self.exprlist if isinstance(e, ConfigVar)])
431 def evaluate(self, depth):
433 "Return unicode result of expansion of the variables."
435 for e in self.exprlist:
436 if isinstance(e, ConfigVar):
437 s.append(e.value(depth+1))
438 elif isinstance(e, unicode):
441 raise ConfigError('Invalid type %s in expression \'%s\'.'%(type(e), self))