5 Lazy conditional string evaluation module for Moe configuration variables.
8 * Each variable has ordered list of operations (definitions), each defining operation either
9 assigns (SET) or appends (APPEND) value of an expression to the variable. Each operation may be guarded by condition(s).
11 * Each condition is a formula (tree consisting of 'AND', 'OR', 'NOT' and '==', '!=' between two expressions.
13 * Expression is a list of strings and variables to be expanded.
15 .. note:: If no 'SET' applies, a variable is still undefined even if some 'APPEND' applies. This might change.
16 .. note:: All expanded data should be (or is converted to) unicode
17 .. todo:: (OPT) Cleanup of unused undefined variables.
18 .. todo:: (OPT) Better variable name checking (no name '.'-structural prefix of another)
19 .. todo:: (OPT) Implemet "subtree" listing.
22 import types, itertools, re, bisect
25 from moe import MoeError
28 "Allowed depth of recursion - includes ALL recursive calls, so should quite high."
31 "Maximum attained depth of recursion - for debug/testing"
34 "Variable name regexp, dots (separators) must be separated from edges and each other."
35 re_VARNAME = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z')
38 def check_depth(depth):
39 "Helper to check for recursion depth."
41 if depth > c_maxdepth:
42 raise CyclicConfigError('Too deep recursion in config evaluation (cyclic substitution?)')
43 if depth > debug_maxdepth:
44 debug_maxdepth = depth
47 class ConfigError(MoeError):
50 class UndefinedError(ConfigError):
53 class VariableNameError(ConfigError):
56 class VariableFixedError(ConfigError):
59 class CyclicConfigError(ConfigError):
63 class ConfigTree(object):
65 Configuration tree containing all the variables.
67 The variables in `self.variables` are referenced directly by the full name.
73 def lookup(self, key, create = True):
75 Lookup and return a variable.
76 If not found and `create` set, check the name and transparently create a new one.
78 if key not in self.variables:
79 if not re_VARNAME.match(key):
80 raise VariableNameError('Invalid variable identifier %r in config', key)
82 raise UndefinedError('Config variable %r undefined.', key)
83 self.variables[key] = ConfigVar(key)
84 return self.variables[key]
86 def __getitem__(self, key):
88 Return the value of an existing variable.
90 return self.lookup(key, create=False).value()
92 def dump(self, prefix=''):
94 Pretty printing of the tree.
95 Returns an iterator of lines (strings).
97 return itertools.chain(*[
98 self.variables[k].dump(prefix) for k in sorted(self.variables.keys())
101 def parse(self, s, source=None, level=0):
102 """Parse `s` (stream/string) into the tree, see `moe.confparser.ConfigParser` for details."""
103 import moe.confparser
104 p = moe.confparser.ConfigParser(text, self, source=source, level=level)
107 def parse_file(self, filename, desc=None, level=0):
108 """Parse an utf-8 file into the tree, see `moe.confparser.ConfigParser` for details.
109 Names the source "`filename` <`desc`>". """
110 f = open(filename, 'rt')
112 filename += " <" + desc + ">"
113 self.parse(f, source=filename, level=level)
116 class ConfigElem(object):
118 Base class for cahed config elements - variables and conditions
121 def __init__(self, name):
122 # Full name with separators, definition for conditions
124 # Vars and conditions depending on value of this one
125 self.dependants = set([])
126 # Cached value (may be None in case of evaluation error)
128 self.cached_val = None
130 def invalidate(self, depth=0):
132 Invalidate cached data and invalidate all dependants.
133 Does nothing if not cached.
137 log.debug('invalidating %s', self)
139 for d in self.dependants:
140 d.invalidate(depth + 1)
142 def value(self, depth=0):
143 "Caching helper calling self.evaluate(), returns a value or throws an exception."
146 self.cached_val = self.evaluate(depth=depth+1)
148 if self.cached_val == None:
149 raise UndefinedError("Unable to evaluate %r."%(self.name,))
150 return self.cached_val
156 class ConfigCondition(ConfigElem):
158 Condition using equality and logic operators.
159 Clause is a tuple-tree in the following recursive form::
161 ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), ('==', e1, e2), ('!=', e1, e2)
163 where e1, e2 are `ConfigExpression`, c1, c2, `ConfigCondition`.
166 def __init__(self, formula, text=None, parent=None):
168 Condition defined by `text` (informative), `formula` as in class definition,
169 `parent` is the parent condition (if any).
172 text = self.formula_string(formula)
173 super(ConfigCondition, self).__init__(text)
174 self.formula = formula
176 # Setup dependencies on used variables (not on the parent condition)
177 for v in self.variables():
178 v.dependants.add(self)
180 self.parent.dependants.add(self)
182 def variables(self, cl=None):
183 "Return an iterator of variables used in formula `cl`"
186 if cl[0] in ['==','!=']:
187 return itertools.chain(cl[1].variables(), cl[2].variables())
188 if cl[0] in ['AND','OR']:
189 return itertools.chain(self.variables(cl[1]), self.variables(cl[2]))
190 return self.variables(cl[1]) # only 'NOT' left
192 def remove_dependencies(self):
193 "Remove self as a dependant from all used variables"
194 for v in self.variables():
195 v.dependants.discard(self)
197 self.parent.dependants.discard(self)
199 def evaluate(self, cl=None, depth=0):
200 """Evaluate formula `cl` (or the entire condition).
201 Partial evaluation for AND and OR. Tests the parent condition first."""
205 if self.parent and not self.parent.value():
207 if cl[0] in ['==','!=']:
208 v = cl[1].evaluate(depth=depth+1) == cl[2].evaluate(depth=depth+1)
209 if cl[0] == '!=': v = not v
211 v1 = self.evaluate(cl=cl[1], depth=depth+1)
214 if cl[0] == 'OR' and v1: return True
215 if cl[0] == 'AND' and not v1: return False
216 return self.evaluate(cl=cl[2], depth=depth+1)
218 def formula_string(self, formula):
219 "Create a string representation of a formula."
220 if formula[0] == 'AND':
221 return itertools.chain(['('], self.formula_string(formula[1]), [' and '], self.formula_string(formula[2]),[')'])
222 elif formula[0] == 'OR':
223 return itertools.chain(['('], self.formula_string(formula[1]), [' or '], self.formula_string(formula[2]),[')'])
224 elif formula[0] == 'NOT':
225 return itertools.chain(['(not '], self.formula_string(formula[1]),[')'])
226 elif formula[0] in ['==', '!=']:
227 return itertools.chain(formula[1], formula[0], formula[2])
228 return iter(['<invalid formula>'])
230 def str(self, parents=False):
231 "Retur the defining expression, if `parents` set, then prefixed with parent conditions."
232 if parents and self.parent:
233 return self.parent.str(parents=True) + u' && ' + self.name
237 return self.str(parents=False)
240 class Operation(object):
241 "Helper class for operation data. Must not be present in more variables or present multiple times."
243 def __init__(self, operation, condition, expression, level=0, source='?'):
244 # operation is currently 'SET' and 'APPEND'
245 self.operation = operation
246 self.condition = condition
247 self.expression = expression
252 return "%s <%d, %s> [%s] %r" % ( {'SET':'=', 'APPEND':'+'}[self.operation], self.level, self.source,
253 (self.condition and self.condition.str(parents=True)) or '', unicode(self.expression))
256 class ConfigVar(ConfigElem):
258 def __init__(self, name):
259 super(ConfigVar, self).__init__(name)
260 # Ordered list of `Operations` (ascending by `level`)
262 # Fixed to value (may be None)
264 self.fixed_val = None
267 "Return a set of variables used in the expressions"
268 return set(sum([ list(op.expression.variables()) for op in self.operations ], []))
272 Fixes the value of the variable. Exception is raised should the variable
273 evaluate to a different value while fixed.
277 self.fixed_val = self.value()
281 "Set the variable to be modifiable again."
284 def value(self, depth=0):
285 "Handle the case when fixed, raise exc. on different evaluation"
286 val = super(ConfigVar,self).value(depth)
287 if self.fixed and self.fixed_val != val:
288 raise VariableFixedError("value of var %s was fixed to %r but evaluated to %r", self.name, self.fixed_val, val)
291 def add_operation(self, operation):
293 Inserts an operation. The operations are sorted by `level` (ascending), new operation goes last among
294 these with the same level.
295 Adds the variable as a dependant of the conditions and variables used in the expressions.
297 # Invalidate cached value
300 pos = bisect.bisect_right([o.level for o in self.operations], operation.level)
301 self.operations.insert(pos, operation)
302 # Create dependencies
303 for v in operation.expression.variables():
304 v.dependants.add(self)
305 if operation.condition:
306 operation.condition.dependants.add(self)
308 def remove_operation(self, operation):
310 Remove the Operation.
311 Also removes the variable as dependant from all conditions and variables used in this
312 operation that are no longer used.
314 # Invalidate cached value
316 # Remove the operation
317 self.operations.remove(operation)
318 # Remove dependencies on variables unused in other operations
319 vs = self.variables()
320 for v in operation.expression.variables():
322 v.dependants.remove(self)
323 # Remove the dependency on the conditions (if not used in another operation)
324 if operation.condition and operation.condition not in [op.condition for op in self.operations]:
325 operation.condition.dependants.remove(self)
327 def evaluate(self, depth=0):
329 Find the last 'SET' operation that applies and return the result of concatenating with all
330 subsequent applicable 'APPEND' operations. The result is the same as performing the operations
332 NOTE: undefined if some 'APPEND' apply but no 'SET' applies.
335 log.debug('evaluating var %r', self.name)
336 # List of strings to be concatenated
338 # Scan for last applicable expression - try each starting from the end, concatenate extensions
339 for i in range(len(self.operations)-1, -1, -1):
340 op = self.operations[i]
341 # Check the guarding condition
342 if (not op.condition) or op.condition.value(depth+1):
343 val.insert(0, op.expression.evaluate(depth=depth+1))
344 if op.operation == 'SET':
348 def dump(self, prefix=''):
350 Pretty printing of the variable. Includes all operations.
351 Returns iterator of lines (unicode strings).
353 # Try to evaluate the variable, but avoid undefined exceptions
356 v = self.value(depth=0)
359 yield prefix+u'%s = %r' % (self.name, v)
360 for op in self.operations:
361 #yield prefix+u' %s [%s] %s' % (op.operation, op.condition and op.condition.str(parents=True), op.expression)
362 yield prefix + u' ' + unicode(op)
365 class ConfigExpression(object):
367 String expression with some unexpanded config variables. Used in variable operations and conditions.
368 Expression is given as a list of unicode strings and ConfigVar variables to be expanded.
371 def __init__(self, exprlist, original = u'<unknown>'):
372 self.exprlist = list(exprlist)
373 # Original defining string
374 self.original = original
375 # Replace strings with unicode
376 for i in range(len(self.exprlist)):
378 if isinstance(e, types.StringTypes):
379 if not isinstance(e, unicode):
380 self.exprlist[i] = unicode(e, 'ascii')
383 "Return an iterator of variables user in the expression"
384 return itertools.ifilter(lambda e: isinstance(e, ConfigVar), self.exprlist)
389 def evaluate(self, depth):
391 "Return unicode result of expansion of the variables."
393 for e in self.exprlist:
394 if isinstance(e, ConfigVar):
395 s.append(e.value(depth+1))
396 elif isinstance(e, unicode):
399 raise ConfigError('Invalid type %s in expression \'%s\'.'%(type(e), self))