2 Lazy conditional string evaluation module for Moe configuration variables.
4 * Each variable has ordered list of operations (definitions), each defining operation either
5 assigns (SET) or appends (APPEND) value of an expression to the variable. Each operation may be guarded by condition(s).
7 * Each condition is a formula (tree consisting of 'AND', 'OR', 'NOT' and '==', '!=' between two expressions.
9 * Expression is a list of strings and variables to be expanded.
11 .. note:: If no 'SET' applies, a variable is still undefined even if some 'APPEND' applies. This might change.
12 .. note:: All expanded data should be (or is converted to) unicode
13 .. todo:: (OPT) Cleanup of unused undefined variables.
14 .. todo:: (OPT) Better variable name checking (no name '.'-structural prefix of another)
15 .. todo:: (OPT) Implemet "subtree" listing.
18 import types, itertools, re, bisect
21 from moe import MoeError
24 "Allowed depth of recursion - includes ALL recursive calls, so should quite high."
27 "Maximum attained depth of recursion - for debug/testing"
30 "Variable name regexp, dots (separators) must be separated from edges and each other."
31 re_VARNAME = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z')
34 def check_depth(depth):
35 "Helper to check for recursion depth."
37 if depth > c_maxdepth:
38 raise CyclicConfigError('Too deep recursion in config evaluation (cyclic substitution?)')
39 if depth > debug_maxdepth:
40 debug_maxdepth = depth
43 class ConfigError(MoeError):
46 class UndefinedError(ConfigError):
49 class VariableNameError(ConfigError):
52 class VariableFixedError(ConfigError):
55 class CyclicConfigError(ConfigError):
59 class ConfigTree(object):
61 Configuration tree containing all the variables.
63 The variables in `self.variables` are referenced directly by the full name.
69 def lookup(self, key, create = True):
71 Lookup and return a variable.
72 If not found and `create` set, check the name and transparently create a new one.
74 if key not in self.variables:
75 if not re_VARNAME.match(key):
76 raise VariableNameError('Invalid variable identifier %r in config', key)
78 raise UndefinedError('Config variable %r undefined.', key)
79 self.variables[key] = ConfigVar(key)
80 return self.variables[key]
82 def __getitem__(self, key):
84 Return the value of an existing variable.
86 return self.lookup(key, create=False).value()
88 def dump(self, prefix=''):
90 Pretty printing of the tree.
91 Returns an iterator of lines (strings).
93 return itertools.chain(*[
94 self.variables[k].dump(prefix) for k in sorted(self.variables.keys())
97 def parse(self, s, source=None, level=0):
98 """Parse `s` (stream/string) into the tree, see `moe.confparser.ConfigParser` for details."""
100 p = moe.confparser.ConfigParser(text, self, source=source, level=level)
103 def parse_file(self, filename, desc=None, level=0):
104 """Parse an utf-8 file into the tree, see `moe.confparser.ConfigParser` for details.
105 Names the source "`filename` <`desc`>". """
106 f = open(filename, 'rt')
108 filename += " <" + desc + ">"
109 self.parse(f, source=filename, level=level)
112 class ConfigElem(object):
114 Base class for cahed config elements - variables and conditions
117 def __init__(self, name):
118 # Full name with separators, definition for conditions
120 # Vars and conditions depending on value of this one
121 self.dependants = set([])
122 # Cached value (may be None in case of evaluation error)
124 self.cached_val = None
126 def invalidate(self, depth=0):
128 Invalidate cached data and invalidate all dependants.
129 Does nothing if not cached.
133 log.debug('invalidating %s', self)
135 for d in self.dependants:
136 d.invalidate(depth + 1)
138 def value(self, depth=0):
139 "Caching helper calling self.evaluate(), returns a value or throws an exception."
142 self.cached_val = self.evaluate(depth=depth+1)
144 if self.cached_val == None:
145 raise UndefinedError("Unable to evaluate %r."%(self.name,))
146 return self.cached_val
152 class ConfigCondition(ConfigElem):
154 Condition using equality and logic operators.
155 Clause is a tuple-tree in the following recursive form::
157 ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), ('==', e1, e2), ('!=', e1, e2)
159 where e1, e2 are `ConfigExpression`, c1, c2, `ConfigCondition`.
162 def __init__(self, formula, text=None, parent=None):
164 Condition defined by `text` (informative), `formula` as in class definition,
165 `parent` is the parent condition (if any).
168 text = self.formula_string(formula)
169 super(ConfigCondition, self).__init__(text)
170 self.formula = formula
172 # Setup dependencies on used variables (not on the parent condition)
173 for v in self.variables():
174 v.dependants.add(self)
176 self.parent.dependants.add(self)
178 def variables(self, cl=None):
179 "Return an iterator of variables used in formula `cl`"
182 if cl[0] in ['==','!=']:
183 return itertools.chain(cl[1].variables(), cl[2].variables())
184 if cl[0] in ['AND','OR']:
185 return itertools.chain(self.variables(cl[1]), self.variables(cl[2]))
186 return self.variables(cl[1]) # only 'NOT' left
188 def remove_dependencies(self):
189 "Remove self as a dependant from all used variables"
190 for v in self.variables():
191 v.dependants.discard(self)
193 self.parent.dependants.discard(self)
195 def evaluate(self, cl=None, depth=0):
196 """Evaluate formula `cl` (or the entire condition).
197 Partial evaluation for AND and OR. Tests the parent condition first."""
201 if self.parent and not self.parent.value():
203 if cl[0] in ['==','!=']:
204 v = cl[1].evaluate(depth=depth+1) == cl[2].evaluate(depth=depth+1)
205 if cl[0] == '!=': v = not v
207 v1 = self.evaluate(cl=cl[1], depth=depth+1)
210 if cl[0] == 'OR' and v1: return True
211 if cl[0] == 'AND' and not v1: return False
212 return self.evaluate(cl=cl[2], depth=depth+1)
214 def formula_string(self, formula):
215 "Create a string representation of a formula."
216 if formula[0] == 'AND':
217 return itertools.chain(['('], self.formula_string(formula[1]), [' and '], self.formula_string(formula[2]),[')'])
218 elif formula[0] == 'OR':
219 return itertools.chain(['('], self.formula_string(formula[1]), [' or '], self.formula_string(formula[2]),[')'])
220 elif formula[0] == 'NOT':
221 return itertools.chain(['(not '], self.formula_string(formula[1]),[')'])
222 elif formula[0] in ['==', '!=']:
223 return itertools.chain(formula[1], formula[0], formula[2])
224 return iter(['<invalid formula>'])
226 def str(self, parents=False):
227 "Retur the defining expression, if `parents` set, then prefixed with parent conditions."
228 if parents and self.parent:
229 return self.parent.str(parents=True) + u' && ' + self.name
233 return self.str(parents=False)
236 class Operation(object):
237 "Helper class for operation data. Must not be present in more variables or present multiple times."
239 def __init__(self, operation, condition, expression, level=0, source='?'):
240 # operation is currently 'SET' and 'APPEND'
241 self.operation = operation
242 self.condition = condition
243 self.expression = expression
248 return "%s <%d, %s> [%s] %r" % ( {'SET':'=', 'APPEND':'+'}[self.operation], self.level, self.source,
249 (self.condition and self.condition.str(parents=True)) or '', unicode(self.expression))
252 class ConfigVar(ConfigElem):
254 def __init__(self, name):
255 super(ConfigVar, self).__init__(name)
256 # Ordered list of `Operations` (ascending by `level`)
258 # Fixed to value (may be None)
260 self.fixed_val = None
263 "Return a set of variables used in the expressions"
264 return set(sum([ list(op.expression.variables()) for op in self.operations ], []))
268 Fixes the value of the variable. Exception is raised should the variable
269 evaluate to a different value while fixed.
273 self.fixed_val = self.value()
277 "Set the variable to be modifiable again."
280 def value(self, depth=0):
281 "Handle the case when fixed, raise exc. on different evaluation"
282 val = super(ConfigVar,self).value(depth)
283 if self.fixed and self.fixed_val != val:
284 raise VariableFixedError("value of var %s was fixed to %r but evaluated to %r", self.name, self.fixed_val, val)
287 def add_operation(self, operation):
289 Inserts an operation. The operations are sorted by `level` (ascending), new operation goes last among
290 these with the same level.
291 Adds the variable as a dependant of the conditions and variables used in the expressions.
293 # Invalidate cached value
296 pos = bisect.bisect_right([o.level for o in self.operations], operation.level)
297 self.operations.insert(pos, operation)
298 # Create dependencies
299 for v in operation.expression.variables():
300 v.dependants.add(self)
301 if operation.condition:
302 operation.condition.dependants.add(self)
304 def remove_operation(self, operation):
306 Remove the Operation.
307 Also removes the variable as dependant from all conditions and variables used in this
308 operation that are no longer used.
310 # Invalidate cached value
312 # Remove the operation
313 self.operations.remove(operation)
314 # Remove dependencies on variables unused in other operations
315 vs = self.variables()
316 for v in operation.expression.variables():
318 v.dependants.remove(self)
319 # Remove the dependency on the conditions (if not used in another operation)
320 if operation.condition and operation.condition not in [op.condition for op in self.operations]:
321 operation.condition.dependants.remove(self)
323 def evaluate(self, depth=0):
325 Find the last 'SET' operation that applies and return the result of concatenating with all
326 subsequent applicable 'APPEND' operations. The result is the same as performing the operations
328 NOTE: undefined if some 'APPEND' apply but no 'SET' applies.
331 log.debug('evaluating var %r', self.name)
332 # List of strings to be concatenated
334 # Scan for last applicable expression - try each starting from the end, concatenate extensions
335 for i in range(len(self.operations)-1, -1, -1):
336 op = self.operations[i]
337 # Check the guarding condition
338 if (not op.condition) or op.condition.value(depth+1):
339 val.insert(0, op.expression.evaluate(depth=depth+1))
340 if op.operation == 'SET':
344 def dump(self, prefix=''):
346 Pretty printing of the variable. Includes all operations.
347 Returns iterator of lines (unicode strings).
349 # Try to evaluate the variable, but avoid undefined exceptions
352 v = self.value(depth=0)
355 yield prefix+u'%s = %r' % (self.name, v)
356 for op in self.operations:
357 #yield prefix+u' %s [%s] %s' % (op.operation, op.condition and op.condition.str(parents=True), op.expression)
358 yield prefix + u' ' + unicode(op)
361 class ConfigExpression(object):
363 String expression with some unexpanded config variables. Used in variable operations and conditions.
364 Expression is given as a list of unicode strings and ConfigVar variables to be expanded.
367 def __init__(self, exprlist, original = u'<unknown>'):
368 self.exprlist = list(exprlist)
369 # Original defining string
370 self.original = original
371 # Replace strings with unicode
372 for i in range(len(self.exprlist)):
374 if isinstance(e, types.StringTypes):
375 if not isinstance(e, unicode):
376 self.exprlist[i] = unicode(e, 'ascii')
379 "Return an iterator of variables user in the expression"
380 return itertools.ifilter(lambda e: isinstance(e, ConfigVar), self.exprlist)
385 def evaluate(self, depth):
387 "Return unicode result of expansion of the variables."
389 for e in self.exprlist:
390 if isinstance(e, ConfigVar):
391 s.append(e.value(depth+1))
392 elif isinstance(e, unicode):
395 raise ConfigError('Invalid type %s in expression \'%s\'.'%(type(e), self))