1 import types, itertools, re
5 Lazy conditional string evaluation module for configuration variables.
8 * Each variable has ordered list of operations (definitions), each SETs or APPENDs an expression
9 to the value. Each operation may be guarded by condition.
11 NOTE: Variable is undefined even if some 'APPEND' apply but no 'SET' applies. This might change.
13 * Each condition is a formula (tree consisting of 'AND', 'OR', 'NOT' and '==', '!=' between
16 * Expression is a list of strings and variables to be expanded.
18 NOTE: All expanded data should be (or is converted to) unicode
20 TODO: Fixing variables.
21 TODO: Cleanup of unused undefined variables.
22 TODO: Better variable name checking (no name '.'-structural prefix of another)
23 TODO: Implemet "subtree" listing.
24 TODO: Test conditions and unicode
28 The configuration syntax is the following (TODO: add whitespaces WSP)
29 TODO: decide '()' around formulas
30 TODO: check escaping in expressions
31 TODO: should whitespace (incl. '\\n') be allowed (almost) everywhere?
32 can comment be anywhere whitespace can?
35 BLOCK + '\\n' = () | STATEMENT ( STATEMENT-SEP STATEMENT )*
37 STATEMENT-SEP = ( '\\n' | ';' )
38 WSP = ( ' ' | '\\t' | '\\n' | COMMENT )*
40 COMMENT = re'#[^\\n]*\\n'
42 STATEMENT = CONDITION | OPERATION | SUBTREE
44 OPERATION = VARNAME ( '=' | '+=' ) EXPRESSION
45 SUBTREE = VARNAME '{' BLOCK '}'
46 CONDITION = 'if' FORMULA '{' BLOCK '}'
48 FORMULA = ( EXPRESSION ( '!=' | '==' ) EXPRESSION ) | '(' AND ')' | '(' OR ')' | NOT
49 AND = FORMULA 'and' FORMULA
50 OR = FORMULA 'or' FORMULA
53 EXPRESSION = '"' ( ECHAR | '{' VARNAME '}' )* '"' | re"'[^'\\n]*'"
54 ECHAR = re'([^\\{}]|\\\\|\\{|\\}|\\n)*'
66 "Variable name regexp, dots (separators) must be separated from edges and each other."
67 re_key = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z')
69 "Allowed depth of recursion -- includes ALL recursive calls, so should quite high."
72 "Maximum attained depth of recursion"
75 def check_depth(depth):
76 "Helper to check for recursion depth."
78 if depth > c_maxdepth:
79 raise ConfigError('Too deep recursion in config evaluation (cyclic substitution?)')
80 if depth > debug_maxdepth:
81 debug_maxdepth = depth
84 class ConfigError(Exception):
88 class ConfigTree(object):
90 Configuration tree containing all the variables.
92 The variables in `self.variables` are referenced directly by the full name.
96 def lookup(self, key, create = True):
98 Lookup and return a variable.
99 If not found and `create` set, check the name and transparently create a new one.
101 if not key in self.variables:
103 raise ConfigError('Config variable %r undefined.', key)
104 if not re_key.match(key):
105 raise ConfigError('Invalid variable identifier %r in config', key)
106 self.variables[key] = ConfigVar(key)
107 return self.variables[key]
108 def dump(self, prefix=''):
110 Pretty printing of the tree.
111 Returns an iterator of lines (strings).
113 return itertools.chain(*[
114 self.variables[k].dump(prefix) for k in sorted(self.variables.keys())
117 class ConfigElem(object):
119 Base class for cahed config elements - variables and conditions
121 def __init__(self, name):
122 # Full name with separators, definition for conditions
124 # Vars and conditions depending on value of this one
125 self.dependants = set([])
126 # Cached value (may be None in case of evaluation error)
128 self.cached_val = None
129 def invalidate(self, depth=0):
131 Invalidate cached data and invalidate all dependants.
132 Does nothing if not cached.
136 log.debug('invalidating %r', self.name)
138 for d in self.dependants:
139 d.invalidate(depth + 1)
140 def value(self, depth=0):
141 "Caching helper calling self.evaluate(), returns a value or throws an exception."
144 self.cached_val = self.evaluate(depth+1)
146 if self.cached_val == None:
147 raise ConfigError("Unable to evaluate %r."%(self.name,))
148 return self.cached_val
152 class ConfigCondition(ConfigElem):
154 Condition using equality and logic operators.
155 Clause is a tuple-tree in the following recursive form:
156 ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1),
157 ('==', e1, e2), ('!=', e1, e2) where e1, e2 are `ConfigExpression`s.
159 def __init__(self, text, clause, parent=None):
161 Condition defined by `text` (informative), `clause` as in class definition,
162 `parent` is the parent condition (if any).
164 super(ConfigVar, self).__init__(text)
167 # Setup dependencies on used variables (not on the parent condition)
168 for v in self.variables():
169 v.dependants.add(self)
171 self.parent.dependants.add(self)
172 def variables(self, cl=None):
173 "Return an iterator of variables used in clause `cl`"
176 if cl[0] in ['==','!=']:
177 return itertools.chain(cl[1].variables(), cl[2].variables())
178 if cl[0] in ['AND','OR']:
179 return itertools.chain(self.variables(cl[1]), self.variables(cl[2]))
180 return self.variables(cl[1]) # only 'NOT' left
181 def remove_dependencies(self):
182 "Remove self as a dependant from all used variables"
183 for v in self.variables():
184 v.dependants.discard(self)
186 self.parent.dependants.discard(self)
187 def evaluate(self, cl=None, depth=0):
188 """Evaluate clause `cl` (or the entire condition).
189 Partial evaluation for AND and OR. Tests the parent condition first."""
193 if self.parent and not self.parent.value():
195 if cl[0] in ['==','!=']:
196 v = cl[1].evaluate(depth+1) == cl[2].evaluate(depth+1)
197 if cl[0] == '!=': v = not v
199 v1 = self.evaluate(cl[1], depth+1)
202 if cl[0] == 'OR' and v1: return True
203 if cl[0] == 'AND' and not v1: return False
204 return self.evaluate(cl[2], depth+1)
205 def str(self, parents=False):
206 "Retur the defining expression, if `parents` set, then prefixed with parent conditions."
207 if parents and self.parent:
208 return self.parent.str(parents=True) + u' && ' + self.name
211 return self.str(parents=False)
213 class ConfigVar(ConfigElem):
214 def __init__(self, name):
215 super(ConfigVar, self).__init__(name)
216 # Ordered list of operations
217 # (operation, condition, expression)
218 # operation is currently 'SET' and 'APPEND'
220 # Fixed to value (may be None) # TODO
222 self.fixed_val = None
224 "Return a set of variables used in the expressions"
225 return set(sum([ list(e[2].variables()) for e in self.operations ], []))
226 def add_operation(self, operation, condition, expression, index=None):
228 Inserts a new operation to position `index` (`None` appends).
229 Adds the variable as a dependant of the conditions and variables used in the expressions.
231 # Invalidate cached value
234 expr = (operation, condition, expression)
236 self.operations.insert(index, expr)
238 self.operations.append(expr)
239 # Create dependencies
240 for v in expression.variables():
241 v.dependants.add(self)
243 condition.dependants.add(self)
244 def remove_operation(self, index):
246 Remove the operation at given index.
247 Also removes the variable as dependant from all conditions and variables used in this
248 operation that are no longer used.
250 # Invalidate cached value
252 # Remove the operation
253 operation, condition, expression = self.operations[index]
254 self.operations.pop(index)
255 # Remove dependencies on variables unused in other operations
256 vs = self.variables()
257 for v in expression.variables():
259 v.dependants.remove(self)
260 # Remove the dependency on the conditions (if not used in another operation)
261 if condition and condition not in [e[1] for e in self.operations]:
262 condition.dependants.remove(self)
263 def evaluate(self, depth=0):
265 Find the last 'SET' operation that applies and return the result of concatenating with all
266 subsequent applicable 'APPEND' operations. The result is the same as performing the operations
268 NOTE: undefined if some 'APPEND' apply but no 'SET' applies.
271 log.debug('evaluating var %r', self.name)
272 # List of strings to be concatenated
274 # Scan for last applicable expression - try each starting from the end, concatenate extensions
275 for i in range(len(self.operations)-1, -1, -1):
276 operation, condition, expr = self.operations[i]
277 # Check the guarding condition
278 if (not condition) or condition.value(depth+1):
279 val.insert(0, expr.evaluate(depth+1))
280 if operation == 'SET':
283 def dump(self, prefix=''):
285 Pretty printing of the variable. Includes all operations.
286 Returns iterator of lines (unicode strings).
288 # Try to evaluate the variable, but avoid undefined exceptions
291 v = self.value(depth=0)
294 yield prefix+u'%s = %r' % (self.name, v)
295 for operation, condition, expr in self.operations:
296 yield prefix+u' %s [%s] %s' % (operation, condition and condition.str(parents=True), expr)
298 class ConfigExpression(object):
300 String expression with some unexpanded config variables. Used in variable operations and conditions.
301 Expression is given as a list of unicode strings and ConfigVar variables to be expanded.
303 def __init__(self, exprlist, original = u'<unknown>'):
304 self.exprlist = exprlist
305 # Original defining string
306 self.original = original
307 # Replace strings with unicode
308 for i in range(len(self.exprlist)):
310 if isinstance(e, types.StringTypes):
311 if not isinstance(e, unicode):
312 self.exprlist[i] = unicode(e, 'ascii')
314 "Return an iterator of variables user in the expression"
315 return itertools.ifilter(lambda e: isinstance(e, ConfigVar), self.exprlist)
318 def evaluate(self, depth):
320 "Return unicode result of expansion of the variables."
322 for e in self.exprlist:
323 if isinstance(e, ConfigVar):
324 s.append(e.value(depth+1))
325 elif isinstance(e, unicode):
328 raise ConfigError('Invalid type %s in expression \'%s\'.'%(type(e), self))