From: Tomas Gavenciak Date: Sat, 10 Jul 2010 08:56:07 +0000 (+0200) Subject: Sanitize by adding newlines detween defs X-Git-Tag: python-dummy-working~44 X-Git-Url: http://mj.ucw.cz/gitweb/?a=commitdiff_plain;h=816ab051157a7f0e42a68f4202297db5699c3373;p=eval.git Sanitize by adding newlines detween defs --- diff --git a/t/moe/conf.py b/t/moe/conf.py index ec657ff..e5602b9 100644 --- a/t/moe/conf.py +++ b/t/moe/conf.py @@ -67,8 +67,10 @@ class ConfigTree(object): The variables in `self.variables` are referenced directly by the full name. """ + def __init__(self): self.variables = {} + def lookup(self, key, create = True): """ Lookup and return a variable. @@ -81,6 +83,7 @@ class ConfigTree(object): raise UndefinedError('Config variable %r undefined.', key) self.variables[key] = ConfigVar(key) return self.variables[key] + def dump(self, prefix=''): """ Pretty printing of the tree. @@ -90,10 +93,12 @@ class ConfigTree(object): self.variables[k].dump(prefix) for k in sorted(self.variables.keys()) ]) + class ConfigElem(object): """ Base class for cahed config elements - variables and conditions """ + def __init__(self, name): # Full name with separators, definition for conditions self.name = name @@ -102,6 +107,7 @@ class ConfigElem(object): # Cached value (may be None in case of evaluation error) self.cached = False self.cached_val = None + def invalidate(self, depth=0): """ Invalidate cached data and invalidate all dependants. @@ -113,6 +119,7 @@ class ConfigElem(object): self.cached = False for d in self.dependants: d.invalidate(depth + 1) + def value(self, depth=0): "Caching helper calling self.evaluate(), returns a value or throws an exception." check_depth(depth) @@ -122,9 +129,11 @@ class ConfigElem(object): if self.cached_val == None: raise UndefinedError("Unable to evaluate %r."%(self.name,)) return self.cached_val + def __str__(self): return self.name + class ConfigCondition(ConfigElem): """ Condition using equality and logic operators. @@ -132,6 +141,7 @@ class ConfigCondition(ConfigElem): ('AND', c1, c1), ('OR', c1, c2), ('NOT', c1), ('==', e1, e2), ('!=', e1, e2) where e1, e2 are `ConfigExpression`s. """ + def __init__(self, formula, text=None, parent=None): """ Condition defined by `text` (informative), `formula` as in class definition, @@ -147,6 +157,7 @@ class ConfigCondition(ConfigElem): v.dependants.add(self) if self.parent: self.parent.dependants.add(self) + def variables(self, cl=None): "Return an iterator of variables used in formula `cl`" if not cl: @@ -156,12 +167,14 @@ class ConfigCondition(ConfigElem): if cl[0] in ['AND','OR']: return itertools.chain(self.variables(cl[1]), self.variables(cl[2])) return self.variables(cl[1]) # only 'NOT' left + def remove_dependencies(self): "Remove self as a dependant from all used variables" for v in self.variables(): v.dependants.discard(self) if self.parent: self.parent.dependants.discard(self) + def evaluate(self, cl=None, depth=0): """Evaluate formula `cl` (or the entire condition). Partial evaluation for AND and OR. Tests the parent condition first.""" @@ -180,6 +193,7 @@ class ConfigCondition(ConfigElem): if cl[0] == 'OR' and v1: return True if cl[0] == 'AND' and not v1: return False return self.evaluate(cl=cl[2], depth=depth+1) + def formula_string(self, formula): "Create a string representation of a formula." if formula[0] == 'AND': @@ -191,16 +205,20 @@ class ConfigCondition(ConfigElem): elif formula[0] in ['==', '!=']: return itertools.chain(formula[1], formula[0], formula[2]) return iter(['']) + def str(self, parents=False): "Retur the defining expression, if `parents` set, then prefixed with parent conditions." if parents and self.parent: return self.parent.str(parents=True) + u' && ' + self.name return self.name + def __str__(self): return self.str(parents=False) + class Operation(object): "Helper class for operation data. Must not be present in more variables or present multiple times." + def __init__(self, operation, condition, expression, level=0, source='?'): # operation is currently 'SET' and 'APPEND' self.operation = operation @@ -208,11 +226,14 @@ class Operation(object): self.expression = expression self.level = level self.source = source + def __str__(self): return "%s <%d, %s> [%s] %r" % ( {'SET':'=', 'APPEND':'+'}[self.operation], self.level, self.source, (self.condition and self.condition.str(parents=True)) or '', unicode(self.expression)) + class ConfigVar(ConfigElem): + def __init__(self, name): super(ConfigVar, self).__init__(name) # Ordered list of `Operations` (ascending by `level`) @@ -220,9 +241,11 @@ class ConfigVar(ConfigElem): # Fixed to value (may be None) self.fixed = False self.fixed_val = None + def variables(self): "Return a set of variables used in the expressions" return set(sum([ list(op.expression.variables()) for op in self.operations ], [])) + def fix(self): """ Fixes the value of the variable. Exception is raised should the variable @@ -232,15 +255,18 @@ class ConfigVar(ConfigElem): return self.fixed = True self.fixed_val = self.value() + def unfix(self): "Set the variable to be modifiable again." self.fixed = False + def value(self, depth=0): "Handle the case when fixed, raise exc. on different evaluation" val = super(ConfigVar,self).value(depth) if self.fixed and self.fixed_val != val: raise VariableFixedError("value of var %s was fixed to %r but evaluated to %r", self.name, self.fixed_val, val) return val + def add_operation(self, operation): """ Inserts an operation. The operations are sorted by `level` (ascending), new operation goes last among @@ -257,6 +283,7 @@ class ConfigVar(ConfigElem): v.dependants.add(self) if operation.condition: operation.condition.dependants.add(self) + def remove_operation(self, operation): """ Remove the Operation. @@ -275,6 +302,7 @@ class ConfigVar(ConfigElem): # Remove the dependency on the conditions (if not used in another operation) if operation.condition and operation.condition not in [op.condition for op in self.operations]: operation.condition.dependants.remove(self) + def evaluate(self, depth=0): """ Find the last 'SET' operation that applies and return the result of concatenating with all @@ -295,6 +323,7 @@ class ConfigVar(ConfigElem): if op.operation == 'SET': return u''.join(val) return None + def dump(self, prefix=''): """ Pretty printing of the variable. Includes all operations. @@ -311,11 +340,13 @@ class ConfigVar(ConfigElem): #yield prefix+u' %s [%s] %s' % (op.operation, op.condition and op.condition.str(parents=True), op.expression) yield prefix + u' ' + unicode(op) + class ConfigExpression(object): """ String expression with some unexpanded config variables. Used in variable operations and conditions. Expression is given as a list of unicode strings and ConfigVar variables to be expanded. """ + def __init__(self, exprlist, original = u''): self.exprlist = exprlist # Original defining string @@ -326,11 +357,14 @@ class ConfigExpression(object): if isinstance(e, types.StringTypes): if not isinstance(e, unicode): self.exprlist[i] = unicode(e, 'ascii') + def variables(self): "Return an iterator of variables user in the expression" return itertools.ifilter(lambda e: isinstance(e, ConfigVar), self.exprlist) + def __str__(self): return self.original + def evaluate(self, depth): check_depth(depth) "Return unicode result of expansion of the variables." diff --git a/t/moe/confparser.py b/t/moe/confparser.py index 4e482ad..ebc453d 100644 --- a/t/moe/confparser.py +++ b/t/moe/confparser.py @@ -48,15 +48,19 @@ import re, types, itertools, logging as log import traceback import moe.conf as conf + class ConfigSyntaxError(conf.ConfigError): + def __init__(self, msg, fname='', line=None, column=None): self.msg = msg self.fname = fname self.line = line self.column = column + def __str__(self): return('ConfigSyntaxError %s:%d:%d: %s'%(self.fname, self.line, self.column, self.msg)) + class ConfigParser(object): c_varname_sep = u'.' c_comment = u'#' @@ -73,6 +77,7 @@ class ConfigParser(object): c_neq = u'!=' c_set = u'=' c_append = u'+=' + def __init__(self, s, tree, fname='', level=0): """Create a config file parser. `s` is either a string, unicode or an open file. File is assumed to be utf-8, string is converted to unicode. @@ -95,20 +100,24 @@ class ConfigParser(object): self.prefix = '' # Prefix of variable name, may begin with '.' self.conditions = [] # Stack of nested conditions, these are chained, so only the last is necessary self.read_ops = [] # List of parsed operations (varname, `Operation`), returned by `self.parse()` + def preread(self, l): "Make sure buf contains at least `l` next characters, return True on succes and False on hitting EOF." if isinstance(self.s, file): self.buf = self.buf[self.bufpos:] + self.s.read(max(l, 1024)).decode('utf8') self.bufpos = 0 return len(self.buf) >= self.bufpos + l + def peek(self, l = 1): "Peek and return next `l` unicode characters or everything until EOF." self.preread(l) return self.buf[self.bufpos:self.bufpos+l] + def peeks(self, s): "Peek and compare next `len(s)` characters to `s`. Converts `s` to unicode. False on hitting EOF." s = unicode(s) return self.peek(len(s)) == s + def next(self, l = 1): "Eat and return next `l` unicode characters. Raise exception on EOF." if not self.preread(l): @@ -124,6 +133,7 @@ class ConfigParser(object): self.line += s.count('\n') self.column = l - rnl - 1 return s + def nexts(self, s): """Compare next `len(s)` characters to `s`. On match, eat them and return True. Otherwise just return False. Converts `s` to unicode. False on hitting EOF.""" @@ -132,17 +142,21 @@ class ConfigParser(object): self.next(len(s)) return True return False + def eof(self): "Check for end-of-stream." return not self.preread(1) + def expect(self, s, msg=None): "Eat and compare next `len(s)` characters to `s`. If not equal, raise an error with `msg`. Unicode." s = unicode(s) if not self.nexts(s): self.syntax_error(msg or u"%r expected."%(s,)) + def syntax_error(self, msg, *args): "Raise a syntax error with file/line/column info" raise ConfigSyntaxError(fname=self.fname, line=self.line, column=self.column, msg=(msg%args)) + def dbg(self): n = None; s = '' for i in traceback.extract_stack(): @@ -150,10 +164,12 @@ class ConfigParser(object): s += ' ' n = i[2] if n: log.debug(s + n + ' ' + repr(self.peek(15)) + '...') + def parse(self): self.read_ops = [] self.p_BLOCK() return self.read_ops + def p_BLOCK(self): self.dbg() # Debug self.p_WS() @@ -168,6 +184,7 @@ class ConfigParser(object): else: self.nexts(';') # NOTE: this is weird - can ';' occur anywhere? Or at most once, but only after any p_WS debris? self.p_WS() + def p_WS(self): self.dbg() # Debug while not self.eof(): @@ -177,11 +194,13 @@ class ConfigParser(object): self.p_COMMENT() else: break + def p_COMMENT(self): self.dbg() # Debug self.expect(self.c_comment, "'#' expected at the beginning of a comment.") while (not self.eof()) and (not self.nexts(self.c_nl)): self.next(1) + def p_STATEMENT(self): self.dbg() # Debug self.p_WS() @@ -195,6 +214,7 @@ class ConfigParser(object): self.p_SUBTREE(varname) else: self.p_OPERATION(varname) + def p_SUBTREE(self, varname=None): self.dbg() # Debug if not varname: @@ -210,6 +230,7 @@ class ConfigParser(object): # close block and self.p_WS() self.expect(self.c_close) + def p_OPERATION(self, varname=None): self.dbg() # Debug if not varname: @@ -235,6 +256,7 @@ class ConfigParser(object): # NOTE/WARNING: The last character of operation will be reported in case of error. v.add_operation(op) self.read_ops.append( (vname, op) ) + def p_CONDITION(self): self.dbg() # Debug self.p_WS() @@ -252,6 +274,7 @@ class ConfigParser(object): self.expect(self.c_close) # Cleanup self.conditions.pop() + def p_VARNAME(self): self.dbg() # Debug vnl = [] @@ -261,6 +284,7 @@ class ConfigParser(object): if not conf.re_VARNAME.match(vn): self.syntax_error('Invalid variable name %r', vn) return vn + def p_EXPRESSION(self): self.dbg() # Debug op = self.next() @@ -308,6 +332,7 @@ class ConfigParser(object): else: expr2.append(i) return conf.ConfigExpression(tuple(expr2), exs) + def p_FORMULA(self): self.dbg() # Debug self.p_WS() diff --git a/t/moe/conftest.py b/t/moe/conftest.py index 31dac56..38ae76e 100644 --- a/t/moe/conftest.py +++ b/t/moe/conftest.py @@ -4,42 +4,55 @@ import logging as log import unittest class TestConfig(unittest.TestCase): + def setUp(s): s.t = conf.ConfigTree() + def parse(s, string, level=0, fname='test'): c=ConfigParser(string, s.t, fname, level) ops = c.parse() c.p_WS() assert c.eof() return ops + def val(s, varname): return s.t.lookup(varname, create=False).value() + def eqparse(s, string, *args, **kwargs): return [(i[0], i[1].operation) for i in s.parse(string, *args, **kwargs)] + class TestParser(TestConfig): s1 = r"""a="1";z{b='2';w{S.Tr_an-g.e='""'}};c.d='\n';e="{a}{b}";e+='{c.d}';a+="\"\n\{\}";f+='Z{a.b}'""" s2 = '\t\n \n ' + s1.replace('=', '= \n ').replace(';', '\t \n\t \n ').replace('+=',' \n += ') + '\n\n ' + def test_noWS(s): assert len(s.parse(s.s1)) == 8 + def test_noWS_COMMENT(s): assert s.eqparse(s.s1+'#COMMENT') == s.eqparse(s.s1+'#') == s.eqparse(s.s1+'#\n') == s.eqparse(s.s1+'\n#') + def test_manyWS(s): assert s.eqparse(s.s2) == s.eqparse(s.s1) + def test_manyWS_COMMENT(s): assert s.eqparse(s.s2.replace('\n',' #COMMENT \n')) == s.eqparse(s.s2.replace('\n','#\n')) == s.eqparse(s.s1) + def test_empty(s): assert s.eqparse('') == s.eqparse('\n') == s.eqparse('') == s.eqparse('a{}') == \ s.eqparse('a.b.c{if ""==\'\' {d.e{\n\n#Nothing\n}} }') == [] + def test_syntax_errors(s): s.assertRaises(ConfigSyntaxError, s.parse, "a=#") s.assertRaises(ConfigSyntaxError, s.parse, "a='\"") s.assertRaises(ConfigSyntaxError, s.parse, 'a="{a@b}"') s.assertRaises(ConfigSyntaxError, s.parse, 'a="A{A"') + def test_error_location(s): try: s.parse('\t \n \n { \n \n ') except ConfigSyntaxError, e: assert e.line == 3 and e.column in range(2,4) + def test_quoting(s): s.parse(' a="\\"\\{a$b\\}\'\n\n\'{z}" ') assert s.t.lookup('z', create=False) @@ -48,6 +61,7 @@ class TestParser(TestConfig): # Variable should not be created s.parse(" a='{z2}' ") s.assertRaises(conf.ConfigError, s.t.lookup, 'z2', create=False) + def test_conditions(s): s.assertRaises(ConfigSyntaxError, s.parse, "if '{a}'=='{b}' and ''!='' {}") s.parse('if ((#C\n (\n (not not not""!="")\n#C\n)\t ) ) {}') @@ -57,23 +71,28 @@ class TestParser(TestConfig): s.assertRaises(ConfigSyntaxError, s.parse, "if ('{a}'=='{b}' not and ''!='') {}") s.assertRaises(ConfigSyntaxError, s.parse, "if ('{a}'=='{b}' ornot ''!='') {}") + class TestConfigEval(TestConfig): + def test_ops(s): s.parse('c+="-C_APP"', level=20) s.parse('a="A"; b="{a}-B"; c="C1-{b}-C2"; a+="FOO"; a="AA"') assert s.val('c') == 'C1-AA-B-C2-C_APP' s.parse('b+="-A:\{{a}\}";a+="A"', level=10) assert s.val('c') == 'C1-AAA-B-A:{AAA}-C2-C_APP' + def test_nested(s): s.parse('a="0"; b{a="1"; b{a="2"; b{a="3"; b{a="4"; b{a="5"}}}}}') assert s.val('b.b.b.a') == '3' s.parse('b.b{b.b{b.a="5MOD"}}') assert s.val('b.b.b.b.b.a') == '5MOD' + def test_escape_chars(s): s.parse(r"""a='{a}\\\\#\n'; b="{a}'\"\{\}"; c='\'; c+="\{{b}\}";""") assert s.val('c') == r"""\{{a}\\\\#\n'"{}}""" ts = 'a="A:"; if "{c1}"=="1" {a+="C1"; b="B"; if ("{c2a}"=="1" or not "{c2b}"=="1") { a+="C2"; '\ 'if ("{c3a}"=="1" and "{c3b}"=="1") { a+="C3" }}}' + def test_cond_chain(s): s.parse(s.ts) s.parse('c1="1"; c2="0"') @@ -91,6 +110,7 @@ class TestConfigEval(TestConfig): # tests condition invalidating s.parse('c2a+="0"') assert s.val('a') == 'A:C1' + def test_cond_eager(s): s.parse(s.ts) # undefined c2b and c3a should not be evaluated @@ -101,22 +121,26 @@ class TestConfigEval(TestConfig): s.assertRaises(conf.UndefinedError, s.val, 'a') s.parse('c1="1"; c2a="1"; c3b="1"') assert s.val('a') == 'A:C1C2C3' + def test_undef(s): s.assertRaises(conf.UndefinedError, s.val, 'a') s.parse('a="{b}"') s.assertRaises(conf.UndefinedError, s.val, 'a') s.parse('b+="1"') s.assertRaises(conf.UndefinedError, s.val, 'b') + def test_loopy_def(s): s.parse('a="A"; a+="{a}"') s.assertRaises(conf.CyclicConfigError, s.val, 'a') s.parse('b="{c}"; c="{b}"') s.assertRaises(conf.CyclicConfigError, s.val, 'b') + def test_varname(s): s.assertRaises(conf.VariableNameError, s.val, 'b/c') s.assertRaises(conf.VariableNameError, s.val, '.b.c') s.assertRaises(conf.VariableNameError, s.val, 'b.c.') s.assertRaises(conf.VariableNameError, s.val, 'b..c') + def test_remove(s): l = s.parse('a="A1"; b="B1"; if "{cond}"=="1" {a+="A2"; b+="B2"}; a+="A3"; b+="B3"; cond="1"') assert s.val('a') == 'A1A2A3' @@ -134,6 +158,7 @@ class TestConfigEval(TestConfig): # TODO: fixing, fail on 1st April # TODO: coverage + if __name__ == '__main__': log.getLogger().setLevel(log.WARN) #log.getLogger().setLevel(log.DEBUG)