From 0afc6bf98bc0301179d0834da6df46178ffeef1c Mon Sep 17 00:00:00 2001 From: Tomas Gavenciak Date: Fri, 28 May 2010 23:50:54 -0400 Subject: [PATCH] Many fixes to config parser, many (passed) tests. Not complete test coverage yet. See notes in conf.test.py --- t/moe/conf.test.py | 113 +++++++++++++++++++++++++++++--------------- t/moe/confparser.py | 104 ++++++++++++++++++++++++++-------------- 2 files changed, 144 insertions(+), 73 deletions(-) diff --git a/t/moe/conf.test.py b/t/moe/conf.test.py index 6c69a8c..d26b4fe 100644 --- a/t/moe/conf.test.py +++ b/t/moe/conf.test.py @@ -1,46 +1,85 @@ -import conf, confparser +import conf +from confparser import * import logging as log import unittest -#log.getLogger().setLevel(log.DEBUG) +class TestConfig(unittest.TestCase): + def setUp(s): + s.t = conf.ConfigTree() + def parse(s, string, level=0, fname='test'): + c=ConfigParser(string, s.t, fname, level) + ops = c.parse() + c.p_WS() + assert c.eof() + return ops + def val(s, varname): + return s.t.lookup(varname, create=False).value() + def eqparse(s, string, *args, **kwargs): + return [(i[0], i[1].operation) for i in s.parse(string, *args, **kwargs)] -vcnt = 3 +class TestParser(TestConfig): + s1 = r"""a="1";z{b='2';w{S.Tr_an-g.e='""'}};c.d='\n';e="{a}{b}";e+='{c.d}';a+="\"\n\{\}";f+='Z{a.b}'""" + s2 = '\t\n \n ' + s1.replace('=', '= \n ').replace(';', '\t \n\t \n ').replace('+=',' \n += ') + '\n\n ' + def test_noWS(s): + assert len(s.parse(s.s1)) == 8 + def test_noWS_COMMENT(s): + assert s.eqparse(s.s1+'#COMMENT') == s.eqparse(s.s1+'#') == s.eqparse(s.s1+'#\n') == s.eqparse(s.s1+'\n#') + def test_manyWS(s): + assert s.eqparse(s.s2) == s.eqparse(s.s1) + def test_manyWS_COMMENT(s): + assert s.eqparse(s.s2.replace('\n',' #COMMENT \n')) == s.eqparse(s.s2.replace('\n','#\n')) == s.eqparse(s.s1) + def test_empty(s): + assert s.eqparse('') == s.eqparse('\n') == s.eqparse('') == s.eqparse('a{}') == \ + s.eqparse('a.b.c{if ""==\'\' {d.e{\n\n#Nothing\n}} }') == [] + def test_syntax_errors(s): + s.assertRaises(ConfigSyntaxError, s.parse, "a=#") + s.assertRaises(ConfigSyntaxError, s.parse, "a='\"") + s.assertRaises(ConfigSyntaxError, s.parse, 'a="{a@b}"') + s.assertRaises(ConfigSyntaxError, s.parse, 'a="A{A"') + def test_error_location(s): + try: s.parse('\t \n \n { \n \n ') + except ConfigSyntaxError, e: + assert e.line == 3 and e.column in range(2,4) + def test_quoting(s): + s.parse(' a="\\"\\{a$b\\}\'\n\n\'{z}" ') + assert s.t.lookup('z', create=False) + # No escaping in '-string + s.assertRaises(ConfigSyntaxError, s.parse, " a='\"\\'\n\n' ") + # Variable should not be created + s.parse(" a='{z2}' ") + s.assertRaises(conf.ConfigError, s.t.lookup, 'z2', create=False) + def test_conditions(s): + s.assertRaises(ConfigSyntaxError, s.parse, "if '{a}'=='{b}' and ''!='' {}") + s.parse('if ((#C\n (\n (not not not""!="")\n#C\n)\t ) ) {}') + s.parse('if (""=="" and not (not not ""!="" or ""=="")){}') + s.parse('if(""==""){a{if(""==""){if(""==""){b{if(""==""){if(""==""){}}}}}}}') + s.assertRaises(ConfigSyntaxError, s.parse, "if notnot'{a}'=='{b}' {}") + s.assertRaises(ConfigSyntaxError, s.parse, "if ('{a}'=='{b}' not and ''!='') {}") + s.assertRaises(ConfigSyntaxError, s.parse, "if ('{a}'=='{b}' ornot ''!='') {}") -def cs(s): - return conf.ConfigExpression([s], s) +class TestConfigEval(TestConfig): + def test_ops(s): + s.parse('c+="-C_APP"', level=20) + s.parse('a="A"; b="{a}-B"; c="C1-{b}-C2"; a+="FOO"; a="AA"') + assert s.val('c') == 'C1-AA-B-C2-C_APP' + s.parse('b+="-A:\{{a}\}";a+="A"', level=10) + assert s.val('c') == 'C1-AAA-B-A:{AAA}-C2-C_APP' + def test_nested(s): + s.parse('a="0"; b{a="1"; b{a="2"; b{a="3"; b{a="4"; b{a="5"}}}}}') + assert s.val('b.b.b.a') == '3' + s.parse('b.b{b.b{b.a="5MOD"}}') + assert s.val('b.b.b.b.b.a') == '5MOD' + def test_escape_chars(s): + s.parse(r"""a='{a}\\\\#\n'; b="{a}'\"\{\}"; c='\'; c+="\{{b}\}";""") + assert s.val('c') == r"""\{{a}\\\\#\n'"{}}""" -# WIP -class Test(unittest.TestCase): - def setUp(self): - self.t = conf.ConfigTree() - def parse(self, s, level=0, fname='test'): - c=confparser.ConfigParser(s, self.t, fname, level) - c.parse() - c.p_WS() - assert c.eof() - s1 = r"""a="1";b='2';c.d='\n';e="{a}{b}";e+='{c.d}';a+="\"\n\{\}";f+='Z{a.b}'#comment""" - def test_parser_nows(self): - self.parse(s1) - -root = conf.ConfigTree() +# TODO: conditions, chaining conditions, undefined (incl +=), loops, removal, fixing, fail on 1st April +# TODO: coverage -for i in range(vcnt): - root.lookup('a.v%d'%i).add_operation(conf.Operation('SET', None, cs('A%d'%i))) - b = root.lookup('b.v%d'%i) - b.add_operation(conf.Operation('APPEND', None, cs(' '))) - b.add_operation(conf.Operation('SET', None, conf.ConfigExpression([root.lookup('a.v%d'%i)], '{a.v%d}'%i))) - b.add_operation(conf.Operation('APPEND', None, cs(' '))) - if i'))) -root.lookup('a.v1').add_operation(conf.Operation('APPEND', None, cs(' '))) -print '\n'.join(root.dump()) +# TODO: log.info('maxdepth: %d', conf.debug_maxdepth) -root.lookup('a.v0').add_operation(conf.Operation('SET', None, cs(''))) -print '\n'.join(root.dump()) -print 'maxdepth: %d'%conf.debug_maxdepth diff --git a/t/moe/confparser.py b/t/moe/confparser.py index 683a3a0..de4a515 100644 --- a/t/moe/confparser.py +++ b/t/moe/confparser.py @@ -42,8 +42,10 @@ ECHAR = re('([^\\{}]|\\\\|\\{|\\}|\\n)*') VARNAME = re('[a-zA-Z0-9-_]+(\.[a-zA-Z0-9-_]+)*') """ -import re, itertools, logging as log - +import re, types, itertools, logging as log +import traceback +import conf + class ConfigSyntaxError(Exception): # TODO: choose a better superclass def __init__(self, msg, fname='', line=None, column=None): @@ -89,11 +91,12 @@ class ConfigParser(object): self.bufpos = 0 self.fname = fname # Filename self.line = 1 - self.col = 1 + self.column = 1 self.tree = tree # ConfTree to fill self.level = level # level of the parsed operations self.prefix = '' # Prefix of variable name, may begin with '.' - self.conds = [] # Stack of nested conditions, these are chained, so only the last is necessary + self.conditions = [] # Stack of nested conditions, these are chained, so only the last is necessary + self.read_ops = [] # List of parsed operations (varname, `Operation`), returned by `self.parse()` def preread(self, l): "Make sure buf contains at least `l` next characters, return True on succes and False on hitting EOF." if isinstance(self.s, file): @@ -103,18 +106,17 @@ class ConfigParser(object): def peek(self, l = 1): "Peek and return next `l` unicode characters or everything until EOF." self.preread(l) - return self.buf[:l] + return self.buf[self.bufpos:self.bufpos+l] def peeks(self, s): "Peek and compare next `len(s)` characters to `s`. Converts `s` to unicode. False on hitting EOF." s = unicode(s) return self.peek(len(s)) == s - return True def next(self, l = 1): "Eat and return next `l` unicode characters. Raise exception on EOF." if not self.preread(l): raise ConfigSyntaxError("Unexpected end of file") s = self.buf[self.bufpos:self.bufpos+l] - bufpos += l + self.bufpos += l rnl = s.rfind('\n') if rnl<0: # no newline @@ -134,27 +136,42 @@ class ConfigParser(object): return False def eof(self): "Check for end-of-stream." - return self.preread(1) - def expected(self, s, msg=None): + return not self.preread(1) + def expect(self, s, msg=None): "Eat and compare next `len(s)` characters to `s`. If not equal, raise an error with `msg`. Unicode." s = unicode(s) if not self.nexts(s): - raise self.syntaxError(msg or u"%r expected."%(s,)) - def syntaxError(self, msg, *args): + self.syntax_error(msg or u"%r expected."%(s,)) + def syntax_error(self, msg, *args): "Raise a syntax error with file/line/column info" - raise ConfSyntaxError(fname=self.fname, line=self.line, column=self.column, msg=(msg%args)) + raise ConfigSyntaxError(fname=self.fname, line=self.line, column=self.column, msg=(msg%args)) + def dbg(self): + n = None; s = '' + for i in traceback.extract_stack(): + if i[2][:2]=='p_': + s += ' ' + n = i[2] + if n: log.debug(s + n + ' ' + repr(self.peek(15)) + '...') def parse(self): - p_BLOCK(self) + self.read_ops = [] + self.p_BLOCK() + return self.read_ops def p_BLOCK(self): + self.dbg() # Debug self.p_WS() - while not self.eof() and not f.peek(self.c_close): + while (not self.eof()) and (not self.peeks(self.c_close)): self.p_STATEMENT() - slef.p_WS() - if not self.peek() in self.c_sep: + l0 = self.line + self.p_WS() + if self.eof() or self.peeks(self.c_close): break - self.p_SEP() + if self.line == l0: # No newline skipped in p_WS + self.expect(';') + else: + self.nexts(';') # NOTE: this is weird - can ';' occur anywhere? Or at most once, but only after any p_WS debris? self.p_WS() - def p_WS(): + def p_WS(self): + self.dbg() # Debug while not self.eof(): if self.peek() in self.c_ws: self.next() @@ -163,10 +180,12 @@ class ConfigParser(object): else: break def p_COMMENT(self): + self.dbg() # Debug self.expect(self.c_comment, "'#' expected at the beginning of a comment.") - while not self.eof() and not self.nexts(self.c_nl): - pass + while (not self.eof()) and (not self.nexts(self.c_nl)): + self.next(1) def p_STATEMENT(self): + self.dbg() # Debug self.p_WS() if self.peeks(self.c_if): self.p_CONDITION() @@ -174,13 +193,12 @@ class ConfigParser(object): # for operation or subtree, read VARNAME varname = self.p_VARNAME() self.p_WS() - if self.nexts(self.c_open): - self.p_BLOCK(varname) - self.p_WS() - self.expect(self.c_close) + if self.peeks(self.c_open): + self.p_SUBTREE(varname) else: self.p_OPERATION(varname) def p_SUBTREE(self, varname=None): + self.dbg() # Debug if not varname: self.p_WS() varname = self.p_VARNAME() @@ -195,6 +213,7 @@ class ConfigParser(object): self.p_WS() self.expect(self.c_close) def p_OPERATION(self, varname=None): + self.dbg() # Debug if not varname: self.p_WS() varname = self.p_VARNAME() @@ -204,23 +223,27 @@ class ConfigParser(object): elif self.nexts(self.c_append): op = 'APPEND' else: - self.syntaxError('Unknown operation.') + self.syntax_error('Unknown operation.') self.p_WS() exp = self.p_EXPRESSION() - v = self.tree.lookup((self.prefix+self.c_varname_sep+varname).lstrip(self.c_varname_sep)) + vname = (self.prefix+self.c_varname_sep+varname).lstrip(self.c_varname_sep) + v = self.tree.lookup(vname) if self.conditions: cnd = self.conditions[-1] else: cnd = None - v.add_operation(conf.Operation(op, cnd, exp, level=self.level, - source="%s:%d:%d"%(self.fname, self.line, self.column))) - # NOTE/WARNING: The last character of operation is reported. + op = conf.Operation(op, cnd, exp, level=self.level, + source="%s:%d:%d"%(self.fname, self.line, self.column)) + # NOTE/WARNING: The last character of operation will be reported in case of error. + v.add_operation(op) + self.read_ops.append( (vname, op) ) def p_CONDITION(self): + self.dbg() # Debug self.p_WS() self.expect(self.c_if) self.p_WS() - f = p_FORMULA(self) - cnd = ConfigCondition(f) + f = self.p_FORMULA() + cnd = conf.ConfigCondition(f) self.conditions.append(cnd) # Parse a block self.p_WS() @@ -231,14 +254,16 @@ class ConfigParser(object): # Cleanup self.conditions.pop() def p_VARNAME(self): + self.dbg() # Debug vnl = [] - while self.peek().isalnum() or self.peek() in u'-_': + while self.peek().isalnum() or self.peek() in u'-_.': vnl.append(self.next()) vn = u''.join(vnl) if not re_VARNAME.match(vn): - self.syntax_error('Invalid variable name') + self.syntax_error('Invalid variable name %r', vn) return vn def p_EXPRESSION(self): + self.dbg() # Debug op = self.next() if op not in '\'"': self.syntax_error('Invalid start of expression') @@ -249,7 +274,7 @@ class ConfigParser(object): exl.append(self.next()) self.expect(op) s = u''.join(exl) - return ConfigExpression((s,), s) + return conf.ConfigExpression((s,), s) # Parse expression with variables exl = [op] expr = [] @@ -279,23 +304,28 @@ class ConfigParser(object): # Concatenate consecutive characters in expr expr2 = [] for i in expr: - if expr2 and isinstance(expr2[-1], unicode): + if expr2 and isinstance(expr2[-1], unicode) and isinstance(i, unicode): expr2[-1] = expr2[-1] + i else: expr2.append(i) - return ConfigExpression(tuple(expr2), exs) + return conf.ConfigExpression(tuple(expr2), exs) def p_FORMULA(self): + self.dbg() # Debug self.p_WS() # Combined logical formula if self.nexts(u'('): f1 = self.p_FORMULA() self.p_WS() if self.nexts(self.c_and): + if self.peek(1).isalnum(): + self.syntax_error('trailing characters after %r', self.c_and) f2 = self.p_FORMULA() self.p_WS() self.expect(u')') return ('AND', f1, f2) elif self.nexts(self.c_or): + if self.peek(1).isalnum(): + self.syntax_error('trailing characters after %r', self.c_or) f2 = self.p_FORMULA() self.p_WS() self.expect(u')') @@ -306,6 +336,8 @@ class ConfigParser(object): else: self.syntax_error("Logic operator or ')' expected") elif self.nexts(self.c_not): + if self.peek().isalnum(): + self.syntax_error('trailing characters after %r', self.c_not) # 'not' formula f = self.p_FORMULA() return ('NOT', f) -- 2.39.2