Simple Moe configuration file syntax parser.
-TODO: decide '()' around formulas
+TODO: decide neccessity of '()' in/around formulas
TODO: check escaping in expressions
TODO: should whitespace (incl. '\\n') be allowed (almost) everywhere?
can comment be anywhere whitespace can?
Generally, whitespace and comments are alowed everywhere except in variable names and inside expressions.
Also, COMMENT must not contain '\\n'.
-FILE, BLOCK, STATEMENT, OPERATION, SUBTREE, CONDITION, FORMULA, AND, OR and NOT eat any preceding whitespace.
+
+FILE, BLOCK, STATEMENT, OPERATION, SUBTREE, CONDITION, FORMULA, AND, OR and NOT eat any preceding whitespace. TODO: check?
The configuration syntax is the following:
STATEMENT = CONDITION | OPERATION | SUBTREE
OPERATION = WS VARNAME WS ( '=' | '+=' ) WS EXPRESSION
-SUBTREE = WS VARNAME WS '{' BLOCK '}'
+SUBTREE = WS VARNAME WS '{' BLOCK WS '}'
CONDITION = WS 'if' FORMULA WS '{' BLOCK WS '}'
FORMULA = WS (( EXPRESSION WS ( '!=' | '==' ) WS EXPRESSION ) | '(' AND WS ')' | '(' OR WS ')' | NOT )
OR = FORMULA WS 'or' FORMULA
NOT = WS 'not' FORMULA
-EXPRESSION = '"' ( ECHAR | '{' VARNAME '}' )* '"' | re"'[^'\\n]*'"
+NOTE: ';' or '\n' is currently required even after CONDITION and SUBTREE block
+ TODO: change to OPERATION only
+NOTE: Formula may contain additional/extra parentheses
+
+EXPRESSION = '"' ( ECHAR | '{' VARNAME '}' )* '"' | re"'[^'\\n]*'" | VARNAME
ECHAR = re('([^\\{}]|\\\\|\\{|\\}|\\n)*')
+VARNAME = re('[a-zA-Z0-9-_]+(\.[a-zA-Z0-9-_]+)*')
"""
-import re, logging as log
+import re, types, itertools, logging as log
+import traceback
+import moe.conf as conf
+
+
+class ConfigSyntaxError(conf.ConfigError):
+
+ def __init__(self, msg, fname='<unknown>', line=None, column=None):
+ self.msg = msg
+ self.fname = fname
+ self.line = line
+ self.column = column
+
+ def __str__(self):
+ return('ConfigSyntaxError %s:%d:%d: %s'%(self.fname, self.line, self.column, self.msg))
+
+
+class ConfigParser(object):
+ c_varname_sep = u'.'
+ c_comment = u'#'
+ c_open = u'{'
+ c_close = u'}'
+ c_ws = u' \t\n'
+ c_sep = u';\n'
+ c_nl = u'\n'
+ c_if = u'if'
+ c_and = u'and'
+ c_or = u'or'
+ c_not = u'not'
+ c_eq = u'=='
+ c_neq = u'!='
+ c_set = u'='
+ c_append = u'+='
+
+ def __init__(self, s, tree, fname='<unknown>', level=0):
+ """Create a config file parser.
+ `s` is either a string, unicode or an open file. File is assumed to be utf-8, string is converted to unicode.
+ `tree` is a ConfigTree to fill the operations into.
+ `fname` is an optional name of the file, for debugging and syntax errors.
+ `level` indicates the precedence the operations should have in the ConfigTree
+ """
+ self.s = s # Unicode, ascii string or an open file
+ self.buf = u"" # Read-buffer for s file, whole unicode string for s string/unicode
+ if isinstance(self.s, types.StringTypes):
+ self.buf = unicode(self.s)
+ elif (not isinstance(self.s, file)) or self.s.closed:
+ raise TypeError("Expected unicode, str or open file.")
+ self.bufpos = 0
+ self.fname = fname # Filename
+ self.line = 1
+ self.column = 1
+ self.tree = tree # ConfTree to fill
+ self.level = level # level of the parsed operations
+ self.prefix = '' # Prefix of variable name, may begin with '.'
+ self.conditions = [] # Stack of nested conditions, these are chained, so only the last is necessary
+ self.read_ops = [] # List of parsed operations (varname, `Operation`), returned by `self.parse()`
+
+ def preread(self, l):
+ "Make sure buf contains at least `l` next characters, return True on succes and False on hitting EOF."
+ if isinstance(self.s, file):
+ self.buf = self.buf[self.bufpos:] + self.s.read(max(l, 1024)).decode('utf8')
+ self.bufpos = 0
+ return len(self.buf) >= self.bufpos + l
+
+ def peek(self, l = 1):
+ "Peek and return next `l` unicode characters or everything until EOF."
+ self.preread(l)
+ return self.buf[self.bufpos:self.bufpos+l]
+
+ def peeks(self, s):
+ "Peek and compare next `len(s)` characters to `s`. Converts `s` to unicode. False on hitting EOF."
+ s = unicode(s)
+ return self.peek(len(s)) == s
+
+ def next(self, l = 1):
+ "Eat and return next `l` unicode characters. Raise exception on EOF."
+ if not self.preread(l):
+ self.syntax_error("Unexpected end of file")
+ s = self.buf[self.bufpos:self.bufpos+l]
+ self.bufpos += l
+ rnl = s.rfind('\n')
+ if rnl<0:
+ # no newline
+ self.column += l
+ else:
+ # some newlines
+ self.line += s.count('\n')
+ self.column = l - rnl - 1
+ return s
+
+ def nexts(self, s):
+ """Compare next `len(s)` characters to `s`. On match, eat them and return True. Otherwise just return False.
+ Converts `s` to unicode. False on hitting EOF."""
+ s = unicode(s)
+ if self.peeks(s):
+ self.next(len(s))
+ return True
+ return False
+
+ def eof(self):
+ "Check for end-of-stream."
+ return not self.preread(1)
+
+ def expect(self, s, msg=None):
+ "Eat and compare next `len(s)` characters to `s`. If not equal, raise an error with `msg`. Unicode."
+ s = unicode(s)
+ if not self.nexts(s):
+ self.syntax_error(msg or u"%r expected."%(s,))
+
+ def syntax_error(self, msg, *args):
+ "Raise a syntax error with file/line/column info"
+ raise ConfigSyntaxError(fname=self.fname, line=self.line, column=self.column, msg=(msg%args))
+
+ def dbg(self):
+ n = None; s = ''
+ for i in traceback.extract_stack():
+ if i[2][:2]=='p_':
+ s += ' '
+ n = i[2]
+ if n: log.debug(s + n + ' ' + repr(self.peek(15)) + '...')
+
+ def parse(self):
+ self.read_ops = []
+ self.p_BLOCK()
+ return self.read_ops
+
+ def p_BLOCK(self):
+ self.dbg() # Debug
+ self.p_WS()
+ while (not self.eof()) and (not self.peeks(self.c_close)):
+ self.p_STATEMENT()
+ l0 = self.line
+ self.p_WS()
+ if self.eof() or self.peeks(self.c_close):
+ break
+ if self.line == l0: # No newline skipped in p_WS
+ self.expect(';')
+ else:
+ self.nexts(';') # NOTE: this is weird - can ';' occur anywhere? Or at most once, but only after any p_WS debris?
+ self.p_WS()
+
+ def p_WS(self):
+ self.dbg() # Debug
+ while not self.eof():
+ if self.peek() in self.c_ws:
+ self.next()
+ elif self.peeks(self.c_comment):
+ self.p_COMMENT()
+ else:
+ break
+
+ def p_COMMENT(self):
+ self.dbg() # Debug
+ self.expect(self.c_comment, "'#' expected at the beginning of a comment.")
+ while (not self.eof()) and (not self.nexts(self.c_nl)):
+ self.next(1)
+
+ def p_STATEMENT(self):
+ self.dbg() # Debug
+ self.p_WS()
+ if self.peeks(self.c_if):
+ self.p_CONDITION()
+ else:
+ # for operation or subtree, read VARNAME
+ varname = self.p_VARNAME()
+ self.p_WS()
+ if self.peeks(self.c_open):
+ self.p_SUBTREE(varname)
+ else:
+ self.p_OPERATION(varname)
+
+ def p_SUBTREE(self, varname=None):
+ self.dbg() # Debug
+ if not varname:
+ self.p_WS()
+ varname = self.p_VARNAME()
+ self.p_WS()
+ self.expect(self.c_open)
+ # backup and extend the variable name prefix
+ p = self.prefix
+ self.prefix = p + self.c_varname_sep + varname
+ self.p_BLOCK()
+ self.prefix = p
+ # close block and
+ self.p_WS()
+ self.expect(self.c_close)
+
+ def p_OPERATION(self, varname=None):
+ self.dbg() # Debug
+ if not varname:
+ self.p_WS()
+ varname = self.p_VARNAME()
+ self.p_WS()
+ if self.nexts(self.c_set):
+ op = 'SET'
+ elif self.nexts(self.c_append):
+ op = 'APPEND'
+ else:
+ self.syntax_error('Unknown operation.')
+ self.p_WS()
+ exp = self.p_EXPRESSION()
+ vname = (self.prefix+self.c_varname_sep+varname).lstrip(self.c_varname_sep)
+ v = self.tree.lookup(vname)
+ if self.conditions:
+ cnd = self.conditions[-1]
+ else:
+ cnd = None
+ op = conf.Operation(op, cnd, exp, level=self.level,
+ source="%s:%d:%d"%(self.fname, self.line, self.column))
+ # NOTE/WARNING: The last character of operation will be reported in case of error.
+ v.add_operation(op)
+ self.read_ops.append( (vname, op) )
+
+ def p_CONDITION(self):
+ self.dbg() # Debug
+ self.p_WS()
+ t = u"condition at %s:%d:%d"%(self.fname, self.line, self.column)
+ self.expect(self.c_if)
+ self.p_WS()
+ f = self.p_FORMULA()
+ cnd = conf.ConfigCondition(f, text=t, parent=(self.conditions and self.conditions[-1]) or None)
+ self.conditions.append(cnd)
+ # Parse a block
+ self.p_WS()
+ self.expect(self.c_open)
+ self.p_BLOCK()
+ self.p_WS()
+ self.expect(self.c_close)
+ # Cleanup
+ self.conditions.pop()
+
+ def p_VARNAME(self):
+ self.dbg() # Debug
+ vnl = []
+ while self.preread(1) and (self.peek().isalnum() or self.peek() in u'-_.'):
+ vnl.append(self.next())
+ vn = u''.join(vnl)
+ if not conf.re_VARNAME.match(vn):
+ self.syntax_error('Invalid variable name %r', vn)
+ return vn
-c_tree_sep = u'.'
-c_comment = u'#'
-c_open = u'{'
-c_close = u'}'
-c_if = u'if'
+ def p_EXPRESSION(self):
+ self.dbg() # Debug
+ if self.peek() not in '\'"':
+ # Expect a variable name
+ varname = self.p_VARNAME()
+ return conf.ConfigExpression((self.tree.lookup(varname),), varname)
+ op = self.next()
+ # Parse literal expression
+ if op == u'\'':
+ exl = []
+ while not self.peeks(op):
+ exl.append(self.next())
+ self.expect(op)
+ s = u''.join(exl)
+ return conf.ConfigExpression((s,), s)
+ # Parse expression with variables
+ exl = [op]
+ expr = []
+ while not self.peeks(op):
+ exl.append(self.peek())
+ if self.nexts(u'\\'):
+ # Escape sequence
+ c = self.next()
+ if c not in u'\\"n' + self.c_open + self.c_close:
+ self.syntax_error('Illeal escape sequence in expression')
+ if c == 'n':
+ expr.append(u'\n')
+ else:
+ expr.append(c)
+ exl.append(c)
+ elif self.nexts(self.c_open):
+ # Parse a variable name in '{}'
+ varname = self.p_VARNAME()
+ self.expect(self.c_close)
+ exl.append(varname)
+ expr.append(self.tree.lookup(varname))
+ else:
+ # Regular character
+ expr.append(self.next())
+ self.expect(op)
+ exs = ''.join(exl)
+ # Concatenate consecutive characters in expr
+ expr2 = []
+ for i in expr:
+ if expr2 and isinstance(expr2[-1], unicode) and isinstance(i, unicode):
+ expr2[-1] = expr2[-1] + i
+ else:
+ expr2.append(i)
+ return conf.ConfigExpression(expr2, exs)
-"Variable name regexp, dots (separators) must be separated from edges and each other."
-VARNAME_re = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z')
+ def p_FORMULA(self):
+ self.dbg() # Debug
+ self.p_WS()
+ # Combined logical formula
+ if self.nexts(u'('):
+ f1 = self.p_FORMULA()
+ self.p_WS()
+ if self.nexts(self.c_and):
+ if self.peek(1).isalnum():
+ self.syntax_error('trailing characters after %r', self.c_and)
+ f2 = self.p_FORMULA()
+ self.p_WS()
+ self.expect(u')')
+ return ('AND', f1, f2)
+ elif self.nexts(self.c_or):
+ if self.peek(1).isalnum():
+ self.syntax_error('trailing characters after %r', self.c_or)
+ f2 = self.p_FORMULA()
+ self.p_WS()
+ self.expect(u')')
+ return ('OR', f1, f2)
+ elif self.nexts(u')'):
+ # Only extra parenthes
+ return f1
+ else:
+ self.syntax_error("Logic operator or ')' expected")
+ elif self.nexts(self.c_not):
+ if self.peek().isalnum():
+ self.syntax_error('trailing characters after %r', self.c_not)
+ # 'not' formula
+ f = self.p_FORMULA()
+ return ('NOT', f)
+ else:
+ # Should be (in)equality condition
+ e1 = self.p_EXPRESSION()
+ self.p_WS()
+ if self.nexts(self.c_eq):
+ self.p_WS()
+ e2 = self.p_EXPRESSION()
+ return ('==', e1, e2)
+ elif self.nexts(self.c_neq):
+ self.p_WS()
+ e2 = self.p_EXPRESSION()
+ return ('!=', e1, e2)
+ else:
+ self.syntax_error("Comparation operator expected")
projects / eval.git / blobdiff