Config parser complete, but completely untested.

Dozens of bugs await!

diff --git a/t/moe/confparser.py b/t/moe/confparser.py
index d7616d55980b72d7a6b157420f542a45c1805d89..683a3a05f7a11a04cfe8f0aa98a20d01f6699547 100644 (file)
--- a/t/moe/confparser.py
+++ b/t/moe/confparser.py
@@ -73,35 +73,68 @@ class ConfigParser(object):
   c_neq = u'!='
   c_set = u'='
   c_append = u'+='
-  def __init__(self, f, tree, fname='<unknown>'):
-    self.f = f         # Stream
+  def __init__(self, s, tree, fname='<unknown>', level=0):
+    """Create a config file parser. 
+    `s` is either a string, unicode or an open file. File is assumed to be utf-8, string is converted to unicode.
+    `tree` is a ConfigTree to fill the operations into.
+    `fname` is an optional name of the file, for debugging and syntax errors. 
+    `level` indicates the precedence the operations should have in the ConfigTree
+    """
+    self.s = s         # Unicode, string or an open file
+    self.buf = u""     # Read-buffer for s file, whole unicode string for s string/unicode
+    if isinstance(self.s, types.StringTypes):
+      self.buf = unicode(self.s)
+    elif (not isinstance(self.s, file)) or self.s.closed:
+      raise TypeError("Expected unicode, str or open file.")
+    self.bufpos = 0
     self.fname = fname # Filename
     self.line = 1      
     self.col = 1
     self.tree = tree   # ConfTree to fill
+    self.level = level # level of the parsed operations
     self.prefix = ''   # Prefix of variable name, may begin with '.'
     self.conds = []    # Stack of nested conditions, these are chained, so only the last is necessary
+  def preread(self, l):
+    "Make sure buf contains at least `l` next characters, return True on succes and False on hitting EOF."
+    if isinstance(self.s, file):
+      self.buf = self.buf[self.bufpos:] + self.s.read(max(l, 1024)).decode('utf8')
+      self.bufpos = 0
+    return len(self.buf) >= self.bufpos + l
   def peek(self, l = 1):
-    "Peek and return next `l` unicode characters."
-    # TODO
-    return ''
+    "Peek and return next `l` unicode characters or everything until EOF."
+    self.preread(l)
+    return self.buf[:l]
   def peeks(self, s):
-    "Peek and compare next `len(s)` characters to `s`. Unicode."
+    "Peek and compare next `len(s)` characters to `s`. Converts `s` to unicode. False on hitting EOF."
     s = unicode(s)
     return self.peek(len(s)) == s
     return True
   def next(self, l = 1):
-    "Eat and return next `l` unicode characters."
-    # TODO
-    return ''
+    "Eat and return next `l` unicode characters. Raise exception on EOF."
+    if not self.preread(l):
+      raise ConfigSyntaxError("Unexpected end of file")
+    s = self.buf[self.bufpos:self.bufpos+l]
+    bufpos += l
+    rnl = s.rfind('\n')
+    if rnl<0:
+      # no newline
+      self.column += l
+    else:
+      # some newlines
+      self.line += s.count('\n')
+      self.column = l - rnl - 1 
+    return s
   def nexts(self, s):
-    "Compare next `len(s)` characters to `s`, eat them and return True if they match. Unicode."
+    """Compare next `len(s)` characters to `s`. On match, eat them and return True. Otherwise just return False. 
+    Converts `s` to unicode. False on hitting EOF."""
     s = unicode(s)
-    return self.next(len(s)) == s
+    if self.peeks(s):
+      self.next(len(s))
+      return True
+    return False
   def eof(self):
     "Check for end-of-stream."
-    # TODO
-    return False
+    return self.preread(1)
   def expected(self, s, msg=None):
     "Eat and compare next `len(s)` characters to `s`. If not equal, raise an error with `msg`. Unicode."
     s = unicode(s)
@@ -133,7 +166,6 @@ class ConfigParser(object):
     self.expect(self.c_comment, "'#' expected at the beginning of a comment.")
     while not self.eof() and not self.nexts(self.c_nl):
       pass
-    self.eof() or self.expect(self.c_nl)
   def p_STATEMENT(self):
     self.p_WS()
     if self.peeks(self.c_if):
@@ -180,7 +212,9 @@ class ConfigParser(object):
       cnd = self.conditions[-1]
     else:
       cnd = None
-    v.add_operation(op, cnd, exp, self.priority) 
+    v.add_operation(conf.Operation(op, cnd, exp, level=self.level, 
+      source="%s:%d:%d"%(self.fname, self.line, self.column))) 
+    # NOTE/WARNING: The last character of operation is reported.  
   def p_CONDITION(self):
     self.p_WS()
     self.expect(self.c_if)