New file for conf parser, move and update conf file syntax

diff --git a/t/moe/conf.py b/t/moe/conf.py
index e5c3b385bc4c43c86d3a926d8c6ff832e3df0544..272ec7bf5bf93082a27dd726cb9e0fdfd16d1428 100644 (file)
--- a/t/moe/conf.py
+++ b/t/moe/conf.py
@@ -1,75 +1,37 @@
-import types, itertools, re
-import logging as log
-
 """
-Lazy conditional string evaluation module for configuration variables.
+conf.py
+-------
 
+Lazy conditional string evaluation module for Moe configuration variables.
 
-* Each variable has ordered list of operations (definitions), each SETs or APPENDs an expression 
-to the value. Each operation may be guarded by condition. 
 
-NOTE: Variable is undefined even if some 'APPEND' apply but no 'SET' applies. This might change.
+* Each variable has ordered list of operations (definitions), each defining operation either 
+assigns (SET) or appends (APPEND) value of an expression to the variable. Each operation may be guarded by condition(s). 
 
-* Each condition is a formula (tree consisting of 'AND', 'OR', 'NOT' and '==', '!=' between
-two expressions.
+NOTE: If no 'SET' applies, a variable is still undefined even if some 'APPEND' applies. This might change.
+
+* Each condition is a formula (tree consisting of 'AND', 'OR', 'NOT' and '==', '!=' between two expressions.
 
 * Expression is a list of strings and variables to be expanded.
 
 NOTE: All expanded data should be (or is converted to) unicode 
 
-TODO: Fixing variables.
+
+TODO: Fixing value of variables.
 TODO: Cleanup of unused undefined variables.
 TODO: Better variable name checking (no name '.'-structural prefix of another)
 TODO: Implemet "subtree" listing.
 TODO: Test conditions and unicode
 """
 
-"""
-The configuration syntax is the following (TODO: add whitespaces WSP)
-TODO: decide '()' around formulas 
-TODO: check escaping in expressions
-TODO: should whitespace (incl. '\\n') be allowed (almost) everywhere?
-      can comment be anywhere whitespace can?
-
-FILE = BLOCK 
-BLOCK + '\\n' = () | STATEMENT ( STATEMENT-SEP STATEMENT )* 
-
-STATEMENT-SEP = ( '\\n' | ';' )
-WSP = ( ' ' | '\\t' | '\\n' | COMMENT )*
-
-COMMENT = re'#[^\\n]*\\n'
-
-STATEMENT = CONDITION | OPERATION | SUBTREE 
-
-OPERATION = VARNAME ( '=' | '+=' ) EXPRESSION
-SUBTREE = VARNAME '{' BLOCK '}'
-CONDITION = 'if' FORMULA '{' BLOCK '}'
-
-FORMULA = ( EXPRESSION ( '!=' | '==' ) EXPRESSION ) | '(' AND ')' | '(' OR ')' | NOT
-AND = FORMULA 'and' FORMULA
-OR = FORMULA 'or' FORMULA
-NOT = 'not' FORMULA 
-
-EXPRESSION = '"' ( ECHAR | '{' VARNAME '}' )* '"' | re"'[^'\\n]*'"
-ECHAR = re'([^\\{}]|\\\\|\\{|\\}|\\n)*'
-"""
-
-
-
-
-c_tree_sep = u'.'
-c_comment = u'#'
-c_open = u'{'
-c_close = u'}'
-c_if = u'if'
-
-"Variable name regexp, dots (separators) must be separated from edges and each other."
-re_key = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z')
+import types, itertools, re
+import logging as log
+from confparser import VARNAME_re
 
-"Allowed depth of recursion -- includes ALL recursive calls, so should quite high."
+"Allowed depth of recursion - includes ALL recursive calls, so should quite high."
 c_maxdepth = 256
 
-"Maximum attained depth of recursion"
+"Maximum attained depth of recursion - for debug/testing"
 debug_maxdepth = 0 
 
 def check_depth(depth):
@@ -101,7 +63,7 @@ class ConfigTree(object):
     if not key in self.variables:
       if not create:
        raise ConfigError('Config variable %r undefined.', key)
-      if not re_key.match(key):
+      if not VARNAME_re.match(key):
        raise ConfigError('Invalid variable identifier %r in config', key)
       self.variables[key] = ConfigVar(key)
     return self.variables[key]

diff --git a/t/moe/confparser.py b/t/moe/confparser.py
new file mode 100644 (file)
index 0000000..352ab7b
--- /dev/null
+++ b/t/moe/confparser.py
@@ -0,0 +1,51 @@
+"""
+confparse.py
+------------
+
+Simple Moe configuration file syntax parser. 
+
+TODO: decide '()' around formulas
+TODO: check escaping in expressions
+TODO: should whitespace (incl. '\\n') be allowed (almost) everywhere?
+      can comment be anywhere whitespace can?
+
+Generally, whitespace and comments are alowed everywhere except in variable names and inside expressions. 
+Also, COMMENT must not contain '\\n'. 
+FILE, BLOCK, STATEMENT, OPERATION, SUBTREE, CONDITION, FORMULA, AND, OR and NOT eat any preceding whitespace.
+
+The configuration syntax is the following:
+
+FILE = BLOCK 
+BLOCK = WS | STATEMENT ( SEP STATEMENT )* 
+
+SEP = ( '\\n' | ';' )
+WS = ( ' ' | '\\t' | '\\n' | COMMENT )*
+
+COMMENT = re('#[^\\n]*\\n')
+
+STATEMENT = CONDITION | OPERATION | SUBTREE
+
+OPERATION = WS VARNAME WS ( '=' | '+=' ) WS EXPRESSION
+SUBTREE = WS VARNAME WS '{' BLOCK '}'
+CONDITION = WS 'if' FORMULA WS '{' BLOCK WS '}'
+
+FORMULA = WS (( EXPRESSION WS ( '!=' | '==' ) WS EXPRESSION ) | '(' AND WS ')' | '(' OR WS ')' | NOT )
+AND = FORMULA WS 'and' FORMULA
+OR = FORMULA WS 'or' FORMULA
+NOT = WS 'not' FORMULA 
+
+EXPRESSION = '"' ( ECHAR | '{' VARNAME '}' )* '"' | re"'[^'\\n]*'"
+ECHAR = re('([^\\{}]|\\\\|\\{|\\}|\\n)*')
+"""
+
+import re, logging as log
+
+c_tree_sep = u'.'
+c_comment = u'#'
+c_open = u'{'
+c_close = u'}'
+c_if = u'if'
+
+"Variable name regexp, dots (separators) must be separated from edges and each other."
+VARNAME_re = re.compile(r'\A([A-Za-z0-9_-]+\.)*[A-Za-z0-9_-]+\Z')
+