Rewriting eval.py with new log, config and timeline

Moving initializaton together, remaining init_* will be called from main pipeline
Most changes are moves and rewrites - the meaning did not change much
The actions are as listed in doc/timeline

diff --git a/t/moe/eval.py b/t/moe/eval.py
index 2f30c6d2e1148a67b33bfc69c84badd2af79293a..3ee55666c5d671406be50add5f08f7bc4f94d958 100644 (file)
--- a/t/moe/eval.py
+++ b/t/moe/eval.py
@@ -12,53 +12,86 @@ import os.path
 import shutil
 
 class Eval:
+    """
+    """
 
     def __init__(self):
-       self.log = moe.log.default
-       self.cfgs = moe.config.MoeConfigStack()
-       self.builtins = moe.config.MoeConfig(type="builtins")
-       self.cfgs.push(self.builtins)
+       self.log = moe.log.Logers()
+       self.config = moe.config.ConfigTree()
        self.main_pipe = moe.pipeline.MoePipeline("main")
        self.test_pipe = moe.pipeline.MoePipeline("test")
        self.stat = moe.status.MoeStatus()
-       pass
 
     def __getitem__(self, key):
-       return self.cfgs[key]
-
-    def init(self, overrides=None):
-       self.log.progress("Initializing... ")
-       self.init_global(overrides)
-       self.init_test()
-       self.init_logs()
-       self.init_task()
-       moe.box.init(self)
-       self.log.progress("OK\n")
-
-    def init_global(self, overrides):
-       main_cfg = moe.config.MoeConfig(name = os.path.join(self['HOME'], "config"), type="main")
-       self.cfgs.push(main_cfg)
-       if overrides:
-           self.cfgs.push(overrides)
-
-    def init_test(self):
+       return self.config[key]
+
+    def debug_dumpconf(self):
+       if self.log.level <= 5:
+         self.log.ddebug('****** Config dump: ******'))
+         self.log.ddebug(self.config.dump('**** '))
+         self.log.ddebug('**************************'))
+    def debug_dumppipe(self, pipe):
+       if self.log.level <= 5:
+         self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name))
+         self.log.ddebug(pipe.dump(prefix='**** '))
+         self.log.ddebug('**************************'))
+       
+    def init(self, overrides=[]):
+       self.log.info("Initializing ...")
+       
+       # set basic builtins
+       self.config.parse('HOME=\'%s\'' % os.getcwd(), source="<builtins>", level=0)
+       self.config.parse('CONFIG="{HOME}/config"', source="<builtins>", level=0)
+       self.config.parse('LOG="{HOME}/log"', source="<builtins>", level=0)
+       self.config.parse('DEBUG_LEVEL="0"', source="<builtins>", level=0)
+       self.config.parse('VERBOSE=""', source="<builtins>", level=0)
+       self.config.parse('TDIR="{HOME}/test"', source="<temp-builtins>", level=0) # -> config
+       self.config.parse('USER_LOG="{TDIR}/log"', source="<temp-builtins>", level=0) # -> config
+       # apply overrides
+       for ov in overrides:
+         self.config.parse(ov, source="<overrides>", level=100)
+       
+       # load config file
+       self.config.fix('CONFIG')
+       with open(self['CONFIG'], 'r') as f:
+         self.config.parse(f, source=self['CONFIG'], level=30)
+       # fix variables
+       self.config.fix(['LOG', 'VERBOSE', 'HOME', 'DEBUG_LEVEL', 'TDIR'])
+       # start logging
+       self.log.open_eval_log(self['LOG'], self['DEBUG_LEVEL'], redirect_fds = True)
+       self.log.open_user_log(self['USER_LOG'])
+       self.debug_dump_config()
+
+       # init and check TDIR
+       self.debug('Cleaning TDIR: %s'%self['TDIR'])
+       self.init_TDIR()
+       
+       # insert hooks into main pipeline
+       # TODO
+
+       # insert custom hooks
+       self.conf.fix('HOOKS')
+       self.main_pipe.configure(self['HOOKS'])
+
+       # go! 
+       self.debug_dump_pipe(self.main_pipe)
+       self.debug('Running main pipeline')
+       self.main_pipe.run(self)
+
+       #self.init_task()
+       #moe.box.init(self)
+
+    def init_TDIR(self):
        test = self['TDIR']
        if os.path.isdir(test):
            shutil.rmtree(test)
-       try:
+       try: #TODO: Remove
            moe.util.mkdir_tree(test)   
        except OSError, err:
            raise moe.MoeError, "Cannot create %s: %s" % (test, err.strerror)
 
-    def init_logs(self):
-       self.log = moe.log.MoeLog()
-       if self["V"]:
-           self.log.verbosity = int(self["V"])
-       self.log.open(os.path.join(self["TDIR"], "log"))
-       self.default_log = self.log
-       moe.log.default = self.log
-       self.log_config(3, "before loading the task")
 
+#TODO ...
     def init_task(self):
        task = self['TASK']
        task_dir = self['PDIR']
@@ -79,17 +112,3 @@ class Eval:
        else:
            raise moe.MoeError, "Unknown task type " + type
 
-    def run(self):
-       self.log_config(2, "for the task pipeline")
-       self.main_pipe.configure(self["HOOKS"])
-       if self.log.verbosity >= 2:
-           self.main_pipe.dump(self.log.log_file, prefix="\t")
-       self.main_pipe.run(self)
-
-    def log_config(self, verb, msg):
-       if self.log.verbosity >= verb:
-           self.log.say("@@@ Configuration stack %s @@@\n" % msg)
-           self.cfgs.dump_defs(self.log.log_file, prefix="\t")
-       if self.log.verbosity >= 99:
-           self.log.say("@@@@@@ Resolved configuration %s @@@@@@\n" % msg)
-           self.cfgs.dump(self.log.log_file, prefix="\t")