Renamed MoeStatus -> Status

[eval.git] / t / moe / eval.py

diff --git a/t/moe/eval.py b/t/moe/eval.py
index 6c6a608ca2ea11474f15d0111faadc0a73fddae2..5eaa11004b01c3a9402433e4b9d9584de43bbb8c 100644 (file)
--- a/t/moe/eval.py
+++ b/t/moe/eval.py
@@ -12,81 +12,134 @@ import os.path
 import shutil
 
 class Eval:
 import shutil
 
 class Eval:

+    """
+    """

 
     def __init__(self):
 
     def __init__(self):

-       self.log = moe.log.default
-       self.cfgs = moe.config.MoeConfigStack()
-       self.builtins = moe.config.MoeConfig(type="builtins")
-       self.cfgs.push(self.builtins)
-       self.main_pipe = moe.pipeline.MoePipeline("main")
-       self.test_pipe = moe.pipeline.MoePipeline("test")
-       self.stat = moe.status.MoeStatus()
-       pass
-
-    def init(self, overrides=None):
-       self.log.progress("Initializing... ")
-       self.init_global(overrides)
-       self.init_test()
-       self.init_logs()
-       self.init_task()
-       moe.box.init(self)
-       self.log.progress("OK\n")
-
-    def init_global(self, overrides):
-       main_cfg = moe.config.MoeConfig(name = os.path.join(self.cfgs['HOME'], "config"), type="main")
-       self.cfgs.push(main_cfg)
-       if overrides:
-           self.cfgs.push(overrides)
-
-    def init_test(self):
-       test = self.cfgs['TDIR']
-       if os.path.isdir(test):
-           shutil.rmtree(test)
-       try:
-           moe.util.mkdir_tree(test)   
-       except OSError, e:
-           raise moe.MoeError, "Cannot create %s: %s" % (test, e.strerror)
-
-    def init_logs(self):
-       self.log = moe.log.MoeLog()
-       if self.cfgs["V"]:
-           self.log.verbosity = int(self.cfgs["V"])
-       self.log.open(os.path.join(self.cfgs["TDIR"], "log"))
-       self.default_log = self.log
-       moe.log.default = self.log
-       self.log_config(3, "before loading the task")
-
-    def init_task(self):
-       task = self.cfgs['TASK']
-       task_dir = self.cfgs['PDIR']
-       if not os.path.isdir(task_dir):
-           raise moe.MoeError, "No such task %s" % task
-
-       task_cfg = moe.config.MoeConfig(name = os.path.join(task_dir, "config"), type='task')
-       self.cfgs.push(task_cfg)
-       self.log_config(3, "after loading the task")
-
-       self.stat["task"] = task
-
-       type = self.cfgs['TASK_TYPE']
-       if type == "batch" or type == "interactive":
-           moe.batch.prepare_pipe(self)
-       elif type == "opendata":
-           raise moe.MoeError, "Opendata tasks not implemented yet"
-       else:
-           raise moe.MoeError, "Unknown task type " + type
+       self.log = moe.log.Loggers()
+       self.config = moe.config.ConfigTree()
+       self.main_pipe = moe.pipeline.Pipeline(self, "main")
+       self.test_pipe = moe.pipeline.Pipeline(self, "test")
+       self.status = moe.status.Status()

 
 

+    def __getitem__(self, key):
+       return self.config[key]
+
+    def init(self, overrides=[]):
+       "Initializes most part of Eval before running the pipeline. See the timeline for details."
+       self.log.info("Initializing ...")
+       
+       # set basic builtins
+       self.config.parse('HOME = \'%s\'' % os.getcwd(), source="<builtins>", level=0)
+       self.config.parse('CONFIG = "{HOME}/config"', source="<builtins>", level=0)
+       self.config.parse('LOG = "{HOME}/log"', source="<builtins>", level=0)
+       self.config.parse('DEBUG_LEVEL = "0"', source="<builtins>", level=0)
+       self.config.parse('VERBOSE = ""', source="<builtins>", level=0)
+       self.config.parse('EXTENSIONS = ""', source="<builtins>", level=0)
+       
+       # apply overrides
+       for ov in overrides:
+           self.config.parse(ov, source="<overrides>", level=100)
+       
+       # load config file
+       self.config.fix('CONFIG')
+       self.config.parse_file(self['CONFIG'], level=30)
+       # fix variables
+       self.config.fix(['LOG', 'USER_LOG', 'VERBOSE', 'HOME', 'DEBUG_LEVEL', 'TDIR'])
+       # start logging
+       self.log.open_eval_log(self['LOG'], int(self['DEBUG_LEVEL']), redirect_fds = True)
+       self.log.open_user_log(self['USER_LOG'])
+       self.debug_dump_config()
+
+       # insert hooks into main pipeline
+       self.main_pipe.insert(5, "Eval.hook_init_dirs", hook_init_dirs)
+       self.main_pipe.insert(15, "Eval.hook_load_task_config", hook_load_task_config)
+       self.main_pipe.insert(20, "Eval.hook_init_tasktype", hook_init_tasktype)
+       self.main_pipe.insert(90, "Eval.hook_write_metadata", hook_write_metadata)
+
+       # ininialize extensions (let them insert hooks) 
+       self.config.fix('EXTENSIONS')
+       exts = self['EXTENSIONS'].split()
+       for e in exts:
+           if not e:
+               raise MoeError, "Invalid extension name: %r" % e
+           self.log.debug("Loading extension %s", e)
+           try:
+               mod = moe.util.load_module('moe.exts.' + e)
+           except ImportError:
+               self.log.exception()
+               raise MoeError, 'Unknown extension: %r' % e
+           mod.init(self)
+       

     def run(self):
     def run(self):

-       self.log_config(2, "for the task pipeline")
-       self.main_pipe.configure(self.cfgs["HOOKS"])
-       if self.log.verbosity >= 2:
-           self.main_pipe.dump(self.log.log_file, prefix="\t")
-       self.main_pipe.run(self)
-
-    def log_config(self, verb, msg):
-       if self.log.verbosity >= verb:
-           self.log.say("@@@ Configuration stack %s @@@\n" % msg)
-           self.cfgs.dump_defs(self.log.log_file, prefix="\t")
-       if self.log.verbosity >= 99:
-           self.log.say("@@@@@@ Resolved configuration %s @@@@@@\n" % msg)
-           self.cfgs.dump(self.log.log_file, prefix="\t")
+       "Run the main pipeline."
+       self.debug_dump_pipe(self.main_pipe)
+       self.log.debug('Running main pipeline')
+       self.main_pipe.run(e=self)
+
+    def debug_dump_config(self):
+       "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)."
+       if self.log.level <= 5:
+           self.log.ddebug(' ****** Config dump: ******')
+           self.log.ddebug('\n'.join(self.config.dump(' * ')))
+           self.log.ddebug(' **************************')
+
+    def debug_dump_pipe(self, pipe):
+       "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)."
+       if self.log.level <= 5:
+           self.log.ddebug(' ****** Pipeline %r dump: ******'%pipe.name)
+           self.log.ddebug('\n'.join(pipe.dump(prefix=' * ')))
+           self.log.ddebug(' **************************')
+    
+    def debug_dump_status(self):
+       "Dumps status metadata at level DDEBUG (only compiles the dump if main level low enough)."
+       if self.log.level <= 5:
+           self.log.ddebug(' ****** Status dump: ******')
+           self.log.ddebug('\n'.join(self.status.dump(prefix=' * ')).rstrip())
+           self.log.ddebug(' **************************')
+
+def hook_init_dirs(e):
+    """(mainline at time 5) Create and check directories, fix directory variables.
+    .. note:: Currently only TDIR."""
+    e.config.fix('TDIR')
+    tdir = e['TDIR']
+    if os.path.isdir(tdir):
+       shutil.rmtree(tdir)
+    moe.util.mkdir_tree(tdir)
+
+def hook_load_task_config(e):
+    """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`."""
+    e.config.fix(['TASK', 'PDIR', 'TASK_CONFIG'])
+    e.log.debug('Loading task config %s', e['TASK_CONFIG'])
+    if not os.path.isdir(e['PDIR']):
+       raise moe.MoeError, "No such task %s in %s" % (e['TASK'], e['PDIR'])
+    e.config.parse_file(e['TASK_CONFIG'], level=50)
+    e.debug_dump_config()
+
+    e.status["task"] = e['TASK']  # Metadata
+
+def hook_init_tasktype(e):
+    """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module."""
+
+    e.config.fix('TASK_TYPE')
+    task_type = e['TASK_TYPE']
+    e.log.debug('Loading module for TASK_TYPE: %r', task_type)
+    if not task_type:
+       raise MoeError, "Invalid TASK_TYPE: %r" % e
+    try:
+       e.tasktype_module = moe.util.load_module('moe.tasktypes.' + task_type)
+    except ImportError:
+       e.log.exception()
+       raise MoeError, 'Unknown TASK_TYPE: %r' % task_type
+    e.tasktype_module.init(e)
+
+def hook_write_metadata(e):
+    """(mainline at time 90) Write status metadata into file `STATUS_FILE`."""
+    e.debug_dump_status()
+    e.log.debug('Writing status file %s', e['STATUS_FILE'])
+    with open(e['STATUS_FILE'], 'w') as f:
+       e.status.write(f)
+    # TODO: dump to ddebug     
+
+
+