import moe
import moe.config
+import moe.box
import moe.log
-import moe.meta
+import moe.status
import moe.pipeline
import moe.batch
import moe.util
import shutil
class Eval:
+ """
+ """
def __init__(self):
- self.log = moe.log.default
- self.cfgs = moe.config.MoeConfigStack()
- self.builtins = moe.config.MoeConfig(type="builtins")
- self.cfgs.push(self.builtins)
+ self.log = moe.log.Logers()
+ self.config = moe.config.ConfigTree()
self.main_pipe = moe.pipeline.MoePipeline("main")
- self.meta = moe.meta.MoeMeta()
- pass
-
- def init(self, overrides=None):
- self.log.progress("Initializing... ")
- self.init_global(overrides)
- self.init_test()
- self.init_logs()
- self.init_task()
- self.log.progress("OK\n")
-
- def init_global(self, overrides):
- main_cfg = moe.config.MoeConfig(name = os.path.join(self.cfgs['HOME'], "config"), type="main")
- self.cfgs.push(main_cfg)
- if overrides:
- self.cfgs.push(overrides)
-
- def init_test(self):
- test = self.cfgs['TEST_DIR']
+ self.test_pipe = moe.pipeline.MoePipeline("test")
+ self.stat = moe.status.MoeStatus()
+
+ def __getitem__(self, key):
+ return self.config[key]
+
+ def init(self, overrides=[]):
+ "Initializes most part of Eval before running the pipeline. See the timeline for details."
+ self.log.info("Initializing ...")
+
+ # set basic builtins
+ self.config.parse('HOME=\'%s\'' % os.getcwd(), source="<builtins>", level=0)
+ self.config.parse('CONFIG="{HOME}/config"', source="<builtins>", level=0)
+ self.config.parse('LOG="{HOME}/log"', source="<builtins>", level=0)
+ self.config.parse('DEBUG_LEVEL="0"', source="<builtins>", level=0)
+ self.config.parse('VERBOSE=""', source="<builtins>", level=0)
+ self.config.parse('TDIR="{HOME}/test"', source="<temp-builtins>", level=0) # -> config
+ self.config.parse('USER_LOG="{TDIR}/log"', source="<temp-builtins>", level=0) # -> config
+ # apply overrides
+ for ov in overrides:
+ self.config.parse(ov, source="<overrides>", level=100)
+
+ # load config file
+ self.config.fix('CONFIG')
+ with open(self['CONFIG'], 'r') as f:
+ self.config.parse(f, source=self['CONFIG'], level=30)
+ # fix variables
+ self.config.fix(['LOG', 'USER_LOG', 'VERBOSE', 'HOME', 'DEBUG_LEVEL', 'TDIR'])
+ # start logging
+ self.log.open_eval_log(self['LOG'], self['DEBUG_LEVEL'], redirect_fds = True)
+ self.log.open_user_log(self['USER_LOG'])
+ self.debug_dump_config()
+
+ # init and check TDIR
+ self.debug('Cleaning TDIR: %s'%self['TDIR'])
+ self.init_TDIR()
+
+ # insert hooks into main pipeline
+ # TODO
+ # TODO moe.box.init(self)
+
+ # insert custom hooks
+ self.conf.fix('HOOKS')
+ self.main_pipe.configure(self['HOOKS'])
+
+ def run(self):
+ "Run the main pipeline."
+ self.debug_dump_pipe(self.main_pipe)
+ self.debug('Running main pipeline')
+ self.main_pipe.run(self)
+
+
+ def init_TDIR(self):
+ test = self['TDIR']
if os.path.isdir(test):
shutil.rmtree(test)
- try:
+ try: #TODO: Remove
moe.util.mkdir_tree(test)
- except OSError, e:
- raise moe.MoeErr, "Cannot create %s: %s" % (test, e.strerror)
-
- def init_logs(self):
- self.log = moe.log.MoeLog()
- if self.cfgs["V"]:
- self.log.verbosity = int(self.cfgs["V"])
- self.log.log_file = open(os.path.join(self.cfgs["TEST_DIR"], "log"), "w")
- self.default_log = self.log
- moe.log.default = self.log
- self.log_config(3, "before loading the task")
+ except OSError, err:
+ raise moe.MoeError, "Cannot create %s: %s" % (test, err.strerror)
+
+ def debug_dump_config(self):
+ "Dumps at level DDEBUG and only compiles the dump if main level low enough."
+ if self.log.level <= 5:
+ self.log.ddebug('****** Config dump: ******'))
+ self.log.ddebug(self.config.dump('**** '))
+ self.log.ddebug('**************************'))
+
+ def debug_dump_pipe(self, pipe):
+ "Dumps at level DDEBUG and only compiles the dump if main level low enough."
+ if self.log.level <= 5:
+ self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name))
+ self.log.ddebug(pipe.dump(prefix='**** '))
+ self.log.ddebug('**************************'))
+#TODO ...
def init_task(self):
- task = self.cfgs['TASK']
- task_dir = self.cfgs['TASK_DIR']
+ task = self['TASK']
+ task_dir = self['PDIR']
if not os.path.isdir(task_dir):
- raise moe.MoeErr, "No such task %s" % task
+ raise moe.MoeError, "No such task %s" % task
task_cfg = moe.config.MoeConfig(name = os.path.join(task_dir, "config"), type='task')
self.cfgs.push(task_cfg)
self.log_config(3, "after loading the task")
- self.meta["task"] = task
+ self.stat["task"] = task
- type = self.cfgs['TASK_TYPE']
+ type = self['TASK_TYPE']
if type == "batch" or type == "interactive":
moe.batch.prepare_pipe(self)
elif type == "opendata":
- raise moe.MoeErr, "Opendata tasks not implemented yet"
+ raise moe.MoeError, "Opendata tasks not implemented yet"
else:
- raise moe.MoeErr, "Unknown task type " + type
-
- def run(self):
- self.log_config(2, "for the task pipeline")
- self.main_pipe.configure(self.cfgs["HOOKS"])
- if self.log.verbosity >= 2:
- self.main_pipe.dump(self.log.log_file, prefix="\t")
- self.main_pipe.run(self)
+ raise moe.MoeError, "Unknown task type " + type
- def log_config(self, verb, msg):
- if self.log.verbosity >= verb:
- self.log.say("@@@ Configuration stack %s @@@\n" % msg)
- self.cfgs.dump_defs(self.log.log_file, prefix="\t")
- if self.log.verbosity >= 99:
- self.log.say("@@@@@@ Resolved configuration %s @@@@@@\n" % msg)
- self.cfgs.dump(self.log.log_file, prefix="\t")