X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;ds=sidebyside;f=t%2Fmoe%2Feval.py;h=bc1ea83e46ebc18c4530b1b53002df76317d7ef9;hb=e9b511e32341f3712df0c285553378a140eaeb97;hp=f3e921a265ba3247fe528eb4078f5e820747bb96;hpb=dcc852db06f099b6c96b681c79832d41777da334;p=moe.git diff --git a/t/moe/eval.py b/t/moe/eval.py index f3e921a..bc1ea83 100644 --- a/t/moe/eval.py +++ b/t/moe/eval.py @@ -12,81 +12,144 @@ import os.path import shutil class Eval: + """ + """ def __init__(self): - self.log = moe.log.default - self.cfgs = moe.config.MoeConfigStack() - self.builtins = moe.config.MoeConfig(type="builtins") - self.cfgs.push(self.builtins) + self.log = moe.log.Logers() + self.config = moe.config.ConfigTree() self.main_pipe = moe.pipeline.MoePipeline("main") self.test_pipe = moe.pipeline.MoePipeline("test") - self.stat = moe.status.MoeStatus() - pass - - def init(self, overrides=None): - self.log.progress("Initializing... ") - self.init_global(overrides) - self.init_test() - self.init_logs() - self.init_task() - moe.box.init(self) - self.log.progress("OK\n") - - def init_global(self, overrides): - main_cfg = moe.config.MoeConfig(name = os.path.join(self.cfgs['HOME'], "config"), type="main") - self.cfgs.push(main_cfg) - if overrides: - self.cfgs.push(overrides) - - def init_test(self): - test = self.cfgs['TDIR'] - if os.path.isdir(test): - shutil.rmtree(test) - try: - moe.util.mkdir_tree(test) - except OSError, e: - raise moe.MoeErr, "Cannot create %s: %s" % (test, e.strerror) - - def init_logs(self): - self.log = moe.log.MoeLog() - if self.cfgs["V"]: - self.log.verbosity = int(self.cfgs["V"]) - self.log.open(os.path.join(self.cfgs["TDIR"], "log")) - self.default_log = self.log - moe.log.default = self.log - self.log_config(3, "before loading the task") - - def init_task(self): - task = self.cfgs['TASK'] - task_dir = self.cfgs['PDIR'] - if not os.path.isdir(task_dir): - raise moe.MoeErr, "No such task %s" % task - - task_cfg = moe.config.MoeConfig(name = os.path.join(task_dir, "config"), type='task') - self.cfgs.push(task_cfg) - self.log_config(3, "after loading the task") - - self.stat["task"] = task - - type = self.cfgs['TASK_TYPE'] - if type == "batch" or type == "interactive": - moe.batch.prepare_pipe(self) - elif type == "opendata": - raise moe.MoeErr, "Opendata tasks not implemented yet" - else: - raise moe.MoeErr, "Unknown task type " + type + self.status = moe.status.MoeStatus() + + def __getitem__(self, key): + return self.config[key] + + def init(self, overrides=[]): + "Initializes most part of Eval before running the pipeline. See the timeline for details." + self.log.info("Initializing ...") + + # set basic builtins + self.config.parse('HOME=\'%s\'' % os.getcwd(), source="", level=0) + self.config.parse('CONFIG="{HOME}/config"', source="", level=0) + self.config.parse('LOG="{HOME}/log"', source="", level=0) + self.config.parse('DEBUG_LEVEL="0"', source="", level=0) + self.config.parse('VERBOSE=""', source="", level=0) + + # Non-basic configuration, should be in default config file + self.config.parse(""" + TDIR = "{HOME}/test" + PDIR = "{HOME}/tasks/{TASK}" + USER_LOG = "{TDIR}/log" + TASK_CONFIG = "{PDIR}/config" + STATUS_FILE = "{TDIR}/status" + EXTENSIONS = "" + # only for testing: + TASK = "sum" + TASK_TYPE = "dummy" + USER = "gavento" + SOURCE = "suma.c" + EXTENSIONS += " dummy " + """, source="", level=0) # -> config + + # apply overrides + for ov in overrides: + self.config.parse(ov, source="", level=100) + + # load config file + self.config.fix('CONFIG') + self.config.parse_file(self['CONFIG'], level=30) + # fix variables + self.config.fix(['LOG', 'USER_LOG', 'VERBOSE', 'HOME', 'DEBUG_LEVEL', 'TDIR']) + # start logging + self.log.open_eval_log(self['LOG'], self['DEBUG_LEVEL'], redirect_fds = True) + self.log.open_user_log(self['USER_LOG']) + self.debug_dump_config() + # init and check TDIR + self.debug('Cleaning TDIR: %s'%self['TDIR']) + self.init_TDIR() + + # insert hooks into main pipeline + self.main_pipe.insert(5, "Eval.hook_init_dirs", self.hook_init_dirs) + self.main_pipe.insert(15, "Eval.hook_load_task_config", self.hook_load_task_config) + self.main_pipe.insert(20, "Eval.hook_init_tasktype", self.hook_init_tasktype) + self.main_pipe.insert(90, "Eval.hook_write_metadata", self.hook_write_metadata) + + # ininialize extensions (let them insert hooks) + self.conf.fix('EXTENSIONS') + exts = self['EXTENSIONS'].split() + for e in exts: + if not e: + raise MoeError, "Invalid extension name: %r" % e + self.log.debug("Loading extension %s", e) + try: + mod = util.load_module('moe.exts.' + e) + except ImportError: + self.log.exception() + raise MoeError, 'Unknown extension: %r' % e + mod.init(self) + def run(self): - self.log_config(2, "for the task pipeline") - self.main_pipe.configure(self.cfgs["HOOKS"]) - if self.log.verbosity >= 2: - self.main_pipe.dump(self.log.log_file, prefix="\t") + "Run the main pipeline." + self.debug_dump_pipe(self.main_pipe) + self.debug('Running main pipeline') self.main_pipe.run(self) - def log_config(self, verb, msg): - if self.log.verbosity >= verb: - self.log.say("@@@ Configuration stack %s @@@\n" % msg) - self.cfgs.dump_defs(self.log.log_file, prefix="\t") - if self.log.verbosity >= 99: - self.log.say("@@@@@@ Resolved configuration %s @@@@@@\n" % msg) - self.cfgs.dump(self.log.log_file, prefix="\t") + def debug_dump_config(self): + "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)." + if self.log.level <= 5: + self.log.ddebug('****** Config dump: ******') + self.log.ddebug(self.config.dump('**** ')) + self.log.ddebug('**************************') + + def debug_dump_pipe(self, pipe): + "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)." + if self.log.level <= 5: + self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name) + self.log.ddebug(pipe.dump(prefix='**** ')) + self.log.ddebug('**************************') + + def hook_init_dirs(self): + """(mainline at time 5) Create and check directories, fix directory variables. + .. note:: Currently only TDIR.""" + self.config.fix('TDIR') + tdir = self['TDIR'] + if os.path.isdir(tdir): + shutil.rmtree(tdir) + moe.util.mkdir_tree(tdir) + + def hook_load_task_config(self): + """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`.""" + self.config.fix(['TASK', 'PDIR', 'TASK_CONFIG']) + self.log.debug('Loading task config %s', self['TASK_CONFIG']) + if not os.path.isdir(self['PDIR']): + raise moe.MoeError, "No such task %s in %s" % (self['TASK'], self['PDIR']) + self.config.parse_file(self['TASK_CONFIG'], level=50) + self.debug_dump_config() + + self.stat["task"] = task # Metadata + + def hook_init_tasktype(self): + """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module.""" + + self.config.fix('TASK_TYPE') + task_type = self['TASK_TYPE'] + self.log.debug('Loading module for TASK_TYPE: %r', task_type) + if not task_type: + raise MoeError, "Invalid TASK_TYPE: %r" % e + try: + self.tasktype_module = utils.load_module('moe.tasktypes.' + task_type) + except ImportError: + self.log.exception() + raise MoeError, 'Unknown TASK_TYPE: %r' % task_type + mod.tasktype_module.init(self) + + def hook_write_metadata(self): + """(mainline at time 90) Write status metadata into file `STATUS_FILE`.""" + self.log.debug('Writing status file %s', self['STATUS_FILE']) + self.status.write(self['STATUS_FILE']) + # TODO: dump to ddebug + + +