X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;f=t%2Fmoe%2Feval.py;h=5eaa11004b01c3a9402433e4b9d9584de43bbb8c;hb=1ed60150aa8d007acd895d0ab7274ba9eb8b0f2a;hp=e8dbdce7ac16ac93f599e644fcec24b527e444fe;hpb=e1637399ace2b325608eb626241a29aedb8e7fe1;p=eval.git diff --git a/t/moe/eval.py b/t/moe/eval.py index e8dbdce..5eaa110 100644 --- a/t/moe/eval.py +++ b/t/moe/eval.py @@ -18,9 +18,9 @@ class Eval: def __init__(self): self.log = moe.log.Loggers() self.config = moe.config.ConfigTree() - self.main_pipe = moe.pipeline.MoePipeline("main") - self.test_pipe = moe.pipeline.MoePipeline("test") - self.status = moe.status.MoeStatus() + self.main_pipe = moe.pipeline.Pipeline(self, "main") + self.test_pipe = moe.pipeline.Pipeline(self, "test") + self.status = moe.status.Status() def __getitem__(self, key): return self.config[key] @@ -47,29 +47,25 @@ class Eval: # fix variables self.config.fix(['LOG', 'USER_LOG', 'VERBOSE', 'HOME', 'DEBUG_LEVEL', 'TDIR']) # start logging - self.log.open_eval_log(self['LOG'], self['DEBUG_LEVEL'], redirect_fds = True) + self.log.open_eval_log(self['LOG'], int(self['DEBUG_LEVEL']), redirect_fds = True) self.log.open_user_log(self['USER_LOG']) self.debug_dump_config() - # init and check TDIR - self.debug('Cleaning TDIR: %s'%self['TDIR']) - self.init_TDIR() - # insert hooks into main pipeline - self.main_pipe.insert(5, "Eval.hook_init_dirs", self.hook_init_dirs) - self.main_pipe.insert(15, "Eval.hook_load_task_config", self.hook_load_task_config) - self.main_pipe.insert(20, "Eval.hook_init_tasktype", self.hook_init_tasktype) - self.main_pipe.insert(90, "Eval.hook_write_metadata", self.hook_write_metadata) + self.main_pipe.insert(5, "Eval.hook_init_dirs", hook_init_dirs) + self.main_pipe.insert(15, "Eval.hook_load_task_config", hook_load_task_config) + self.main_pipe.insert(20, "Eval.hook_init_tasktype", hook_init_tasktype) + self.main_pipe.insert(90, "Eval.hook_write_metadata", hook_write_metadata) # ininialize extensions (let them insert hooks) - self.conf.fix('EXTENSIONS') + self.config.fix('EXTENSIONS') exts = self['EXTENSIONS'].split() for e in exts: if not e: raise MoeError, "Invalid extension name: %r" % e self.log.debug("Loading extension %s", e) try: - mod = util.load_module('moe.exts.' + e) + mod = moe.util.load_module('moe.exts.' + e) except ImportError: self.log.exception() raise MoeError, 'Unknown extension: %r' % e @@ -78,63 +74,72 @@ class Eval: def run(self): "Run the main pipeline." self.debug_dump_pipe(self.main_pipe) - self.debug('Running main pipeline') - self.main_pipe.run(self) + self.log.debug('Running main pipeline') + self.main_pipe.run(e=self) def debug_dump_config(self): "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)." if self.log.level <= 5: - self.log.ddebug('****** Config dump: ******') - self.log.ddebug(self.config.dump('**** ')) - self.log.ddebug('**************************') + self.log.ddebug(' ****** Config dump: ******') + self.log.ddebug('\n'.join(self.config.dump(' * '))) + self.log.ddebug(' **************************') def debug_dump_pipe(self, pipe): "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)." if self.log.level <= 5: - self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name) - self.log.ddebug(pipe.dump(prefix='**** ')) - self.log.ddebug('**************************') - - def hook_init_dirs(self): - """(mainline at time 5) Create and check directories, fix directory variables. - .. note:: Currently only TDIR.""" - self.config.fix('TDIR') - tdir = self['TDIR'] - if os.path.isdir(tdir): - shutil.rmtree(tdir) - moe.util.mkdir_tree(tdir) + self.log.ddebug(' ****** Pipeline %r dump: ******'%pipe.name) + self.log.ddebug('\n'.join(pipe.dump(prefix=' * '))) + self.log.ddebug(' **************************') - def hook_load_task_config(self): - """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`.""" - self.config.fix(['TASK', 'PDIR', 'TASK_CONFIG']) - self.log.debug('Loading task config %s', self['TASK_CONFIG']) - if not os.path.isdir(self['PDIR']): - raise moe.MoeError, "No such task %s in %s" % (self['TASK'], self['PDIR']) - self.config.parse_file(self['TASK_CONFIG'], level=50) - self.debug_dump_config() - - self.stat["task"] = task # Metadata - - def hook_init_tasktype(self): - """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module.""" - - self.config.fix('TASK_TYPE') - task_type = self['TASK_TYPE'] - self.log.debug('Loading module for TASK_TYPE: %r', task_type) - if not task_type: - raise MoeError, "Invalid TASK_TYPE: %r" % e - try: - self.tasktype_module = utils.load_module('moe.tasktypes.' + task_type) - except ImportError: - self.log.exception() - raise MoeError, 'Unknown TASK_TYPE: %r' % task_type - mod.tasktype_module.init(self) - - def hook_write_metadata(self): - """(mainline at time 90) Write status metadata into file `STATUS_FILE`.""" - self.log.debug('Writing status file %s', self['STATUS_FILE']) - self.status.write(self['STATUS_FILE']) - # TODO: dump to ddebug + def debug_dump_status(self): + "Dumps status metadata at level DDEBUG (only compiles the dump if main level low enough)." + if self.log.level <= 5: + self.log.ddebug(' ****** Status dump: ******') + self.log.ddebug('\n'.join(self.status.dump(prefix=' * ')).rstrip()) + self.log.ddebug(' **************************') + +def hook_init_dirs(e): + """(mainline at time 5) Create and check directories, fix directory variables. + .. note:: Currently only TDIR.""" + e.config.fix('TDIR') + tdir = e['TDIR'] + if os.path.isdir(tdir): + shutil.rmtree(tdir) + moe.util.mkdir_tree(tdir) + +def hook_load_task_config(e): + """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`.""" + e.config.fix(['TASK', 'PDIR', 'TASK_CONFIG']) + e.log.debug('Loading task config %s', e['TASK_CONFIG']) + if not os.path.isdir(e['PDIR']): + raise moe.MoeError, "No such task %s in %s" % (e['TASK'], e['PDIR']) + e.config.parse_file(e['TASK_CONFIG'], level=50) + e.debug_dump_config() + + e.status["task"] = e['TASK'] # Metadata + +def hook_init_tasktype(e): + """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module.""" + + e.config.fix('TASK_TYPE') + task_type = e['TASK_TYPE'] + e.log.debug('Loading module for TASK_TYPE: %r', task_type) + if not task_type: + raise MoeError, "Invalid TASK_TYPE: %r" % e + try: + e.tasktype_module = moe.util.load_module('moe.tasktypes.' + task_type) + except ImportError: + e.log.exception() + raise MoeError, 'Unknown TASK_TYPE: %r' % task_type + e.tasktype_module.init(e) + +def hook_write_metadata(e): + """(mainline at time 90) Write status metadata into file `STATUS_FILE`.""" + e.debug_dump_status() + e.log.debug('Writing status file %s', e['STATUS_FILE']) + with open(e['STATUS_FILE'], 'w') as f: + e.status.write(f) + # TODO: dump to ddebug