X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;ds=inline;f=t%2Fmoe%2Feval.py;h=5a93480a638461787be5b5a7a01ab0df67634f56;hb=ed3ca9e005f68d9f8d1c7e5aee80bab39acd0206;hp=bcf0c5330dcb5b8771c6c993e327df74daaf8b34;hpb=6f80b543eafc0e1c9644de1efb4db9ea97bab30d;p=moe.git diff --git a/t/moe/eval.py b/t/moe/eval.py index bcf0c53..5a93480 100644 --- a/t/moe/eval.py +++ b/t/moe/eval.py @@ -18,8 +18,8 @@ class Eval: def __init__(self): self.log = moe.log.Loggers() self.config = moe.config.ConfigTree() - self.main_pipe = moe.pipeline.MoePipeline("main") - self.test_pipe = moe.pipeline.MoePipeline("test") + self.main_pipe = moe.pipeline.Pipeline(self, "main") + self.test_pipe = moe.pipeline.Pipeline(self, "test") self.status = moe.status.MoeStatus() def __getitem__(self, key): @@ -52,20 +52,20 @@ class Eval: self.debug_dump_config() # insert hooks into main pipeline - self.main_pipe.insert(5, "Eval.hook_init_dirs", self.hook_init_dirs) - self.main_pipe.insert(15, "Eval.hook_load_task_config", self.hook_load_task_config) - self.main_pipe.insert(20, "Eval.hook_init_tasktype", self.hook_init_tasktype) - self.main_pipe.insert(90, "Eval.hook_write_metadata", self.hook_write_metadata) + self.main_pipe.insert(5, "Eval.hook_init_dirs", hook_init_dirs) + self.main_pipe.insert(15, "Eval.hook_load_task_config", hook_load_task_config) + self.main_pipe.insert(20, "Eval.hook_init_tasktype", hook_init_tasktype) + self.main_pipe.insert(90, "Eval.hook_write_metadata", hook_write_metadata) # ininialize extensions (let them insert hooks) - self.conf.fix('EXTENSIONS') + self.config.fix('EXTENSIONS') exts = self['EXTENSIONS'].split() for e in exts: if not e: raise MoeError, "Invalid extension name: %r" % e self.log.debug("Loading extension %s", e) try: - mod = util.load_module('moe.exts.' + e) + mod = moe.util.load_module('moe.exts.' + e) except ImportError: self.log.exception() raise MoeError, 'Unknown extension: %r' % e @@ -74,63 +74,72 @@ class Eval: def run(self): "Run the main pipeline." self.debug_dump_pipe(self.main_pipe) - self.debug('Running main pipeline') - self.main_pipe.run(self) + self.log.debug('Running main pipeline') + self.main_pipe.run(e=self) def debug_dump_config(self): "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)." if self.log.level <= 5: - self.log.ddebug('****** Config dump: ******') - self.log.ddebug(self.config.dump('**** ')) - self.log.ddebug('**************************') + self.log.ddebug(' ****** Config dump: ******') + self.log.ddebug('\n'.join(self.config.dump(' * '))) + self.log.ddebug(' **************************') def debug_dump_pipe(self, pipe): "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)." if self.log.level <= 5: - self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name) - self.log.ddebug(pipe.dump(prefix='**** ')) - self.log.ddebug('**************************') - - def hook_init_dirs(self): - """(mainline at time 5) Create and check directories, fix directory variables. - .. note:: Currently only TDIR.""" - self.config.fix('TDIR') - tdir = self['TDIR'] - if os.path.isdir(tdir): - shutil.rmtree(tdir) - moe.util.mkdir_tree(tdir) + self.log.ddebug(' ****** Pipeline %r dump: ******'%pipe.name) + self.log.ddebug('\n'.join(pipe.dump(prefix=' * '))) + self.log.ddebug(' **************************') - def hook_load_task_config(self): - """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`.""" - self.config.fix(['TASK', 'PDIR', 'TASK_CONFIG']) - self.log.debug('Loading task config %s', self['TASK_CONFIG']) - if not os.path.isdir(self['PDIR']): - raise moe.MoeError, "No such task %s in %s" % (self['TASK'], self['PDIR']) - self.config.parse_file(self['TASK_CONFIG'], level=50) - self.debug_dump_config() - - self.stat["task"] = task # Metadata - - def hook_init_tasktype(self): - """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module.""" - - self.config.fix('TASK_TYPE') - task_type = self['TASK_TYPE'] - self.log.debug('Loading module for TASK_TYPE: %r', task_type) - if not task_type: - raise MoeError, "Invalid TASK_TYPE: %r" % e - try: - self.tasktype_module = utils.load_module('moe.tasktypes.' + task_type) - except ImportError: - self.log.exception() - raise MoeError, 'Unknown TASK_TYPE: %r' % task_type - mod.tasktype_module.init(self) - - def hook_write_metadata(self): - """(mainline at time 90) Write status metadata into file `STATUS_FILE`.""" - self.log.debug('Writing status file %s', self['STATUS_FILE']) - self.status.write(self['STATUS_FILE']) - # TODO: dump to ddebug + def debug_dump_status(self): + "Dumps status metadata at level DDEBUG (only compiles the dump if main level low enough)." + if self.log.level <= 5: + self.log.ddebug(' ****** Status dump: ******') + self.log.ddebug('\n'.join(self.status.dump(prefix=' * ')).rstrip()) + self.log.ddebug(' **************************') + +def hook_init_dirs(e): + """(mainline at time 5) Create and check directories, fix directory variables. + .. note:: Currently only TDIR.""" + e.config.fix('TDIR') + tdir = e['TDIR'] + if os.path.isdir(tdir): + shutil.rmtree(tdir) + moe.util.mkdir_tree(tdir) + +def hook_load_task_config(e): + """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`.""" + e.config.fix(['TASK', 'PDIR', 'TASK_CONFIG']) + e.log.debug('Loading task config %s', e['TASK_CONFIG']) + if not os.path.isdir(e['PDIR']): + raise moe.MoeError, "No such task %s in %s" % (e['TASK'], e['PDIR']) + e.config.parse_file(e['TASK_CONFIG'], level=50) + e.debug_dump_config() + + e.status["task"] = e['TASK'] # Metadata + +def hook_init_tasktype(e): + """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module.""" + + e.config.fix('TASK_TYPE') + task_type = e['TASK_TYPE'] + e.log.debug('Loading module for TASK_TYPE: %r', task_type) + if not task_type: + raise MoeError, "Invalid TASK_TYPE: %r" % e + try: + e.tasktype_module = moe.util.load_module('moe.tasktypes.' + task_type) + except ImportError: + e.log.exception() + raise MoeError, 'Unknown TASK_TYPE: %r' % task_type + e.tasktype_module.init(e) + +def hook_write_metadata(e): + """(mainline at time 90) Write status metadata into file `STATUS_FILE`.""" + e.debug_dump_status() + e.log.debug('Writing status file %s', e['STATUS_FILE']) + with open(e['STATUS_FILE'], 'w') as f: + e.status.write(f) + # TODO: dump to ddebug