From 68ef17dd956952f2a559bf9250628aa366884521 Mon Sep 17 00:00:00 2001 From: Tomas Gavenciak Date: Sat, 18 Sep 2010 18:10:08 +0200 Subject: [PATCH] Pipeline functions now get keyword arguments No dependence on order, Eval argument called `e` Fixed dummy extension and task type --- t/moe/eval.py | 99 ++++++++++++++++++++-------------------- t/moe/exts/dummy.py | 9 ++-- t/moe/pipeline.py | 4 +- t/moe/tasktypes/dummy.py | 6 +-- 4 files changed, 61 insertions(+), 57 deletions(-) diff --git a/t/moe/eval.py b/t/moe/eval.py index 46dff84..f15f9ee 100644 --- a/t/moe/eval.py +++ b/t/moe/eval.py @@ -52,10 +52,10 @@ class Eval: self.debug_dump_config() # insert hooks into main pipeline - self.main_pipe.insert(5, "Eval.hook_init_dirs", self.hook_init_dirs) - self.main_pipe.insert(15, "Eval.hook_load_task_config", self.hook_load_task_config) - self.main_pipe.insert(20, "Eval.hook_init_tasktype", self.hook_init_tasktype) - self.main_pipe.insert(90, "Eval.hook_write_metadata", self.hook_write_metadata) + self.main_pipe.insert(5, "Eval.hook_init_dirs", hook_init_dirs) + self.main_pipe.insert(15, "Eval.hook_load_task_config", hook_load_task_config) + self.main_pipe.insert(20, "Eval.hook_init_tasktype", hook_init_tasktype) + self.main_pipe.insert(90, "Eval.hook_write_metadata", hook_write_metadata) # ininialize extensions (let them insert hooks) self.config.fix('EXTENSIONS') @@ -74,63 +74,64 @@ class Eval: def run(self): "Run the main pipeline." self.debug_dump_pipe(self.main_pipe) - self.debug('Running main pipeline') - self.main_pipe.run(self) + self.log.debug('Running main pipeline') + self.main_pipe.run(e=self) def debug_dump_config(self): "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)." if self.log.level <= 5: self.log.ddebug('****** Config dump: ******') - self.log.ddebug('\n'.join(self.config.dump('** '))) + self.log.ddebug('\n'.join(self.config.dump('* '))) self.log.ddebug('**************************') def debug_dump_pipe(self, pipe): "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)." if self.log.level <= 5: - self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name) - self.log.ddebug(pipe.dump(prefix='**** ')) + self.log.ddebug('****** Pipeline %r dump: ******'%pipe.name) + self.log.ddebug(pipe.pipe) + self.log.ddebug(pipe.dump(prefix='* ')) self.log.ddebug('**************************') - def hook_init_dirs(self): - """(mainline at time 5) Create and check directories, fix directory variables. - .. note:: Currently only TDIR.""" - self.config.fix('TDIR') - tdir = self['TDIR'] - if os.path.isdir(tdir): - shutil.rmtree(tdir) - moe.util.mkdir_tree(tdir) - - def hook_load_task_config(self): - """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`.""" - self.config.fix(['TASK', 'PDIR', 'TASK_CONFIG']) - self.log.debug('Loading task config %s', self['TASK_CONFIG']) - if not os.path.isdir(self['PDIR']): - raise moe.MoeError, "No such task %s in %s" % (self['TASK'], self['PDIR']) - self.config.parse_file(self['TASK_CONFIG'], level=50) - self.debug_dump_config() - - self.stat["task"] = task # Metadata - - def hook_init_tasktype(self): - """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module.""" - - self.config.fix('TASK_TYPE') - task_type = self['TASK_TYPE'] - self.log.debug('Loading module for TASK_TYPE: %r', task_type) - if not task_type: - raise MoeError, "Invalid TASK_TYPE: %r" % e - try: - self.tasktype_module = utils.load_module('moe.tasktypes.' + task_type) - except ImportError: - self.log.exception() - raise MoeError, 'Unknown TASK_TYPE: %r' % task_type - mod.tasktype_module.init(self) - - def hook_write_metadata(self): - """(mainline at time 90) Write status metadata into file `STATUS_FILE`.""" - self.log.debug('Writing status file %s', self['STATUS_FILE']) - self.status.write(self['STATUS_FILE']) - # TODO: dump to ddebug +def hook_init_dirs(e): + """(mainline at time 5) Create and check directories, fix directory variables. + .. note:: Currently only TDIR.""" + e.config.fix('TDIR') + tdir = e['TDIR'] + if os.path.isdir(tdir): + shutil.rmtree(tdir) + moe.util.mkdir_tree(tdir) + +def hook_load_task_config(e): + """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`.""" + e.config.fix(['TASK', 'PDIR', 'TASK_CONFIG']) + e.log.debug('Loading task config %s', self['TASK_CONFIG']) + if not os.path.isdir(e['PDIR']): + raise moe.MoeError, "No such task %s in %s" % (e['TASK'], self['PDIR']) + e.config.parse_file(self['TASK_CONFIG'], level=50) + e.debug_dump_config() + + e.status["task"] = task # Metadata + +def hook_init_tasktype(e): + """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module.""" + + e.config.fix('TASK_TYPE') + task_type = e['TASK_TYPE'] + e.log.debug('Loading module for TASK_TYPE: %r', task_type) + if not task_type: + raise MoeError, "Invalid TASK_TYPE: %r" % e + try: + e.tasktype_module = utils.load_module('moe.tasktypes.' + task_type) + except ImportError: + e.log.exception() + raise MoeError, 'Unknown TASK_TYPE: %r' % task_type + mod.tasktype_module.init(e) + +def hook_write_metadata(e): + """(mainline at time 90) Write status metadata into file `STATUS_FILE`.""" + e.log.debug('Writing status file %s', self['STATUS_FILE']) + e.status.write(self['STATUS_FILE']) + # TODO: dump to ddebug diff --git a/t/moe/exts/dummy.py b/t/moe/exts/dummy.py index 7a070a2..3ec5816 100644 --- a/t/moe/exts/dummy.py +++ b/t/moe/exts/dummy.py @@ -7,16 +7,19 @@ Adds several info-printing hooks to both pipelines. def init(e): - def hook_m_0(self): + def hook_m_0(e): e.log.info('Hey! It\'s me, the dummy extension in your main pipeline! (at time 0)') + e.main_pipe.insert(0, 'exts.dummy.hook_m_0', hook_m_0) - def hook_m_79(self): + def hook_m_79(e): e.log.info('Me, the dummy extension, requies no cleanup! (at time 79)') + e.main_pipe.insert(79, 'exts.dummy.hook_m_79', hook_m_79) - def hook_t_42(self): + def hook_t_42(e): t = 'It\'s test %s and the dummy extension did nothing! (at time 42)' % e['TEST'] e.log.info(t) e.log.user.info(t) + e.test_pipe.insert(42, 'exts.dummy.hook_t_42', hook_t_42) diff --git a/t/moe/pipeline.py b/t/moe/pipeline.py index 22422e6..a064c61 100644 --- a/t/moe/pipeline.py +++ b/t/moe/pipeline.py @@ -34,7 +34,7 @@ class MoePipeline: for pri,name,fun in self.pipe: file.write("%s%03d %s\n" % (prefix,pri,name)) - def run(self, *args): + def run(self, *args, **kwargs): self.index = 0 min_pri = -1 while self.index < len(self.pipe): @@ -42,7 +42,7 @@ class MoePipeline: if pri >= min_pri: moe.log.default.verbose(">> Running %s:%s\n" % (self.name,name)) try: - fun(*args) + fun(*args, **kwargs) except MoeAbortPipeline, err: min_pri = err.skip_to else: diff --git a/t/moe/tasktypes/dummy.py b/t/moe/tasktypes/dummy.py index 446c4e7..c8b7c73 100644 --- a/t/moe/tasktypes/dummy.py +++ b/t/moe/tasktypes/dummy.py @@ -8,12 +8,12 @@ Runs the test pipeline for each TEST def init(e): - def hook_m_50(self): + def hook_m_50(e): e.log.info('Here should be compiling') e.main_pipe.insert(50, 'dummy tasktype hook_m_50', hook_m_50) - def hook_m_60(self): + def hook_m_60(e): e.log.info('Here should be test pipeline running') tests = e['TESTS'].split() for t in tests: @@ -29,7 +29,7 @@ def init(e): e.main_pipe.insert(60, 'dummy tasktype hook_m_60', hook_m_60) - def hook_t_30(self): + def hook_t_30(e): e.log.info("Maybe we should do something? Nah...") e.test_pipe.insert(30, 'dummy tasktype hook_t_30', hook_t_30) -- 2.39.2