From df430d8cc46be638ba60c3cd6d27dfba1caa0b5c Mon Sep 17 00:00:00 2001 From: Tomas Gavenciak Date: Sat, 11 Sep 2010 17:59:38 +0200 Subject: [PATCH] Timeline change, multiple changes in Eval I am sorry for this giga-commit, but it would be too much work to split it Update (some) docs, small timeline change (move loading tasktype from init to pipeline) Eval.stat -> Eval.status (cosmetic) Add basic (builtins) and testing configuration Eval.init fills mainline with basic hooks Eval.init loads and initializes extensions Increase indentation (2 by gavento -> 4 by MJ) Write all main hooks in Eval Eval.hook_init_tasktype loads tasktype module It may not seem so, but most of the important code is reused :-) --- t/doc/timeline.rst | 24 ++++---- t/moe/eval.py | 143 ++++++++++++++++++++++++++++----------------- 2 files changed, 104 insertions(+), 63 deletions(-) diff --git a/t/doc/timeline.rst b/t/doc/timeline.rst index 4c5383e..df67a09 100644 --- a/t/doc/timeline.rst +++ b/t/doc/timeline.rst @@ -1,12 +1,14 @@ .. _timeline: +.. currentmodule:: moe.eval + Evaluation timeline ==================== `Eval.init` ------------ -:func:`Eval.init ` is run before the main pipeline and does the following: +:func:`Eval.init` is run before the main pipeline and does the following: ======= ========== Time Action @@ -16,21 +18,21 @@ Time Action \ Fix `CONFIG` and load the config file `CONFIG` \ Fix `LOG`, `USER_LOG`, `VERBOSE`, `HOME`, `DEBUG_LEVEL`, `TDIR` \ Start logging system (to `LOG` and `USER_LOG`) -\ Check basic settings, directories and rights (currently checking only `TDIR`) -\ Insert predefined hooks ([m]) and `HOOKS` to main pipeline +\ Insert predefined hooks ([m]) and custom `HOOKS` to main pipeline ======= ========== Main pipeline ------------- -Ran by `Eval.run`. +Ran by :func:`Eval.run`. ======= ========== Time Action ======= ========== -10 [m] Task **must** be ready (rsynced, pulled). Locate and fix `PDIR` -15 [m] Load `TASK_CONFIG_FILE` (dep. on `PDIR`) -20 [m] Check `TASK_TYPE` and depending on that, initialize the module and insert future actions +5 [m :func:`Eval.hook_init_dirs`] Check basic settings and directories +10 *Task must be ready (rsynced, pulled)* +15 [m :func:`Eval.hook_load_task_config`] Load task config file +20 [m :func:`Eval.hook_init_tasktype`] Import and initialize the task-type module **For 'standard':** ------------------ 25 [tt] Locate the source @@ -44,11 +46,11 @@ Time Action 30 [tt] Initialize sandbox module (if needed) 40 [tt] Initialize judge module, insert its hooks 60 [tt] Run test pipeline for each `TEST` of `TESTS` -**Recovery** +**Cleanup** ------------------ 70 Any failure jumps to this time -80 [*] Cleanup must be finished -\ Write metafile +80 Cleanup *must* be finished +90 [m :func:`Eval.hook_write_metadata`] Write metafile ======= ========== Test pipeline @@ -71,7 +73,7 @@ Time Action 20 [tt] Locate the data file, copy to 40 [tt] Run judge (inside sandbox) 50 [tt] Read/parse judge result -**Recovery** +**Cleanup** -------------------------------- 70 Any failure jumps to this time 80 [*] Cleanup must be finished diff --git a/t/moe/eval.py b/t/moe/eval.py index 1e54b4e..bc1ea83 100644 --- a/t/moe/eval.py +++ b/t/moe/eval.py @@ -20,7 +20,7 @@ class Eval: self.config = moe.config.ConfigTree() self.main_pipe = moe.pipeline.MoePipeline("main") self.test_pipe = moe.pipeline.MoePipeline("test") - self.stat = moe.status.MoeStatus() + self.status = moe.status.MoeStatus() def __getitem__(self, key): return self.config[key] @@ -35,11 +35,26 @@ class Eval: self.config.parse('LOG="{HOME}/log"', source="", level=0) self.config.parse('DEBUG_LEVEL="0"', source="", level=0) self.config.parse('VERBOSE=""', source="", level=0) - self.config.parse('TDIR="{HOME}/test"', source="", level=0) # -> config - self.config.parse('USER_LOG="{TDIR}/log"', source="", level=0) # -> config + + # Non-basic configuration, should be in default config file + self.config.parse(""" + TDIR = "{HOME}/test" + PDIR = "{HOME}/tasks/{TASK}" + USER_LOG = "{TDIR}/log" + TASK_CONFIG = "{PDIR}/config" + STATUS_FILE = "{TDIR}/status" + EXTENSIONS = "" + # only for testing: + TASK = "sum" + TASK_TYPE = "dummy" + USER = "gavento" + SOURCE = "suma.c" + EXTENSIONS += " dummy " + """, source="", level=0) # -> config + # apply overrides for ov in overrides: - self.config.parse(ov, source="", level=100) + self.config.parse(ov, source="", level=100) # load config file self.config.fix('CONFIG') @@ -56,61 +71,85 @@ class Eval: self.init_TDIR() # insert hooks into main pipeline - # TODO - # TODO moe.box.init(self) - - # insert custom hooks - self.conf.fix('HOOKS') - self.main_pipe.configure(self['HOOKS']) - + self.main_pipe.insert(5, "Eval.hook_init_dirs", self.hook_init_dirs) + self.main_pipe.insert(15, "Eval.hook_load_task_config", self.hook_load_task_config) + self.main_pipe.insert(20, "Eval.hook_init_tasktype", self.hook_init_tasktype) + self.main_pipe.insert(90, "Eval.hook_write_metadata", self.hook_write_metadata) + + # ininialize extensions (let them insert hooks) + self.conf.fix('EXTENSIONS') + exts = self['EXTENSIONS'].split() + for e in exts: + if not e: + raise MoeError, "Invalid extension name: %r" % e + self.log.debug("Loading extension %s", e) + try: + mod = util.load_module('moe.exts.' + e) + except ImportError: + self.log.exception() + raise MoeError, 'Unknown extension: %r' % e + mod.init(self) + def run(self): "Run the main pipeline." self.debug_dump_pipe(self.main_pipe) self.debug('Running main pipeline') self.main_pipe.run(self) - - def init_TDIR(self): - test = self['TDIR'] - if os.path.isdir(test): - shutil.rmtree(test) - try: #TODO: Remove - moe.util.mkdir_tree(test) - except OSError, err: - raise moe.MoeError, "Cannot create %s: %s" % (test, err.strerror) - def debug_dump_config(self): - "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)." - if self.log.level <= 5: - self.log.ddebug('****** Config dump: ******') - self.log.ddebug(self.config.dump('**** ')) - self.log.ddebug('**************************') + "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)." + if self.log.level <= 5: + self.log.ddebug('****** Config dump: ******') + self.log.ddebug(self.config.dump('**** ')) + self.log.ddebug('**************************') def debug_dump_pipe(self, pipe): - "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)." - if self.log.level <= 5: - self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name) - self.log.ddebug(pipe.dump(prefix='**** ')) - self.log.ddebug('**************************') - -#TODO ... - def init_task(self): - task = self['TASK'] - task_dir = self['PDIR'] - if not os.path.isdir(task_dir): - raise moe.MoeError, "No such task %s" % task - - task_cfg = moe.config.MoeConfig(name = os.path.join(task_dir, "config"), type='task') - self.cfgs.push(task_cfg) - self.log_config(3, "after loading the task") - - self.stat["task"] = task - - type = self['TASK_TYPE'] - if type == "batch" or type == "interactive": - moe.batch.prepare_pipe(self) - elif type == "opendata": - raise moe.MoeError, "Opendata tasks not implemented yet" - else: - raise moe.MoeError, "Unknown task type " + type + "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)." + if self.log.level <= 5: + self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name) + self.log.ddebug(pipe.dump(prefix='**** ')) + self.log.ddebug('**************************') + + def hook_init_dirs(self): + """(mainline at time 5) Create and check directories, fix directory variables. + .. note:: Currently only TDIR.""" + self.config.fix('TDIR') + tdir = self['TDIR'] + if os.path.isdir(tdir): + shutil.rmtree(tdir) + moe.util.mkdir_tree(tdir) + + def hook_load_task_config(self): + """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`.""" + self.config.fix(['TASK', 'PDIR', 'TASK_CONFIG']) + self.log.debug('Loading task config %s', self['TASK_CONFIG']) + if not os.path.isdir(self['PDIR']): + raise moe.MoeError, "No such task %s in %s" % (self['TASK'], self['PDIR']) + self.config.parse_file(self['TASK_CONFIG'], level=50) + self.debug_dump_config() + + self.stat["task"] = task # Metadata + + def hook_init_tasktype(self): + """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module.""" + + self.config.fix('TASK_TYPE') + task_type = self['TASK_TYPE'] + self.log.debug('Loading module for TASK_TYPE: %r', task_type) + if not task_type: + raise MoeError, "Invalid TASK_TYPE: %r" % e + try: + self.tasktype_module = utils.load_module('moe.tasktypes.' + task_type) + except ImportError: + self.log.exception() + raise MoeError, 'Unknown TASK_TYPE: %r' % task_type + mod.tasktype_module.init(self) + + def hook_write_metadata(self): + """(mainline at time 90) Write status metadata into file `STATUS_FILE`.""" + self.log.debug('Writing status file %s', self['STATUS_FILE']) + self.status.write(self['STATUS_FILE']) + # TODO: dump to ddebug + + -- 2.39.2