.. _timeline:
+.. currentmodule:: moe.eval
+
Evaluation timeline
====================
`Eval.init`
------------
-:func:`Eval.init <moe.eval.Eval.init>` is run before the main pipeline and does the following:
+:func:`Eval.init` is run before the main pipeline and does the following:
======= ==========
Time Action
\ Fix `CONFIG` and load the config file `CONFIG`
\ Fix `LOG`, `USER_LOG`, `VERBOSE`, `HOME`, `DEBUG_LEVEL`, `TDIR`
\ Start logging system (to `LOG` and `USER_LOG`)
-\ Check basic settings, directories and rights (currently checking only `TDIR`)
-\ Insert predefined hooks ([m]) and `HOOKS` to main pipeline
+\ Insert predefined hooks ([m]) and custom `HOOKS` to main pipeline
======= ==========
Main pipeline
-------------
-Ran by `Eval.run`.
+Ran by :func:`Eval.run`.
======= ==========
Time Action
======= ==========
-10 [m] Task **must** be ready (rsynced, pulled). Locate and fix `PDIR`
-15 [m] Load `TASK_CONFIG_FILE` (dep. on `PDIR`)
-20 [m] Check `TASK_TYPE` and depending on that, initialize the module and insert future actions
+5 [m :func:`Eval.hook_init_dirs`] Check basic settings and directories
+10 *Task must be ready (rsynced, pulled)*
+15 [m :func:`Eval.hook_load_task_config`] Load task config file
+20 [m :func:`Eval.hook_init_tasktype`] Import and initialize the task-type module
**For 'standard':**
------------------
25 [tt] Locate the source
30 [tt] Initialize sandbox module (if needed)
40 [tt] Initialize judge module, insert its hooks
60 [tt] Run test pipeline for each `TEST` of `TESTS`
-**Recovery**
+**Cleanup**
------------------
70 Any failure jumps to this time
-80 [*] Cleanup must be finished
-\ Write metafile
+80 Cleanup *must* be finished
+90 [m :func:`Eval.hook_write_metadata`] Write metafile
======= ==========
Test pipeline
20 [tt] Locate the data file, copy to
40 [tt] Run judge (inside sandbox)
50 [tt] Read/parse judge result
-**Recovery**
+**Cleanup**
--------------------------------
70 Any failure jumps to this time
80 [*] Cleanup must be finished
self.config = moe.config.ConfigTree()
self.main_pipe = moe.pipeline.MoePipeline("main")
self.test_pipe = moe.pipeline.MoePipeline("test")
- self.stat = moe.status.MoeStatus()
+ self.status = moe.status.MoeStatus()
def __getitem__(self, key):
return self.config[key]
self.config.parse('LOG="{HOME}/log"', source="<builtins>", level=0)
self.config.parse('DEBUG_LEVEL="0"', source="<builtins>", level=0)
self.config.parse('VERBOSE=""', source="<builtins>", level=0)
- self.config.parse('TDIR="{HOME}/test"', source="<temp-builtins>", level=0) # -> config
- self.config.parse('USER_LOG="{TDIR}/log"', source="<temp-builtins>", level=0) # -> config
+
+ # Non-basic configuration, should be in default config file
+ self.config.parse("""
+ TDIR = "{HOME}/test"
+ PDIR = "{HOME}/tasks/{TASK}"
+ USER_LOG = "{TDIR}/log"
+ TASK_CONFIG = "{PDIR}/config"
+ STATUS_FILE = "{TDIR}/status"
+ EXTENSIONS = ""
+ # only for testing:
+ TASK = "sum"
+ TASK_TYPE = "dummy"
+ USER = "gavento"
+ SOURCE = "suma.c"
+ EXTENSIONS += " dummy "
+ """, source="<temp-builtins>", level=0) # -> config
+
# apply overrides
for ov in overrides:
- self.config.parse(ov, source="<overrides>", level=100)
+ self.config.parse(ov, source="<overrides>", level=100)
# load config file
self.config.fix('CONFIG')
self.init_TDIR()
# insert hooks into main pipeline
- # TODO
- # TODO moe.box.init(self)
-
- # insert custom hooks
- self.conf.fix('HOOKS')
- self.main_pipe.configure(self['HOOKS'])
-
+ self.main_pipe.insert(5, "Eval.hook_init_dirs", self.hook_init_dirs)
+ self.main_pipe.insert(15, "Eval.hook_load_task_config", self.hook_load_task_config)
+ self.main_pipe.insert(20, "Eval.hook_init_tasktype", self.hook_init_tasktype)
+ self.main_pipe.insert(90, "Eval.hook_write_metadata", self.hook_write_metadata)
+
+ # ininialize extensions (let them insert hooks)
+ self.conf.fix('EXTENSIONS')
+ exts = self['EXTENSIONS'].split()
+ for e in exts:
+ if not e:
+ raise MoeError, "Invalid extension name: %r" % e
+ self.log.debug("Loading extension %s", e)
+ try:
+ mod = util.load_module('moe.exts.' + e)
+ except ImportError:
+ self.log.exception()
+ raise MoeError, 'Unknown extension: %r' % e
+ mod.init(self)
+
def run(self):
"Run the main pipeline."
self.debug_dump_pipe(self.main_pipe)
self.debug('Running main pipeline')
self.main_pipe.run(self)
-
- def init_TDIR(self):
- test = self['TDIR']
- if os.path.isdir(test):
- shutil.rmtree(test)
- try: #TODO: Remove
- moe.util.mkdir_tree(test)
- except OSError, err:
- raise moe.MoeError, "Cannot create %s: %s" % (test, err.strerror)
-
def debug_dump_config(self):
- "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)."
- if self.log.level <= 5:
- self.log.ddebug('****** Config dump: ******')
- self.log.ddebug(self.config.dump('**** '))
- self.log.ddebug('**************************')
+ "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)."
+ if self.log.level <= 5:
+ self.log.ddebug('****** Config dump: ******')
+ self.log.ddebug(self.config.dump('**** '))
+ self.log.ddebug('**************************')
def debug_dump_pipe(self, pipe):
- "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)."
- if self.log.level <= 5:
- self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name)
- self.log.ddebug(pipe.dump(prefix='**** '))
- self.log.ddebug('**************************')
-
-#TODO ...
- def init_task(self):
- task = self['TASK']
- task_dir = self['PDIR']
- if not os.path.isdir(task_dir):
- raise moe.MoeError, "No such task %s" % task
-
- task_cfg = moe.config.MoeConfig(name = os.path.join(task_dir, "config"), type='task')
- self.cfgs.push(task_cfg)
- self.log_config(3, "after loading the task")
-
- self.stat["task"] = task
-
- type = self['TASK_TYPE']
- if type == "batch" or type == "interactive":
- moe.batch.prepare_pipe(self)
- elif type == "opendata":
- raise moe.MoeError, "Opendata tasks not implemented yet"
- else:
- raise moe.MoeError, "Unknown task type " + type
+ "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)."
+ if self.log.level <= 5:
+ self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name)
+ self.log.ddebug(pipe.dump(prefix='**** '))
+ self.log.ddebug('**************************')
+
+ def hook_init_dirs(self):
+ """(mainline at time 5) Create and check directories, fix directory variables.
+ .. note:: Currently only TDIR."""
+ self.config.fix('TDIR')
+ tdir = self['TDIR']
+ if os.path.isdir(tdir):
+ shutil.rmtree(tdir)
+ moe.util.mkdir_tree(tdir)
+
+ def hook_load_task_config(self):
+ """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`."""
+ self.config.fix(['TASK', 'PDIR', 'TASK_CONFIG'])
+ self.log.debug('Loading task config %s', self['TASK_CONFIG'])
+ if not os.path.isdir(self['PDIR']):
+ raise moe.MoeError, "No such task %s in %s" % (self['TASK'], self['PDIR'])
+ self.config.parse_file(self['TASK_CONFIG'], level=50)
+ self.debug_dump_config()
+
+ self.stat["task"] = task # Metadata
+
+ def hook_init_tasktype(self):
+ """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module."""
+
+ self.config.fix('TASK_TYPE')
+ task_type = self['TASK_TYPE']
+ self.log.debug('Loading module for TASK_TYPE: %r', task_type)
+ if not task_type:
+ raise MoeError, "Invalid TASK_TYPE: %r" % e
+ try:
+ self.tasktype_module = utils.load_module('moe.tasktypes.' + task_type)
+ except ImportError:
+ self.log.exception()
+ raise MoeError, 'Unknown TASK_TYPE: %r' % task_type
+ mod.tasktype_module.init(self)
+
+ def hook_write_metadata(self):
+ """(mainline at time 90) Write status metadata into file `STATUS_FILE`."""
+ self.log.debug('Writing status file %s', self['STATUS_FILE'])
+ self.status.write(self['STATUS_FILE'])
+ # TODO: dump to ddebug
+
+
projects / eval.git / commitdiff