From 8175ec1a0fe914d582d4f4936cb5d2857e4d6edc Mon Sep 17 00:00:00 2001 From: Tomas Gavenciak Date: Fri, 10 Sep 2010 21:44:28 +0200 Subject: [PATCH] Updated documentation and tidyed up eval.py Updated INIT part of the timeline Shifted some stuff in eval.py around (re)Created Eval.run() (run in init did not make sense) Made an observation: Tried porgrameer make many mitsakes. --- t/doc/index.rst | 14 ++++----- t/doc/{eval_timing.rst => timeline.rst} | 38 ++++++++++++++++--------- t/moe/eval.py | 33 +++++++++++---------- 3 files changed, 49 insertions(+), 36 deletions(-) rename t/doc/{eval_timing.rst => timeline.rst} (68%) diff --git a/t/doc/index.rst b/t/doc/index.rst index d30fb17..c0ef801 100644 --- a/t/doc/index.rst +++ b/t/doc/index.rst @@ -7,16 +7,14 @@ Contents: .. toctree:: :maxdepth: 2 + timeline.rst log.rst config.rst - eval_timing.rst -.. automodule:: moe.config +Indices and tables +================== -.. Indices and tables - ================== - -.. * :ref:`genindex` - * :ref:`modindex` - * :ref:`search` +* :ref:`genindex` +* :ref:`modindex` +* :ref:`search` diff --git a/t/doc/eval_timing.rst b/t/doc/timeline.rst similarity index 68% rename from t/doc/eval_timing.rst rename to t/doc/timeline.rst index c35ae12..37a2f5e 100644 --- a/t/doc/eval_timing.rst +++ b/t/doc/timeline.rst @@ -1,41 +1,53 @@ Evaluation timeline ==================== +Eval.init +--------- + +`Eval.init` is run before the main pipeline, does the following: + +======= ========== +Time Action +======= ========== +\ Set builtins `HOME`, `CONFIG`, `LOG`, `DEBUG_LEVEL`, `VERBOSE`, also (temp?) `TDIR`, `USER_LOG` +\ Apply command-line overrides +\ Fix `CONFIG` and load the config file `CONFIG` +\ Fix `LOG`, `USER_LOG`, `VERBOSE`, `HOME`, `DEBUG_LEVEL`, `TDIR` +\ Start logging system (to `LOG` and `USER_LOG`) +\ Check basic settings, directories and rights (currently checking only `TDIR`) +\ Insert predefined hooks ([m]) and `HOOKS` to main pipeline +======= ========== + Main pipeline ------------- -======= ======================== +Ran by `Eval.run`. + +======= ========== Time Action -======= ======================== -\ Apply command-line overrides, set basic builtins (`CONFIG_FILE`, `VERBOSITY`, ...) -\ FIX: `CONFIG_FILE` -\ Load `CONFIG_FILE` -\ FIX: `LOG_FILE`, `VERBOSITY`, `HOME`, `TDIR`, `BOXDIR`, `HOOKS`, `TEST_HOOKS`, ... -\ Start logging to `LOG_FILE` -\ Insert predefined hooks, `HOOKS` and `TEST_HOOKS` to pipelines -5 [m] Check basic settings, directories and rights +======= ========== 10 [m] Task **must** be ready (rsynced, pulled). Locate and fix `PDIR` 15 [m] Load `TASK_CONFIG_FILE` (dep. on `PDIR`) 20 [m] Check `TASK_TYPE` and depending on that, initialize the module and insert future actions **For 'standard':** --------------------------------- +------------------ 25 [tt] Locate the source 30 [tt] Initialize sandbox module 35 [tt] Initialize compiler module, insert its hooks 50 [cpl] Compile the source, perhaps using the sandbox 60 [tt] Run test pipeline for each `TEST` of `TESTS` **For 'opendata':** --------------------------------- +------------------ 25 [tt] Locate the data files (as a dir) 30 [tt] Initialize sandbox module (if needed) 40 [tt] Initialize judge module, insert its hooks 60 [tt] Run test pipeline for each `TEST` of `TESTS` **Recovery** --------------------------------- +------------------ 70 Any failure jumps to this time 80 [*] Cleanup must be finished \ Write metafile -======= ======================== +======= ========== Test pipeline ------------- diff --git a/t/moe/eval.py b/t/moe/eval.py index 3ee5566..c92ce8a 100644 --- a/t/moe/eval.py +++ b/t/moe/eval.py @@ -25,18 +25,8 @@ class Eval: def __getitem__(self, key): return self.config[key] - def debug_dumpconf(self): - if self.log.level <= 5: - self.log.ddebug('****** Config dump: ******')) - self.log.ddebug(self.config.dump('**** ')) - self.log.ddebug('**************************')) - def debug_dumppipe(self, pipe): - if self.log.level <= 5: - self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name)) - self.log.ddebug(pipe.dump(prefix='**** ')) - self.log.ddebug('**************************')) - def init(self, overrides=[]): + "Initializes most part of Eval before running the pipeline. See the timeline for details." self.log.info("Initializing ...") # set basic builtins @@ -56,7 +46,7 @@ class Eval: with open(self['CONFIG'], 'r') as f: self.config.parse(f, source=self['CONFIG'], level=30) # fix variables - self.config.fix(['LOG', 'VERBOSE', 'HOME', 'DEBUG_LEVEL', 'TDIR']) + self.config.fix(['LOG', 'USER_LOG', 'VERBOSE', 'HOME', 'DEBUG_LEVEL', 'TDIR']) # start logging self.log.open_eval_log(self['LOG'], self['DEBUG_LEVEL'], redirect_fds = True) self.log.open_user_log(self['USER_LOG']) @@ -68,18 +58,18 @@ class Eval: # insert hooks into main pipeline # TODO + # TODO moe.box.init(self) # insert custom hooks self.conf.fix('HOOKS') self.main_pipe.configure(self['HOOKS']) - # go! + def run(self): + "Run the main pipeline." self.debug_dump_pipe(self.main_pipe) self.debug('Running main pipeline') self.main_pipe.run(self) - #self.init_task() - #moe.box.init(self) def init_TDIR(self): test = self['TDIR'] @@ -90,6 +80,19 @@ class Eval: except OSError, err: raise moe.MoeError, "Cannot create %s: %s" % (test, err.strerror) + def debug_dump_config(self): + "Dumps at level DDEBUG and only compiles the dump if main level low enough." + if self.log.level <= 5: + self.log.ddebug('****** Config dump: ******')) + self.log.ddebug(self.config.dump('**** ')) + self.log.ddebug('**************************')) + + def debug_dump_pipe(self, pipe): + "Dumps at level DDEBUG and only compiles the dump if main level low enough." + if self.log.level <= 5: + self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name)) + self.log.ddebug(pipe.dump(prefix='**** ')) + self.log.ddebug('**************************')) #TODO ... def init_task(self): -- 2.39.2