def __init__(self):
self.log = moe.log.Loggers()
self.config = moe.config.ConfigTree()
- self.main_pipe = moe.pipeline.MoePipeline("main")
- self.test_pipe = moe.pipeline.MoePipeline("test")
- self.status = moe.status.MoeStatus()
+ self.main_pipe = moe.pipeline.Pipeline(self, "main")
+ self.test_pipe = moe.pipeline.Pipeline(self, "test")
+ self.status = moe.status.Status()
def __getitem__(self, key):
return self.config[key]
# fix variables
self.config.fix(['LOG', 'USER_LOG', 'VERBOSE', 'HOME', 'DEBUG_LEVEL', 'TDIR'])
# start logging
- self.log.open_eval_log(self['LOG'], self['DEBUG_LEVEL'], redirect_fds = True)
+ self.log.open_eval_log(self['LOG'], int(self['DEBUG_LEVEL']), redirect_fds = True)
self.log.open_user_log(self['USER_LOG'])
self.debug_dump_config()
- # init and check TDIR
- self.debug('Cleaning TDIR: %s'%self['TDIR'])
- self.init_TDIR()
-
# insert hooks into main pipeline
- self.main_pipe.insert(5, "Eval.hook_init_dirs", self.hook_init_dirs)
- self.main_pipe.insert(15, "Eval.hook_load_task_config", self.hook_load_task_config)
- self.main_pipe.insert(20, "Eval.hook_init_tasktype", self.hook_init_tasktype)
- self.main_pipe.insert(90, "Eval.hook_write_metadata", self.hook_write_metadata)
+ self.main_pipe.insert(5, "Eval.hook_init_dirs", hook_init_dirs)
+ self.main_pipe.insert(15, "Eval.hook_load_task_config", hook_load_task_config)
+ self.main_pipe.insert(20, "Eval.hook_init_tasktype", hook_init_tasktype)
+ self.main_pipe.insert(90, "Eval.hook_write_metadata", hook_write_metadata)
# ininialize extensions (let them insert hooks)
- self.conf.fix('EXTENSIONS')
+ self.config.fix('EXTENSIONS')
exts = self['EXTENSIONS'].split()
for e in exts:
if not e:
raise MoeError, "Invalid extension name: %r" % e
self.log.debug("Loading extension %s", e)
try:
- mod = util.load_module('moe.exts.' + e)
+ mod = moe.util.load_module('moe.exts.' + e)
except ImportError:
self.log.exception()
raise MoeError, 'Unknown extension: %r' % e
def run(self):
"Run the main pipeline."
self.debug_dump_pipe(self.main_pipe)
- self.debug('Running main pipeline')
- self.main_pipe.run(self)
+ self.log.debug('Running main pipeline')
+ self.main_pipe.run(e=self)
def debug_dump_config(self):
"Dumps config at level DDEBUG (only compiles the dump if main level is low enough)."
if self.log.level <= 5:
- self.log.ddebug('****** Config dump: ******')
- self.log.ddebug(self.config.dump('**** '))
- self.log.ddebug('**************************')
+ self.log.ddebug(' ****** Config dump: ******')
+ self.log.ddebug('\n'.join(self.config.dump(' * ')))
+ self.log.ddebug(' **************************')
def debug_dump_pipe(self, pipe):
"Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)."
if self.log.level <= 5:
- self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name)
- self.log.ddebug(pipe.dump(prefix='**** '))
- self.log.ddebug('**************************')
-
- def hook_init_dirs(self):
- """(mainline at time 5) Create and check directories, fix directory variables.
- .. note:: Currently only TDIR."""
- self.config.fix('TDIR')
- tdir = self['TDIR']
- if os.path.isdir(tdir):
- shutil.rmtree(tdir)
- moe.util.mkdir_tree(tdir)
+ self.log.ddebug(' ****** Pipeline %r dump: ******'%pipe.name)
+ self.log.ddebug('\n'.join(pipe.dump(prefix=' * ')))
+ self.log.ddebug(' **************************')
- def hook_load_task_config(self):
- """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`."""
- self.config.fix(['TASK', 'PDIR', 'TASK_CONFIG'])
- self.log.debug('Loading task config %s', self['TASK_CONFIG'])
- if not os.path.isdir(self['PDIR']):
- raise moe.MoeError, "No such task %s in %s" % (self['TASK'], self['PDIR'])
- self.config.parse_file(self['TASK_CONFIG'], level=50)
- self.debug_dump_config()
-
- self.stat["task"] = task # Metadata
-
- def hook_init_tasktype(self):
- """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module."""
-
- self.config.fix('TASK_TYPE')
- task_type = self['TASK_TYPE']
- self.log.debug('Loading module for TASK_TYPE: %r', task_type)
- if not task_type:
- raise MoeError, "Invalid TASK_TYPE: %r" % e
- try:
- self.tasktype_module = utils.load_module('moe.tasktypes.' + task_type)
- except ImportError:
- self.log.exception()
- raise MoeError, 'Unknown TASK_TYPE: %r' % task_type
- mod.tasktype_module.init(self)
-
- def hook_write_metadata(self):
- """(mainline at time 90) Write status metadata into file `STATUS_FILE`."""
- self.log.debug('Writing status file %s', self['STATUS_FILE'])
- self.status.write(self['STATUS_FILE'])
- # TODO: dump to ddebug
+ def debug_dump_status(self):
+ "Dumps status metadata at level DDEBUG (only compiles the dump if main level low enough)."
+ if self.log.level <= 5:
+ self.log.ddebug(' ****** Status dump: ******')
+ self.log.ddebug('\n'.join(self.status.dump(prefix=' * ')).rstrip())
+ self.log.ddebug(' **************************')
+
+def hook_init_dirs(e):
+ """(mainline at time 5) Create and check directories, fix directory variables.
+ .. note:: Currently only TDIR."""
+ e.config.fix('TDIR')
+ tdir = e['TDIR']
+ if os.path.isdir(tdir):
+ shutil.rmtree(tdir)
+ moe.util.mkdir_tree(tdir)
+
+def hook_load_task_config(e):
+ """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`."""
+ e.config.fix(['TASK', 'PDIR', 'TASK_CONFIG'])
+ e.log.debug('Loading task config %s', e['TASK_CONFIG'])
+ if not os.path.isdir(e['PDIR']):
+ raise moe.MoeError, "No such task %s in %s" % (e['TASK'], e['PDIR'])
+ e.config.parse_file(e['TASK_CONFIG'], level=50)
+ e.debug_dump_config()
+
+ e.status["task"] = e['TASK'] # Metadata
+
+def hook_init_tasktype(e):
+ """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module."""
+
+ e.config.fix('TASK_TYPE')
+ task_type = e['TASK_TYPE']
+ e.log.debug('Loading module for TASK_TYPE: %r', task_type)
+ if not task_type:
+ raise MoeError, "Invalid TASK_TYPE: %r" % e
+ try:
+ e.tasktype_module = moe.util.load_module('moe.tasktypes.' + task_type)
+ except ImportError:
+ e.log.exception()
+ raise MoeError, 'Unknown TASK_TYPE: %r' % task_type
+ e.tasktype_module.init(e)
+
+def hook_write_metadata(e):
+ """(mainline at time 90) Write status metadata into file `STATUS_FILE`."""
+ e.debug_dump_status()
+ e.log.debug('Writing status file %s', e['STATUS_FILE'])
+ with open(e['STATUS_FILE'], 'w') as f:
+ e.status.write(f)
+ # TODO: dump to ddebug
projects / eval.git / blobdiff