19 self.log = moe.log.Logers()
20 self.config = moe.config.ConfigTree()
21 self.main_pipe = moe.pipeline.MoePipeline("main")
22 self.test_pipe = moe.pipeline.MoePipeline("test")
23 self.status = moe.status.MoeStatus()
25 def __getitem__(self, key):
26 return self.config[key]
28 def init(self, overrides=[]):
29 "Initializes most part of Eval before running the pipeline. See the timeline for details."
30 self.log.info("Initializing ...")
33 self.config.parse('HOME=\'%s\'' % os.getcwd(), source="<builtins>", level=0)
34 self.config.parse('CONFIG="{HOME}/config"', source="<builtins>", level=0)
35 self.config.parse('LOG="{HOME}/log"', source="<builtins>", level=0)
36 self.config.parse('DEBUG_LEVEL="0"', source="<builtins>", level=0)
37 self.config.parse('VERBOSE=""', source="<builtins>", level=0)
39 # Non-basic configuration, should be in default config file
42 PDIR = "{HOME}/tasks/{TASK}"
43 USER_LOG = "{TDIR}/log"
44 TASK_CONFIG = "{PDIR}/config"
45 STATUS_FILE = "{TDIR}/status"
52 EXTENSIONS += " dummy "
53 """, source="<temp-builtins>", level=0) # -> config
57 self.config.parse(ov, source="<overrides>", level=100)
60 self.config.fix('CONFIG')
61 self.config.parse_file(self['CONFIG'], level=30)
63 self.config.fix(['LOG', 'USER_LOG', 'VERBOSE', 'HOME', 'DEBUG_LEVEL', 'TDIR'])
65 self.log.open_eval_log(self['LOG'], self['DEBUG_LEVEL'], redirect_fds = True)
66 self.log.open_user_log(self['USER_LOG'])
67 self.debug_dump_config()
70 self.debug('Cleaning TDIR: %s'%self['TDIR'])
73 # insert hooks into main pipeline
74 self.main_pipe.insert(5, "Eval.hook_init_dirs", self.hook_init_dirs)
75 self.main_pipe.insert(15, "Eval.hook_load_task_config", self.hook_load_task_config)
76 self.main_pipe.insert(20, "Eval.hook_init_tasktype", self.hook_init_tasktype)
77 self.main_pipe.insert(90, "Eval.hook_write_metadata", self.hook_write_metadata)
79 # ininialize extensions (let them insert hooks)
80 self.conf.fix('EXTENSIONS')
81 exts = self['EXTENSIONS'].split()
84 raise MoeError, "Invalid extension name: %r" % e
85 self.log.debug("Loading extension %s", e)
87 mod = util.load_module('moe.exts.' + e)
90 raise MoeError, 'Unknown extension: %r' % e
94 "Run the main pipeline."
95 self.debug_dump_pipe(self.main_pipe)
96 self.debug('Running main pipeline')
97 self.main_pipe.run(self)
99 def debug_dump_config(self):
100 "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)."
101 if self.log.level <= 5:
102 self.log.ddebug('****** Config dump: ******')
103 self.log.ddebug(self.config.dump('**** '))
104 self.log.ddebug('**************************')
106 def debug_dump_pipe(self, pipe):
107 "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)."
108 if self.log.level <= 5:
109 self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name)
110 self.log.ddebug(pipe.dump(prefix='**** '))
111 self.log.ddebug('**************************')
113 def hook_init_dirs(self):
114 """(mainline at time 5) Create and check directories, fix directory variables.
115 .. note:: Currently only TDIR."""
116 self.config.fix('TDIR')
118 if os.path.isdir(tdir):
120 moe.util.mkdir_tree(tdir)
122 def hook_load_task_config(self):
123 """(mainline at time 15) Load `TASK_CONFIG` and check `PDIR`, fixes `TASK`, `PDIR`, `TASK_CONFIG`."""
124 self.config.fix(['TASK', 'PDIR', 'TASK_CONFIG'])
125 self.log.debug('Loading task config %s', self['TASK_CONFIG'])
126 if not os.path.isdir(self['PDIR']):
127 raise moe.MoeError, "No such task %s in %s" % (self['TASK'], self['PDIR'])
128 self.config.parse_file(self['TASK_CONFIG'], level=50)
129 self.debug_dump_config()
131 self.stat["task"] = task # Metadata
133 def hook_init_tasktype(self):
134 """(mainline at time 20) Fix `TASK_TYPE`, initialize task type module."""
136 self.config.fix('TASK_TYPE')
137 task_type = self['TASK_TYPE']
138 self.log.debug('Loading module for TASK_TYPE: %r', task_type)
140 raise MoeError, "Invalid TASK_TYPE: %r" % e
142 self.tasktype_module = utils.load_module('moe.tasktypes.' + task_type)
145 raise MoeError, 'Unknown TASK_TYPE: %r' % task_type
146 mod.tasktype_module.init(self)
148 def hook_write_metadata(self):
149 """(mainline at time 90) Write status metadata into file `STATUS_FILE`."""
150 self.log.debug('Writing status file %s', self['STATUS_FILE'])
151 self.status.write(self['STATUS_FILE'])
152 # TODO: dump to ddebug