19 self.log = moe.log.Logers()
20 self.config = moe.config.ConfigTree()
21 self.main_pipe = moe.pipeline.MoePipeline("main")
22 self.test_pipe = moe.pipeline.MoePipeline("test")
23 self.stat = moe.status.MoeStatus()
25 def __getitem__(self, key):
26 return self.config[key]
28 def init(self, overrides=[]):
29 "Initializes most part of Eval before running the pipeline. See the timeline for details."
30 self.log.info("Initializing ...")
33 self.config.parse('HOME=\'%s\'' % os.getcwd(), source="<builtins>", level=0)
34 self.config.parse('CONFIG="{HOME}/config"', source="<builtins>", level=0)
35 self.config.parse('LOG="{HOME}/log"', source="<builtins>", level=0)
36 self.config.parse('DEBUG_LEVEL="0"', source="<builtins>", level=0)
37 self.config.parse('VERBOSE=""', source="<builtins>", level=0)
38 self.config.parse('TDIR="{HOME}/test"', source="<temp-builtins>", level=0) # -> config
39 self.config.parse('USER_LOG="{TDIR}/log"', source="<temp-builtins>", level=0) # -> config
42 self.config.parse(ov, source="<overrides>", level=100)
45 self.config.fix('CONFIG')
46 with open(self['CONFIG'], 'r') as f:
47 self.config.parse(f, source=self['CONFIG'], level=30)
49 self.config.fix(['LOG', 'USER_LOG', 'VERBOSE', 'HOME', 'DEBUG_LEVEL', 'TDIR'])
51 self.log.open_eval_log(self['LOG'], self['DEBUG_LEVEL'], redirect_fds = True)
52 self.log.open_user_log(self['USER_LOG'])
53 self.debug_dump_config()
56 self.debug('Cleaning TDIR: %s'%self['TDIR'])
59 # insert hooks into main pipeline
61 # TODO moe.box.init(self)
64 self.conf.fix('HOOKS')
65 self.main_pipe.configure(self['HOOKS'])
68 "Run the main pipeline."
69 self.debug_dump_pipe(self.main_pipe)
70 self.debug('Running main pipeline')
71 self.main_pipe.run(self)
76 if os.path.isdir(test):
79 moe.util.mkdir_tree(test)
81 raise moe.MoeError, "Cannot create %s: %s" % (test, err.strerror)
83 def debug_dump_config(self):
84 "Dumps config at level DDEBUG (only compiles the dump if main level is low enough)."
85 if self.log.level <= 5:
86 self.log.ddebug('****** Config dump: ******')
87 self.log.ddebug(self.config.dump('**** '))
88 self.log.ddebug('**************************')
90 def debug_dump_pipe(self, pipe):
91 "Dumps pipeline `pipe` at level DDEBUG (only compiles the dump if main level low enough)."
92 if self.log.level <= 5:
93 self.log.ddebug('****** Pipeline %r dump: ******'%pipe,name)
94 self.log.ddebug(pipe.dump(prefix='**** '))
95 self.log.ddebug('**************************')
100 task_dir = self['PDIR']
101 if not os.path.isdir(task_dir):
102 raise moe.MoeError, "No such task %s" % task
104 task_cfg = moe.config.MoeConfig(name = os.path.join(task_dir, "config"), type='task')
105 self.cfgs.push(task_cfg)
106 self.log_config(3, "after loading the task")
108 self.stat["task"] = task
110 type = self['TASK_TYPE']
111 if type == "batch" or type == "interactive":
112 moe.batch.prepare_pipe(self)
113 elif type == "opendata":
114 raise moe.MoeError, "Opendata tasks not implemented yet"
116 raise moe.MoeError, "Unknown task type " + type