From fdf195df30677b8552bb7fedc34f760120268a30 Mon Sep 17 00:00:00 2001 From: Martin Mares Date: Mon, 10 Aug 2009 00:38:32 +0200 Subject: [PATCH] Logging --- t/moe/batch.py | 5 ++--- t/moe/config.py | 25 +++++++++++++------------ t/moe/eval.py | 29 +++++++++++++++++++++++++---- t/moe/log.py | 24 ++++++++++++++++++------ t/moe/pipeline.py | 4 ++-- t/test.py | 8 -------- 6 files changed, 60 insertions(+), 35 deletions(-) diff --git a/t/moe/batch.py b/t/moe/batch.py index fb891da..28b585c 100644 --- a/t/moe/batch.py +++ b/t/moe/batch.py @@ -1,7 +1,6 @@ #!/usr/bin/env python import os.path -import moe.log import moe.eval import moe.util @@ -45,12 +44,12 @@ def locate(e, filename=None): orig_path = os.path.join(dir, file) norm_ext = normalize_ext(e, ext) - moe.log.verbose("Found solution %s\n" % orig_path) + e.log.verbose("Found solution %s\n" % orig_path) copy = e.cfgs["TASK"] + "." + norm_ext copy_path = os.path.join(e.cfgs["TEST_DIR"], copy) if file != copy: - moe.log.verbose("Renaming to %s\n" % copy) + e.log.verbose("Renaming to %s\n" % copy) moe.util.link_or_copy(orig_path, copy_path) e.builtins.set("SRC", copy) diff --git a/t/moe/config.py b/t/moe/config.py index ec0ad50..c68a899 100644 --- a/t/moe/config.py +++ b/t/moe/config.py @@ -15,8 +15,9 @@ class MoeConfigEvalErr(Exception): class MoeConfig: """Moe configuration file. Should be immutable once a part of a stack.""" - def __init__(self, file=None, name=None): + def __init__(self, file=None, name=None, type=""): self.vars = {} + self.type = type if file is not None: self.load(file) elif name is not None: @@ -100,8 +101,9 @@ class MoeConfig: list.append(("s", s[:p])) s = s[p:] - def dump(self, file=sys.stdout): + def dump(self, file=sys.stdout, prefix=""): for k,v in self.vars.items(): + file.write(prefix) file.write(k) if len(v) > 0 and v[0][0] == "a": file.write("+") @@ -170,31 +172,30 @@ class MoeConfigStack: seen[k] = None return seen.keys() - def dump(self, file=sys.stdout): + def dump(self, file=sys.stdout, prefix=""): for k in sorted(self.keys()): v = self[k] - file.write("%s=%s\n" % (k,v)) + file.write("%s%s=%s\n" % (prefix,k,v)) - def dump_defs(self, file=sys.stdout): - file.write("Configuration stack:\n") + def dump_defs(self, file=sys.stdout, prefix=""): level = 0 for cfg in self.stk: level += 1 - file.write("(level %d)\n" % level) - cfg.dump(file) - file.write("(end)\n") + file.write("%s(level %d: %s)\n" % (prefix,level,cfg.type)) + cfg.dump(file, prefix + "\t") + file.write("%s(end)\n" % prefix) def apply_overrides(self, prefix): newstk = [] for cfg in self.stk: - over = MoeConfig() + over = MoeConfig(type = cfg.type + '-overrides') changed = False for k in cfg.vars.keys(): if k.startswith(prefix): over.vars[k[len(prefix):]] = cfg.vars[k] changed = True if changed: - clean = MoeConfig() + clean = MoeConfig(type = cfg.type) for k in cfg.vars.keys(): if not k.startswith(prefix): clean.vars[k] = cfg.vars[k] @@ -210,7 +211,7 @@ def parse_overrides(argv): argv0 = argv.pop(0) while len(argv) > 0 and argv[0].find("=") >= 0: if cfg is None: - cfg = MoeConfig() + cfg = MoeConfig(type='cmdline') cfg.parse_line(argv.pop(0)) argv.insert(0, argv0) return cfg diff --git a/t/moe/eval.py b/t/moe/eval.py index 9c9f5ac..2d5c976 100644 --- a/t/moe/eval.py +++ b/t/moe/eval.py @@ -15,8 +15,9 @@ class MoeEvalErr(Exception): class Eval: def __init__(self): + self.log = moe.log.default self.cfgs = moe.config.MoeConfigStack() - self.builtins = moe.config.MoeConfig() + self.builtins = moe.config.MoeConfig(type="builtins") self.cfgs.push(self.builtins) self.main_pipe = moe.pipeline.MoePipeline("main") self.meta = moe.meta.MoeMeta() @@ -25,11 +26,11 @@ class Eval: def init(self, overrides=None): self.init_global(overrides) self.init_test() - ## FIXME: Initialize logging early on + self.init_logs() self.init_task() def init_global(self, overrides): - main_cfg = moe.config.MoeConfig(name = os.path.join(self.cfgs['HOME'], "config")) + main_cfg = moe.config.MoeConfig(name = os.path.join(self.cfgs['HOME'], "config"), type="main") self.cfgs.push(main_cfg) if overrides: self.cfgs.push(overrides) @@ -43,13 +44,24 @@ class Eval: except OSError, e: raise MoeEvalErr, "Cannot create %s: %s" % (test, e.strerror) + def init_logs(self): + self.log = moe.log.MoeLog() + if self.cfgs["V"]: + self.log.verbosity = int(self.cfgs["V"]) + self.log.log_file = open(os.path.join(self.cfgs["TEST_DIR"], "log"), "w") + self.default_log = self.log + self.log_config(3, "before loading the task") + def init_task(self): task = self.cfgs['TASK'] task_dir = self.cfgs['TASK_DIR'] if not os.path.isdir(task_dir): raise MoeEvalErr, "No such task %s" % task - task_cfg = moe.config.MoeConfig(name = os.path.join(task_dir, "config")) + + task_cfg = moe.config.MoeConfig(name = os.path.join(task_dir, "config"), type='task') self.cfgs.push(task_cfg) + self.log_config(3, "after loading the task") + self.meta["task"] = task type = self.cfgs['TASK_TYPE'] @@ -61,6 +73,15 @@ class Eval: raise MoeEvalErr, "Unknown task type " + type def run(self): + self.log_config(2, "for the task pipeline") self.main_pipe.configure(self.cfgs["HOOKS"]) self.main_pipe.dump() self.main_pipe.run(self) + + def log_config(self, verb, msg): + if self.log.verbosity >= verb: + self.log.say("@@@ Configuration stack %s @@@\n" % msg) + self.cfgs.dump_defs(self.log.log_file, prefix="\t") + if self.log.verbosity >= 99: + self.log.say("@@@@@@ Resolved configuration %s @@@@@@\n" % msg) + self.cfgs.dump(self.log.log_file, prefix="\t") diff --git a/t/moe/log.py b/t/moe/log.py index 4b8e586..0695b79 100644 --- a/t/moe/log.py +++ b/t/moe/log.py @@ -2,11 +2,23 @@ import sys -def say(msg): - sys.stdout.write(msg + "\n") +class MoeLog: -def verbose(msg): - sys.stdout.write(msg) + def __init__(self): + self.verbosity = 0 + self.progress_file = sys.stdout + self.log_file = None -def progress(msg): - sys.stdout.write(msg) + def say(self, msg): + if self.log_file: + self.log_file.write(msg) + + def verbose(self, msg): + if self.verbosity and self.log_file: + self.log_file.write(msg) + + def progress(self, msg): + if self.progress: + self.progress.write(msg) + +default = MoeLog() diff --git a/t/moe/pipeline.py b/t/moe/pipeline.py index 30c4c58..e7fc9a7 100644 --- a/t/moe/pipeline.py +++ b/t/moe/pipeline.py @@ -32,14 +32,14 @@ class MoePipeline: self.index = 0 while self.index < len(self.pipe): (pri,name,fun) = self.pipe[self.index] - moe.log.verbose(">> Running %s:%s\n" % (self.name,name)) + moe.log.default.verbose(">> Running %s:%s\n" % (self.name,name)) fun(*args) self.index += 1 self.index = -1 def add_hook(self, name): modname = "moe.hooks." + name - moe.log.verbose(">> Loading hook %s\n" % name) + moe.log.default.verbose(">> Loading hook %s\n" % name) if not sys.modules.has_key(modname): ## FIXME: Configuration variable for the hook directory? try: diff --git a/t/test.py b/t/test.py index 122d850..e736b32 100755 --- a/t/test.py +++ b/t/test.py @@ -17,20 +17,12 @@ e.builtins.set("TASK", "sum") e.builtins.set("CONTESTANT", "mj") e.init(overrides) -print "\nTask configuration:" -e.cfgs.dump() -print - if len(sys.argv) > 1: file = sys.argv[1] else: file = None moe.batch.locate(e, file) -print "\nConfiguration after per-extension overrides:" -e.cfgs.dump() -print - e.run() print "\nFinal meta file:" -- 2.39.2