X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;f=t%2Fmoe%2Ftestcase.py;h=36d335b7f72e7360b534e6f6a0e76cbcf16fc820;hb=a249fb797c925052e3ad692d887eb2004d66c455;hp=fba4128ca14b9f22636e993f80fc8ebc8862e83f;hpb=dcc852db06f099b6c96b681c79832d41777da334;p=moe.git diff --git a/t/moe/testcase.py b/t/moe/testcase.py index fba4128..36d335b 100644 --- a/t/moe/testcase.py +++ b/t/moe/testcase.py @@ -6,9 +6,7 @@ import moe.config import moe.eval import moe.log import shutil - -def judge(e): - pass +import traceback def configure_test(e, test): e.cfgs = moe.config.MoeConfigStack(e.cfgs) @@ -16,30 +14,79 @@ def configure_test(e, test): e.test_builtins.set("TEST", test) e.cfgs.push(e.test_builtins) - test_cf = os.path.join(e.cfgs["PDIR"], test + ".config") + test_cf = os.path.join(e["PDIR"], test + ".config") if os.path.exists(test_cf): cfg = moe.config.MoeConfig(name=test_cf, type="test") e.cfgs.push(cfg) e.cfgs.apply_overrides("TEST_" + test + "_") - ext = e.cfgs["EXT"] + ext = e["EXT"] if ext != "": e.cfgs.apply_overrides("EXT_" + ext + "_") log = moe.log.MoeLog() log.verbosity = e.log.verbosity - log.open(os.path.join(e.cfgs["TDIR"], test + ".log")) + log.open(os.path.join(e["TDIR"], test + ".log")) log.say("Test case %s\n\n" % test) e.log = log + moe.log.default = log e.log_config(2, "for the test") + e.test_stat = moe.status.MoeStatus() + e.test_stat["id"] = test + e.stat.get_list("tests").append(e.test_stat) + +def collect_status(e): + sf = os.path.join(e["TDIR"], e["TESTCASE_STATUS"]) + if os.path.exists(sf): + e.log.verbose("Reading status from %s\n" % sf) + stat = moe.status.MoeStatus() + stat.read(name=sf) + if e.log.verbosity > 1: + stat.write_nested(e.log.log_file, 1) + for k in stat.keys(): + e.test_stat[k] = stat[k] + os.unlink(sf) + else: + e.log.verbose("No status file present\n") + +def tmpname(e): + return os.path.join(e["TDIR"], e["TEST"] + ".tmp") + +def collect_tmp_line(tmp_file): + if os.path.exists(tmp_file): + f = open(tmp_file, "r") + v = f.readline().rstrip("\n") + f.close() + os.unlink(tmp_file) + else: + v = "" + return v + +def collect_verdict(e, verdict_file): + v = collect_tmp_line(verdict_file) + if len(v) >= 4 and v[0].isalnum and v[1].isalnum and v[2] == ":" and v[3] == " ": + e.test_stat["status"] = v[0:2] + e.log.verbose("Judge's status: %s\n" % v[0:2]) + v = v[4:] + v.strip() + if v != "": + e.test_stat["message"] = v + e.log.verbose("Judge's verdict: %s\n" % v) + +def collect_points(e): + p = collect_tmp_line(os.path.join(e["TDIR"], e["TESTCASE_PTS"])) + if p != "": + e.log.verbose("Judge has supplied points: %s\n" % p) + e.test_stat["points"] = p + def setup(e): - pdir = e.cfgs["PDIR"] - tdir = e.cfgs["TDIR"] - inn = e.cfgs["TESTCASE_IN"] - out = e.cfgs["TESTCASE_OUT"] - ok = e.cfgs["TESTCASE_OK"] + pdir = e["PDIR"] + tdir = e["TDIR"] + inn = e["TESTCASE_IN"] + out = e["TESTCASE_OUT"] + ok = e["TESTCASE_OK"] if os.path.exists(os.path.join(pdir, inn)): moe.util.link_or_copy(os.path.join(pdir, inn), os.path.join(tdir, inn)) @@ -47,50 +94,130 @@ def setup(e): moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok)) def judge(e): - judge = e.cfgs["OUTPUT_CHECK"] - if judge == "": + cmd = e["OUTPUT_CHECK"] + if cmd == "": return + verdict_file = tmpname(e) + cmd = "exec 2>%s ; %s" % (verdict_file,cmd) e.log.progress(" ") - e.log.verbose("Checking output: %s\n" % judge) + e.log.verbose("Checking output: %s\n" % cmd) + e.log.flush() + rc = os.system(cmd) + collect_verdict(e, verdict_file) + collect_points(e) + collect_status(e) + if os.WIFEXITED(rc): + if os.WEXITSTATUS(rc) == 0: + return + elif os.WEXITSTATUS(rc) == 1: + raise moe.TestError("Wrong answer", "WA") + raise moe.MoeError("Judge failure") + +def points(e): + if e.test_stat["points"] is None: + e.test_stat["points"] = e["POINTS_PER_TEST"] + +def filter(e): + cmd = e["OUTPUT_FILTER"] + if cmd == "": + return + + os.rename(os.path.join(e["TDIR"], e["TESTCASE_OUT"]), os.path.join(e["TDIR"], e["TESTCASE_RAW"])) + e.log.progress(" ") + e.log.verbose("Filtering output: %s\n" % cmd) + e.log.flush() + rc = os.system(cmd) + if os.WIFEXITED(rc) and os.WEXITSTATUS(rc) == 0: + if not os.path.exists(os.path.join(e["TDIR"], e["TESTCASE_OUT"])): + raise moe.MoeError("Filter has generated no output") + else: + raise moe.MoeError("Filter failure") + +def syntax(e): + cmd = e["SYNTAX_CHECK"] + if cmd == "": + return + verdict_file = tmpname(e) + cmd = "exec 2>%s ; %s" % (verdict_file,cmd) + + e.log.progress(" ") + e.log.verbose("Checking syntax: %s\n" % cmd) e.log.flush() - rc = os.system(judge) - ## FIXME: The judge might want to return a status file + rc = os.system(cmd) + collect_verdict(e, verdict_file) + collect_status(e) if os.WIFEXITED(rc): if os.WEXITSTATUS(rc) == 0: return elif os.WEXITSTATUS(rc) == 1: - raise moe.SolutionErr("Wrong answer", "WA") - raise moe.MoeErr("Judge failure") + raise moe.TestError("Wrong syntax", "SY") + raise moe.MoeError("Syntax checker failure") def run_test(e, test): configure_test(e, test) ## FIXME: interactive tasks - e.test_pipe.configure(e.cfgs["TESTCASE_HOOKS"]) + e.test_pipe.configure(e["TESTCASE_HOOKS"]) if e.log.verbosity >= 2: e.test_pipe.dump(e.log.log_file, prefix="\t") e.test_pipe.run(e) - e.log.progress("OK\n") +def wrap_run_test(e, test): + try: + run_test(e, test) + except moe.MoeError, err: + raise moe.TestError(err, "XX") + except Exception: + if e["DEBUG"]: + moe.log.fatal_exception() + traceback.print_exc(file = e.log.log_file) + raise moe.TestError("Internal exception", "XX") + +def conclude_test(e): + stat = e.test_stat + if not stat["points"]: + stat["points"] = 0 + + if e.log.verbosity > 1: + e.log.verbose("Final status:\n") + stat.write_nested(e.log.log_file, 1) + + if stat["status"]: + msg = "%s: %s" % (stat["status"], e.test_stat["message"]) + else: + msg = "OK" + msg += " (%s points" % stat["points"] + if stat["time"]: + msg += ", %s s" % stat["time"] + if stat["mem"]: + msg += ", %d MB" % ((int(stat["mem"]) + 524288) // 1048576) + msg += ")\n" + e.log.progress(msg) + e.log.say(msg) def run_tests(e): - ## FIXME: output filter e.test_pipe.insert(0, "setup", setup) - e.test_pipe.insert(400, "judge", judge) + e.test_pipe.insert(400, "filter", filter) + e.test_pipe.insert(500, "syntax", syntax) + e.test_pipe.insert(600, "judge", judge) + e.test_pipe.insert(700, "points", points) - for test in e.cfgs["TESTS"].split(): + for test in e["TESTS"].split(): e.log.progress("Test %s: " % test) old_cfgs = e.cfgs old_log = e.log try: - run_test(e, test) - except moe.MoeErr, err: - e.log.progress("FAILED: %s\n" % err) - ## FIXME: write it to the status file - except moe.SolutionErr, err: - e.log.progress("%s\n" % err) + wrap_run_test(e, test) + except moe.TestError, err: + if not e.test_stat["status"]: + e.test_stat["status"] = err.stat_code + if not e.test_stat["message"]: + e.test_stat["message"] = err.message + + conclude_test(e) e.cfgs = old_cfgs e.log = old_log + moe.log.default = old_log