X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;f=t%2Fmoe%2Ftestcase.py;h=36d335b7f72e7360b534e6f6a0e76cbcf16fc820;hb=a249fb797c925052e3ad692d887eb2004d66c455;hp=e49bd8c3d1428793fc373c66ebf057b0ca5217a9;hpb=a609f96f445c9d27420fa54a13eb7e587f1bd560;p=moe.git diff --git a/t/moe/testcase.py b/t/moe/testcase.py index e49bd8c..36d335b 100644 --- a/t/moe/testcase.py +++ b/t/moe/testcase.py @@ -6,9 +6,7 @@ import moe.config import moe.eval import moe.log import shutil - -def judge(e): - pass +import traceback def configure_test(e, test): e.cfgs = moe.config.MoeConfigStack(e.cfgs) @@ -16,19 +14,19 @@ def configure_test(e, test): e.test_builtins.set("TEST", test) e.cfgs.push(e.test_builtins) - test_cf = os.path.join(e.cfgs["PDIR"], test + ".config") + test_cf = os.path.join(e["PDIR"], test + ".config") if os.path.exists(test_cf): cfg = moe.config.MoeConfig(name=test_cf, type="test") e.cfgs.push(cfg) e.cfgs.apply_overrides("TEST_" + test + "_") - ext = e.cfgs["EXT"] + ext = e["EXT"] if ext != "": e.cfgs.apply_overrides("EXT_" + ext + "_") log = moe.log.MoeLog() log.verbosity = e.log.verbosity - log.open(os.path.join(e.cfgs["TDIR"], test + ".log")) + log.open(os.path.join(e["TDIR"], test + ".log")) log.say("Test case %s\n\n" % test) e.log = log moe.log.default = log @@ -40,7 +38,7 @@ def configure_test(e, test): e.stat.get_list("tests").append(e.test_stat) def collect_status(e): - sf = os.path.join(e.cfgs["TDIR"], e.cfgs["TESTCASE_STATUS"]) + sf = os.path.join(e["TDIR"], e["TESTCASE_STATUS"]) if os.path.exists(sf): e.log.verbose("Reading status from %s\n" % sf) stat = moe.status.MoeStatus() @@ -53,12 +51,42 @@ def collect_status(e): else: e.log.verbose("No status file present\n") +def tmpname(e): + return os.path.join(e["TDIR"], e["TEST"] + ".tmp") + +def collect_tmp_line(tmp_file): + if os.path.exists(tmp_file): + f = open(tmp_file, "r") + v = f.readline().rstrip("\n") + f.close() + os.unlink(tmp_file) + else: + v = "" + return v + +def collect_verdict(e, verdict_file): + v = collect_tmp_line(verdict_file) + if len(v) >= 4 and v[0].isalnum and v[1].isalnum and v[2] == ":" and v[3] == " ": + e.test_stat["status"] = v[0:2] + e.log.verbose("Judge's status: %s\n" % v[0:2]) + v = v[4:] + v.strip() + if v != "": + e.test_stat["message"] = v + e.log.verbose("Judge's verdict: %s\n" % v) + +def collect_points(e): + p = collect_tmp_line(os.path.join(e["TDIR"], e["TESTCASE_PTS"])) + if p != "": + e.log.verbose("Judge has supplied points: %s\n" % p) + e.test_stat["points"] = p + def setup(e): - pdir = e.cfgs["PDIR"] - tdir = e.cfgs["TDIR"] - inn = e.cfgs["TESTCASE_IN"] - out = e.cfgs["TESTCASE_OUT"] - ok = e.cfgs["TESTCASE_OK"] + pdir = e["PDIR"] + tdir = e["TDIR"] + inn = e["TESTCASE_IN"] + out = e["TESTCASE_OUT"] + ok = e["TESTCASE_OK"] if os.path.exists(os.path.join(pdir, inn)): moe.util.link_or_copy(os.path.join(pdir, inn), os.path.join(tdir, inn)) @@ -66,33 +94,71 @@ def setup(e): moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok)) def judge(e): - judge = e.cfgs["OUTPUT_CHECK"] - if judge == "": + cmd = e["OUTPUT_CHECK"] + if cmd == "": return + verdict_file = tmpname(e) + cmd = "exec 2>%s ; %s" % (verdict_file,cmd) e.log.progress(" ") - e.log.verbose("Checking output: %s\n" % judge) + e.log.verbose("Checking output: %s\n" % cmd) e.log.flush() - rc = os.system(judge) - ## FIXME: parse stderr of the judge to get the status message + rc = os.system(cmd) + collect_verdict(e, verdict_file) + collect_points(e) collect_status(e) if os.WIFEXITED(rc): if os.WEXITSTATUS(rc) == 0: return elif os.WEXITSTATUS(rc) == 1: - raise moe.TestErr("Wrong answer", "WA") - raise moe.MoeErr("Judge failure") + raise moe.TestError("Wrong answer", "WA") + raise moe.MoeError("Judge failure") def points(e): - ## FIXME: check $TEST.pts if e.test_stat["points"] is None: - e.test_stat["points"] = e.cfgs["POINTS_PER_TEST"] + e.test_stat["points"] = e["POINTS_PER_TEST"] + +def filter(e): + cmd = e["OUTPUT_FILTER"] + if cmd == "": + return + + os.rename(os.path.join(e["TDIR"], e["TESTCASE_OUT"]), os.path.join(e["TDIR"], e["TESTCASE_RAW"])) + e.log.progress(" ") + e.log.verbose("Filtering output: %s\n" % cmd) + e.log.flush() + rc = os.system(cmd) + if os.WIFEXITED(rc) and os.WEXITSTATUS(rc) == 0: + if not os.path.exists(os.path.join(e["TDIR"], e["TESTCASE_OUT"])): + raise moe.MoeError("Filter has generated no output") + else: + raise moe.MoeError("Filter failure") + +def syntax(e): + cmd = e["SYNTAX_CHECK"] + if cmd == "": + return + verdict_file = tmpname(e) + cmd = "exec 2>%s ; %s" % (verdict_file,cmd) + + e.log.progress(" ") + e.log.verbose("Checking syntax: %s\n" % cmd) + e.log.flush() + rc = os.system(cmd) + collect_verdict(e, verdict_file) + collect_status(e) + if os.WIFEXITED(rc): + if os.WEXITSTATUS(rc) == 0: + return + elif os.WEXITSTATUS(rc) == 1: + raise moe.TestError("Wrong syntax", "SY") + raise moe.MoeError("Syntax checker failure") def run_test(e, test): configure_test(e, test) ## FIXME: interactive tasks - e.test_pipe.configure(e.cfgs["TESTCASE_HOOKS"]) + e.test_pipe.configure(e["TESTCASE_HOOKS"]) if e.log.verbosity >= 2: e.test_pipe.dump(e.log.log_file, prefix="\t") e.test_pipe.run(e) @@ -100,8 +166,13 @@ def run_test(e, test): def wrap_run_test(e, test): try: run_test(e, test) - except moe.MoeErr, err: - raise moe.TestErr(err, "XX") + except moe.MoeError, err: + raise moe.TestError(err, "XX") + except Exception: + if e["DEBUG"]: + moe.log.fatal_exception() + traceback.print_exc(file = e.log.log_file) + raise moe.TestError("Internal exception", "XX") def conclude_test(e): stat = e.test_stat @@ -126,22 +197,23 @@ def conclude_test(e): e.log.say(msg) def run_tests(e): - ## FIXME: output filter - ## FIXME: syntax checks e.test_pipe.insert(0, "setup", setup) - e.test_pipe.insert(400, "judge", judge) - e.test_pipe.insert(500, "points", points) + e.test_pipe.insert(400, "filter", filter) + e.test_pipe.insert(500, "syntax", syntax) + e.test_pipe.insert(600, "judge", judge) + e.test_pipe.insert(700, "points", points) - for test in e.cfgs["TESTS"].split(): + for test in e["TESTS"].split(): e.log.progress("Test %s: " % test) old_cfgs = e.cfgs old_log = e.log try: wrap_run_test(e, test) - except moe.TestErr, err: + except moe.TestError, err: if not e.test_stat["status"]: e.test_stat["status"] = err.stat_code + if not e.test_stat["message"]: e.test_stat["message"] = err.message conclude_test(e)