X-Git-Url: http://mj.ucw.cz/gitweb/?a=blobdiff_plain;f=t%2Fmoe%2Ftestcase.py;h=36d335b7f72e7360b534e6f6a0e76cbcf16fc820;hb=750990d7dbbaf2740f290a26dab527ce4444cbfb;hp=358fa0b1365ec37600932cc3dbca6c10c6a5d40b;hpb=dbb3cc4764a4bba8dca262f63ed4eaa85543f588;p=moe.git diff --git a/t/moe/testcase.py b/t/moe/testcase.py index 358fa0b..36d335b 100644 --- a/t/moe/testcase.py +++ b/t/moe/testcase.py @@ -6,6 +6,7 @@ import moe.config import moe.eval import moe.log import shutil +import traceback def configure_test(e, test): e.cfgs = moe.config.MoeConfigStack(e.cfgs) @@ -13,19 +14,19 @@ def configure_test(e, test): e.test_builtins.set("TEST", test) e.cfgs.push(e.test_builtins) - test_cf = os.path.join(e.cfgs["PDIR"], test + ".config") + test_cf = os.path.join(e["PDIR"], test + ".config") if os.path.exists(test_cf): cfg = moe.config.MoeConfig(name=test_cf, type="test") e.cfgs.push(cfg) e.cfgs.apply_overrides("TEST_" + test + "_") - ext = e.cfgs["EXT"] + ext = e["EXT"] if ext != "": e.cfgs.apply_overrides("EXT_" + ext + "_") log = moe.log.MoeLog() log.verbosity = e.log.verbosity - log.open(os.path.join(e.cfgs["TDIR"], test + ".log")) + log.open(os.path.join(e["TDIR"], test + ".log")) log.say("Test case %s\n\n" % test) e.log = log moe.log.default = log @@ -37,7 +38,7 @@ def configure_test(e, test): e.stat.get_list("tests").append(e.test_stat) def collect_status(e): - sf = os.path.join(e.cfgs["TDIR"], e.cfgs["TESTCASE_STATUS"]) + sf = os.path.join(e["TDIR"], e["TESTCASE_STATUS"]) if os.path.exists(sf): e.log.verbose("Reading status from %s\n" % sf) stat = moe.status.MoeStatus() @@ -51,16 +52,20 @@ def collect_status(e): e.log.verbose("No status file present\n") def tmpname(e): - return os.path.join(e.cfgs["TDIR"], e.cfgs["TEST"] + ".tmp") + return os.path.join(e["TDIR"], e["TEST"] + ".tmp") -def collect_verdict(e, verdict_file): - try: - f = open(verdict_file, "r") +def collect_tmp_line(tmp_file): + if os.path.exists(tmp_file): + f = open(tmp_file, "r") v = f.readline().rstrip("\n") f.close() - except IOError, OSError: + os.unlink(tmp_file) + else: v = "" + return v +def collect_verdict(e, verdict_file): + v = collect_tmp_line(verdict_file) if len(v) >= 4 and v[0].isalnum and v[1].isalnum and v[2] == ":" and v[3] == " ": e.test_stat["status"] = v[0:2] e.log.verbose("Judge's status: %s\n" % v[0:2]) @@ -70,17 +75,18 @@ def collect_verdict(e, verdict_file): e.test_stat["message"] = v e.log.verbose("Judge's verdict: %s\n" % v) - try: - os.unlink(verdict_file) - except IOError, OSError: - pass +def collect_points(e): + p = collect_tmp_line(os.path.join(e["TDIR"], e["TESTCASE_PTS"])) + if p != "": + e.log.verbose("Judge has supplied points: %s\n" % p) + e.test_stat["points"] = p def setup(e): - pdir = e.cfgs["PDIR"] - tdir = e.cfgs["TDIR"] - inn = e.cfgs["TESTCASE_IN"] - out = e.cfgs["TESTCASE_OUT"] - ok = e.cfgs["TESTCASE_OK"] + pdir = e["PDIR"] + tdir = e["TDIR"] + inn = e["TESTCASE_IN"] + out = e["TESTCASE_OUT"] + ok = e["TESTCASE_OK"] if os.path.exists(os.path.join(pdir, inn)): moe.util.link_or_copy(os.path.join(pdir, inn), os.path.join(tdir, inn)) @@ -88,7 +94,7 @@ def setup(e): moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok)) def judge(e): - cmd = e.cfgs["OUTPUT_CHECK"] + cmd = e["OUTPUT_CHECK"] if cmd == "": return verdict_file = tmpname(e) @@ -99,6 +105,7 @@ def judge(e): e.log.flush() rc = os.system(cmd) collect_verdict(e, verdict_file) + collect_points(e) collect_status(e) if os.WIFEXITED(rc): if os.WEXITSTATUS(rc) == 0: @@ -108,15 +115,50 @@ def judge(e): raise moe.MoeError("Judge failure") def points(e): - ## FIXME: check $TEST.pts if e.test_stat["points"] is None: - e.test_stat["points"] = e.cfgs["POINTS_PER_TEST"] + e.test_stat["points"] = e["POINTS_PER_TEST"] + +def filter(e): + cmd = e["OUTPUT_FILTER"] + if cmd == "": + return + + os.rename(os.path.join(e["TDIR"], e["TESTCASE_OUT"]), os.path.join(e["TDIR"], e["TESTCASE_RAW"])) + e.log.progress(" ") + e.log.verbose("Filtering output: %s\n" % cmd) + e.log.flush() + rc = os.system(cmd) + if os.WIFEXITED(rc) and os.WEXITSTATUS(rc) == 0: + if not os.path.exists(os.path.join(e["TDIR"], e["TESTCASE_OUT"])): + raise moe.MoeError("Filter has generated no output") + else: + raise moe.MoeError("Filter failure") + +def syntax(e): + cmd = e["SYNTAX_CHECK"] + if cmd == "": + return + verdict_file = tmpname(e) + cmd = "exec 2>%s ; %s" % (verdict_file,cmd) + + e.log.progress(" ") + e.log.verbose("Checking syntax: %s\n" % cmd) + e.log.flush() + rc = os.system(cmd) + collect_verdict(e, verdict_file) + collect_status(e) + if os.WIFEXITED(rc): + if os.WEXITSTATUS(rc) == 0: + return + elif os.WEXITSTATUS(rc) == 1: + raise moe.TestError("Wrong syntax", "SY") + raise moe.MoeError("Syntax checker failure") def run_test(e, test): configure_test(e, test) ## FIXME: interactive tasks - e.test_pipe.configure(e.cfgs["TESTCASE_HOOKS"]) + e.test_pipe.configure(e["TESTCASE_HOOKS"]) if e.log.verbosity >= 2: e.test_pipe.dump(e.log.log_file, prefix="\t") e.test_pipe.run(e) @@ -126,6 +168,11 @@ def wrap_run_test(e, test): run_test(e, test) except moe.MoeError, err: raise moe.TestError(err, "XX") + except Exception: + if e["DEBUG"]: + moe.log.fatal_exception() + traceback.print_exc(file = e.log.log_file) + raise moe.TestError("Internal exception", "XX") def conclude_test(e): stat = e.test_stat @@ -150,13 +197,13 @@ def conclude_test(e): e.log.say(msg) def run_tests(e): - ## FIXME: output filter - ## FIXME: syntax checks e.test_pipe.insert(0, "setup", setup) - e.test_pipe.insert(400, "judge", judge) - e.test_pipe.insert(500, "points", points) + e.test_pipe.insert(400, "filter", filter) + e.test_pipe.insert(500, "syntax", syntax) + e.test_pipe.insert(600, "judge", judge) + e.test_pipe.insert(700, "points", points) - for test in e.cfgs["TESTS"].split(): + for test in e["TESTS"].split(): e.log.progress("Test %s: " % test) old_cfgs = e.cfgs old_log = e.log