import moe.eval
import moe.log
import shutil
-
-def judge(e):
- pass
+import traceback
def configure_test(e, test):
e.cfgs = moe.config.MoeConfigStack(e.cfgs)
e.test_builtins.set("TEST", test)
e.cfgs.push(e.test_builtins)
- test_cf = os.path.join(e.cfgs["PDIR"], test + ".config")
+ test_cf = os.path.join(e["PDIR"], test + ".config")
if os.path.exists(test_cf):
cfg = moe.config.MoeConfig(name=test_cf, type="test")
e.cfgs.push(cfg)
e.cfgs.apply_overrides("TEST_" + test + "_")
- ext = e.cfgs["EXT"]
+ ext = e["EXT"]
if ext != "":
e.cfgs.apply_overrides("EXT_" + ext + "_")
log = moe.log.MoeLog()
log.verbosity = e.log.verbosity
- log.open(os.path.join(e.cfgs["TDIR"], test + ".log"))
+ log.open(os.path.join(e["TDIR"], test + ".log"))
log.say("Test case %s\n\n" % test)
e.log = log
+ moe.log.default = log
e.log_config(2, "for the test")
+ e.test_stat = moe.status.MoeStatus()
+ e.test_stat["id"] = test
+ e.stat.get_list("tests").append(e.test_stat)
+
+def collect_status(e):
+ sf = os.path.join(e["TDIR"], e["TESTCASE_STATUS"])
+ if os.path.exists(sf):
+ e.log.verbose("Reading status from %s\n" % sf)
+ stat = moe.status.MoeStatus()
+ stat.read(name=sf)
+ if e.log.verbosity > 1:
+ stat.write_nested(e.log.log_file, 1)
+ for k in stat.keys():
+ e.test_stat[k] = stat[k]
+ os.unlink(sf)
+ else:
+ e.log.verbose("No status file present\n")
+
+def tmpname(e):
+ return os.path.join(e["TDIR"], e["TEST"] + ".tmp")
+
+def collect_tmp_line(tmp_file):
+ if os.path.exists(tmp_file):
+ f = open(tmp_file, "r")
+ v = f.readline().rstrip("\n")
+ f.close()
+ os.unlink(tmp_file)
+ else:
+ v = ""
+ return v
+
+def collect_verdict(e, verdict_file):
+ v = collect_tmp_line(verdict_file)
+ if len(v) >= 4 and v[0].isalnum and v[1].isalnum and v[2] == ":" and v[3] == " ":
+ e.test_stat["status"] = v[0:2]
+ e.log.verbose("Judge's status: %s\n" % v[0:2])
+ v = v[4:]
+ v.strip()
+ if v != "":
+ e.test_stat["message"] = v
+ e.log.verbose("Judge's verdict: %s\n" % v)
+
+def collect_points(e):
+ p = collect_tmp_line(os.path.join(e["TDIR"], e["TESTCASE_PTS"]))
+ if p != "":
+ e.log.verbose("Judge has supplied points: %s\n" % p)
+ e.test_stat["points"] = p
+
def setup(e):
- pdir = e.cfgs["PDIR"]
- tdir = e.cfgs["TDIR"]
- inn = e.cfgs["TESTCASE_IN"]
- out = e.cfgs["TESTCASE_OUT"]
- ok = e.cfgs["TESTCASE_OK"]
+ pdir = e["PDIR"]
+ tdir = e["TDIR"]
+ inn = e["TESTCASE_IN"]
+ out = e["TESTCASE_OUT"]
+ ok = e["TESTCASE_OK"]
if os.path.exists(os.path.join(pdir, inn)):
moe.util.link_or_copy(os.path.join(pdir, inn), os.path.join(tdir, inn))
moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok))
def judge(e):
- judge = e.cfgs["OUTPUT_CHECK"]
- if judge == "":
+ cmd = e["OUTPUT_CHECK"]
+ if cmd == "":
return
+ verdict_file = tmpname(e)
+ cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
e.log.progress("<check> ")
- e.log.verbose("Checking output: %s\n" % judge)
+ e.log.verbose("Checking output: %s\n" % cmd)
+ e.log.flush()
+ rc = os.system(cmd)
+ collect_verdict(e, verdict_file)
+ collect_points(e)
+ collect_status(e)
+ if os.WIFEXITED(rc):
+ if os.WEXITSTATUS(rc) == 0:
+ return
+ elif os.WEXITSTATUS(rc) == 1:
+ raise moe.TestError("Wrong answer", "WA")
+ raise moe.MoeError("Judge failure")
+
+def points(e):
+ if e.test_stat["points"] is None:
+ e.test_stat["points"] = e["POINTS_PER_TEST"]
+
+def filter(e):
+ cmd = e["OUTPUT_FILTER"]
+ if cmd == "":
+ return
+
+ os.rename(os.path.join(e["TDIR"], e["TESTCASE_OUT"]), os.path.join(e["TDIR"], e["TESTCASE_RAW"]))
+ e.log.progress("<filter> ")
+ e.log.verbose("Filtering output: %s\n" % cmd)
+ e.log.flush()
+ rc = os.system(cmd)
+ if os.WIFEXITED(rc) and os.WEXITSTATUS(rc) == 0:
+ if not os.path.exists(os.path.join(e["TDIR"], e["TESTCASE_OUT"])):
+ raise moe.MoeError("Filter has generated no output")
+ else:
+ raise moe.MoeError("Filter failure")
+
+def syntax(e):
+ cmd = e["SYNTAX_CHECK"]
+ if cmd == "":
+ return
+ verdict_file = tmpname(e)
+ cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
+
+ e.log.progress("<syntax> ")
+ e.log.verbose("Checking syntax: %s\n" % cmd)
e.log.flush()
- rc = os.system(judge)
- ## FIXME: The judge might want to return a status file
+ rc = os.system(cmd)
+ collect_verdict(e, verdict_file)
+ collect_status(e)
if os.WIFEXITED(rc):
if os.WEXITSTATUS(rc) == 0:
return
elif os.WEXITSTATUS(rc) == 1:
- raise moe.SolutionErr("Wrong answer", "WA")
- raise moe.MoeErr("Judge failure")
+ raise moe.TestError("Wrong syntax", "SY")
+ raise moe.MoeError("Syntax checker failure")
def run_test(e, test):
configure_test(e, test)
## FIXME: interactive tasks
- e.test_pipe.configure(e.cfgs["TESTCASE_HOOKS"])
+ e.test_pipe.configure(e["TESTCASE_HOOKS"])
if e.log.verbosity >= 2:
e.test_pipe.dump(e.log.log_file, prefix="\t")
e.test_pipe.run(e)
- e.log.progress("OK\n")
+def wrap_run_test(e, test):
+ try:
+ run_test(e, test)
+ except moe.MoeError, err:
+ raise moe.TestError(err, "XX")
+ except Exception:
+ if e["DEBUG"]:
+ moe.log.fatal_exception()
+ traceback.print_exc(file = e.log.log_file)
+ raise moe.TestError("Internal exception", "XX")
+
+def conclude_test(e):
+ stat = e.test_stat
+ if not stat["points"]:
+ stat["points"] = 0
+
+ if e.log.verbosity > 1:
+ e.log.verbose("Final status:\n")
+ stat.write_nested(e.log.log_file, 1)
+
+ if stat["status"]:
+ msg = "%s: %s" % (stat["status"], e.test_stat["message"])
+ else:
+ msg = "OK"
+ msg += " (%s points" % stat["points"]
+ if stat["time"]:
+ msg += ", %s s" % stat["time"]
+ if stat["mem"]:
+ msg += ", %d MB" % ((int(stat["mem"]) + 524288) // 1048576)
+ msg += ")\n"
+ e.log.progress(msg)
+ e.log.say(msg)
def run_tests(e):
- ## FIXME: output filter
e.test_pipe.insert(0, "setup", setup)
- e.test_pipe.insert(400, "judge", judge)
+ e.test_pipe.insert(400, "filter", filter)
+ e.test_pipe.insert(500, "syntax", syntax)
+ e.test_pipe.insert(600, "judge", judge)
+ e.test_pipe.insert(700, "points", points)
- for test in e.cfgs["TESTS"].split():
+ for test in e["TESTS"].split():
e.log.progress("Test %s: " % test)
old_cfgs = e.cfgs
old_log = e.log
try:
- run_test(e, test)
- except moe.MoeErr, err:
- e.log.progress("FAILED: %s\n" % err)
- ## FIXME: write it to the status file
- except moe.SolutionErr, err:
- e.log.progress("%s\n" % err)
+ wrap_run_test(e, test)
+ except moe.TestError, err:
+ if not e.test_stat["status"]:
+ e.test_stat["status"] = err.stat_code
+ if not e.test_stat["message"]:
+ e.test_stat["message"] = err.message
+
+ conclude_test(e)
e.cfgs = old_cfgs
e.log = old_log
+ moe.log.default = old_log
projects / eval.git / blobdiff