From a609f96f445c9d27420fa54a13eb7e587f1bd560 Mon Sep 17 00:00:00 2001 From: Martin Mares Date: Wed, 12 Aug 2009 01:53:20 +0200 Subject: [PATCH] Status files are now handled almost properly --- doc/meta | 2 ++ t/config | 1 + t/moe/__init__.py | 3 ++ t/moe/batch.py | 21 +++++++------- t/moe/pipeline.py | 1 + t/moe/status.py | 3 ++ t/moe/testcase.py | 73 +++++++++++++++++++++++++++++++++++++++++------ t/test.py | 8 +++--- 8 files changed, 89 insertions(+), 23 deletions(-) diff --git a/doc/meta b/doc/meta index 63b7723..fce7e12 100644 --- a/doc/meta +++ b/doc/meta @@ -39,6 +39,8 @@ queue-done: timestamp of the finish of evaluation Added by the evaluator ~~~~~~~~~~~~~~~~~~~~~~ +lang: language of the submitted source (file-name extension; missing for open-data problems) +error: human-readable error message for the whole submit (e.g., unrecognized language) test( results of a single test -- mandatory: id: the name of the test diff --git a/t/config b/t/config index 0c5c43f..96ac31e 100644 --- a/t/config +++ b/t/config @@ -10,6 +10,7 @@ TASK_TYPE=batch TESTCASE_IN=${TEST}.in TESTCASE_OUT=${TEST}.out TESTCASE_OK=${TEST}.ok +TESTCASE_STATUS=${TEST}.stat # HOOKS # TESTCASE_HOOKS diff --git a/t/moe/__init__.py b/t/moe/__init__.py index 26e4f24..2ce5499 100644 --- a/t/moe/__init__.py +++ b/t/moe/__init__.py @@ -14,3 +14,6 @@ class SolutionErr(Exception): return self.message else: return "%s: %s" % (self.stat_code, self.message) + +class TestErr(SolutionErr): + pass diff --git a/t/moe/batch.py b/t/moe/batch.py index 4fb4f1f..4e3dcf9 100644 --- a/t/moe/batch.py +++ b/t/moe/batch.py @@ -63,6 +63,7 @@ def locate(e, filename=None): e.cfgs.apply_overrides("EXT_" + norm_ext + "_") e.stat["source"] = file + e.stat["lang"] = ext e.log.progress(file + "\n") def compile_init(e): @@ -85,7 +86,7 @@ def compile_run(e): rc = moe.box.run(e, e.cfgs["COMP_SANDBOX_OPTS"], cc) if rc > 0: e.log.progress("FAILED\n") - ## FIXME: status file + ## FIXME: status file ... or generate an exception? raise moe.pipeline.MoeAbortPipeline(200) moe.box.show(e, "compiler output") @@ -103,11 +104,10 @@ def test_in(e): in_type = e.cfgs["IN_TYPE"] or e.cfgs["IO_TYPE"] out_type = e.cfgs["OUT_TYPE"] or e.cfgs["IO_TYPE"] is_interactive = e.cfgs["TASK_TYPE"] == "interactive" - sandbox_opts = "-M" + os.path.join(tdir, e.cfgs["TEST"] + ".status") + sandbox_opts = "-M" + os.path.join(tdir, e.cfgs["TESTCASE_STATUS"]) if not os.path.exists(os.path.join(tdir, e.cfgs["EXE"])): - ## FIXME: status file - raise SolutionErr, "Compilation failed" + raise TestErr("Compilation failed", "CE") shutil.copyfile(os.path.join(tdir, e.cfgs["EXE"]), os.path.join(boxdir, e.cfgs["EXE"])) os.chmod(os.path.join(boxdir, e.cfgs["EXE"]), 0555) @@ -116,11 +116,11 @@ def test_in(e): e.log.verbose("Input file: %s (copied from %s)\n" % (in_name, os.path.join(e.cfgs["PDIR"], inn))) shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, in_name)) if not is_interactive: - sandbox_opts = " -i/dev/null" + sandbox_opts += " -i/dev/null" elif in_type == "stdio": e.log.verbose("Input file: (copied from %s)\n" % os.path.join(e.cfgs["PDIR"], inn)) shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, ".stdin")) - sandbox_opts = " -i.stdin" + sandbox_opts += " -i.stdin" elif in_type == "none": e.log.verbose("Input file: \n") if not is_interactive: @@ -153,10 +153,11 @@ def test_run(e): e.log.verbose("Memory limit: %s KB\n" % e.cfgs["MEM_LIMIT"]) moe.box.show(e, "test input") e.log.progress(" ") - moe.box.run(e, e.cfgs["TEST_SANDBOX_OPTS"], e.cfgs["TEST_EXEC_CMD"]) + rc = moe.box.run(e, e.cfgs["TEST_SANDBOX_OPTS"], e.cfgs["TEST_EXEC_CMD"]) + moe.testcase.collect_status(e) moe.box.show(e, "test output") - ## FIXME: Parse the status file and delete it - ### Check for runtime errors reported by the box + if rc > 0: + raise moe.TestErr("Wrong answer", "WA") def test_collect(e): tdir = e.cfgs["TDIR"] @@ -169,7 +170,7 @@ def test_collect(e): elif out_type == "stdio": out_path = ".stdout" if not os.path.exists(os.path.join(boxdir, out_path)): - raise moe.SolutionErr("No output file", "NO") + raise moe.TestErr("No output file", "NO") shutil.copyfile(os.path.join(boxdir, out_path), os.path.join(tdir, e.cfgs["TESTCASE_OUT"])) def tests(e): diff --git a/t/moe/pipeline.py b/t/moe/pipeline.py index 11d809d..7726110 100644 --- a/t/moe/pipeline.py +++ b/t/moe/pipeline.py @@ -49,6 +49,7 @@ class MoePipeline: moe.log.default.verbose(">> Skipping %s:%s\n" % (self.name,name)) self.index += 1 self.index = -1 + moe.log.default.verbose(">> Pipeline %s finished\n" % self.name) def add_hook(self, name): modname = "moe.hooks." + name diff --git a/t/moe/status.py b/t/moe/status.py index 936659e..3d72db4 100644 --- a/t/moe/status.py +++ b/t/moe/status.py @@ -26,6 +26,9 @@ class MoeStatus: def __setitem__(self, k, v): self.stat[k] = v + def keys(self): + return self.stat.keys() + def get_list(self, k): m = self.stat if not m.has_key(k): diff --git a/t/moe/testcase.py b/t/moe/testcase.py index fba4128..e49bd8c 100644 --- a/t/moe/testcase.py +++ b/t/moe/testcase.py @@ -31,9 +31,28 @@ def configure_test(e, test): log.open(os.path.join(e.cfgs["TDIR"], test + ".log")) log.say("Test case %s\n\n" % test) e.log = log + moe.log.default = log e.log_config(2, "for the test") + e.test_stat = moe.status.MoeStatus() + e.test_stat["id"] = test + e.stat.get_list("tests").append(e.test_stat) + +def collect_status(e): + sf = os.path.join(e.cfgs["TDIR"], e.cfgs["TESTCASE_STATUS"]) + if os.path.exists(sf): + e.log.verbose("Reading status from %s\n" % sf) + stat = moe.status.MoeStatus() + stat.read(name=sf) + if e.log.verbosity > 1: + stat.write_nested(e.log.log_file, 1) + for k in stat.keys(): + e.test_stat[k] = stat[k] + os.unlink(sf) + else: + e.log.verbose("No status file present\n") + def setup(e): pdir = e.cfgs["PDIR"] tdir = e.cfgs["TDIR"] @@ -55,14 +74,20 @@ def judge(e): e.log.verbose("Checking output: %s\n" % judge) e.log.flush() rc = os.system(judge) - ## FIXME: The judge might want to return a status file + ## FIXME: parse stderr of the judge to get the status message + collect_status(e) if os.WIFEXITED(rc): if os.WEXITSTATUS(rc) == 0: return elif os.WEXITSTATUS(rc) == 1: - raise moe.SolutionErr("Wrong answer", "WA") + raise moe.TestErr("Wrong answer", "WA") raise moe.MoeErr("Judge failure") +def points(e): + ## FIXME: check $TEST.pts + if e.test_stat["points"] is None: + e.test_stat["points"] = e.cfgs["POINTS_PER_TEST"] + def run_test(e, test): configure_test(e, test) @@ -72,12 +97,40 @@ def run_test(e, test): e.test_pipe.dump(e.log.log_file, prefix="\t") e.test_pipe.run(e) - e.log.progress("OK\n") +def wrap_run_test(e, test): + try: + run_test(e, test) + except moe.MoeErr, err: + raise moe.TestErr(err, "XX") + +def conclude_test(e): + stat = e.test_stat + if not stat["points"]: + stat["points"] = 0 + + if e.log.verbosity > 1: + e.log.verbose("Final status:\n") + stat.write_nested(e.log.log_file, 1) + + if stat["status"]: + msg = "%s: %s" % (stat["status"], e.test_stat["message"]) + else: + msg = "OK" + msg += " (%s points" % stat["points"] + if stat["time"]: + msg += ", %s s" % stat["time"] + if stat["mem"]: + msg += ", %d MB" % ((int(stat["mem"]) + 524288) // 1048576) + msg += ")\n" + e.log.progress(msg) + e.log.say(msg) def run_tests(e): ## FIXME: output filter + ## FIXME: syntax checks e.test_pipe.insert(0, "setup", setup) e.test_pipe.insert(400, "judge", judge) + e.test_pipe.insert(500, "points", points) for test in e.cfgs["TESTS"].split(): e.log.progress("Test %s: " % test) @@ -85,12 +138,14 @@ def run_tests(e): old_log = e.log try: - run_test(e, test) - except moe.MoeErr, err: - e.log.progress("FAILED: %s\n" % err) - ## FIXME: write it to the status file - except moe.SolutionErr, err: - e.log.progress("%s\n" % err) + wrap_run_test(e, test) + except moe.TestErr, err: + if not e.test_stat["status"]: + e.test_stat["status"] = err.stat_code + e.test_stat["message"] = err.message + + conclude_test(e) e.cfgs = old_cfgs e.log = old_log + moe.log.default = old_log diff --git a/t/test.py b/t/test.py index 8b8f3f1..07a220f 100755 --- a/t/test.py +++ b/t/test.py @@ -5,6 +5,7 @@ sys.path.append('.') import moe import moe.config +import moe.log import moe.eval import moe.pipeline import moe.batch @@ -33,9 +34,8 @@ except moe.MoeErr, err: e.log.shout("FATAL: %s\n" % err) sys.exit(1) except moe.SolutionErr, err: - ## FIXME: In this case, we might write the status file + e.stat["error"] = err e.log.shout("%s\n" % err) - sys.exit(1) -print "\nFinal status file:" -e.stat.write() +moe.log.default.progress_file.write("\nFinal status file:\n") +e.stat.write(file=moe.log.default.progress_file) -- 2.39.2