From: Martin Mares <mj@ucw.cz>
Date: Fri, 14 Aug 2009 08:25:33 +0000 (+0200)
Subject: Implemented syntax checkers and output filters
X-Git-Tag: python-dummy-working~70
X-Git-Url: http://mj.ucw.cz/gitweb/?a=commitdiff_plain;h=44aa05d5c58e206ec005635ad86a75c2c70f5a75;p=eval.git

Implemented syntax checkers and output filters
---

diff --git a/doc/meta b/doc/meta
index fce7e12..424c882 100644
--- a/doc/meta
+++ b/doc/meta
@@ -55,6 +55,7 @@ test(		results of a single test
 			WA = wrong answer
 			PA = partial answer
 			NO = no output file generated
+			SY = syntax error
 			PE = protocol error (in case of interactive tasks)
 			XX = internal error (e.g., error when calling judge)
   message:	human-readable status message (not intended for machine parsing)
diff --git a/t/config b/t/config
index 2ad11ea..411443f 100644
--- a/t/config
+++ b/t/config
@@ -10,6 +10,7 @@ TESTCASE_IN=${TEST}.in
 TESTCASE_OUT=${TEST}.out
 TESTCASE_OK=${TEST}.ok
 TESTCASE_STATUS=${TEST}.stat
+TESTCASE_RAW=${TEST}.raw
 # backward compatibility
 TESTCASE_PTS=${TEST}.pts
 
@@ -131,3 +132,21 @@ TEST_SANDBOX_OPTS=-a2 -f -m$MEM_LIMIT -k$STACK_LIMIT -t$TIME_LIMIT $BOX_EXTRAS $
 
 # Extra options to be overridden in task configuration
 BOX_EXTRAS=
+
+### Hook priorities:
+
+# Task pipeline for batch and interactive tasks:
+#	100	compile-init
+#	150	compile-run
+#	190	compile-done
+#	200	batch-tests
+
+# Test pipeline:
+#	000	setup		copy input and correct output to $TDIR
+#	100	prepare		copy input and executables to the sandbox
+#	200	run		run inside the sandbox
+#	300	collect		copy output out of the sandbox
+#	400	filter		filter the output ($OUTPUT_FILTER)
+#	500	syntax		check syntax of the output ($SYNTAX_CHECK)
+#	600	judge		check correctness of the output ($OUTPUT_CHECK)
+#	700	points		award $POINTS_PER_TEST points unless already done
diff --git a/t/moe/testcase.py b/t/moe/testcase.py
index d1184d3..e129a53 100644
--- a/t/moe/testcase.py
+++ b/t/moe/testcase.py
@@ -117,6 +117,42 @@ def points(e):
     if e.test_stat["points"] is None:
 	e.test_stat["points"] = e["POINTS_PER_TEST"]
 
+def filter(e):
+    cmd = e["OUTPUT_FILTER"]
+    if cmd == "":
+	return
+
+    os.rename(os.path.join(e["TDIR"], e["TESTCASE_OUT"]), os.path.join(e["TDIR"], e["TESTCASE_RAW"]))
+    e.log.progress("<filter> ")
+    e.log.verbose("Filtering output: %s\n" % cmd)
+    e.log.flush()
+    rc = os.system(cmd)
+    if os.WIFEXITED(rc) and os.WEXITSTATUS(rc) == 0:
+	if not os.path.exists(os.path.join(e["TDIR"], e["TESTCASE_OUT"])):
+	    raise moe.MoeError("Filter has generated no output")
+    else:
+	raise moe.MoeError("Filter failure")
+
+def syntax(e):
+    cmd = e["SYNTAX_CHECK"]
+    if cmd == "":
+	return
+    verdict_file = tmpname(e)
+    cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
+
+    e.log.progress("<syntax> ")
+    e.log.verbose("Checking syntax: %s\n" % cmd)
+    e.log.flush()
+    rc = os.system(cmd)
+    collect_verdict(e, verdict_file)
+    collect_status(e)
+    if os.WIFEXITED(rc):
+	if os.WEXITSTATUS(rc) == 0:
+	    return
+	elif os.WEXITSTATUS(rc) == 1:
+	    raise moe.TestError("Wrong syntax", "SY")
+    raise moe.MoeError("Syntax checker failure")
+
 def run_test(e, test):
     configure_test(e, test)
 
@@ -155,11 +191,11 @@ def conclude_test(e):
     e.log.say(msg)
 
 def run_tests(e):
-    ## FIXME: output filter
-    ## FIXME: syntax checks
     e.test_pipe.insert(0, "setup", setup)
-    e.test_pipe.insert(400, "judge", judge)
-    e.test_pipe.insert(500, "points", points)
+    e.test_pipe.insert(400, "filter", filter)
+    e.test_pipe.insert(500, "syntax", syntax)
+    e.test_pipe.insert(600, "judge", judge)
+    e.test_pipe.insert(700, "points", points)
 
     for test in e["TESTS"].split():
 	e.log.progress("Test %s: " % test)