Implemented syntax checkers and output filters

diff --git a/doc/meta b/doc/meta
index fce7e129637d030a68b75ebd0420a75ed9ca6a40..424c882986d15f98ea2236e5e8a64b8d07eb40ad 100644 (file)
--- a/doc/meta
+++ b/doc/meta
@@ -55,6 +55,7 @@ test(         results of a single test
                        WA = wrong answer
                        PA = partial answer
                        NO = no output file generated
+                       SY = syntax error
                        PE = protocol error (in case of interactive tasks)
                        XX = internal error (e.g., error when calling judge)
   message:     human-readable status message (not intended for machine parsing)

diff --git a/t/config b/t/config
index 2ad11eaee3fc2d55b6d3d9496ef48bdfd65d8ecd..411443fc32ab39839f67ff1dd79353c00227f3d4 100644 (file)
--- a/t/config
+++ b/t/config
@@ -10,6 +10,7 @@ TESTCASE_IN=${TEST}.in
 TESTCASE_OUT=${TEST}.out
 TESTCASE_OK=${TEST}.ok
 TESTCASE_STATUS=${TEST}.stat
+TESTCASE_RAW=${TEST}.raw
 # backward compatibility
 TESTCASE_PTS=${TEST}.pts
 
@@ -131,3 +132,21 @@ TEST_SANDBOX_OPTS=-a2 -f -m$MEM_LIMIT -k$STACK_LIMIT -t$TIME_LIMIT $BOX_EXTRAS $
 
 # Extra options to be overridden in task configuration
 BOX_EXTRAS=
+
+### Hook priorities:
+
+# Task pipeline for batch and interactive tasks:
+#      100     compile-init
+#      150     compile-run
+#      190     compile-done
+#      200     batch-tests
+
+# Test pipeline:
+#      000     setup           copy input and correct output to $TDIR
+#      100     prepare         copy input and executables to the sandbox
+#      200     run             run inside the sandbox
+#      300     collect         copy output out of the sandbox
+#      400     filter          filter the output ($OUTPUT_FILTER)
+#      500     syntax          check syntax of the output ($SYNTAX_CHECK)
+#      600     judge           check correctness of the output ($OUTPUT_CHECK)
+#      700     points          award $POINTS_PER_TEST points unless already done

diff --git a/t/moe/testcase.py b/t/moe/testcase.py
index d1184d30a8faf218a1c3e0fb4bcca3c44b8ddb98..e129a5305d88ac87d40ac9b3638302a561d8d307 100644 (file)
--- a/t/moe/testcase.py
+++ b/t/moe/testcase.py
@@ -117,6 +117,42 @@ def points(e):
     if e.test_stat["points"] is None:
        e.test_stat["points"] = e["POINTS_PER_TEST"]
 
+def filter(e):
+    cmd = e["OUTPUT_FILTER"]
+    if cmd == "":
+       return
+
+    os.rename(os.path.join(e["TDIR"], e["TESTCASE_OUT"]), os.path.join(e["TDIR"], e["TESTCASE_RAW"]))
+    e.log.progress("<filter> ")
+    e.log.verbose("Filtering output: %s\n" % cmd)
+    e.log.flush()
+    rc = os.system(cmd)
+    if os.WIFEXITED(rc) and os.WEXITSTATUS(rc) == 0:
+       if not os.path.exists(os.path.join(e["TDIR"], e["TESTCASE_OUT"])):
+           raise moe.MoeError("Filter has generated no output")
+    else:
+       raise moe.MoeError("Filter failure")
+
+def syntax(e):
+    cmd = e["SYNTAX_CHECK"]
+    if cmd == "":
+       return
+    verdict_file = tmpname(e)
+    cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
+
+    e.log.progress("<syntax> ")
+    e.log.verbose("Checking syntax: %s\n" % cmd)
+    e.log.flush()
+    rc = os.system(cmd)
+    collect_verdict(e, verdict_file)
+    collect_status(e)
+    if os.WIFEXITED(rc):
+       if os.WEXITSTATUS(rc) == 0:
+           return
+       elif os.WEXITSTATUS(rc) == 1:
+           raise moe.TestError("Wrong syntax", "SY")
+    raise moe.MoeError("Syntax checker failure")
+
 def run_test(e, test):
     configure_test(e, test)
 
@@ -155,11 +191,11 @@ def conclude_test(e):
     e.log.say(msg)
 
 def run_tests(e):
-    ## FIXME: output filter
-    ## FIXME: syntax checks
     e.test_pipe.insert(0, "setup", setup)
-    e.test_pipe.insert(400, "judge", judge)
-    e.test_pipe.insert(500, "points", points)
+    e.test_pipe.insert(400, "filter", filter)
+    e.test_pipe.insert(500, "syntax", syntax)
+    e.test_pipe.insert(600, "judge", judge)
+    e.test_pipe.insert(700, "points", points)
 
     for test in e["TESTS"].split():
        e.log.progress("Test %s: " % test)