Judges can return single-line verdicts on stderr instead of producing a status file

diff --git a/t/moe/testcase.py b/t/moe/testcase.py
index 158f3742c36098896d78d8481751742166db6659..358fa0b1365ec37600932cc3dbca6c10c6a5d40b 100644 (file)
--- a/t/moe/testcase.py
+++ b/t/moe/testcase.py
@@ -7,9 +7,6 @@ import moe.eval
 import moe.log
 import shutil
 
-def judge(e):
-    pass
-
 def configure_test(e, test):
     e.cfgs = moe.config.MoeConfigStack(e.cfgs)
     e.test_builtins = moe.config.MoeConfig(type="test-builtins")
@@ -53,6 +50,31 @@ def collect_status(e):
     else:
        e.log.verbose("No status file present\n")
 
+def tmpname(e):
+    return os.path.join(e.cfgs["TDIR"], e.cfgs["TEST"] + ".tmp")
+
+def collect_verdict(e, verdict_file):
+    try:
+       f = open(verdict_file, "r")
+       v = f.readline().rstrip("\n")
+       f.close()
+    except IOError, OSError:
+       v = ""
+
+    if len(v) >= 4 and v[0].isalnum and v[1].isalnum and v[2] == ":" and v[3] == " ":
+       e.test_stat["status"] = v[0:2]
+       e.log.verbose("Judge's status: %s\n" % v[0:2])
+       v = v[4:]
+    v.strip()
+    if v != "":
+       e.test_stat["message"] = v
+       e.log.verbose("Judge's verdict: %s\n" % v)
+
+    try:
+       os.unlink(verdict_file)
+    except IOError, OSError:
+       pass
+
 def setup(e):
     pdir = e.cfgs["PDIR"]
     tdir = e.cfgs["TDIR"]
@@ -66,15 +88,17 @@ def setup(e):
        moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok))
 
 def judge(e):
-    judge = e.cfgs["OUTPUT_CHECK"]
-    if judge == "":
+    cmd = e.cfgs["OUTPUT_CHECK"]
+    if cmd == "":
        return
+    verdict_file = tmpname(e)
+    cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
 
     e.log.progress("<check> ")
-    e.log.verbose("Checking output: %s\n" % judge)
+    e.log.verbose("Checking output: %s\n" % cmd)
     e.log.flush()
-    rc = os.system(judge)
-    ## FIXME: parse stderr of the judge to get the status message
+    rc = os.system(cmd)
+    collect_verdict(e, verdict_file)
     collect_status(e)
     if os.WIFEXITED(rc):
        if os.WEXITSTATUS(rc) == 0:
@@ -142,6 +166,7 @@ def run_tests(e):
        except moe.TestError, err:
            if not e.test_stat["status"]:
                e.test_stat["status"] = err.stat_code
+           if not e.test_stat["message"]:
                e.test_stat["message"] = err.message
 
        conclude_test(e)