Status files are now handled almost properly

diff --git a/doc/meta b/doc/meta
index 63b7723f8b8f98daa702db3b39d3a0137799ec6d..fce7e129637d030a68b75ebd0420a75ed9ca6a40 100644 (file)
--- a/doc/meta
+++ b/doc/meta
@@ -39,6 +39,8 @@ queue-done:   timestamp of the finish of evaluation
 
 Added by the evaluator
 ~~~~~~~~~~~~~~~~~~~~~~
+lang:          language of the submitted source (file-name extension; missing for open-data problems)
+error:         human-readable error message for the whole submit (e.g., unrecognized language)
 test(          results of a single test
   -- mandatory:
   id:          the name of the test

diff --git a/t/config b/t/config
index 0c5c43f45085418f2de61de84e9b1032e4e38e8c..96ac31e4c11ee469f6c81a42e0c7b714cfd926e1 100644 (file)
--- a/t/config
+++ b/t/config
@@ -10,6 +10,7 @@ TASK_TYPE=batch
 TESTCASE_IN=${TEST}.in
 TESTCASE_OUT=${TEST}.out
 TESTCASE_OK=${TEST}.ok
+TESTCASE_STATUS=${TEST}.stat
 
 # HOOKS
 # TESTCASE_HOOKS

diff --git a/t/moe/__init__.py b/t/moe/__init__.py
index 26e4f24ebf7951e5fc46bce03361b014eb3fd75d..2ce549975d842130c7c3fe8876b24f690709f790 100644 (file)
--- a/t/moe/__init__.py
+++ b/t/moe/__init__.py
@@ -14,3 +14,6 @@ class SolutionErr(Exception):
            return self.message
        else:
            return "%s: %s" % (self.stat_code, self.message)
+
+class TestErr(SolutionErr):
+    pass

diff --git a/t/moe/batch.py b/t/moe/batch.py
index 4fb4f1f5ff96a46c4ba07a5329554d09d6228515..4e3dcf92684327ad7b04b12de74395ac3865598e 100644 (file)
--- a/t/moe/batch.py
+++ b/t/moe/batch.py
@@ -63,6 +63,7 @@ def locate(e, filename=None):
     e.cfgs.apply_overrides("EXT_" + norm_ext + "_")
 
     e.stat["source"] = file
+    e.stat["lang"] = ext
     e.log.progress(file + "\n")
 
 def compile_init(e):
@@ -85,7 +86,7 @@ def compile_run(e):
     rc = moe.box.run(e, e.cfgs["COMP_SANDBOX_OPTS"], cc)
     if rc > 0:
        e.log.progress("FAILED\n")
-       ## FIXME: status file
+       ## FIXME: status file ... or generate an exception?
        raise moe.pipeline.MoeAbortPipeline(200)
     moe.box.show(e, "compiler output")
 
@@ -103,11 +104,10 @@ def test_in(e):
     in_type = e.cfgs["IN_TYPE"] or e.cfgs["IO_TYPE"]
     out_type = e.cfgs["OUT_TYPE"] or e.cfgs["IO_TYPE"]
     is_interactive = e.cfgs["TASK_TYPE"] == "interactive"
-    sandbox_opts = "-M" + os.path.join(tdir, e.cfgs["TEST"] + ".status")
+    sandbox_opts = "-M" + os.path.join(tdir, e.cfgs["TESTCASE_STATUS"])
 
     if not os.path.exists(os.path.join(tdir, e.cfgs["EXE"])):
-       ## FIXME: status file
-       raise SolutionErr, "Compilation failed"
+       raise TestErr("Compilation failed", "CE")
     shutil.copyfile(os.path.join(tdir, e.cfgs["EXE"]), os.path.join(boxdir, e.cfgs["EXE"]))
     os.chmod(os.path.join(boxdir, e.cfgs["EXE"]), 0555)
 
@@ -116,11 +116,11 @@ def test_in(e):
        e.log.verbose("Input file: %s (copied from %s)\n" % (in_name, os.path.join(e.cfgs["PDIR"], inn)))
        shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, in_name))
        if not is_interactive:
-           sandbox_opts = " -i/dev/null"
+           sandbox_opts += " -i/dev/null"
     elif in_type == "stdio":
        e.log.verbose("Input file: <stdin> (copied from %s)\n" % os.path.join(e.cfgs["PDIR"], inn))
        shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, ".stdin"))
-       sandbox_opts = " -i.stdin"
+       sandbox_opts += " -i.stdin"
     elif in_type == "none":
        e.log.verbose("Input file: <none>\n")
        if not is_interactive:
@@ -153,10 +153,11 @@ def test_run(e):
     e.log.verbose("Memory limit: %s KB\n" % e.cfgs["MEM_LIMIT"])
     moe.box.show(e, "test input")
     e.log.progress("<run> ")
-    moe.box.run(e, e.cfgs["TEST_SANDBOX_OPTS"], e.cfgs["TEST_EXEC_CMD"])
+    rc = moe.box.run(e, e.cfgs["TEST_SANDBOX_OPTS"], e.cfgs["TEST_EXEC_CMD"])
+    moe.testcase.collect_status(e)
     moe.box.show(e, "test output")
-    ## FIXME: Parse the status file and delete it
-    ### Check for runtime errors reported by the box
+    if rc > 0:
+       raise moe.TestErr("Wrong answer", "WA")
 
 def test_collect(e):
     tdir = e.cfgs["TDIR"]
@@ -169,7 +170,7 @@ def test_collect(e):
     elif out_type == "stdio":
        out_path = ".stdout"
     if not os.path.exists(os.path.join(boxdir, out_path)):
-       raise moe.SolutionErr("No output file", "NO")
+       raise moe.TestErr("No output file", "NO")
     shutil.copyfile(os.path.join(boxdir, out_path), os.path.join(tdir, e.cfgs["TESTCASE_OUT"]))
 
 def tests(e):

diff --git a/t/moe/pipeline.py b/t/moe/pipeline.py
index 11d809d64bd1d378f3a82bff62b05457ca1f7ee7..7726110ccf32b2bd313324121b20ac75ee36eb82 100644 (file)
--- a/t/moe/pipeline.py
+++ b/t/moe/pipeline.py
@@ -49,6 +49,7 @@ class MoePipeline:
                moe.log.default.verbose(">> Skipping %s:%s\n" % (self.name,name))
            self.index += 1
        self.index = -1
+       moe.log.default.verbose(">> Pipeline %s finished\n" % self.name)
 
     def add_hook(self, name):
        modname = "moe.hooks." + name

diff --git a/t/moe/status.py b/t/moe/status.py
index 936659e6408a907650d8aea60887e6ec5ded5370..3d72db476de48df25caf8c566cb1f14a429f922d 100644 (file)
--- a/t/moe/status.py
+++ b/t/moe/status.py
@@ -26,6 +26,9 @@ class MoeStatus:
     def __setitem__(self, k, v):
        self.stat[k] = v
 
+    def keys(self):
+       return self.stat.keys()
+
     def get_list(self, k):
        m = self.stat
        if not m.has_key(k):

diff --git a/t/moe/testcase.py b/t/moe/testcase.py
index fba4128ca14b9f22636e993f80fc8ebc8862e83f..e49bd8c3d1428793fc373c66ebf057b0ca5217a9 100644 (file)
--- a/t/moe/testcase.py
+++ b/t/moe/testcase.py
@@ -31,9 +31,28 @@ def configure_test(e, test):
     log.open(os.path.join(e.cfgs["TDIR"], test + ".log"))
     log.say("Test case %s\n\n" % test)
     e.log = log
+    moe.log.default = log
 
     e.log_config(2, "for the test")
 
+    e.test_stat = moe.status.MoeStatus()
+    e.test_stat["id"] = test
+    e.stat.get_list("tests").append(e.test_stat)
+
+def collect_status(e):
+    sf = os.path.join(e.cfgs["TDIR"], e.cfgs["TESTCASE_STATUS"])
+    if os.path.exists(sf):
+       e.log.verbose("Reading status from %s\n" % sf)
+       stat = moe.status.MoeStatus()
+       stat.read(name=sf)
+       if e.log.verbosity > 1:
+           stat.write_nested(e.log.log_file, 1)
+       for k in stat.keys():
+           e.test_stat[k] = stat[k]
+       os.unlink(sf)
+    else:
+       e.log.verbose("No status file present\n")
+
 def setup(e):
     pdir = e.cfgs["PDIR"]
     tdir = e.cfgs["TDIR"]
@@ -55,14 +74,20 @@ def judge(e):
     e.log.verbose("Checking output: %s\n" % judge)
     e.log.flush()
     rc = os.system(judge)
-    ## FIXME: The judge might want to return a status file
+    ## FIXME: parse stderr of the judge to get the status message
+    collect_status(e)
     if os.WIFEXITED(rc):
        if os.WEXITSTATUS(rc) == 0:
            return
        elif os.WEXITSTATUS(rc) == 1:
-           raise moe.SolutionErr("Wrong answer", "WA")
+           raise moe.TestErr("Wrong answer", "WA")
     raise moe.MoeErr("Judge failure")
 
+def points(e):
+    ## FIXME: check $TEST.pts
+    if e.test_stat["points"] is None:
+       e.test_stat["points"] = e.cfgs["POINTS_PER_TEST"]
+
 def run_test(e, test):
     configure_test(e, test)
 
@@ -72,12 +97,40 @@ def run_test(e, test):
        e.test_pipe.dump(e.log.log_file, prefix="\t")
     e.test_pipe.run(e)
 
-    e.log.progress("OK\n")
+def wrap_run_test(e, test):
+    try:
+       run_test(e, test)
+    except moe.MoeErr, err:
+       raise moe.TestErr(err, "XX")
+
+def conclude_test(e):
+    stat = e.test_stat
+    if not stat["points"]:
+       stat["points"] = 0
+
+    if e.log.verbosity > 1:
+       e.log.verbose("Final status:\n")
+       stat.write_nested(e.log.log_file, 1)
+
+    if stat["status"]:
+       msg = "%s: %s" % (stat["status"], e.test_stat["message"])
+    else:
+       msg = "OK"
+    msg += " (%s points" % stat["points"]
+    if stat["time"]:
+       msg += ", %s s" % stat["time"]
+    if stat["mem"]:
+       msg += ", %d MB" % ((int(stat["mem"]) + 524288) // 1048576)
+    msg += ")\n"
+    e.log.progress(msg)
+    e.log.say(msg)
 
 def run_tests(e):
     ## FIXME: output filter
+    ## FIXME: syntax checks
     e.test_pipe.insert(0, "setup", setup)
     e.test_pipe.insert(400, "judge", judge)
+    e.test_pipe.insert(500, "points", points)
 
     for test in e.cfgs["TESTS"].split():
        e.log.progress("Test %s: " % test)
@@ -85,12 +138,14 @@ def run_tests(e):
        old_log = e.log
 
        try:
-           run_test(e, test)
-       except moe.MoeErr, err:
-           e.log.progress("FAILED: %s\n" % err)
-           ## FIXME: write it to the status file
-       except moe.SolutionErr, err:
-           e.log.progress("%s\n" % err)
+           wrap_run_test(e, test)
+       except moe.TestErr, err:
+           if not e.test_stat["status"]:
+               e.test_stat["status"] = err.stat_code
+               e.test_stat["message"] = err.message
+
+       conclude_test(e)
        
        e.cfgs = old_cfgs
        e.log = old_log
+       moe.log.default = old_log

diff --git a/t/test.py b/t/test.py
index 8b8f3f124fc796a820be1cafedf4f460d5812db5..07a220fe96a51bb7e11e96abc46341ab2ae43d55 100755 (executable)
--- a/t/test.py
+++ b/t/test.py
@@ -5,6 +5,7 @@ sys.path.append('.')
 
 import moe
 import moe.config
+import moe.log
 import moe.eval
 import moe.pipeline
 import moe.batch
@@ -33,9 +34,8 @@ except moe.MoeErr, err:
     e.log.shout("FATAL: %s\n" % err)
     sys.exit(1)
 except moe.SolutionErr, err:
-    ## FIXME: In this case, we might write the status file
+    e.stat["error"] = err
     e.log.shout("%s\n" % err)
-    sys.exit(1)
 
-print "\nFinal status file:"
-e.stat.write()
+moe.log.default.progress_file.write("\nFinal status file:\n")
+e.stat.write(file=moe.log.default.progress_file)