From: Martin Mares <mj@ucw.cz>
Date: Tue, 11 Aug 2009 22:46:13 +0000 (+0200)
Subject: Batch task testing works (sort of)
X-Git-Tag: python-dummy-working~78
X-Git-Url: http://mj.ucw.cz/gitweb/?a=commitdiff_plain;h=dcc852db06f099b6c96b681c79832d41777da334;p=eval.git

Batch task testing works (sort of)
---

diff --git a/doc/meta b/doc/meta
index 84fbdb0..63b7723 100644
--- a/doc/meta
+++ b/doc/meta
@@ -52,6 +52,7 @@ test(		results of a single test
 			TO = timeout
 			WA = wrong answer
 			PA = partial answer
+			NO = no output file generated
 			PE = protocol error (in case of interactive tasks)
 			XX = internal error (e.g., error when calling judge)
   message:	human-readable status message (not intended for machine parsing)
diff --git a/t/config b/t/config
index d9387bc..0c5c43f 100644
--- a/t/config
+++ b/t/config
@@ -1,13 +1,19 @@
 # HOME set automatically
 # CONTESTANT set automatically
 # TASK set automatically
-## FIXME: Rename?
-TASK_DIR="${HOME}/problems/${TASK}"
-SOL_DIR="${HOME}/solutions/${CONTESTANT}/${TASK}"
-TEST_DIR="${HOME}/testing/${CONTESTANT}/${TASK}"
+PDIR="${HOME}/problems/${TASK}"
+SDIR="${HOME}/solutions/${CONTESTANT}/${TASK}"
+TDIR="${HOME}/testing/${CONTESTANT}/${TASK}"
 
 TASK_TYPE=batch
 
+TESTCASE_IN=${TEST}.in
+TESTCASE_OUT=${TEST}.out
+TESTCASE_OK=${TEST}.ok
+
+# HOOKS
+# TESTCASE_HOOKS
+
 ### Programming language settings
 
 # Known source file extensions
@@ -27,9 +33,9 @@ ALIAS_EXT_p=pas
 COMP=false
 
 # Sandbox options used when compiling
-COMP_SANDBOX_OPTS='-m262144 -w60 -e -i/dev/null'
+COMP_SANDBOX_OPTS="-m262144 -w60 -e -i/dev/null"
 
-# EXE is auto, but can be overridden
+EXE=$TASK
 
 # Command used to execute the compiled program, may be ./$PROGRAM (default) or an
 # interpreter with $PROGRAM as a parameter.
@@ -38,15 +44,15 @@ TEST_EXEC_CMD=./$EXE
 ## Settings for individual languages
 
 # C
-EXT_c_COMP='/usr/bin/gcc -std=gnu99 -O2 -g -o $EXE $EXTRA_CFLAGS $SRC -lm'
+EXT_c_COMP="/usr/bin/gcc -std=gnu99 -O2 -g -o $EXE $EXTRA_CFLAGS $SRC -lm"
 EXTRA_CFLAGS=
 
 # C++
-EXT_cpp_COMP='/usr/bin/g++ -O2 -g -o $EXE $EXTRA_CXXFLAGS $SRC -lm'
+EXT_cpp_COMP="/usr/bin/g++ -O2 -g -o $EXE $EXTRA_CXXFLAGS $SRC -lm"
 EXTRA_CXXFLAGS=
 
 # Pascal
-EXT_pas_COMP='/usr/bin/fpc -Ci -g -O2 -Sg -o$EXE $EXTRA_PFLAGS $SRC'
+EXT_pas_COMP="/usr/bin/fpc -Ci -g -O2 -Sg -o$EXE $EXTRA_PFLAGS $SRC"
 EXTRA_PFLAGS=
 
 ### Per-task configuration variables (default values, override in per-task config)
@@ -54,3 +60,73 @@ EXTRA_PFLAGS=
 # List of extra files needed for compilation. They are copied to the compiler
 # sandbox from the problem's directory. XXX: or tdir
 #COMP_EXTRAS="extras.h"
+
+# Task type:
+# batch		off-line task
+# interactive	interactive task communicating via stdio with a testing program
+# open-data	open-data task (i.e., we don't submit program, but output files)
+TASK_TYPE=batch
+
+# I/O type (IO_TYPE sets defaults for IN_TYPE and OUT_TYPE)
+# file		input from $PROBLEM.in, output to $PROBLEM.out (possible even for interactive tasks)
+# stdio		input from stdin, output to stdout
+# dir 		input from all files in the directory $TEST.in; these are copied to $BOXDIR
+# 		and if they include .stdin, it will be available as program's std. input.
+# none		no input/output
+IO_TYPE=stdio
+#IN_TYPE=stdio
+#OUT_TYPE=stdio
+
+IN_NAME=$TASK.in
+OUT_NAME=$TASK.out
+
+# A list of all tests
+TESTS="1 2 3 4 5 6 7 8 9 10"
+
+# A list of public tests (executed by submit and check scripts)
+SAMPLE_TESTS="0"
+
+# Number of points per test
+POINTS_PER_TEST=1
+
+# Time limit in seconds (can be fractional, but beware of noise)
+TIME_LIMIT=10
+
+# Memory limit in kilobytes
+MEM_LIMIT=16384
+
+# Stack size limit in kilobytes (0=limited only by MEM_LIMIT)
+STACK_LIMIT=0
+
+# Command used for filtering of program output (optional)
+# If turned on, program output (*.raw) is ran through this filter and the
+# checkers are applied to the output of the filter (*.out).
+# Can exit with code 1 if there is a syntax error in the output.
+#OUTPUT_FILTER=tr -d '\r' <$TDIR/$TEST.raw >$TDIR/$TEST.out
+
+# Command used to check output syntax (optional)
+# Returns exit code 1 if syntax is wrong, 0 if correct
+# fd1 is connect to evaluator log, feel free to log anything
+# fd2 is an optional one-line verdict
+#SYNTAX_CHECK=grep -v -- - $TDIR/$TEST.out
+
+# Command used to check output correctness
+# Returns exit code 1 if output is incorrect, 0 if correct
+# fd1 is connect to evaluator log, feel free to log anything
+# fd2 is an optional one-line verdict
+# The checker can generate $TDIR/$TEST.pts to assign points irregularly
+OUTPUT_CHECK=diff -bBu $TDIR/$TEST.ok $TDIR/$TEST.out
+
+# Checker for interactive tasks
+# Returns exit code 1 if test failed, 0 if passed
+# fd0 and fd1 are connected to fd1 and fd0 of the program tested
+# fd2 is an optional one-line verdict
+# The checker can generate $TDIR/$TEST.pts to assign points irregularly
+#IC_CHECK=$PDIR/checker $PDIR/$TEST.in $PDIR/$TEST.chk
+
+# Sandbox options used when testing
+TEST_SANDBOX_OPTS=-a2 -f -m$MEM_LIMIT -k$STACK_LIMIT -t$TIME_LIMIT $BOX_EXTRAS $BOX_IO_OPTS
+
+# Extra options to be overridden in task configuration
+BOX_EXTRAS=
+
diff --git a/t/moe/__init__.py b/t/moe/__init__.py
index 19957e2..26e4f24 100644
--- a/t/moe/__init__.py
+++ b/t/moe/__init__.py
@@ -4,4 +4,13 @@ class MoeErr(Exception):
     pass
 
 class SolutionErr(Exception):
-    pass
+
+    def __init__(self, message, stat_code=None):
+	self.stat_code = stat_code
+	self.message = message
+
+    def __str__(self):
+	if self.stat_code is None:
+	    return self.message
+	else:
+	    return "%s: %s" % (self.stat_code, self.message)
diff --git a/t/moe/batch.py b/t/moe/batch.py
index 2a0ef1e..4fb4f1f 100644
--- a/t/moe/batch.py
+++ b/t/moe/batch.py
@@ -5,6 +5,8 @@ import moe
 import moe.box
 import moe.eval
 import moe.util
+import moe.pipeline
+import moe.testcase
 import shutil
 
 def normalize_ext(e, ext):
@@ -26,7 +28,7 @@ def locate(e, filename=None):
     else:
 	dir, file = os.path.split(filename)
     if dir == "":
-	dir = e.cfgs["SOL_DIR"]
+	dir = e.cfgs["SDIR"]
 
     base, ext = os.path.splitext(file)
     if ext != "":
@@ -51,25 +53,25 @@ def locate(e, filename=None):
     e.log.verbose("Found solution %s\n" % orig_path)
 
     copy = e.cfgs["TASK"] + "." + norm_ext
-    copy_path = os.path.join(e.cfgs["TEST_DIR"], copy)
+    copy_path = os.path.join(e.cfgs["TDIR"], copy)
     if file != copy:
 	e.log.verbose("Renaming to %s\n" % copy)
     moe.util.link_or_copy(orig_path, copy_path)
 
     e.builtins.set("SRC", copy)
     e.builtins.set("EXT", norm_ext)
-    e.cfgs.apply_overrides("EXT_" + norm_ext)
+    e.cfgs.apply_overrides("EXT_" + norm_ext + "_")
 
     e.stat["source"] = file
     e.log.progress(file + "\n")
 
 def compile_init(e):
-    e.log.progress("Compiling: ")
+    e.log.progress("Compiling... ")
     boxdir = moe.box.setup(e)
-    pdir = e.cfgs["TASK_DIR"]
-    tdir = e.cfgs["TEST_DIR"]
+    pdir = e.cfgs["PDIR"]
+    tdir = e.cfgs["TDIR"]
     shutil.copyfile(os.path.join(tdir, e.cfgs["SRC"]), os.path.join(boxdir, e.cfgs["SRC"]))
-    for x in e.cfgs["EXTRAS"].split() + e.cfgs["COMP_EXTRAS"].split()
+    for x in e.cfgs["EXTRAS"].split() + e.cfgs["COMP_EXTRAS"].split():
 	xx = os.path.join(tdir, x)
 	if not os.path.isfile(xx):
 	    xx = os.path.join(pdir, x)
@@ -83,14 +85,99 @@ def compile_run(e):
     rc = moe.box.run(e, e.cfgs["COMP_SANDBOX_OPTS"], cc)
     if rc > 0:
 	e.log.progress("FAILED\n")
-	## FIXME: fill in the status file and abort the pipeline?
+	## FIXME: status file
+	raise moe.pipeline.MoeAbortPipeline(200)
     moe.box.show(e, "compiler output")
 
 def compile_done(e):
+    try:
+	shutil.copyfile(os.path.join(e.cfgs["BOXDIR"], e.cfgs["EXE"]), os.path.join(e.cfgs["TDIR"], e.cfgs["EXE"]))
+    except IOError:
+	raise moe.MoeErr, "Compiler succeeded, but produced no output"
     e.log.progress("OK\n")
 
+def test_in(e):
+    tdir = e.cfgs["TDIR"]
+    boxdir = moe.box.setup(e)
+    inn = e.cfgs["TESTCASE_IN"]
+    in_type = e.cfgs["IN_TYPE"] or e.cfgs["IO_TYPE"]
+    out_type = e.cfgs["OUT_TYPE"] or e.cfgs["IO_TYPE"]
+    is_interactive = e.cfgs["TASK_TYPE"] == "interactive"
+    sandbox_opts = "-M" + os.path.join(tdir, e.cfgs["TEST"] + ".status")
+
+    if not os.path.exists(os.path.join(tdir, e.cfgs["EXE"])):
+	## FIXME: status file
+	raise SolutionErr, "Compilation failed"
+    shutil.copyfile(os.path.join(tdir, e.cfgs["EXE"]), os.path.join(boxdir, e.cfgs["EXE"]))
+    os.chmod(os.path.join(boxdir, e.cfgs["EXE"]), 0555)
+
+    if in_type == "file":
+	in_name = e.cfgs["IN_NAME"]
+	e.log.verbose("Input file: %s (copied from %s)\n" % (in_name, os.path.join(e.cfgs["PDIR"], inn)))
+	shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, in_name))
+	if not is_interactive:
+	    sandbox_opts = " -i/dev/null"
+    elif in_type == "stdio":
+	e.log.verbose("Input file: <stdin> (copied from %s)\n" % os.path.join(e.cfgs["PDIR"], inn))
+	shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, ".stdin"))
+	sandbox_opts = " -i.stdin"
+    elif in_type == "none":
+	e.log.verbose("Input file: <none>\n")
+	if not is_interactive:
+	    sandbox_opts += " -i/dev/null"
+    elif in_type == "dir":
+	## FIXME
+	raise MoeErr, "Directory input not yet implemented"
+    else:
+	raise MoeErr, "Unknown input type %s" % in_type
+
+    if out_type == "file":
+	out_name = e.cfgs["OUT_NAME"]
+	e.log.verbose("Output file: %s\n" % out_name)
+	if not is_interactive:
+	    sandbox_opts += " -o/dev/null"
+    elif out_type == "stdio":
+	e.log.verbose("Output file: <stdout>\n")
+	sandbox_opts += " -o.stdout"
+    elif out_type == "none":
+	e.log.verbose("Output file: <none>\n")
+	if not is_interactive:
+	    sandbox_opts += " -o/dev/null"
+    else:
+	raise MoeErr, "Unknown output type %s" % out_type
+
+    e.test_builtins.set("BOX_IO_OPTS", sandbox_opts)
+
+def test_run(e):
+    e.log.verbose("Time limit: %s s\n" % e.cfgs["TIME_LIMIT"])
+    e.log.verbose("Memory limit: %s KB\n" % e.cfgs["MEM_LIMIT"])
+    moe.box.show(e, "test input")
+    e.log.progress("<run> ")
+    moe.box.run(e, e.cfgs["TEST_SANDBOX_OPTS"], e.cfgs["TEST_EXEC_CMD"])
+    moe.box.show(e, "test output")
+    ## FIXME: Parse the status file and delete it
+    ### Check for runtime errors reported by the box
+
+def test_collect(e):
+    tdir = e.cfgs["TDIR"]
+    boxdir = e.cfgs["BOXDIR"]
+    out_type = e.cfgs["OUT_TYPE"] or e.cfgs["IO_TYPE"]
+    is_interactive = e.cfgs["TASK_TYPE"] == "interactive"
+
+    if out_type == "file":
+	out_path = e.cfgs["OUT_NAME"]
+    elif out_type == "stdio":
+	out_path = ".stdout"
+    if not os.path.exists(os.path.join(boxdir, out_path)):
+	raise moe.SolutionErr("No output file", "NO")
+    shutil.copyfile(os.path.join(boxdir, out_path), os.path.join(tdir, e.cfgs["TESTCASE_OUT"]))
+
 def tests(e):
-    pass
+    e.log.progress("\n")
+    e.test_pipe.insert(100, "prepare", test_in)
+    e.test_pipe.insert(200, "run", test_run)
+    e.test_pipe.insert(300, "collect", test_collect)
+    moe.testcase.run_tests(e)
 
 def prepare_pipe(e):
     e.main_pipe.insert(100, "compile-init", compile_init)
diff --git a/t/moe/box.py b/t/moe/box.py
index bd02395..321eedd 100644
--- a/t/moe/box.py
+++ b/t/moe/box.py
@@ -39,7 +39,12 @@ def show(e, msg):
 def run(e, opts, cmd):
     c = e.cfgs["BOXCMD"] + " " + opts + " -- " + cmd
     e.log.verbose("Sandbox: %s\n" % c)
-    rc = os.system(c)
-    if rc > 1:
-    	raise moe.MoeErr, "Sandbox failed"
-    return rc
+    e.log.flush()
+    st = os.system(c)
+    if os.WIFEXITED(st):
+    	rc = os.WEXITSTATUS(st)
+	if rc > 1:
+	    raise moe.MoeErr, "Sandbox failed with rc=%d" % rc
+	return rc
+    else:
+	raise moe.MoeErr, "Sandbox failed with exit status 0x%04x" % rc
diff --git a/t/moe/config.py b/t/moe/config.py
index 71cfbae..7f015da 100644
--- a/t/moe/config.py
+++ b/t/moe/config.py
@@ -153,7 +153,7 @@ class MoeConfigStack:
 	    cfg = self.stk[pos]
 	    if cfg.vars.has_key(k):
 		new = cfg.vars[k]
-		if new[0][0] == "a":
+		if len(new) > 0 and new[0][0] == "a":
 		    v = self.do_get(k, pos-1)
 		else:
 		    v = ""
diff --git a/t/moe/eval.py b/t/moe/eval.py
index bc2e5c0..f3e921a 100644
--- a/t/moe/eval.py
+++ b/t/moe/eval.py
@@ -19,6 +19,7 @@ class Eval:
 	self.builtins = moe.config.MoeConfig(type="builtins")
 	self.cfgs.push(self.builtins)
 	self.main_pipe = moe.pipeline.MoePipeline("main")
+	self.test_pipe = moe.pipeline.MoePipeline("test")
 	self.stat = moe.status.MoeStatus()
 	pass
 
@@ -38,7 +39,7 @@ class Eval:
 	    self.cfgs.push(overrides)
 
     def init_test(self):
-	test = self.cfgs['TEST_DIR']
+	test = self.cfgs['TDIR']
 	if os.path.isdir(test):
 	    shutil.rmtree(test)
 	try:
@@ -50,14 +51,14 @@ class Eval:
 	self.log = moe.log.MoeLog()
 	if self.cfgs["V"]:
 	    self.log.verbosity = int(self.cfgs["V"])
-	self.log.open(os.path.join(self.cfgs["TEST_DIR"], "log"))
+	self.log.open(os.path.join(self.cfgs["TDIR"], "log"))
 	self.default_log = self.log
 	moe.log.default = self.log
 	self.log_config(3, "before loading the task")
 
     def init_task(self):
 	task = self.cfgs['TASK']
-	task_dir = self.cfgs['TASK_DIR']
+	task_dir = self.cfgs['PDIR']
 	if not os.path.isdir(task_dir):
 	    raise moe.MoeErr, "No such task %s" % task
 
diff --git a/t/moe/pipeline.py b/t/moe/pipeline.py
index bc407a2..11d809d 100644
--- a/t/moe/pipeline.py
+++ b/t/moe/pipeline.py
@@ -9,6 +9,11 @@ import moe.log
 class MoePipeError(moe.MoeErr):
     """Failure of the MoePipeline."""
 
+class MoeAbortPipeline(Exception):
+
+    def __init__(self, skip_to=999):
+	self.skip_to = skip_to
+
 class MoePipeline:
     """Moe pipeline."""
 
@@ -31,10 +36,17 @@ class MoePipeline:
 
     def run(self, *args):
 	self.index = 0
+	min_pri = -1
 	while self.index < len(self.pipe):
 	    (pri,name,fun) = self.pipe[self.index]
-	    moe.log.default.verbose(">> Running %s:%s\n" % (self.name,name))
-	    fun(*args)
+	    if pri >= min_pri:
+		moe.log.default.verbose(">> Running %s:%s\n" % (self.name,name))
+		try:
+		    fun(*args)
+		except MoeAbortPipeline, err:
+		    min_pri = err.skip_to
+	    else:
+		moe.log.default.verbose(">> Skipping %s:%s\n" % (self.name,name))
 	    self.index += 1
 	self.index = -1
 
diff --git a/t/moe/testcase.py b/t/moe/testcase.py
new file mode 100644
index 0000000..fba4128
--- /dev/null
+++ b/t/moe/testcase.py
@@ -0,0 +1,96 @@
+#!/usr/bin/env python
+
+import os.path
+import moe
+import moe.config
+import moe.eval
+import moe.log
+import shutil
+
+def judge(e):
+    pass
+
+def configure_test(e, test):
+    e.cfgs = moe.config.MoeConfigStack(e.cfgs)
+    e.test_builtins = moe.config.MoeConfig(type="test-builtins")
+    e.test_builtins.set("TEST", test)
+    e.cfgs.push(e.test_builtins)
+
+    test_cf = os.path.join(e.cfgs["PDIR"], test + ".config")
+    if os.path.exists(test_cf):
+	cfg = moe.config.MoeConfig(name=test_cf, type="test")
+	e.cfgs.push(cfg)
+
+    e.cfgs.apply_overrides("TEST_" + test + "_")
+    ext = e.cfgs["EXT"]
+    if ext != "":
+	e.cfgs.apply_overrides("EXT_" + ext + "_")
+
+    log = moe.log.MoeLog()
+    log.verbosity = e.log.verbosity
+    log.open(os.path.join(e.cfgs["TDIR"], test + ".log"))
+    log.say("Test case %s\n\n" % test)
+    e.log = log
+
+    e.log_config(2, "for the test")
+
+def setup(e):
+    pdir = e.cfgs["PDIR"]
+    tdir = e.cfgs["TDIR"]
+    inn = e.cfgs["TESTCASE_IN"]
+    out = e.cfgs["TESTCASE_OUT"]
+    ok = e.cfgs["TESTCASE_OK"]
+
+    if os.path.exists(os.path.join(pdir, inn)):
+	moe.util.link_or_copy(os.path.join(pdir, inn), os.path.join(tdir, inn))
+    if os.path.exists(os.path.join(pdir, out)):
+	moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok))
+
+def judge(e):
+    judge = e.cfgs["OUTPUT_CHECK"]
+    if judge == "":
+	return
+
+    e.log.progress("<check> ")
+    e.log.verbose("Checking output: %s\n" % judge)
+    e.log.flush()
+    rc = os.system(judge)
+    ## FIXME: The judge might want to return a status file
+    if os.WIFEXITED(rc):
+	if os.WEXITSTATUS(rc) == 0:
+	    return
+	elif os.WEXITSTATUS(rc) == 1:
+	    raise moe.SolutionErr("Wrong answer", "WA")
+    raise moe.MoeErr("Judge failure")
+
+def run_test(e, test):
+    configure_test(e, test)
+
+    ## FIXME: interactive tasks
+    e.test_pipe.configure(e.cfgs["TESTCASE_HOOKS"])
+    if e.log.verbosity >= 2:
+	e.test_pipe.dump(e.log.log_file, prefix="\t")
+    e.test_pipe.run(e)
+
+    e.log.progress("OK\n")
+
+def run_tests(e):
+    ## FIXME: output filter
+    e.test_pipe.insert(0, "setup", setup)
+    e.test_pipe.insert(400, "judge", judge)
+
+    for test in e.cfgs["TESTS"].split():
+	e.log.progress("Test %s: " % test)
+	old_cfgs = e.cfgs
+	old_log = e.log
+
+	try:
+	    run_test(e, test)
+	except moe.MoeErr, err:
+	    e.log.progress("FAILED: %s\n" % err)
+	    ## FIXME: write it to the status file
+	except moe.SolutionErr, err:
+	    e.log.progress("%s\n" % err)
+	
+	e.cfgs = old_cfgs
+	e.log = old_log