From dcc852db06f099b6c96b681c79832d41777da334 Mon Sep 17 00:00:00 2001 From: Martin Mares Date: Wed, 12 Aug 2009 00:46:13 +0200 Subject: [PATCH] Batch task testing works (sort of) --- doc/meta | 1 + t/config | 94 +++++++++++++++++++++++++++++++++++++---- t/moe/__init__.py | 11 ++++- t/moe/batch.py | 105 ++++++++++++++++++++++++++++++++++++++++++---- t/moe/box.py | 13 ++++-- t/moe/config.py | 2 +- t/moe/eval.py | 7 ++-- t/moe/pipeline.py | 16 ++++++- t/moe/testcase.py | 96 ++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 316 insertions(+), 29 deletions(-) create mode 100644 t/moe/testcase.py diff --git a/doc/meta b/doc/meta index 84fbdb0..63b7723 100644 --- a/doc/meta +++ b/doc/meta @@ -52,6 +52,7 @@ test( results of a single test TO = timeout WA = wrong answer PA = partial answer + NO = no output file generated PE = protocol error (in case of interactive tasks) XX = internal error (e.g., error when calling judge) message: human-readable status message (not intended for machine parsing) diff --git a/t/config b/t/config index d9387bc..0c5c43f 100644 --- a/t/config +++ b/t/config @@ -1,13 +1,19 @@ # HOME set automatically # CONTESTANT set automatically # TASK set automatically -## FIXME: Rename? -TASK_DIR="${HOME}/problems/${TASK}" -SOL_DIR="${HOME}/solutions/${CONTESTANT}/${TASK}" -TEST_DIR="${HOME}/testing/${CONTESTANT}/${TASK}" +PDIR="${HOME}/problems/${TASK}" +SDIR="${HOME}/solutions/${CONTESTANT}/${TASK}" +TDIR="${HOME}/testing/${CONTESTANT}/${TASK}" TASK_TYPE=batch +TESTCASE_IN=${TEST}.in +TESTCASE_OUT=${TEST}.out +TESTCASE_OK=${TEST}.ok + +# HOOKS +# TESTCASE_HOOKS + ### Programming language settings # Known source file extensions @@ -27,9 +33,9 @@ ALIAS_EXT_p=pas COMP=false # Sandbox options used when compiling -COMP_SANDBOX_OPTS='-m262144 -w60 -e -i/dev/null' +COMP_SANDBOX_OPTS="-m262144 -w60 -e -i/dev/null" -# EXE is auto, but can be overridden +EXE=$TASK # Command used to execute the compiled program, may be ./$PROGRAM (default) or an # interpreter with $PROGRAM as a parameter. @@ -38,15 +44,15 @@ TEST_EXEC_CMD=./$EXE ## Settings for individual languages # C -EXT_c_COMP='/usr/bin/gcc -std=gnu99 -O2 -g -o $EXE $EXTRA_CFLAGS $SRC -lm' +EXT_c_COMP="/usr/bin/gcc -std=gnu99 -O2 -g -o $EXE $EXTRA_CFLAGS $SRC -lm" EXTRA_CFLAGS= # C++ -EXT_cpp_COMP='/usr/bin/g++ -O2 -g -o $EXE $EXTRA_CXXFLAGS $SRC -lm' +EXT_cpp_COMP="/usr/bin/g++ -O2 -g -o $EXE $EXTRA_CXXFLAGS $SRC -lm" EXTRA_CXXFLAGS= # Pascal -EXT_pas_COMP='/usr/bin/fpc -Ci -g -O2 -Sg -o$EXE $EXTRA_PFLAGS $SRC' +EXT_pas_COMP="/usr/bin/fpc -Ci -g -O2 -Sg -o$EXE $EXTRA_PFLAGS $SRC" EXTRA_PFLAGS= ### Per-task configuration variables (default values, override in per-task config) @@ -54,3 +60,73 @@ EXTRA_PFLAGS= # List of extra files needed for compilation. They are copied to the compiler # sandbox from the problem's directory. XXX: or tdir #COMP_EXTRAS="extras.h" + +# Task type: +# batch off-line task +# interactive interactive task communicating via stdio with a testing program +# open-data open-data task (i.e., we don't submit program, but output files) +TASK_TYPE=batch + +# I/O type (IO_TYPE sets defaults for IN_TYPE and OUT_TYPE) +# file input from $PROBLEM.in, output to $PROBLEM.out (possible even for interactive tasks) +# stdio input from stdin, output to stdout +# dir input from all files in the directory $TEST.in; these are copied to $BOXDIR +# and if they include .stdin, it will be available as program's std. input. +# none no input/output +IO_TYPE=stdio +#IN_TYPE=stdio +#OUT_TYPE=stdio + +IN_NAME=$TASK.in +OUT_NAME=$TASK.out + +# A list of all tests +TESTS="1 2 3 4 5 6 7 8 9 10" + +# A list of public tests (executed by submit and check scripts) +SAMPLE_TESTS="0" + +# Number of points per test +POINTS_PER_TEST=1 + +# Time limit in seconds (can be fractional, but beware of noise) +TIME_LIMIT=10 + +# Memory limit in kilobytes +MEM_LIMIT=16384 + +# Stack size limit in kilobytes (0=limited only by MEM_LIMIT) +STACK_LIMIT=0 + +# Command used for filtering of program output (optional) +# If turned on, program output (*.raw) is ran through this filter and the +# checkers are applied to the output of the filter (*.out). +# Can exit with code 1 if there is a syntax error in the output. +#OUTPUT_FILTER=tr -d '\r' <$TDIR/$TEST.raw >$TDIR/$TEST.out + +# Command used to check output syntax (optional) +# Returns exit code 1 if syntax is wrong, 0 if correct +# fd1 is connect to evaluator log, feel free to log anything +# fd2 is an optional one-line verdict +#SYNTAX_CHECK=grep -v -- - $TDIR/$TEST.out + +# Command used to check output correctness +# Returns exit code 1 if output is incorrect, 0 if correct +# fd1 is connect to evaluator log, feel free to log anything +# fd2 is an optional one-line verdict +# The checker can generate $TDIR/$TEST.pts to assign points irregularly +OUTPUT_CHECK=diff -bBu $TDIR/$TEST.ok $TDIR/$TEST.out + +# Checker for interactive tasks +# Returns exit code 1 if test failed, 0 if passed +# fd0 and fd1 are connected to fd1 and fd0 of the program tested +# fd2 is an optional one-line verdict +# The checker can generate $TDIR/$TEST.pts to assign points irregularly +#IC_CHECK=$PDIR/checker $PDIR/$TEST.in $PDIR/$TEST.chk + +# Sandbox options used when testing +TEST_SANDBOX_OPTS=-a2 -f -m$MEM_LIMIT -k$STACK_LIMIT -t$TIME_LIMIT $BOX_EXTRAS $BOX_IO_OPTS + +# Extra options to be overridden in task configuration +BOX_EXTRAS= + diff --git a/t/moe/__init__.py b/t/moe/__init__.py index 19957e2..26e4f24 100644 --- a/t/moe/__init__.py +++ b/t/moe/__init__.py @@ -4,4 +4,13 @@ class MoeErr(Exception): pass class SolutionErr(Exception): - pass + + def __init__(self, message, stat_code=None): + self.stat_code = stat_code + self.message = message + + def __str__(self): + if self.stat_code is None: + return self.message + else: + return "%s: %s" % (self.stat_code, self.message) diff --git a/t/moe/batch.py b/t/moe/batch.py index 2a0ef1e..4fb4f1f 100644 --- a/t/moe/batch.py +++ b/t/moe/batch.py @@ -5,6 +5,8 @@ import moe import moe.box import moe.eval import moe.util +import moe.pipeline +import moe.testcase import shutil def normalize_ext(e, ext): @@ -26,7 +28,7 @@ def locate(e, filename=None): else: dir, file = os.path.split(filename) if dir == "": - dir = e.cfgs["SOL_DIR"] + dir = e.cfgs["SDIR"] base, ext = os.path.splitext(file) if ext != "": @@ -51,25 +53,25 @@ def locate(e, filename=None): e.log.verbose("Found solution %s\n" % orig_path) copy = e.cfgs["TASK"] + "." + norm_ext - copy_path = os.path.join(e.cfgs["TEST_DIR"], copy) + copy_path = os.path.join(e.cfgs["TDIR"], copy) if file != copy: e.log.verbose("Renaming to %s\n" % copy) moe.util.link_or_copy(orig_path, copy_path) e.builtins.set("SRC", copy) e.builtins.set("EXT", norm_ext) - e.cfgs.apply_overrides("EXT_" + norm_ext) + e.cfgs.apply_overrides("EXT_" + norm_ext + "_") e.stat["source"] = file e.log.progress(file + "\n") def compile_init(e): - e.log.progress("Compiling: ") + e.log.progress("Compiling... ") boxdir = moe.box.setup(e) - pdir = e.cfgs["TASK_DIR"] - tdir = e.cfgs["TEST_DIR"] + pdir = e.cfgs["PDIR"] + tdir = e.cfgs["TDIR"] shutil.copyfile(os.path.join(tdir, e.cfgs["SRC"]), os.path.join(boxdir, e.cfgs["SRC"])) - for x in e.cfgs["EXTRAS"].split() + e.cfgs["COMP_EXTRAS"].split() + for x in e.cfgs["EXTRAS"].split() + e.cfgs["COMP_EXTRAS"].split(): xx = os.path.join(tdir, x) if not os.path.isfile(xx): xx = os.path.join(pdir, x) @@ -83,14 +85,99 @@ def compile_run(e): rc = moe.box.run(e, e.cfgs["COMP_SANDBOX_OPTS"], cc) if rc > 0: e.log.progress("FAILED\n") - ## FIXME: fill in the status file and abort the pipeline? + ## FIXME: status file + raise moe.pipeline.MoeAbortPipeline(200) moe.box.show(e, "compiler output") def compile_done(e): + try: + shutil.copyfile(os.path.join(e.cfgs["BOXDIR"], e.cfgs["EXE"]), os.path.join(e.cfgs["TDIR"], e.cfgs["EXE"])) + except IOError: + raise moe.MoeErr, "Compiler succeeded, but produced no output" e.log.progress("OK\n") +def test_in(e): + tdir = e.cfgs["TDIR"] + boxdir = moe.box.setup(e) + inn = e.cfgs["TESTCASE_IN"] + in_type = e.cfgs["IN_TYPE"] or e.cfgs["IO_TYPE"] + out_type = e.cfgs["OUT_TYPE"] or e.cfgs["IO_TYPE"] + is_interactive = e.cfgs["TASK_TYPE"] == "interactive" + sandbox_opts = "-M" + os.path.join(tdir, e.cfgs["TEST"] + ".status") + + if not os.path.exists(os.path.join(tdir, e.cfgs["EXE"])): + ## FIXME: status file + raise SolutionErr, "Compilation failed" + shutil.copyfile(os.path.join(tdir, e.cfgs["EXE"]), os.path.join(boxdir, e.cfgs["EXE"])) + os.chmod(os.path.join(boxdir, e.cfgs["EXE"]), 0555) + + if in_type == "file": + in_name = e.cfgs["IN_NAME"] + e.log.verbose("Input file: %s (copied from %s)\n" % (in_name, os.path.join(e.cfgs["PDIR"], inn))) + shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, in_name)) + if not is_interactive: + sandbox_opts = " -i/dev/null" + elif in_type == "stdio": + e.log.verbose("Input file: (copied from %s)\n" % os.path.join(e.cfgs["PDIR"], inn)) + shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, ".stdin")) + sandbox_opts = " -i.stdin" + elif in_type == "none": + e.log.verbose("Input file: \n") + if not is_interactive: + sandbox_opts += " -i/dev/null" + elif in_type == "dir": + ## FIXME + raise MoeErr, "Directory input not yet implemented" + else: + raise MoeErr, "Unknown input type %s" % in_type + + if out_type == "file": + out_name = e.cfgs["OUT_NAME"] + e.log.verbose("Output file: %s\n" % out_name) + if not is_interactive: + sandbox_opts += " -o/dev/null" + elif out_type == "stdio": + e.log.verbose("Output file: \n") + sandbox_opts += " -o.stdout" + elif out_type == "none": + e.log.verbose("Output file: \n") + if not is_interactive: + sandbox_opts += " -o/dev/null" + else: + raise MoeErr, "Unknown output type %s" % out_type + + e.test_builtins.set("BOX_IO_OPTS", sandbox_opts) + +def test_run(e): + e.log.verbose("Time limit: %s s\n" % e.cfgs["TIME_LIMIT"]) + e.log.verbose("Memory limit: %s KB\n" % e.cfgs["MEM_LIMIT"]) + moe.box.show(e, "test input") + e.log.progress(" ") + moe.box.run(e, e.cfgs["TEST_SANDBOX_OPTS"], e.cfgs["TEST_EXEC_CMD"]) + moe.box.show(e, "test output") + ## FIXME: Parse the status file and delete it + ### Check for runtime errors reported by the box + +def test_collect(e): + tdir = e.cfgs["TDIR"] + boxdir = e.cfgs["BOXDIR"] + out_type = e.cfgs["OUT_TYPE"] or e.cfgs["IO_TYPE"] + is_interactive = e.cfgs["TASK_TYPE"] == "interactive" + + if out_type == "file": + out_path = e.cfgs["OUT_NAME"] + elif out_type == "stdio": + out_path = ".stdout" + if not os.path.exists(os.path.join(boxdir, out_path)): + raise moe.SolutionErr("No output file", "NO") + shutil.copyfile(os.path.join(boxdir, out_path), os.path.join(tdir, e.cfgs["TESTCASE_OUT"])) + def tests(e): - pass + e.log.progress("\n") + e.test_pipe.insert(100, "prepare", test_in) + e.test_pipe.insert(200, "run", test_run) + e.test_pipe.insert(300, "collect", test_collect) + moe.testcase.run_tests(e) def prepare_pipe(e): e.main_pipe.insert(100, "compile-init", compile_init) diff --git a/t/moe/box.py b/t/moe/box.py index bd02395..321eedd 100644 --- a/t/moe/box.py +++ b/t/moe/box.py @@ -39,7 +39,12 @@ def show(e, msg): def run(e, opts, cmd): c = e.cfgs["BOXCMD"] + " " + opts + " -- " + cmd e.log.verbose("Sandbox: %s\n" % c) - rc = os.system(c) - if rc > 1: - raise moe.MoeErr, "Sandbox failed" - return rc + e.log.flush() + st = os.system(c) + if os.WIFEXITED(st): + rc = os.WEXITSTATUS(st) + if rc > 1: + raise moe.MoeErr, "Sandbox failed with rc=%d" % rc + return rc + else: + raise moe.MoeErr, "Sandbox failed with exit status 0x%04x" % rc diff --git a/t/moe/config.py b/t/moe/config.py index 71cfbae..7f015da 100644 --- a/t/moe/config.py +++ b/t/moe/config.py @@ -153,7 +153,7 @@ class MoeConfigStack: cfg = self.stk[pos] if cfg.vars.has_key(k): new = cfg.vars[k] - if new[0][0] == "a": + if len(new) > 0 and new[0][0] == "a": v = self.do_get(k, pos-1) else: v = "" diff --git a/t/moe/eval.py b/t/moe/eval.py index bc2e5c0..f3e921a 100644 --- a/t/moe/eval.py +++ b/t/moe/eval.py @@ -19,6 +19,7 @@ class Eval: self.builtins = moe.config.MoeConfig(type="builtins") self.cfgs.push(self.builtins) self.main_pipe = moe.pipeline.MoePipeline("main") + self.test_pipe = moe.pipeline.MoePipeline("test") self.stat = moe.status.MoeStatus() pass @@ -38,7 +39,7 @@ class Eval: self.cfgs.push(overrides) def init_test(self): - test = self.cfgs['TEST_DIR'] + test = self.cfgs['TDIR'] if os.path.isdir(test): shutil.rmtree(test) try: @@ -50,14 +51,14 @@ class Eval: self.log = moe.log.MoeLog() if self.cfgs["V"]: self.log.verbosity = int(self.cfgs["V"]) - self.log.open(os.path.join(self.cfgs["TEST_DIR"], "log")) + self.log.open(os.path.join(self.cfgs["TDIR"], "log")) self.default_log = self.log moe.log.default = self.log self.log_config(3, "before loading the task") def init_task(self): task = self.cfgs['TASK'] - task_dir = self.cfgs['TASK_DIR'] + task_dir = self.cfgs['PDIR'] if not os.path.isdir(task_dir): raise moe.MoeErr, "No such task %s" % task diff --git a/t/moe/pipeline.py b/t/moe/pipeline.py index bc407a2..11d809d 100644 --- a/t/moe/pipeline.py +++ b/t/moe/pipeline.py @@ -9,6 +9,11 @@ import moe.log class MoePipeError(moe.MoeErr): """Failure of the MoePipeline.""" +class MoeAbortPipeline(Exception): + + def __init__(self, skip_to=999): + self.skip_to = skip_to + class MoePipeline: """Moe pipeline.""" @@ -31,10 +36,17 @@ class MoePipeline: def run(self, *args): self.index = 0 + min_pri = -1 while self.index < len(self.pipe): (pri,name,fun) = self.pipe[self.index] - moe.log.default.verbose(">> Running %s:%s\n" % (self.name,name)) - fun(*args) + if pri >= min_pri: + moe.log.default.verbose(">> Running %s:%s\n" % (self.name,name)) + try: + fun(*args) + except MoeAbortPipeline, err: + min_pri = err.skip_to + else: + moe.log.default.verbose(">> Skipping %s:%s\n" % (self.name,name)) self.index += 1 self.index = -1 diff --git a/t/moe/testcase.py b/t/moe/testcase.py new file mode 100644 index 0000000..fba4128 --- /dev/null +++ b/t/moe/testcase.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python + +import os.path +import moe +import moe.config +import moe.eval +import moe.log +import shutil + +def judge(e): + pass + +def configure_test(e, test): + e.cfgs = moe.config.MoeConfigStack(e.cfgs) + e.test_builtins = moe.config.MoeConfig(type="test-builtins") + e.test_builtins.set("TEST", test) + e.cfgs.push(e.test_builtins) + + test_cf = os.path.join(e.cfgs["PDIR"], test + ".config") + if os.path.exists(test_cf): + cfg = moe.config.MoeConfig(name=test_cf, type="test") + e.cfgs.push(cfg) + + e.cfgs.apply_overrides("TEST_" + test + "_") + ext = e.cfgs["EXT"] + if ext != "": + e.cfgs.apply_overrides("EXT_" + ext + "_") + + log = moe.log.MoeLog() + log.verbosity = e.log.verbosity + log.open(os.path.join(e.cfgs["TDIR"], test + ".log")) + log.say("Test case %s\n\n" % test) + e.log = log + + e.log_config(2, "for the test") + +def setup(e): + pdir = e.cfgs["PDIR"] + tdir = e.cfgs["TDIR"] + inn = e.cfgs["TESTCASE_IN"] + out = e.cfgs["TESTCASE_OUT"] + ok = e.cfgs["TESTCASE_OK"] + + if os.path.exists(os.path.join(pdir, inn)): + moe.util.link_or_copy(os.path.join(pdir, inn), os.path.join(tdir, inn)) + if os.path.exists(os.path.join(pdir, out)): + moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok)) + +def judge(e): + judge = e.cfgs["OUTPUT_CHECK"] + if judge == "": + return + + e.log.progress(" ") + e.log.verbose("Checking output: %s\n" % judge) + e.log.flush() + rc = os.system(judge) + ## FIXME: The judge might want to return a status file + if os.WIFEXITED(rc): + if os.WEXITSTATUS(rc) == 0: + return + elif os.WEXITSTATUS(rc) == 1: + raise moe.SolutionErr("Wrong answer", "WA") + raise moe.MoeErr("Judge failure") + +def run_test(e, test): + configure_test(e, test) + + ## FIXME: interactive tasks + e.test_pipe.configure(e.cfgs["TESTCASE_HOOKS"]) + if e.log.verbosity >= 2: + e.test_pipe.dump(e.log.log_file, prefix="\t") + e.test_pipe.run(e) + + e.log.progress("OK\n") + +def run_tests(e): + ## FIXME: output filter + e.test_pipe.insert(0, "setup", setup) + e.test_pipe.insert(400, "judge", judge) + + for test in e.cfgs["TESTS"].split(): + e.log.progress("Test %s: " % test) + old_cfgs = e.cfgs + old_log = e.log + + try: + run_test(e, test) + except moe.MoeErr, err: + e.log.progress("FAILED: %s\n" % err) + ## FIXME: write it to the status file + except moe.SolutionErr, err: + e.log.progress("%s\n" % err) + + e.cfgs = old_cfgs + e.log = old_log -- 2.39.2