TO = timeout
WA = wrong answer
PA = partial answer
+ NO = no output file generated
PE = protocol error (in case of interactive tasks)
XX = internal error (e.g., error when calling judge)
message: human-readable status message (not intended for machine parsing)
# HOME set automatically
# CONTESTANT set automatically
# TASK set automatically
-## FIXME: Rename?
-TASK_DIR="${HOME}/problems/${TASK}"
-SOL_DIR="${HOME}/solutions/${CONTESTANT}/${TASK}"
-TEST_DIR="${HOME}/testing/${CONTESTANT}/${TASK}"
+PDIR="${HOME}/problems/${TASK}"
+SDIR="${HOME}/solutions/${CONTESTANT}/${TASK}"
+TDIR="${HOME}/testing/${CONTESTANT}/${TASK}"
TASK_TYPE=batch
+TESTCASE_IN=${TEST}.in
+TESTCASE_OUT=${TEST}.out
+TESTCASE_OK=${TEST}.ok
+
+# HOOKS
+# TESTCASE_HOOKS
+
### Programming language settings
# Known source file extensions
COMP=false
# Sandbox options used when compiling
-COMP_SANDBOX_OPTS='-m262144 -w60 -e -i/dev/null'
+COMP_SANDBOX_OPTS="-m262144 -w60 -e -i/dev/null"
-# EXE is auto, but can be overridden
+EXE=$TASK
# Command used to execute the compiled program, may be ./$PROGRAM (default) or an
# interpreter with $PROGRAM as a parameter.
## Settings for individual languages
# C
-EXT_c_COMP='/usr/bin/gcc -std=gnu99 -O2 -g -o $EXE $EXTRA_CFLAGS $SRC -lm'
+EXT_c_COMP="/usr/bin/gcc -std=gnu99 -O2 -g -o $EXE $EXTRA_CFLAGS $SRC -lm"
EXTRA_CFLAGS=
# C++
-EXT_cpp_COMP='/usr/bin/g++ -O2 -g -o $EXE $EXTRA_CXXFLAGS $SRC -lm'
+EXT_cpp_COMP="/usr/bin/g++ -O2 -g -o $EXE $EXTRA_CXXFLAGS $SRC -lm"
EXTRA_CXXFLAGS=
# Pascal
-EXT_pas_COMP='/usr/bin/fpc -Ci -g -O2 -Sg -o$EXE $EXTRA_PFLAGS $SRC'
+EXT_pas_COMP="/usr/bin/fpc -Ci -g -O2 -Sg -o$EXE $EXTRA_PFLAGS $SRC"
EXTRA_PFLAGS=
### Per-task configuration variables (default values, override in per-task config)
# List of extra files needed for compilation. They are copied to the compiler
# sandbox from the problem's directory. XXX: or tdir
#COMP_EXTRAS="extras.h"
+
+# Task type:
+# batch off-line task
+# interactive interactive task communicating via stdio with a testing program
+# open-data open-data task (i.e., we don't submit program, but output files)
+TASK_TYPE=batch
+
+# I/O type (IO_TYPE sets defaults for IN_TYPE and OUT_TYPE)
+# file input from $PROBLEM.in, output to $PROBLEM.out (possible even for interactive tasks)
+# stdio input from stdin, output to stdout
+# dir input from all files in the directory $TEST.in; these are copied to $BOXDIR
+# and if they include .stdin, it will be available as program's std. input.
+# none no input/output
+IO_TYPE=stdio
+#IN_TYPE=stdio
+#OUT_TYPE=stdio
+
+IN_NAME=$TASK.in
+OUT_NAME=$TASK.out
+
+# A list of all tests
+TESTS="1 2 3 4 5 6 7 8 9 10"
+
+# A list of public tests (executed by submit and check scripts)
+SAMPLE_TESTS="0"
+
+# Number of points per test
+POINTS_PER_TEST=1
+
+# Time limit in seconds (can be fractional, but beware of noise)
+TIME_LIMIT=10
+
+# Memory limit in kilobytes
+MEM_LIMIT=16384
+
+# Stack size limit in kilobytes (0=limited only by MEM_LIMIT)
+STACK_LIMIT=0
+
+# Command used for filtering of program output (optional)
+# If turned on, program output (*.raw) is ran through this filter and the
+# checkers are applied to the output of the filter (*.out).
+# Can exit with code 1 if there is a syntax error in the output.
+#OUTPUT_FILTER=tr -d '\r' <$TDIR/$TEST.raw >$TDIR/$TEST.out
+
+# Command used to check output syntax (optional)
+# Returns exit code 1 if syntax is wrong, 0 if correct
+# fd1 is connect to evaluator log, feel free to log anything
+# fd2 is an optional one-line verdict
+#SYNTAX_CHECK=grep -v -- - $TDIR/$TEST.out
+
+# Command used to check output correctness
+# Returns exit code 1 if output is incorrect, 0 if correct
+# fd1 is connect to evaluator log, feel free to log anything
+# fd2 is an optional one-line verdict
+# The checker can generate $TDIR/$TEST.pts to assign points irregularly
+OUTPUT_CHECK=diff -bBu $TDIR/$TEST.ok $TDIR/$TEST.out
+
+# Checker for interactive tasks
+# Returns exit code 1 if test failed, 0 if passed
+# fd0 and fd1 are connected to fd1 and fd0 of the program tested
+# fd2 is an optional one-line verdict
+# The checker can generate $TDIR/$TEST.pts to assign points irregularly
+#IC_CHECK=$PDIR/checker $PDIR/$TEST.in $PDIR/$TEST.chk
+
+# Sandbox options used when testing
+TEST_SANDBOX_OPTS=-a2 -f -m$MEM_LIMIT -k$STACK_LIMIT -t$TIME_LIMIT $BOX_EXTRAS $BOX_IO_OPTS
+
+# Extra options to be overridden in task configuration
+BOX_EXTRAS=
+
pass
class SolutionErr(Exception):
- pass
+
+ def __init__(self, message, stat_code=None):
+ self.stat_code = stat_code
+ self.message = message
+
+ def __str__(self):
+ if self.stat_code is None:
+ return self.message
+ else:
+ return "%s: %s" % (self.stat_code, self.message)
import moe.box
import moe.eval
import moe.util
+import moe.pipeline
+import moe.testcase
import shutil
def normalize_ext(e, ext):
else:
dir, file = os.path.split(filename)
if dir == "":
- dir = e.cfgs["SOL_DIR"]
+ dir = e.cfgs["SDIR"]
base, ext = os.path.splitext(file)
if ext != "":
e.log.verbose("Found solution %s\n" % orig_path)
copy = e.cfgs["TASK"] + "." + norm_ext
- copy_path = os.path.join(e.cfgs["TEST_DIR"], copy)
+ copy_path = os.path.join(e.cfgs["TDIR"], copy)
if file != copy:
e.log.verbose("Renaming to %s\n" % copy)
moe.util.link_or_copy(orig_path, copy_path)
e.builtins.set("SRC", copy)
e.builtins.set("EXT", norm_ext)
- e.cfgs.apply_overrides("EXT_" + norm_ext)
+ e.cfgs.apply_overrides("EXT_" + norm_ext + "_")
e.stat["source"] = file
e.log.progress(file + "\n")
def compile_init(e):
- e.log.progress("Compiling: ")
+ e.log.progress("Compiling... ")
boxdir = moe.box.setup(e)
- pdir = e.cfgs["TASK_DIR"]
- tdir = e.cfgs["TEST_DIR"]
+ pdir = e.cfgs["PDIR"]
+ tdir = e.cfgs["TDIR"]
shutil.copyfile(os.path.join(tdir, e.cfgs["SRC"]), os.path.join(boxdir, e.cfgs["SRC"]))
- for x in e.cfgs["EXTRAS"].split() + e.cfgs["COMP_EXTRAS"].split()
+ for x in e.cfgs["EXTRAS"].split() + e.cfgs["COMP_EXTRAS"].split():
xx = os.path.join(tdir, x)
if not os.path.isfile(xx):
xx = os.path.join(pdir, x)
rc = moe.box.run(e, e.cfgs["COMP_SANDBOX_OPTS"], cc)
if rc > 0:
e.log.progress("FAILED\n")
- ## FIXME: fill in the status file and abort the pipeline?
+ ## FIXME: status file
+ raise moe.pipeline.MoeAbortPipeline(200)
moe.box.show(e, "compiler output")
def compile_done(e):
+ try:
+ shutil.copyfile(os.path.join(e.cfgs["BOXDIR"], e.cfgs["EXE"]), os.path.join(e.cfgs["TDIR"], e.cfgs["EXE"]))
+ except IOError:
+ raise moe.MoeErr, "Compiler succeeded, but produced no output"
e.log.progress("OK\n")
+def test_in(e):
+ tdir = e.cfgs["TDIR"]
+ boxdir = moe.box.setup(e)
+ inn = e.cfgs["TESTCASE_IN"]
+ in_type = e.cfgs["IN_TYPE"] or e.cfgs["IO_TYPE"]
+ out_type = e.cfgs["OUT_TYPE"] or e.cfgs["IO_TYPE"]
+ is_interactive = e.cfgs["TASK_TYPE"] == "interactive"
+ sandbox_opts = "-M" + os.path.join(tdir, e.cfgs["TEST"] + ".status")
+
+ if not os.path.exists(os.path.join(tdir, e.cfgs["EXE"])):
+ ## FIXME: status file
+ raise SolutionErr, "Compilation failed"
+ shutil.copyfile(os.path.join(tdir, e.cfgs["EXE"]), os.path.join(boxdir, e.cfgs["EXE"]))
+ os.chmod(os.path.join(boxdir, e.cfgs["EXE"]), 0555)
+
+ if in_type == "file":
+ in_name = e.cfgs["IN_NAME"]
+ e.log.verbose("Input file: %s (copied from %s)\n" % (in_name, os.path.join(e.cfgs["PDIR"], inn)))
+ shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, in_name))
+ if not is_interactive:
+ sandbox_opts = " -i/dev/null"
+ elif in_type == "stdio":
+ e.log.verbose("Input file: <stdin> (copied from %s)\n" % os.path.join(e.cfgs["PDIR"], inn))
+ shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, ".stdin"))
+ sandbox_opts = " -i.stdin"
+ elif in_type == "none":
+ e.log.verbose("Input file: <none>\n")
+ if not is_interactive:
+ sandbox_opts += " -i/dev/null"
+ elif in_type == "dir":
+ ## FIXME
+ raise MoeErr, "Directory input not yet implemented"
+ else:
+ raise MoeErr, "Unknown input type %s" % in_type
+
+ if out_type == "file":
+ out_name = e.cfgs["OUT_NAME"]
+ e.log.verbose("Output file: %s\n" % out_name)
+ if not is_interactive:
+ sandbox_opts += " -o/dev/null"
+ elif out_type == "stdio":
+ e.log.verbose("Output file: <stdout>\n")
+ sandbox_opts += " -o.stdout"
+ elif out_type == "none":
+ e.log.verbose("Output file: <none>\n")
+ if not is_interactive:
+ sandbox_opts += " -o/dev/null"
+ else:
+ raise MoeErr, "Unknown output type %s" % out_type
+
+ e.test_builtins.set("BOX_IO_OPTS", sandbox_opts)
+
+def test_run(e):
+ e.log.verbose("Time limit: %s s\n" % e.cfgs["TIME_LIMIT"])
+ e.log.verbose("Memory limit: %s KB\n" % e.cfgs["MEM_LIMIT"])
+ moe.box.show(e, "test input")
+ e.log.progress("<run> ")
+ moe.box.run(e, e.cfgs["TEST_SANDBOX_OPTS"], e.cfgs["TEST_EXEC_CMD"])
+ moe.box.show(e, "test output")
+ ## FIXME: Parse the status file and delete it
+ ### Check for runtime errors reported by the box
+
+def test_collect(e):
+ tdir = e.cfgs["TDIR"]
+ boxdir = e.cfgs["BOXDIR"]
+ out_type = e.cfgs["OUT_TYPE"] or e.cfgs["IO_TYPE"]
+ is_interactive = e.cfgs["TASK_TYPE"] == "interactive"
+
+ if out_type == "file":
+ out_path = e.cfgs["OUT_NAME"]
+ elif out_type == "stdio":
+ out_path = ".stdout"
+ if not os.path.exists(os.path.join(boxdir, out_path)):
+ raise moe.SolutionErr("No output file", "NO")
+ shutil.copyfile(os.path.join(boxdir, out_path), os.path.join(tdir, e.cfgs["TESTCASE_OUT"]))
+
def tests(e):
- pass
+ e.log.progress("\n")
+ e.test_pipe.insert(100, "prepare", test_in)
+ e.test_pipe.insert(200, "run", test_run)
+ e.test_pipe.insert(300, "collect", test_collect)
+ moe.testcase.run_tests(e)
def prepare_pipe(e):
e.main_pipe.insert(100, "compile-init", compile_init)
def run(e, opts, cmd):
c = e.cfgs["BOXCMD"] + " " + opts + " -- " + cmd
e.log.verbose("Sandbox: %s\n" % c)
- rc = os.system(c)
- if rc > 1:
- raise moe.MoeErr, "Sandbox failed"
- return rc
+ e.log.flush()
+ st = os.system(c)
+ if os.WIFEXITED(st):
+ rc = os.WEXITSTATUS(st)
+ if rc > 1:
+ raise moe.MoeErr, "Sandbox failed with rc=%d" % rc
+ return rc
+ else:
+ raise moe.MoeErr, "Sandbox failed with exit status 0x%04x" % rc
cfg = self.stk[pos]
if cfg.vars.has_key(k):
new = cfg.vars[k]
- if new[0][0] == "a":
+ if len(new) > 0 and new[0][0] == "a":
v = self.do_get(k, pos-1)
else:
v = ""
self.builtins = moe.config.MoeConfig(type="builtins")
self.cfgs.push(self.builtins)
self.main_pipe = moe.pipeline.MoePipeline("main")
+ self.test_pipe = moe.pipeline.MoePipeline("test")
self.stat = moe.status.MoeStatus()
pass
self.cfgs.push(overrides)
def init_test(self):
- test = self.cfgs['TEST_DIR']
+ test = self.cfgs['TDIR']
if os.path.isdir(test):
shutil.rmtree(test)
try:
self.log = moe.log.MoeLog()
if self.cfgs["V"]:
self.log.verbosity = int(self.cfgs["V"])
- self.log.open(os.path.join(self.cfgs["TEST_DIR"], "log"))
+ self.log.open(os.path.join(self.cfgs["TDIR"], "log"))
self.default_log = self.log
moe.log.default = self.log
self.log_config(3, "before loading the task")
def init_task(self):
task = self.cfgs['TASK']
- task_dir = self.cfgs['TASK_DIR']
+ task_dir = self.cfgs['PDIR']
if not os.path.isdir(task_dir):
raise moe.MoeErr, "No such task %s" % task
class MoePipeError(moe.MoeErr):
"""Failure of the MoePipeline."""
+class MoeAbortPipeline(Exception):
+
+ def __init__(self, skip_to=999):
+ self.skip_to = skip_to
+
class MoePipeline:
"""Moe pipeline."""
def run(self, *args):
self.index = 0
+ min_pri = -1
while self.index < len(self.pipe):
(pri,name,fun) = self.pipe[self.index]
- moe.log.default.verbose(">> Running %s:%s\n" % (self.name,name))
- fun(*args)
+ if pri >= min_pri:
+ moe.log.default.verbose(">> Running %s:%s\n" % (self.name,name))
+ try:
+ fun(*args)
+ except MoeAbortPipeline, err:
+ min_pri = err.skip_to
+ else:
+ moe.log.default.verbose(">> Skipping %s:%s\n" % (self.name,name))
self.index += 1
self.index = -1
--- /dev/null
+#!/usr/bin/env python
+
+import os.path
+import moe
+import moe.config
+import moe.eval
+import moe.log
+import shutil
+
+def judge(e):
+ pass
+
+def configure_test(e, test):
+ e.cfgs = moe.config.MoeConfigStack(e.cfgs)
+ e.test_builtins = moe.config.MoeConfig(type="test-builtins")
+ e.test_builtins.set("TEST", test)
+ e.cfgs.push(e.test_builtins)
+
+ test_cf = os.path.join(e.cfgs["PDIR"], test + ".config")
+ if os.path.exists(test_cf):
+ cfg = moe.config.MoeConfig(name=test_cf, type="test")
+ e.cfgs.push(cfg)
+
+ e.cfgs.apply_overrides("TEST_" + test + "_")
+ ext = e.cfgs["EXT"]
+ if ext != "":
+ e.cfgs.apply_overrides("EXT_" + ext + "_")
+
+ log = moe.log.MoeLog()
+ log.verbosity = e.log.verbosity
+ log.open(os.path.join(e.cfgs["TDIR"], test + ".log"))
+ log.say("Test case %s\n\n" % test)
+ e.log = log
+
+ e.log_config(2, "for the test")
+
+def setup(e):
+ pdir = e.cfgs["PDIR"]
+ tdir = e.cfgs["TDIR"]
+ inn = e.cfgs["TESTCASE_IN"]
+ out = e.cfgs["TESTCASE_OUT"]
+ ok = e.cfgs["TESTCASE_OK"]
+
+ if os.path.exists(os.path.join(pdir, inn)):
+ moe.util.link_or_copy(os.path.join(pdir, inn), os.path.join(tdir, inn))
+ if os.path.exists(os.path.join(pdir, out)):
+ moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok))
+
+def judge(e):
+ judge = e.cfgs["OUTPUT_CHECK"]
+ if judge == "":
+ return
+
+ e.log.progress("<check> ")
+ e.log.verbose("Checking output: %s\n" % judge)
+ e.log.flush()
+ rc = os.system(judge)
+ ## FIXME: The judge might want to return a status file
+ if os.WIFEXITED(rc):
+ if os.WEXITSTATUS(rc) == 0:
+ return
+ elif os.WEXITSTATUS(rc) == 1:
+ raise moe.SolutionErr("Wrong answer", "WA")
+ raise moe.MoeErr("Judge failure")
+
+def run_test(e, test):
+ configure_test(e, test)
+
+ ## FIXME: interactive tasks
+ e.test_pipe.configure(e.cfgs["TESTCASE_HOOKS"])
+ if e.log.verbosity >= 2:
+ e.test_pipe.dump(e.log.log_file, prefix="\t")
+ e.test_pipe.run(e)
+
+ e.log.progress("OK\n")
+
+def run_tests(e):
+ ## FIXME: output filter
+ e.test_pipe.insert(0, "setup", setup)
+ e.test_pipe.insert(400, "judge", judge)
+
+ for test in e.cfgs["TESTS"].split():
+ e.log.progress("Test %s: " % test)
+ old_cfgs = e.cfgs
+ old_log = e.log
+
+ try:
+ run_test(e, test)
+ except moe.MoeErr, err:
+ e.log.progress("FAILED: %s\n" % err)
+ ## FIXME: write it to the status file
+ except moe.SolutionErr, err:
+ e.log.progress("%s\n" % err)
+
+ e.cfgs = old_cfgs
+ e.log = old_log
projects / eval.git / commitdiff