# The Evaluator -- Shell Function Library
-# (c) 2001 Martin Mares <mj@ucw.cz>
+# (c) 2001--2004 Martin Mares <mj@ucw.cz>
# General settings
shopt -s dotglob
function die
{
+ # Report an internal error
+ echo >&2 "$@"
+ [ -n "$HAVE_LOG" ] && echo "Fatal error: $@"
+ exit 2
+}
+
+function fatal
+{
+ # Report a fatal error in the program being tested
echo >&2 "$@"
[ -n "$HAVE_LOG" ] && echo "Fatal error: $@"
exit 1
function box-init
{
pstart "Preparing sandbox... "
- [ -n "$TEST_USER" ] || die "TEST_USER not set. Please fix."
- if [ $TEST_USER == $EVAL_USER ] ; then
+ if [ -z "$TEST_USER" -o "$TEST_USER" == $EVAL_USER ] ; then
pcont "running locally (INSECURE), "
+ TEST_USER=$EVAL_USER
BOXDIR=box
BOXCMD=bin/box
mkdir -p box
else
pcont "used account $TEST_USER, "
- BOXDIR=$MO_ROOT/$TEST_USER
+ BOXDIR=$MO_ROOT/eval/$TEST_USER
BOXCMD=bin/box-$TEST_USER
fi
[ -d $BOXDIR -a -f $BOXCMD ] || die "Sandbox set up incorrectly"
function dir-init
{
pstart "Initializing... "
+ HDIR=.
PDIR=problems/$PROBLEM
SDIR=solutions/$CONTESTANT/$PROBLEM
TDIR=testing/$CONTESTANT/$PROBLEM
+ TMPDIR=tmp
[ -d $PDIR ] || die "Problem $PROBLEM not known"
- [ -d $SDIR ] || die "Solution of $PROBLEM not found"
- mkdir -p $TDIR
- rm -rf $TDIR
- cp -a $SDIR $TDIR
+ [ -d $SDIR ] || fatal "Solution of $PROBLEM not found"
+ mkdir -p $TDIR $TMPDIR
+ rm -rf $TDIR $TMPDIR
+ mkdir -p $TDIR $TMPDIR
cat >$TDIR/log <<EOF
Testing solution of $PROBLEM by $CONTESTANT
Test started at `date`
Problem directory: $PDIR
Testing directory: $TDIR
EOF
- >$TDIR/points
pend "OK"
}
function locate-source
{
pstart "Finding source... "
+ if [ -n "$1" ] ; then
+ SRCBASE=$(echo $1 | sed 's/\.\([^.]\+\)//')
+ SRCEXT=$(echo $1 | sed 's/.*\.\([^.]\+\)/\1/')
+ if [ -n "$SRCEXT" ] ; then
+ # Full name given, so just check the extension and existence
+ SRCN=$1
+ if [ ${SRCN:0:1} == / ] ; then
+ SRCN=${SRCN:1}
+ SDIR=
+ fi
+ [ -f "$SDIR/$SRCN" ] || die "Cannot find source file $SDIR/$SRCN"
+ SRCEXT_OK=
+ for a in $EXTENSIONS ; do
+ if [ $a == $SRCEXT ] ; then
+ pend $SDIR/$SRCN
+ echo "Explicitly set source file: $SDIR/$SRCN"
+ return 0
+ fi
+ done
+ die "Unknown extension .$SRCEXT"
+ fi
+ else
+ SBASE=$PROBLEM
+ fi
for a in $EXTENSIONS ; do
- if [ -f $SDIR/$PROBLEM.$a ] ; then
+ if [ -f $SDIR/$SBASE.$a ] ; then
[ -z "$SRCN" ] || die "Multiple source files found: $SDIR/$PROBLEM.$a and $SDIR/$SRCN. Please fix."
- SRCN=$PROBLEM.$a
+ SRCN=$SBASE.$a
SRCEXT=$a
fi
done
- [ -n "$SRCN" ] || die "NOT FOUND"
+ [ -n "$SRCN" ] || fatal "NOT FOUND"
pend $SRCN
echo "Found source file: $SDIR/$SRCN"
}
-# Compilation (compile SDIR/SRCN with PDIR/EXTRAS to EXE=TDIR/PROBLEM)
+# Compilation (compile SDIR/SRCN with PDIR/COMP_EXTRAS to EXE=TDIR/PROBLEM)
function compile
{
pstart "Compiling... "
- cp -a $SDIR/$SRCN $TDIR/$SRCN
- if [ -n "$EXTRAS" ] ; then
- echo "Extras: $EXTRAS"
- for a in $EXTRAS ; do cp $PDIR/$a $TDIR/ ; done
+ # Beware, the original SRCN can be a strange user-supplied name
+ SRC=$PROBLEM.$SRCEXT
+ cp -a $SDIR/$SRCN $TDIR/$SRC
+ if [ -n "$COMP_EXTRAS" ] ; then
+ echo "Extras: $COMP_EXTRAS"
+ for a in $COMP_EXTRAS ; do cp $PDIR/$a $TDIR/ ; done
fi
box-clean
- for a in $SRCN $EXTRAS ; do cp $TDIR/$a $BOXDIR/ ; done
- SRC=$SRCN
+ for a in $SRC $COMP_EXTRAS ; do cp $TDIR/$a $BOXDIR/ ; done
EXE=$PROBLEM
CCMD=COMP_$SRCEXT
- CCMD="`eval echo ${!CCMD}`"
- COMP_SANDBOX_OPTS="`eval echo $COMP_SANDBOX_OPTS`"
+ CCMD=`eval echo ${!CCMD}`
+ COMP_SANDBOX_OPTS=`eval echo $COMP_SANDBOX_OPTS`
echo "Compiler command: $CCMD"
echo "Compiler sandbox options: $COMP_SANDBOX_OPTS"
+ eval $COMP_SANDBOX_INIT
echo "Compiler input files:"
ls -Al $BOXDIR
pend "OK"
}
-# Running of test program with file input/output
+# Running of test program according to current task type (returns exit code and TEST_MSG)
function test-run
+{
+ test-run-$TASK_TYPE
+}
+
+function test-result
+{
+ P=$1
+ M=$2
+ if [ -s $TDIR/$TEST.pts ] ; then
+ P=`cat $TDIR/$TEST.pts`
+ rm $TDIR/$TEST.pts
+ fi
+ echo "Verdict: $M"
+ echo "Points: $P"
+ test-verdict $P "$M"
+}
+
+function test-prolog
{
pcont "<init> "
box-clean
echo "Executable file: $TDIR/$PROBLEM"
+ if [ ! -x $TDIR/$PROBLEM ] ; then
+ test-result 0 "Compile error."
+ fi
cp $TDIR/$PROBLEM $BOXDIR/
- echo "Input: $TDIR/$PROBLEM"
- ln $PDIR/$TEST.in $TDIR/$TEST.in
- cp $PDIR/$TEST.in $BOXDIR/$PROBLEM.in
- echo "Input files:"
+ BOX_EXTRAS=
+ IN_TYPE=${IN_TYPE:-$IO_TYPE}
+ OUT_TYPE=${OUT_TYPE:-$IO_TYPE}
+ case $IN_TYPE in
+ file) echo "Input file: $PROBLEM.in (from $PDIR/$TEST.in)"
+ ln $PDIR/$TEST.in $TDIR/$TEST.in
+ cp $PDIR/$TEST.in $BOXDIR/$PROBLEM.in
+ BOX_EXTRAS="$BOX_EXTRAS -i/dev/null"
+ ;;
+ stdio) echo "Input file: <stdin> (from $PDIR/$TEST.in)"
+ ln $PDIR/$TEST.in $TDIR/$TEST.in
+ cp $PDIR/$TEST.in $BOXDIR/.stdin
+ BOX_EXTRAS="$BOX_EXTRAS -i.stdin"
+ ;;
+ none) echo "Input file: <none>"
+ ;;
+ *) die "Unknown IN_TYPE $IN_TYPE"
+ ;;
+ esac
+ case $OUT_TYPE in
+ file) echo "Output file: $PROBLEM.out"
+ BOX_EXTRAS="$BOX_EXTRAS -o/dev/null"
+ ;;
+ stdio) echo "Output file: <stdout>"
+ BOX_EXTRAS="$BOX_EXTRAS -o.stdout"
+ ;;
+ none) echo "Output file: <none>"
+ ;;
+ *) die "Unknown OUT_TYPE $OUT_TYPE"
+ ;;
+ esac
+ echo "Timeout: $TIME_LIMIT s"
+ echo "Memory: $MEM_LIMIT KB"
+ eval $SANDBOX_INIT
+ echo "Sandbox contents before start:"
ls -Al $BOXDIR
+}
+function test-epilog
+{
+ echo "Sandbox contents after exit:"
+ ls -Al $BOXDIR
+ case ${OUT_TYPE:-$IO_TYPE} in
+ file) [ -f $BOXDIR/$PROBLEM.out ] || test-result 0 "No output file."
+ cp $BOXDIR/$PROBLEM.out $TDIR/$TEST.out
+ ;;
+ stdio) [ -f $BOXDIR/.stdout ] || test-result 0 "No output file."
+ cp $BOXDIR/.stdout $TDIR/$TEST.out
+ ;;
+ esac
+}
+
+# Running of test program with file input/output
+
+function test-run-file
+{
+ test-prolog
pcont "<run> "
- echo "Timeout: $TIME_LIMIT s"
- echo "Memory: $MEM_LIMIT KB"
- BOXOPTS="`eval echo $TEST_SANDBOX_OPTS`"
+ BOXOPTS="`eval echo $TEST_SANDBOX_OPTS`$BOX_EXTRAS"
echo "Sandbox options: $BOXOPTS"
- if ! $BOXCMD $BOXOPTS -- ./$PROBLEM 2>$TDIR/exec.out ; then
- TEST_MSG="`cat $TDIR/exec.out`"
- pend "$TEST_MSG"
- echo "$TEST_MSG"
- echo >>$PTSFILE "0 $TEST_MSG"
- return 1
+ if ! $BOXCMD $BOXOPTS -- ./$PROBLEM 2>$TMPDIR/exec.out ; then
+ cat $TMPDIR/exec.out
+ MSG=`tail -1 $TMPDIR/exec.out`
+ test-result 0 "$MSG"
fi
- cat $TDIR/exec.out
- rm $TDIR/exec.out
- echo "Output files:"
- ls -Al $BOXDIR
- if [ ! -s $BOXDIR/$PROBLEM.out ] ; then
- pend "No output file."
- echo "No output file."
- echo >>$PTSFILE "0 No output."
- return 1
+ cat $TMPDIR/exec.out
+ test-epilog
+}
+
+# Running of interactive test programs
+
+function test-run-interactive
+{
+ test-prolog
+ pcont "<run> "
+ BOXOPTS="`eval echo $TEST_SANDBOX_OPTS`$BOX_EXTRAS"
+ echo "Sandbox options: $BOXOPTS"
+ ICCMD=`eval echo $IA_CHECK`
+ echo "Interactive checker: $ICCMD"
+ if ! $HDIR/bin/iwrapper $BOXCMD $BOXOPTS -- ./$PROBLEM @@ $ICCMD 2>$TMPDIR/exec.out ; then
+ cat $TMPDIR/exec.out
+ MSG="`tail -1 $TMPDIR/exec.out`"
+ test-result 0 "$MSG"
fi
- cp $BOXDIR/$PROBLEM.out $TDIR/$TEST.out
+ cat $TMPDIR/exec.out
+ test-epilog
}
# Syntax checks
{
[ -n "$SYNTAX_CHECK" ] || return 0
pcont "<syntax> "
- SCHECK="`eval echo $SYNTAX_CHECK`"
+ SCHECK=`eval echo $SYNTAX_CHECK`
echo "Syntax check command: $SCHECK"
- $SCHECK && return 0
- pend "Wrong syntax."
- echo "Wrong syntax."
- echo >>$PTSFILE "0 Wrong syntax."
- return 1
+ if ! eval $SCHECK 2>$TMPDIR/exec.out ; then
+ cat $TMPDIR/exec.out
+ MSG=`tail -1 $TMPDIR/exec.out`
+ if [ -z "$MSG" ] ; then MSG="Wrong syntax." ; fi
+ test-result 0 "$MSG"
+ fi
+ cat $TMPDIR/exec.out
}
# Output checks
function output-check
{
+ [ -n "$OUTPUT_CHECK" ] || return 0
pcont "<check> "
- ln $PDIR/$TEST.out $TDIR/$TEST.ok
- OCHECK="`eval echo $OUTPUT_CHECK`"
+ [ -f $PDIR/$TEST.out ] && ln $PDIR/$TEST.out $TDIR/$TEST.ok
+ OCHECK=`eval echo $OUTPUT_CHECK`
echo "Output check command: $OCHECK"
- $OCHECK && return 0
- pend "Wrong answer."
- echo "Wrong answer."
- echo >>$PTSFILE "0 Wrong answer."
- return 1
+ if ! eval $OCHECK 2>$TMPDIR/exec.out ; then
+ cat $TMPDIR/exec.out
+ MSG=`tail -1 $TMPDIR/exec.out`
+ if [ -z "$MSG" ] ; then MSG="Wrong answer." ; fi
+ test-result 0 "$MSG"
+ fi
+ cat $TMPDIR/exec.out
+ MSG=`tail -1 $TMPDIR/exec.out`
+ if [ -z "$MSG" ] ; then MSG="OK" ; fi
+ test-result $POINTS_PER_TEST "$MSG"
}
# Setup of public commands
function public-setup
{
- PDIR=$MO_PUBLIC/problems/$PROBLEM
+ HDIR=$MO_ROOT
+ PDIR=$MO_ROOT/problems/$PROBLEM
SDIR=.
TDIR=~/.test
+ TMPDIR=~/.test
[ -d $PDIR ] || die "Unknown problem $PROBLEM"
pstart "Initializing... "
BOXDIR=~/.box
mkdir -p $BOXDIR
rm -rf $BOXDIR/*
- BOXCMD="$MO_PUBLIC/bin/box -c$BOXDIR"
+ BOXCMD="$MO_ROOT/bin/box -c$BOXDIR"
exec >log
pend "OK (see 'log' for details)"
}
+
+# Locate output of open data problem, test case TEST
+
+function open-locate
+{
+ [ -f $PDIR/$TEST.in ] || die "Unknown test $TEST"
+ if [ -n "$1" ] ; then
+ SRCN=$1
+ else
+ SRCN=$SDIR/$PROBLEM$TEST.out
+ fi
+ [ -f $SRCN ] || fatal "Output file $SRCN not found"
+}