From: Tomas Gavenciak <gavento@ucw.cz>
Date: Fri, 24 Sep 2010 14:13:12 +0000 (+0200)
Subject: Moving things around, WIP batch tasktype via hooks
X-Git-Url: http://mj.ucw.cz/gitweb/?a=commitdiff_plain;h=538b950bac4d1f3baea968c5f3a5adb406218f05;p=eval.git

Moving things around, WIP batch tasktype via hooks

Trying to make testutils a collection of useful functions and hooks
and to make tasktypes/batch as minimalistic as possible
(and to make writing new task types easy as well)
---

diff --git a/t/moe/batch.py b/t/moe/batch.py
deleted file mode 100644
index 9c83983..0000000
--- a/t/moe/batch.py
+++ /dev/null
@@ -1,193 +0,0 @@
-#!/usr/bin/env python
-
-import os.path
-import moe
-import moe.box
-import moe.eval
-import moe.util
-import moe.pipeline
-import moe.testcase
-import shutil
-
-def normalize_ext(e, ext):
-    alias = e["ALIAS_EXT_" + ext]
-    return alias if alias != "" else ext
-
-def try_ext(e, ext):
-    for e in e["EXTENSIONS"].split():
-	if e == ext:
-	    return
-    raise moe.MoeError, "Unknown extension: " + ext
-
-def locate(e, filename=None):
-    e.log.progress("Locating source... ")
-    task = e["TASK"]
-    if filename is None:
-	dir = ""
-	file = task
-    else:
-	dir, file = os.path.split(filename)
-    if dir == "":
-	dir = e["SDIR"]
-
-    base, ext = os.path.splitext(file)
-    if ext != "":
-	if not os.path.exists(os.path.join(dir, file)):
-	    raise moe.SolutionError, "No solution of %s called %s found" % (task,file)
-	ext = ext[1:]
-	try_ext(e, ext)
-    else:
-	found = []
-	for ext in e["EXTENSIONS"].split():
-	    if os.path.exists(os.path.join(dir, base + "." + ext)):
-		found.append(ext)
-	if len(found) == 0:
-	    raise moe.SolutionError, "No solution of %s found" % task
-	if len(found) > 1:
-	    raise moe.SolutionError, "Multiple solutions of %s found" % task
-	ext = found[0]
-	file = base + "." + ext
-
-    orig_path = os.path.join(dir, file)
-    norm_ext = normalize_ext(e, ext)
-    e.log.verbose("Found solution %s\n" % orig_path)
-
-    copy = e["TASK"] + "." + norm_ext
-    copy_path = os.path.join(e["TDIR"], copy)
-    if file != copy:
-	e.log.verbose("Renaming to %s\n" % copy)
-    moe.util.link_or_copy(orig_path, copy_path)
-
-    e.builtins.set("SRC", copy)
-    e.builtins.set("EXT", norm_ext)
-    e.cfgs.apply_overrides("EXT_" + norm_ext + "_")
-
-    e.stat["source"] = file
-    e.stat["lang"] = ext
-    e.log.progress(file + "\n")
-
-def compile_init(e):
-    e.log.progress("Compiling... ")
-    boxdir = moe.box.setup(e)
-    pdir = e["PDIR"]
-    tdir = e["TDIR"]
-    shutil.copyfile(os.path.join(tdir, e["SRC"]), os.path.join(boxdir, e["SRC"]))
-    for x in e["EXTRAS"].split() + e["COMP_EXTRAS"].split():
-	xx = os.path.join(tdir, x)
-	if not os.path.isfile(xx):
-	    xx = os.path.join(pdir, x)
-	e.log.verbose("Copying extra file %s\n" % xx)
-	shutil.copyfile(xx, os.path.join(boxdir, x))
-
-def compile_run(e):
-    moe.box.show(e, "compiler input")
-    cc = e["COMP"]
-    e.log.verbose("Compilation command: %s\n" % cc)
-    rc = moe.box.run(e, e["COMP_SANDBOX_OPTS"], cc)
-    if rc > 0:
-	e.log.progress("FAILED\n")
-	## FIXME: status file ... or generate an exception?
-	raise moe.pipeline.MoeAbortPipeline(200)
-    moe.box.show(e, "compiler output")
-
-def compile_done(e):
-    try:
-	shutil.copyfile(os.path.join(e["BOXDIR"], e["EXE"]), os.path.join(e["TDIR"], e["EXE"]))
-    except IOError:
-	raise moe.MoeError, "Compiler succeeded, but produced no output"
-    e.log.progress("OK\n")
-
-def test_in(e):
-    tdir = e["TDIR"]
-    boxdir = moe.box.setup(e)
-    inn = e["TESTCASE_IN"]
-    in_type = e["IN_TYPE"] or e["IO_TYPE"]
-    out_type = e["OUT_TYPE"] or e["IO_TYPE"]
-    is_interactive = e["TASK_TYPE"] == "interactive"
-    sandbox_opts = "-M" + os.path.join(tdir, e["TESTCASE_STATUS"])
-
-    if not os.path.exists(os.path.join(tdir, e["EXE"])):
-	raise TestError("Compilation failed", "CE")
-    shutil.copyfile(os.path.join(tdir, e["EXE"]), os.path.join(boxdir, e["EXE"]))
-    os.chmod(os.path.join(boxdir, e["EXE"]), 0555)
-
-    if in_type == "file":
-	in_name = e["IN_NAME"]
-	e.log.verbose("Input file: %s (copied from %s)\n" % (in_name, os.path.join(e["PDIR"], inn)))
-	try:
-	    shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, in_name))
-	except IOError:
-	    raise moe.MoeError, "Input file not found"
-	if not is_interactive:
-	    sandbox_opts += " -i/dev/null"
-    elif in_type == "stdio":
-	e.log.verbose("Input file: <stdin> (copied from %s)\n" % os.path.join(e["PDIR"], inn))
-	try:
-	    shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, ".stdin"))
-	except IOError:
-	    raise moe.MoeError, "Input file not found"
-	sandbox_opts += " -i.stdin"
-    elif in_type == "none":
-	e.log.verbose("Input file: <none>\n")
-	if not is_interactive:
-	    sandbox_opts += " -i/dev/null"
-    elif in_type == "dir":
-	## FIXME
-	raise MoeError, "Directory input not yet implemented"
-    else:
-	raise MoeError, "Unknown input type %s" % in_type
-
-    if out_type == "file":
-	out_name = e["OUT_NAME"]
-	e.log.verbose("Output file: %s\n" % out_name)
-	if not is_interactive:
-	    sandbox_opts += " -o/dev/null"
-    elif out_type == "stdio":
-	e.log.verbose("Output file: <stdout>\n")
-	sandbox_opts += " -o.stdout"
-    elif out_type == "none":
-	e.log.verbose("Output file: <none>\n")
-	if not is_interactive:
-	    sandbox_opts += " -o/dev/null"
-    else:
-	raise MoeError, "Unknown output type %s" % out_type
-
-    e.test_builtins.set("BOX_IO_OPTS", sandbox_opts)
-
-def test_run(e):
-    e.log.verbose("Time limit: %s s\n" % e["TIME_LIMIT"])
-    e.log.verbose("Memory limit: %s KB\n" % e["MEM_LIMIT"])
-    moe.box.show(e, "test input")
-    e.log.progress("<run> ")
-    rc = moe.box.run(e, e["TEST_SANDBOX_OPTS"], e["TEST_EXEC_CMD"])
-    moe.testcase.collect_status(e)
-    moe.box.show(e, "test output")
-    if rc > 0:
-	raise moe.TestError("Wrong answer", "WA")
-
-def test_collect(e):
-    tdir = e["TDIR"]
-    boxdir = e["BOXDIR"]
-    out_type = e["OUT_TYPE"] or e["IO_TYPE"]
-    is_interactive = e["TASK_TYPE"] == "interactive"
-
-    if out_type == "file":
-	out_path = e["OUT_NAME"]
-    elif out_type == "stdio":
-	out_path = ".stdout"
-    if not os.path.exists(os.path.join(boxdir, out_path)):
-	raise moe.TestError("No output file", "NO")
-    shutil.copyfile(os.path.join(boxdir, out_path), os.path.join(tdir, e["TESTCASE_OUT"]))
-
-def tests(e):
-    e.log.progress("\n")
-    e.test_pipe.insert(100, "prepare", test_in)
-    e.test_pipe.insert(200, "run", test_run)
-    e.test_pipe.insert(300, "collect", test_collect)
-    moe.testcase.run_tests(e)
-
-def prepare_pipe(e):
-    e.main_pipe.insert(100, "compile-init", compile_init)
-    e.main_pipe.insert(150, "compile-run", compile_run)
-    e.main_pipe.insert(190, "compile-done", compile_done)
-    e.main_pipe.insert(200, "batch-tests", tests)
diff --git a/t/moe/tasktypes/batch.py b/t/moe/tasktypes/batch.py
new file mode 100644
index 0000000..9c83983
--- /dev/null
+++ b/t/moe/tasktypes/batch.py
@@ -0,0 +1,193 @@
+#!/usr/bin/env python
+
+import os.path
+import moe
+import moe.box
+import moe.eval
+import moe.util
+import moe.pipeline
+import moe.testcase
+import shutil
+
+def normalize_ext(e, ext):
+    alias = e["ALIAS_EXT_" + ext]
+    return alias if alias != "" else ext
+
+def try_ext(e, ext):
+    for e in e["EXTENSIONS"].split():
+	if e == ext:
+	    return
+    raise moe.MoeError, "Unknown extension: " + ext
+
+def locate(e, filename=None):
+    e.log.progress("Locating source... ")
+    task = e["TASK"]
+    if filename is None:
+	dir = ""
+	file = task
+    else:
+	dir, file = os.path.split(filename)
+    if dir == "":
+	dir = e["SDIR"]
+
+    base, ext = os.path.splitext(file)
+    if ext != "":
+	if not os.path.exists(os.path.join(dir, file)):
+	    raise moe.SolutionError, "No solution of %s called %s found" % (task,file)
+	ext = ext[1:]
+	try_ext(e, ext)
+    else:
+	found = []
+	for ext in e["EXTENSIONS"].split():
+	    if os.path.exists(os.path.join(dir, base + "." + ext)):
+		found.append(ext)
+	if len(found) == 0:
+	    raise moe.SolutionError, "No solution of %s found" % task
+	if len(found) > 1:
+	    raise moe.SolutionError, "Multiple solutions of %s found" % task
+	ext = found[0]
+	file = base + "." + ext
+
+    orig_path = os.path.join(dir, file)
+    norm_ext = normalize_ext(e, ext)
+    e.log.verbose("Found solution %s\n" % orig_path)
+
+    copy = e["TASK"] + "." + norm_ext
+    copy_path = os.path.join(e["TDIR"], copy)
+    if file != copy:
+	e.log.verbose("Renaming to %s\n" % copy)
+    moe.util.link_or_copy(orig_path, copy_path)
+
+    e.builtins.set("SRC", copy)
+    e.builtins.set("EXT", norm_ext)
+    e.cfgs.apply_overrides("EXT_" + norm_ext + "_")
+
+    e.stat["source"] = file
+    e.stat["lang"] = ext
+    e.log.progress(file + "\n")
+
+def compile_init(e):
+    e.log.progress("Compiling... ")
+    boxdir = moe.box.setup(e)
+    pdir = e["PDIR"]
+    tdir = e["TDIR"]
+    shutil.copyfile(os.path.join(tdir, e["SRC"]), os.path.join(boxdir, e["SRC"]))
+    for x in e["EXTRAS"].split() + e["COMP_EXTRAS"].split():
+	xx = os.path.join(tdir, x)
+	if not os.path.isfile(xx):
+	    xx = os.path.join(pdir, x)
+	e.log.verbose("Copying extra file %s\n" % xx)
+	shutil.copyfile(xx, os.path.join(boxdir, x))
+
+def compile_run(e):
+    moe.box.show(e, "compiler input")
+    cc = e["COMP"]
+    e.log.verbose("Compilation command: %s\n" % cc)
+    rc = moe.box.run(e, e["COMP_SANDBOX_OPTS"], cc)
+    if rc > 0:
+	e.log.progress("FAILED\n")
+	## FIXME: status file ... or generate an exception?
+	raise moe.pipeline.MoeAbortPipeline(200)
+    moe.box.show(e, "compiler output")
+
+def compile_done(e):
+    try:
+	shutil.copyfile(os.path.join(e["BOXDIR"], e["EXE"]), os.path.join(e["TDIR"], e["EXE"]))
+    except IOError:
+	raise moe.MoeError, "Compiler succeeded, but produced no output"
+    e.log.progress("OK\n")
+
+def test_in(e):
+    tdir = e["TDIR"]
+    boxdir = moe.box.setup(e)
+    inn = e["TESTCASE_IN"]
+    in_type = e["IN_TYPE"] or e["IO_TYPE"]
+    out_type = e["OUT_TYPE"] or e["IO_TYPE"]
+    is_interactive = e["TASK_TYPE"] == "interactive"
+    sandbox_opts = "-M" + os.path.join(tdir, e["TESTCASE_STATUS"])
+
+    if not os.path.exists(os.path.join(tdir, e["EXE"])):
+	raise TestError("Compilation failed", "CE")
+    shutil.copyfile(os.path.join(tdir, e["EXE"]), os.path.join(boxdir, e["EXE"]))
+    os.chmod(os.path.join(boxdir, e["EXE"]), 0555)
+
+    if in_type == "file":
+	in_name = e["IN_NAME"]
+	e.log.verbose("Input file: %s (copied from %s)\n" % (in_name, os.path.join(e["PDIR"], inn)))
+	try:
+	    shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, in_name))
+	except IOError:
+	    raise moe.MoeError, "Input file not found"
+	if not is_interactive:
+	    sandbox_opts += " -i/dev/null"
+    elif in_type == "stdio":
+	e.log.verbose("Input file: <stdin> (copied from %s)\n" % os.path.join(e["PDIR"], inn))
+	try:
+	    shutil.copyfile(os.path.join(tdir, inn), os.path.join(boxdir, ".stdin"))
+	except IOError:
+	    raise moe.MoeError, "Input file not found"
+	sandbox_opts += " -i.stdin"
+    elif in_type == "none":
+	e.log.verbose("Input file: <none>\n")
+	if not is_interactive:
+	    sandbox_opts += " -i/dev/null"
+    elif in_type == "dir":
+	## FIXME
+	raise MoeError, "Directory input not yet implemented"
+    else:
+	raise MoeError, "Unknown input type %s" % in_type
+
+    if out_type == "file":
+	out_name = e["OUT_NAME"]
+	e.log.verbose("Output file: %s\n" % out_name)
+	if not is_interactive:
+	    sandbox_opts += " -o/dev/null"
+    elif out_type == "stdio":
+	e.log.verbose("Output file: <stdout>\n")
+	sandbox_opts += " -o.stdout"
+    elif out_type == "none":
+	e.log.verbose("Output file: <none>\n")
+	if not is_interactive:
+	    sandbox_opts += " -o/dev/null"
+    else:
+	raise MoeError, "Unknown output type %s" % out_type
+
+    e.test_builtins.set("BOX_IO_OPTS", sandbox_opts)
+
+def test_run(e):
+    e.log.verbose("Time limit: %s s\n" % e["TIME_LIMIT"])
+    e.log.verbose("Memory limit: %s KB\n" % e["MEM_LIMIT"])
+    moe.box.show(e, "test input")
+    e.log.progress("<run> ")
+    rc = moe.box.run(e, e["TEST_SANDBOX_OPTS"], e["TEST_EXEC_CMD"])
+    moe.testcase.collect_status(e)
+    moe.box.show(e, "test output")
+    if rc > 0:
+	raise moe.TestError("Wrong answer", "WA")
+
+def test_collect(e):
+    tdir = e["TDIR"]
+    boxdir = e["BOXDIR"]
+    out_type = e["OUT_TYPE"] or e["IO_TYPE"]
+    is_interactive = e["TASK_TYPE"] == "interactive"
+
+    if out_type == "file":
+	out_path = e["OUT_NAME"]
+    elif out_type == "stdio":
+	out_path = ".stdout"
+    if not os.path.exists(os.path.join(boxdir, out_path)):
+	raise moe.TestError("No output file", "NO")
+    shutil.copyfile(os.path.join(boxdir, out_path), os.path.join(tdir, e["TESTCASE_OUT"]))
+
+def tests(e):
+    e.log.progress("\n")
+    e.test_pipe.insert(100, "prepare", test_in)
+    e.test_pipe.insert(200, "run", test_run)
+    e.test_pipe.insert(300, "collect", test_collect)
+    moe.testcase.run_tests(e)
+
+def prepare_pipe(e):
+    e.main_pipe.insert(100, "compile-init", compile_init)
+    e.main_pipe.insert(150, "compile-run", compile_run)
+    e.main_pipe.insert(190, "compile-done", compile_done)
+    e.main_pipe.insert(200, "batch-tests", tests)
diff --git a/t/moe/testcase.py b/t/moe/testcase.py
deleted file mode 100644
index 36d335b..0000000
--- a/t/moe/testcase.py
+++ /dev/null
@@ -1,223 +0,0 @@
-#!/usr/bin/env python
-
-import os.path
-import moe
-import moe.config
-import moe.eval
-import moe.log
-import shutil
-import traceback
-
-def configure_test(e, test):
-    e.cfgs = moe.config.MoeConfigStack(e.cfgs)
-    e.test_builtins = moe.config.MoeConfig(type="test-builtins")
-    e.test_builtins.set("TEST", test)
-    e.cfgs.push(e.test_builtins)
-
-    test_cf = os.path.join(e["PDIR"], test + ".config")
-    if os.path.exists(test_cf):
-	cfg = moe.config.MoeConfig(name=test_cf, type="test")
-	e.cfgs.push(cfg)
-
-    e.cfgs.apply_overrides("TEST_" + test + "_")
-    ext = e["EXT"]
-    if ext != "":
-	e.cfgs.apply_overrides("EXT_" + ext + "_")
-
-    log = moe.log.MoeLog()
-    log.verbosity = e.log.verbosity
-    log.open(os.path.join(e["TDIR"], test + ".log"))
-    log.say("Test case %s\n\n" % test)
-    e.log = log
-    moe.log.default = log
-
-    e.log_config(2, "for the test")
-
-    e.test_stat = moe.status.MoeStatus()
-    e.test_stat["id"] = test
-    e.stat.get_list("tests").append(e.test_stat)
-
-def collect_status(e):
-    sf = os.path.join(e["TDIR"], e["TESTCASE_STATUS"])
-    if os.path.exists(sf):
-	e.log.verbose("Reading status from %s\n" % sf)
-	stat = moe.status.MoeStatus()
-	stat.read(name=sf)
-	if e.log.verbosity > 1:
-	    stat.write_nested(e.log.log_file, 1)
-	for k in stat.keys():
-	    e.test_stat[k] = stat[k]
-	os.unlink(sf)
-    else:
-	e.log.verbose("No status file present\n")
-
-def tmpname(e):
-    return os.path.join(e["TDIR"], e["TEST"] + ".tmp")
-
-def collect_tmp_line(tmp_file):
-    if os.path.exists(tmp_file):
-	f = open(tmp_file, "r")
-	v = f.readline().rstrip("\n")
-	f.close()
-	os.unlink(tmp_file)
-    else:
-	v = ""
-    return v
-
-def collect_verdict(e, verdict_file):
-    v = collect_tmp_line(verdict_file)
-    if len(v) >= 4 and v[0].isalnum and v[1].isalnum and v[2] == ":" and v[3] == " ":
-	e.test_stat["status"] = v[0:2]
-	e.log.verbose("Judge's status: %s\n" % v[0:2])
-	v = v[4:]
-    v.strip()
-    if v != "":
-	e.test_stat["message"] = v
-	e.log.verbose("Judge's verdict: %s\n" % v)
-
-def collect_points(e):
-    p = collect_tmp_line(os.path.join(e["TDIR"], e["TESTCASE_PTS"]))
-    if p != "":
-	e.log.verbose("Judge has supplied points: %s\n" % p)
-	e.test_stat["points"] = p
-
-def setup(e):
-    pdir = e["PDIR"]
-    tdir = e["TDIR"]
-    inn = e["TESTCASE_IN"]
-    out = e["TESTCASE_OUT"]
-    ok = e["TESTCASE_OK"]
-
-    if os.path.exists(os.path.join(pdir, inn)):
-	moe.util.link_or_copy(os.path.join(pdir, inn), os.path.join(tdir, inn))
-    if os.path.exists(os.path.join(pdir, out)):
-	moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok))
-
-def judge(e):
-    cmd = e["OUTPUT_CHECK"]
-    if cmd == "":
-	return
-    verdict_file = tmpname(e)
-    cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
-
-    e.log.progress("<check> ")
-    e.log.verbose("Checking output: %s\n" % cmd)
-    e.log.flush()
-    rc = os.system(cmd)
-    collect_verdict(e, verdict_file)
-    collect_points(e)
-    collect_status(e)
-    if os.WIFEXITED(rc):
-	if os.WEXITSTATUS(rc) == 0:
-	    return
-	elif os.WEXITSTATUS(rc) == 1:
-	    raise moe.TestError("Wrong answer", "WA")
-    raise moe.MoeError("Judge failure")
-
-def points(e):
-    if e.test_stat["points"] is None:
-	e.test_stat["points"] = e["POINTS_PER_TEST"]
-
-def filter(e):
-    cmd = e["OUTPUT_FILTER"]
-    if cmd == "":
-	return
-
-    os.rename(os.path.join(e["TDIR"], e["TESTCASE_OUT"]), os.path.join(e["TDIR"], e["TESTCASE_RAW"]))
-    e.log.progress("<filter> ")
-    e.log.verbose("Filtering output: %s\n" % cmd)
-    e.log.flush()
-    rc = os.system(cmd)
-    if os.WIFEXITED(rc) and os.WEXITSTATUS(rc) == 0:
-	if not os.path.exists(os.path.join(e["TDIR"], e["TESTCASE_OUT"])):
-	    raise moe.MoeError("Filter has generated no output")
-    else:
-	raise moe.MoeError("Filter failure")
-
-def syntax(e):
-    cmd = e["SYNTAX_CHECK"]
-    if cmd == "":
-	return
-    verdict_file = tmpname(e)
-    cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
-
-    e.log.progress("<syntax> ")
-    e.log.verbose("Checking syntax: %s\n" % cmd)
-    e.log.flush()
-    rc = os.system(cmd)
-    collect_verdict(e, verdict_file)
-    collect_status(e)
-    if os.WIFEXITED(rc):
-	if os.WEXITSTATUS(rc) == 0:
-	    return
-	elif os.WEXITSTATUS(rc) == 1:
-	    raise moe.TestError("Wrong syntax", "SY")
-    raise moe.MoeError("Syntax checker failure")
-
-def run_test(e, test):
-    configure_test(e, test)
-
-    ## FIXME: interactive tasks
-    e.test_pipe.configure(e["TESTCASE_HOOKS"])
-    if e.log.verbosity >= 2:
-	e.test_pipe.dump(e.log.log_file, prefix="\t")
-    e.test_pipe.run(e)
-
-def wrap_run_test(e, test):
-    try:
-	run_test(e, test)
-    except moe.MoeError, err:
-	raise moe.TestError(err, "XX")
-    except Exception:
-	if e["DEBUG"]:
-	    moe.log.fatal_exception()
-	traceback.print_exc(file = e.log.log_file)
-	raise moe.TestError("Internal exception", "XX")
-
-def conclude_test(e):
-    stat = e.test_stat
-    if not stat["points"]:
-	stat["points"] = 0
-
-    if e.log.verbosity > 1:
-	e.log.verbose("Final status:\n")
-	stat.write_nested(e.log.log_file, 1)
-
-    if stat["status"]:
-	msg = "%s: %s" % (stat["status"], e.test_stat["message"])
-    else:
-	msg = "OK"
-    msg += " (%s points" % stat["points"]
-    if stat["time"]:
-	msg += ", %s s" % stat["time"]
-    if stat["mem"]:
-	msg += ", %d MB" % ((int(stat["mem"]) + 524288) // 1048576)
-    msg += ")\n"
-    e.log.progress(msg)
-    e.log.say(msg)
-
-def run_tests(e):
-    e.test_pipe.insert(0, "setup", setup)
-    e.test_pipe.insert(400, "filter", filter)
-    e.test_pipe.insert(500, "syntax", syntax)
-    e.test_pipe.insert(600, "judge", judge)
-    e.test_pipe.insert(700, "points", points)
-
-    for test in e["TESTS"].split():
-	e.log.progress("Test %s: " % test)
-	old_cfgs = e.cfgs
-	old_log = e.log
-
-	try:
-	    wrap_run_test(e, test)
-	except moe.TestError, err:
-	    if not e.test_stat["status"]:
-		e.test_stat["status"] = err.stat_code
-	    if not e.test_stat["message"]:
-		e.test_stat["message"] = err.message
-
-	conclude_test(e)
-	
-	e.cfgs = old_cfgs
-	e.log = old_log
-	moe.log.default = old_log
diff --git a/t/moe/testutils.py b/t/moe/testutils.py
new file mode 100644
index 0000000..f45caf8
--- /dev/null
+++ b/t/moe/testutils.py
@@ -0,0 +1,241 @@
+#!/usr/bin/env python
+
+import os.path
+import moe
+import moe.config
+import moe.eval
+import moe.log
+import shutil
+import traceback
+import re
+
+# Allowed test names
+testname_regexp = re.compile('\A[\w]+\Z')
+
+def hook_run_tests(e):
+    """Hook that runs the test pipeline for each test in `TESTS`.
+    Opens per-test log `TEST_LOG`.
+    `TEST` is set, checked for invalid characters and fixed in each iteration.
+
+    .. todo :: Different log-level for per-test log?
+    """ 
+    e.log.info('Running test pipeline for each test')
+    e.config.fix('TESTS')
+    tests = e['TESTS'].split()
+    e.log.debug('TESTS: %r', tests)
+    for t in tests:
+	if not testname_regexp.match(t):
+	    raise MoeError("Invalid test name %r", t)
+	e.log.user.info('TEST %s ...' % t)
+	with e.config.parse("TEST='"+t+"'", level=70, source='<hook_run_tests>'):
+	    try:
+		e.config.fix('TEST')
+		e.log.open_test_log(e['TEST_LOG'], e.log.level) 
+		e.log.info(' *** Test case %s *** ' % t)
+		e.debug_dump_config()
+		e.debug_dump_pipe(e.test_pipe)
+		e.test_pipe.run(e=e)
+	    except:
+		e.log.test.exception()
+		raise
+	    finally:
+		e.config.unfix('TEST')
+		e.log.close_test_log()
+
+
+def configure_test(e, test):
+    test_cf = os.path.join(e["PDIR"], test + ".config")
+    if os.path.exists(test_cf):
+	cfg = moe.config.MoeConfig(name=test_cf, type="test")
+	e.cfgs.push(cfg)
+
+    e.log_config(2, "for the test")
+
+    e.test_stat = moe.status.MoeStatus()
+    e.test_stat["id"] = test
+    e.stat.get_list("tests").append(e.test_stat)
+
+def collect_status(e):
+    sf = os.path.join(e["TDIR"], e["TESTCASE_STATUS"])
+    if os.path.exists(sf):
+	e.log.verbose("Reading status from %s\n" % sf)
+	stat = moe.status.MoeStatus()
+	stat.read(name=sf)
+	if e.log.verbosity > 1:
+	    stat.write_nested(e.log.log_file, 1)
+	for k in stat.keys():
+	    e.test_stat[k] = stat[k]
+	os.unlink(sf)
+    else:
+	e.log.verbose("No status file present\n")
+
+def tmpname(e):
+    return os.path.join(e["TDIR"], e["TEST"] + ".tmp")
+
+def collect_tmp_line(tmp_file):
+    if os.path.exists(tmp_file):
+	f = open(tmp_file, "r")
+	v = f.readline().rstrip("\n")
+	f.close()
+	os.unlink(tmp_file)
+    else:
+	v = ""
+    return v
+
+def collect_verdict(e, verdict_file):
+    v = collect_tmp_line(verdict_file)
+    if len(v) >= 4 and v[0].isalnum and v[1].isalnum and v[2] == ":" and v[3] == " ":
+	e.test_stat["status"] = v[0:2]
+	e.log.verbose("Judge's status: %s\n" % v[0:2])
+	v = v[4:]
+    v.strip()
+    if v != "":
+	e.test_stat["message"] = v
+	e.log.verbose("Judge's verdict: %s\n" % v)
+
+def collect_points(e):
+    p = collect_tmp_line(os.path.join(e["TDIR"], e["TESTCASE_PTS"]))
+    if p != "":
+	e.log.verbose("Judge has supplied points: %s\n" % p)
+	e.test_stat["points"] = p
+
+def setup(e):
+    pdir = e["PDIR"]
+    tdir = e["TDIR"]
+    inn = e["TESTCASE_IN"]
+    out = e["TESTCASE_OUT"]
+    ok = e["TESTCASE_OK"]
+
+    if os.path.exists(os.path.join(pdir, inn)):
+	moe.util.link_or_copy(os.path.join(pdir, inn), os.path.join(tdir, inn))
+    if os.path.exists(os.path.join(pdir, out)):
+	moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok))
+
+def judge(e):
+    cmd = e["OUTPUT_CHECK"]
+    if cmd == "":
+	return
+    verdict_file = tmpname(e)
+    cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
+
+    e.log.progress("<check> ")
+    e.log.verbose("Checking output: %s\n" % cmd)
+    e.log.flush()
+    rc = os.system(cmd)
+    collect_verdict(e, verdict_file)
+    collect_points(e)
+    collect_status(e)
+    if os.WIFEXITED(rc):
+	if os.WEXITSTATUS(rc) == 0:
+	    return
+	elif os.WEXITSTATUS(rc) == 1:
+	    raise moe.TestError("Wrong answer", "WA")
+    raise moe.MoeError("Judge failure")
+
+def points(e):
+    if e.test_stat["points"] is None:
+	e.test_stat["points"] = e["POINTS_PER_TEST"]
+
+def filter(e):
+    cmd = e["OUTPUT_FILTER"]
+    if cmd == "":
+	return
+
+    os.rename(os.path.join(e["TDIR"], e["TESTCASE_OUT"]), os.path.join(e["TDIR"], e["TESTCASE_RAW"]))
+    e.log.progress("<filter> ")
+    e.log.verbose("Filtering output: %s\n" % cmd)
+    e.log.flush()
+    rc = os.system(cmd)
+    if os.WIFEXITED(rc) and os.WEXITSTATUS(rc) == 0:
+	if not os.path.exists(os.path.join(e["TDIR"], e["TESTCASE_OUT"])):
+	    raise moe.MoeError("Filter has generated no output")
+    else:
+	raise moe.MoeError("Filter failure")
+
+def syntax(e):
+    cmd = e["SYNTAX_CHECK"]
+    if cmd == "":
+	return
+    verdict_file = tmpname(e)
+    cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
+
+    e.log.progress("<syntax> ")
+    e.log.verbose("Checking syntax: %s\n" % cmd)
+    e.log.flush()
+    rc = os.system(cmd)
+    collect_verdict(e, verdict_file)
+    collect_status(e)
+    if os.WIFEXITED(rc):
+	if os.WEXITSTATUS(rc) == 0:
+	    return
+	elif os.WEXITSTATUS(rc) == 1:
+	    raise moe.TestError("Wrong syntax", "SY")
+    raise moe.MoeError("Syntax checker failure")
+
+def run_test(e, test):
+    configure_test(e, test)
+
+    ## FIXME: interactive tasks
+    e.test_pipe.configure(e["TESTCASE_HOOKS"])
+    if e.log.verbosity >= 2:
+	e.test_pipe.dump(e.log.log_file, prefix="\t")
+    e.test_pipe.run(e)
+
+def wrap_run_test(e, test):
+    try:
+	run_test(e, test)
+    except moe.MoeError, err:
+	raise moe.TestError(err, "XX")
+    except Exception:
+	if e["DEBUG"]:
+	    moe.log.fatal_exception()
+	traceback.print_exc(file = e.log.log_file)
+	raise moe.TestError("Internal exception", "XX")
+
+def conclude_test(e):
+    stat = e.test_stat
+    if not stat["points"]:
+	stat["points"] = 0
+
+    if e.log.verbosity > 1:
+	e.log.verbose("Final status:\n")
+	stat.write_nested(e.log.log_file, 1)
+
+    if stat["status"]:
+	msg = "%s: %s" % (stat["status"], e.test_stat["message"])
+    else:
+	msg = "OK"
+    msg += " (%s points" % stat["points"]
+    if stat["time"]:
+	msg += ", %s s" % stat["time"]
+    if stat["mem"]:
+	msg += ", %d MB" % ((int(stat["mem"]) + 524288) // 1048576)
+    msg += ")\n"
+    e.log.progress(msg)
+    e.log.say(msg)
+
+def run_tests(e):
+    e.test_pipe.insert(0, "setup", setup)
+    e.test_pipe.insert(400, "filter", filter)
+    e.test_pipe.insert(500, "syntax", syntax)
+    e.test_pipe.insert(600, "judge", judge)
+    e.test_pipe.insert(700, "points", points)
+
+    for test in e["TESTS"].split():
+	e.log.progress("Test %s: " % test)
+	old_cfgs = e.cfgs
+	old_log = e.log
+
+	try:
+	    wrap_run_test(e, test)
+	except moe.TestError, err:
+	    if not e.test_stat["status"]:
+		e.test_stat["status"] = err.stat_code
+	    if not e.test_stat["message"]:
+		e.test_stat["message"] = err.message
+
+	conclude_test(e)
+	
+	e.cfgs = old_cfgs
+	e.log = old_log
+	moe.log.default = old_log
diff --git a/t/test.py b/t/test.py
index 15470ab..cc61e99 100755
--- a/t/test.py
+++ b/t/test.py
@@ -3,12 +3,7 @@
 import sys
 #sys.path.append('.')
 
-import moe
-import moe.config
-import moe.log
 import moe.eval
-import moe.pipeline
-import moe.batch
 import os
 
 e = moe.eval.Eval()