12 from moe.logs import *
16 testname_regexp = re.compile('\A[\w]+\Z')
18 def hook_run_tests(e):
19 """Hook that runs the test pipeline for each test in `TESTS`.
20 Opens per-test log `TEST_LOG`.
21 `TEST` is set, checked for invalid characters and fixed in each iteration.
23 .. todo :: Different log-level for per-test log?
25 log.info('Running test pipeline for each test')
27 tests = e['TESTS'].split()
28 log.debug('TESTS: %r', tests)
30 if not testname_regexp.match(t):
31 raise MoeError("Invalid test name %r", t)
32 userlog.info('TEST %s ...' % t)
33 with e.config.parse("TEST='"+t+"'", level=70, source='<hook_run_tests>'):
36 moe.logs.open_test_log(e['TEST_LOG'], log.level)
37 log.info(' *** Test case %s *** ' % t)
39 e.debug_dump_pipe(e.test_pipe)
42 (testlog or log).exception("Exception falls through hook_run_tests()")
45 e.config.unfix('TEST')
46 moe.logs.close_test_log()
48 def configure_test(e, test):
49 test_cf = os.path.join(e["PDIR"], test + ".config")
50 if os.path.exists(test_cf):
51 cfg = moe.config.MoeConfig(name=test_cf, type="test")
54 e.log_config(2, "for the test")
56 e.test_stat = moe.status.MoeStatus()
57 e.test_stat["id"] = test
58 e.stat.get_list("tests").append(e.test_stat)
60 def collect_status(e):
61 sf = os.path.join(e["TDIR"], e["TESTCASE_STATUS"])
62 if os.path.exists(sf):
63 e.log.verbose("Reading status from %s\n" % sf)
64 stat = moe.status.MoeStatus()
66 if e.log.verbosity > 1:
67 stat.write_nested(e.log.log_file, 1)
69 e.test_stat[k] = stat[k]
72 e.log.verbose("No status file present\n")
75 return os.path.join(e["TDIR"], e["TEST"] + ".tmp")
77 def collect_tmp_line(tmp_file):
78 if os.path.exists(tmp_file):
79 f = open(tmp_file, "r")
80 v = f.readline().rstrip("\n")
87 def collect_verdict(e, verdict_file):
88 v = collect_tmp_line(verdict_file)
89 if len(v) >= 4 and v[0].isalnum and v[1].isalnum and v[2] == ":" and v[3] == " ":
90 e.test_stat["status"] = v[0:2]
91 e.log.verbose("Judge's status: %s\n" % v[0:2])
95 e.test_stat["message"] = v
96 e.log.verbose("Judge's verdict: %s\n" % v)
98 def collect_points(e):
99 p = collect_tmp_line(os.path.join(e["TDIR"], e["TESTCASE_PTS"]))
101 e.log.verbose("Judge has supplied points: %s\n" % p)
102 e.test_stat["points"] = p
107 inn = e["TESTCASE_IN"]
108 out = e["TESTCASE_OUT"]
109 ok = e["TESTCASE_OK"]
111 if os.path.exists(os.path.join(pdir, inn)):
112 moe.util.link_or_copy(os.path.join(pdir, inn), os.path.join(tdir, inn))
113 if os.path.exists(os.path.join(pdir, out)):
114 moe.util.link_or_copy(os.path.join(pdir, out), os.path.join(tdir, ok))
117 cmd = e["OUTPUT_CHECK"]
120 verdict_file = tmpname(e)
121 cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
123 e.log.progress("<check> ")
124 e.log.verbose("Checking output: %s\n" % cmd)
127 collect_verdict(e, verdict_file)
131 if os.WEXITSTATUS(rc) == 0:
133 elif os.WEXITSTATUS(rc) == 1:
134 raise moe.TestError("Wrong answer", "WA")
135 raise moe.MoeError("Judge failure")
138 if e.test_stat["points"] is None:
139 e.test_stat["points"] = e["POINTS_PER_TEST"]
142 cmd = e["OUTPUT_FILTER"]
146 os.rename(os.path.join(e["TDIR"], e["TESTCASE_OUT"]), os.path.join(e["TDIR"], e["TESTCASE_RAW"]))
147 e.log.progress("<filter> ")
148 e.log.verbose("Filtering output: %s\n" % cmd)
151 if os.WIFEXITED(rc) and os.WEXITSTATUS(rc) == 0:
152 if not os.path.exists(os.path.join(e["TDIR"], e["TESTCASE_OUT"])):
153 raise moe.MoeError("Filter has generated no output")
155 raise moe.MoeError("Filter failure")
158 cmd = e["SYNTAX_CHECK"]
161 verdict_file = tmpname(e)
162 cmd = "exec 2>%s ; %s" % (verdict_file,cmd)
164 e.log.progress("<syntax> ")
165 e.log.verbose("Checking syntax: %s\n" % cmd)
168 collect_verdict(e, verdict_file)
171 if os.WEXITSTATUS(rc) == 0:
173 elif os.WEXITSTATUS(rc) == 1:
174 raise moe.TestError("Wrong syntax", "SY")
175 raise moe.MoeError("Syntax checker failure")
177 def run_test(e, test):
178 configure_test(e, test)
180 ## FIXME: interactive tasks
181 e.test_pipe.configure(e["TESTCASE_HOOKS"])
182 if e.log.verbosity >= 2:
183 e.test_pipe.dump(e.log.log_file, prefix="\t")
186 def wrap_run_test(e, test):
189 except moe.MoeError, err:
190 raise moe.TestError(err, "XX")
193 moe.log.fatal_exception()
194 traceback.print_exc(file = e.log.log_file)
195 raise moe.TestError("Internal exception", "XX")
197 def conclude_test(e):
199 if not stat["points"]:
202 if e.log.verbosity > 1:
203 e.log.verbose("Final status:\n")
204 stat.write_nested(e.log.log_file, 1)
207 msg = "%s: %s" % (stat["status"], e.test_stat["message"])
210 msg += " (%s points" % stat["points"]
212 msg += ", %s s" % stat["time"]
214 msg += ", %d MB" % ((int(stat["mem"]) + 524288) // 1048576)
220 e.test_pipe.insert(0, "setup", setup)
221 e.test_pipe.insert(400, "filter", filter)
222 e.test_pipe.insert(500, "syntax", syntax)
223 e.test_pipe.insert(600, "judge", judge)
224 e.test_pipe.insert(700, "points", points)
226 for test in e["TESTS"].split():
227 e.log.progress("Test %s: " % test)
232 wrap_run_test(e, test)
233 except moe.TestError, err:
234 if not e.test_stat["status"]:
235 e.test_stat["status"] = err.stat_code
236 if not e.test_stat["message"]:
237 e.test_stat["message"] = err.message
243 moe.log.default = old_log