1 | // ================================================================ // |
---|
2 | // // |
---|
3 | // File : UnitTester.cxx // |
---|
4 | // Purpose : // |
---|
5 | // // |
---|
6 | // Coded by Ralf Westram (coder@reallysoft.de) in February 2010 // |
---|
7 | // Institute of Microbiology (Technical University Munich) // |
---|
8 | // http://www.arb-home.de/ // |
---|
9 | // // |
---|
10 | // ================================================================ // |
---|
11 | |
---|
12 | #include "UnitTester.hxx" |
---|
13 | |
---|
14 | #include <cstdarg> |
---|
15 | #include <cstdlib> |
---|
16 | #include <cstdio> |
---|
17 | #include <cstring> |
---|
18 | #include <cerrno> |
---|
19 | #include <climits> |
---|
20 | |
---|
21 | #include <sys/time.h> |
---|
22 | #include <sys/stat.h> |
---|
23 | |
---|
24 | #include <arb_simple_assert.h> |
---|
25 | #include <arb_backtrace.h> |
---|
26 | #include <arb_pathlen.h> |
---|
27 | #include <arb_sleep.h> |
---|
28 | |
---|
29 | #include <test_unit.h> |
---|
30 | #include <ut_valgrinded.h> |
---|
31 | #include <valgrind.h> |
---|
32 | |
---|
33 | #include <SuppressOutput.h> |
---|
34 | |
---|
35 | #include <SigHandler.h> |
---|
36 | #include <setjmp.h> |
---|
37 | #include <unistd.h> |
---|
38 | #include <string> |
---|
39 | #include <signal.h> |
---|
40 | |
---|
41 | #include <mod_rlimit.h> |
---|
42 | #include "../SOURCE_TOOLS/arb_main.h" |
---|
43 | |
---|
44 | #define ut_assert(cond) arb_assert(cond) |
---|
45 | |
---|
46 | #if defined(DEVEL_ELMAR) |
---|
47 | #define COLORED_MESSAGES |
---|
48 | #endif |
---|
49 | |
---|
50 | #if defined(DEBUG) |
---|
51 | # if defined(DEVEL_RALF) |
---|
52 | |
---|
53 | // whether TEST_VALID_LOCATION works or not seems to depend on how well compiler version and system match. |
---|
54 | // now works with gcc 7.3 on my current development machine. |
---|
55 | # define TEST_VALID_LOCATION |
---|
56 | |
---|
57 | # endif |
---|
58 | #endif |
---|
59 | |
---|
60 | #ifdef COLORED_MESSAGES |
---|
61 | |
---|
62 | #define ESC_BOLD "\033[1m" |
---|
63 | #define ESC_RED "\033[31m" |
---|
64 | #define ESC_GREEN "\033[32m" |
---|
65 | #define ESC_YELLOW "\033[33m" |
---|
66 | #define ESC_RESET_COL "\033[39m" |
---|
67 | #define ESC_RESET_ALL "\033[0m" |
---|
68 | |
---|
69 | #endif |
---|
70 | |
---|
71 | using namespace std; |
---|
72 | |
---|
73 | // -------------------------------------------------------------------------------- |
---|
74 | |
---|
75 | struct Globals : virtual Noncopyable { |
---|
76 | bool inside_test; |
---|
77 | bool running_on_valgrind; |
---|
78 | char *runDir; |
---|
79 | pid_t pid; |
---|
80 | const char *libname; |
---|
81 | |
---|
82 | Globals() |
---|
83 | : inside_test(false), |
---|
84 | runDir(NULp), |
---|
85 | pid(getpid()), |
---|
86 | libname(NULp) |
---|
87 | { |
---|
88 | running_on_valgrind = (RUNNING_ON_VALGRIND>0); |
---|
89 | } |
---|
90 | ~Globals() { |
---|
91 | free(runDir); |
---|
92 | } |
---|
93 | |
---|
94 | inline void setup_test_precondition() { |
---|
95 | TEST_EXPECT_ZERO_OR_SHOW_ERRNO(chdir(runDir)); |
---|
96 | } |
---|
97 | inline void setup_test_postcondition() { |
---|
98 | TEST_ANNOTATE(NULp); |
---|
99 | } |
---|
100 | }; |
---|
101 | |
---|
102 | static Globals GLOBAL; |
---|
103 | |
---|
104 | // -------------------------------------------------------------------------------- |
---|
105 | // #define TRACE_PREFIX "UnitTester:0: " |
---|
106 | #define TRACE_PREFIX "UnitTester: " |
---|
107 | |
---|
108 | __ATTR__FORMAT(1) static void trace(const char *format, ...) { |
---|
109 | va_list parg; |
---|
110 | |
---|
111 | fflush(stdout); |
---|
112 | fflush(stderr); |
---|
113 | |
---|
114 | #if defined(COLORED_MESSAGES) |
---|
115 | fputs(ESC_BOLD, stderr); |
---|
116 | #endif |
---|
117 | fputs(TRACE_PREFIX, stderr); |
---|
118 | va_start(parg, format); |
---|
119 | vfprintf(stderr, format, parg); |
---|
120 | va_end(parg); |
---|
121 | #if defined(COLORED_MESSAGES) |
---|
122 | fputs(ESC_RESET_ALL, stderr); |
---|
123 | #endif |
---|
124 | fputc('\n', stderr); |
---|
125 | fflush(stderr); |
---|
126 | } |
---|
127 | |
---|
128 | // -------------------------------------------------------------------------------- |
---|
129 | |
---|
130 | #ifdef COLORED_MESSAGES |
---|
131 | static const char *readable_result[] = { |
---|
132 | ESC_GREEN "OK" ESC_RESET_COL, |
---|
133 | ESC_RED "TRAPPED" ESC_RESET_COL, |
---|
134 | ESC_RED "VIOLATED" ESC_RESET_COL, |
---|
135 | ESC_RED "INTERRUPTED" ESC_RESET_COL, |
---|
136 | ESC_RED "THREW" ESC_RESET_COL, |
---|
137 | ESC_RED "INVALID" ESC_RESET_COL, |
---|
138 | ESC_RED "{unknown}" ESC_RESET_COL, |
---|
139 | }; |
---|
140 | #else |
---|
141 | static const char *readable_result[] = { |
---|
142 | "OK" , |
---|
143 | "TRAPPED" , |
---|
144 | "VIOLATED" , |
---|
145 | "INTERRUPTED" , |
---|
146 | "THREW" , |
---|
147 | "INVALID" , // use TEST_PUBLISH to make it valid |
---|
148 | "{unknown}" , |
---|
149 | }; |
---|
150 | #endif |
---|
151 | |
---|
152 | // -------------------------------------------------------------------------------- |
---|
153 | |
---|
154 | static jmp_buf UNITTEST_return_after_segv; |
---|
155 | static bool terminate_was_called = false; |
---|
156 | |
---|
157 | enum TrapCode { |
---|
158 | TRAP_UNEXPECTED = 668, |
---|
159 | TRAP_SEGV, |
---|
160 | TRAP_INT, |
---|
161 | TRAP_TERM, |
---|
162 | }; |
---|
163 | |
---|
164 | __ATTR__NORETURN static void UNITTEST_sigsegv_handler(int sig) { |
---|
165 | if (GLOBAL.inside_test) { |
---|
166 | int trap_code; |
---|
167 | const char *backtrace_cause = NULp; |
---|
168 | switch (sig) { |
---|
169 | case SIGSEGV: { |
---|
170 | trap_code = TRAP_SEGV; |
---|
171 | |
---|
172 | arb_test::GlobalTestData& test_data = arb_test::test_data(); |
---|
173 | if (!test_data.assertion_failed) { // not caused by assertion |
---|
174 | backtrace_cause = "Caught SIGSEGV not caused by assertion"; |
---|
175 | } |
---|
176 | break; |
---|
177 | } |
---|
178 | case SIGINT: |
---|
179 | trap_code = TRAP_INT; |
---|
180 | backtrace_cause = "Caught SIGINT (deadlock in test function?)"; |
---|
181 | break; |
---|
182 | |
---|
183 | case SIGTERM: |
---|
184 | trap_code = TRAP_TERM; |
---|
185 | if (terminate_was_called) { |
---|
186 | backtrace_cause = "Caught SIGTERM, cause std::terminate() has been called in test-code (might be an invalid throw)"; |
---|
187 | } |
---|
188 | else { |
---|
189 | backtrace_cause = "Caught SIGTERM (deadlock in uninterruptable test function?)"; |
---|
190 | } |
---|
191 | break; |
---|
192 | |
---|
193 | default: |
---|
194 | trap_code = TRAP_UNEXPECTED; |
---|
195 | test_assert(0, true); |
---|
196 | break; |
---|
197 | } |
---|
198 | if (backtrace_cause) demangle_backtrace(BackTraceInfo(0), stderr, backtrace_cause); |
---|
199 | siglongjmp(UNITTEST_return_after_segv, trap_code); // suppress signal |
---|
200 | } |
---|
201 | |
---|
202 | const char *signame = NULp; |
---|
203 | switch (sig) { |
---|
204 | case SIGSEGV: signame = "SEGV"; break; |
---|
205 | case SIGINT: signame = "INT"; break; |
---|
206 | case SIGTERM: signame = "TERM"; break; |
---|
207 | } |
---|
208 | |
---|
209 | fputs("[UnitTester catched unexpected signal ", stderr); |
---|
210 | if (signame) fputs(signame, stderr); else fprintf(stderr, "%i", sig); |
---|
211 | fputs("]\n", stderr); |
---|
212 | BackTraceInfo(0).dump(stderr, "Unexpected signal (NOT raised in test-code)"); |
---|
213 | exit(EXIT_FAILURE); |
---|
214 | } |
---|
215 | |
---|
216 | #define SECOND 1000000 |
---|
217 | |
---|
218 | static void terminate_called() { |
---|
219 | // example-test triggering call of terminate_called(): ../CORE/arb_signal.cxx@TEST_throw_during_throw |
---|
220 | terminate_was_called = true; |
---|
221 | raise(SIGTERM); |
---|
222 | // GBK_dump_backtrace(stdout, "just raised SIGTERM - wtf am i doing here"); fflush(stdout); |
---|
223 | } |
---|
224 | |
---|
225 | static UnitTestResult execute_guarded_ClientCode(UnitTest_function fun, long *duration_usec) { |
---|
226 | terminate_handler old_terminate = std::set_terminate(terminate_called); |
---|
227 | |
---|
228 | SigHandler old_int_handler = INSTALL_SIGHANDLER(SIGINT, UNITTEST_sigsegv_handler, "execute_guarded"); |
---|
229 | SigHandler old_term_handler = INSTALL_SIGHANDLER(SIGTERM, UNITTEST_sigsegv_handler, "execute_guarded"); |
---|
230 | SigHandler old_segv_handler = INSTALL_SIGHANDLER(SIGSEGV, UNITTEST_sigsegv_handler, "execute_guarded"); |
---|
231 | |
---|
232 | // ---------------------------------------- |
---|
233 | // start of critical section |
---|
234 | // (need volatile for modified local auto variables, see man longjump) |
---|
235 | volatile timeval t1; |
---|
236 | volatile UnitTestResult result = TEST_OK; |
---|
237 | volatile int trapped = sigsetjmp(UNITTEST_return_after_segv, 1); |
---|
238 | |
---|
239 | if (trapped) { |
---|
240 | switch (trapped) { |
---|
241 | case TRAP_UNEXPECTED: ut_assert(0); // fall-through |
---|
242 | case TRAP_SEGV: result = TEST_TRAPPED; break; |
---|
243 | |
---|
244 | case TRAP_INT: |
---|
245 | case TRAP_TERM: result = TEST_INTERRUPTED; break; |
---|
246 | } |
---|
247 | } |
---|
248 | else { // normal execution |
---|
249 | GLOBAL.inside_test = true; |
---|
250 | |
---|
251 | gettimeofday((timeval*)&t1, NULp); |
---|
252 | |
---|
253 | arb_test::test_data().assertion_failed = false; |
---|
254 | arb_test::test_data().running_test = true; |
---|
255 | |
---|
256 | // Note: fun() may do several ugly things, e.g. |
---|
257 | // - segfault (handled) |
---|
258 | // - never return (handled by caller) |
---|
259 | // - exit() or abort() (std::terminate is handled, direct call to abort is not) |
---|
260 | // - throw an exception (catched by caller of execute_guarded_ClientCode) |
---|
261 | // - change working dir (should always be reset before i get called) |
---|
262 | |
---|
263 | fun(); |
---|
264 | |
---|
265 | // sleepms(10000); // simulate a deadlock |
---|
266 | } |
---|
267 | // end of critical section |
---|
268 | // ---------------------------------------- |
---|
269 | |
---|
270 | timeval t2; |
---|
271 | gettimeofday(&t2, NULp); |
---|
272 | *duration_usec = (t2.tv_sec - t1.tv_sec) * SECOND + (t2.tv_usec - t1.tv_usec); |
---|
273 | |
---|
274 | GLOBAL.inside_test = false; |
---|
275 | |
---|
276 | UNINSTALL_SIGHANDLER(SIGSEGV, UNITTEST_sigsegv_handler, old_segv_handler, "execute_guarded"); |
---|
277 | UNINSTALL_SIGHANDLER(SIGTERM, UNITTEST_sigsegv_handler, old_term_handler, "execute_guarded"); |
---|
278 | UNINSTALL_SIGHANDLER(SIGINT, UNITTEST_sigsegv_handler, old_int_handler, "execute_guarded"); |
---|
279 | |
---|
280 | terminate_handler handler = std::set_terminate(old_terminate); |
---|
281 | if (handler != terminate_called) { |
---|
282 | fputs("Error: test-code has modified std::terminate (this is invalid)\n", stderr); |
---|
283 | fflush(stderr); |
---|
284 | result = TEST_INVALID; |
---|
285 | } |
---|
286 | |
---|
287 | return result; |
---|
288 | } |
---|
289 | |
---|
290 | inline bool kill_verbose(pid_t pid, int sig, const char *signame) { |
---|
291 | int result = kill(pid, sig); |
---|
292 | if (result != 0) { |
---|
293 | fprintf(stderr, "Failed to send %s to test (%s)\n", signame, strerror(errno)); |
---|
294 | fflush(stderr); |
---|
295 | } |
---|
296 | return result == 0; |
---|
297 | } |
---|
298 | |
---|
299 | class Flag { |
---|
300 | string flagname; |
---|
301 | |
---|
302 | public: |
---|
303 | Flag(const string& flagname_) : flagname(flagname_) {} |
---|
304 | |
---|
305 | bool is_up() const { struct stat stt; return stat(flagname.c_str(), &stt) == 0 && S_ISREG(stt.st_mode); } |
---|
306 | void raise() { |
---|
307 | fprintf(stderr, "Raising flag '%s'\n", flagname.c_str()); fflush(stderr); |
---|
308 | ut_assert(!is_up()); |
---|
309 | FILE *fp = fopen(flagname.c_str(), "w"); |
---|
310 | fclose(fp); |
---|
311 | } |
---|
312 | void lower() { |
---|
313 | fprintf(stderr, "Lowering flag '%s'\n", flagname.c_str()); fflush(stderr); |
---|
314 | ut_assert(is_up()); |
---|
315 | TEST_EXPECT_ZERO_OR_SHOW_ERRNO(unlink(flagname.c_str())); |
---|
316 | } |
---|
317 | |
---|
318 | void lower_if_up() { if (is_up()) lower(); } |
---|
319 | }; |
---|
320 | |
---|
321 | inline Flag getLocalFlag(const char *flagname) { |
---|
322 | string localname = string(GLOBAL.runDir)+"/../flags/"+flagname+'.'+GLOBAL.libname; |
---|
323 | return Flag(localname); |
---|
324 | } |
---|
325 | |
---|
326 | static bool flag_callback(arb_test::FlagAction action, const char *name) { |
---|
327 | using namespace arb_test; |
---|
328 | |
---|
329 | Flag flag = getLocalFlag(name); |
---|
330 | bool result = true; |
---|
331 | |
---|
332 | switch (action) { |
---|
333 | case FLAG_RAISE: |
---|
334 | if (!flag.is_up()) flag.raise(); |
---|
335 | break; |
---|
336 | case FLAG_IS_RAISED: |
---|
337 | result = flag.is_up(); |
---|
338 | break; |
---|
339 | } |
---|
340 | |
---|
341 | return result; |
---|
342 | } |
---|
343 | |
---|
344 | inline bool been_inside_environment() { |
---|
345 | return getLocalFlag(ANY_SETUP).is_up() || arb_test::test_data().entered_mutex_loop; |
---|
346 | } |
---|
347 | inline bool did_valgrinded_syscall() { |
---|
348 | return seen_valgrinded_call(); |
---|
349 | } |
---|
350 | inline void reset_test_local_flags() { |
---|
351 | getLocalFlag(ANY_SETUP).lower_if_up(); |
---|
352 | } |
---|
353 | |
---|
354 | void sleepms(long ms) { |
---|
355 | long seconds = ms/1000; |
---|
356 | long rest_ms = ms - seconds*1000; |
---|
357 | |
---|
358 | if (seconds) ARB_sleep(seconds, SEC); |
---|
359 | ARB_sleep(rest_ms, MS); |
---|
360 | } |
---|
361 | |
---|
362 | #if (DEADLOCKGUARD == 1) |
---|
363 | __ATTR__NORETURN static void deadlockguard(long max_allowed_duration_ms, bool detect_environment_calls) { |
---|
364 | // this function is completely incompatible with debuggers |
---|
365 | sleepms(max_allowed_duration_ms); |
---|
366 | |
---|
367 | if (detect_environment_calls && been_inside_environment()) { |
---|
368 | fprintf(stderr, "[test_environment has been called. Added %li ms tolerance]\n", MAX_EXEC_MS_ENV); |
---|
369 | fflush(stderr); |
---|
370 | sleepms(MAX_EXEC_MS_ENV); |
---|
371 | max_allowed_duration_ms += MAX_EXEC_MS_ENV; |
---|
372 | } |
---|
373 | |
---|
374 | if (did_valgrinded_syscall()) { |
---|
375 | const long additional = max_allowed_duration_ms*3 + MAX_EXEC_MS_VGSYS; |
---|
376 | fprintf(stderr, "[external calls are running valgrinded. Added %li ms tolerance]\n", additional); |
---|
377 | fflush(stderr); |
---|
378 | sleepms(additional); |
---|
379 | max_allowed_duration_ms += additional; |
---|
380 | } |
---|
381 | |
---|
382 | const int aBIT = 50; // 50 milliseconds |
---|
383 | |
---|
384 | fprintf(stderr, |
---|
385 | "[deadlockguard woke up after %li ms]\n" |
---|
386 | "[interrupting possibly deadlocked test]\n", |
---|
387 | max_allowed_duration_ms); fflush(stderr); |
---|
388 | kill_verbose(GLOBAL.pid, SIGINT, "SIGINT"); |
---|
389 | sleepms(aBIT); // give parent a chance to kill me |
---|
390 | |
---|
391 | fprintf(stderr, "[test still running -> terminate]\n"); fflush(stderr); |
---|
392 | kill_verbose(GLOBAL.pid, SIGTERM, "SIGTERM"); |
---|
393 | sleepms(aBIT); // give parent a chance to kill me |
---|
394 | |
---|
395 | fprintf(stderr, "[still running -> kill]\n"); fflush(stderr); |
---|
396 | kill_verbose(GLOBAL.pid, SIGKILL, "SIGKILL"); // commit suicide |
---|
397 | // parent had his chance |
---|
398 | fprintf(stderr, "[still alive after suicide -> perplexed]\n"); fflush(stderr); |
---|
399 | exit(EXIT_FAILURE); |
---|
400 | } |
---|
401 | #endif |
---|
402 | |
---|
403 | UnitTestResult execute_guarded(UnitTest_function fun, long *duration_usec, long max_allowed_duration_ms, bool detect_environment_calls) { |
---|
404 | if (GLOBAL.running_on_valgrind) max_allowed_duration_ms *= 4; |
---|
405 | UnitTestResult result; |
---|
406 | |
---|
407 | pid_t child_pid = fork(); |
---|
408 | if (child_pid) { // parent |
---|
409 | try { result = execute_guarded_ClientCode(fun, duration_usec); } |
---|
410 | catch (...) { result = TEST_THREW; } |
---|
411 | if (kill(child_pid, SIGKILL) != 0) { |
---|
412 | fprintf(stderr, "Failed to kill deadlock-guard (%s)\n", strerror(errno)); fflush(stderr); |
---|
413 | } |
---|
414 | } |
---|
415 | else { // child |
---|
416 | #if (DEADLOCKGUARD == 1) |
---|
417 | deadlockguard(max_allowed_duration_ms, detect_environment_calls); |
---|
418 | #else |
---|
419 | #warning DEADLOCKGUARD has been disabled (not default!) |
---|
420 | detect_environment_calls = !!detect_environment_calls; // dont warn |
---|
421 | #endif |
---|
422 | exit(EXIT_FAILURE); |
---|
423 | } |
---|
424 | |
---|
425 | return result; |
---|
426 | } |
---|
427 | |
---|
428 | // -------------------------------------------------------------------------------- |
---|
429 | |
---|
430 | class SimpleTester { |
---|
431 | const UnitTest_simple *tests; |
---|
432 | const UnitTest_simple *postcond; |
---|
433 | size_t count; |
---|
434 | size_t postcount; |
---|
435 | double duration_ms; |
---|
436 | |
---|
437 | bool perform(size_t which); |
---|
438 | UnitTestResult run_postcond(long *duration_usec, long max_allowed_duration_ms, bool print_errors); |
---|
439 | |
---|
440 | public: |
---|
441 | SimpleTester(const UnitTest_simple *tests_, const UnitTest_simple *postcond_) |
---|
442 | : tests(tests_), |
---|
443 | postcond(postcond_), |
---|
444 | duration_ms(0.0) |
---|
445 | { |
---|
446 | for (count = 0; tests[count].fun; ++count) {} |
---|
447 | for (postcount = 0; postcond[postcount].fun; ++postcount) {} |
---|
448 | } |
---|
449 | |
---|
450 | size_t perform_all(); |
---|
451 | size_t get_test_count() const { return count; } |
---|
452 | |
---|
453 | double overall_duration_ms() const { return duration_ms; } |
---|
454 | }; |
---|
455 | |
---|
456 | |
---|
457 | size_t SimpleTester::perform_all() { |
---|
458 | // returns number of passed tests |
---|
459 | |
---|
460 | trace("performing %zu simple tests..", count); |
---|
461 | size_t passed = 0; |
---|
462 | for (size_t c = 0; c<count; ++c) { |
---|
463 | GLOBAL.setup_test_precondition(); |
---|
464 | passed += perform(c); |
---|
465 | GLOBAL.setup_test_postcondition(); |
---|
466 | } |
---|
467 | return passed; |
---|
468 | } |
---|
469 | |
---|
470 | |
---|
471 | UnitTestResult SimpleTester::run_postcond(long *duration_usec_sum, long abort_after_ms, bool print_errors) { |
---|
472 | UnitTestResult result = TEST_OK; |
---|
473 | *duration_usec_sum = 0; |
---|
474 | for (size_t pc = 0; pc<postcount && result == TEST_OK; ++pc) { |
---|
475 | long duration_usec; |
---|
476 | result = execute_guarded(postcond[pc].fun, &duration_usec, abort_after_ms, true); // <-- call postcond |
---|
477 | *duration_usec_sum += duration_usec; |
---|
478 | |
---|
479 | if (result != TEST_OK && print_errors) { |
---|
480 | postcond[pc].print_error(stderr, result); |
---|
481 | } |
---|
482 | } |
---|
483 | return result; |
---|
484 | } |
---|
485 | |
---|
486 | void UnitTest_simple::print_error(FILE *out, UnitTestResult result) const { |
---|
487 | switch (result) { |
---|
488 | case TEST_OK: |
---|
489 | break; |
---|
490 | |
---|
491 | case TEST_TRAPPED: |
---|
492 | fprintf(out, "%s: Error: %s failed (details above)\n", location, name); |
---|
493 | break; |
---|
494 | |
---|
495 | case TEST_FAILED_POSTCONDITION: |
---|
496 | fprintf(out, "%s: Error: %s succeeded, but violates general postconditions (see above)\n", location, name); |
---|
497 | break; |
---|
498 | |
---|
499 | case TEST_INTERRUPTED: |
---|
500 | fprintf(out, "%s: Error: %s has been interrupted (details above)\n", location, name); |
---|
501 | break; |
---|
502 | |
---|
503 | case TEST_THREW: |
---|
504 | fprintf(out, "%s: Error: %s has thrown an exception\n", location, name); |
---|
505 | break; |
---|
506 | |
---|
507 | case TEST_INVALID: |
---|
508 | fprintf(out, "%s: Error: %s is invalid (see above)\n", location, name); |
---|
509 | break; |
---|
510 | |
---|
511 | case TEST_UNKNOWN_RESULT: |
---|
512 | ut_assert(0); |
---|
513 | break; |
---|
514 | } |
---|
515 | } |
---|
516 | |
---|
517 | bool SimpleTester::perform(size_t which) { |
---|
518 | ut_assert(which<count); |
---|
519 | |
---|
520 | const UnitTest_simple& test = tests[which]; |
---|
521 | UnitTest_function fun = test.fun; |
---|
522 | |
---|
523 | reset_test_local_flags(); |
---|
524 | |
---|
525 | bool marked_as_slow = strlen(test.name) >= 10 && memcmp(test.name, "TEST_SLOW_", 10) == 0; |
---|
526 | const long abort_after_ms = marked_as_slow ? MAX_EXEC_MS_SLOW : MAX_EXEC_MS_NORMAL; |
---|
527 | |
---|
528 | #if defined(TEST_VALID_LOCATION) |
---|
529 | bool invalid = !test.location; // in NDEBUG mode location is always missing |
---|
530 | #else |
---|
531 | bool invalid = false; |
---|
532 | #endif |
---|
533 | |
---|
534 | UnitTestResult result = TEST_UNKNOWN_RESULT; |
---|
535 | unsigned tryMax = 1; // normally only try once; see ./README.txt@TEST_ALLOW_RETRY |
---|
536 | for (unsigned tryCount = 1; tryCount<=tryMax && result != TEST_OK; ++tryCount) { |
---|
537 | long duration_usec; |
---|
538 | result = invalid ? TEST_INVALID : execute_guarded(fun, &duration_usec, abort_after_ms, true); // <--- call unittest |
---|
539 | double duration_ms_this = invalid ? 0.0 : duration_usec/1000.0; |
---|
540 | |
---|
541 | duration_ms += duration_ms_this; |
---|
542 | |
---|
543 | long duration_usec_post = 0; |
---|
544 | UnitTestResult postcond_result = TEST_UNKNOWN_RESULT; |
---|
545 | |
---|
546 | if (result == TEST_OK) { |
---|
547 | // if test is ok -> check postconditions |
---|
548 | postcond_result = run_postcond(&duration_usec_post, MAX_EXEC_MS_NORMAL, true); |
---|
549 | if (postcond_result != TEST_OK) { |
---|
550 | result = TEST_FAILED_POSTCONDITION; // fail test, if any postcondition fails |
---|
551 | } |
---|
552 | } |
---|
553 | |
---|
554 | trace("* %s = %s (%.1f ms)", test.name, readable_result[result], duration_ms_this); |
---|
555 | |
---|
556 | if (result == TEST_OK) { |
---|
557 | if (duration_ms_this >= WARN_SLOW_ABOVE_MS) { // long test duration |
---|
558 | if (!marked_as_slow) { |
---|
559 | bool accept_slow = GLOBAL.running_on_valgrind || did_valgrinded_syscall(); |
---|
560 | if (!accept_slow) { |
---|
561 | fprintf(stderr, "%s: Warning: Name of slow tests shall start with TEST_SLOW_ (it'll be run after other tests)\n", |
---|
562 | test.location); |
---|
563 | } |
---|
564 | } |
---|
565 | } |
---|
566 | } |
---|
567 | |
---|
568 | test.print_error(stderr, result); |
---|
569 | |
---|
570 | if (result != TEST_OK) { // test did not succeed |
---|
571 | if (tryCount == 1) { // 1st attempt failed |
---|
572 | tryMax = arb_test::test_data().allowed_retries(); |
---|
573 | if (tryMax>1) { |
---|
574 | fprintf(stderr, "%s: Warning: test failed, but allows %u retries\n", test.location, tryMax-1); |
---|
575 | } |
---|
576 | } |
---|
577 | |
---|
578 | if (result != TEST_FAILED_POSTCONDITION) { // postconditions not checked yet |
---|
579 | // also check postconditions when test failed |
---|
580 | // (because they are used to clean up side effects of failing tests) |
---|
581 | SuppressOutput here; |
---|
582 | postcond_result = run_postcond(&duration_usec_post, MAX_EXEC_MS_NORMAL, false); // but do not print anything |
---|
583 | } |
---|
584 | } |
---|
585 | |
---|
586 | double duration_ms_postcond = duration_usec_post/1000.0; |
---|
587 | duration_ms += duration_ms_postcond; |
---|
588 | } |
---|
589 | |
---|
590 | return result == TEST_OK; |
---|
591 | } |
---|
592 | |
---|
593 | |
---|
594 | // -------------------------------------------------------------------------------- |
---|
595 | |
---|
596 | static char text_buffer[100]; |
---|
597 | |
---|
598 | inline const char *as_text(size_t z) { sprintf(text_buffer, "%zu", z); return text_buffer; } |
---|
599 | inline const char *as_text(double d) { sprintf(text_buffer, "%.1f", d); return text_buffer; } |
---|
600 | |
---|
601 | inline void appendValue(string& report, const char *tag, const char *value) { report = report+' '+tag+'='+value; } |
---|
602 | |
---|
603 | inline void appendValue(string& report, const char *tag, size_t value) { appendValue(report, tag, as_text(value)); } |
---|
604 | inline void appendValue(string& report, const char *tag, double value) { appendValue(report, tag, as_text(value)); } |
---|
605 | |
---|
606 | static const char *generateReport(const char *libname, size_t tests, size_t skipped, size_t passed, double duration_ms, size_t warnings) { |
---|
607 | // generate a report consumed by reporter.pl@parse_log |
---|
608 | |
---|
609 | static string report; |
---|
610 | report.clear(); |
---|
611 | |
---|
612 | size_t tested = tests-skipped; |
---|
613 | size_t failed = tested-passed; |
---|
614 | |
---|
615 | appendValue(report, "target", libname); |
---|
616 | appendValue(report, "time", duration_ms); |
---|
617 | appendValue(report, "tests", tests); |
---|
618 | if (skipped) appendValue(report, "skipped", skipped); |
---|
619 | if (tests) { |
---|
620 | if (!failed) appendValue(report, "passed", "ALL"); |
---|
621 | else appendValue(report, "passed", passed); |
---|
622 | } |
---|
623 | if (failed) appendValue(report, "failed", failed); |
---|
624 | if (warnings) appendValue(report, "warnings", warnings); |
---|
625 | |
---|
626 | return report.c_str()+1; // skip leading space |
---|
627 | } |
---|
628 | |
---|
629 | const size_t Mb = 1024*1024; |
---|
630 | const size_t Gb = 1024*Mb; |
---|
631 | |
---|
632 | UnitTester::UnitTester(const char *libname, const UnitTest_simple *simple_tests, int warn_level, size_t skippedTests, const UnitTest_simple *postcond) { |
---|
633 | // this is the "main()" of the unit test |
---|
634 | // it is invoked from code generated by sym2testcode.pl@InvokeUnitTester |
---|
635 | |
---|
636 | #if !defined(LEAKS_SANITIZED) && !defined(UNDEF_SANITIZED) |
---|
637 | // restrict memory available for test-code (does not work together with sanitizer) |
---|
638 | ModRLimit restrict_virtual_memory(RLIMIT_AS, 1 * Gb); |
---|
639 | #endif |
---|
640 | |
---|
641 | start_of_main(); |
---|
642 | |
---|
643 | TEST_EXPECT_ZERO_OR_SHOW_ERRNO(chdir("run")); |
---|
644 | GLOBAL.runDir = getcwd(NULp, ARB_PATH_MAX); |
---|
645 | GLOBAL.libname = libname; |
---|
646 | |
---|
647 | size_t tests = 0; |
---|
648 | size_t passed = 0; |
---|
649 | |
---|
650 | { |
---|
651 | arb_test::GlobalTestData& global = arb_test::test_data(); |
---|
652 | |
---|
653 | global.init(flag_callback); |
---|
654 | global.show_warnings = (warn_level != 0); |
---|
655 | |
---|
656 | double duration_ms = 0; |
---|
657 | { |
---|
658 | SimpleTester simple_tester(simple_tests, postcond); |
---|
659 | |
---|
660 | tests = simple_tester.get_test_count(); |
---|
661 | if (tests) { |
---|
662 | passed = simple_tester.perform_all(); |
---|
663 | duration_ms = simple_tester.overall_duration_ms(); |
---|
664 | } |
---|
665 | } |
---|
666 | |
---|
667 | trace("%s", generateReport(libname, |
---|
668 | tests+skippedTests, skippedTests, passed, |
---|
669 | duration_ms, global.warnings)); |
---|
670 | } |
---|
671 | |
---|
672 | arb_test::GlobalTestData::erase_instance(); |
---|
673 | exit(tests == passed ? EXIT_SUCCESS : EXIT_FAILURE); |
---|
674 | } |
---|
675 | |
---|
676 | |
---|