From 2c21050956206b5e7962e86f4bdbaade1a44b6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miroslav=20Bajto=C5=A1?= Date: Sat, 20 Apr 2013 06:43:31 +0200 Subject: [PATCH] test: add RETURN_SKIP and RETURN_TODO macros Added two new flags to identify tests that are intentionally ignored (usually because we don't want to implement the tested functionality on current platform) and test serving as TODO list (usually indicating that the tested functionality should be implemented on current plaform in the near future.) --- test/runner-unix.c | 28 ++++++++++++++ test/runner-win.c | 40 ++++++++++++++++++++ test/runner.c | 94 +++++++++++++++++++++++++++++++++++++--------- test/runner.h | 5 +++ test/task.h | 24 ++++++++++++ 5 files changed, 174 insertions(+), 17 deletions(-) diff --git a/test/runner-unix.c b/test/runner-unix.c index eefb7008..b66df51b 100644 --- a/test/runner-unix.c +++ b/test/runner-unix.c @@ -286,6 +286,34 @@ int process_copy_output(process_info_t *p, int fd) { } +/* Copy the last line of the stdio output buffer to `buffer` */ +int process_read_last_line(process_info_t *p, + char* buffer, + size_t buffer_len) { + char* ptr; + + int r = fseek(p->stdout_file, 0, SEEK_SET); + if (r < 0) { + perror("fseek"); + return -1; + } + + buffer[0] = '\0'; + + while (fgets(buffer, buffer_len, p->stdout_file) != NULL) { + for (ptr = buffer; *ptr && *ptr != '\r' && *ptr != '\n'; ptr++); + *ptr = '\0'; + } + + if (ferror(p->stdout_file)) { + perror("read"); + buffer[0] = '\0'; + return -1; + } + return 0; +} + + /* Return the name that was specified when `p` was started by process_start */ char* process_get_name(process_info_t *p) { return p->name; diff --git a/test/runner-win.c b/test/runner-win.c index 3aae1c3e..5d232594 100644 --- a/test/runner-win.c +++ b/test/runner-win.c @@ -248,6 +248,46 @@ int process_copy_output(process_info_t *p, int fd) { } +int process_read_last_line(process_info_t *p, + char * buffer, + size_t buffer_len) { + DWORD size; + DWORD read; + DWORD start; + OVERLAPPED overlapped; + + ASSERT(buffer_len > 0); + + size = GetFileSize(p->stdio_out, NULL); + if (size == INVALID_FILE_SIZE) + return -1; + + if (size == 0) { + buffer[0] = '\0'; + return 1; + } + + memset(&overlapped, 0, sizeof overlapped); + if (size >= buffer_len) + overlapped.Offset = size - buffer_len - 1; + + if (!ReadFile(p->stdio_out, buffer, buffer_len - 1, &read, &overlapped)) + return -1; + + for (start = read - 1; start >= 0; start--) { + if (buffer[start] == '\n' || buffer[start] == '\r') + break; + } + + if (start > 0) + memmove(buffer, buffer + start, read - start); + + buffer[read - start] = '\0'; + + return 0; +} + + char* process_get_name(process_info_t *p) { return p->name; } diff --git a/test/runner.c b/test/runner.c index bda1080a..d8e9ddeb 100644 --- a/test/runner.c +++ b/test/runner.c @@ -31,12 +31,25 @@ char executable_path[PATHMAX] = { '\0' }; int tap_output = 0; -static void log_progress(int total, int passed, int failed, const char* name) { +static void log_progress(int total, + int passed, + int failed, + int todos, + int skipped, + const char* name) { + int progress; + if (total == 0) total = 1; - LOGF("[%% %3d|+ %3d|- %3d]: %s", (int) ((passed + failed) / ((double) total) * 100.0), - passed, failed, name); + progress = 100 * (passed + failed + skipped + todos) / total; + LOGF("[%% %3d|+ %3d|- %3d|T %3d|S %3d]: %s", + progress, + passed, + failed, + todos, + skipped, + name); } @@ -78,7 +91,13 @@ const char* fmt(double d) { int run_tests(int timeout, int benchmark_output) { - int total, passed, failed, current; + int total; + int passed; + int failed; + int todos; + int skipped; + int current; + int test_result; task_entry_t* task; /* Count the number of tests. */ @@ -96,6 +115,8 @@ int run_tests(int timeout, int benchmark_output) { /* Run all tests. */ passed = 0; failed = 0; + todos = 0; + skipped = 0; current = 1; for (task = TASKS; task->main; task++) { if (task->is_helper) { @@ -106,13 +127,15 @@ int run_tests(int timeout, int benchmark_output) { rewind_cursor(); if (!benchmark_output && !tap_output) { - log_progress(total, passed, failed, task->task_name); + log_progress(total, passed, failed, todos, skipped, task->task_name); } - if (run_test(task->task_name, timeout, benchmark_output, current) == 0) { - passed++; - } else { - failed++; + test_result = run_test(task->task_name, timeout, benchmark_output, current); + switch (test_result) { + case TEST_OK: passed++; break; + case TEST_TODO: todos++; break; + case TEST_SKIP: skipped++; break; + default: failed++; } current++; } @@ -121,13 +144,50 @@ int run_tests(int timeout, int benchmark_output) { rewind_cursor(); if (!benchmark_output && !tap_output) { - log_progress(total, passed, failed, "Done.\n"); + log_progress(total, passed, failed, todos, skipped, "Done.\n"); } return failed; } +void log_tap_result(int test_count, + const char* test, + int status, + process_info_t* process) { + const char* result; + const char* directive; + char reason[1024]; + + switch (status) { + case TEST_OK: + result = "ok"; + directive = ""; + break; + case TEST_TODO: + result = "not ok"; + directive = " # TODO "; + break; + case TEST_SKIP: + result = "ok"; + directive = " # SKIP "; + break; + default: + result = "not ok"; + directive = ""; + } + + if ((status == TEST_SKIP || status == TEST_TODO) && + process_output_size(process) > 0) { + process_read_last_line(process, reason, sizeof reason); + } else { + reason[0] = '\0'; + } + + LOGF("%s %d - %s%s%s\n", result, test_count, test, directive, reason); +} + + int run_test(const char* test, int timeout, int benchmark_output, @@ -231,7 +291,7 @@ int run_test(const char* test, } status = process_reap(main_proc); - if (status != 0) { + if (status != TEST_OK) { snprintf(errmsg, sizeof errmsg, "exit code %d", @@ -255,17 +315,17 @@ out: FATAL("process_wait failed"); } - if (tap_output) { - if (status == 0) - LOGF("ok %d - %s\n", test_count, test); - else - LOGF("not ok %d - %s\n", test_count, test); - } + if (tap_output) + log_tap_result(test_count, test, status, &processes[i]); /* Show error and output from processes if the test failed. */ if (status != 0 || task->show_output) { if (tap_output) { LOGF("#"); + } else if (status == TEST_TODO) { + LOGF("\n`%s` todo\n", test); + } else if (status == TEST_SKIP) { + LOGF("\n`%s` skipped\n", test); } else if (status != 0) { LOGF("\n`%s` failed: %s\n", test, errmsg); } else { diff --git a/test/runner.h b/test/runner.h index 8be9e39f..aa7f2054 100644 --- a/test/runner.h +++ b/test/runner.h @@ -143,6 +143,11 @@ long int process_output_size(process_info_t *p); /* Copy the contents of the stdio output buffer to `fd`. */ int process_copy_output(process_info_t *p, int fd); +/* Copy the last line of the stdio output buffer to `buffer` */ +int process_read_last_line(process_info_t *p, + char * buffer, + size_t buffer_len); + /* Return the name that was specified when `p` was started by process_start */ char* process_get_name(process_info_t *p); diff --git a/test/task.h b/test/task.h index de7c80ec..308201e5 100644 --- a/test/task.h +++ b/test/task.h @@ -119,4 +119,28 @@ void uv_sleep(int msec); /* Format big numbers nicely. WARNING: leaks memory. */ const char* fmt(double d); +/* Reserved test exit codes. */ +enum test_status { + TEST_OK = 0, + TEST_TODO, + TEST_SKIP +}; + +#define RETURN_OK() \ + do { \ + return TEST_OK; \ + } while (0) + +#define RETURN_TODO(explanation) \ + do { \ + LOGF("%s\n", explanation); \ + return TEST_TODO; \ + } while (0) + +#define RETURN_SKIP(explanation) \ + do { \ + LOGF("%s\n", explanation); \ + return TEST_SKIP; \ + } while (0) + #endif /* TASK_H_ */