diff --git a/Makefile b/Makefile index 0b4f38bb..bad95dbd 100644 --- a/Makefile +++ b/Makefile @@ -97,15 +97,21 @@ test/echo.o: test/echo.c test/echo.h $(CC) $(CPPFLAGS) $(CFLAGS) -c test/echo.c -o test/echo.o -.PHONY: clean clean-platform distclean distclean-platform test benchmark +.PHONY: clean clean-platform distclean distclean-platform test bench test: test/run-tests$(E) test/run-tests +test-%: test/run-tests$(E) + test/run-tests $(@:test-%=%) + bench: test/run-benchmarks$(E) test/run-benchmarks +bench-%: test/run-benchmarks$(E) + test/run-benchmarks $(@:bench-%=%) + clean: clean-platform $(RM) -f src/*.o *.a test/run-tests$(E) test/run-benchmarks$(E) diff --git a/test/run-benchmarks.c b/test/run-benchmarks.c index 43d25a63..bdef4a4b 100644 --- a/test/run-benchmarks.c +++ b/test/run-benchmarks.c @@ -34,26 +34,14 @@ int main(int argc, char **argv) { - task_entry_t *task; - platform_init(argc, argv); - if (argc > 1) { - /* A specific process was requested. */ - return run_process(argv[1]); - - } else { - /* Run all benchmarks. */ - task = (task_entry_t*)&TASKS; - for (task = (task_entry_t*)&TASKS; task->main; task++) { - if (task->is_helper) { - continue; - } - - run_task(task, BENCHMARK_TIMEOUT, 1); - } - LOG("Done.\n"); - - return 0; + switch (argc) { + case 1: return run_tests(BENCHMARK_TIMEOUT, 1); + case 2: return run_test(argv[1], BENCHMARK_TIMEOUT, 1); + case 3: return run_test_part(argv[1], argv[2]); + default: + LOGF("Too many arguments.\n"); + return 1; } } diff --git a/test/run-tests.c b/test/run-tests.c index f83006bd..7ae6c6c7 100644 --- a/test/run-tests.c +++ b/test/run-tests.c @@ -33,54 +33,15 @@ #define TEST_TIMEOUT 5000 -static void log_progress(int total, int passed, int failed, char* name) { - LOGF("[%% %3d|+ %3d|- %3d]: %s", (passed + failed) / total * 100, - passed, failed, name); -} - - int main(int argc, char **argv) { - int total, passed, failed; - task_entry_t* task; - platform_init(argc, argv); - if (argc > 1) { - /* A specific process was requested. */ - return run_process(argv[1]); - - } else { - /* Count the number of tests. */ - total = 0; - task = (task_entry_t*)&TASKS; - for (task = (task_entry_t*)&TASKS; task->main; task++) { - if (!task->is_helper) { - total++; - } - } - - /* Run all tests. */ - passed = 0; - failed = 0; - task = (task_entry_t*)&TASKS; - for (task = (task_entry_t*)&TASKS; task->main; task++) { - if (task->is_helper) { - continue; - } - - rewind_cursor(); - log_progress(total, passed, failed, task->task_name); - - if (run_task(task, TEST_TIMEOUT, 0)) { - passed++; - } else { - failed++; - } - } - - rewind_cursor(); - log_progress(total, passed, failed, "Done.\n"); - - return 0; + switch (argc) { + case 1: return run_tests(TEST_TIMEOUT, 0); + case 2: return run_test(argv[1], TEST_TIMEOUT, 0); + case 3: return run_test_part(argv[1], argv[2]); + default: + LOGF("Too many arguments.\n"); + return 1; } } diff --git a/test/runner-unix.c b/test/runner-unix.c index fa4c63bb..0bbfd62e 100644 --- a/test/runner-unix.c +++ b/test/runner-unix.c @@ -68,9 +68,9 @@ void platform_init(int argc, char **argv) { } -/* Invoke "arv[0] test-name". Store process info in *p. */ +/* Invoke "argv[0] test-name [test-part]". Store process info in *p. */ /* Make sure that all stdio output of the processes is buffered up. */ -int process_start(char* name, process_info_t* p) { +int process_start(char* name, char* part, process_info_t* p) { FILE* stdout_file = tmpfile(); if (!stdout_file) { perror("tmpfile"); @@ -92,7 +92,7 @@ int process_start(char* name, process_info_t* p) { dup2(fileno(stdout_file), STDOUT_FILENO); dup2(fileno(stdout_file), STDERR_FILENO); - char* args[3] = { executable_path, name, NULL }; + char* args[] = { executable_path, name, part, NULL }; execvp(executable_path, args); perror("execvp()"); _exit(127); diff --git a/test/runner-win.c b/test/runner-win.c index 09458a6b..fc08839e 100644 --- a/test/runner-win.c +++ b/test/runner-win.c @@ -52,7 +52,7 @@ void platform_init(int argc, char **argv) { } -int process_start(char *name, process_info_t *p) { +int process_start(char *name, char *part, process_info_t *p) { HANDLE file = INVALID_HANDLE_VALUE; HANDLE nul = INVALID_HANDLE_VALUE; WCHAR path[MAX_PATH], filename[MAX_PATH]; @@ -97,12 +97,24 @@ int process_start(char *name, process_info_t *p) { if (result == 0 || result == sizeof(image)) goto error; - if (_snwprintf((wchar_t*)&args, - sizeof(args) / sizeof(wchar_t), - L"\"%s\" %S", - image, - name) < 0) - goto error; + if (part) { + if (_snwprintf((wchar_t*)args, + sizeof(args) / sizeof(wchar_t), + L"\"%s\" %S %S", + image, + name, + part) < 0) { + goto error; + } + } else { + if (_snwprintf((wchar_t*)args, + sizeof(args) / sizeof(wchar_t), + L"\"%s\" %S", + image, + name) < 0) { + goto error; + } + } memset((void*)&si, 0, sizeof(si)); si.cb = sizeof(si); diff --git a/test/runner.c b/test/runner.c index 7b6590f6..9309414a 100644 --- a/test/runner.c +++ b/test/runner.c @@ -26,102 +26,155 @@ char executable_path[PATHMAX] = { '\0' }; -/* Start a specific process declared by TEST_ENTRY or TEST_HELPER. */ -/* Returns the exit code of the specific process. */ -int run_process(char* name) { - task_entry_t *test; - for (test = (task_entry_t*)&TASKS; test->main; test++) { - if (strcmp(name, test->process_name) == 0) { - return test->main(); - } - } - - LOGF("Test process %s not found!\n", name); - return 255; +static void log_progress(int total, int passed, int failed, const char* name) { + LOGF("[%% %3d|+ %3d|- %3d]: %s", (passed + failed) / total * 100, + passed, failed, name); } -/* - * Runs all processes associated with a particular test or benchmark. - * It returns 1 if the test succeeded, 0 if it failed. - * If the test fails it prints diagnostic information. - * If benchmark_output is nonzero, the output from the main process is - * always shown. - */ -int run_task(task_entry_t *test, int timeout, int benchmark_output) { - int i, result, success; - char errmsg[256]; - task_entry_t *helper; - int process_count; - process_info_t processes[MAX_PROCESSES]; - process_info_t *main_process; +int run_tests(int timeout, int benchmark_output) { + int total, passed, failed; + task_entry_t* task; - success = 0; - - process_count = 0; - - /* Start all helpers for this test first. */ - for (helper = (task_entry_t*)&TASKS; helper->main; helper++) { - if (helper->is_helper && - strcmp(test->task_name, helper->task_name) == 0) { - if (process_start(helper->process_name, &processes[process_count]) == -1) { - snprintf((char*)&errmsg, - sizeof(errmsg), - "process `%s` failed to start.", - helper->process_name); - goto finalize; - } - process_count++; + /* Count the number of tests. */ + total = 0; + for (task = TASKS; task->main; task++) { + if (!task->is_helper) { + total++; } } - /* Wait a little bit to allow servers to start. Racy. */ - uv_sleep(100); + /* Run all tests. */ + passed = 0; + failed = 0; + for (task = TASKS; task->main; task++) { + if (task->is_helper) { + continue; + } - /* Start the main test process. */ - if (process_start(test->process_name, &processes[process_count]) == -1) { - snprintf((char*)&errmsg, sizeof(errmsg), "process `%s` failed to start.", - test->process_name); - goto finalize; + rewind_cursor(); + log_progress(total, passed, failed, task->task_name); + + if (run_test(task->task_name, timeout, benchmark_output) == 0) { + passed++; + } else { + failed++; + } } - main_process = &processes[process_count]; - process_count++; - /* Wait for the main process to terminate. */ - result = process_wait(main_process, 1, timeout); + rewind_cursor(); + log_progress(total, passed, failed, "Done.\n"); + + return 0; +} + + +int run_test(const char* test, int timeout, int benchmark_output) { + char errmsg[1024] = "no error"; + process_info_t processes[1024]; + process_info_t *main_proc; + task_entry_t* task; + int process_count; + int result; + int status; + int i; + + status = 255; + process_count = 0; + + /* Start the helpers first. */ + for (task = TASKS; task->main; task++) { + if (strcmp(test, task->task_name) != 0) { + continue; + } + + /* Skip the test itself. */ + if (!task->is_helper) { + continue; + } + + if (process_start(task->task_name, + task->process_name, + &processes[process_count]) == -1) { + snprintf(errmsg, + sizeof errmsg, + "Process `%s` failed to start.", + task->process_name); + goto out; + } + + process_count++; + } + + /* Give the helpers time to settle. Race-y, fix this. */ + uv_sleep(250); + + /* Now start the test itself. */ + for (main_proc = NULL, task = TASKS; task->main; task++) { + if (strcmp(test, task->task_name) != 0) { + continue; + } + + if (task->is_helper) { + continue; + } + + if (process_start(task->task_name, + task->process_name, + &processes[process_count]) == -1) { + snprintf(errmsg, + sizeof errmsg, + "Process `%s` failed to start.", + task->process_name); + goto out; + } + + main_proc = &processes[process_count]; + process_count++; + break; + } + + if (main_proc == NULL) { + snprintf(errmsg, + sizeof errmsg, + "No test with that name: %s", + test); + goto out; + } + + result = process_wait(main_proc, 1, timeout); if (result == -1) { FATAL("process_wait failed"); } else if (result == -2) { - snprintf((char*)&errmsg, sizeof(errmsg), "timeout."); - goto finalize; + /* Don't have to clean up the process, process_wait() has killed it. */ + snprintf(errmsg, + sizeof errmsg, + "timeout"); + goto out; } - /* Reap the main process. */ - result = process_reap(main_process); - if (result != 0) { - snprintf((char*)&errmsg, sizeof(errmsg), "exit code %d.", result); - goto finalize; + status = process_reap(main_proc); + if (status != 0) { + snprintf(errmsg, + sizeof errmsg, + "exit code %d", + status); } - /* Yes! did it. */ - success = 1; - -finalize: - /* Kill all (helper) processes that are still running. */ - for (i = 0; i < process_count; i++) { - /* If terminate fails the process is probably already closed. */ +out: + /* Reap running processes except the main process, it's already dead. */ + for (i = 0; i < process_count - 1; i++) { process_terminate(&processes[i]); } - /* Wait until all processes have really terminated. */ - if (process_wait((process_info_t*)&processes, process_count, -1) < 0) { + if (process_wait(processes, process_count - 1, -1) < 0) { FATAL("process_wait failed"); } /* Show error and output from processes if the test failed. */ - if (!success) { - LOGF("\n`%s` failed: %s\n", test->task_name, errmsg); + if (status != 0) { + LOGF("\n`%s` failed: %s\n", test, errmsg); for (i = 0; i < process_count; i++) { switch (process_output_size(&processes[i])) { @@ -142,30 +195,25 @@ finalize: } } LOG("=============================================================\n"); + } - /* In benchmark mode show concise output from the main process. */ - } else if (benchmark_output) { - switch (process_output_size(main_process)) { - case -1: - LOGF("%s: (unavailabe)\n", test->task_name); - break; + return status; +} - case 0: - LOGF("%s: (no output)\n", test->task_name); - break; - default: - for (i = 0; i < process_count; i++) { - process_copy_output(&processes[i], fileno(stderr)); - } - break; +/* Returns the status code of the task part + * or 255 if no matching task was not found. + */ +int run_test_part(const char* test, const char* part) { + task_entry_t* task; + + for (task = TASKS; task->main; task++) { + if (strcmp(test, task->task_name) == 0 + && strcmp(part, task->process_name) == 0) { + return task->main(); } } - /* Clean up all process handles. */ - for (i = 0; i < process_count; i++) { - process_cleanup(&processes[i]); - } - - return success; + LOGF("No test part with that name: %s:%s\n", test, part); + return 255; } diff --git a/test/runner.h b/test/runner.h index 11d49802..59eff1ed 100644 --- a/test/runner.h +++ b/test/runner.h @@ -41,13 +41,6 @@ typedef struct { } task_entry_t, bench_entry_t; -/* Runs an individual task; returns 1 if the test succeeded, 0 if it failed. */ -/* If the test fails it prints diagnostic information. */ -/* If benchmark_output is nonzero, the output from the main process is -/* always shown. */ -int run_task(task_entry_t *test, int timeout, int benchmark_output); - - /* * Macros used by test-list.h and benchmark-list.h. */ @@ -95,13 +88,20 @@ extern char executable_path[PATHMAX]; /* The array that is filled by test-list.h or benchmark-list.h */ extern task_entry_t TASKS[]; -/* Start a specific process declared by TEST_ENTRY or TEST_HELPER. */ -/* Returns the exit code of the specific process. */ -int run_task(task_entry_t *test, int timeout, int benchmark_output); +/* + * Run all tests. + */ +int run_tests(int timeout, int benchmark_output); -/* Start a specific process declared by TEST_ENTRY or TEST_HELPER. */ -/* Returns the exit code of the specific process. */ -int run_process(char* name); +/* + * Run a single test. Starts up any helpers. + */ +int run_test(const char* test, int timeout, int benchmark_output); + +/* + * Run a test part, i.e. the test or one of its helpers. + */ +int run_test_part(const char* test, const char* part); /* @@ -113,9 +113,9 @@ int run_process(char* name); /* Do platform-specific initialization. */ void platform_init(); -/* Invoke "arv[0] test-name". Store process info in *p. */ +/* Invoke "argv[0] test-name [test-part]". Store process info in *p. */ /* Make sure that all stdio output of the processes is buffered up. */ -int process_start(char *name, process_info_t *p); +int process_start(char *name, char* part, process_info_t *p); /* Wait for all `n` processes in `vec` to terminate. */ /* Time out after `timeout` msec, or never if timeout == -1 */