test: add RETURN_SKIP and RETURN_TODO macros

Added two new flags to identify tests that are intentionally ignored
(usually because we don't want to implement the tested functionality
on current platform) and test serving as TODO list (usually indicating
that the tested functionality should be implemented on current plaform
in the near future.)

test/runner-unix.c

@@ -286,6 +286,34 @@ int process_copy_output(process_info_t *p, int fd) {
 }
 
 
+/* Copy the last line of the stdio output buffer to `buffer` */
+int process_read_last_line(process_info_t *p,
+                           char* buffer,
+                           size_t buffer_len) {
+  char* ptr;
+
+  int r = fseek(p->stdout_file, 0, SEEK_SET);
+  if (r < 0) {
+    perror("fseek");
+    return -1;
+  }
+
+  buffer[0] = '\0';
+
+  while (fgets(buffer, buffer_len, p->stdout_file) != NULL) {
+    for (ptr = buffer; *ptr && *ptr != '\r' && *ptr != '\n'; ptr++);
+    *ptr = '\0';
+  }
+
+  if (ferror(p->stdout_file)) {
+    perror("read");
+    buffer[0] = '\0';
+    return -1;
+  }
+  return 0;
+}
+
+
 /* Return the name that was specified when `p` was started by process_start */
 char* process_get_name(process_info_t *p) {
   return p->name;

test/runner-win.c

@@ -248,6 +248,46 @@ int process_copy_output(process_info_t *p, int fd) {
 }
 
 
+int process_read_last_line(process_info_t *p,
+                           char * buffer,
+                           size_t buffer_len) {
+  DWORD size;
+  DWORD read;
+  DWORD start;
+  OVERLAPPED overlapped;
+
+  ASSERT(buffer_len > 0);
+
+  size = GetFileSize(p->stdio_out, NULL);
+  if (size == INVALID_FILE_SIZE)
+    return -1;
+
+  if (size == 0) {
+    buffer[0] = '\0';
+    return 1;
+  }
+
+  memset(&overlapped, 0, sizeof overlapped);
+  if (size >= buffer_len)
+    overlapped.Offset = size - buffer_len - 1;
+
+  if (!ReadFile(p->stdio_out, buffer, buffer_len - 1, &read, &overlapped))
+    return -1;
+
+  for (start = read - 1; start >= 0; start--) {
+    if (buffer[start] == '\n' || buffer[start] == '\r')
+      break;
+  }
+
+  if (start > 0)
+    memmove(buffer, buffer + start, read - start);
+
+  buffer[read - start] = '\0';
+
+  return 0;
+}
+
+
 char* process_get_name(process_info_t *p) {
   return p->name;
 }

test/runner.c

@@ -31,12 +31,25 @@ char executable_path[PATHMAX] = { '\0' };
 int tap_output = 0;
 
 
-static void log_progress(int total, int passed, int failed, const char* name) {
+static void log_progress(int total,
+                         int passed,
+                         int failed,
+                         int todos,
+                         int skipped,
+                         const char* name) {
+  int progress;
+
   if (total == 0)
     total = 1;
 
-  LOGF("[%% %3d|+ %3d|- %3d]: %s", (int) ((passed + failed) / ((double) total) * 100.0),
-      passed, failed, name);
+  progress = 100 * (passed + failed + skipped + todos) / total;
+  LOGF("[%% %3d|+ %3d|- %3d|T %3d|S %3d]: %s",
+       progress,
+       passed,
+       failed,
+       todos,
+       skipped,
+       name);
 }
 
 
@@ -78,7 +91,13 @@ const char* fmt(double d) {
 
 
 int run_tests(int timeout, int benchmark_output) {
-  int total, passed, failed, current;
+  int total;
+  int passed;
+  int failed;
+  int todos;
+  int skipped;
+  int current;
+  int test_result;
   task_entry_t* task;
 
   /* Count the number of tests. */
@@ -96,6 +115,8 @@ int run_tests(int timeout, int benchmark_output) {
   /* Run all tests. */
   passed = 0;
   failed = 0;
+  todos = 0;
+  skipped = 0;
   current = 1;
   for (task = TASKS; task->main; task++) {
     if (task->is_helper) {
@@ -106,13 +127,15 @@ int run_tests(int timeout, int benchmark_output) {
       rewind_cursor();
 
     if (!benchmark_output && !tap_output) {
-      log_progress(total, passed, failed, task->task_name);
+      log_progress(total, passed, failed, todos, skipped, task->task_name);
     }
 
-    if (run_test(task->task_name, timeout, benchmark_output, current) == 0) {
-      passed++;
-    } else {
-      failed++;
+    test_result = run_test(task->task_name, timeout, benchmark_output, current);
+    switch (test_result) {
+    case TEST_OK: passed++; break;
+    case TEST_TODO: todos++; break;
+    case TEST_SKIP: skipped++; break;
+    default: failed++;
     }
     current++;
   }
@@ -121,13 +144,50 @@ int run_tests(int timeout, int benchmark_output) {
     rewind_cursor();
 
   if (!benchmark_output && !tap_output) {
-    log_progress(total, passed, failed, "Done.\n");
+    log_progress(total, passed, failed, todos, skipped, "Done.\n");
   }
 
   return failed;
 }
 
 
+void log_tap_result(int test_count,
+                    const char* test,
+                    int status,
+                    process_info_t* process) {
+  const char* result;
+  const char* directive;
+  char reason[1024];
+
+  switch (status) {
+  case TEST_OK:
+    result = "ok";
+    directive = "";
+    break;
+  case TEST_TODO:
+    result = "not ok";
+    directive = " # TODO ";
+    break;
+  case TEST_SKIP:
+    result = "ok";
+    directive = " # SKIP ";
+    break;
+  default:
+    result = "not ok";
+    directive = "";
+  }
+
+  if ((status == TEST_SKIP || status == TEST_TODO) &&
+      process_output_size(process) > 0) {
+    process_read_last_line(process, reason, sizeof reason);
+  } else {
+    reason[0] = '\0';
+  }
+
+  LOGF("%s %d - %s%s%s\n", result, test_count, test, directive, reason);
+}
+
+
 int run_test(const char* test,
              int timeout,
              int benchmark_output,
@@ -231,7 +291,7 @@ int run_test(const char* test,
   }
 
   status = process_reap(main_proc);
-  if (status != 0) {
+  if (status != TEST_OK) {
     snprintf(errmsg,
              sizeof errmsg,
              "exit code %d",
@@ -255,17 +315,17 @@ out:
     FATAL("process_wait failed");
   }
 
-  if (tap_output) {
-    if (status == 0)
-      LOGF("ok %d - %s\n", test_count, test);
-    else
-      LOGF("not ok %d - %s\n", test_count, test);
-  }
+  if (tap_output)
+    log_tap_result(test_count, test, status, &processes[i]);
 
   /* Show error and output from processes if the test failed. */
   if (status != 0 || task->show_output) {
     if (tap_output) {
       LOGF("#");
+    } else if (status == TEST_TODO) {
+      LOGF("\n`%s` todo\n", test);
+    } else if (status == TEST_SKIP) {
+      LOGF("\n`%s` skipped\n", test);
     } else if (status != 0) {
       LOGF("\n`%s` failed: %s\n", test, errmsg);
     } else {

test/runner.h

@@ -143,6 +143,11 @@ long int process_output_size(process_info_t *p);
 /* Copy the contents of the stdio output buffer to `fd`. */
 int process_copy_output(process_info_t *p, int fd);
 
+/* Copy the last line of the stdio output buffer to `buffer` */
+int process_read_last_line(process_info_t *p,
+                           char * buffer,
+                           size_t buffer_len);
+
 /* Return the name that was specified when `p` was started by process_start */
 char* process_get_name(process_info_t *p);
 

test/task.h

@@ -119,4 +119,28 @@ void uv_sleep(int msec);
 /* Format big numbers nicely. WARNING: leaks memory. */
 const char* fmt(double d);
 
+/* Reserved test exit codes. */
+enum test_status {
+  TEST_OK = 0,
+  TEST_TODO,
+  TEST_SKIP
+};
+
+#define RETURN_OK()                                                           \
+  do {                                                                        \
+    return TEST_OK;                                                           \
+  } while (0)
+
+#define RETURN_TODO(explanation)                                              \
+  do {                                                                        \
+    LOGF("%s\n", explanation);                                                \
+    return TEST_TODO;                                                         \
+  } while (0)
+
+#define RETURN_SKIP(explanation)                                              \
+  do {                                                                        \
+    LOGF("%s\n", explanation);                                                \
+    return TEST_SKIP;                                                         \
+  } while (0)
+
 #endif /* TASK_H_ */