unix: use ngx_queue_move when iterating over lists

Replace uses of ngx_queue_foreach when the list can get modified while
iterating over it, in particular when a callback is made into the
user's code.  This should fix a number of spurious failures that
people have been reporting.

This is a backport of commit 442b8a5 from the v1.x branch.

PR-URL: https://github.com/libuv/libuv/pull/566
Reviewed-By: Saúl Ibarra Corretgé <saghul@gmail.com>

include/uv-private/ngx-queue.h

@@ -131,6 +131,9 @@ struct ngx_queue_s {
     ((type *) ((unsigned char *) q - offsetof(type, link)))
 
 
+/* Important note: mutating the list while ngx_queue_foreach is
+ * iterating over its elements results in undefined behavior.
+ */
 #define ngx_queue_foreach(q, h)                                               \
     for ((q) = ngx_queue_head(h);                                             \
          (q) != ngx_queue_sentinel(h) && !ngx_queue_empty(h);                 \

src/unix/async.c

@@ -74,12 +74,18 @@ void uv__async_close(uv_async_t* handle) {
 static void uv__async_event(uv_loop_t* loop,
                             struct uv__async* w,
                             unsigned int nevents) {
+  ngx_queue_t queue;
   ngx_queue_t* q;
   uv_async_t* h;
 
-  ngx_queue_foreach(q, &loop->async_handles) {
+  ngx_queue_move(&loop->async_handles, &queue);
+  while (!ngx_queue_empty(&queue)) {
+    q = ngx_queue_head(&queue);
     h = ngx_queue_data(q, uv_async_t, queue);
 
+    ngx_queue_remove(q);
+    ngx_queue_insert_tail(&loop->async_handles, q);
+
     if (cmpxchgi(&h->pending, 1, 0) == 0)
       continue;
 

src/unix/linux-inotify.c

@@ -119,6 +119,7 @@ static void uv__inotify_read(uv_loop_t* loop,
   const struct uv__inotify_event* e;
   struct watcher_list* w;
   uv_fs_event_t* h;
+  ngx_queue_t queue;
   ngx_queue_t* q;
   const char* path;
   ssize_t size;
@@ -158,8 +159,14 @@ static void uv__inotify_read(uv_loop_t* loop,
        */
       path = e->len ? (const char*) (e + 1) : basename_r(w->path);
 
-      ngx_queue_foreach(q, &w->watchers) {
+      ngx_queue_move(&w->watchers, &queue);
+      while (!ngx_queue_empty(&queue)) {
+        q = ngx_queue_head(&queue);
         h = ngx_queue_data(q, uv_fs_event_t, watchers);
+
+        ngx_queue_remove(q);
+        ngx_queue_insert_tail(&w->watchers, q);
+
         h->cb(h, path, events, 0);
       }
     }

src/unix/loop-watcher.c

@@ -48,9 +48,14 @@
                                                                               \
   void uv__run_##name(uv_loop_t* loop) {                                      \
     uv_##name##_t* h;                                                         \
+    ngx_queue_t queue;                                                        \
     ngx_queue_t* q;                                                           \
-    ngx_queue_foreach(q, &loop->name##_handles) {                             \
+    ngx_queue_move(&loop->name##_handles, &queue);                            \
+    while (!ngx_queue_empty(&queue)) {                                        \
+      q = ngx_queue_head(&queue);                                             \
       h = ngx_queue_data(q, uv_##name##_t, queue);                            \
+      ngx_queue_remove(q);                                                    \
+      ngx_queue_insert_tail(&loop->name##_handles, q);                        \
       h->name##_cb(h, 0);                                                     \
     }                                                                         \
   }                                                                           \

src/unix/signal.c

@@ -231,6 +231,8 @@ void uv__signal_loop_cleanup(uv_loop_t* loop) {
   /* Stop all the signal watchers that are still attached to this loop. This
    * ensures that the (shared) signal tree doesn't contain any invalid entries
    * entries, and that signal handlers are removed when appropriate.
+   * It's safe to use QUEUE_FOREACH here because the handles and the handle
+   * queue are not modified by uv__signal_stop().
    */
   ngx_queue_foreach(q, &loop->handle_queue) {
     uv_handle_t* handle = ngx_queue_data(q, uv_handle_t, handle_queue);

src/uv-common.c

@@ -362,11 +362,18 @@ unsigned long uv_thread_self(void) {
 
 
 void uv_walk(uv_loop_t* loop, uv_walk_cb walk_cb, void* arg) {
+  ngx_queue_t queue;
   ngx_queue_t* q;
   uv_handle_t* h;
 
-  ngx_queue_foreach(q, &loop->handle_queue) {
+  ngx_queue_move(&loop->handle_queue, &queue);
+  while (!ngx_queue_empty(&queue)) {
+    q = ngx_queue_head(&queue);
     h = ngx_queue_data(q, uv_handle_t, handle_queue);
+
+    ngx_queue_remove(q);
+    ngx_queue_insert_tail(&loop->handle_queue, q);
+
     if (h->flags & UV__HANDLE_INTERNAL) continue;
     walk_cb(h, arg);
   }