linux: add some more iouring backed fs ops (#4012)

Specifically: `link`, `mkdir`, `rename`, `symlink` and `unlink`.

src/unix/fs.c

@@ -899,7 +899,7 @@ out:
 
 
 #ifdef __linux__
-static unsigned uv__kernel_version(void) {
+unsigned uv__kernel_version(void) {
   static _Atomic unsigned cached_version;
   struct utsname u;
   unsigned version;
@@ -1926,6 +1926,9 @@ int uv_fs_link(uv_loop_t* loop,
                uv_fs_cb cb) {
   INIT(LINK);
   PATH2;
+  if (cb != NULL)
+    if (uv__iou_fs_link(loop, req))
+      return 0;
   POST;
 }
 
@@ -1938,6 +1941,9 @@ int uv_fs_mkdir(uv_loop_t* loop,
   INIT(MKDIR);
   PATH;
   req->mode = mode;
+  if (cb != NULL)
+    if (uv__iou_fs_mkdir(loop, req))
+      return 0;
   POST;
 }
 
@@ -2089,6 +2095,9 @@ int uv_fs_rename(uv_loop_t* loop,
                  uv_fs_cb cb) {
   INIT(RENAME);
   PATH2;
+  if (cb != NULL)
+    if (uv__iou_fs_rename(loop, req))
+      return 0;
   POST;
 }
 
@@ -2135,6 +2144,9 @@ int uv_fs_symlink(uv_loop_t* loop,
   INIT(SYMLINK);
   PATH2;
   req->flags = flags;
+  if (cb != NULL)
+    if (uv__iou_fs_symlink(loop, req))
+      return 0;
   POST;
 }
 
@@ -2142,6 +2154,9 @@ int uv_fs_symlink(uv_loop_t* loop,
 int uv_fs_unlink(uv_loop_t* loop, uv_fs_t* req, const char* path, uv_fs_cb cb) {
   INIT(UNLINK);
   PATH;
+  if (cb != NULL)
+    if (uv__iou_fs_unlink(loop, req))
+      return 0;
   POST;
 }
 

src/unix/internal.h

@@ -335,20 +335,30 @@ int uv__iou_fs_close(uv_loop_t* loop, uv_fs_t* req);
 int uv__iou_fs_fsync_or_fdatasync(uv_loop_t* loop,
                                   uv_fs_t* req,
                                   uint32_t fsync_flags);
+int uv__iou_fs_link(uv_loop_t* loop, uv_fs_t* req);
+int uv__iou_fs_mkdir(uv_loop_t* loop, uv_fs_t* req);
 int uv__iou_fs_open(uv_loop_t* loop, uv_fs_t* req);
 int uv__iou_fs_read_or_write(uv_loop_t* loop,
                              uv_fs_t* req,
                              int is_read);
+int uv__iou_fs_rename(uv_loop_t* loop, uv_fs_t* req);
 int uv__iou_fs_statx(uv_loop_t* loop,
                      uv_fs_t* req,
                      int is_fstat,
                      int is_lstat);
+int uv__iou_fs_symlink(uv_loop_t* loop, uv_fs_t* req);
+int uv__iou_fs_unlink(uv_loop_t* loop, uv_fs_t* req);
 #else
 #define uv__iou_fs_close(loop, req) 0
 #define uv__iou_fs_fsync_or_fdatasync(loop, req, fsync_flags) 0
+#define uv__iou_fs_link(loop, req) 0
+#define uv__iou_fs_mkdir(loop, req) 0
 #define uv__iou_fs_open(loop, req) 0
 #define uv__iou_fs_read_or_write(loop, req, is_read) 0
+#define uv__iou_fs_rename(loop, req) 0
 #define uv__iou_fs_statx(loop, req, is_fstat, is_lstat) 0
+#define uv__iou_fs_symlink(loop, req) 0
+#define uv__iou_fs_unlink(loop, req) 0
 #endif
 
 #if defined(__APPLE__)
@@ -429,6 +439,7 @@ int uv__statx(int dirfd,
               struct uv__statx* statxbuf);
 void uv__statx_to_stat(const struct uv__statx* statxbuf, uv_stat_t* buf);
 ssize_t uv__getrandom(void* buf, size_t buflen, unsigned flags);
+unsigned uv__kernel_version(void);
 #endif
 
 typedef int (*uv__peersockfunc)(int, struct sockaddr*, socklen_t*);

src/unix/linux.c

@@ -150,6 +150,11 @@ enum {
   UV__IORING_OP_CLOSE = 19,
   UV__IORING_OP_STATX = 21,
   UV__IORING_OP_EPOLL_CTL = 29,
+  UV__IORING_OP_RENAMEAT = 35,
+  UV__IORING_OP_UNLINKAT = 36,
+  UV__IORING_OP_MKDIRAT = 37,
+  UV__IORING_OP_SYMLINKAT = 38,
+  UV__IORING_OP_LINKAT = 39,
 };
 
 enum {
@@ -162,6 +167,10 @@ enum {
   UV__IORING_SQ_CQ_OVERFLOW = 2u,
 };
 
+enum {
+  UV__MKDIRAT_SYMLINKAT_LINKAT = 1u,
+};
+
 struct uv__io_cqring_offsets {
   uint32_t head;
   uint32_t tail;
@@ -507,6 +516,10 @@ static void uv__iou_init(int epollfd,
   iou->sqelen = sqelen;
   iou->ringfd = ringfd;
   iou->in_flight = 0;
+  iou->flags = 0;
+
+  if (uv__kernel_version() >= /* 5.15.0 */ 0x050F00)
+    iou->flags |= UV__MKDIRAT_SYMLINKAT_LINKAT;
 
   for (i = 0; i <= iou->sqmask; i++)
     iou->sqarray[i] = i;  /* Slot -> sqe identity mapping. */
@@ -758,6 +771,55 @@ int uv__iou_fs_fsync_or_fdatasync(uv_loop_t* loop,
 }
 
 
+int uv__iou_fs_link(uv_loop_t* loop, uv_fs_t* req) {
+  struct uv__io_uring_sqe* sqe;
+  struct uv__iou* iou;
+
+  iou = &uv__get_internal_fields(loop)->iou;
+
+  if (!(iou->flags & UV__MKDIRAT_SYMLINKAT_LINKAT))
+    return 0;
+
+  sqe = uv__iou_get_sqe(iou, loop, req);
+  if (sqe == NULL)
+    return 0;
+
+  sqe->addr = (uintptr_t) req->path;
+  sqe->fd = AT_FDCWD;
+  sqe->addr2 = (uintptr_t) req->new_path;
+  sqe->len = AT_FDCWD;
+  sqe->opcode = UV__IORING_OP_LINKAT;
+
+  uv__iou_submit(iou);
+
+  return 1;
+}
+
+
+int uv__iou_fs_mkdir(uv_loop_t* loop, uv_fs_t* req) {
+  struct uv__io_uring_sqe* sqe;
+  struct uv__iou* iou;
+
+  iou = &uv__get_internal_fields(loop)->iou;
+
+  if (!(iou->flags & UV__MKDIRAT_SYMLINKAT_LINKAT))
+    return 0;
+
+  sqe = uv__iou_get_sqe(iou, loop, req);
+  if (sqe == NULL)
+    return 0;
+
+  sqe->addr = (uintptr_t) req->path;
+  sqe->fd = AT_FDCWD;
+  sqe->len = req->mode;
+  sqe->opcode = UV__IORING_OP_MKDIRAT;
+
+  uv__iou_submit(iou);
+
+  return 1;
+}
+
+
 int uv__iou_fs_open(uv_loop_t* loop, uv_fs_t* req) {
   struct uv__io_uring_sqe* sqe;
   struct uv__iou* iou;
@@ -780,6 +842,72 @@ int uv__iou_fs_open(uv_loop_t* loop, uv_fs_t* req) {
 }
 
 
+int uv__iou_fs_rename(uv_loop_t* loop, uv_fs_t* req) {
+  struct uv__io_uring_sqe* sqe;
+  struct uv__iou* iou;
+
+  iou = &uv__get_internal_fields(loop)->iou;
+
+  sqe = uv__iou_get_sqe(iou, loop, req);
+  if (sqe == NULL)
+    return 0;
+
+  sqe->addr = (uintptr_t) req->path;
+  sqe->fd = AT_FDCWD;
+  sqe->addr2 = (uintptr_t) req->new_path;
+  sqe->len = AT_FDCWD;
+  sqe->opcode = UV__IORING_OP_RENAMEAT;
+
+  uv__iou_submit(iou);
+
+  return 1;
+}
+
+
+int uv__iou_fs_symlink(uv_loop_t* loop, uv_fs_t* req) {
+  struct uv__io_uring_sqe* sqe;
+  struct uv__iou* iou;
+
+  iou = &uv__get_internal_fields(loop)->iou;
+
+  if (!(iou->flags & UV__MKDIRAT_SYMLINKAT_LINKAT))
+    return 0;
+
+  sqe = uv__iou_get_sqe(iou, loop, req);
+  if (sqe == NULL)
+    return 0;
+
+  sqe->addr = (uintptr_t) req->path;
+  sqe->fd = AT_FDCWD;
+  sqe->addr2 = (uintptr_t) req->new_path;
+  sqe->opcode = UV__IORING_OP_SYMLINKAT;
+
+  uv__iou_submit(iou);
+
+  return 1;
+}
+
+
+int uv__iou_fs_unlink(uv_loop_t* loop, uv_fs_t* req) {
+  struct uv__io_uring_sqe* sqe;
+  struct uv__iou* iou;
+
+  iou = &uv__get_internal_fields(loop)->iou;
+
+  sqe = uv__iou_get_sqe(iou, loop, req);
+  if (sqe == NULL)
+    return 0;
+
+  sqe->addr = (uintptr_t) req->path;
+  sqe->fd = AT_FDCWD;
+  sqe->opcode = UV__IORING_OP_UNLINKAT;
+
+  uv__iou_submit(iou);
+
+  return 1;
+}
+
+
 int uv__iou_fs_read_or_write(uv_loop_t* loop,
                              uv_fs_t* req,
                              int is_read) {

src/uv-common.h

@@ -415,6 +415,7 @@ struct uv__iou {
   size_t sqelen;
   int ringfd;
   uint32_t in_flight;
+  uint32_t flags;
 };
 #endif  /* __linux__ */
 

test/test-threadpool-cancel.c

@@ -98,11 +98,16 @@ static int known_broken(uv_req_t* req) {
     case UV_FS_FDATASYNC:
     case UV_FS_FSTAT:
     case UV_FS_FSYNC:
+    case UV_FS_LINK:
     case UV_FS_LSTAT:
+    case UV_FS_MKDIR:
     case UV_FS_OPEN:
     case UV_FS_READ:
+    case UV_FS_RENAME:
     case UV_FS_STAT:
+    case UV_FS_SYMLINK:
     case UV_FS_WRITE:
+    case UV_FS_UNLINK:
       return 1;
     default:  /* Squelch -Wswitch warnings. */
       break;