tool_main: exit at once if out of file descriptors

If the main_checkfds function cannot create new file descriptors in an
attempt to detect of stdin, stdout or stderr are closed.

Also changed the check to use fcntl() to check if the descriptors are
open, which avoids superfluously calling pipe() if they all already are.

Follow-up to facfa19cdd

Reported-by: Trail of Bits

Closes #9663

src/tool_main.c

@@ -33,6 +33,10 @@
 #include <signal.h>
 #endif
 
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
 #ifdef USE_NSS
 #include <nspr.h>
 #include <plarenas.h>
@@ -80,29 +84,26 @@ int _CRT_glob = 0;
 /* if we build a static library for unit tests, there is no main() function */
 #ifndef UNITTESTS
 
+#if defined(HAVE_PIPE) && defined(HAVE_FCNTL)
 /*
  * Ensure that file descriptors 0, 1 and 2 (stdin, stdout, stderr) are
  * open before starting to run.  Otherwise, the first three network
  * sockets opened by curl could be used for input sources, downloaded data
  * or error logs as they will effectively be stdin, stdout and/or stderr.
  */
-static void main_checkfds(void)
+static int main_checkfds(void)
 {
-#ifdef HAVE_PIPE
-  int fd[2] = { STDIN_FILENO, STDIN_FILENO };
-  while(fd[0] == STDIN_FILENO ||
-        fd[0] == STDOUT_FILENO ||
-        fd[0] == STDERR_FILENO ||
-        fd[1] == STDIN_FILENO ||
-        fd[1] == STDOUT_FILENO ||
-        fd[1] == STDERR_FILENO)
-    if(pipe(fd) < 0)
-      return;   /* Out of handles. This isn't really a big problem now, but
-                   will be when we try to create a socket later. */
-  close(fd[0]);
-  close(fd[1]);
-#endif
+  int fd[2];
+  while(fcntl(STDIN_FILENO, F_GETFD) ||
+        fcntl(STDOUT_FILENO, F_GETFD) ||
+        fcntl(STDERR_FILENO, F_GETFD))
+    if(pipe(fd))
+      return 1;
+  return 0;
 }
+#else
+#define main_checkfds() 0
+#endif
 
 #ifdef CURLDEBUG
 static void memory_tracking_init(void)
@@ -259,7 +260,10 @@ int main(int argc, char *argv[])
   }
 #endif
 
-  main_checkfds();
+  if(main_checkfds()) {
+    fprintf(stderr, "curl: out of file descriptors\n");
+    return CURLE_FAILED_INIT;
+  }
 
 #if defined(HAVE_SIGNAL) && defined(SIGPIPE)
   (void)signal(SIGPIPE, SIG_IGN);