build: detect and use _setmode() with Cygwin/MSYS, also use on Windows

Before this patch `setmode()` was not detected with Cygwin/MSYS, because
it's a macro, not a function, and detection is looking for a function.

Switching to symbol detection doesn't work because it mis-detects it on
BSD systems which features a function with the same name but different
functionality and arguments.

Fix it by looking for a `_setmode()` function on Cygwin/MSYS, and use it
if available.

`_setmode()` is recommended over `setmode()` by Windows documentation so
use that on Windows too. It seems to be available on all supported
compilers, so omit detection.

https://learn.microsoft.com/cpp/c-runtime-library/reference/posix-setmode
https://learn.microsoft.com/cpp/c-runtime-library/reference/setmode

Officially Windows requires argument `_O_BINARY` with an underscore.
`O_BINARY` is also supported but bound to conditions. Continue to use it
for simplicity. Cygwin supports `O_BINARY` (no underscore).

Closes #15169

CMake/Platforms/WindowsCache.cmake

@@ -105,6 +105,7 @@ set(HAVE_FSETXATTR 0)
 set(HAVE_LIBSOCKET 0)
 set(HAVE_SETLOCALE 1)
 set(HAVE_SETMODE 1)
+set(HAVE__SETMODE 1)
 set(HAVE_GETPEERNAME 1)
 set(HAVE_GETSOCKNAME 1)
 set(HAVE_GETHOSTNAME 1)

CMakeLists.txt

@@ -1524,6 +1524,10 @@ check_symbol_exists("setlocale"       "${CURL_INCLUDES}" HAVE_SETLOCALE)
 check_symbol_exists("setmode"         "${CURL_INCLUDES}" HAVE_SETMODE)
 check_symbol_exists("setrlimit"       "${CURL_INCLUDES}" HAVE_SETRLIMIT)
 
+if(WIN32 OR CYGWIN)
+  check_function_exists("_setmode" HAVE__SETMODE)
+endif()
+
 if(NOT _ssl_enabled)
   check_symbol_exists("arc4random" "${CURL_INCLUDES};stdlib.h" HAVE_ARC4RANDOM)
 endif()

acinclude.m4

@@ -1553,9 +1553,9 @@ AC_DEFUN([CURL_PREPARE_BUILDINFO], [
         curl_pflags="${curl_pflags} UNIX";;
     esac
   fi
-  case $host_os in
-    cygwin*|msys*) curl_pflags="${curl_pflags} CYGWIN";;
-  esac
+  if test "$curl_cv_cygwin" = 'yes'; then
+    curl_pflags="${curl_pflags} CYGWIN"
+  fi
   case $host_os in
     msys*) curl_pflags="${curl_pflags} MSYS";;
   esac

configure.ac

@@ -605,6 +605,11 @@ case $host in
     ;;
 esac
 
+curl_cv_cygwin='no'
+case $host_os in
+  cygwin*|msys*) curl_cv_cygwin='yes';;
+esac
+
 AM_CONDITIONAL([HAVE_WINDRES],
   [test "$curl_cv_native_windows" = "yes" && test -n "${RC}"])
 
@@ -4120,6 +4125,10 @@ AC_CHECK_FUNCS([\
   utimes \
 ])
 
+if test "$curl_cv_native_windows" = 'yes' -o "$curl_cv_cygwin" = 'yes'; then
+  AC_CHECK_FUNCS([_setmode])
+fi
+
 if test -z "$ssl_backends"; then
   AC_CHECK_FUNCS([arc4random])
 fi

lib/config-win32.h

@@ -155,6 +155,9 @@
 /* Define if you have the setmode function. */
 #define HAVE_SETMODE 1
 
+/* Define if you have the _setmode function. */
+#define HAVE__SETMODE 1
+
 /* Define if you have the socket function. */
 #define HAVE_SOCKET 1
 

lib/curl_config.h.cmake

@@ -481,6 +481,9 @@
 /* Define to 1 if you have the `setmode' function. */
 #cmakedefine HAVE_SETMODE 1
 
+/* Define to 1 if you have the `_setmode' function. */
+#cmakedefine HAVE__SETMODE 1
+
 /* Define to 1 if you have the `setrlimit' function. */
 #cmakedefine HAVE_SETRLIMIT 1
 

src/tool_binmode.c

@@ -23,7 +23,7 @@
  ***************************************************************************/
 #include "tool_setup.h"
 
-#ifdef HAVE_SETMODE
+#if defined(HAVE_SETMODE) || defined(HAVE__SETMODE)
 
 #ifdef HAVE_IO_H
 #  include <io.h>
@@ -42,6 +42,8 @@ void set_binmode(FILE *stream)
 #ifdef O_BINARY
 #  ifdef __HIGHC__
   _setmode(stream, O_BINARY);
+#  elif defined(HAVE__SETMODE)
+  (void)_setmode(fileno(stream), O_BINARY);
 #  else
   (void)setmode(fileno(stream), O_BINARY);
 #  endif
@@ -50,4 +52,4 @@ void set_binmode(FILE *stream)
 #endif
 }
 
-#endif /* HAVE_SETMODE */
+#endif /* HAVE_SETMODE || HAVE__SETMODE */

src/tool_binmode.h

@@ -25,7 +25,7 @@
  ***************************************************************************/
 #include "tool_setup.h"
 
-#ifdef HAVE_SETMODE
+#if defined(HAVE_SETMODE) || defined(HAVE__SETMODE)
 
 void set_binmode(FILE *stream);
 
@@ -33,6 +33,6 @@ void set_binmode(FILE *stream);
 
 #define set_binmode(x) Curl_nop_stmt
 
-#endif /* HAVE_SETMODE */
+#endif /* HAVE_SETMODE || HAVE__SETMODE */
 
 #endif /* HEADER_CURL_TOOL_BINMODE_H */