runtests: support multi-target cmake, drop workarounds from CI

Support multi-target cmake builds via `CURL_DIRSUFFIX` env. For example:
`export CURL_DIRSUFFIX=Debug/`.

Multi-target generators place their output to `src/<subdir>/`,
`lib/<subdir>/`, `tests/server/<subdir>`, `tests/libtest/<subdir>` and
`tests/unit/<subdir>/` by default. Before this patch, `runtests.pl`
couldn't run on such builds because it expected the binaries under the
their `<subdir>`-less directories. This patch allows to set such subdir
and make `runtests.pl` find the binaries. In CI we use multi-target
builds with tests for MSVC. It also helps Xcode-generator builds, though
in CI we don't have such job running tests.

There may be better solutions to configure this, but passing a custom
value to `runtests.pl` including its subprocesses is somewhat tricky.
The reason the configuration value expects the slash at the end is
because MSYS is automagically expanding the env to a (wrong) absolute
path if the slash is in the front.

Also:
- drop the `-DCMAKE_RUNTIME_OUTPUT_DIRECTORY_*=` workaround from CI.
- replace `resolve` references in tests with a new `%RESOLVE` variable.
  It didn't use a filename extension before. After this patch it uses
  `exe_ext('TOOL')`. I'm not sure if this is the correct choice vs.
  `exe_ext('SRV')`.
- fix `-c` option format in manual.
- fix some whitespace.

Note, in CI we still tweak `CMAKE_RUNTIME_OUTPUT_DIRECTORY_*` in jobs
which share steps between `./configure` and cmake. It's easier that way.

Ref: #15000
Cherry-picked from #16394

Closes #16452

.github/workflows/windows.yml

@@ -827,8 +827,6 @@ jobs:
               -DCMAKE_EXE_LINKER_FLAGS="-INCREMENTAL:NO ${ldflags}" \
               -DCMAKE_SHARED_LINKER_FLAGS="-INCREMENTAL:NO ${ldflags}" \
               -DCMAKE_VS_GLOBALS="TrackFileAccess=false${vsglobals}" \
-              -DCMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG= \
-              -DCMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE= \
               -DCMAKE_UNITY_BUILD=ON -DCURL_TEST_BUNDLES=ON \
               -DCURL_WERROR=ON \
               -DBUILD_SHARED_LIBS=OFF \
@@ -860,8 +858,8 @@ jobs:
         run: |
           PATH=/usr/bin find . \( -name '*.exe' -o -name '*.dll' -o -name '*.lib' \) -exec file '{}' \;
           if [ '${{ matrix.plat }}' != 'uwp' ]; then  # Missing: ucrtbased.dll, VCRUNTIME140D.dll, VCRUNTIME140D_APP.dll
-            PATH="$PWD/bld/lib:$PATH"
+            PATH="$PWD/bld/lib/${{ matrix.type }}:$PATH"
-            bld/src/curl.exe --disable --version
+            'bld/src/${{ matrix.type }}/curl.exe' --disable --version
           fi
 
       - name: 'build tests'
@@ -896,13 +894,14 @@ jobs:
         if: ${{ matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
         timeout-minutes: 10
         run: |
+          export CURL_DIRSUFFIX='${{ matrix.type }}/'
           export TFLAGS='-j8 ~WebSockets ~SCP ~612 ${{ matrix.tflags }}'
           if [[ '${{ matrix.install }}' = *'libssh2[core,zlib]'* ]]; then
             TFLAGS+=' ~SFTP'
           elif [[ '${{ matrix.install }}' = *'libssh '* ]]; then
             TFLAGS+=' ~614'  # 'SFTP pre-quote chmod' SFTP, pre-quote, directory
           fi
-          PATH="$PWD/bld/lib:$PATH:/c/Program Files (x86)/stunnel/bin:/c/Program Files/OpenSSH-Win64"
+          PATH="$PWD/bld/lib/${{ matrix.type }}:$PATH:/c/Program Files (x86)/stunnel/bin:/c/Program Files/OpenSSH-Win64"
           PATH="/c/msys64/usr/bin:$PATH"
           cmake --build bld --config '${{ matrix.type }}' --target test-ci
 

appveyor.sh

@@ -55,8 +55,6 @@ if [ "${BUILD_SYSTEM}" = 'CMake' ]; then
     # shellcheck disable=SC2086
     cmake -B "_bld${_chkprefill}" -G "${PRJ_GEN}" ${TARGET} \
       -DCMAKE_VS_GLOBALS=TrackFileAccess=false \
-      -DCMAKE_RUNTIME_OUTPUT_DIRECTORY_DEBUG= \
-      -DCMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE= \
       -DCMAKE_UNITY_BUILD="${UNITY}" -DCURL_TEST_BUNDLES=ON \
       -DCURL_WERROR=ON \
       -DBUILD_SHARED_LIBS="${SHARED}" \
@@ -78,9 +76,9 @@ if [ "${BUILD_SYSTEM}" = 'CMake' ]; then
   echo 'curl_config.h'; grep -F '#define' _bld/lib/curl_config.h | sort || true
   # shellcheck disable=SC2086
   cmake --build _bld --config "${PRJ_CFG}" --parallel 2 -- ${BUILD_OPT:-}
-  [ "${SHARED}" = 'ON' ] && PATH="$PWD/_bld/lib:$PATH"
+  [ "${SHARED}" = 'ON' ] && PATH="$PWD/_bld/lib/${PRJ_CFG}:$PATH"
   [ "${OPENSSL}" = 'ON' ] && PATH="${openssl_root}:$PATH"
-  curl='_bld/src/curl.exe'
+  curl="_bld/src/${PRJ_CFG}/curl.exe"
 elif [ "${BUILD_SYSTEM}" = 'VisualStudioSolution' ]; then
   (
     cd projects
@@ -131,6 +129,7 @@ fi
 
 if [ "${TFLAGS}" != 'skipall' ] && \
    [ "${TFLAGS}" != 'skiprun' ]; then
+  export CURL_DIRSUFFIX="${PRJ_CFG}/"
   if [ -x "$(cygpath "${SYSTEMROOT}/System32/curl.exe")" ]; then
     TFLAGS+=" -ac $(cygpath "${SYSTEMROOT}/System32/curl.exe")"
   elif [ -x "$(cygpath 'C:/msys64/usr/bin/curl.exe')" ]; then

tests/FILEFORMAT.md

@@ -163,6 +163,7 @@ Available substitute variables include:
 - `%POSIX_PWD` - Current directory somewhat MinGW friendly
 - `%PROXYPORT` - Port number of the HTTP proxy
 - `%PWD` - Current directory
+- `%RESOLVE` - server/resolve command
 - `%RTSP6PORT` - IPv6 port number of the RTSP server
 - `%RTSPPORT` - Port number of the RTSP server
 - `%SMBPORT` - Port number of the SMB server

tests/data/test1083

@@ -39,7 +39,7 @@ HTTP-IPv6 GET with ip6-localhost --interface
 -g "http://%HOST6IP:%HTTP6PORT/%TESTNUMBER" --interface ip6-localhost
 </command>
 <precheck>
-%PERL -e "if ('%CLIENT6IP' ne '[::1]') {print 'Test requires default test client host address';} else {exec './server/resolve --ipv6 ip6-localhost'; print 'Cannot run precheck resolve';}"
+%PERL -e "if ('%CLIENT6IP' ne '[::1]') {print 'Test requires default test client host address';} else {exec '%RESOLVE --ipv6 ip6-localhost'; print 'Cannot run precheck resolve';}"
 </precheck>
 </client>
 

tests/data/test1085

@@ -34,7 +34,7 @@ HTTP-IPv6 GET with invalid --interface
 # Ensure the IPv6 stack is operational before running this test (other tests
 # use the startup of the IPv6 test server as a substitute check for this).
 <precheck>
-./server/resolve --ipv6 ::1
+%RESOLVE --ipv6 ::1
 </precheck>
 </client>
 

tests/data/test241

@@ -36,7 +36,7 @@ HTTP-IPv6 GET (using ip6-localhost)
 -g "http://ip6-localhost:%HTTP6PORT/%TESTNUMBER"
 </command>
 <precheck>
-./server/resolve --ipv6 ip6-localhost
+%RESOLVE --ipv6 ip6-localhost
 </precheck>
 </client>
 

tests/ftpserver.pl

@@ -68,16 +68,13 @@ use serverhelp qw(
     servername_str
     server_pidfilename
     server_logfilename
+    server_exe
     mainsockf_pidfilename
     mainsockf_logfilename
     datasockf_pidfilename
     datasockf_logfilename
     );
 
-use pathhelp qw(
-    exe_ext
-    );
-
 use globalconfig qw(
     $SERVERCMD
     $LOCKDIR
@@ -410,7 +407,7 @@ sub sysread_or_die {
 }
 
 sub startsf {
-    my @mainsockfcmd = ("./server/sockfilt".exe_ext('SRV'),
+    my @mainsockfcmd = (server_exe('sockfilt'),
         "--ipv$ipvnum",
         "--port", $port,
         "--pidfile", $mainsockf_pidfile,
@@ -2472,10 +2469,10 @@ sub PASV_ftp {
     logmsg "DATA sockfilt for passive data channel starting...\n";
 
     # We fire up a new sockfilt to do the data transfer for us.
-    my @datasockfcmd = ("./server/sockfilt".exe_ext('SRV'),
+    my @datasockfcmd = (server_exe('sockfilt'),
         "--ipv$ipvnum", "--port", 0,
         "--pidfile", $datasockf_pidfile,
-        "--logfile",  $datasockf_logfile);
+        "--logfile", $datasockf_logfile);
     if($nodataconn) {
         push(@datasockfcmd, '--bindonly');
     }
@@ -2694,7 +2691,7 @@ sub PORT_ftp {
     logmsg "DATA sockfilt for active data channel starting...\n";
 
     # We fire up a new sockfilt to do the data transfer for us.
-    my @datasockfcmd = ("./server/sockfilt".exe_ext('SRV'),
+    my @datasockfcmd = (server_exe('sockfilt'),
         "--ipv$ipvnum", "--connect", $port, "--addr", $addr,
         "--pidfile", $datasockf_pidfile,
         "--logfile", $datasockf_logfile);

tests/globalconfig.pm

@@ -42,6 +42,8 @@ BEGIN {
         $DATE
         $has_shared
         $LIBDIR
+        $UNITDIR
+        $SRVDIR
         $listonly
         $LOCKDIR
         $LOGDIR
@@ -102,9 +104,11 @@ our $perlcmd=shell_quote($^X);
 our $perl="$perlcmd -I. " . shell_quote("-I$srcdir"); # invoke perl like this
 our $LOGDIR="log";  # root of the log directory; this will be different for
                     # each runner in multiprocess mode
-our $LIBDIR="./libtest";
+our $LIBDIR="./libtest/" . ($ENV{'CURL_DIRSUFFIX'} || '');
+our $UNITDIR="./unit/" . ($ENV{'CURL_DIRSUFFIX'} || '');
+our $SRVDIR="./server/" . ($ENV{'CURL_DIRSUFFIX'} || '');
 our $TESTDIR="$srcdir/data";
-our $CURL="../src/curl".exe_ext('TOOL'); # what curl binary to run on the tests
+our $CURL="../src/" . ($ENV{'CURL_DIRSUFFIX'} || '') . "curl".exe_ext('TOOL'); # what curl binary to run on the tests
 our $VCURL=$CURL;  # what curl binary to use to verify the servers with
                    # VCURL is handy to set to the system one when the one you
                    # just built hangs or crashes and thus prevent verification

tests/http-server.pl

@@ -36,10 +36,7 @@ use File::Basename;
 use serverhelp qw(
     server_pidfilename
     server_logfilename
-    );
+    server_exe
-
-use pathhelp qw(
-    exe_ext
     );
 
 my $verbose = 0;     # set to 1 for debugging
@@ -187,8 +184,8 @@ if($ipvnum eq 'unix') {
 $flags .= "--srcdir \"$srcdir\"";
 
 if($verbose) {
-    print STDERR "RUN: server/sws".exe_ext('SRV')." $flags\n";
+    print STDERR "RUN: ".server_exe('sws')." $flags\n";
 }
 
 $| = 1;
-exec("exec server/sws".exe_ext('SRV')." $flags");
+exec("exec ".server_exe('sws')." $flags");

tests/rtspserver.pl

@@ -34,10 +34,7 @@ BEGIN {
 use serverhelp qw(
     server_pidfilename
     server_logfilename
-    );
+    server_exe
-
-use pathhelp qw(
-    exe_ext
     );
 
 my $verbose = 0;     # set to 1 for debugging
@@ -133,4 +130,4 @@ $flags .= "--pidfile \"$pidfile\" ".
 $flags .= "--ipv$ipvnum --port $port --srcdir \"$srcdir\"";
 
 $| = 1;
-exec("exec server/rtspd".exe_ext('SRV')." $flags");
+exec("exec ".server_exe('rtspd')." $flags");

tests/runner.pm

@@ -126,7 +126,6 @@ our $tortalloc;
 
 # local variables
 my %oldenv;       # environment variables before test is started
-my $UNITDIR="./unit";
 my $CURLLOG="$LOGDIR/commands.log"; # all command lines run
 my $defserverlogslocktimeout = 5; # timeout to await server logs lock removal
 my $defpostcommanddelay = 0; # delay between command and postcheck sections
@@ -263,7 +262,7 @@ sub event_loop {
 #
 sub checktestcmd {
     my ($cmd)=@_;
-    my @testpaths=("$LIBDIR/.libs", "$LIBDIR");
+    my @testpaths=($LIBDIR . ".libs", "$LIBDIR");
     return checkcmd($cmd, @testpaths);
 }
 
@@ -922,18 +921,18 @@ sub singletest_run {
 
         if($tool =~ /^lib/) {
             if($bundle) {
-                $CMDLINE="$LIBDIR/libtests";
+                $CMDLINE=$LIBDIR . "libtests";
             }
             else {
-                $CMDLINE="$LIBDIR/$tool";
+                $CMDLINE=$LIBDIR . $tool;
             }
         }
         elsif($tool =~ /^unit/) {
             if($bundle) {
-                $CMDLINE="$UNITDIR/units";
+                $CMDLINE=$UNITDIR . "units";
             }
             else {
-                $CMDLINE="$UNITDIR/$tool";
+                $CMDLINE=$UNITDIR . $tool;
             }
         }
 
@@ -1035,12 +1034,12 @@ sub singletest_run {
     if ($torture) {
         $cmdres = torture($CMDLINE,
                           $testnum,
-                          "$gdb --directory $LIBDIR " . shell_quote($DBGCURL) . " -x $LOGDIR/gdbcmd");
+                          "$gdb --directory $LIBDIR " . shell_quote($DBGCURL) . " -x $LOGDIR" . "gdbcmd");
     }
     elsif($gdbthis == 1) {
         # gdb
         my $GDBW = ($gdbxwin) ? "-w" : "";
-        runclient("$gdb --directory $LIBDIR " . shell_quote($DBGCURL) . " $GDBW -x $LOGDIR/gdbcmd");
+        runclient("$gdb --directory $LIBDIR " . shell_quote($DBGCURL) . " $GDBW -x $LOGDIR" . "gdbcmd");
         $cmdres=0; # makes it always continue after a debugged run
     }
     elsif($gdbthis == 2) {

tests/runtests.md

@@ -96,7 +96,7 @@ Display test results in automake style output (`PASS/FAIL: [number] [name]`).
 Run tests via bundled test binaries. Bundled test binaries contain all tests,
 and the test name passed as the first argument selects which test run.
 
-## `-c\<curl\>`
+## `-c \<curl\>`
 
 Provide a path to a custom curl binary to run the tests with. Default is the
 curl executable in the build tree.

tests/runtests.pl

@@ -84,6 +84,9 @@ use Digest::MD5 qw(md5);
 use List::Util 'sum';
 use I18N::Langinfo qw(langinfo CODESET);
 
+use serverhelp qw(
+    server_exe
+    );
 use pathhelp qw(
     exe_ext
     sys_native_current_path
@@ -510,7 +513,7 @@ sub checksystemfeatures {
     @version = <$versout>;
     close($versout);
 
-    open(my $disabledh, "-|", "server/disabled".exe_ext('TOOL'));
+    open(my $disabledh, "-|", server_exe('disabled', 'TOOL'));
     @disabled = <$disabledh>;
     close($disabledh);
 
@@ -766,7 +769,7 @@ sub checksystemfeatures {
         # client has IPv6 support
 
         # check if the HTTP server has it!
-        my $cmd = "server/sws".exe_ext('SRV')." --version";
+        my $cmd = server_exe('sws')." --version";
         my @sws = `$cmd`;
         if($sws[0] =~ /IPv6/) {
             # HTTP server has IPv6 support!
@@ -774,7 +777,7 @@ sub checksystemfeatures {
         }
 
         # check if the FTP server has it!
-        $cmd = "server/sockfilt".exe_ext('SRV')." --version";
+        $cmd = server_exe('sockfilt')." --version";
         @sws = `$cmd`;
         if($sws[0] =~ /IPv6/) {
             # FTP server has IPv6 support!
@@ -784,7 +787,7 @@ sub checksystemfeatures {
 
     if($feature{"UnixSockets"}) {
         # client has Unix sockets support, check whether the HTTP server has it
-        my $cmd = "server/sws".exe_ext('SRV')." --version";
+        my $cmd = server_exe('sws')." --version";
         my @sws = `$cmd`;
         $http_unix = 1 if($sws[0] =~ /unix/);
     }

tests/serverhelp.pm

@@ -45,6 +45,7 @@ BEGIN {
         server_cmdfilename
         server_inputfilename
         server_outputfilename
+        server_exe
         mainsockf_pidfilename
         mainsockf_logfilename
         datasockf_pidfilename
@@ -59,6 +60,10 @@ BEGIN {
     }
 }
 
+use globalconfig;
+use pathhelp qw(
+    exe_ext
+    );
 
 our $logfile;  # server log file name, for logmsg
 
@@ -230,6 +235,18 @@ sub server_outputfilename {
 }
 
 
+#***************************************************************************
+# Return filename for a server executable
+#
+sub server_exe {
+    my ($name, $ext) = @_;
+    if(!defined $ext) {
+        $ext = 'SRV';
+    }
+    return $SRVDIR . $name . exe_ext($ext);
+}
+
+
 #***************************************************************************
 # Return file name for main or primary sockfilter pid file.
 #

tests/servers.pm

@@ -77,6 +77,7 @@ use serverhelp qw(
     server_pidfilename
     server_portfilename
     server_logfilename
+    server_exe
     );
 
 use sshhelp qw(
@@ -1959,8 +1960,8 @@ sub runmqttserver {
     my $logfile = server_logfilename($LOGDIR, $proto, $ipvnum, $idnum);
 
     # start our MQTT server - on a random port!
-    my $cmd="server/mqttd".exe_ext('SRV').
+    my $cmd=server_exe('mqttd').
-        " --port 0 ".
+        " --port 0".
         " --pidfile $pidfile".
         " --portfile $portfile".
         " --config $LOGDIR/$SERVERCMD".
@@ -2017,7 +2018,7 @@ sub runsocksserver {
     # start our socks server, get commands from the FTP cmd file
     my $cmd="";
     if($is_unix) {
-        $cmd="server/socksd".exe_ext('SRV').
+        $cmd=server_exe('socksd').
             " --pidfile $pidfile".
             " --reqfile $LOGDIR/$SOCKSIN".
             " --logfile $logfile".
@@ -2025,8 +2026,8 @@ sub runsocksserver {
             " --backend $HOSTIP".
             " --config $LOGDIR/$SERVERCMD";
     } else {
-        $cmd="server/socksd".exe_ext('SRV').
+        $cmd=server_exe('socksd').
-            " --port 0 ".
+            " --port 0".
             " --pidfile $pidfile".
             " --portfile $portfile".
             " --reqfile $LOGDIR/$SOCKSIN".
@@ -3114,6 +3115,8 @@ sub subvariables {
     $$thing =~ s/${prefix}VERNUM/$CURLVERNUM/g;
     $$thing =~ s/${prefix}DATE/$DATE/g;
     $$thing =~ s/${prefix}TESTNUMBER/$testnum/g;
+    my $resolve = server_exe('resolve', 'TOOL');
+    $$thing =~ s/${prefix}RESOLVE/$resolve/g;
 
     # POSIX/MSYS/Cygwin curl needs: file://localhost/d/path/to
     # Windows native    curl needs: file://localhost/D:/path/to

tests/tftpserver.pl

@@ -34,10 +34,7 @@ BEGIN {
 use serverhelp qw(
     server_pidfilename
     server_logfilename
-    );
+    server_exe
-
-use pathhelp qw(
-    exe_ext
     );
 
 my $verbose = 0;     # set to 1 for debugging
@@ -130,4 +127,4 @@ $flags .= "--pidfile \"$pidfile\" ".
 $flags .= "--ipv$ipvnum --port $port --srcdir \"$srcdir\"";
 
 $| = 1;
-exec("exec server/tftpd".exe_ext('SRV')." $flags");
+exec("exec ".server_exe('tftpd')." $flags");