luo980/curl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

CI: improvements in test reliability and performance

Browse Source 
- CI default: raise parallelism to 20
- CI valgind: set parallelism to 6
- CI non-native: adapt parallelism for OS builds
- CI Windows: no longer ignore FTP, TFTP, MQTT and SMTP
- CI Windows: restrict test timeout to 10 minutes
- CI Windows: do not run tests for msh3 build
- tests, various: restrict curl invocation to ipv4 to avoid talking to
  any ipv6 test server running in parallel
- tests: requiring http/2 server no longer needs to mention http server
- test 190: use a fixed timeout of 10 seconds instead of %FTPTIME2, as
  that value rises under parallel load beyond what the server is waiting
  and then produces different results
- test 1540: add debug logging
- testrunner: add verify checks for http/2 and http/3 server

Closes #15040
This commit is contained in:
Stefan Eissing 2024-09-25 15:14:50 +02:00 committed by Daniel Stenberg
parent aca28abac7
commit 7f3d598276
No known key found for this signature in database
GPG Key ID: 5CC908FDB71E12C2
9 changed files with 26 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/cygwin.yml vendored
View File

@ -122,7 +122,7 @@ jobs:
if: ${{ matrix.build == 'automake' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
timeout-minutes: 15
run: |
export TFLAGS='-j8 ${{ matrix.tflags }} ~615'
export TFLAGS='-j20 ${{ matrix.tflags }} ~615'
if [ -x "$(cygpath "${SYSTEMROOT}/System32/curl.exe")" ]; then
TFLAGS+=" -ac $(cygpath "${SYSTEMROOT}/System32/curl.exe")"
fi

2
.github/workflows/linux.yml vendored
View File

@ -649,7 +649,7 @@ jobs:
run: |
export TFLAGS='${{ matrix.build.tflags }}'
if [[ '${{ matrix.build.install_packages }}' = *'valgrind'* ]]; then
TFLAGS+=' -j4'
TFLAGS+=' -j6'
fi
if [[ '${{ matrix.build.install_packages }}' = *'heimdal-dev'* ]]; then
TFLAGS+=' ~2077 ~2078' # valgrind errors

4
.github/workflows/macos.yml vendored
View File

@ -231,7 +231,7 @@ jobs:
- name: 'run tests'
timeout-minutes: 20
run: |
export TFLAGS='-j10 ${{ matrix.tflags }}'
export TFLAGS='-j20 ${{ matrix.tflags }}'
TFLAGS+=' ~2037 ~2041' # flaky
if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then
TFLAGS+=' ~RTSP' # 567 568 569 570 571 572 577 689 3100
@ -381,7 +381,7 @@ jobs:
- name: 'cmake run tests'
timeout-minutes: ${{ matrix.build.torture && 20 || 10 }}
run: |
export TFLAGS='-j10 ${{ matrix.build.tflags }}'
export TFLAGS='-j20 ${{ matrix.build.tflags }}'
if [ -z '${{ matrix.build.torture }}' ]; then
TFLAGS+=' ~2037 ~2041' # flaky
if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then

7
.github/workflows/non-native.yml vendored
View File

@ -66,7 +66,7 @@ jobs:
bld/src/curl --disable --version
if [ '${{ matrix.arch }}' = 'x86_64' ]; then # Slow on emulated CPU
cmake --build bld --config Debug --parallel 3 --target testdeps
export TFLAGS='-j0' # flakies: ~389 ~392 ~TFTP and more
export TFLAGS='-j4'
cmake --build bld --config Debug --target test-ci
fi
echo '::group::build examples'
@ -146,7 +146,7 @@ jobs:
src/curl --disable --version
if [ '${{ matrix.arch }}' = 'x86_64' ]; then # Slow on emulated CPU
make -j3 -C tests
make test-ci V=1 TFLAGS='-j8 ~FTP'
make test-ci V=1 TFLAGS='-j4'
fi
echo '::group::build examples'
make -j3 examples
@ -176,7 +176,6 @@ jobs:
bld/src/curl --disable --version
if [ '${{ matrix.arch }}' = 'x86_64' ]; then # Slow on emulated CPU
cmake --build bld --config Debug --parallel 3 --target testdeps
export TFLAGS='-j8 ~FTP'
cmake --build bld --config Debug --target test-ci
fi
echo '::group::build examples'
@ -206,7 +205,7 @@ jobs:
gmake -j3 install
src/curl --disable --version
gmake -j3 -C tests
gmake test-ci V=1 TFLAGS='-j12 ~MQTT ~FTP'
gmake test-ci V=1
echo '::group::build examples'
gmake -j3 examples
echo '::endgroup::'

22
.github/workflows/windows.yml vendored
View File

@ -149,11 +149,11 @@ jobs:
- name: 'autotools run tests'
if: ${{ matrix.build == 'autotools' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
timeout-minutes: 14
timeout-minutes: 10
run: |
export TFLAGS='-j14 ${{ matrix.tflags }} ~SCP'
export TFLAGS='-j8 ${{ matrix.tflags }} ~SCP'
if [ '${{ matrix.sys }}' != 'msys' ]; then
TFLAGS+=' ~TFTP ~MQTT ~WebSockets ~SMTP ~FTP'
TFLAGS+=' ~2301 ~2302' # WebSockets'
TFLAGS+=' ~612 ~613 ~616 ~618' # SFTP
else
TFLAGS+=' ~SFTP'
@ -243,11 +243,11 @@ jobs:
- name: 'cmake run tests'
if: ${{ matrix.build == 'cmake' && matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
timeout-minutes: 14
timeout-minutes: 10
run: |
export TFLAGS='-j14 ${{ matrix.tflags }} ~SCP'
export TFLAGS='-j8 ${{ matrix.tflags }} ~SCP'
if [ '${{ matrix.sys }}' != 'msys' ]; then
TFLAGS+=' ~TFTP ~MQTT ~WebSockets ~SMTP ~FTP'
TFLAGS+=' ~WebSockets'
TFLAGS+=' ~612 ~613 ~616 ~618' # SFTP
else
TFLAGS+=' ~SFTP'
@ -374,10 +374,10 @@ jobs:
- name: 'cmake run tests'
if: ${{ matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
timeout-minutes: 14
timeout-minutes: 10
run: |
PATH="$(cygpath "${USERPROFILE}")/my-cache/${{ matrix.dir }}/bin:/c/msys64/usr/bin:$PATH"
export TFLAGS='-j14 ~TFTP ~MQTT ~WebSockets ~FTP ${{ matrix.tflags }}'
export TFLAGS='-j4 ~WebSockets ${{ matrix.tflags }}'
if [ -x "$(cygpath "${SYSTEMROOT}/System32/curl.exe")" ]; then
TFLAGS+=" -ac $(cygpath "${SYSTEMROOT}/System32/curl.exe")"
fi
@ -531,7 +531,7 @@ jobs:
arch: 'x64'
plat: 'windows'
type: 'Debug'
tflags: '~1516'
tflags: 'skipall'
config: '-DENABLE_DEBUG=ON -DENABLE_UNICODE=OFF -DCURL_USE_SCHANNEL=OFF -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_LIBSSH2=ON -DUSE_MSH3=ON -DCURL_USE_GSASL=ON'
fail-fast: false
steps:
@ -617,9 +617,9 @@ jobs:
- name: 'cmake run tests'
if: ${{ matrix.tflags != 'skipall' && matrix.tflags != 'skiprun' }}
timeout-minutes: 16
timeout-minutes: 10
run: |
export TFLAGS='-j14 ~TFTP ~MQTT ~WebSockets ~SMTP ~FTP ~SCP ~612 ${{ matrix.tflags }}'
export TFLAGS='-j8 ~WebSockets ~SCP ~612 ${{ matrix.tflags }}'
if [[ '${{ matrix.install }}' = *'libssh2[core,zlib]'* ]]; then
TFLAGS+=' ~SFTP'
elif [[ '${{ matrix.install }}' = *'libssh '* ]]; then

2
tests/CMakeLists.txt
View File

@ -105,7 +105,7 @@ add_runtests(test-am "-a -am")
add_runtests(test-full "-a -p -r")
# ~flaky means that it ignores results of tests using the flaky keyword
add_runtests(test-nonflaky "-a -p ~flaky ~timing-dependent")
add_runtests(test-ci "-a -p ~flaky ~timing-dependent -r -rm -j2")
add_runtests(test-ci "-a -p ~flaky ~timing-dependent -r -rm -j20")
add_runtests(test-torture "-a -t -j2")
add_runtests(test-event "-a -e")

2
tests/Makefile.am
View File

@ -140,7 +140,7 @@ TEST_E = -a -e
TEST_NF = -a -p ~flaky ~timing-dependent
# special CI target derived from nonflaky with CI-specific flags
TEST_CI = $(TEST_NF) -r -rm -j2
TEST_CI = $(TEST_NF) -r -rm -j20
PYTEST = pytest
endif

4
tests/server/util.c
View File

@ -115,7 +115,9 @@ void logmsg(const char *msg, ...)
mvsnprintf(buffer, sizeof(buffer), msg, ap);
va_end(ap);
logfp = fopen(serverlogfile, "ab");
do {
logfp = fopen(serverlogfile, "ab");
} while(!logfp && (errno == EINTR));
if(logfp) {
fprintf(logfp, "%s %s\n", timebuf, buffer);
fclose(logfp);

3
tests/serverhelp.pm
View File

@ -86,6 +86,9 @@ sub logmsg {
localtime($seconds);
$now = sprintf("%02d:%02d:%02d ", $hour, $min, $sec);
}
# we see warnings on Windows run that $logfile is used uninitialized
# TODO: not found yet where this comes from
$logfile = "serverhelp_uninitialized.log" if(!$logfile);
if(open(my $logfilefh, ">>", "$logfile")) {
print $logfilefh $now;
print $logfilefh @_;