luo980/curl
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

GHA: speed up builds in torture jobs, tidy up

Browse Source 
- use cmake with ninja.
- drop unnecessary packages.

Also:
- Linux: switch to Ubuntu 20.24 runner. Initially for newer packages,
  then left it there for variation and as canary.
- Linux: make it easier to enable valgrind by dropping `-n` tflags.
- Linux: show `curl -V` after build.
- Linux: sync job names with macOS.
- Linux, macOS: review and adjust job timeouts.
- Linux, macOS: sync job configs more:
  Linux: enable libssh2, macOS: enable brotli, zstd, libssh2.
- macOS: prefer `source` (over manual `PATH` editing) for `venv` setup.
- macOS: drop redundant `-DENABLE_CURLDEBUG=ON`.
- macOS: drop redundant `-n` tflags. There is no valgrind on macOS.
- macOS: allow overriding test parallelism per job.
- macOS: unlock all disabled tests for torture jobs.

Speed-ups (configure/build stage):
- Linux !FTP: 83s -> 36s
  before: https://github.com/curl/curl/actions/runs/10948030198/job/30398134647
  after: https://github.com/curl/curl/actions/runs/10961987675/job/30440179978?pr=14972
- Linux FTP: 79s -> 32s
  before: https://github.com/curl/curl/actions/runs/10948030198/job/30398137070
  after: https://github.com/curl/curl/actions/runs/10961987675/job/30440182406?pr=14972
- macOS !FTP: 98s -> 29s
  before: https://github.com/curl/curl/actions/runs/10960141275/job/30434081412?pr=14972
  after: https://github.com/curl/curl/actions/runs/10960141275/job/30434082357?pr=14972
- macOS FTP: 129s -> 29s
  before: https://github.com/curl/curl/actions/runs/10960141275/job/30434082041?pr=14972
  after: https://github.com/curl/curl/actions/runs/10960141275/job/30434083050?pr=14972

Closes #14972
This commit is contained in:
Viktor Szakats 2024-09-19 22:38:04 +02:00
parent 60c3d04465
commit c5e3d8ba94
No known key found for this signature in database
GPG Key ID: B5ABD165E2AEF201
2 changed files with 66 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
.github/workflows/macos.yml vendored
View File

@ -135,16 +135,6 @@ jobs:
configure: --enable-debug --with-openssl=$(brew --prefix openssl) --enable-websockets configure: --enable-debug --with-openssl=$(brew --prefix openssl) --enable-websockets
macos-version-min: '10.9' macos-version-min: '10.9'
tflags: -e tflags: -e
- name: 'OpenSSL torture !FTP'
compiler: clang
configure: --enable-debug --disable-shared --disable-threaded-resolver --with-openssl=$(brew --prefix openssl) --enable-websockets
tflags: -n -t --shallow=25 !FTP
macos-version-min: '10.9'
- name: 'OpenSSL torture FTP'
compiler: clang
configure: --enable-debug --disable-shared --disable-threaded-resolver --with-openssl=$(brew --prefix openssl) --enable-websockets
tflags: -n -t --shallow=20 FTP
macos-version-min: '10.9'
- name: 'OpenSSL libssh2 !ldap 10.15' - name: 'OpenSSL libssh2 !ldap 10.15'
compiler: clang compiler: clang
configure: --enable-debug --disable-ldap --with-openssl=$(brew --prefix openssl) --enable-websockets configure: --enable-debug --disable-ldap --with-openssl=$(brew --prefix openssl) --enable-websockets
@ -241,7 +231,7 @@ jobs:
- name: 'run tests' - name: 'run tests'
timeout-minutes: 20 timeout-minutes: 20
run: | run: |
export TFLAGS='${{ matrix.tflags }} -j10' export TFLAGS='-j10 ${{ matrix.tflags }}'
TFLAGS+=' ~2037 ~2041' # flaky TFLAGS+=' ~2037 ~2041' # flaky
if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then
TFLAGS+=' ~RTSP' # 567 568 569 570 571 572 577 689 3100 TFLAGS+=' ~RTSP' # 567 568 569 570 571 572 577 689 3100
@ -257,7 +247,7 @@ jobs:
TFLAGS+=' ~313' # SecureTransport does not support crl file TFLAGS+=' ~313' # SecureTransport does not support crl file
TFLAGS+=' ~1631 ~1632' # SecureTransport is not able to shutdown ftp over https gracefully yet TFLAGS+=' ~1631 ~1632' # SecureTransport is not able to shutdown ftp over https gracefully yet
fi fi
PATH="$HOME/venv/bin:$PATH" source $HOME/venv/bin/activate
rm -f $HOME/.curlrc rm -f $HOME/.curlrc
make -C bld V=1 test-ci make -C bld V=1 test-ci
@ -268,7 +258,7 @@ jobs:
cmake: cmake:
name: 'CM ${{ matrix.compiler }} ${{ matrix.build.name }}' name: 'CM ${{ matrix.compiler }} ${{ matrix.build.name }}'
runs-on: 'macos-latest' runs-on: 'macos-latest'
timeout-minutes: 10 timeout-minutes: 30
env: env:
DEVELOPER_DIR: "/Applications/Xcode${{ matrix.xcode && format('_{0}', matrix.xcode) || '' }}.app/Contents/Developer" DEVELOPER_DIR: "/Applications/Xcode${{ matrix.xcode && format('_{0}', matrix.xcode) || '' }}.app/Contents/Developer"
CC: ${{ matrix.compiler }} CC: ${{ matrix.compiler }}
@ -285,8 +275,8 @@ jobs:
install: libssh install: libssh
generate: -DOPENSSL_ROOT_DIR=$(brew --prefix openssl) -DBUILD_STATIC_LIBS=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON generate: -DOPENSSL_ROOT_DIR=$(brew --prefix openssl) -DBUILD_STATIC_LIBS=ON -DCURL_USE_LIBSSH2=OFF -DCURL_USE_LIBSSH=ON
macos-version-min: '10.9' macos-version-min: '10.9'
- name: 'SecureTransport ws debug+' - name: 'SecureTransport ws debug'
generate: -DCURL_USE_SECTRANSP=ON -DENABLE_WEBSOCKETS=ON -DENABLE_DEBUG=ON -DENABLE_CURLDEBUG=ON generate: -DCURL_USE_SECTRANSP=ON -DENABLE_WEBSOCKETS=ON -DENABLE_DEBUG=ON
macos-version-min: '10.8' macos-version-min: '10.8'
- name: 'LibreSSL !ldap heimdal c-ares +examples' - name: 'LibreSSL !ldap heimdal c-ares +examples'
install: libressl heimdal install: libressl heimdal
@ -300,9 +290,20 @@ jobs:
install: gnutls nettle krb5 install: gnutls nettle krb5
generate: -DCURL_USE_GNUTLS=ON -DCURL_USE_OPENSSL=OFF -DCURL_USE_GSSAPI=ON -DGSS_ROOT_DIR=$(brew --prefix krb5) -DCURL_DISABLE_LDAP=ON generate: -DCURL_USE_GNUTLS=ON -DCURL_USE_OPENSSL=OFF -DCURL_USE_GSSAPI=ON -DGSS_ROOT_DIR=$(brew --prefix krb5) -DCURL_DISABLE_LDAP=ON
macos-version-min: '10.15' macos-version-min: '10.15'
- name: 'OpenSSL torture !FTP'
generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=$(brew --prefix openssl) -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DENABLE_WEBSOCKETS=ON
tflags: -t --shallow=25 !FTP
macos-version-min: '10.9'
torture: true
- name: 'OpenSSL torture FTP'
generate: -DENABLE_DEBUG=ON -DBUILD_SHARED_LIBS=OFF -DENABLE_THREADED_RESOLVER=OFF -DOPENSSL_ROOT_DIR=$(brew --prefix openssl) -DCURL_BROTLI=ON -DCURL_ZSTD=ON
tflags: -t --shallow=20 FTP
macos-version-min: '10.9'
torture: true
exclude: exclude:
- { compiler: llvm@15, build: { macos-version-min: '10.15' } } - { compiler: llvm@15, build: { macos-version-min: '10.15' } }
- { compiler: llvm@15, build: { macos-version-min: '10.9' } } - { compiler: llvm@15, build: { macos-version-min: '10.9' } }
- { compiler: gcc-12, build: { torture: true } }
steps: steps:
- name: 'brew install' - name: 'brew install'
run: | run: |
@ -378,25 +379,27 @@ jobs:
python3 -m pip install impacket python3 -m pip install impacket
- name: 'cmake run tests' - name: 'cmake run tests'
timeout-minutes: 10 timeout-minutes: ${{ matrix.build.torture && 20 || 10 }}
run: | run: |
export TFLAGS='${{ matrix.build.tflags }} -j10' export TFLAGS='-j10 ${{ matrix.build.tflags }}'
TFLAGS+=' ~2037 ~2041' # flaky if [ -z '${{ matrix.build.torture }}' ]; then
if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then TFLAGS+=' ~2037 ~2041' # flaky
TFLAGS+=' ~RTSP' # 567 568 569 570 571 572 577 689 3100 if [[ '${{ matrix.compiler }}' = 'gcc'* ]]; then
TFLAGS+=' ~1156 ~1539' # HTTP Content-Range, Content-Length TFLAGS+=' ~RTSP' # 567 568 569 570 571 572 577 689 3100
TFLAGS+=' ~1156 ~1539' # HTTP Content-Range, Content-Length
if [[ '${{ matrix.build.generate }}' = *'-DCURL_USE_SECTRANSP=ON'* ]]; then
TFLAGS+=' ~2100' # 2100:'HTTP GET using DoH' https://github.com/curl/curl/actions/runs/9942146678/job/27462937524#step:15:5059
TFLAGS+=' ~HTTP/2' # 2400 2401 2402 2403 2404 2406, SecureTransport + nghttp2
else
TFLAGS+=' ~2402 ~2404' # non-SecureTransport + nghttp2
fi
fi
if [[ '${{ matrix.build.generate }}' = *'-DCURL_USE_SECTRANSP=ON'* ]]; then if [[ '${{ matrix.build.generate }}' = *'-DCURL_USE_SECTRANSP=ON'* ]]; then
TFLAGS+=' ~2100' # 2100:'HTTP GET using DoH' https://github.com/curl/curl/actions/runs/9942146678/job/27462937524#step:15:5059 TFLAGS+=' ~313' # SecureTransport does not support crl file
TFLAGS+=' ~HTTP/2' # 2400 2401 2402 2403 2404 2406, SecureTransport + nghttp2 TFLAGS+=' ~1631 ~1632' # SecureTransport is not able to shutdown ftp over https gracefully yet
else
TFLAGS+=' ~2402 ~2404' # non-SecureTransport + nghttp2
fi fi
fi fi
if [[ '${{ matrix.build.generate }}' = *'-DCURL_USE_SECTRANSP=ON'* ]]; then source $HOME/venv/bin/activate
TFLAGS+=' ~313' # SecureTransport does not support crl file
TFLAGS+=' ~1631 ~1632' # SecureTransport is not able to shutdown ftp over https gracefully yet
fi
PATH="$HOME/venv/bin:$PATH"
rm -f $HOME/.curlrc rm -f $HOME/.curlrc
ninja -C bld test-ci ninja -C bld test-ci
@ -408,7 +411,7 @@ jobs:
if: true # Set to `true` to enable this test matrix. It runs quickly. if: true # Set to `true` to enable this test matrix. It runs quickly.
name: "${{ matrix.build == 'cmake' && 'CM' || 'AM' }} ${{ matrix.compiler }} ${{ matrix.image }} ${{ matrix.xcode }} ${{ matrix.config }}" name: "${{ matrix.build == 'cmake' && 'CM' || 'AM' }} ${{ matrix.compiler }} ${{ matrix.image }} ${{ matrix.xcode }} ${{ matrix.config }}"
runs-on: ${{ matrix.image }} runs-on: ${{ matrix.image }}
timeout-minutes: 30 timeout-minutes: 10
env: env:
DEVELOPER_DIR: "/Applications/Xcode${{ matrix.xcode && format('_{0}', matrix.xcode) || '' }}.app/Contents/Developer" DEVELOPER_DIR: "/Applications/Xcode${{ matrix.xcode && format('_{0}', matrix.xcode) || '' }}.app/Contents/Developer"
CC: ${{ matrix.compiler }} CC: ${{ matrix.compiler }}

64
.github/workflows/torture.yml vendored
View File

@ -40,52 +40,52 @@ concurrency:
permissions: {} permissions: {}
env:
MAKEFLAGS: -j 5
jobs: jobs:
autotools: cmake:
name: ${{ matrix.build.name }} name: '${{ matrix.build.name }}'
runs-on: 'ubuntu-latest' runs-on: 'ubuntu-24.04'
timeout-minutes: 90 timeout-minutes: 30
strategy: strategy:
fail-fast: false fail-fast: false
matrix: matrix:
build: build:
- name: torture - name: 'OpenSSL torture !FTP'
install: libpsl-dev libbrotli-dev libzstd-dev zlib1g-dev libnghttp2-dev libssh2-1-dev libc-ares-dev install: libpsl-dev libbrotli-dev libzstd-dev zlib1g-dev libnghttp2-dev libssh2-1-dev libc-ares-dev
configure: --with-openssl --enable-debug --enable-ares --enable-websockets generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON -DENABLE_WEBSOCKETS=ON
tflags: -n -t --shallow=25 !FTP -j10 tflags: -t --shallow=25 !FTP
- name: torture-ftp - name: 'OpenSSL torture FTP'
install: libpsl-dev libbrotli-dev libzstd-dev zlib1g-dev libnghttp2-dev libssh2-1-dev libc-ares-dev install: libpsl-dev libbrotli-dev libzstd-dev zlib1g-dev libnghttp2-dev libssh2-1-dev libc-ares-dev
configure: --with-openssl --enable-debug --enable-ares generate: -DCURL_USE_OPENSSL=ON -DENABLE_DEBUG=ON -DENABLE_ARES=ON
tflags: -n -t --shallow=20 FTP -j10 tflags: -t --shallow=20 FTP
steps: steps:
- run: | - run: |
sudo rm -f /etc/apt/sources.list.d/microsoft-prod.list sudo apt-get install cmake ninja-build pkgconf stunnel4 ${{ matrix.build.install }}
sudo apt-get update python3 -m venv $HOME/venv
sudo apt-get install libtool autoconf automake pkgconf stunnel4 ${{ matrix.build.install }} source $HOME/venv/bin/activate
sudo python3 -m pip install impacket python3 -m pip install impacket
name: 'install prereqs and impacket' name: 'install prereqs'
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4 - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4
- run: autoreconf -fi - run: |
name: 'autoreconf' cmake -G Ninja -DCMAKE_UNITY_BUILD=ON -DCURL_WERROR=ON \
-DCURL_BROTLI=ON -DCURL_ZSTD=ON \
${{ matrix.build.generate }}
name: 'cmake configure'
- run: cmake --build . --verbose
name: 'cmake build'
- run: ./src/curl -V
name: 'check curl -V output'
- run: cmake --build . --verbose --target testdeps
name: 'build tests'
- run: | - run: |
./configure --disable-dependency-tracking --enable-unity --enable-warnings --enable-werror \ source $HOME/venv/bin/activate
${{ matrix.build.configure }} cmake --build . --verbose --target test-torture
name: 'configure'
- run: make V=1
name: 'make'
- run: make V=1 -C tests
name: 'make tests'
- run: make V=1 test-torture
name: 'run tests' name: 'run tests'
env: env:
TFLAGS: "${{ matrix.build.tflags }}" TFLAGS: '-j10 ${{ matrix.build.tflags }}'