GHA: improve vcpkg cache, add BoringSSL ECH and LibreSSL MSVC jobs

- cache on a per-package basis. Replace manual caching with a built-in solution. It shares cached package builds between jobs, e.g. libssh2 only builds once per platform (instead of once per job). Individual packages are built as needed (not the whole per-job tree). It also fixes the duplicate cache entry issues. Ref: https://learn.microsoft.com/en-us/vcpkg/consume/binary-caching-github-actions-cache Follow-up to e26cbe20cb #13979 Follow-up to cb22cfca69 #14077 - add BoringSSL job with ECH enabled. The first such job in the curl CI. - add LibreSSL job. - use vcpkg pre-installed on the runner image, instead of rolling our own. This is quicker, simpler and more robust. Follow-up to e26cbe20cb #13979 - show pre-installed vcpkg and ports version. - drop `gsasl` dependency till it reaches the pre-installed vcpkg ports. - re-add `find .` to see the binaries generated. - simplify setting up `PATH`. - exclude failing tests for any job enabling WinIDN. - drop collecting and uploading log archives. We already dump CMake logs, and our build doesn't use Ninja. Rest of files weren't generated by the curl build. We don't aim to debug vcpkg package builds. Closes #14090
2024-07-07 15:39:31 +02:00 · 2024-07-07 15:39:31 +02:00 · f99c08dba4
commit f99c08dba4
parent cb22cfca69
1 changed files with 50 additions and 69 deletions
--- a/.github/workflows/windows.yml
+++ b/.github/workflows/windows.yml
@ -423,25 +423,25 @@ jobs:
    runs-on: windows-latest
    timeout-minutes: 55
    env:
+      VCPKG_BINARY_SOURCES: 'clear;x-gha,readwrite'
      VCPKG_DISABLE_METRICS: '1'
    strategy:
      matrix:
        include:
-          # name should be unique and without special chars. It creates a folder with that name for the cache.
        - name: 'schannel'
-          install: 'brotli zlib zstd libpsl nghttp2 libssh2[core,zlib] pkgconf gsasl'
-          arch: 'x64'
-          plat: 'windows'
-          type: 'Debug'
-          tflags: '~1516 ~2301 ~2302 ~2303 ~2307 ~165 ~1448 ~2046 ~2047'
-          config: '-DENABLE_DEBUG=ON -DENABLE_UNICODE=ON  -DCURL_USE_SCHANNEL=ON  -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DCURL_USE_LIBPSL=ON -DUSE_NGHTTP2=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_LIBSSH2=ON -DUSE_WIN32_IDN=ON -DCURL_USE_GSASL=ON'
-        - name: 'openssl'
-          install: 'brotli zlib zstd libpsl nghttp2 nghttp3 openssl libssh2 pkgconf gsasl'
+          install: 'brotli zlib zstd libpsl nghttp2 libssh2[core,zlib]'
          arch: 'x64'
          plat: 'windows'
          type: 'Debug'
          tflags: '~1516 ~2301 ~2302 ~2303 ~2307'
-          config: '-DENABLE_DEBUG=ON -DENABLE_UNICODE=OFF -DCURL_USE_SCHANNEL=OFF -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DCURL_USE_LIBPSL=ON -DUSE_NGHTTP2=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_LIBSSH2=ON -DCURL_USE_OPENSSL=ON -DUSE_OPENSSL_QUIC=ON -DCURL_USE_GSASL=ON'
+          config: '-DENABLE_DEBUG=ON -DENABLE_UNICODE=ON  -DCURL_USE_SCHANNEL=ON  -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DCURL_USE_LIBPSL=ON -DUSE_NGHTTP2=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_LIBSSH2=ON -DUSE_WIN32_IDN=ON'
+        - name: 'openssl'
+          install: 'brotli zlib zstd libpsl nghttp2 nghttp3 openssl libssh2'
+          arch: 'x64'
+          plat: 'windows'
+          type: 'Debug'
+          tflags: '~1516 ~2301 ~2302 ~2303 ~2307'
+          config: '-DENABLE_DEBUG=ON -DENABLE_UNICODE=OFF -DCURL_USE_SCHANNEL=OFF -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DCURL_USE_LIBPSL=ON -DUSE_NGHTTP2=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_LIBSSH2=ON -DCURL_USE_OPENSSL=ON -DUSE_OPENSSL_QUIC=ON'
        - name: 'openssl'
          install: 'brotli zlib zstd nghttp2 nghttp3 openssl libssh2'
          arch: 'x64'
@ -449,6 +449,20 @@ jobs:
          type: 'Debug'
          tflags: 'skipall'
          config: '-DENABLE_DEBUG=ON -DENABLE_UNICODE=OFF -DCURL_USE_SCHANNEL=OFF -DCURL_BROTLI=ON -DCURL_ZSTD=ON                      -DUSE_NGHTTP2=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_LIBSSH2=ON -DCURL_USE_OPENSSL=ON -DUSE_OPENSSL_QUIC=ON'
+        - name: 'libressl'
+          install: 'brotli zlib zstd libpsl nghttp2 libressl libssh2[core,zlib]'
+          arch: 'x64'
+          plat: 'windows'
+          type: 'Debug'
+          tflags: '~1516 ~2301 ~2302 ~2303 ~2307'
+          config: '-DENABLE_DEBUG=ON -DENABLE_UNICODE=OFF -DCURL_USE_SCHANNEL=OFF -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DCURL_USE_LIBPSL=ON -DUSE_NGHTTP2=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_LIBSSH2=ON -DCURL_USE_OPENSSL=ON'
+        - name: 'boringssl ECH'
+          install: 'brotli zlib zstd libpsl nghttp2 boringssl libssh2[core,zlib]'
+          arch: 'x64'
+          plat: 'windows'
+          type: 'Debug'
+          tflags: '~1516 ~2301 ~2302 ~2303 ~2307'
+          config: '-DENABLE_DEBUG=ON -DENABLE_UNICODE=OFF -DCURL_USE_SCHANNEL=OFF -DCURL_BROTLI=ON -DCURL_ZSTD=ON -DCURL_USE_LIBPSL=ON -DUSE_NGHTTP2=ON -DBUILD_SHARED_LIBS=OFF -DCURL_USE_LIBSSH2=ON -DCURL_USE_OPENSSL=ON -DUSE_HTTPSRR=ON -DUSE_ECH=ON'
        - name: 'no ssl'
          install: 'brotli zlib zstd libpsl'
          arch: 'x64'
@ -460,46 +474,38 @@ jobs:
    steps:
      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4

-      - name: 'restore cache dependencies'
-        uses: actions/cache/restore@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
+      - name: 'vcpkg cache setup'
+        uses: actions/github-script@v7
        with:
-          path: 'cache/${{ matrix.arch }}-${{ matrix.plat }}-${{ matrix.name }}'
-          key: ${{ matrix.arch }}-${{ matrix.plat }}-${{ matrix.name }}
-          restore-keys: ${{ matrix.arch }}-${{ matrix.plat }}-${{ matrix.name }}
+          script: |
+            core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
+            core.exportVariable('ACTIONS_RUNTIME_TOKEN', process.env.ACTIONS_RUNTIME_TOKEN || '');

-      - name: 'vcpkg init'
-        timeout-minutes: 5
-        if: success()
+      - name: 'vcpkg versions'
+        timeout-minutes: 1
        shell: bash
        run: |
-          git clone --quiet --depth 1 https://github.com/microsoft/vcpkg/
-          vcpkg/bootstrap-vcpkg.sh -disableMetrics
+          git -C "$VCPKG_INSTALLATION_ROOT" show --no-patch --format='%H %ai'
+          vcpkg version

      - name: 'vcpkg build'
        timeout-minutes: 25
        shell: bash
        run: |
-          export VCPKG_ROOT="$PWD/vcpkg"
-          export CURRENT_BINARY_CACHE='cache/${{ matrix.arch }}-${{ matrix.plat }}-${{ matrix.name }}'
-          mkdir -p "$CURRENT_BINARY_CACHE"
-          export VCPKG_BINARY_SOURCES="clear;files,$GITHUB_WORKSPACE/$CURRENT_BINARY_CACHE,readwrite;"
-          vcpkg/vcpkg x-set-installed ${{ matrix.install }} --triplet=${{ matrix.arch }}-${{ matrix.plat }}
-
-      - name: 'save cache dependencies'
-        id: cache-save
-        uses: actions/cache/save@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4
-        with:
-          path: 'cache/${{ matrix.arch }}-${{ matrix.plat }}-${{ matrix.name }}'
-          key: ${{ matrix.arch }}-${{ matrix.plat }}-${{ matrix.name }}-${{ hashFiles('vcpkg/installed/vcpkg/info/*', 'vcpkg/installed/vcpkg/updates/*') }}
+          vcpkg x-set-installed ${{ matrix.install }} '--triplet=${{ matrix.arch }}-${{ matrix.plat }}'

      - name: 'cmake configure'
        timeout-minutes: 5
        shell: bash
        run: |
-          export VCPKG_ROOT="$PWD/vcpkg"
-          cmake -B bld \
-            "-DCMAKE_TOOLCHAIN_FILE=$VCPKG_ROOT/scripts/buildsystems/vcpkg.cmake" \
-            "-DVCPKG_INSTALLED_DIR=$VCPKG_ROOT/installed" \
+          if [[ '${{ matrix.install }}' = *'libressl'* ]]; then
+            # without this, CMake gets confused about the non-vcpkg OpenSSL
+            # installed on the runner and fails when linking.
+            options+=" -DOPENSSL_ROOT_DIR=$VCPKG_INSTALLATION_ROOT/installed/${{ matrix.arch }}-${{ matrix.plat }}"
+          fi
+          cmake -B bld ${options} \
+            "-DCMAKE_TOOLCHAIN_FILE=$VCPKG_INSTALLATION_ROOT/scripts/buildsystems/vcpkg.cmake" \
+            "-DVCPKG_INSTALLED_DIR=$VCPKG_INSTALLATION_ROOT/installed" \
            '-DVCPKG_TARGET_TRIPLET=${{ matrix.arch }}-${{ matrix.plat }}' \
            -DCMAKE_VS_GLOBALS=TrackFileAccess=false \
            '-DCMAKE_BUILD_TYPE=${{ matrix.type }}' \
@ -511,40 +517,15 @@ jobs:
            -DCMAKE_RUNTIME_OUTPUT_DIRECTORY_RELEASE= \
            ${{ matrix.config }} || { cat bld/CMakeFiles/CMake*.yaml; false; }

-      - name: 'prepare logs on failure'
-        timeout-minutes: 5
-        if: failure()
-        shell: bash
-        run: |
-          export VCPKG_ROOT="$GITHUB_WORKSPACE/vcpkg"
-          mkdir -p bld
-          7z a -t7z -r -mx=9 logs.7z \
-            "$VCPKG_ROOT"/buildtrees/*.log \
-            bld/.ninja_log \
-            bld/build.ninja \
-            bld/install_manifest.txt \
-            bld/vcpkg-manifest-install.log \
-            bld/CMakeFiles/CMake*.yaml
-
-      - name: 'upload logs on failure'
-        timeout-minutes: 10
-        if: failure()
-        uses: actions/upload-artifact@0b2256b8c012f0828dc542b3febcab082c67f72b # v4
-        with:
-          name: windows_logs_${{matrix.name}}_${{github.event.pull_request.head.sha}}
-          path: logs.7z
-
      - name: 'cmake build'
        timeout-minutes: 5
        shell: bash
        run: |
          cmake --build bld --config '${{ matrix.type }}' --parallel 5
+          find . -name '*.exe' -o -name '*.dll' | grep -v '/examples/'
          if [ '${{ matrix.plat }}' != 'uwp' ]; then
-            export VCPKG_ROOT="$PWD/vcpkg"
-            export VCPKG_TRIPLET_DIR="$VCPKG_ROOT/installed/${{ matrix.arch }}-${{ matrix.plat }}"
-            export debug_lib="$VCPKG_TRIPLET_DIR/debug/bin"
-            export release_lib="$VCPKG_TRIPLET_DIR/bin"
-            export PATH="$PWD/bld/lib/:$release_lib:$debug_lib:$PATH"
+            export VCPKG_TRIPLET_DIR="$VCPKG_INSTALLATION_ROOT/installed/${{ matrix.arch }}-${{ matrix.plat }}"
+            export PATH="$PWD/bld/lib/:$VCPKG_TRIPLET_DIR/bin:$VCPKG_TRIPLET_DIR/debug/bin:$PATH"
            bld/src/curl.exe --disable --version
          fi

@ -561,9 +542,9 @@ jobs:
        shell: bash
        run: |
          export TFLAGS='-j14 !TFTP !MQTT !WebSockets !SMTP ${{ matrix.tflags }}'
-          export VCPKG_ROOT="$PWD/vcpkg"
-          export VCPKG_TRIPLET_DIR="$VCPKG_ROOT/installed/${{ matrix.arch }}-${{ matrix.plat }}"
-          export debug_lib="$VCPKG_TRIPLET_DIR/debug/bin"
-          export release_lib="$VCPKG_TRIPLET_DIR/bin"
-          export PATH="$PWD/bld/lib/:$release_lib:$debug_lib:$PATH"
+          if [[ '${{ matrix.config }}' = *'-DUSE_WIN32_IDN=ON'* ]]; then
+            TFLAGS+=' ~165 ~1448 ~2046 ~2047'
+          fi
+          export VCPKG_TRIPLET_DIR="$VCPKG_INSTALLATION_ROOT/installed/${{ matrix.arch }}-${{ matrix.plat }}"
+          export PATH="$PWD/bld/lib/:$VCPKG_TRIPLET_DIR/bin:$VCPKG_TRIPLET_DIR/debug/bin:$PATH"
          cmake --build bld --config '${{ matrix.type }}' --target test-ci