mirror of
https://github.com/xroche/httrack.git
synced 2026-06-29 05:26:32 +03:00
c1a8c5ffa8
Both linters fetched a tool over the network. The format job pulled the git-clang-format driver from raw.githubusercontent.com, which 429 rate-limits the shared runner egress IPs; a 429 failed the job and left the cache empty, so every later run cold-missed and 429'd again. The lint job similarly fetched the shfmt release binary from github.com. Both are unnecessary. The clang-format-19 package already installed ships the matching git-clang-format driver (/usr/bin/git-clang-format-19); symlink it to the unsuffixed name. And ubuntu-24.04 (noble) ships shfmt 3.8.0 in universe, exactly the pinned version, so install it from apt too. This drops both fetches, both actions/cache steps, and the LLVM_TAG / SHFMT_VERSION env: no network call, nothing to rate-limit. Each tool's version now tracks its apt package, same as clang-format itself. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com>
396 lines
14 KiB
YAML
396 lines
14 KiB
YAML
# Build and test on x86-64 and arm64, and lint the shell scripts.
|
|
name: CI
|
|
|
|
on:
|
|
push:
|
|
branches: [master]
|
|
pull_request:
|
|
workflow_dispatch:
|
|
|
|
# Least privilege: the workflow only needs to read the repo.
|
|
permissions:
|
|
contents: read
|
|
|
|
# Cancel superseded runs on the same branch or PR.
|
|
concurrency:
|
|
group: ci-${{ github.ref }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
build:
|
|
name: build (${{ matrix.arch }}, ${{ matrix.cc }})
|
|
runs-on: ${{ matrix.runner }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- { arch: x86-64, runner: ubuntu-24.04, cc: gcc }
|
|
- { arch: x86-64, runner: ubuntu-24.04, cc: clang }
|
|
- { arch: arm64, runner: ubuntu-24.04-arm, cc: gcc }
|
|
- { arch: arm64, runner: ubuntu-24.04-arm, cc: clang }
|
|
env:
|
|
CC: ${{ matrix.cc }}
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Install build dependencies
|
|
run: |
|
|
set -euo pipefail
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends \
|
|
build-essential clang autoconf automake libtool autoconf-archive \
|
|
zlib1g-dev libssl-dev
|
|
|
|
- name: Configure
|
|
run: |
|
|
set -euo pipefail
|
|
# Regenerate from configure.ac/Makefile.am to validate them; the
|
|
# committed generated files already let a plain checkout build.
|
|
autoreconf -fi
|
|
./configure
|
|
|
|
- name: Build
|
|
run: make -j"$(nproc)"
|
|
|
|
- name: Test
|
|
run: make check
|
|
|
|
- name: Print the test log on failure
|
|
if: failure()
|
|
run: cat tests/test-suite.log 2>/dev/null || true
|
|
|
|
# Portability: build and test on macOS (Darwin/clang) on a native runner --
|
|
# no VM. The tree has no __APPLE__ branches, so Darwin exercises the
|
|
# generic-Unix path on a second libc and kernel. brew's openssl@3 is keg-only,
|
|
# so point configure at it; everything else is in the SDK or default paths.
|
|
macos:
|
|
name: build (macOS arm64, clang)
|
|
runs-on: macos-14
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Install build dependencies
|
|
run: |
|
|
set -euo pipefail
|
|
brew install autoconf automake libtool autoconf-archive
|
|
|
|
- name: Configure
|
|
run: |
|
|
set -euo pipefail
|
|
ssl="$(brew --prefix openssl@3)"
|
|
autoreconf -fi
|
|
./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib"
|
|
|
|
- name: Build
|
|
run: make -j"$(sysctl -n hw.ncpu)"
|
|
|
|
- name: Test
|
|
run: make check
|
|
|
|
- name: Print the test log on failure
|
|
if: failure()
|
|
run: cat tests/test-suite.log 2>/dev/null || true
|
|
|
|
# Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
|
|
# -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
|
|
# and bounds math can truncate or wrap in ways 64-bit never reveals (the axis
|
|
# the overflow-safe bounds work targets). --build (not --host) keeps configure
|
|
# out of cross mode, so the i386 binary still runs the test suite here.
|
|
linux-i386:
|
|
name: build (linux i386, gcc -m32)
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Install build dependencies (multilib + 32-bit libs)
|
|
run: |
|
|
set -euo pipefail
|
|
sudo dpkg --add-architecture i386
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends \
|
|
build-essential gcc-multilib autoconf automake libtool \
|
|
autoconf-archive zlib1g-dev:i386 libssl-dev:i386
|
|
|
|
- name: Configure
|
|
run: |
|
|
set -euo pipefail
|
|
autoreconf -fi
|
|
./configure --build=i686-pc-linux-gnu CC="gcc -m32"
|
|
|
|
- name: Build
|
|
run: make -j"$(nproc)"
|
|
|
|
- name: Test
|
|
run: make check
|
|
|
|
- name: Print the test log on failure
|
|
if: failure()
|
|
run: cat tests/test-suite.log 2>/dev/null || true
|
|
|
|
# Memory safety: build and run the suite under AddressSanitizer +
|
|
# UndefinedBehaviorSanitizer. The offline engine self-tests drive the parsers
|
|
# that chew on untrusted crawled input (charset, mime, HTML, entities, IDNA,
|
|
# filters, cache) straight through the sanitizers, so a buffer overrun,
|
|
# use-after-free, or signed overflow there fails the build instead of slipping
|
|
# past a plain -O2 build. gcc's runtimes; one job is enough (the bug class is
|
|
# arch-independent and the matrix already covers compile portability).
|
|
sanitize:
|
|
name: sanitize (ASan+UBSan, gcc)
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Install build dependencies
|
|
run: |
|
|
set -euo pipefail
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends \
|
|
build-essential autoconf automake libtool autoconf-archive \
|
|
zlib1g-dev libssl-dev
|
|
|
|
- name: Configure (sanitized)
|
|
run: |
|
|
set -euo pipefail
|
|
autoreconf -fi
|
|
./configure CC=gcc \
|
|
CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
|
|
LDFLAGS="-fsanitize=address,undefined"
|
|
|
|
- name: Build
|
|
run: make -j"$(nproc)"
|
|
|
|
- name: Test (sanitized)
|
|
# Leaks at exit are out of scope (the CLI frees little on the way out);
|
|
# we want memory-safety errors, so turn leak detection off and make every
|
|
# other finding abort the run.
|
|
#
|
|
# Poison fresh allocations with 0xCA and freed blocks with 0xCB (decimal
|
|
# 202/203) so memory never reads back as accidental zeros: a missing-NUL
|
|
# fread buffer then runs strlen off into the redzone instead of stopping
|
|
# at a lucky zero. Distinct bytes tell the two apart in a dump (0xCA =
|
|
# uninitialized, 0xCB = use-after-free). ASan caps its malloc fill at 4096
|
|
# bytes by default, so max_malloc_fill_size lifts it to cover large cache
|
|
# buffers; free_fill flags use-after-free reads.
|
|
env:
|
|
ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
|
|
UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
|
|
run: make check
|
|
|
|
- name: Print the test log on failure
|
|
if: failure()
|
|
run: cat tests/test-suite.log 2>/dev/null || true
|
|
|
|
# Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
|
|
# the configuration users on minimal systems build, and one libssl is not even
|
|
# installed here so configure cannot silently re-enable it. The matrix above
|
|
# always has libssl, so the #if HTS_USEOPENSSL branches would otherwise never
|
|
# be compiled and could rot unnoticed.
|
|
no-ssl:
|
|
name: build (no openssl, --disable-https)
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Install build dependencies (no libssl)
|
|
run: |
|
|
set -euo pipefail
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends \
|
|
build-essential autoconf automake libtool autoconf-archive zlib1g-dev
|
|
|
|
- name: Configure (https disabled)
|
|
run: |
|
|
set -euo pipefail
|
|
autoreconf -fi
|
|
./configure --disable-https
|
|
|
|
- name: Build
|
|
run: make -j"$(nproc)"
|
|
|
|
- name: Test
|
|
run: make check
|
|
|
|
- name: Print the test log on failure
|
|
if: failure()
|
|
run: cat tests/test-suite.log 2>/dev/null || true
|
|
|
|
# Validate the Debian packaging via the same script maintainers release with.
|
|
# One amd64/gcc run is enough: packaging (control/rules/manifest/lintian/quilt
|
|
# source build) is arch- and compiler-independent, and the build matrix above
|
|
# already covers compile portability. lintian runs with --fail-on=error.
|
|
deb:
|
|
name: deb package (lintian)
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Install packaging toolchain
|
|
run: |
|
|
set -euo pipefail
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends \
|
|
build-essential autoconf automake libtool autoconf-archive \
|
|
zlib1g-dev libssl-dev \
|
|
debhelper devscripts lintian fakeroot
|
|
|
|
# --unsigned: CI has no GPG key (also skips the release sig/checksums).
|
|
# debuild builds every package, then lintian gates on errors.
|
|
#
|
|
# DEB_BUILD_OPTIONS trims work CI does not need (release builds via
|
|
# mkdeb.sh are untouched): noautodbgsym drops the -dbgsym packages whose
|
|
# LTO payloads are slow to compress and that CI never ships; parallel uses
|
|
# every core. We let debuild run its test pass -- the only one now that
|
|
# mkdeb no longer runs its own -- so CI exercises the packaged tests.
|
|
- name: Build Debian packages
|
|
run: |
|
|
export DEB_BUILD_OPTIONS="noautodbgsym parallel=$(nproc)"
|
|
bash tools/mkdeb.sh --unsigned --no-release-artifacts
|
|
|
|
# Release-tarball integrity: `make distcheck` rolls the dist tarball, then
|
|
# configures, builds and tests it out-of-tree from a read-only source tree and
|
|
# checks nothing is left behind. Catches a file referenced in *_SOURCES or
|
|
# EXTRA_DIST but missing from the tarball -- the same "ships broken to users"
|
|
# class as a stale committed Makefile.in.
|
|
distcheck:
|
|
name: distcheck (release tarball)
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: recursive
|
|
|
|
- name: Install build dependencies
|
|
run: |
|
|
set -euo pipefail
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends \
|
|
build-essential autoconf automake libtool autoconf-archive \
|
|
zlib1g-dev libssl-dev
|
|
|
|
- name: distcheck
|
|
run: |
|
|
set -euo pipefail
|
|
autoreconf -fi
|
|
./configure
|
|
make -j"$(nproc)" distcheck
|
|
|
|
dco:
|
|
name: DCO sign-off
|
|
# Only checkable on a PR, where we have the base..head commit range.
|
|
if: github.event_name == 'pull_request'
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Every commit must be signed off
|
|
env:
|
|
BASE: ${{ github.event.pull_request.base.sha }}
|
|
HEAD: ${{ github.event.pull_request.head.sha }}
|
|
run: |
|
|
set -euo pipefail
|
|
fail=0
|
|
# --no-merges: merge commits are GitHub-generated and carry no sign-off.
|
|
for sha in $(git rev-list --no-merges "$BASE..$HEAD"); do
|
|
if [ -z "$(git log -1 --format='%(trailers:key=Signed-off-by)' "$sha")" ]; then
|
|
echo "Missing Signed-off-by: $(git log -1 --format='%h %s' "$sha")"
|
|
fail=1
|
|
fi
|
|
done
|
|
if [ "$fail" -ne 0 ]; then
|
|
echo
|
|
echo "Sign commits with 'git commit -s'; fix a branch with 'git rebase --signoff $BASE'."
|
|
echo "See CONTRIBUTING.md (Developer Certificate of Origin)."
|
|
exit 1
|
|
fi
|
|
|
|
lint:
|
|
name: lint (shellcheck, shfmt)
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
|
|
- name: Install linters
|
|
run: |
|
|
set -euo pipefail
|
|
sudo apt-get update
|
|
# noble ships shfmt 3.8.0 (universe), matching the pinned local dev
|
|
# version; use it rather than fetching a release binary from github.com.
|
|
sudo apt-get install -y --no-install-recommends shellcheck shfmt
|
|
shfmt --version
|
|
|
|
# Lint the scripts we maintain; the legacy scripts are a separate cleanup.
|
|
- name: shellcheck
|
|
run: shellcheck man/makeman.sh tools/mkdeb.sh .githooks/pre-commit tests/*.test tests/check-network.sh
|
|
|
|
- name: shfmt
|
|
run: shfmt -d -i 4 man/makeman.sh tools/mkdeb.sh .githooks/pre-commit
|
|
|
|
# Check clang-format on CHANGED LINES ONLY. The engine predates clang-format
|
|
# (it was shaped by an old Visual Studio formatter) and does not round-trip,
|
|
# so we never reformat the whole tree -- only the lines a PR touches.
|
|
format:
|
|
name: format (clang-format-19, changed lines)
|
|
if: github.event_name == 'pull_request'
|
|
runs-on: ubuntu-24.04
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
|
|
- name: Install clang-format 19 (pinned, from apt.llvm.org)
|
|
run: |
|
|
set -euo pipefail
|
|
# ubuntu-24.04's native clang-format is 18; pin 19 to match local dev.
|
|
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \
|
|
| sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc >/dev/null
|
|
echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-19 main" \
|
|
| sudo tee /etc/apt/sources.list.d/llvm-19.list >/dev/null
|
|
sudo apt-get update
|
|
sudo apt-get install -y --no-install-recommends clang-format-19
|
|
# The clang-format-19 package ships the git-clang-format driver;
|
|
# expose it unsuffixed so "git clang-format" finds it.
|
|
sudo ln -sf /usr/bin/git-clang-format-19 /usr/local/bin/git-clang-format
|
|
clang-format-19 --version
|
|
|
|
- name: Check formatting of changed lines
|
|
run: |
|
|
set -euo pipefail
|
|
git fetch --no-tags origin \
|
|
"+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
|
|
base="origin/${{ github.base_ref }}"
|
|
set +e
|
|
diff="$(git clang-format --binary clang-format-19 --style=file \
|
|
--diff --extensions c,h "$base")"
|
|
rc=$?
|
|
set -e
|
|
# Classify by output, not exit code: a non-empty diff means "not
|
|
# clean" (git-clang-format may exit 0 or 1 on a diff). A nonzero exit
|
|
# with clean output is a real checker error.
|
|
case "$diff" in
|
|
"" | "no modified files to format" | *"did not modify any files"*)
|
|
if [ "$rc" -ne 0 ]; then
|
|
echo "::error::git clang-format failed (exit $rc): checker error."
|
|
exit 1
|
|
fi
|
|
echo "Formatting OK: changed C lines are clang-format-clean." ;;
|
|
*)
|
|
echo "$diff"
|
|
echo "::error::Changed C lines are not clang-format-clean."
|
|
echo "Fix locally with: git clang-format --binary clang-format-19 $base"
|
|
exit 1 ;;
|
|
esac
|