httrack/.github/workflows/ci.yml

# Build and test on x86-64 and arm64, and lint the shell scripts.
name: CI

on:
  push:
    branches: [master]
  pull_request:
  workflow_dispatch:

# Least privilege: the workflow only needs to read the repo.
permissions:
  contents: read

# Cancel superseded runs on the same branch or PR.
concurrency:
  group: ci-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build:
    name: build (${{ matrix.arch }}, ${{ matrix.cc }})
    runs-on: ${{ matrix.runner }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - { arch: x86-64, runner: ubuntu-24.04, cc: gcc }
          - { arch: x86-64, runner: ubuntu-24.04, cc: clang }
          - { arch: arm64, runner: ubuntu-24.04-arm, cc: gcc }
          - { arch: arm64, runner: ubuntu-24.04-arm, cc: clang }
    env:
      CC: ${{ matrix.cc }}
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Install build dependencies
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential clang autoconf automake libtool autoconf-archive \
            zlib1g-dev libssl-dev libbrotli-dev libzstd-dev

      - name: Configure
        run: |
          set -euo pipefail
          # Regenerate from configure.ac/Makefile.am to validate them; the
          # committed generated files already let a plain checkout build.
          autoreconf -fi
          ./configure
          # a missing decoder would silently drop the coding from Accept-Encoding
          grep -q "define HTS_USEBROTLI 1" config.h
          grep -q "define HTS_USEZSTD 1" config.h

      - name: Build
        run: make -j"$(nproc)"

      - name: Test
        run: |
          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
          make check -j"$jobs"

      - name: Print the test log on failure
        if: failure()
        run: cat tests/test-suite.log 2>/dev/null || true

  # Reproduce the Debian buildds: they build in a minimal chroot with no
  # python3, so the local-server tests must SKIP (exit 77), not fail. GitHub
  # runners ship python3, so every other job hides this path; here we remove it
  # before `make check`. This is the guard that would have caught the 3.49.10-1
  # FTBFS (28_local-pause failed instead of skipping when python3 was absent).
  buildd-no-python3:
    name: build (no python3, Debian buildd)
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Install build dependencies
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential autoconf automake libtool autoconf-archive \
            zlib1g-dev libssl-dev libbrotli-dev libzstd-dev

      - name: Configure
        run: |
          set -euo pipefail
          autoreconf -fi
          ./configure

      - name: Build
        run: make -j"$(nproc)"

      - name: Test without python3
        run: |
          set -euo pipefail
          # Hide every python3* so `command -v python3` fails like it does in the
          # buildd chroot; masking with /bin/false would still resolve.
          sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
            -exec mv {} {}.hidden \;
          ! command -v python3
          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
          make check -j"$jobs"

      - name: Print the test log on failure
        if: failure()
        run: cat tests/test-suite.log 2>/dev/null || true

  # Portability: build and test on macOS (Darwin/clang) on a native runner --
  # no VM. The tree has no __APPLE__ branches, so Darwin exercises the
  # generic-Unix path on a second libc and kernel. brew's openssl@3 is keg-only,
  # so point configure at it; everything else is in the SDK or default paths.
  macos:
    name: build (macOS arm64, clang)
    runs-on: macos-14
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Install build dependencies
        run: |
          set -euo pipefail
          brew install autoconf automake libtool autoconf-archive brotli zstd

      - name: Configure
        run: |
          set -euo pipefail
          ssl="$(brew --prefix openssl@3)"
          brewp="$(brew --prefix)"
          autoreconf -fi
          ./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
            LDFLAGS="-L${ssl}/lib -L${brewp}/lib"
          grep -q "define HTS_USEBROTLI 1" config.h
          grep -q "define HTS_USEZSTD 1" config.h

      - name: Build
        run: make -j"$(sysctl -n hw.ncpu)"

      - name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
        # 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
        # instantly like Linux; alias them onto lo0 so they don't stall to timeout.
        run: |
          set -euo pipefail
          sudo ifconfig lo0 alias 127.0.0.2 up
          sudo ifconfig lo0 alias 127.0.0.3 up

      - name: Test
        run: |
          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
          make check -j"$jobs"

      - name: Print the test log on failure
        if: failure()
        run: cat tests/test-suite.log 2>/dev/null || true

  # Runtime smoke of the WebHTTrack launcher on macOS: it carries a Darwin-only
  # browser path (open -W) and nothing else exercises htsserver. Install into a
  # temp prefix, then check webhttrack brings up htsserver and serves the UI.
  webhttrack-macos:
    name: webhttrack smoke (macOS arm64)
    runs-on: macos-14
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Install build dependencies
        run: |
          set -euo pipefail
          brew install autoconf automake libtool autoconf-archive brotli zstd

      - name: Build and install into a temp prefix
        run: |
          set -euo pipefail
          ssl="$(brew --prefix openssl@3)"
          brewp="$(brew --prefix)"
          autoreconf -fi
          ./configure CPPFLAGS="-I${ssl}/include -I${brewp}/include" \
            LDFLAGS="-L${ssl}/lib -L${brewp}/lib" \
            --prefix="$RUNNER_TEMP/inst"
          make -j"$(sysctl -n hw.ncpu)"
          make install

      - name: Smoke-test webhttrack
        run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"

  # Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
  # -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
  # and bounds math can truncate or wrap in ways 64-bit never reveals (the axis
  # the overflow-safe bounds work targets). --build (not --host) keeps configure
  # out of cross mode, so the i386 binary still runs the test suite here.
  linux-i386:
    name: build (linux i386, gcc -m32)
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Install build dependencies (multilib + 32-bit libs)
        run: |
          set -euo pipefail
          sudo dpkg --add-architecture i386
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential gcc-multilib autoconf automake libtool \
            autoconf-archive zlib1g-dev:i386 libssl-dev:i386 \
            libbrotli-dev:i386 libzstd-dev:i386

      - name: Configure
        run: |
          set -euo pipefail
          autoreconf -fi
          ./configure --build=i686-pc-linux-gnu CC="gcc -m32"

      - name: Build
        run: make -j"$(nproc)"

      - name: Test
        run: |
          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
          make check -j"$jobs"

      - name: Print the test log on failure
        if: failure()
        run: cat tests/test-suite.log 2>/dev/null || true

  # Memory safety: build and run the suite under AddressSanitizer +
  # UndefinedBehaviorSanitizer. The offline engine self-tests drive the parsers
  # that chew on untrusted crawled input (charset, mime, HTML, entities, IDNA,
  # filters, cache) straight through the sanitizers, so a buffer overrun,
  # use-after-free, or signed overflow there fails the build instead of slipping
  # past a plain -O2 build. gcc's runtimes; one job is enough (the bug class is
  # arch-independent and the matrix already covers compile portability).
  sanitize:
    name: sanitize (ASan+UBSan, gcc)
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Install build dependencies
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential autoconf automake libtool autoconf-archive \
            zlib1g-dev libssl-dev libbrotli-dev libzstd-dev

      - name: Configure (sanitized)
        run: |
          set -euo pipefail
          autoreconf -fi
          ./configure CC=gcc \
            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
            LDFLAGS="-fsanitize=address,undefined"

      - name: Build
        run: make -j"$(nproc)"

      - name: Test (sanitized)
        # Leaks at exit are out of scope (the CLI frees little on the way out);
        # we want memory-safety errors, so turn leak detection off and make every
        # other finding abort the run.
        #
        # Poison fresh allocations with 0xCA and freed blocks with 0xCB (decimal
        # 202/203) so memory never reads back as accidental zeros: a missing-NUL
        # fread buffer then runs strlen off into the redzone instead of stopping
        # at a lucky zero. Distinct bytes tell the two apart in a dump (0xCA =
        # uninitialized, 0xCB = use-after-free). ASan caps its malloc fill at 4096
        # bytes by default, so max_malloc_fill_size lifts it to cover large cache
        # buffers; free_fill flags use-after-free reads.
        env:
          ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
        run: |
          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
          make check -j"$jobs"

      - name: Print the test log on failure
        if: failure()
        run: cat tests/test-suite.log 2>/dev/null || true

  # MemorySanitizer catches reads of uninitialized memory (#143's stack-garbage
  # size filter) that ASan/UBSan miss. It flags any byte an uninstrumented lib
  # wrote, so the job stays in our own code: offline self-tests only, no openssl
  # (--disable-https), no zlib cache tests, static (the runtime is not in .so's).
  msan:
    name: msan (MemorySanitizer, clang)
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Install build dependencies
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential clang autoconf automake libtool autoconf-archive \
            zlib1g-dev

      - name: Configure (MSan, static, no https)
        run: |
          set -euo pipefail
          autoreconf -fi
          ./configure CC=clang \
            CFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
            LDFLAGS="-fsanitize=memory" \
            --disable-https --disable-shared --enable-static

      - name: Build
        run: make -j"$(nproc)"

      - name: Test (offline self-tests under MSan)
        env:
          MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
        run: |
          set -euo pipefail
          # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
          # skipped here (uninstrumented libz floods MSan with false positives).
          tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
          make check -j"$jobs" TESTS="$tests"

      - name: Print the test log on failure
        if: failure()
        run: cat tests/test-suite.log 2>/dev/null || true

  # libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
  # parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
  # (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
  # catastrophic backtracking -- and pins the regression seeds for the bugs the
  # fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
  fuzz:
    name: fuzz (libFuzzer corpus replay, clang)
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Install build dependencies
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential clang autoconf automake libtool autoconf-archive \
            zlib1g-dev libssl-dev libbrotli-dev libzstd-dev

      - name: Configure (fuzzers, static)
        run: |
          set -euo pipefail
          autoreconf -fi
          ./configure CC=clang \
            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
            LDFLAGS="-fsanitize=address,undefined" \
            --enable-fuzzers --disable-shared

      - name: Build
        run: make -j"$(nproc)"

      - name: Replay corpora
        env:
          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
        run: bash fuzz/run-fuzzers.sh fuzz check

  # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
  # the configuration users on minimal systems build, and one libssl is not even
  # installed here so configure cannot silently re-enable it. The matrix above
  # always has libssl, so the #if HTS_USEOPENSSL branches would otherwise never
  # be compiled and could rot unnoticed.
  no-ssl:
    name: build (no openssl, --disable-https)
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Install build dependencies (no libssl)
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential autoconf automake libtool autoconf-archive zlib1g-dev

      - name: Configure (https disabled)
        run: |
          set -euo pipefail
          autoreconf -fi
          ./configure --disable-https

      - name: Build
        run: make -j"$(nproc)"

      - name: Test
        run: |
          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
          make check -j"$jobs"

      - name: Print the test log on failure
        if: failure()
        run: cat tests/test-suite.log 2>/dev/null || true

  # Validate the Debian packaging via the same script maintainers release with.
  # One amd64/gcc run is enough: packaging (control/rules/manifest/lintian/quilt
  # source build) is arch- and compiler-independent, and the build matrix above
  # already covers compile portability. mkdeb.sh runs lintian as an explicit gate
  # (debuild does not propagate lintian's exit) with --fail-on=error,warning.
  deb:
    name: deb package (lintian)
    runs-on: ubuntu-24.04
    # Build and gate inside Debian sid, the upload target. A Debian dpkg-deb
    # produces archive-legal xz members (an Ubuntu host defaults to zstd, which
    # the archive's lintian rejects), and sid's lintian carries the same
    # data-driven checks (embedded-lib fingerprints and the like) the buildds and
    # UDD apply -- so issues surface here instead of after upload.
    container: debian:sid
    steps:
      - name: Install packaging toolchain
        run: |
          set -euo pipefail
          apt-get update
          apt-get install -y --no-install-recommends \
            ca-certificates git \
            build-essential autoconf automake libtool autoconf-archive \
            zlib1g-dev libssl-dev libbrotli-dev libzstd-dev \
            debhelper devscripts lintian fakeroot

      - uses: actions/checkout@v6
        with:
          submodules: recursive

      # --unsigned: CI has no GPG key (also skips the release sig/checksums).
      # mkdeb builds every package then runs the lintian gate (--fail-on=error,
      # warning); debuild runs the packaged test pass.
      #
      # DEB_BUILD_OPTIONS trims work CI does not need (release builds via
      # mkdeb.sh are untouched): noautodbgsym drops the -dbgsym packages whose
      # LTO payloads are slow to compress and that CI never ships; parallel uses
      # every core.
      - name: Build and lint Debian packages
        run: |
          set -euo pipefail
          # The workspace volume is owned by the host runner uid, but the
          # container runs as root, so mkdeb's git calls (superproject and the
          # coucal submodule) trip "dubious ownership"; mark them all safe.
          git config --global --add safe.directory "*"
          export DEB_BUILD_OPTIONS="noautodbgsym parallel=$(nproc)"
          bash tools/mkdeb.sh --unsigned --no-release-artifacts

  # Release-tarball integrity: `make distcheck` rolls the dist tarball, then
  # configures, builds and tests it out-of-tree from a read-only source tree and
  # checks nothing is left behind. Catches a file referenced in *_SOURCES or
  # EXTRA_DIST but missing from the tarball -- the same "ships broken to users"
  # class as a stale committed Makefile.in.
  distcheck:
    name: distcheck (release tarball)
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6
        with:
          submodules: recursive

      - name: Install build dependencies
        run: |
          set -euo pipefail
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential autoconf automake libtool autoconf-archive \
            zlib1g-dev libssl-dev libbrotli-dev libzstd-dev

      - name: distcheck
        run: |
          set -euo pipefail
          autoreconf -fi
          ./configure
          make -j"$(nproc)" distcheck

  dco:
    name: DCO sign-off
    # Only checkable on a PR, where we have the base..head commit range.
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Every commit must be signed off
        env:
          BASE: ${{ github.event.pull_request.base.sha }}
          HEAD: ${{ github.event.pull_request.head.sha }}
        run: |
          set -euo pipefail
          fail=0
          # --no-merges: merge commits are GitHub-generated and carry no sign-off.
          for sha in $(git rev-list --no-merges "$BASE..$HEAD"); do
            if [ -z "$(git log -1 --format='%(trailers:key=Signed-off-by)' "$sha")" ]; then
              echo "Missing Signed-off-by: $(git log -1 --format='%h %s' "$sha")"
              fail=1
            fi
          done
          if [ "$fail" -ne 0 ]; then
            echo
            echo "Sign commits with 'git commit -s'; fix a branch with 'git rebase --signoff $BASE'."
            echo "See CONTRIBUTING.md (Developer Certificate of Origin)."
            exit 1
          fi

  lint:
    name: lint (shellcheck, shfmt)
    runs-on: ubuntu-24.04
    # Every tracked shell script; the globs expand at run time. Kept here so the
    # shellcheck and shfmt steps below cannot drift apart.
    env:
      SHELL_SCRIPTS: >-
        .githooks/pre-commit
        bootstrap
        build.sh
        html/div/search.sh
        man/makeman.sh
        src/htsbasiccharsets.sh
        src/htsentities.sh
        src/webhttrack
        tests/*.sh
        tests/*.test
        tools/mkdeb.sh
    steps:
      - uses: actions/checkout@v6

      - name: Install linters
        run: |
          set -euo pipefail
          sudo apt-get update
          # noble ships shfmt 3.8.0 (universe), matching the pinned local dev
          # version; use it rather than fetching a release binary from github.com.
          sudo apt-get install -y --no-install-recommends shellcheck shfmt
          shfmt --version

      - name: shellcheck
        run: shellcheck $SHELL_SCRIPTS

      - name: shfmt
        run: shfmt -d -i 4 $SHELL_SCRIPTS

      # MSBuild rejects a malformed .vcxproj with a bare MSB4025 and no build, so
      # catch it here in seconds rather than on a Windows runner minutes in.
      - name: XML well-formedness (MSBuild project files)
        run: |
          set -euo pipefail
          for f in src/*.vcxproj; do
            python3 -c "import sys,xml.dom.minidom; xml.dom.minidom.parse(sys.argv[1])" "$f"
            echo "ok $f"
          done

  # Check clang-format on CHANGED LINES ONLY. The engine predates clang-format
  # (it was shaped by an old Visual Studio formatter) and does not round-trip,
  # so we never reformat the whole tree -- only the lines a PR touches.
  format:
    name: format (clang-format-19, changed lines)
    if: github.event_name == 'pull_request'
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@v6
        with:
          fetch-depth: 0

      - name: Install clang-format 19 (pinned, from apt.llvm.org)
        run: |
          set -euo pipefail
          # ubuntu-24.04's native clang-format is 18; pin 19 to match local dev.
          wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key \
            | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc >/dev/null
          echo "deb http://apt.llvm.org/noble/ llvm-toolchain-noble-19 main" \
            | sudo tee /etc/apt/sources.list.d/llvm-19.list >/dev/null
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends clang-format-19
          # The clang-format-19 package ships the git-clang-format driver;
          # expose it unsuffixed so "git clang-format" finds it.
          sudo ln -sf /usr/bin/git-clang-format-19 /usr/local/bin/git-clang-format
          clang-format-19 --version

      - name: Check formatting of changed lines
        run: |
          set -euo pipefail
          git fetch --no-tags origin \
            "+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
          base="origin/${{ github.base_ref }}"
          set +e
          diff="$(git clang-format --binary clang-format-19 --style=file \
                    --diff --extensions c,h "$base")"
          rc=$?
          set -e
          # Classify by output, not exit code: a non-empty diff means "not
          # clean" (git-clang-format may exit 0 or 1 on a diff). A nonzero exit
          # with clean output is a real checker error.
          case "$diff" in
            "" | "no modified files to format" | *"did not modify any files"*)
              if [ "$rc" -ne 0 ]; then
                echo "::error::git clang-format failed (exit $rc): checker error."
                exit 1
              fi
              echo "Formatting OK: changed C lines are clang-format-clean." ;;
            *)
              echo "$diff"
              echo "::error::Changed C lines are not clang-format-clean."
              echo "Fix locally with: git clang-format --binary clang-format-19 $base"
              exit 1 ;;
          esac