Read the .arc bytes with od instead of piping into head

`tail -c +N file | head -c 3` leaves tail writing into a pipe head has already
closed. Under this test's `set -o pipefail` that SIGPIPE becomes the script's
exit status, so 154 dies at 141 having printed nothing. GNU coreutils happens
to survive it, which is why the red only shows on the uutils leg. od seeks to
the offset itself and reads no more than it needs, so no second process is
left holding the other end.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.gitattributes

@@ -7,5 +7,6 @@
 *.test text eol=lf
 *.sh text eol=lf
 
-# Scoped to this file so a union can never touch build logic.
-tests/tests-list.mk merge=union
+# Scoped to this file so a union can never touch build logic. LF because a CRLF
+# checkout would make every line of it a stray to check-tests-list.sh.
+tests/tests-list.mk merge=union text eol=lf

.github/workflows/ci.yml

@@ -41,7 +41,8 @@ jobs:
           sudo apt-get update
           sudo apt-get install -y --no-install-recommends \
             build-essential clang autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
+            zlib1g-dev libssl-dev libbrotli-dev libzstd-dev \
+            appstream appstream-util
 
       - name: Configure
         run: |
@@ -137,6 +138,66 @@ jobs:
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # The Ubuntu buildds run a development release every runner above is well behind.
+  # That gap is what shipped 3.49.18 broken on every Ubuntu architecture but
+  # riscv64, when 25.10 made uutils the default coreutils (#1042).
+  ubuntu-devel:
+    name: build (Ubuntu devel, uutils coreutils)
+    runs-on: ubuntu-24.04
+    container: ubuntu:devel
+    timeout-minutes: 40
+    steps:
+      # Before checkout: the container has no git, so submodules would be lost.
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          apt-get update
+          DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+            build-essential autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev libbrotli-dev libzstd-dev \
+            git ca-certificates python3 procps
+
+      - name: Assert the coreutils dispatch on argv[0]
+        # The property #1042 broke on, not the version banner, which uutils has
+        # already reworded once. Without it the job quietly becomes a slower copy
+        # of the one above.
+        run: |
+          set -euo pipefail
+          ln -s "$(command -v sleep)" /tmp/not-a-coreutil
+          if /tmp/not-a-coreutil 0; then
+            echo "::error::coreutils here take any argv[0]; this image no longer guards uutils" >&2
+            exit 1
+          fi
+
+      - uses: actions/checkout@v7
+        with:
+          submodules: recursive
+
+      - name: Configure
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure
+          # A renamed dev package would disable a codec or TLS here, and the tests
+          # that cover them would SKIP rather than fail.
+          grep -q "define HTS_USEBROTLI 1" config.h
+          grep -q "define HTS_USEZSTD 1" config.h
+          grep -q "define HTS_USEOPENSSL 1" config.h
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Test
+        timeout-minutes: 25
+        run: |
+          set -euo pipefail
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
+
+      - name: Print the test log on failure
+        if: failure()
+        run: cat tests/test-suite.log 2>/dev/null || true
+
   # Portability: build and test on macOS (Darwin/clang) on a native runner --
   # no VM. The tree has no __APPLE__ branches, so Darwin exercises the
   # generic-Unix path on a second libc and kernel. brew's openssl@3 is keg-only,
@@ -510,6 +571,33 @@ jobs:
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
         run: bash fuzz/run-fuzzers.sh fuzz check
 
+  # Catches #1030: the job above links ASan, whose runtime already defines the
+  # sancov hooks the instrumentation calls, hiding a program left without one.
+  fuzz-no-sanitizer:
+    name: build (fuzzers, no sanitizer runtime, clang)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential clang autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev libbrotli-dev libzstd-dev
+
+      - name: Configure (fuzzers, static)
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure CC=clang --enable-fuzzers --disable-shared
+
+      - name: Build
+        run: make -j"$(nproc)"
+
   # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
   # the configuration users on minimal systems build, and one libssl is not even
   # installed here so configure cannot silently re-enable it. The matrix above
@@ -715,7 +803,8 @@ jobs:
           sudo apt-get update
           # noble ships shfmt 3.8.0 (universe), matching the pinned local dev
           # version; use it rather than fetching a release binary from github.com.
-          sudo apt-get install -y --no-install-recommends shellcheck shfmt appstream
+          sudo apt-get install -y --no-install-recommends shellcheck shfmt appstream \
+            appstream-util
           shfmt --version
           appstreamcli --version
 
@@ -725,6 +814,11 @@ jobs:
       - name: shfmt
         run: shfmt -d -i 4 $SHELL_SCRIPTS
 
+      # A registration whose file is gone stops "make check" before it reports
+      # anything, so catch it here rather than as a missing line in a build log.
+      - name: tests-list.mk registrations
+        run: bash tests/check-tests-list.sh
+
       # MSBuild rejects a malformed .vcxproj with a bare MSB4025 and no build, so
       # catch it here in seconds rather than on a Windows runner minutes in.
       - name: XML well-formedness (MSBuild project files)
@@ -742,6 +836,8 @@ jobs:
           set -euo pipefail
           appstreamcli validate --no-net --explain \
             html/server/div/com.httrack.WebHTTrack.metainfo.xml
+          appstream-util validate-relax --nonet \
+            html/server/div/com.httrack.WebHTTrack.metainfo.xml
 
   # Check clang-format on CHANGED LINES ONLY. The engine predates clang-format
   # (it was shaped by an old Visual Studio formatter) and does not round-trip,

.github/workflows/cross-arch.yml

@@ -0,0 +1,116 @@
+# Cross-compiles for the Debian architectures no GitHub runner exists for.
+# 3.49.17 failed on four of them at once and every failure was a compile or
+# assemble error that x86-64 and arm64 cannot produce, so the buildd was the
+# first thing to see the code.
+name: Cross-arch
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+  workflow_dispatch:
+
+# Least privilege: the workflow only needs to read the repo.
+permissions:
+  contents: read
+
+# Cancel superseded runs on the same branch or PR.
+concurrency:
+  group: cross-arch-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  cross:
+    name: cross (${{ matrix.arch }})
+    runs-on: ubuntu-24.04
+    # sid, not stable: the point is to predict what the sid buildds will say.
+    container: debian:sid
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          # arch selects the hardening flags, triplet is the toolchain prefix.
+          # The first four are 3.49.17's failures.
+          - { arch: armhf, triplet: arm-linux-gnueabihf }
+          - { arch: powerpc, triplet: powerpc-linux-gnu }
+          - { arch: hppa, triplet: hppa-linux-gnu }
+          - { arch: loong64, triplet: loongarch64-linux-gnu }
+          - { arch: sh4, triplet: sh4-linux-gnu }
+          - { arch: m68k, triplet: m68k-linux-gnu }
+          - { arch: sparc64, triplet: sparc64-linux-gnu }
+          - { arch: riscv64, triplet: riscv64-linux-gnu }
+          - { arch: s390x, triplet: s390x-linux-gnu }
+    env:
+      DEB_HOST_ARCH: ${{ matrix.arch }}
+      TRIPLET: ${{ matrix.triplet }}
+      ZPREFIX: /tmp/zlib-${{ matrix.arch }}
+    steps:
+      - name: Install the toolchain
+        run: |
+          set -euo pipefail
+          # deb-src, so zlib below comes from the archive and not a pinned URL.
+          # In place: a second stanza has to repeat Signed-By, and apt rejects
+          # the same source carrying two spellings of the keyring.
+          sed -i 's/^Types: deb$/Types: deb deb-src/' \
+            /etc/apt/sources.list.d/debian.sources
+          apt-get update
+          apt-get install -y --no-install-recommends \
+            build-essential autoconf automake libtool autoconf-archive \
+            dpkg-dev git ca-certificates \
+            "gcc-$TRIPLET" "libc6-dev-$DEB_HOST_ARCH-cross"
+
+      - uses: actions/checkout@v7
+        with:
+          submodules: recursive
+
+      - name: Cross-build zlib
+        run: |
+          set -euo pipefail
+          # Mandatory, and no ports arch has a zlib1g-dev:<arch> to
+          # multiarch-install. -fPIC because libhttrack.so links it.
+          mkdir -p /tmp/zsrc && cd /tmp/zsrc
+          # Unpatched: Debian's arch-specific patches expect its own rules to
+          # build them (s390x's vector CRC pulls a header they add), and we only
+          # need something to link -lz against.
+          apt-get source --download-only zlib1g
+          dpkg-source --skip-patches -x ./*.dsc src
+          cd src
+          CHOST="$TRIPLET" CC="$TRIPLET-gcc" CFLAGS="-O2 -fPIC" \
+            ./configure --prefix="$ZPREFIX" --static
+          # configure detects s390x's vector CRC, whose sources live in the
+          # contrib/ the .dfsg repack drops: drop the define and the object it
+          # would build. Nothing here needs a fast CRC.
+          sed -i -E 's/(-DHAVE_S390X_VX|crc32_vx\.l?o)//g' Makefile
+          make -j"$(nproc)"
+          make install
+
+      - name: Configure
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          mkdir -p /tmp/bld && cd /tmp/bld
+          # The buildd's own per-arch flags: -D_FILE_OFFSET_BITS=64 is the shim
+          # hazard, and -fstack-clash-protection is not offered everywhere.
+          eval "$(dpkg-buildflags --export=sh)"
+          # A cross AC_TRY_RUN answers "cross", which would swap in the bundled
+          # snprintf that no buildd ever builds. Every glibc passes these.
+          # --disable-https: no ports arch has a cross libssl, so the TLS paths
+          # are the one part of the tree this matrix does not compile.
+          "$GITHUB_WORKSPACE/configure" --host="$TRIPLET" \
+            --build="$(dpkg-architecture -qDEB_BUILD_GNU_TYPE)" --disable-https \
+            ac_cv_have_working_snprintf=yes ac_cv_have_working_vsnprintf=yes \
+            CPPFLAGS="${CPPFLAGS:-} -I$ZPREFIX/include" \
+            LDFLAGS="${LDFLAGS:-} -L$ZPREFIX/lib"
+
+      - name: Build
+        run: make -C /tmp/bld -j"$(nproc)"
+
+      - name: Build the test artifacts
+        # Nothing runs here, but the shims are where three of the four 3.49.17
+        # failures were: they only build under `check`. Asserted, because an
+        # empty TESTS= would otherwise pass having built nothing.
+        run: |
+          set -euo pipefail
+          make -C /tmp/bld -j"$(nproc)" check TESTS=
+          test -f /tmp/bld/tests/.libs/libaltstackprobe.so

.github/workflows/windows-build.yml

@@ -14,6 +14,9 @@ on:
 
 permissions:
   contents: read
+  # The suite watchdog reports as a commit status, the only channel that
+  # outlives the runner it is reporting on (#795).
+  statuses: write
 
 # Cancel superseded runs on the same branch or PR.
 concurrency:
@@ -37,6 +40,9 @@ jobs:
       - uses: actions/checkout@v7
         with:
           submodules: recursive   # coucal lives in src/coucal
+          # Or the job token stays in .git/config, where every test the suite
+          # runs can read it.
+          persist-credentials: false
 
       # Located through vswhere rather than microsoft/setup-msbuild: the repo
       # only allows GitHub-owned actions.
@@ -171,6 +177,14 @@ jobs:
         shell: bash
         working-directory: tests
         timeout-minutes: 45
+        env:
+          # Through the environment, never argv, which the process list exposes.
+          WATCHDOG_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          WATCHDOG_REPO: ${{ github.repository }}
+          # github.sha here is the PR's merge commit, so statuses posted against it stay out of the PR's checks UI.
+          WATCHDOG_SHA: ${{ github.sha }}
+          WATCHDOG_CONTEXT: windows-suite (${{ matrix.platform }}, ${{ matrix.configuration }})
+          WATCHDOG_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
         run: |
           bash ./ci-windows-suite.sh \
             "$(cygpath -u "$GITHUB_WORKSPACE")/src/${{ matrix.platform }}/${{ matrix.configuration }}"

AGENTS.md

@@ -20,7 +20,9 @@ the operational checklist: toolchain, invariants, and how to ship a change.
   `-c16` bigcrawl the way Python's default backlog of 5 did.
   Or run `sh build.sh` to do bootstrap + configure + make in one shot.
 - A `tests/NN_*.test` runs only if listed in `tests/tests-list.mk`; an
-  unregistered file is silently skipped.
+  unregistered file is silently skipped. A registration whose file is gone makes
+  `make check` exit 2 with no `# TOTAL:` line. `tests/check-tests-list.sh` (CI
+  lint and `231_tests-list-bijection.test`) enforces the pairing both ways.
 - `make check` prepends the build's `src/` to `PATH`, but a hand-run `.test` does
   not — an installed `/usr/bin/httrack` then shadows your build. Run via `make
   check`, or `PATH="<bld>/src:$PATH"` for a manual run.
@@ -38,6 +40,10 @@ the operational checklist: toolchain, invariants, and how to ship a change.
   match that is not on the last line is exposed. Capture the reply, assert the
   status line it must carry (an empty, truncated or redirected one is
   marker-free too), then match with a here-string: `grep -q M <<<"$reply"`.
+  `head -c N` and `head -n N` end the same way, and there the reds are
+  platform-specific: GNU `tail | head -c 3` survives, the uutils coreutils leg
+  turns the same line into exit 141 with no output at all. Let the reader seek
+  instead: `od -An -c -j <skip> -N <len> file`.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.17], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.18], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,6 +29,9 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
+# 3:11:0: revision-only bump. #1001 moved SOCaddr_inetntoa_ out of htsnet.h, adding
+# an export where callers had an inline; nothing changed or went away.
+# 3:10:0: revision-only bump; only the version macro moved, the engine is untouched.
 # 3:9:0: revision-only bump. #991 and #1005 each added an export
 # (hts_set_thread_hooks, escape_control_url); nothing changed or went away.
 # 3:8:0: revision-only bump, no ABI change.
@@ -37,7 +40,7 @@ AM_INIT_AUTOMAKE([subdir-objects])
 # moves nothing). Soname stays .so.3: HTTrackQt is the only consumer of the installed
 # headers, so a libhttrack4 rename isn't worth it.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:9:0"
+VERSION_INFO="3:11:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 
@@ -392,7 +395,10 @@ AC_CHECK_TYPE(sa_family_t, [], [AC_DEFINE([sa_family_t], [uint16_t], [sa_family_
 AX_CHECK_ALIGNED_ACCESS_REQUIRED
 
 # check for various headers
-AC_CHECK_HEADERS([execinfo.h sys/ioctl.h])
+AC_CHECK_HEADERS([execinfo.h sys/ioctl.h sys/random.h])
+
+## CSPRNG for the --single-file mark; /dev/urandom is the fallback
+AC_CHECK_FUNCS([getrandom])
 
 ### zlib (mandatory)
 CHECK_ZLIB()
@@ -547,7 +553,7 @@ AC_DEFINE(HAVE_STRNLEN, 1,[Check for strnlen])], AC_MSG_RESULT([not found]))
 ## Online unit tests
 AC_MSG_CHECKING(whether to enable online unit tests)
 AC_ARG_ENABLE([online-unit-tests],
-	[AS_HELP_STRING([--enable-online-unit-tests=@<:@yes/no/auto@:>@],[Enable online-unit-tests @<:@default=yes@:>@])],
+	[AS_HELP_STRING([--enable-online-unit-tests=@<:@yes/no/auto@:>@],[Enable online-unit-tests @<:@default=no@:>@])],
 	[
 	case "${enableval}" in
 	no|yes|auto)
@@ -589,6 +595,10 @@ if test x"$fuzzers" = x"yes"; then
 	AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
 		[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
 		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
+	# Ordinary programs are instrumented too, so they need that runtime at link.
+	AX_CHECK_LINK_FLAG([-fsanitize=fuzzer-no-link],
+		[DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -fsanitize=fuzzer-no-link"],
+		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
 	# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
 	if test x"$enable_shared" != x"no"; then
 		AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])

debian/changelog

@@ -1,3 +1,15 @@
+httrack (3.49.18-1) unstable; urgency=medium
+
+  * New upstream release, fixing the 3.49.17-1 build failures on armhf,
+    powerpc, hppa and loong64: the test suite's mmap interposer did not
+    compile on the first three, and the alternate-stack test insisted on a
+    longer backtrace than loong64's unwinder produces. Nothing in the engine
+    changed. Upstream now cross-builds the test shims for the ports
+    architectures on every push, so the next such break surfaces before the
+    buildds do.
+
+ -- Xavier Roche <xavier@debian.org>  Wed, 05 Aug 2026 23:00:24 +0200
+
 httrack (3.49.17-1) unstable; urgency=medium
 
   * New upstream release: security fixes in the FTP and HTTP request paths and

fuzz/Makefile.am

@@ -58,7 +58,9 @@ EXTRA_DIST = README.md run-fuzzers.sh \
 	corpus/singlefile/img-src.html corpus/singlefile/link-rel.html \
 	corpus/singlefile/style-block.html corpus/singlefile/style-attr.html \
 	corpus/singlefile/srcset.html corpus/singlefile/rawtext.html \
-	corpus/singlefile/malformed.html corpus/singlefile/many-attrs.html \
+	corpus/singlefile/malformed.html \
+	corpus/singlefile/mark-at-eof.html corpus/singlefile/mark-only.html \
+	corpus/singlefile/mark-degenerate.html corpus/singlefile/traversal.html \
 	corpus/sitemap/urlset.xml corpus/sitemap/sitemapindex.xml \
 	corpus/sitemap/truncated.xml corpus/sitemap/urlset.xml.gz \
 	corpus/arc/roundtrip.arc corpus/arc/truncated.arc \

fuzz/corpus/singlefile/img-src.html

@@ -1,3 +1,4 @@
-<img src="a.png">
-<img src=big.png alt=over-cap>
-<img src="../escape.png"><img src="/abs.png"><img src="data:,x">
+<img src="
a.png">
+<img src=
big.png alt=over-cap>
+<img src="
../escape.png"><img src="
/abs.png">
+<img src="data:,
x">

fuzz/corpus/singlefile/link-rel.html

@@ -1,4 +1,3 @@
-<link rel="stylesheet" href="s.css">
-<link rel=icon href=a.png>
-<link rel="next" href="p2.html">
-<link rel="preload" href="j.js">
+<link rel="stylesheet" href="
s.css">
+<link rel="icon" href="
a.png">
+<link rel="canonical" href="
other.html">

fuzz/corpus/singlefile/malformed.html

@@ -1,5 +1,3 @@
-<img src="unterminated.png
-<div style="background:url(a.png">
-<style>@import url(
-<!-- unterminated comment
-<a href=
+<p a=1 b=2 c=3 d=4 e=5 f=6 g=7 h=8 i=9 j=10 title="> <img src=
a.png>">x</p>
+<!--><img src="
a.png">
+<img src="
a.png" >

fuzz/corpus/singlefile/many-attrs.html

@@ -1 +0,0 @@
-<img src="a.png" a0="v" a1="v" a2="v" a3="v" a4="v" a5="v" a6="v" a7="v" a8="v" a9="v" a10="v" a11="v" a12="v" a13="v" a14="v" a15="v" a16="v" a17="v" a18="v" a19="v" a20="v" a21="v" a22="v" a23="v" a24="v" a25="v" a26="v" a27="v" a28="v" a29="v" a30="v" a31="v" a32="v" a33="v" a34="v" a35="v" a36="v" a37="v" a38="v" a39="v" a40="v" a41="v" a42="v" a43="v" a44="v" a45="v" a46="v" a47="v" a48="v" a49="v" a50="v" a51="v" a52="v" a53="v" a54="v" a55="v" a56="v" a57="v" a58="v" a59="v" a60="v" a61="v" a62="v" a63="v" a64="v" a65="v" a66="v" a67="v" a68="v" a69="v">

fuzz/corpus/singlefile/mark-at-eof.html

@@ -0,0 +1 @@
+
a.png

fuzz/corpus/singlefile/mark-degenerate.html

@@ -0,0 +1,5 @@
+



+=
=
+"
"
+a.png#!htsinlin
+#!htsinlin

fuzz/corpus/singlefile/mark-only.html

@@ -0,0 +1 @@
+


fuzz/corpus/singlefile/rawtext.html

@@ -1,4 +1,3 @@
-<script>var s="</scripting>"; if(a</b) x=1;</script>
-<script src="j.js"></script>
-<textarea></textareas></textarea>
-<title></titles></title>
+<script src="
j.js"></script>
+<script>var u="
a.png";</script>
+<textarea>
a.png</textarea>

fuzz/corpus/singlefile/srcset.html

@@ -1,2 +1 @@
-<img srcset="a.png 1x, big.png 2x, a.png 100w">
-<source srcset="a.png,, a.png 2x," src="a.png">
+<img srcset="
a.png 1x, 
big.png 2x, 
a.png 480w">

fuzz/corpus/singlefile/style-attr.html

@@ -1,2 +1,2 @@
-<div style="background:url(a.png);list-style:url('a.png')"></div>
-<p style='background:url("a.png")'>x</p>
+<div style="background:url(
a.png)"></div>
+<div style='background:url("
a.png")'></div>

fuzz/corpus/singlefile/style-block.html

@@ -1,5 +1,4 @@
-<style>@import "s.css";
-@import url(sub/b.css);
-div{background:url(a.png)}
-/* url(a.png) */ p:after{content:"url(a.png)"}
+<style>@import "
s.css";
+@import url(
sub/b.css);
+div{background:url(
a.png)}
 </style>

fuzz/corpus/singlefile/traversal.html

@@ -0,0 +1,4 @@
+<img src="
../../../../etc/passwd">
+<img src="
a.png#frag?q=1">
+<img src="
sub/../a.png">
+<img src="
./a.png">

fuzz/fuzz-singlefile.c

@@ -60,6 +60,10 @@ static void sf_cleanup(void) {
   (void) remove(sf_root);
 }
 
+/* One httrackp for the whole run: the mark secret lives on it, so a fresh one
+   per input would make every mark in the corpus unrecognisable. */
+static httrackp *sf_opt = NULL;
+
 /* A missing asset would silently reduce the target to its parser half. */
 static void sf_write(const char *name, const char *data, size_t len) {
   char path[700];
@@ -76,6 +80,48 @@ static void sf_text(const char *name, const char *data) {
   sf_write(name, data, strlen(data));
 }
 
+/* Append ref plus a real mark for it. The secret is drawn per httrackp, so a
+   fixture cannot spell one; every mark the target sees is built here. */
+static void sf_marked(String *out, const char *ref, char cls) {
+  char mark[SINGLEFILE_MARK_MAX];
+
+  StringCat(*out, ref);
+  StringCat(*out,
+            singlefile_mark(sf_opt, mark, sizeof(mark), cls, strlen(ref)));
+}
+
+/* \001<ref>\002 in an input becomes <ref> plus its mark, which is the only
+   way a corpus file can reach the mark parser at all. */
+static void sf_expand_input(const char *in, size_t len, String *out) {
+  size_t i, start = 0;
+
+  StringClear(*out);
+  for (i = 0; i < len; i++) {
+    if (in[i] == '\001') {
+      start = StringLength(*out);
+    } else if (in[i] == '\002') {
+      char mark[SINGLEFILE_MARK_MAX];
+
+      StringCat(*out, singlefile_mark(sf_opt, mark, sizeof(mark),
+                                      SINGLEFILE_CLASS_ANY,
+                                      StringLength(*out) - start));
+    } else {
+      StringAddchar(*out, in[i]);
+    }
+  }
+}
+
+static void sf_css(const char *name, const char *pre, const char *ref,
+                   const char *post) {
+  String body = STRING_EMPTY;
+
+  StringCopy(body, pre);
+  sf_marked(&body, ref, SINGLEFILE_CLASS_ANY);
+  StringCat(body, post);
+  sf_write(name, StringBuff(body), StringLength(body));
+  StringFree(body);
+}
+
 static void sf_init(void) {
   static const char png[] = "\x89PNG\r\n\x1a\n";
   static const char big[4096] = "\x89PNG";
@@ -83,6 +129,9 @@ static void sf_init(void) {
   char path[700];
 
   hts_init();
+  sf_opt = hts_create_opt();
+  sf_opt->log = sf_opt->errlog = NULL;
+  sf_opt->single_file_max_size = FUZZ_SF_CAP;
   snprintf(sf_root, sizeof(sf_root), "%s/httrack-fuzz-sf-XXXXXX",
            tmp != NULL && tmp[0] != '\0' ? tmp : "/tmp");
   if (mkdtemp(sf_root) == NULL)
@@ -95,37 +144,50 @@ static void sf_init(void) {
   sf_write("a.png", png, sizeof(png) - 1);
   sf_write("big.png", big, sizeof(big));
   sf_text("j.js", "var x=1;\n");
-  /* @import plus a url(), so an inlined stylesheet recurses and its own
-     relative reference is rebased. */
-  sf_text("s.css", "@import url(sub/b.css);\ndiv{background:url(a.png)}\n");
-  sf_text("sub/b.css", "p{background:url(../a.png)}\n");
+  /* Marked, so an inlined stylesheet recurses into its own marks and its
+     un-inlinable reference is rebased; unmarked assets leave the target as a
+     bare scan that reaches nothing. */
+  {
+    String css = STRING_EMPTY;
+
+    StringCopy(css, "@import url(");
+    sf_marked(&css, "sub/b.css", SINGLEFILE_CLASS_CSS);
+    StringCat(css, ");\ndiv{background:url(");
+    sf_marked(&css, "a.png", SINGLEFILE_CLASS_ANY);
+    StringCat(css, ")}\np{background:url(");
+    sf_marked(&css, "big.png", SINGLEFILE_CLASS_ANY);
+    StringCat(css, ")}\n");
+    sf_write("s.css", StringBuff(css), StringLength(css));
+    StringFree(css);
+  }
+  sf_css("sub/b.css", "p{background:url(", "../a.png", ")}\n");
 }
 
 int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   static int inited = 0;
 
   String out = STRING_EMPTY;
-  httrackp *opt;
-  /* Exact-length, unterminated: the rewriter is span-based, so ASan bounds a
-     read past html_len instead of it landing on a terminator. */
-  char *html = malloct(size != 0 ? size : 1);
+  String in = STRING_EMPTY;
+  char *html;
+  size_t html_len;
 
   if (!inited) {
     sf_init();
     inited = 1;
   }
-  memcpy(html, data, size);
-
-  opt = hts_create_opt();
-  opt->log = opt->errlog = NULL;
-  opt->single_file_max_size = FUZZ_SF_CAP;
+  sf_expand_input((const char *) data, size, &in);
+  html_len = StringLength(in);
+  /* Exact-length, unterminated: the rewriter is span-based, so ASan bounds a
+     read past html_len instead of it landing on a terminator. */
+  html = malloct(html_len != 0 ? html_len : 1);
+  memcpy(html, StringBuff(in), html_len);
+  StringFree(in);
 
   StringClear(out);
-  (void) singlefile_rewrite_html(opt, sf_root, sf_page, html, size,
+  (void) singlefile_rewrite_html(sf_opt, sf_root, sf_page, html, html_len,
                                  SINGLEFILE_MAX_PAGE_SIZE, &out);
 
   StringFree(out);
   freet(html);
-  hts_free_opt(opt);
   return 0;
 }

history.txt

@@ -4,6 +4,11 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
+3.49-18
++ Fixed: the 3.49.17 package build failed on four architectures: the test suite's mmap interposer did not compile on armhf, powerpc or hppa, and the alternate-stack test rejected loong64's shorter backtrace (#1023)
++ Fixed: the test suite failed on a host with no ps command, such as a Fedora build root; its hang diagnostics now read /proc directly (#1021)
++ Changed: multiple internal test and CI improvements, including a cross-compile matrix covering the Debian ports architectures
+
 3.49-17
 + New: a pkg-config file, libhttrack.pc, ships with the development headers (#1018)
 + Fixed: a crawled page could read the WebHTTrack session id and drive the control panel; the id is now unguessable and cross-origin commands are refused (#877)

html/Makefile.am

@@ -15,7 +15,7 @@ WebIcon64x64dir = $(datadir)/icons/hicolor/64x64/apps
 WebIcon128x128dir = $(datadir)/icons/hicolor/128x128/apps
 WebIcon256x256dir = $(datadir)/icons/hicolor/256x256/apps
 WebIconScalabledir = $(datadir)/icons/hicolor/scalable/apps
-VFolderEntrydir = $(prefix)/share/applications
+VFolderEntrydir = $(datadir)/applications
 MetaInfodir = $(datadir)/metainfo
 
 # Wildcards are globbed against $(srcdir): a bare "*.html" is resolved against

html/changes.html

@@ -155,6 +155,8 @@ rather than on <tt>url</tt> when a mirror is known to carry legacy-charset URLs.
 
 <h3 id="example">Example</h3>
 
+One captured run; <tt>generator</tt> and <tt>date</tt> will read differently in yours.
+
 <pre>
 {
   "schema": 1,

html/server/div/com.httrack.WebHTTrack.metainfo.xml

@@ -26,7 +26,6 @@
     </ul>
   </description>
   <launchable type="desktop-id">WebHTTrack.desktop</launchable>
-  <icon type="stock">httrack</icon>
   <categories>
     <category>Network</category>
   </categories>
@@ -51,6 +50,13 @@
   <content_rating type="oars-1.1"/>
   <!-- Newest first; tests/01_engine-version-macros.test enforces it. -->
   <releases>
+    <release version="3.49.18" date="2026-08-05">
+      <description>
+        <ul>
+          <li>The program itself is unchanged: this release only fixes building HTTrack from source on some Linux distributions and processors</li>
+        </ul>
+      </description>
+    </release>
     <release version="3.49.17" date="2026-08-05">
       <description>
         <ul>

html/server/div/httrack16x16.xpm

@@ -1,37 +1,24 @@
 /* XPM */
 static char *httrack16x16[] = {
 /* columns rows colors chars-per-pixel */
-"16 16 15 1 ",
-"  c #0D0D10",
-". c #191921",
-"X c #2A2A38",
-"o c #363647",
-"O c #424257",
-"+ c #53536E",
-"@ c #717197",
-"# c #7E7EA8",
-"$ c #8585B1",
-"% c #9898CB",
-"& c #9999CC",
-"* c #9A9ACE",
-"= c #9696C9",
-"- c #9C9CD0",
-"; c #9999CC",
+"16 16 2 1 ",
+"  c #040404",
+". c #9999CC",
 /* pixels */
-"&&&&&&&&&%%&&&&&",
-"&&&&&&&&%%%%&&&&",
-"&&&&&&&&%%%%&&&&",
-"&%%&&&%%%%%%%%%&",
-"%+o$&%+o@ooooo+%",
-"%o $-%X @X.  .O%",
-"%o @$$X $=@ o$=&",
-"%o  .   $-# o&&&",
-"%o oOo. $-# o%&&",
-"%o $-%X $-# o%&&",
-"%X $&%X $-# o%&&",
-"%+o$&%+o$&$o+%&&",
-"%%%&&&%%&&&%%&&&",
-"%%&&&&&&&&&&&&&&",
-"%&&&&&&&&&&&&&&&",
-"&&&&&&&&&&&&&&&&"
+"................",
+"................",
+"................",
+"................",
+".  ...  .      .",
+".  ...  .      .",
+".  ...  ...  ...",
+".       ...  ...",
+".       ...  ...",
+".  ...  ...  ...",
+".  ...  ...  ...",
+".  ...  ...  ...",
+"................",
+"................",
+"................",
+"................"
 };

lang/Polski.txt

@@ -539,9 +539,9 @@ U
 TimeOut(s)
 Przekroczenie(a) czasu oczekiwania
 Persistent connections (Keep-Alive)
-Podtrzymywane połączenia (Keep-Alive)
+Podtrzymywane po³¹czenia (Keep-Alive)
 Reduce connection time and type lookup time using persistent connections
-Zmniejsz czas połączenia i czas sprawdzania typu dzięki użyciu podtrzymywanych połączeń
+Zmniejsz czas po³¹czenia i czas sprawdzania typu dziêki u¿yciu podtrzymywanych po³¹czeñ
 Retries
 Ponowienia
 Size limit

libtest/readme.txt

@@ -29,8 +29,7 @@ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 GNU General Public License for more details.
 
 You should have received a copy of the GNU General Public License
-along with this program; if not, write to the Free Software
-Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+along with this program. If not, see <http://www.gnu.org/licenses/>.
 
 ========================================================================
     MAKEFILE PROJECT : libtest Project Overview

src/Makefile.am

@@ -43,7 +43,8 @@ htsserver_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS) libhttrack.la
 proxytrack_LDADD = $(THREADS_LIBS) $(SOCKET_LIBS)
 
 httrack_CFLAGS = $(AM_CFLAGS) $(CFLAGS_PIE)
-proxytrack_CFLAGS = $(AM_CFLAGS) $(CFLAGS_PIE) -DNO_MALLOCT -DZLIB_CONST -DHTS_INTHASH_USES_MD5
+# -DHTS_NO_LIBHTTRACK: see htsnet.h's HTSNET_API guard.
+proxytrack_CFLAGS = $(AM_CFLAGS) $(CFLAGS_PIE) -DNO_MALLOCT -DZLIB_CONST -DHTS_INTHASH_USES_MD5 -DHTS_NO_LIBHTTRACK
 htsserver_CFLAGS = $(AM_CFLAGS) $(CFLAGS_PIE) -DZLIB_CONST -DHTS_INTHASH_USES_MD5
 
 # @RPATH_ORIGIN_LDFLAGS@ makes a copied tree find libhttrack next to it (#906),
@@ -55,11 +56,12 @@ htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE) @RPATH_ORIGIN_LDFLAGS@
 lib_LTLIBRARIES = libhttrack.la
 
 htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h htsstats.h \
+	htsrandom.c htsrandom.h \
 	htscmdline.c htscmdline.h \
 	htsurlport.c htsurlport.h
 proxytrack_SOURCES = proxy/main.c \
 	proxy/proxytrack.c proxy/store.c \
-	htsurlport.c htsurlport.h \
+	htsurlport.c htsurlport.h htsnet.c \
 	coucal/coucal.c htsmd5.c md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c
 
@@ -77,10 +79,10 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htscache_selftest.c htsdns_selftest.c htsselftest.c \
 	htscatchurl.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
 	htscmdline.c htshelp.c htslib.c htsurlport.c htscoremain.c \
-	htsname.c htsrobots.c htstools.c htswizard.c \
+	htsname.c htsrobots.c htstools.c htsrandom.c htswizard.c \
 	htsalias.c htsthread.c htsindex.c htsbauth.c \
 	htscrashtest.c \
-	htsmd5.c htscodec.c htswarc.c htschanges.c htssinglefile.c htssitemap.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
+	htsmd5.c htsnet.c htscodec.c htswarc.c htschanges.c htssinglefile.c htssitemap.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
 	htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
 	md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
@@ -91,7 +93,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
 	htsmodules.h htsname.h htsnet.h htssniff.h \
 	htsopt.h htsrobots.h htsthread.h  \
-	htscrashtest.h htstools.h htswizard.h htswrap.h htscodec.h htswarc.h htschanges.h htssinglefile.h htssitemap.h htsproxy.h htszlib.h  \
+	htscrashtest.h htstools.h htsrandom.h htswizard.h htswrap.h htscodec.h htswarc.h htschanges.h htssinglefile.h htssitemap.h htsproxy.h htszlib.h  \
 	htsstrings.h htsarrays.h httrack-library.h \
 	htscharset.h punycode.h htsencoding.h \
 	htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \

src/htsback.c

@@ -225,6 +225,9 @@ void back_delete_all(httrackp * opt, cache_back * cache, struct_back * sback) {
   if (sback != NULL) {
     int i;
 
+    /* An FTP worker writes through its slot until it returns, so nothing here
+       may wipe or free one under it. */
+    ftp_stop_workers();
     // delete live slots
     for(i = 0; i < sback->count; i++) {
       back_delete(opt, cache, sback, i);

src/htscoremain.c

@@ -1334,6 +1334,16 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   opt->mimehtml = 0;
                   com++;
                 }
+                if (opt->mimehtml && opt->single_file) {
+                  HTS_PANIC_PRINTF(
+                      "-%M and --single-file are two ways to make one "
+                      "self-contained file, so pick one: MIME (-%M) carries "
+                      "text parts without the base64 tax and stores a shared "
+                      "asset once; single-file HTML opens anywhere by "
+                      "double-click.");
+                  htsmain_free();
+                  return -1;
+                }
                 break;
               case 'k':
                 opt->nokeepalive = 0;
@@ -1945,6 +1955,16 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     com++;
                   }
                 }
+                if (opt->single_file && opt->mimehtml) {
+                  HTS_PANIC_PRINTF(
+                      "-%M and --single-file are two ways to make one "
+                      "self-contained file, so pick one: MIME (-%M) carries "
+                      "text parts without the base64 tax and stores a shared "
+                      "asset once; single-file HTML opens anywhere by "
+                      "double-click.");
+                  htsmain_free();
+                  return -1;
+                }
                 break;
               case 'm': // sitemap / sitemap-url: seed the crawl from sitemaps
                 if (*(com + 1) == 'u') { // --sitemap-url URL: explicit sitemap

src/htsftp.c

@@ -40,6 +40,9 @@ Please visit our Website: http://www.httrack.com
 
 #include "htscore.h"
 #include "htsthread.h"
+
+#include <limits.h>
+
 #ifdef _WIN32
 #else
 //inet_ntoa
@@ -60,12 +63,55 @@ Please visit our Website: http://www.httrack.com
 
 #if USE_BEGINTHREAD
 
+/* Live FTP workers. Each writes through its backlog slot and reads opt for its
+   whole run, so neither may be freed while the list is not empty (#1051).
+   Registration happens at spawn, not at thread entry: a worker that has not
+   been scheduled yet must already hold the list against a teardown. */
+static FTPDownloadStruct *ftp_workers = NULL;
+static htsmutex ftp_workers_mutex = HTSMUTEX_INIT;
+
+static void ftp_worker_register(FTPDownloadStruct *worker) {
+  hts_mutexlock(&ftp_workers_mutex);
+  worker->pNext = ftp_workers;
+  ftp_workers = worker;
+  hts_mutexrelease(&ftp_workers_mutex);
+}
+
+static void ftp_worker_unregister(FTPDownloadStruct *worker) {
+  FTPDownloadStruct **prev;
+
+  hts_mutexlock(&ftp_workers_mutex);
+  for (prev = &ftp_workers; *prev != NULL; prev = &(*prev)->pNext) {
+    if (*prev == worker) {
+      *prev = worker->pNext;
+      break;
+    }
+  }
+  hts_mutexrelease(&ftp_workers_mutex);
+}
+
+void ftp_stop_workers(void) {
+  int wait;
+
+  do {
+    FTPDownloadStruct *worker;
+
+    hts_mutexlock(&ftp_workers_mutex);
+    /* Idempotent, and re-raised each pass so a worker registered meanwhile is
+       not missed. A registered worker's slot is still live by construction:
+       back_delete_all() drains before back_free(). */
+    for (worker = ftp_workers; worker != NULL; worker = worker->pNext)
+      worker->pBack->stop_ftp = HTS_TRUE;
+    wait = ftp_workers != NULL;
+    hts_mutexrelease(&ftp_workers_mutex);
+    if (wait)
+      Sleep(100);
+  } while (wait);
+}
+
 void back_launch_ftp(void *pP) {
   FTPDownloadStruct *pStruct = (FTPDownloadStruct *) pP;
 
-  if (pStruct == NULL)
-    return;
-
   if (pStruct == NULL) {
 #if FTP_DEBUG
     printf("[ftp error: no args]\n");
@@ -84,11 +130,15 @@ void back_launch_ftp(void *pP) {
   // prêt
   pStruct->pBack->status = STATUS_FTP_READY;
 
-  /* Delete structure */
-  free(pP);
-
   /* Uninitialize */
   hts_uninit();
+
+  /* The status store above was this worker's last read of the slot and of opt;
+     deregistering lets the engine free both. */
+  ftp_worker_unregister(pStruct);
+
+  /* Delete structure */
+  free(pP);
   return;
 }
 
@@ -98,7 +148,16 @@ void launch_ftp(FTPDownloadStruct * params) {
 #if FTP_DEBUG
   printf("[Launching main ftp thread]\n");
 #endif
-  hts_newthread(back_launch_ftp, (void *) params);
+  ftp_worker_register(params);
+  if (hts_newthread(back_launch_ftp, (void *) params) != 0) {
+    /* Nobody would ever reap a slot left in STATUS_FTP_TRANSFER, and
+       ftp_stop_workers() would never return. */
+    ftp_worker_unregister(params);
+    strcpybuff(params->pBack->r.msg, "Unable to launch FTP thread");
+    params->pBack->r.statuscode = STATUSCODE_INVALID;
+    params->pBack->status = STATUS_FTP_READY;
+    free(params);
+  }
 }
 
 #else
@@ -122,42 +181,48 @@ void launch_ftp(FTPDownloadStruct * params) {
   return 0; \
   }
 
-/* Bounded split of a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
-void ftp_split_userpass(const char *src, const char *end, char *user,
-                        size_t user_size, char *pass, size_t pass_size) {
-  size_t n = 0;
+/* Split a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
+hts_boolean ftp_split_userpass(const char *src, const char *end, char *user,
+                               size_t user_size, char *pass, size_t pass_size) {
+  size_t len = 0, user_len, pass_len;
+  const char *colon;
 
   assertf(user_size > 0 && pass_size > 0); /* the size-1 math underflows on 0 */
+  assertf(end > src);                      /* end is one past the '@' */
 
-  while (src[n] != '\0' && src[n] != ':') {
-    if (n < user_size - 1)
-      user[n] = src[n];
-    n++;
-  }
-  user[n < user_size ? n : user_size - 1] = '\0';
-  pass[0] = '\0';
-  if (src[n] == ':') { // password follows the colon
-    const size_t base = n + 1;
-    size_t k = 0;
-
-    while (&src[base + k + 1] < end && src[base + k] != '\0') {
-      if (k < pass_size - 1)
-        pass[k] = src[base + k];
-      k++;
-    }
-    pass[k < pass_size ? k : pass_size - 1] = '\0';
-  }
+  user[0] = pass[0] = '\0'; // fail safe for a caller that ignores the result
+  while (len < (size_t) (end - src) - 1 && src[len] != '\0')
+    len++;
+  colon = memchr(src, ':', len);
+  user_len = colon != NULL ? (size_t) (colon - src) : len;
+  pass_len = colon != NULL ? len - user_len - 1 : 0;
+  /* a clipped name is another account's, so refuse rather than log in as it */
+  if (user_len >= user_size || pass_len >= pass_size)
+    return HTS_FALSE;
+  memcpy(user, src, user_len);
+  user[user_len] = '\0';
+  if (pass_len != 0)
+    memcpy(pass, colon + 1, pass_len);
+  pass[pass_len] = '\0';
+  return HTS_TRUE;
 }
 
 /* Build "<verb> <path>" (see htsftp.h). */
-void ftp_command(char *line, size_t line_size, const char *verb,
-                 const char *path) {
+hts_boolean ftp_command(char *line, size_t line_size, const char *verb,
+                        const char *path) {
+  int n;
+
   /* A leading '-' would reach a server that shells out to ls as a flag. */
   if (path[0] == '-' || strchr(path, ' ') != NULL ||
       strchr(path, '\"') != NULL || strchr(path, '\'') != NULL)
-    snprintf(line, line_size, "%s \"%s\"", verb, path);
+    n = snprintf(line, line_size, "%s \"%s\"", verb, path);
   else
-    snprintf(line, line_size, "%s %s", verb, path);
+    n = snprintf(line, line_size, "%s %s", verb, path);
+  if (n < 0 || (size_t) n >= line_size) {
+    line[0] = '\0'; // fail safe for a caller that ignores the result
+    return HTS_FALSE;
+  }
+  return HTS_TRUE;
 }
 
 /* MDTM reply "213 YYYYMMDDHHMMSS[.frac]" (RFC 3659, UTC) into tm_time. */
@@ -211,7 +276,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   httrackp *opt = pStruct->pOpt;
   char user[256] = "anonymous";
   char pass[256] = "user@";
-  char line_retr[2048];
+  char line_retr[FTP_LINE_SIZE];
   int port = 21;
 
 #if FTP_PASV
@@ -220,7 +285,9 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   char BIGSTK adr_ip[1024];
   char *adr, *real_adr;
   char BIGSTK ftp_path[CATBUFF_SIZE]; // the decoded URL path, screened once
-  int timeout = 300;            // timeout
+  /* opt->timeout == 0 means unlimited; use INT_MAX so 0 doesn't make the read
+     return instantly. */
+  const int timeout = opt->timeout > 0 ? opt->timeout : INT_MAX;
   int timeout_onfly = 8;        // attente réponse supplémentaire
   int transfer_list = 0;        // directory
   int rest_understood = 0;      // rest command understood
@@ -234,8 +301,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   //
   line_retr[0] = adr_ip[0] = ftp_path[0] = '\0';
 
-  timeout = 300;
-
   // effacer
   strcpybuff(back->r.msg, "");
   back->r.statuscode = 0;
@@ -251,7 +316,11 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   while(*real_adr == '/')
     real_adr++;                 // sauter /
   if ((adr = jump_identification(real_adr)) != real_adr) {      // user
-    ftp_split_userpass(real_adr, adr, user, sizeof(user), pass, sizeof(pass));
+    if (!ftp_split_userpass_buf(real_adr, adr, user, pass)) {
+      strcpybuff(back->r.msg, "FTP user name or password too long");
+      back->r.statuscode = STATUSCODE_INVALID;
+      _HALT_FTP return 0;
+    }
   }
   // Calculer RETR <nom>
   {
@@ -262,13 +331,19 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
       if (strnotempty(a)) {
         const size_t len_a =
             strlen(unescape_http(ftp_path, sizeof(ftp_path), a));
+        hts_boolean fits;
 
         if (len_a > 0 &&
             ftp_path[len_a - 1] == '/') { /* obviously a directory listing */
           transfer_list = 1;
-          ftp_command(line_retr, sizeof(line_retr), "LIST -A", ftp_path);
+          fits = ftp_command_line(line_retr, "LIST -A", ftp_path);
         } else {
-          ftp_command(line_retr, sizeof(line_retr), "RETR", ftp_path);
+          fits = ftp_command_line(line_retr, "RETR", ftp_path);
+        }
+        if (!fits) {
+          strcpybuff(back->r.msg, "FTP path too long");
+          back->r.statuscode = STATUSCODE_INVALID;
+          _HALT_FTP return 0;
         }
       } else {
         transfer_list = 1;
@@ -298,6 +373,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     SOCaddr server;
     char *a;
     char _adr[256];
+    size_t adr_len;
     const char *error = "unknown error";
 
     _adr[0] = '\0';
@@ -315,9 +391,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
         back->r.statuscode = STATUSCODE_INVALID; // permanent, unlike a DNS miss
         _HALT_FTP return 0;
       }
-      strncatbuff(_adr, adr, (int) (a - adr));
+      adr_len = (size_t) (a - adr);
     } else
-      strcpybuff(_adr, adr);
+      adr_len = strlen(adr);
+    // no resolvable name is this long, and clipping would query another host
+    if (adr_len >= sizeof(_adr)) {
+      htsblk_failf(&back->r, "Host name too long");
+      back->r.statuscode = STATUSCODE_INVALID;
+      _HALT_FTP return 0;
+    }
+    strncatbuff(_adr, adr, (int) adr_len);
 
     // récupérer adresse résolue
     strcpybuff(back->info, "host name");
@@ -356,19 +439,22 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     _CHECK_HALT_FTP;
 
     {
-      char BIGSTK line[1024];
+      char BIGSTK line[FTP_LINE_SIZE];
+
+      /* line_retr is copied here verbatim; a narrower line[] would clip it. */
+      HTS_COMPILE_ASSERT(sizeof(line) == sizeof(line_retr));
 
       // envoi du login
 
       // --USER--
-      get_ftp_line(soc_ctl, line, sizeof(line), timeout);     // en tête
+      get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt); // en tête
       _CHECK_HALT_FTP;
 
       if (line[0] == '2') {     // ok, connecté
         strcpybuff(back->info, "login: user");
         snprintf(line, sizeof(line), "USER %s", user);
         send_line(soc_ctl, line);
-        get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+        get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
         _CHECK_HALT_FTP;
         if ((line[0] == '3') || (line[0] == '2')) {
           // --PASS--
@@ -376,12 +462,12 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             strcpybuff(back->info, "login: pass");
             snprintf(line, sizeof(line), "PASS %s", pass);
             send_line(soc_ctl, line);
-            get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+            get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
             _CHECK_HALT_FTP;
           }
           if (line[0] == '2') { // ok
             send_line(soc_ctl, "TYPE I");
-            get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+            get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
             _CHECK_HALT_FTP;
             if (line[0] == '2') {
               // ok
@@ -414,7 +500,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
           strcpybuff(back->info, "pasv");
           snprintf(line, sizeof(line), "PASV");
           send_line(soc_ctl, line);
-          get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+          get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
         } else {                /* ipv6 */
           line[0] = '\0';
         }
@@ -476,7 +562,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
           strcpybuff(back->info, "pasv");
           snprintf(line, sizeof(line), "EPSV");
           send_line(soc_ctl, line);
-          get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+          get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
           _CHECK_HALT_FTP;
           if (line[0] == '2') { /* got it */
             char *a;
@@ -514,13 +600,12 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 #endif
           // SIZE
           if (back->r.statuscode != -1) {
-            if (!transfer_list) {
-              ftp_command(line, sizeof(line), "SIZE", ftp_path);
-
+            // a clipped probe would size and date a different file
+            if (!transfer_list && ftp_command_line(line, "SIZE", ftp_path)) {
               // SIZE?
               strcpybuff(back->info, "size");
               send_line(soc_ctl, line);
-              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+              get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
               _CHECK_HALT_FTP;
               if (line[0] == '2') {     // SIZE compris, ALORS tester REST (sinon pas tester: cf probleme des txt.gz decompresses a la volee)
                 char *szstr = strchr(line, ' ');
@@ -537,22 +622,24 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 }
 
                 // MDTM?
-                ftp_command(line, sizeof(line), "MDTM", ftp_path);
-                strcpybuff(back->info, "mdtm");
-                send_line(soc_ctl, line);
-                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-                _CHECK_HALT_FTP;
-                if (ftp_parse_mdtm(line, &remote_tm)) {
-                  char date[256];
+                if (ftp_command_line(line, "MDTM", ftp_path)) {
+                  strcpybuff(back->info, "mdtm");
+                  send_line(soc_ctl, line);
+                  get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
+                  _CHECK_HALT_FTP;
+                  if (ftp_parse_mdtm(line, &remote_tm)) {
+                    char date[256];
 
-                  time_rfc822(date, &remote_tm);
-                  /* Stamp the mirror as the HTTP path does, so a later pass
-                     compares server-clock times instead of crossing clocks. */
-                  back->r.lastmodified[0] = '\0';
-                  strlncatbuff(back->r.lastmodified, date,
-                               sizeof(back->r.lastmodified),
-                               sizeof(back->r.lastmodified) - 1);
-                  remote_mtime = timegm(&remote_tm);
+                    time_rfc822(date, &remote_tm);
+                    /* Stamp the mirror as the HTTP path does, so a later pass
+                       compares server-clock times instead of crossing clocks.
+                     */
+                    back->r.lastmodified[0] = '\0';
+                    strlncatbuff(back->r.lastmodified, date,
+                                 sizeof(back->r.lastmodified),
+                                 sizeof(back->r.lastmodified) - 1);
+                    remote_mtime = timegm(&remote_tm);
+                  }
                 }
 
                 /* Only over a copy back_add() judged partial: on --update every
@@ -564,13 +651,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   snprintf(line, sizeof(line), "REST " LLintP,
                            (LLint) back->range_req_size);
                   send_line(soc_ctl, line);
-                  get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+                  get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
                   _CHECK_HALT_FTP;
                   if ((line[0] == '3') || (line[0] == '2')) {   // ok
                     rest_understood = 1;
                   } // else never mind
                 }
-              }                 // sinon tant pis 
+              } // sinon tant pis
             }
           }
 #if FTP_PASV
@@ -613,9 +700,11 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               SOCaddr_initport(server, port_pasv);
               if (connect(soc_dat, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
                 strcpybuff(back->info, "retr");
-                strcpybuff(line, line_retr);
+                // clip, never abort: this line is built from a crawled URL
+                line[0] = '\0';
+                strlncatbuff(line, line_retr, sizeof(line), sizeof(line) - 1);
                 send_line(soc_ctl, line);
-                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+                get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
                 _CHECK_HALT_FTP;
                 if (line[0] == '1') {
                   // OK
@@ -656,13 +745,15 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
         if ((soc_servdat = get_datasocket(line, sizeof(line))) != INVALID_SOCKET) {
           _CHECK_HALT_FTP;
           send_line(soc_ctl, line);     // envoi du RETR
-          get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+          get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
           _CHECK_HALT_FTP;
           if (line[0] == '2') { // ok
             strcpybuff(back->info, "retr");
-            strcpybuff(line, line_retr);
+            // clip, never abort: this line is built from a crawled URL
+            line[0] = '\0';
+            strlncatbuff(line, line_retr, sizeof(line), sizeof(line) - 1);
             send_line(soc_ctl, line);
-            get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+            get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
             _CHECK_HALT_FTP;
             if (line[0] == '1') {
               //T_SOC soc_dat;
@@ -717,14 +808,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             while((len > 0) && (!stop_ftp(back))) {
               // attendre les données
               len = 1;          // pas d'erreur pour le moment
-              switch (wait_socket_receive(soc_dat, timeout)) {
+              switch (wait_socket_receive(back, soc_dat, timeout, opt)) {
               case -1:
                 strcpybuff(back->r.msg, "FTP read error");
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
               case 0:
-                htsblk_failf(&back->r, "Time out (%d)", timeout);
+                /* stop_ftp() names the real reason when it is the stop that
+                   cut the wait short. */
+                if (!stop_ftp(back))
+                  htsblk_failf(&back->r, "Time out (%d)", timeout);
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
@@ -784,9 +878,9 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 
           // 226 Transfer complete?
           if (back->r.statuscode != -1) {
-            if (wait_socket_receive(soc_ctl, timeout_onfly) > 0) {
+            if (wait_socket_receive(back, soc_ctl, timeout_onfly, opt) > 0) {
               // récupérer 226 transfer complete
-              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+              get_ftp_line(back, soc_ctl, line, sizeof(line), timeout, opt);
               if (line[0] == '2') {     // OK
                 strcpybuff(back->r.msg, "OK");
                 back->r.statuscode = HTTP_OK;
@@ -809,7 +903,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     _CHECK_HALT_FTP;
     strcpybuff(back->info, "quit");
     send_line(soc_ctl, "QUIT"); // bye bye
-    get_ftp_line(soc_ctl, NULL, 0, timeout);
+    get_ftp_line(back, soc_ctl, NULL, 0, timeout, opt);
 #ifdef _WIN32
     closesocket(soc_ctl);
 #else
@@ -922,7 +1016,8 @@ FILE *dd = NULL;
 // routines de réception/émission
 // 0 = ERROR
 int send_line(T_SOC soc, const char *data) {
-  char BIGSTK line[1024];
+  char BIGSTK line[FTP_LINE_SIZE + 2]; // room for the CRLF of a maximal command
+  int n;
 
   // backstop: the driver fails earlier, but no injected byte reaches the wire
   if (!hts_is_control_free(data))
@@ -942,7 +1037,10 @@ int send_line(T_SOC soc, const char *data) {
   printf("---> %s", data);
   fflush(stdout);
 #endif
-  snprintf(line, sizeof(line), "%s\x0d\x0a", data);
+  // an unterminated command would blend into whatever the server reads next
+  n = snprintf(line, sizeof(line), "%s\x0d\x0a", data);
+  if (n < 0 || n >= (int) sizeof(line))
+    return 0;
   if (check_socket_connect(soc) != 1) {
 #if FTP_DEBUG
     printf("!SOC WRITE ERROR\n");
@@ -962,7 +1060,8 @@ int send_line(T_SOC soc, const char *data) {
 #endif
 }
 
-int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
+int get_ftp_line(lien_back *back, T_SOC soc, char *ptrline, size_t line_size,
+                 int timeout, const httrackp *opt) {
   char BIGSTK data[1024];
   int i, ok, multiline;
 
@@ -978,7 +1077,7 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
     char b;
 
     // vérifier données
-    switch (wait_socket_receive(soc, timeout)) {
+    switch (wait_socket_receive(back, soc, timeout, opt)) {
     case -1:                   // erreur de lecture
       if (ptrline)
         snprintf(ptrline, line_size, "500 *read error");
@@ -1089,18 +1188,35 @@ int check_socket_connect(T_SOC soc) {
   return 0;
 }
 
+/* Clip a wait to what is left of --max-time: back_wait() can't abort this slot
+   the way it aborts an HTTP one. */
+static int ftp_wait_left(const httrackp *opt, int timeout) {
+  if (opt != NULL && opt->maxtime > 0) {
+    const TStamp left =
+        (TStamp) opt->maxtime - (time_local() - HTS_STAT.stat_timestart);
+
+    if (left < (TStamp) timeout)
+      timeout = left > 0 ? (int) left : 0;
+  }
+  return timeout;
+}
+
 // attendre des données
-int wait_socket_receive(T_SOC soc, int timeout) {
-  // attendre les données
+int wait_socket_receive(lien_back *back, T_SOC soc, int timeout,
+                        const httrackp *opt) {
   TStamp ltime = time_local();
   int r;
 
+  timeout = ftp_wait_left(opt, timeout);
+
 #if FTP_DEBUG
   printf("\x0dWaiting for data ");
   fflush(stdout);
 #endif
-  while((!(r = check_socket(soc)))
-        && (((int) ((TStamp) (time_local() - ltime))) < timeout)) {
+  /* A stop raised by the engine ends the wait on the next 100ms tick instead of
+     after the full timeout, so teardown does not sit on a silent server. */
+  while ((!(r = check_socket(soc))) && (back == NULL || !back->stop_ftp) &&
+         (((int) ((TStamp) (time_local() - ltime))) < timeout)) {
     Sleep(100);
 #if FTP_DEBUG
     printf(".");

src/htsftp.h

@@ -55,13 +55,20 @@ typedef struct FTPDownloadStruct FTPDownloadStruct;
 struct FTPDownloadStruct {
   lien_back *pBack;
   httrackp *pOpt;
+  FTPDownloadStruct *pNext; /* live-worker list, owned by htsftp.c */
 };
 
 /* Library internal definictions */
 #ifdef HTS_INTERNAL_BYTECODE
+/* Capacity of every FTP control-line buffer; send_line() adds the CRLF. */
+#define FTP_LINE_SIZE 1024
+
 #if USE_BEGINTHREAD
 void launch_ftp(FTPDownloadStruct * params);
 void back_launch_ftp(void *pP);
+/* Cancel every live FTP worker and block until each stops touching its backlog
+   slot and opt. Call before freeing either. */
+void ftp_stop_workers(void);
 #else
 void launch_ftp(FTPDownloadStruct * params, char *path, char *exec);
 int back_launch_ftp(FTPDownloadStruct * params);
@@ -69,23 +76,43 @@ int back_launch_ftp(FTPDownloadStruct * params);
 
 int run_launch_ftp(FTPDownloadStruct * params);
 int send_line(T_SOC soc, const char *data);
-int get_ftp_line(T_SOC soc, char *line, size_t line_size, int timeout);
+/* Read one control-channel reply into line[line_size]. Returns 0 on error.
+   Optional back ends the wait on a stop, opt clips it to --max-time. */
+int get_ftp_line(lien_back *back, T_SOC soc, char *line, size_t line_size,
+                 int timeout, const httrackp *opt);
 /* Split a "user[:pass]@" prefix (end = jump_identification result) into
-   bounded, NUL-terminated user/pass buffers, truncating to fit.
+   NUL-terminated user/pass buffers. Returns HTS_FALSE and empties both when a
+   field does not fit, as a clipped one would name another account.
    Both sizes must be nonzero. */
-void ftp_split_userpass(const char *src, const char *end, char *user,
-                        size_t user_size, char *pass, size_t pass_size);
-/* Build "<verb> <path>" into line[line_size], truncating to fit. The path is
-   quoted whenever a bare one would give the server a second token; it must
-   already have been screened for control bytes. */
-void ftp_command(char *line, size_t line_size, const char *verb,
-                 const char *path);
+hts_boolean ftp_split_userpass(const char *src, const char *end, char *user,
+                               size_t user_size, char *pass, size_t pass_size);
+/* ftp_split_userpass() into the caller's fixed buffers; a buffer too wide for
+   its "USER <user>" line would be clipped again when the command is built. */
+#define ftp_split_userpass_buf(src, end, user, pass)                           \
+  (HTS_COMPILE_ASSERT(sizeof(user) + sizeof("USER ") - 1 <= FTP_LINE_SIZE &&   \
+                      sizeof(pass) + sizeof("PASS ") - 1 <= FTP_LINE_SIZE),    \
+   ftp_split_userpass((src), (end), (user), sizeof(user), (pass),              \
+                      sizeof(pass)))
+/* Build "<verb> <path>" into line[line_size]. The path is quoted whenever a
+   bare one would give the server a second token; it must already have been
+   screened for control bytes. Returns HTS_FALSE and empties line when the
+   command does not fit, as a clipped one would name a different file. */
+hts_boolean ftp_command(char *line, size_t line_size, const char *verb,
+                        const char *path);
+/* ftp_command() into a control line of the one capacity every FTP buffer has;
+   anything narrower fails the build rather than refusing a path that fits. */
+#define ftp_command_line(line, verb, path)                                     \
+  (HTS_COMPILE_ASSERT(sizeof(line) == FTP_LINE_SIZE),                          \
+   ftp_command((line), sizeof(line), (verb), (path)))
 T_SOC get_datasocket(char *to_send, size_t to_send_size);
 int stop_ftp(lien_back * back);
 char *linejmp(char *line);
 int check_socket(T_SOC soc);
 int check_socket_connect(T_SOC soc);
-int wait_socket_receive(T_SOC soc, int timeout);
+/* Wait up to timeout seconds for soc to become readable. Optional back ends the
+   wait on a stop, opt clips it to --max-time. */
+int wait_socket_receive(lien_back *back, T_SOC soc, int timeout,
+                        const httrackp *opt);
 #endif
 
 #endif

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-17"
-#define HTTRACK_VERSIONID "3.49.17"
+#define HTTRACK_VERSION "3.49-18"
+#define HTTRACK_VERSIONID "3.49.18"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 
@@ -273,6 +273,9 @@ typedef int hts_tristate;
 /* True when A is a non-NULL, non-empty string. */
 #define strnotempty(A) (((A) != NULL && (A)[0] != '\0'))
 
+/* Compile-time check, usable as an expression. */
+#define HTS_COMPILE_ASSERT(cond) ((void) sizeof(char[(cond) ? 1 : -1]))
+
 /* 'inline' where the dialect supports it (C++), nothing in plain C. */
 #ifdef __cplusplus
 #define HTS_INLINE inline

src/htslib.c

@@ -47,6 +47,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsbauth.h"
 #include "htsthread.h"
 #include "htsback.h"
+#include "htsftp.h"
 #include "htswrap.h"
 #include "htsmd5.h"
 #include "htsmodules.h"
@@ -6055,6 +6056,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->changes_state = NULL;
   opt->single_file = HTS_FALSE;
   opt->single_file_max_size = SINGLEFILE_DEFAULT_MAX_SIZE;
+  opt->singlefile_state = NULL;
   StringCopy(opt->why_url, "");
   opt->pause_min_ms = 0;
   opt->pause_max_ms = 0;
@@ -6126,6 +6128,9 @@ HTSEXT_API size_t hts_sizeof_opt(void) {
 
 HTSEXT_API void hts_free_opt(httrackp * opt) {
   if (opt != NULL) {
+    /* An FTP worker reads opt for its whole run, and not every caller drains
+       its threads the way httrack.c does before getting here. */
+    ftp_stop_workers();
 
     /* Alocated callbacks */
 
@@ -6209,6 +6214,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
     StringFree(opt->warc_file);
     StringFree(opt->sitemap_url);
     hts_sitemap_free(opt); /* backstop: httpmirror's early-return paths */
+    singlefile_free(opt);
 
     hts_changes_free_opt(opt);
 

src/htsnet.c

@@ -0,0 +1,52 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: Out-of-line htsnet.h helpers, kept here so the installed */
+/*       header needs nothing beyond strict ISO C                 */
+/* Author: Xavier Roche                                           */
+/* ------------------------------------------------------------ */
+
+#include "htsnet.h"
+
+HTSNET_API void SOCaddr_inetntoa_(char *namebuf, size_t namebuflen,
+                                  SOCaddr *const ss, const char *file,
+                                  const int line) {
+  assertf_(namebuf != NULL, file, line);
+  assertf_(ss != NULL, file, line);
+
+  if (getnameinfo(&ss->m_addr.sa, sizeof(ss->m_addr), namebuf, namebuflen, NULL,
+                  0, NI_NUMERICHOST) == 0) {
+    /* remove scope id(s) */
+    char *const pos = strchr(namebuf, '%');
+    if (pos != NULL) {
+      *pos = '\0';
+    }
+  } else {
+    namebuf[0] = '\0';
+  }
+}

src/htsnet.h

@@ -273,25 +273,20 @@ static HTS_UNUSED socklen_t SOCaddr_copyaddr_(SOCaddr *const server,
                       __LINE__);                                               \
   } while (0)
 
-/** Write the numeric (dotted/colon) host of ss into namebuf (capacity
-    namebuflen), scope id stripped. On failure namebuf becomes "". */
-static HTS_UNUSED void SOCaddr_inetntoa_(char *namebuf, size_t namebuflen,
-                                         SOCaddr *const ss, const char *file,
-                                         const int line) {
-  assertf_(namebuf != NULL, file, line);
-  assertf_(ss != NULL, file, line);
+/* proxytrack compiles htsnet.c in rather than linking the library, and MSVC
+   rejects a dllimport definition. */
+#ifdef HTS_NO_LIBHTTRACK
+#define HTSNET_API
+#else
+#define HTSNET_API HTSEXT_API
+#endif
 
-  if (getnameinfo(&ss->m_addr.sa, sizeof(ss->m_addr), namebuf, namebuflen, NULL,
-                  0, NI_NUMERICHOST) == 0) {
-    /* remove scope id(s) */
-    char *const pos = strchr(namebuf, '%');
-    if (pos != NULL) {
-      *pos = '\0';
-    }
-  } else {
-    namebuf[0] = '\0';
-  }
-}
+/** Write the numeric (dotted/colon) host of ss into namebuf (capacity
+    namebuflen), scope id stripped. On failure namebuf becomes "". Out of line:
+    getnameinfo() isn't declared to a strict-ISO translation unit (#1001). */
+HTSNET_API void SOCaddr_inetntoa_(char *namebuf, size_t namebuflen,
+                                  SOCaddr *const ss, const char *file,
+                                  const int line);
 
 /** Numeric host of ss into namebuf (capacity namebuflen); "" on failure. */
 #define SOCaddr_inetntoa(namebuf, namebuflen, ss)                              \

src/htsopt.h

@@ -564,6 +564,7 @@ struct httrackp {
      rather than in htsoptstate because that struct is embedded by value, so
      growing it would shift every httrackp field declared after it. */
   void *sitemap_state; /**< hts_sitemap_state*, or NULL. Tail: ABI */
+  void *singlefile_state; /**< hts_singlefile_state*, or NULL. Tail: ABI */
 };
 
 /* Running statistics for a mirror. */

src/htsparse.c

@@ -59,6 +59,7 @@ Please visit our Website: http://www.httrack.com
 
 // parser
 #include "htsparse.h"
+#include "htssinglefile.h"
 #include "htsback.h"
 
 // arrays
@@ -398,6 +399,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
     FILE *fp = NULL;                  // fichier écrit localement 
     const char *html = r->adr;        // pointeur (on parcours)
     const char *lastsaved;            // adresse du dernier octet sauvé + 1
+    hts_boolean sf_may_mark = HTS_FALSE; // --single-file: mark this one?
 
     hts_log_print(opt, LOG_DEBUG, "scanning file %s%s (%s)..", urladr(), urlfil(),
                   savename());
@@ -413,6 +415,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           r->adr[i] = ' ';
         }
       }
+      /* Only chance can put this run's secret in a fetched document. */
+      if (opt->single_file) {
+        sf_may_mark = singlefile_may_mark(opt, r->adr, (size_t) r->size);
+      }
     }
 
     // Indexing!
@@ -1831,6 +1837,12 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             int quoteinscript = 0;
             int noquote = 0;
             const char *tag_attr_start = html;
+            /* Kept because the value scan below clears intag on an unquoted
+               value ending at '>', and --single-file must still know the tag.
+               inscript_locked marks a whole CSS or JS file, the one context
+               that legitimately has no tag. */
+            const char *const sf_tag = intag_start_valid ? intag_name : NULL;
+            const int sf_tagless_body = inscript_locked;
 
             // si nofollow ou un stop a été déclenché, réécrire tous les liens en externe
             if ((nofollow)
@@ -3100,6 +3112,19 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                         if ((opt->getmode & HTS_GETMODE_HTML) && (ptr > 0)) {
 
+                          /* --single-file: tag the reference for the
+                             end-of-mirror pass. A fragment, so a mirror left
+                             marked by an interrupted run still browses. */
+                          /* A NULL tag reads as "a CSS or JS body", so a tag
+                             we cannot name must not reach the classifier. */
+                          const char sf_class =
+                              (opt->single_file && sf_may_mark && !in_media &&
+                               p_type == 0 && !p_searchMETAURL &&
+                               (sf_tag != NULL || sf_tagless_body))
+                                  ? singlefile_ref_class(sf_tag, tag_attr_start)
+                                  : 0;
+                          const size_t sf_start = TypedArraySize(output_buffer);
+
                           // écrire le lien modifié, relatif
                           // Note: escape all chars, even >127 (no UTF)
                           HT_ADD_HTMLESCAPED_FULL(tempo);
@@ -3113,6 +3138,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                               HT_ADD_HTMLESCAPED(a);
                             }
                           }
+
+                          /* Measured on the appended bytes, not tempo: the
+                             escape can change the length. The query sits inside
+                             the span so inlining drops it. */
+                          if (sf_class != 0) {
+                            char sf_mark[SINGLEFILE_MARK_MAX];
+
+                            HT_ADD(singlefile_mark(
+                                opt, sf_mark, sizeof(sf_mark), sf_class,
+                                TypedArraySize(output_buffer) - sf_start));
+                          }
                         }
                         lastsaved = eadr - 1;   // dernier écrit+1 (enfin euh apres on fait un ++ alors hein)
                       } else {

src/htsrandom.c

@@ -0,0 +1,101 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* System CSPRNG. See htsrandom.h.                               */
+/* ------------------------------------------------------------ */
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+
+#include "htsrandom.h"
+
+#ifdef _WIN32
+#include <windows.h>
+#else
+#include <errno.h>
+#include <stdio.h>
+#ifdef HAVE_SYS_RANDOM_H
+#include <sys/random.h>
+#endif
+#endif
+
+#ifdef _WIN32
+/* RtlGenRandom, resolved at runtime so no import library is needed. */
+typedef BOOLEAN(WINAPI *hts_rtlgenrandom_t)(PVOID buffer, ULONG length);
+#endif
+
+hts_boolean hts_random_bytes(void *buf, size_t len) {
+#ifdef _WIN32
+  hts_boolean ok = HTS_FALSE;
+  HMODULE dll = LoadLibraryA("advapi32.dll");
+
+  if (dll != NULL) {
+    hts_rtlgenrandom_t gen =
+        (hts_rtlgenrandom_t) GetProcAddress(dll, "SystemFunction036");
+
+    /* the ULONG cast must not truncate; callers ask for a few dozen bytes */
+    if (gen != NULL && len <= 0x10000 && gen(buf, (ULONG) len)) {
+      ok = HTS_TRUE;
+    }
+    FreeLibrary(dll);
+  }
+  return ok;
+#else
+  unsigned char *const p = (unsigned char *) buf;
+  size_t got = 0;
+  FILE *fp;
+
+#ifdef HAVE_GETRANDOM
+  while (got < len) {
+    const ssize_t n = getrandom(p + got, len - got, 0);
+
+    if (n < 0) {
+      if (errno == EINTR)
+        continue;
+      break; /* pre-3.17 kernel or a seccomp filter: try /dev/urandom */
+    }
+    got += (size_t) n;
+  }
+  if (got == len)
+    return HTS_TRUE;
+#endif
+  fp = fopen("/dev/urandom", "rb");
+  if (fp == NULL)
+    return HTS_FALSE;
+  while (got < len) {
+    const size_t n = fread(p + got, 1, len - got, fp);
+
+    if (n == 0) /* a short read is a hard failure, not partial credit */
+      break;
+    got += n;
+  }
+  fclose(fp);
+  return got == len ? HTS_TRUE : HTS_FALSE;
+#endif
+}

src/htsrandom.h

@@ -0,0 +1,52 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* System CSPRNG. Its own file because htsserver links libhttrack but cannot
+   see a hidden symbol inside it, and both need this one. */
+/* ------------------------------------------------------------ */
+
+#ifndef HTS_RANDOM_DEFH
+#define HTS_RANDOM_DEFH
+
+#include "htsglobal.h"
+
+#include <stddef.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+/* Fill buf with len cryptographically strong random bytes. HTS_FALSE when no
+   such source is available: the caller must fail, never fall back to rand(). */
+hts_boolean hts_random_bytes(void *buf, size_t len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/htsselftest.c

@@ -4494,52 +4494,160 @@ static int st_ftpline(httrackp *opt, int argc, char **argv) {
   assertf(send(sv[1], "\r\n", 2, 0) == 2); // end the line so we return
   deletesoc(sv[1]);
   line[0] = '\0';
-  get_ftp_line(sv[0], line, sizeof(line), 5);
+  get_ftp_line(NULL, sv[0], line, sizeof(line), 5, NULL);
   deletesoc(sv[0]);
   printf("ftp-line self-test OK (bounded %d-byte reply)\n",
          (int) sizeof(flood));
   return 0;
 }
 
-/* ftp_split_userpass: well-formed split, plus a hostile over-long userinfo
-   that pre-fix overran user[256]/pass[256]. */
+/* ftp_split_userpass: the split itself, and userinfo refused rather than
+   clipped into another account's name (#1032). */
 static int st_ftpuser(httrackp *opt, int argc, char **argv) {
-  char user[256], pass[256];
-  char in[1200];
+  static const size_t caps[] = {16, 256}; /* asymmetric: a shared bound shows */
+  char ubuf[256 + 32], pbuf[sizeof(ubuf)], poison[sizeof(ubuf)];
+  char in[2 * 256 + 8];
+  size_t c, over;
 
   (void) opt;
   (void) argc;
   (void) argv;
+  memset(poison, '#', sizeof(poison));
   {
     const char ok[] = "bob:secret@host/f"; // '@' at index 10
 
-    ftp_split_userpass(ok, ok + 11, user, sizeof(user), pass, sizeof(pass));
-    assertf(strcmp(user, "bob") == 0);
-    assertf(strcmp(pass, "secret") == 0);
+    assertf(ftp_split_userpass(ok, ok + 11, ubuf, sizeof(ubuf), pbuf,
+                               sizeof(pbuf)) == HTS_TRUE);
+    assertf(strcmp(ubuf, "bob") == 0);
+    assertf(strcmp(pbuf, "secret") == 0);
   }
-  memset(in, 'u', 400);
-  in[400] = ':';
-  memset(in + 401, 'p', 400);
-  in[801] = '@';
-  in[802] = '\0';
-  ftp_split_userpass(in, in + 802, user, sizeof(user), pass, sizeof(pass));
-  assertf(strlen(user) == sizeof(user) - 1);
-  assertf(strlen(pass) == sizeof(pass) - 1);
   {
-    /* tight sizes + guard byte catch an off-by-one the 256 case can't */
-    char ubuf[16], pbuf[16];
+    const char ok[] = "bob@host/f"; // no password: the '@' still ends the user
 
-    memset(ubuf, 'Z', sizeof(ubuf));
-    memset(pbuf, 'Z', sizeof(pbuf));
-    ftp_split_userpass(in, in + 802, ubuf, 8, pbuf, 8);
-    assertf(strcmp(ubuf, "uuuuuuu") == 0);
-    assertf(strcmp(pbuf, "ppppppp") == 0);
-    assertf(ubuf[8] == 'Z' && pbuf[8] == 'Z');
+    assertf(ftp_split_userpass(ok, ok + 4, ubuf, sizeof(ubuf), pbuf,
+                               sizeof(pbuf)) == HTS_TRUE);
+    assertf(strcmp(ubuf, "bob") == 0);
+    assertf(pbuf[0] == '\0');
+  }
+  {
+    const char ok[] = "u@relay:pw@gw/f"; // only the last '@' ends the userinfo
+
+    assertf(ftp_split_userpass(ok, ok + 11, ubuf, sizeof(ubuf), pbuf,
+                               sizeof(pbuf)) == HTS_TRUE);
+    assertf(strcmp(ubuf, "u@relay") == 0);
+    assertf(strcmp(pbuf, "pw") == 0);
+  }
+  for (c = 0; c < sizeof(caps) / sizeof(caps[0]); c++) {
+    const size_t ucap = caps[c], pcap = caps[1 - c];
+
+    /* overshoot the user, the pass, then a bare name bounded only by '@' */
+    for (over = 0; over <= 2; over++) {
+      const size_t cap = over == 1 ? pcap : ucap;
+      size_t len;
+
+      for (len = cap - 2; len <= cap + 1; len++) {
+        const size_t user_len = over == 1 ? 1 : len;
+        const size_t pass_len = over == 0 ? 1 : (over == 1 ? len : 0);
+        const size_t total = user_len + pass_len + (over == 2 ? 1 : 2);
+        const hts_boolean fits = len < cap ? HTS_TRUE : HTS_FALSE;
+
+        memset(in, 'u', user_len);
+        if (over != 2) {
+          in[user_len] = ':';
+          memset(in + user_len + 1, 'p', pass_len);
+        }
+        in[total - 1] = '@';
+        in[total] = '\0';
+        memcpy(ubuf, poison, sizeof(ubuf)); /* a zero canary would hide a NUL */
+        memcpy(pbuf, poison, sizeof(pbuf));
+        assertf(ftp_split_userpass(in, in + total, ubuf, ucap, pbuf, pcap) ==
+                fits);
+        if (fits) {
+          assertf(strlen(ubuf) == user_len && strlen(pbuf) == pass_len);
+          assertf(ubuf[user_len - 1] == 'u');
+          assertf(pass_len == 0 || pbuf[pass_len - 1] == 'p');
+        } else {
+          assertf(ubuf[0] == '\0' && pbuf[0] == '\0'); /* fail safe */
+        }
+        /* the whole tail: one canary byte misses a write just past it */
+        assertf(memcmp(ubuf + ucap, poison, sizeof(ubuf) - ucap) == 0);
+        assertf(memcmp(pbuf + pcap, poison, sizeof(pbuf) - pcap) == 0);
+      }
+    }
   }
   printf("ftp-userpass self-test OK\n");
   return 0;
 }
 
+/* Both quoting forms at two capacities: the quoted form is two bytes wider
+   (#1019). */
+static int st_ftpcmdlen(httrackp *opt, int argc, char **argv) {
+  static const size_t caps[] = {32, FTP_LINE_SIZE};
+  char BIGSTK buf[FTP_LINE_SIZE + 32];
+  char BIGSTK poison[FTP_LINE_SIZE + 32];
+  char BIGSTK path[FTP_LINE_SIZE + 2];
+  char BIGSTK wire[FTP_LINE_SIZE * 2];
+  size_t c, got = 0;
+  T_SOC sv[2];
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  memset(poison, '#', sizeof(poison));
+  for (c = 0; c < sizeof(caps) / sizeof(caps[0]); c++) {
+    const size_t cap = caps[c];
+    int quoted;
+
+    for (quoted = 0; quoted <= 1; quoted++) {
+      const size_t verb = 5 + 2 * (size_t) quoted; /* "RETR " plus quotes */
+      const size_t fit = cap - 1 - verb; /* longest path still fitting */
+      size_t len;
+
+      for (len = fit - 1; len <= fit + 1; len++) {
+        memset(path, 'p', len);
+        path[len] = '\0';
+        if (quoted)
+          path[0] = ' '; /* any of these forces the quoted form */
+        memcpy(buf, poison, sizeof(buf)); /* a zero canary would hide a NUL */
+        if (len > fit) {
+          assertf(ftp_command(buf, cap, "RETR", path) == HTS_FALSE);
+          assertf(buf[0] == '\0'); /* fail-safe for an ignored result */
+        } else {
+          assertf(ftp_command(buf, cap, "RETR", path) == HTS_TRUE);
+          assertf(strlen(buf) == verb + len);
+          assertf(strncmp(buf, "RETR ", 5) == 0);
+          assertf(buf[verb + len - 1] == (quoted ? '\"' : 'p'));
+        }
+        /* the whole tail: one canary byte misses a write just past it */
+        assertf(memcmp(buf + cap, poison, sizeof(buf) - cap) == 0);
+      }
+    }
+  }
+
+  /* send_line() adds the CRLF and drops, rather than truncates, an over-length
+     line. */
+  memset(path, 'q', FTP_LINE_SIZE);
+  path[FTP_LINE_SIZE] = '\0';
+  assertf(st_socketpair(sv) == 0);
+  assertf(send_line(sv[0], path) == 0); /* one byte too long: never sent */
+  path[FTP_LINE_SIZE - 1] = '\0';
+  assertf(send_line(sv[0], path) != 0);
+  deletesoc(sv[0]);
+  for (;;) {
+    const int n = (int) recv(sv[1], wire + got, (int) (sizeof(wire) - got), 0);
+
+    if (n <= 0)
+      break;
+    got += (size_t) n;
+  }
+  deletesoc(sv[1]);
+  assertf(got == FTP_LINE_SIZE + 1); /* the maximal command alone */
+  assertf(memcmp(wire, path, FTP_LINE_SIZE - 1) == 0);
+  assertf(memcmp(wire + FTP_LINE_SIZE - 1, "\r\n", 2) == 0);
+  printf("ftp-cmdlen self-test OK (%d bytes sent)\n", (int) got);
+  return 0;
+}
+
 /* send_line() must drop a command line carrying a control byte (#1010). */
 static int st_ftpctrl(httrackp *opt, int argc, char **argv) {
   /* Verb and URL path as run_launch_ftp() hands them over, then the line the
@@ -5776,11 +5884,47 @@ static hts_boolean sf_try_put(const char *dir, const char *rel,
   return HTS_TRUE;
 }
 
+/* Expand a fixture: \001<ref>\002 becomes <ref> plus this run's mark for it.
+   The secret is random per run, so a fixture cannot spell a mark itself. */
+static void sf_expand_fixture(httrackp *opt, const char *in, size_t len,
+                              String *out) {
+  size_t i, start = 0;
+
+  StringClear(*out);
+  for (i = 0; i < len; i++) {
+    if (in[i] == '\001') {
+      start = StringLength(*out);
+    } else if (in[i] == '\002') {
+      char mark[SINGLEFILE_MARK_MAX];
+
+      StringCat(*out,
+                singlefile_mark(opt, mark, sizeof(mark), SINGLEFILE_CLASS_ANY,
+                                StringLength(*out) - start));
+    } else {
+      StringAddchar(*out, in[i]);
+    }
+  }
+}
+
 static void sf_put(const char *dir, const char *rel, const void *data,
                    size_t len) {
   assertf(sf_try_put(dir, rel, data, len));
 }
 
+/* sf_put() for a text fixture, expanding its \001<ref>\002 delimiters. Binary
+   fixtures must not go through here: sf_png carries those very bytes. */
+static size_t sf_put_marked(httrackp *opt, const char *dir, const char *rel,
+                            const void *data, size_t len) {
+  String body = STRING_EMPTY;
+  size_t written;
+
+  sf_expand_fixture(opt, (const char *) data, len, &body);
+  assertf(sf_try_put(dir, rel, StringBuff(body), StringLength(body)));
+  written = StringLength(body);
+  StringFree(body);
+  return written;
+}
+
 /* Number of times needle occurs in hay. */
 static int sf_count(const char *hay, const char *needle) {
   const size_t l = strlen(needle);
@@ -5892,98 +6036,99 @@ static const char sf_svg[] = "<svg><g id=\"icon-a\"/></svg>";
 
 static const char sf_page[] =
     "<html><head>\n"
-    "<link rel=\"stylesheet\" href=\"css/main.css\">\n"
-    "<link rel=\"canonical\" href=\"other.html\">\n"
+    "<link rel=\"stylesheet\" href=\"\001css/main.css\002\">\n"
+    "<link rel=\"canonical\" href=\"\001other.html\002\">\n"
     "<title>t</title>\n"
-    "<style>body { background: url(\"img/a%20b.png\"); }\n"
-    /* Everything the escape set has to neutralise before a fragment reaches an
-       unquoted url() token: whitespace, '<'/'>', then a quote, a paren, a
-       backslash and a high byte together. */
-    "b { background: url('img/sprite.svg#w x'); }\n"
-    "i { background: url('img/sprite.svg#lt<s>gt'); }\n"
-    "u { background: url(\"img/sprite.svg#z'(\\\xC3\xA9\"); }</style>\n"
+    "<style>body { background: url(\"\001img/a%20b.png\002\"); }\n"
+    /* A fragment is document text the mark never covers, so it reaches the
+       data: URI exactly as written, in the quoting the author gave it. */
+    "b { background: url('\001img/sprite.svg\002#w x'); }\n"
+    "i { background: url('\001img/sprite.svg\002#lt<s>gt'); }\n"
+    "u { background: url(\"\001img/sprite.svg\002#z'(\\\xC3\xA9\"); }</style>\n"
     "</head><body>\n"
-    "<img src=\"img/a%20b.png\" srcset=\"img/a%20b.png 1x, img/big.png 2x\">\n"
-    "<link rel=\"icon\" href=\"icon.png\">\n"
-    "<link rel=\"preload\" as=\"font\" href=\"font/f.woff2\">\n"
+    "<img src=\"\001img/a%20b.png\002\" srcset=\"\001img/a%20b.png\002 "
+    "1x, \001img/big.png\002 2x\">\n"
+    "<link rel=\"icon\" href=\"\001icon.png\002\">\n"
+    "<link rel=\"preload\" as=\"font\" href=\"\001font/f.woff2\002\">\n"
     "<img src=\"data:image/gif;base64,QUJD\">\n"
     /* Each has a real file where its guard's removal would land it; without
        that they stay links either way, the target merely being absent. */
     "<img src=\"http://example.com/x.png\">\n"
     "<img src=\"//example.com/x.png\">\n"
-    "<input type=\"image\" src=\"img/in.png\">\n"
+    "<input type=\"image\" src=\"\001img/in.png\002\">\n"
     /* Lazy loading: src is the placeholder, the real image rides data-src. */
-    "<img src=\"img/ph.png\" data-src=\"img/lz.png\" "
-    "data-srcset=\"img/lz2.png 2x\" lowsrc=\"img/low.png\">\n"
-    "<object data=\"img/ob.png\"></object>\n"
-    "<embed src=\"img/em.png\">\n"
-    "<img data-src=\"other.html\">\n"
+    "<img src=\"\001img/ph.png\002\" data-src=\"\001img/lz.png\002\" "
+    "data-srcset=\"\001img/lz2.png\002 2x\" "
+    "lowsrc=\"\001img/low.png\002\">\n"
+    "<object data=\"\001img/ob.png\002\"></object>\n"
+    "<embed src=\"\001img/em.png\002\">\n"
+    "<img data-src=\"\001other.html\002\">\n"
     /* What a first pass emits: re-resolving it is what a second pass must not
        do, and the fallback type would inline whatever the walk found. */
     "<link rel=\"stylesheet\" href=\"data:text/css;base64,QUJD\">\n"
-    "<video poster=\"img/po.png\" controls>"
-    "<source src=\"v.mp4\" type=\"video/mp4\"></video>\n"
-    "<svg><image href=\"img/sv.png\"/></svg>\n"
-    "<table background=\"img/bg.png\"><tr><td>x</td></tr></table>\n"
+    "<video poster=\"\001img/po.png\002\" controls>"
+    "<source src=\"\001v.mp4\002\" type=\"video/mp4\"></video>\n"
+    "<svg><image href=\"\001img/sv.png\002\"/></svg>\n"
+    "<table background=\"\001img/bg.png\002\"><tr><td>x</td></tr></table>\n"
     /* The second is what bites: drop the clamp and its leading ".." lands it
        back on <root>/img/a b.png. The first can only 404 either way. */
-    "<img src=\"../escape.png\">\n"
-    "<img src=\"../img/a%20b.png\">\n"
+    "<img src=\"\001../escape.png\002\">\n"
+    "<img src=\"\001../img/a%20b.png\002\">\n"
     "<a href=\"img/a%20b.png\">link</a>\n"
-    "<script src=\"js/app.js\"></script>\n"
+    "<script src=\"\001js/app.js\002\"></script>\n"
     "<script>var s = \"</scripting>\"; var t = \"<img src='img/a%20b.png'>\";"
     "</script>\n"
     /* A fragment selects inside the asset, so it has to survive onto the
        data: URI; the query named the remote resource and must not. */
-    "<img src=\"img/sprite.svg#icon-a\">\n"
-    "<img srcset=\"img/sprite.svg#icon-b 2x\">\n"
-    "<svg><image xlink:href=\"img/sprite.svg#icon-c\"/></svg>\n"
-    "<div style=\"background:url(img/sprite.svg#icon-d)\"></div>\n"
-    "<div style=\"background:url('img/sprite.svg#i)e')\"></div>\n"
-    "<img src=\"img/sprite.svg?v=1#icon-f\">\n"
-    /* Already carries the document's own escapes: re-encoding either would
-       make the browser look for a different id. */
-    "<img src=\"img/sprite.svg#g&amp;h%2Di\">\n"
-    /* A '"' left raw here would end the attribute the rewriter re-quotes. */
-    "<img src='img/sprite.svg#q\"z'>\n"
-    "<img src=\"missing.png\" >\n"
-    "<!--><img src=\"img/a%20b.png\">\n"
-    "<div style=\"background:url(img/a%20b.png)\"></div>\n"
-    "<div style='content:\"x\"; background:url(img/a%20b.png)'></div>\n"
+    "<img src=\"\001img/sprite.svg\002#icon-a\">\n"
+    "<img srcset=\"\001img/sprite.svg\002#icon-b 2x\">\n"
+    "<svg><image xlink:href=\"\001img/sprite.svg\002#icon-c\"/></svg>\n"
+    "<div style=\"background:url(\001img/sprite.svg\002#icon-d)\"></div>\n"
+    "<div style=\"background:url('\001img/sprite.svg\002#i)e')\"></div>\n"
+    "<img src=\"\001img/sprite.svg?v=1\002#icon-f\">\n"
+    "<img src=\"\001img/sprite.svg\002#g&amp;h%2Di\">\n"
+    "<img src='\001img/sprite.svg\002#q\"z'>\n"
+    "<img src=\"\001missing.png\002\" >\n"
+    "<!--><img src=\"\001img/a%20b.png\002\">\n"
+    "<div style=\"background:url(\001img/a%20b.png\002)\"></div>\n"
+    "<div style='content:\"x\"; "
+    "background:url(\001img/a%20b.png\002)'></div>\n"
     "</body></html>\n";
 
 /* Lay a small mirror down under root. */
-static void sf_fixture(const char *root) {
+static void sf_fixture(httrackp *opt, const char *root) {
   /* The over-cap url() is what drives the rebase fallback: a reference an
      inlined stylesheet could not embed has to come out relative to the page,
      not to the stylesheet, or it dangles. */
   static const char css[] =
-      "@import \"sub/nested.css\";\n"
-      "@import url(\"sub/two.css\");\n"
+      "@import \"\001sub/nested.css\002\";\n"
+      "@import url(\"\001sub/two.css\002\");\n"
       "@import \"a\\\"url(../img/a b.png)b.css\";\n"
-      "@font-face { font-family: f; src: url(../font/f.woff2); }\n"
-      "body { background: url(../img/a b.png); }\n"
-      "div { background: url(../img/big.png); }\n"
-      "div.s { background: url(../img/big-sprite.svg#icon-g); }\n"
+      "@font-face { font-family: f; src: url(\001../font/f.woff2\002); }\n"
+      "body { background: url(\001../img/a%20b.png\002); }\n"
+      "div { background: url(\001../img/big.png\002); }\n"
+      "div.s { background: url(\001../img/big-sprite.svg\002#icon-g); }\n"
       /* A name whose escapes the rebase has to put back, unlike a fragment's;
          the '#' has to come back encoded or it reads as one. */
-      "div.h { background: url(../img/b&amp;c%25d%23e.png); }\n"
+      "div.h { background: url(\001../img/b&amp;c%25d%23e.png\002); }\n"
       "/* url(../img/never.png) */\n";
-  static const char nested[] = "div { background: url(../../img/a b.png); }\n";
-  static const char two[] = "p { background: url(../../img/a b.png); }\n";
-  static const char deep[] =
-      "<html><head><link rel=\"stylesheet\" href=\"../../css/main.css\">\n"
-      "</head><body>d</body></html>\n";
+  static const char nested[] =
+      "div { background: url(\001../../img/a%20b.png\002); }\n";
+  static const char two[] =
+      "p { background: url(\001../../img/a%20b.png\002); }\n";
+  static const char deep[] = "<html><head><link rel=\"stylesheet\" "
+                             "href=\"\001../../css/main.css\002\">\n"
+                             "</head><body>d</body></html>\n";
   static const char js[] = "var app = 1;\n";
   char big[4096];
 
   memset(big, 'B', sizeof(big));
-  sf_put(root, "page.html", sf_page, sizeof(sf_page) - 1);
-  sf_put(root, "deep/sub/page.html", deep, sizeof(deep) - 1);
+  sf_put_marked(opt, root, "page.html", sf_page, sizeof(sf_page) - 1);
+  sf_put_marked(opt, root, "deep/sub/page.html", deep, sizeof(deep) - 1);
   sf_put(root, "other.html", "<html>o</html>", 14);
-  sf_put(root, "css/main.css", css, sizeof(css) - 1);
-  sf_put(root, "css/sub/nested.css", nested, sizeof(nested) - 1);
-  sf_put(root, "css/sub/two.css", two, sizeof(two) - 1);
+  sf_put_marked(opt, root, "css/main.css", css, sizeof(css) - 1);
+  sf_put_marked(opt, root, "css/sub/nested.css", nested, sizeof(nested) - 1);
+  sf_put_marked(opt, root, "css/sub/two.css", two, sizeof(two) - 1);
   sf_put(root, "js/app.js", js, sizeof(js) - 1);
   sf_put(root, "img/a b.png", sf_png, SF_PNG_LEN);
   sf_put(root, "img/big.png", big, sizeof(big));
@@ -6024,7 +6169,7 @@ static void sf_fixture(const char *root) {
     }
     StringCat(wide,
               " title=\"> <img src=img/a%20b.png> \">end</p></body></html>");
-    sf_put(root, "wide.html", StringBuff(wide), StringLength(wide));
+    sf_put_marked(opt, root, "wide.html", StringBuff(wide), StringLength(wide));
     StringFree(wide);
   }
   sf_put(root, "v.mp4",
@@ -6050,7 +6195,7 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
   sf_put(argv[0], "escape.png", sf_png,
          SF_PNG_LEN); /* just outside the mirror */
   fconcat(root, sizeof(root), argv[0], "mirror/");
-  sf_fixture(root);
+  sf_fixture(opt, root);
   fconcat(page, sizeof(page), root, "page.html");
 
   /* Cap between the small assets and big.png. */
@@ -6118,36 +6263,33 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
   sf_check(strstr(out, "var t = \"<img src='img/a%20b.png'>\";") != NULL,
            "script body rewritten past a </scripting> lookalike");
 
-  /* Nothing an attribute value cannot hold: url() stays unquoted, and a quote
-     that was already in the CSS is escaped. */
-  sf_check(strstr(out, "url(\"data:") == NULL,
-           "a quoted url() would end a style attribute");
-  sf_check(strstr(out, "style=\"content:&quot;x&quot;; background:url(data:") !=
-               NULL,
-           "quote inside a rewritten style attribute not escaped");
+  /* Only the marked reference is touched: the value keeps its own quoting, so
+     nothing can be emitted that the attribute could not already hold. */
+  sf_check(strstr(out, "style='content:\"x\"; background:url(data:") != NULL,
+           "style attribute re-quoted instead of substituted in place");
 
-  /* Fragments: kept on the replacement, escaped where they would close the
-     url() token, and never joined by the query the mirrored name dropped. */
+  /* Fragments: the mark covers the reference only, so what followed it comes
+     back byte-identical -- including the escapes the document already carried
+     -- while the query rode inside the mark and went with it. */
   sf_check(strstr(out, "img/sprite.svg") == NULL,
            "a fragment-bearing reference was left a link");
   sf_check(sf_count(out, "#icon-a\"") == 1, "img src fragment dropped");
   sf_check(sf_count(out, "#icon-b 2x\"") == 1, "srcset fragment dropped");
   sf_check(sf_count(out, "#icon-c\"") == 1, "xlink:href fragment dropped");
   sf_check(sf_count(out, "#icon-d)") == 1, "style url() fragment dropped");
-  sf_check(sf_count(out, "#i%29e)") == 1,
-           "a fragment closing the url() token was not escaped");
+  sf_check(sf_count(out, "#i)e')") == 1,
+           "a fragment closing the url() token was rewritten");
   sf_check(sf_count(out, "#icon-f\"") == 1, "fragment after a query dropped");
   sf_check(strstr(out, "?v=1") == NULL, "query carried onto the data: URI");
   sf_check(sf_count(out, "#g&amp;h%2Di\"") == 1,
            "an escape the document already carried was encoded again");
-  sf_check(sf_count(out, "#q%22z\"") == 1,
-           "a quote in a fragment was not escaped");
-  sf_check(sf_count(out, "#w%20x)") == 1,
-           "whitespace in a fragment was not escaped");
-  sf_check(sf_count(out, "#lt%3Cs%3Egt)") == 1,
-           "'<'/'>' in a fragment were not escaped");
-  sf_check(sf_count(out, "#z%27%28%5C%C3%A9)") == 1,
-           "a quote, paren, backslash or high byte was left raw");
+  sf_check(sf_count(out, "#q\"z'") == 1, "a quote in a fragment was rewritten");
+  sf_check(sf_count(out, "#w x')") == 1,
+           "whitespace in a fragment was rewritten");
+  sf_check(sf_count(out, "#lt<s>gt')") == 1,
+           "'<'/'>' in a fragment were rewritten");
+  sf_check(sf_count(out, "#z'(\\\xC3\xA9\")") == 1,
+           "a quote, paren, backslash or high byte was rewritten");
 
   sf_check(strstr(out, "img/big.png 2x") != NULL, "over-cap asset inlined");
   sf_check(strstr(out, " 1x") != NULL, "srcset descriptor lost");
@@ -6167,7 +6309,7 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
              "url() inside a CSS comment was rewritten");
     sf_check(strstr(css, "url(data:font/woff2;base64,") != NULL,
              "@font-face src not inlined");
-    sf_check(strstr(css, "@import url(data:text/css;base64,") != NULL,
+    sf_check(strstr(css, "@import url(\"data:text/css;base64,") != NULL,
              "@import url() form not inlined");
     sf_check(strstr(css, "url(../img/a b.png)b.css") != NULL,
              "url() inside a string with an escaped quote was rewritten");
@@ -6235,7 +6377,7 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
 
   /* Same page, a cap above big.png: it now inlines. */
   fconcat(root, sizeof(root), argv[0], "mirror2/");
-  sf_fixture(root);
+  sf_fixture(opt, root);
   fconcat(page, sizeof(page), root, "page.html");
   opt->single_file_max_size = 1024 * 1024;
   sf_check(singlefile_rewrite_file(opt, root, page),
@@ -6249,21 +6391,37 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
   /* Nothing inlines, so the rewriter must be byte transparent. Assert on its
      output: the file it declined to write could not have changed regardless. */
   fconcat(root, sizeof(root), argv[0], "mirror3/");
-  sf_fixture(root);
+  sf_fixture(opt, root);
   fconcat(page, sizeof(page), root, "page.html");
   opt->single_file_max_size = 1;
-  sf_check(!singlefile_rewrite_file(opt, root, page), "one-byte cap inlined");
+  /* The page is still rewritten: nothing inlines, but the marks must go. */
+  sf_check(singlefile_rewrite_file(opt, root, page),
+           "one-byte cap left the marks in place");
+  {
+    char *capped = readfile_utf8(page);
+
+    /* Only the two data: URIs the fixture itself ships. */
+    sf_check(capped != NULL && sf_count(capped, ";base64,") == 2,
+             "one-byte cap inlined");
+    freet(capped);
+  }
   {
     String verbatim = STRING_EMPTY;
 
     StringClear(verbatim);
-    (void) singlefile_rewrite_html(opt, root, page, sf_page,
-                                   sizeof(sf_page) - 1,
+    String marked = STRING_EMPTY;
+
+    sf_expand_fixture(opt, sf_page, sizeof(sf_page) - 1, &marked);
+    (void) singlefile_rewrite_html(opt, root, page, StringBuff(marked),
+                                   StringLength(marked),
                                    SINGLEFILE_MAX_PAGE_SIZE, &verbatim);
-    sf_check(StringLength(verbatim) == sizeof(sf_page) - 1 &&
-                 memcmp(StringBuff(verbatim), sf_page, sizeof(sf_page) - 1) ==
-                     0,
-             "a page with nothing to inline was re-serialized differently");
+    /* Mark-transparent, not byte-transparent: a reference that cannot be
+       inlined loses its mark and keeps everything else. */
+    sf_check(StringLength(verbatim) < StringLength(marked),
+             "a page with nothing to inline kept its marks");
+    sf_check(strstr(StringBuff(verbatim), singlefile_intro(opt)) == NULL,
+             "an un-inlinable reference kept its mark");
+    StringFree(marked);
     StringFree(verbatim);
   }
   (void) outlen;
@@ -6272,26 +6430,29 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
      run is the control: it proves the fan-out is real, so the small one was
      cut short by the budget and not by the fixture. */
   {
-    static const char bomb_css[] = "@import \"b.css\";@import \"b.css\";"
-                                   "@import \"b.css\";@import \"b.css\";\n";
-    static const char bomb_html[] = "<html><head>"
-                                    "<link rel=\"stylesheet\" href=\"b.css\">"
-                                    "</head></html>\n";
-    const size_t css_len = sizeof(bomb_css) - 1;
-    String small = STRING_EMPTY, large = STRING_EMPTY;
+    static const char bomb_css[] =
+        "@import \"\001b.css\002\";@import \"\001b.css\002\";"
+        "@import \"\001b.css\002\";@import \"\001b.css\002\";\n";
+    static const char bomb_html[] =
+        "<html><head>"
+        "<link rel=\"stylesheet\" href=\"\001b.css\002\">"
+        "</head></html>\n";
+    size_t css_len;
+    String small = STRING_EMPTY, large = STRING_EMPTY, bomb = STRING_EMPTY;
 
     fconcat(root, sizeof(root), argv[0], "bomb/");
-    sf_put(root, "b.css", bomb_css, css_len);
+    css_len = sf_put_marked(opt, root, "b.css", bomb_css, sizeof(bomb_css) - 1);
     fconcat(page, sizeof(page), root, "page.html");
     opt->single_file_max_size = 1024 * 1024;
     StringClear(small);
     StringClear(large);
-    (void) singlefile_rewrite_html(opt, root, page, bomb_html,
-                                   sizeof(bomb_html) - 1, (LLint) css_len * 3,
+    sf_expand_fixture(opt, bomb_html, sizeof(bomb_html) - 1, &bomb);
+    (void) singlefile_rewrite_html(opt, root, page, StringBuff(bomb),
+                                   StringLength(bomb), (LLint) css_len * 3,
                                    &small);
-    (void) singlefile_rewrite_html(opt, root, page, bomb_html,
-                                   sizeof(bomb_html) - 1,
-                                   SINGLEFILE_MAX_PAGE_SIZE, &large);
+    (void) singlefile_rewrite_html(opt, root, page, StringBuff(bomb),
+                                   StringLength(bomb), SINGLEFILE_MAX_PAGE_SIZE,
+                                   &large);
     sf_check(StringLength(large) > 4096, "the @import bomb did not fan out");
     sf_check(StringLength(small) < StringLength(large) / 8,
              "the per-page budget did not cut the fan-out short");
@@ -6301,6 +6462,7 @@ static int st_singlefile(httrackp *opt, int argc, char **argv) {
              "the per-page budget was not charged as each asset was taken");
     StringFree(small);
     StringFree(large);
+    StringFree(bomb);
   }
 
   if (!sf_colon_ok)
@@ -8120,6 +8282,8 @@ static const struct selftest_entry {
     {"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
     {"ftp-ctrlchars", "", "send_line rejects a control byte in an FTP command",
      st_ftpctrl},
+    {"ftp-cmdlen", "",
+     "an FTP command too long for its control line is refused", st_ftpcmdlen},
     {"warc", "<dir>", "WARC/1.1 writer: framing, digests, revisit dedup",
      st_warc},
     {"warc-trunc", "<dir>", "WARC-Truncated on a cap-truncated body",

src/htssinglefile.h

@@ -55,18 +55,66 @@ extern "C" {
    so a few hundred bytes of hostile CSS can otherwise ask for gigabytes. */
 #define SINGLEFILE_MAX_PAGE_SIZE (64 * 1024 * 1024)
 
-/* Rewrite every HTML page the mirror produced. No-op unless opt->single_file;
-   call once the tree is final, after the update purge. */
+/* The mark htsparse appends to a saved reference the pass may inline:
+
+     #!<16-hex secret>.<class>.<len>
+
+   A fragment, so a mirror left marked by an interrupted run still browses.
+   The secret is 64 CSPRNG bits drawn once per run and never written to disk,
+   which is what makes the mark unforgeable: a site cannot spell one, so no
+   sanitiser has to keep hostile bytes away from it. <len> is the byte length
+   of the reference text immediately preceding the mark, so the pass never has
+   to guess where that reference starts. <class> is the context htsparse saw,
+   checked against the resolved type so a mismatch fails loudly. */
+#define SINGLEFILE_MARK_INTRO "#!"
+#define SINGLEFILE_SECRET_HEX 16
+
+/* Bytes before the ".<class>.<len>" tail. */
+#define SINGLEFILE_INTRO_LEN                                                   \
+  (sizeof(SINGLEFILE_MARK_INTRO) - 1 + SINGLEFILE_SECRET_HEX)
+
+/* <class>: what the referencing context expects. */
+#define SINGLEFILE_CLASS_ANY '-'
+#define SINGLEFILE_CLASS_CSS 'c'
+#define SINGLEFILE_CLASS_JS 'j'
+
+/* Enough for the intro, the secret, both separators and a 20-digit length. */
+#define SINGLEFILE_MARK_MAX 64
+
+/* Release this run's secret. */
+void singlefile_free(httrackp *opt);
+
+/* "#!<secret>" for this run, or NULL if no CSPRNG was available (in which case
+   nothing may be marked). Draws the secret on first use. */
+const char *singlefile_intro(httrackp *opt);
+
+/* HTS_FALSE if [body,len) already contains this run's intro, in which case the
+   document must not be marked: RFC 2046 gives the generator the same duty for
+   a MIME boundary. Only a 2^-64 coincidence can trip it, since the intro is
+   not something fetched content can spell. */
+hts_boolean singlefile_may_mark(httrackp *opt, const char *body, size_t len);
+
+/* Write the mark for a reference of reflen bytes into buf; returns buf. */
+const char *singlefile_mark(httrackp *opt, char *buf, size_t bufsize, char cls,
+                            size_t reflen);
+
+/* The class a reference in this context may inline as, or 0 to leave it alone.
+   tag_name points just past the '<' of the enclosing start tag, or NULL when
+   there is none (inside a stylesheet or a script); attr at the attribute name.
+   Everything htsparse detects is inlinable unless it names a page. */
+char singlefile_ref_class(const char *tag_name, const char *attr);
+
+/* Rewrite every HTML page the mirror produced, then strip the marks left in
+   the assets. No-op unless opt->single_file; call once the tree is final,
+   after the update purge. */
 void singlefile_process_mirror(httrackp *opt);
 
-/* Rewrite one HTML document held in memory, appending the result to out.
-   root is the mirror directory that references may not escape; page_path is
-   the document's own path under it (both UTF-8, '/' or native separators).
-   page_budget caps the total inlined bytes, since nested @import fans out
+/* Expand the marks in the document held in memory, appending the result to
+   out. root is the mirror directory that references may not escape; page_path
+   is the document's own path under it (both UTF-8, '/' or native separators).
+   page_budget caps the total inlined bytes, since a nested @import fans out
    multiplicatively; the mirror pass passes SINGLEFILE_MAX_PAGE_SIZE.
-   Returns HTS_TRUE if at least one reference was replaced; out may still
-   differ from the input when that is HTS_FALSE, since a style or srcset value
-   is re-serialized in place. */
+   Returns HTS_TRUE if at least one reference was replaced. */
 hts_boolean singlefile_rewrite_html(httrackp *opt, const char *root,
                                     const char *page_path, const char *html,
                                     size_t html_len, LLint page_budget,

src/htsweb.c

@@ -67,6 +67,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsurlport.h"
 #include "htsweb.h"
 #include "htscharset.h"
+#include "htsrandom.h"
 
 #if USE_BEGINTHREAD==0
 #error fatal: no threads support
@@ -101,50 +102,6 @@ static void htsweb_sig_brpipe(int code) {
   /* ignore */
 }
 
-#ifdef _WIN32
-/* RtlGenRandom, resolved at runtime so no import library is needed. */
-typedef BOOLEAN(WINAPI *hts_rtlgenrandom_t)(PVOID buffer, ULONG length);
-#endif
-
-/* Fill buffer with system entropy; HTS_FALSE if no source answered. There is
-   deliberately no weak fallback: the caller mints a secret with it. */
-static hts_boolean hts_random_bytes(void *buffer, size_t size) {
-#ifdef _WIN32
-  hts_boolean ok = HTS_FALSE;
-  HMODULE dll = LoadLibraryA("advapi32.dll");
-
-  if (dll != NULL) {
-    hts_rtlgenrandom_t gen =
-        (hts_rtlgenrandom_t) GetProcAddress(dll, "SystemFunction036");
-
-    /* the ULONG cast must not truncate; callers ask for a few dozen bytes */
-    if (gen != NULL && size <= 0x10000 && gen(buffer, (ULONG) size)) {
-      ok = HTS_TRUE;
-    }
-    FreeLibrary(dll);
-  }
-  return ok;
-#else
-  unsigned char *dst = (unsigned char *) buffer;
-  size_t done = 0;
-  FILE *fp = fopen("/dev/urandom", "rb");
-
-  if (fp == NULL) {
-    return HTS_FALSE;
-  }
-  while (done < size) {
-    const size_t n = fread(dst + done, 1, size - done, fp);
-
-    if (n == 0) { /* short read is a hard failure, not partial credit */
-      break;
-    }
-    done += n;
-  }
-  fclose(fp);
-  return done == size ? HTS_TRUE : HTS_FALSE;
-#endif
-}
-
 /* Threads that never return; no wait may count on them draining. */
 static int nonjoinable_threads = 0;
 

src/htswizard.c

@@ -61,11 +61,6 @@ Please visit our Website: http://www.httrack.com
     assertf((*opt->filters.filptr) < opt->maxfilter);                          \
   } while (0)
 
-typedef struct htspair_t {
-  const char *tag;
-  const char *attr;
-} htspair_t;
-
 /* "embedded" */
 htspair_t hts_detect_embed[] = {
   {"img", "src"},
@@ -133,11 +128,13 @@ int hts_acceptlink(httrackp * opt, int ptr,
   return forbidden_url;
 }
 
-static int cmp_token(const char *tag, const char *cmp) {
+hts_boolean hts_cmp_tag_token(const char *tag, const char *cmp) {
   int p;
 
-  return (strncasecmp(tag, cmp, (p = (int) strlen(cmp))) == 0
-          && !isalnum((unsigned char) tag[p]));
+  return (tag != NULL && strncasecmp(tag, cmp, (p = (int) strlen(cmp))) == 0 &&
+          !isalnum((unsigned char) tag[p]))
+             ? HTS_TRUE
+             : HTS_FALSE;
 }
 
 /* TRUE if (tag, attribute) matches an embedded-asset pair in the table */
@@ -145,7 +142,8 @@ static hts_boolean is_embed_pair(const htspair_t *table, const char *tag,
                                  const char *attribute) {
   int i;
   for (i = 0; table[i].tag != NULL; i++) {
-    if (cmp_token(tag, table[i].tag) && cmp_token(attribute, table[i].attr))
+    if (hts_cmp_tag_token(tag, table[i].tag) &&
+        hts_cmp_tag_token(attribute, table[i].attr))
       return HTS_TRUE;
   }
   return HTS_FALSE;

src/htswizard.h

@@ -56,6 +56,19 @@ hts_boolean hts_robots_forbids(httrackp *opt, const char *adr, const char *fil,
                                hts_boolean filters_decided,
                                hts_boolean filters_refused);
 
+/* A (tag, attribute) pair naming a reference kind. */
+#ifndef HTS_DEF_DEFSTRUCT_htspair_t
+#define HTS_DEF_DEFSTRUCT_htspair_t
+
+typedef struct htspair_t {
+  const char *tag;
+  const char *attr;
+} htspair_t;
+#endif
+
+/* HTS_TRUE if tag starts with the whole token cmp; NULL tag never matches. */
+hts_boolean hts_cmp_tag_token(const char *tag, const char *cmp);
+
 int hts_acceptlink(httrackp * opt, int ptr,
                    const char *adr, const char *fil,
                    const char *tag, const char *attribute,

src/libhttrack.vcxproj

@@ -126,10 +126,12 @@
     <ClCompile Include="htscmdline.c" />
     <ClCompile Include="htsurlport.c" />
     <ClCompile Include="htsmd5.c" />
+    <ClCompile Include="htsnet.c" />
     <ClCompile Include="htsmodules.c" />
     <ClCompile Include="htsname.c" />
     <ClCompile Include="htsparse.c" />
     <ClCompile Include="htsproxy.c" />
+    <ClCompile Include="htsrandom.c" />
     <ClCompile Include="htsrobots.c" />
     <ClCompile Include="htsselftest.c" />
     <ClCompile Include="htssinglefile.c" />

src/proxytrack.vcxproj

@@ -57,7 +57,7 @@
     <ClCompile>
       <!-- Matches proxytrack_CFLAGS in Makefile.am. Standalone: it does not link
            libhttrack, it only borrows headers (hts_effective_mime is a macro). -->
-      <PreprocessorDefinitions>WIN32;_CONSOLE;_MBCS;NO_MALLOCT;ZLIB_CONST;HTS_INTHASH_USES_MD5;ZLIB_DLL;WINVER=0x0601;_WIN32_WINNT=0x0601;_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
+      <PreprocessorDefinitions>WIN32;_CONSOLE;_MBCS;NO_MALLOCT;HTS_NO_LIBHTTRACK;ZLIB_CONST;HTS_INTHASH_USES_MD5;ZLIB_DLL;WINVER=0x0601;_WIN32_WINNT=0x0601;_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <AdditionalIncludeDirectories>$(MSBuildThisFileDirectory);$(MSBuildThisFileDirectory)coucal;$(MSBuildThisFileDirectory)proxy;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
       <WarningLevel>Level3</WarningLevel>
       <MultiProcessorCompilation>true</MultiProcessorCompilation>
@@ -100,6 +100,7 @@
     <ClCompile Include="proxy\proxytrack.c" />
     <ClCompile Include="proxy\store.c" />
     <ClCompile Include="htsurlport.c" />
+    <ClCompile Include="htsnet.c" />
     <ClCompile Include="coucal\coucal.c" />
     <ClCompile Include="htsmd5.c" />
     <ClCompile Include="md5.c" />

src/version.rc

@@ -17,8 +17,8 @@
 #endif
 
 VS_VERSION_INFO VERSIONINFO
-  FILEVERSION 3, 49, 17, 0
-  PRODUCTVERSION 3, 49, 17, 0
+  FILEVERSION 3, 49, 18, 0
+  PRODUCTVERSION 3, 49, 18, 0
   FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
 #ifdef _DEBUG
   FILEFLAGS VS_FF_DEBUG
@@ -35,12 +35,12 @@ BEGIN
     BEGIN
       VALUE "CompanyName", "Xavier Roche"
       VALUE "FileDescription", VER_FILE_DESCRIPTION
-      VALUE "FileVersion", "3.49.17"
+      VALUE "FileVersion", "3.49.18"
       VALUE "InternalName", VER_ORIGINAL_FILENAME
       VALUE "LegalCopyright", "Copyright (C) 1998-2026 Xavier Roche and other contributors. GNU GPL v3 or later."
       VALUE "OriginalFilename", VER_ORIGINAL_FILENAME
       VALUE "ProductName", "HTTrack Website Copier"
-      VALUE "ProductVersion", "3.49-17"
+      VALUE "ProductVersion", "3.49-18"
     END
   END
   BLOCK "VarFileInfo"

src/webhttrack.vcxproj

@@ -97,6 +97,7 @@
   <ItemGroup>
     <ClCompile Include="htsserver.c" />
     <ClCompile Include="htsweb.c" />
+    <ClCompile Include="htsrandom.c" />
     <ClCompile Include="htscmdline.c" />
     <ClCompile Include="htsurlport.c" />
   </ItemGroup>

tests/01_engine-ftp-userpass.test

@@ -1,8 +1,43 @@
 #!/bin/bash
 #
+# Over-long FTP userinfo must fail the link, never log in under a clipped name
+# (#1032).
 
 set -euo pipefail
 
-# ftp_split_userpass bounds an over-long user:pass@ from a hostile ftp:// URL.
-out=$(httrack -O /dev/null -#test=ftp-userpass run)
-grep -q "ftp-userpass self-test OK" <<<"$out"
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftpuser.XXXXXX") || exit 1
+trap 'set +e; rm -rf "${tmpdir}"' EXIT
+trap 'exit 1' HUP INT QUIT TERM
+
+out=$(httrack -O "${tmpdir}/st" "-#test=ftp-userpass" run 2>&1) ||
+    fail "self-test exited non-zero: ${out}"
+grep -q "ftp-userpass self-test OK" <<<"${out}" || fail "unexpected output: ${out}"
+
+# The reachable half: a URL carries the userinfo straight into user[256]/pass[256].
+rep() { awk -v n="$1" 'BEGIN { while (i++ < n) printf "a" }'; }
+
+probe() {
+    rm -rf "${tmpdir}/mir"
+    httrack "ftp://$1@127.0.0.1:1/f.txt" -O "${tmpdir}/mir" -q >/dev/null 2>&1 ||
+        fail "an FTP URL with ${#1}-byte userinfo crashed the engine"
+    cat "${tmpdir}/mir/hts-log.txt"
+}
+
+# The bare name has no ':' to bound it, only the '@', and is the worse branch.
+for u in "$(rep 256):pw" "u:$(rep 256)" "$(rep 256)"; do
+    log=$(probe "${u}")
+    grep -q "FTP user name or password too long" <<<"${log}" ||
+        fail "over-long userinfo was not refused: ${log}"
+done
+
+# 255 still fits, and a plain login is the outsider a widened gate would refuse.
+for u in "$(rep 255):pw" "u:$(rep 255)" "$(rep 255)" "bob:secret"; do
+    log=$(probe "${u}")
+    grep -q "Unable to connect to the server" <<<"${log}" ||
+        fail "userinfo that fits did not reach the connect: ${log}"
+done

tests/102_local-ftp-refetch.test

@@ -52,19 +52,7 @@ serverlog="${tmpdir}/server.out"
 "$python" "$server" --root "$(nativepath "$root")" \
     --mode-file "$(nativepath "$mode")" >"$serverlog" 2>&1 &
 serverpid=$!
-port=
-for _ in $(seq 1 300); do
-    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "ftp server exited early: $(cat "$serverlog")" >&2
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
-    exit 1
-}
+port=$(discover_server_port "$serverlog" "$serverpid") || exit 1
 host="127.0.0.1_${port}"
 urls=()
 for name in keep empty stay; do

tests/105_suite-timeout.test

@@ -23,8 +23,11 @@ fail() {
 }
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_sutmo.XXXXXX") || exit 1
-trap 'set +e; rm -rf "$tmp"' EXIT
-trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
+# A colon-free root for the PATH shim below: MSYS hands out a drive-letter TMPDIR
+# and a PATH entry carrying a colon is read as two, which leaves it unreachable.
+shim=$(mktemp -d /tmp/httrack_sutmoshim.XXXXXX 2>/dev/null) || shim="$tmp/starve"
+trap 'set +e; rm -rf "$tmp" "$shim"' EXIT
+trap 'rm -rf "$tmp" "$shim"' HUP INT QUIT PIPE TERM
 out="$tmp/out"
 
 # --- a hanging test is killed, named, and reported 124 ----------------------
@@ -65,17 +68,20 @@ grep -q 'slow but healthy' "$out" || fail "healthy test output lost"
 
 # The killed tree must really be gone, or the next test inherits its ports.
 sleep 1
-! grep -q "$tmp/90_wedged.test" <<<"$(ps -A -o args 2>/dev/null)" ||
+! grep -q "$tmp/90_wedged.test" <<<"$(ps_snapshot)" ||
     fail "the wedged test survived the guard"
 
 # --- the budget is wall clock, not a count of poll iterations ----------------
 # Starve the poll and a counted budget stretches with it: 10 polls of a 0.1s tick
 # become 40s, so the guard fires long after the step it was meant to beat (#795).
+# HTTRACK_POLL_SLEEP forces the forked tick, the only one a shimmed sleep reaches;
+# the loop it stretches is the same one poll_wait's fd tick drives.
 (
-    starve_sleep "$tmp/starve" 4 || fail "could not install the slow sleep"
+    starve_sleep "$shim" 4 || fail "could not install the slow sleep"
     start=$SECONDS
     rc=0
-    HTTRACK_TEST_TIMEOUT=1 bash "$driver" "$tmp/90_wedged.test" >"$out" 2>&1 || rc=$?
+    HTTRACK_POLL_SLEEP=1 HTTRACK_TEST_TIMEOUT=1 \
+        bash "$driver" "$tmp/90_wedged.test" >"$out" 2>&1 || rc=$?
     elapsed=$((SECONDS - start))
     test "$rc" -eq 124 || fail "starved guard reported $rc, want 124"
     # Generous: the diagnostics dump runs inside this window too.
@@ -139,6 +145,142 @@ HTTRACK_TEST_TIMEOUT=600 bash "$driver" "$tmp/94_pacer.test" >"$out" 2>&1 || rc=
 test "$rc" -eq 0 || fail "a 60s step under a 600s budget reported $rc, want 0"
 grep -q 'the pacer let it through' "$out" || fail "the pacer skipped a test that fits"
 
+# --- the process lists survive a host with no ps ----------------------------
+# Fedora's build root ships no procps, and the guard then named nothing (#1021).
+if ! is_windows && test -r /proc/self/stat; then
+    mkdir -p "$tmp/nops"
+    printf '#!/bin/sh\nexit 127\n' >"$tmp/nops/ps"
+    chmod +x "$tmp/nops/ps"
+    # A shell copy, not a renamed sleep: uutils refuses an argv[0] it does not own
+    # (#1042). Either works, since both lists match on the basename alone.
+    cp "${BASH:-$(command -v sh)}" "$tmp/httrack" || fail "no shell to copy"
+    "$tmp/httrack" -c : || fail "cannot exec out of $tmp (noexec?)"
+    # Read-write, so the open does not block and the timeout can bound it: the
+    # fakes then fork nothing for the SIGABRT to orphan, and a test killed
+    # outright leaks no process named httrack onto the runner.
+    mkfifo "$tmp/hold"
+    hold="read -t 60 _ <>'$tmp/hold'"
+    # Started from a subshell so ppid and pgid differ: swapping those two columns
+    # is invisible whenever the test shell happens to lead its own group.
+    (
+        "$tmp/httrack" -c "$hold" &
+        echo $! >"$tmp/fake.pid"
+    )
+    # Its own group, to prove the filter keeps an outsider out.
+    (
+        set -m
+        "$tmp/httrack" -c "$hold" &
+        echo $! >"$tmp/outside.pid"
+    )
+    fake=$(cat "$tmp/fake.pid")
+    outside=$(cat "$tmp/outside.pid")
+    trap 'set +e; kill "$fake" "$outside" 2>/dev/null; rm -rf "$tmp"' EXIT
+    # /proc holds the pid from the fork on, so a fake that never exec'd would fail
+    # some later assertion instead of this one.
+    running() {
+        local i=0 argv0
+        while test "$i" -lt 20; do
+            { read -r -d '' argv0 <"/proc/$1/cmdline"; } 2>/dev/null &&
+                test "${argv0##*/}" = httrack && return 0
+            sleep 0.1
+            i=$((i + 1))
+        done
+        return 1
+    }
+    running "$fake" || fail "the fake engine never started"
+    running "$outside" || fail "the outsider never started"
+    # ppid and pgid of $1: the oracle the listing is checked against.
+    procfields() { awk '{ sub(/.*\) /, ""); print $2, $3 }' "/proc/$1/stat"; }
+    procstate() { awk '{ sub(/.*\) /, ""); print $1 }' "/proc/$1/stat" 2>/dev/null || echo gone; }
+    read -r fppid pgid _ <<<"$(procfields "$fake")"
+    read -r _ opgid _ <<<"$(procfields "$outside")"
+    test "$fppid" != "$pgid" || fail "the fake shares ppid and pgid ($pgid)"
+    test "$opgid" != "$pgid" || fail "the outsider landed in the group under test"
+    (
+        PATH="$tmp/nops:$PATH"
+        ! ps -A >/dev/null 2>&1 || fail "the ps shim did not take"
+        snap=$(list_stray_processes "$pgid" group)
+        # Line 1 is the header every consumer drops; a row there would be lost.
+        case "$(head -n1 <<<"$snap")" in
+        [0-9]*) fail "the no-ps listing has no header: $snap" ;;
+        esac
+        ! grep -qE "^$outside " <<<"$snap" ||
+            fail "no-ps group list reported pid $outside, in another group"
+        read -r _ rppid rpgid _ <<<"$(grep -E "^$fake " <<<"$snap")"
+        test "$rppid $rpgid" = "$fppid $pgid" ||
+            fail "no-ps row for $fake reads ppid/pgid '$rppid $rpgid', want '$fppid $pgid'"
+        engines=$(list_engine_pids "$pgid")
+        grep -qx "$fake" <<<"$engines" || fail "no-ps engine list lost pid $fake"
+        ! grep -qx "$outside" <<<"$engines" ||
+            fail "no-ps engine list reached outside the group (pid $outside)"
+        # A platform with no stack mechanism must say so and leave the engine
+        # alone. Shimmed, because no CI leg is one: the Hurd branch below runs
+        # only on the buildds that failed for want of it.
+        # shellcheck disable=SC2016 # the shim reads them, not us
+        printf '#!/bin/sh\ncase "$1" in -s) echo GNU;; *) exec %s "$@";; esac\n' \
+            "$(command -v uname)" >"$tmp/nops/uname"
+        chmod +x "$tmp/nops/uname"
+        grep -q "no stack mechanism known for GNU" <<<"$(request_engine_backtraces "$pgid")" ||
+            fail "a platform with no stack mechanism reported nothing"
+        case "$(procstate "$fake")" in
+        Z | gone) fail "the no-mechanism branch signalled pid $fake" ;;
+        esac
+        rm -f "$tmp/nops/uname"
+
+        # The chain, not just its input: this is what reported nothing on Fedora.
+        stacks=$(request_engine_backtraces "$pgid")
+        ! grep -q 'no engine process left' <<<"$stacks" ||
+            fail "no-ps stack request found no engine to signal: $stacks"
+        # Only Linux signals; elsewhere (Hurd's uname reports "GNU") the chain
+        # owes the reader its no-mechanism report instead of silence.
+        if test "$(uname -s)" = Linux; then
+            # Gone, or a zombie: the fake is reparented, and a container's pid 1
+            # does not always reap. kill -0 succeeds on a zombie, proving nothing.
+            case "$(procstate "$fake")" in
+            Z | gone) ;;
+            *) fail "the SIGABRT never reached pid $fake" ;;
+            esac
+        else
+            grep -q 'no stack mechanism known' <<<"$stacks" ||
+                fail "no stack mechanism was reported on $(uname -s): $stacks"
+        fi
+        # With neither source the dump must say so, not print an empty section.
+        # shellcheck disable=SC2317 # reached through ps_snapshot
+        proc_snapshot() { return 1; }
+        grep -q 'no process list' <<<"$(list_stray_processes "$pgid" group)" ||
+            fail "a host with no ps and no /proc produced no notice"
+    )
+    kill "$fake" "$outside" 2>/dev/null || true
+fi
+
+# --- an emulated buildd hides the engine behind its binfmt interpreter -------
+# Synthetic rows: no runner here has qemu-user, and the column shift is the point.
+if ! is_windows; then
+    (
+        # shellcheck disable=SC2317 # reached through the two matchers below
+        ps_snapshot() {
+            cat <<'EOF'
+PID PPID PGID ELAPSED S COMMAND
+11 1 40 12 S /usr/libexec/qemu-binfmt/hppa-binfmt-P /bld/src/httrack -q http://h/
+12 1 40 12 S /usr/bin/qemu-hppa-static /usr/bin/python3 /t/local-server.py 8080
+13 1 40 12 S /bld/src/httrack -q http://h/
+14 1 40 12 S /usr/bin/strace /bld/src/httrack -q http://h/
+15 1 40 12 S /usr/bin/qemu-img convert /srv/local-server.py out.raw
+16 1 40 12 S /usr/bin/qemu-nbd /mnt/httrack
+17 1 40 12 S /opt/my-custom-binfmt /mnt/httrack
+EOF
+        }
+        # Rows 14 to 17 are the controls: shifting past anything but an emulator
+        # would take a wrapper, or a disk image, for the engine and kill it.
+        engines=$(list_engine_pids 40)
+        test "$engines" = $'11\n13' ||
+            fail "emulated engine list is '$engines', want pids 11 and 13"
+        named=$(list_stray_processes 0 named | awk 'NR > 1 { print $1 }')
+        test "$named" = $'11\n12\n13' ||
+            fail "emulated stray list is '$named', want pids 11, 12 and 13"
+    )
+fi
+
 # --- a wedged crawl yields a symbolized engine stack ------------------------
 # The whole point of the dump: name the frame the engine is stuck in. Windows has
 # neither half (MSYS signals do not reach a native httrack.exe, and that build has

tests/10_crawl-simple.test

@@ -3,6 +3,8 @@
 
 set -euo pipefail
 
-bash check-network.sh || ! echo "skipping online unit tests" || exit 77
+testdir=$(cd "$(dirname "$0")" && pwd)
 
-bash crawl-test.sh --errors 0 --files 5 httrack http://ut.httrack.com/simple/basic.html
+bash "$testdir/check-network.sh" || ! echo "skipping online unit tests" || exit 77
+
+bash "$testdir/crawl-test.sh" --errors 0 --files 5 httrack http://ut.httrack.com/simple/basic.html

tests/110_local-ftp-parallel.test

@@ -52,19 +52,7 @@ done
 serverlog="${tmpdir}/server.out"
 "$python" "$server" --root "$(nativepath "$root")" >"$serverlog" 2>&1 &
 serverpid=$!
-port=
-for _ in $(seq 1 300); do
-    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "ftp server exited early: $(cat "$serverlog")" >&2
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
-    exit 1
-}
+port=$(discover_server_port "$serverlog" "$serverpid") || exit 1
 host="127.0.0.1_${port}"
 
 urls=()

tests/111_local-ftp-update-rest.test

@@ -64,19 +64,7 @@ serverlog="${tmpdir}/server.out"
     --mode-file "$(nativepath "$mode")" --log "$(nativepath "$cmds")" \
     >"$serverlog" 2>&1 &
 serverpid=$!
-port=
-for _ in $(seq 1 300); do
-    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "ftp server exited early: $(cat "$serverlog")" >&2
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
-    exit 1
-}
+port=$(discover_server_port "$serverlog" "$serverpid") || exit 1
 host="127.0.0.1_${port}"
 url="ftp://127.0.0.1:${port}/a.bin"
 mirror="${out}/${host}/a.bin"

tests/11_crawl-cookies.test

@@ -3,9 +3,11 @@
 
 set -euo pipefail
 
-bash check-network.sh || ! echo "skipping online unit tests" || exit 77
+testdir=$(cd "$(dirname "$0")" && pwd)
 
-bash crawl-test.sh --errors 0 --files 3 \
+bash "$testdir/check-network.sh" || ! echo "skipping online unit tests" || exit 77
+
+bash "$testdir/crawl-test.sh" --errors 0 --files 3 \
     --found ut.httrack.com/cookies/third.html \
     --found ut.httrack.com/cookies/second.html \
     --found ut.httrack.com/cookies/entrance.html \

tests/11_crawl-idna.test

@@ -3,10 +3,12 @@
 
 set -euo pipefail
 
-bash check-network.sh || ! echo "skipping online unit tests" || exit 77
+testdir=$(cd "$(dirname "$0")" && pwd)
+
+bash "$testdir/check-network.sh" || ! echo "skipping online unit tests" || exit 77
 
 # unicode tests
-bash crawl-test.sh \
+bash "$testdir/crawl-test.sh" \
     --errors 1 --files 5 \
     --found 'café.ut.httrack.com/unicode-links/café3860.html' \
     --found 'café.ut.httrack.com/unicode-links/café30f4.html' \
@@ -16,7 +18,7 @@ bash crawl-test.sh \
     '+*.ut.httrack.com/*' --robots=0
 
 # unicode tests (bogus links)
-bash crawl-test.sh \
+bash "$testdir/crawl-test.sh" \
     --errors 0 --files 1 \
     --found 'ut.httrack.com/unicode-links/idna_bogus.html' \
     httrack 'http://ut.httrack.com/unicode-links/idna_bogus.html' \

tests/11_crawl-international.test

@@ -3,10 +3,12 @@
 
 set -euo pipefail
 
-bash check-network.sh || ! echo "skipping online unit tests" || exit 77
+testdir=$(cd "$(dirname "$0")" && pwd)
+
+bash "$testdir/check-network.sh" || ! echo "skipping online unit tests" || exit 77
 
 # unicode tests
-bash crawl-test.sh \
+bash "$testdir/crawl-test.sh" \
     --errors 1 --files 10 \
     --found ut.httrack.com/unicode-links/caf%a91bce.html \
     --found ut.httrack.com/unicode-links/café30f4.html \
@@ -21,7 +23,7 @@ bash crawl-test.sh \
     --found ut.httrack.com/unicode-links/utf8.html \
     httrack http://ut.httrack.com/unicode-links/utf8.html
 
-bash crawl-test.sh \
+bash "$testdir/crawl-test.sh" \
     --errors 2 --files 9 \
     --found ut.httrack.com/unicode-links/café3860.html \
     --found ut.httrack.com/unicode-links/café9fa8.html \
@@ -36,7 +38,7 @@ bash crawl-test.sh \
     --found ut.httrack.com/unicode-links/default.html \
     httrack http://ut.httrack.com/unicode-links/default.html
 
-bash crawl-test.sh \
+bash "$testdir/crawl-test.sh" \
     --errors 2 --files 9 \
     --found ut.httrack.com/unicode-links/caf%a9ae52.html \
     --found ut.httrack.com/unicode-links/caf%a9bf59.html \
@@ -51,7 +53,7 @@ bash crawl-test.sh \
     --found ut.httrack.com/unicode-links/iso88591.html \
     httrack http://ut.httrack.com/unicode-links/iso88591.html
 
-bash crawl-test.sh \
+bash "$testdir/crawl-test.sh" \
     --errors 4 --files 9 \
     --found ut.httrack.com/unicode-links/caf%a8%a6c72a.html \
     --found ut.httrack.com/unicode-links/caf%a9bf59.html \

tests/11_crawl-longurl.test

@@ -3,10 +3,12 @@
 
 set -euo pipefail
 
-bash check-network.sh || ! echo "skipping online unit tests" || exit 77
+testdir=$(cd "$(dirname "$0")" && pwd)
+
+bash "$testdir/check-network.sh" || ! echo "skipping online unit tests" || exit 77
 
 # http://code.google.com/p/httrack/issues/detail?id=42&can=1
 # we expect 2 errors only because other links are too longs (to be modified if suitable)
-bash crawl-test.sh --errors 2 --files 1 \
+bash "$testdir/crawl-test.sh" --errors 2 --files 1 \
     --found ut.httrack.com/overflow/longquerywithaccents.html \
     httrack http://ut.httrack.com/overflow/longquerywithaccents.php

tests/11_crawl-parsing.test

@@ -3,10 +3,12 @@
 
 set -euo pipefail
 
-bash check-network.sh || ! echo "skipping online unit tests" || exit 77
+testdir=$(cd "$(dirname "$0")" && pwd)
+
+bash "$testdir/check-network.sh" || ! echo "skipping online unit tests" || exit 77
 
 # http://code.google.com/p/httrack/issues/detail?id=4&can=1
-bash crawl-test.sh --errors 0 --files 4 \
+bash "$testdir/crawl-test.sh" --errors 0 --files 4 \
     --found ut.httrack.com/parsing/back5e1f.gif \
     --found ut.httrack.com/parsing/events.html \
     --found ut.httrack.com/parsing/fade230f4.gif \
@@ -14,21 +16,21 @@ bash crawl-test.sh --errors 0 --files 4 \
     httrack http://ut.httrack.com/parsing/events.html
 
 # http://code.google.com/p/httrack/issues/detail?id=2&can=1
-bash crawl-test.sh --errors 0 --files 3 \
+bash "$testdir/crawl-test.sh" --errors 0 --files 3 \
     --found ut.httrack.com/parsing/background-image.css \
     --found ut.httrack.com/parsing/background-image.html \
     --found ut.httrack.com/parsing/fade.gif \
     httrack http://ut.httrack.com/parsing/background-image.html
 
 # javascript parsing
-bash crawl-test.sh --errors 0 --files 3 \
+bash "$testdir/crawl-test.sh" --errors 0 --files 3 \
     --found ut.httrack.com/parsing/back.gif \
     --found ut.httrack.com/parsing/fade.gif \
     --found ut.httrack.com/parsing/javascript.html \
     httrack http://ut.httrack.com/parsing/javascript.html
 
 # handling of + before query string
-bash crawl-test.sh --errors 0 --files 6 \
+bash "$testdir/crawl-test.sh" --errors 0 --files 6 \
     --found ut.httrack.com/parsing/escaping.html \
     --found "ut.httrack.com/parsing/foo bar30f4.html" \
     --found "ut.httrack.com/parsing/foo bar5e1f.html" \
@@ -39,7 +41,7 @@ bash crawl-test.sh --errors 0 --files 6 \
 
 # handling of # encoded in filename
 # see http://code.google.com/p/httrack/issues/detail?id=25
-bash crawl-test.sh --errors 2 --files 4 \
+bash "$testdir/crawl-test.sh" --errors 2 --files 4 \
     --found "ut.httrack.com/parsing/escaping2.html" \
     --found "ut.httrack.com/parsing/++foo++bar++plus++.html" \
     --found "ut.httrack.com/parsing/foo#bar#.html" \

tests/121_local-ftp-nocache-splice.test

@@ -79,19 +79,7 @@ serverlog="${tmpdir}/server.out"
     --mode-file "$(nativepath "$mode")" --log "$(nativepath "$cmds")" \
     >"$serverlog" 2>&1 &
 serverpid=$!
-port=
-for _ in $(seq 1 300); do
-    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "ftp server exited early: $(cat "$serverlog")" >&2
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
-    exit 1
-}
+port=$(discover_server_port "$serverlog" "$serverpid") || exit 1
 host="127.0.0.1_${port}"
 url="ftp://127.0.0.1:${port}/a.bin"
 mirror="${out}/${host}/a.bin"

tests/12_crawl_https.test

@@ -3,11 +3,13 @@
 
 set -euo pipefail
 
-bash check-network.sh || ! echo "skipping online unit tests" || exit 77
+testdir=$(cd "$(dirname "$0")" && pwd)
+
+bash "$testdir/check-network.sh" || ! echo "skipping online unit tests" || exit 77
 
 if test "${HTTPS_SUPPORT:-}" == "no"; then
     echo "no https support compiled, skipping"
     exit 77
 fi
 
-bash crawl-test.sh --errors 0 --files 5 httrack https://ut.httrack.com/simple/basic.html
+bash "$testdir/crawl-test.sh" --errors 0 --files 5 httrack https://ut.httrack.com/simple/basic.html

tests/153_local-proxytrack-quiet.test

@@ -126,8 +126,13 @@ wait_drained() { # log
 
 quietport=$(freeport)
 : >"$dir/quiet.log" # the drainer creates it asynchronously
-env -u HTS_LOG "$python" -c "$pty_exec" "$dir/quiet.log" \
-    proxytrack "127.0.0.1:$quietport" "127.0.0.1:$(freeport)" "$dir/in.arc" &
+# Subshell, not env(1): uutils' env forks rather than execs, so stop_server would
+# kill the wrapper and leave proxytrack holding the pty open (#1042).
+(
+    unset HTS_LOG
+    exec "$python" -c "$pty_exec" "$dir/quiet.log" \
+        proxytrack "127.0.0.1:$quietport" "127.0.0.1:$(freeport)" "$dir/in.arc"
+) &
 quietpid=$!
 wait_listen "$dir/quiet.log" "$quietpid"
 

tests/154_local-proxytrack-arc-roundtrip.test

@@ -58,11 +58,13 @@ test "$(convert "$dir/legacy.arc" "$dir/c.arc")" = 1 ||
 # a third would put the record out of its reach again.
 hdrlen=$(head -1 "$dir/a.arc" | wc -c)
 declared=$(head -1 "$dir/a.arc" | awk '{print $NF}')
-sep=$(tail -c +$((hdrlen + declared + 1)) "$dir/a.arc" | head -c 3 | od -An -c | tr -d ' ')
+# od seeks for itself: piping the archive into `head -c` leaves the writer with
+# a closed pipe, and under pipefail that SIGPIPE is the test's exit status.
+sep=$(od -An -c -j $((hdrlen + declared)) -N 3 "$dir/a.arc" | tr -d ' ')
 test "$sep" = '\n\nh' || fail "version block ends on '$sep', expected two newlines"
 # The block itself must not end on one either: counting the blank line and then
 # writing a second one round-trips, but leaves the length a byte too long.
-inner=$(tail -c +$((hdrlen + declared - 1)) "$dir/a.arc" | head -c 2 | od -An -c | tr -d ' ')
+inner=$(od -An -c -j $((hdrlen + declared - 2)) -N 2 "$dir/a.arc" | tr -d ' ')
 test "$inner" != '\n\n' || fail "declared length still counts the blank line"
 
 # An empty archive is not a corrupt one...

tests/171_watchdog-heartbeat.test

@@ -8,6 +8,10 @@
 
 set -euo pipefail
 
+# Before the driver is sourced: with these set, anything it starts posts a commit
+# status on whatever the surrounding shell was last looking at.
+unset WATCHDOG_TOKEN WATCHDOG_REPO WATCHDOG_SHA WATCHDOG_URL
+
 testdir=$(cd "$(dirname "$0")" && pwd)
 # shellcheck source=tests/ci-windows-suite.sh
 . "${testdir}/ci-windows-suite.sh"
@@ -138,7 +142,8 @@ test "$killed" -le 10080 || fail "killed at $killed, want 9960 within a tick"
 
 # taskkill is a grandchild of its own target, so a leaves-first /T would reap the
 # watchdog before the root (#953): kill_tree here never returns, and the stubs
-# record the order, which the code under test cannot write to.
+# record the order, which the code under test cannot write to. The reporter goes
+# first: the kill runs no EXIT trap, and the tree walk reaches this heartbeat.
 rec="$tmp/killed"
 : >"$rec"
 vnow=9000
@@ -146,6 +151,7 @@ ticks=0
 printf 'RUN 96_wedged.test at 0s\n' >"$progress"
 rc=0
 (
+    watchdog=777
     kill_pid() { echo "DIRECT $1" >>"$rec"; }
     kill_tree() {
         echo "TREE $1" >>"$rec"
@@ -154,10 +160,12 @@ rc=0
     ci_suite_heartbeat 960 360 "$progress" 900 4242 >"$tmp/hedge" 2>&1
 ) || rc=$?
 test "$rc" -eq 9 || fail "the tree kill never fired: watchdog returned $rc"
-test "$(sed -n 1p "$rec")" = "DIRECT 4242" ||
+test "$(sed -n 1p "$rec")" = "DIRECT 777" ||
+    fail "the reporter was not killed ahead of the suite: $(tr '\n' '/' <"$rec")"
+test "$(sed -n 2p "$rec")" = "DIRECT 4242" ||
     fail "the target was not signalled directly ahead of the tree walk: $(tr '\n' '/' <"$rec")"
-test "$(sed -n 2p "$rec")" = "TREE 4242" ||
+test "$(sed -n 3p "$rec")" = "TREE 4242" ||
     fail "the tree was not killed after the direct signal: $(tr '\n' '/' <"$rec")"
-test "$(sed -n '$=' "$rec")" -eq 2 || fail "extra kills: $(tr '\n' '/' <"$rec")"
+test "$(sed -n '$=' "$rec")" -eq 3 || fail "extra kills: $(tr '\n' '/' <"$rec")"
 
 echo "heartbeat OK"

tests/172_ci-windows-driver.test

@@ -6,6 +6,10 @@
 
 set -euo pipefail
 
+# Before the driver is sourced: with these set, anything it starts posts a commit
+# status on whatever the surrounding shell was last looking at.
+unset WATCHDOG_TOKEN WATCHDOG_REPO WATCHDOG_SHA WATCHDOG_URL
+
 testdir=$(cd "$(dirname "$0")" && pwd)
 driver="$testdir/ci-windows-suite.sh"
 
@@ -39,36 +43,96 @@ grep -q usage <<<"$out" || fail "argless run said: $out"
 # sibling's.
 run=$tmp/run
 mkdir -p "$run"
-cp "$driver" "$testdir/testlib.sh" "$testdir/test-timeout.sh" "$run/"
+cp "$driver" "$testdir/testlib.sh" "$testdir/test-timeout.sh" \
+    "$testdir/ci-windows-watchdog.ps1" "$run/"
 chmod u+w "$run"/*.sh # distcheck's srcdir is read-only, and cp carries that over
 cat >>"$run/testlib.sh" <<'EOF'
 reap_leftover_processes() { return 0; }
 EOF
+# Records who sourced this copy, so assert_staged can tell the staged driver from
+# the srcdir one. test-timeout.sh sources it too, hence the caller and not a bare
+# marker file.
+cat >>"$run/testlib.sh" <<EOF
+printf '%s\n' "\${BASH_SOURCE[1]##*/}" >>"$tmp/sourced-by"
+EOF
+: >"$tmp/sourced-by"
+
+# The suite under test has to be the staged copy: only it sources the testlib.sh
+# neutered above, and the srcdir one would reap sibling engines under "make check -j".
+assert_staged() {
+    grep -qx 'ci-windows-suite.sh' "$tmp/sourced-by" ||
+        fail "the srcdir driver ran, not the staged copy: $(sort -u "$tmp/sourced-by" | tr '\n' ' ')"
+    : >"$tmp/sourced-by"
+}
 
 bin=$tmp/bin
 mkdir -p "$bin"
 printf '#!/bin/sh\nexit 0\n' >"$bin/httrack"
-# shellcheck disable=SC2016 # $2 is the stub's own argument
-printf '#!/bin/sh\necho "$2"\n' >"$bin/cygpath"
+# The first child the driver forks: the token has to be gone before it runs.
+# shellcheck disable=SC2016 # $2 and the token are the stub's own expansions
+printf '#!/bin/sh\necho "TOKEN=${WATCHDOG_TOKEN:-}" >>%s\necho "$2"\n' "$tmp/cygpathtoken" \
+    >"$bin/cygpath"
 chmod +x "$bin/httrack" "$bin/cygpath"
-test -x "$bin/httrack" || {
+# The driver puts this bindir first on PATH, so this is the interpreter it finds.
+# Unconditional, because a real PowerShell here runs the real watchdog, and only
+# the scrub above stands between that and api.github.com.
+wdargv=$tmp/wdargv
+wdmark=$tmp/wdmark
+: >"$wdargv"
+cat >"$bin/pwsh" <<EOF
+#!/bin/sh
+{ echo "TOKEN=\${WATCHDOG_TOKEN:-}"; printf '%s\n' "\$@"; } >>"$wdargv"
+echo "watchdog ready"
+# Ticks rather than sleeps: a pid can be reused, a running reporter cannot.
+n=0
+while :; do
+    n=\$((n + 1))
+    echo "\$n" >"$wdmark.part"
+    mv "$wdmark.part" "$wdmark"
+    sleep 0.2
+done
+EOF
+chmod +x "$bin/pwsh"
+# Ahead of any command -v, which asks access(X_OK) and so reads false here.
+test -x "$bin/pwsh" || {
     echo "SKIP: ${TMPDIR:-/tmp} is noexec, the stub bindir cannot be run"
     exit 77
 }
+test "$(PATH="$bin:$PATH" command -v pwsh)" = "$bin/pwsh" ||
+    fail "the stub does not shadow PowerShell: $(PATH="$bin:$PATH" command -v pwsh)"
+
+# The stub advances $wdmark every 0.2s, so a marker that moves says it still runs.
+# Sampled three times, or a loaded box starving the stub would read as a kill.
+still_ticking() {
+    local a b
+    a=$(cat "$wdmark" 2>/dev/null || true)
+    for _ in 1 2 3; do
+        sleep 1
+        b=$(cat "$wdmark" 2>/dev/null || true)
+        test -n "$b" && test "$a" != "$b" && return 0
+    done
+    return 1
+}
 
 # One per glob the driver enumerates: an empty category is counted as a failing
 # test named after the unexpanded pattern, which would drown the accounting.
-for t in 00_runnable 10_engine-pass 13_zlib-pass 14_local-pass 15_watchdog-pass \
+for t in 00_runnable 13_zlib-pass 14_local-pass 15_watchdog-pass \
     16_crawl_proxy_https 17_crawl-log-salvage; do
     printf '#!/bin/sh\nexit 0\n' >"$run/$t.test"
 done
+# One test reports back what it inherited: the token the step is handed can post
+# commit statuses, and a forged one has already reached a commit under test.
+# shellcheck disable=SC2016 # the stub expands it, not this shell
+printf '#!/bin/sh\necho "TOKEN=${WATCHDOG_TOKEN:-}" >%s\nexit 0\n' "$tmp/childtoken" \
+    >"$run/10_engine-pass.test"
 printf '#!/bin/sh\nexit 77\n' >"$run/11_engine-skip.test"
 printf '#!/bin/sh\nexit 3\n' >"$run/12_engine-fail.test"
 
 cd "$run"
 rc=0
-out=$(RUNNER_TEMP="$tmp" GITHUB_STEP_SUMMARY="$tmp/summary" \
+out=$(RUNNER_TEMP="$tmp" GITHUB_STEP_SUMMARY="$tmp/summary" WATCHDOG_TOKEN=s3cr3t \
     bash ./ci-windows-suite.sh "$bin" 2>&1) || rc=$?
+assert_staged
 # The skip set and the pass floor are pinned to the real suite, so a stub run
 # ends on the floor; what is under test is the tally that reaches it.
 test "$rc" -eq 1 || fail "stub suite exited $rc, want 1 from the pass floor"
@@ -77,6 +141,20 @@ grep -q '^ran=9 pass=7 fail=1 skip=1$' <<<"$out" ||
 grep -q '::error::only 7 tests passed (1 skipped)' <<<"$out" ||
     fail "the pass floor did not report the count"
 grep -q 'FAIL 12_engine-fail.test (exit 3)' <<<"$out" || fail "the failing test was not named"
+
+# This launches the watchdog the way the real driver does, not through a test
+# sourcing the helper, proving it starts and holds a token nothing else sees.
+grep -q '^watchdog ready$' watchdog.log || fail "no watchdog was launched: $(cat watchdog.log)"
+test -r "$tmp/childtoken" || fail "the test that reads its environment did not run"
+grep -qx 'TOKEN=' "$tmp/childtoken" ||
+    fail "a test inherited the status token: $(cat "$tmp/childtoken")"
+grep -qx 'TOKEN=s3cr3t' "$wdargv" ||
+    fail "the watchdog was launched without its token: $(cat "$wdargv")"
+test -s "$tmp/cygpathtoken" || fail "the cygpath stub never ran"
+grep -q 'TOKEN=s3cr3t' "$tmp/cygpathtoken" &&
+    fail "the driver forked a child before scrubbing its token: $(cat "$tmp/cygpathtoken")"
+# An orphan runs out its own deadline, re-posting a frozen tail past the step (#795).
+! still_ticking || fail "the driver exited leaving its reporter running"
 # Positive control for the gate below: with every category matched it stays silent.
 grep -q 'matched no tests' <<<"$out" && fail "a full suite reported an empty category"
 
@@ -92,6 +170,7 @@ rm -f "$run/13_zlib-pass.test"
 rc=0
 out=$(RUNNER_TEMP="$tmp" GITHUB_STEP_SUMMARY="$tmp/summary" \
     bash ./ci-windows-suite.sh "$bin" 2>&1) || rc=$?
+assert_staged
 test "$rc" -eq 1 || fail "an empty category exited $rc, want 1"
 grep -q '::error::test category zlib matched no tests' <<<"$out" ||
     fail "the empty category was not named: $out"
@@ -105,9 +184,40 @@ rm -f "$run/00_runnable.test"
 rc=0
 out=$(RUNNER_TEMP="$tmp" GITHUB_STEP_SUMMARY="$tmp/summary" \
     bash ./ci-windows-suite.sh "$bin" 2>&1) || rc=$?
+assert_staged
 test "$rc" -eq 1 || fail "an empty single-test category exited $rc, want 1"
 grep -q '::error::test category runnable matched no tests' <<<"$out" ||
     fail "the empty single-test category was not named: $out"
 grep -q 'FAIL 00_runnable' <<<"$out" && fail "the unexpanded name was run as a test: $out"
 
+# The wedge branch has to end the reporter itself: its kill of the suite is a
+# TerminateProcess, which runs no EXIT trap. Forked the way the driver forks it,
+# since the branch reads $watchdog from the shell it was forked from.
+wedge_leg() (
+    PATH="$bin:$PATH"
+    export PATH
+    prog=$run/wedge-progress.log
+    printf 'RUN 99_wedged.test at 0s\n' >"$prog"
+    ci_watchdog_pid=''
+    ci_start_native_watchdog "$prog" || fail "the wedge leg launched no watchdog"
+    watchdog=$ci_watchdog_pid
+    still_ticking || fail "the wedge leg's reporter was not running to begin with"
+    # Its own process group, so the tree kill below cannot reach this shell.
+    set -m
+    sleep 300 &
+    victim=$!
+    set +m
+    ci_suite_heartbeat 0 1 "$prog" 0 "$victim" >"$tmp/wedge.out" 2>&1 &
+    wait "$!"
+    grep -q '::error title=suite watchdog::' "$tmp/wedge.out" ||
+        fail "the wedge branch was never reached: $(cat "$tmp/wedge.out")"
+    ! still_ticking || fail "the wedge kill orphaned the reporter"
+)
+# Held back: bash announces every job it reaps from a signal, which reads as a
+# failure in a log that passed.
+wedge_leg >"$tmp/wedge.log" 2>&1 || {
+    cat "$tmp/wedge.log" >&2
+    exit 1
+}
+
 echo "ci-windows driver OK"

tests/183_altstack-worker.test

@@ -63,11 +63,24 @@ grep -q "^Caught signal 11$" <<<"$worker" || fail "-#c=threadstack: no 'Caught s
 # worker's, which is what had no alternate stack.
 worker_frames=$(frame_count "$worker")
 plain_frames=$(frame_count "$plain")
-test "$worker_frames" -ge 100 ||
-    fail "-#c=threadstack: $worker_frames frames, expected a runaway recursion"
 test "$plain_frames" -lt 100 ||
     fail "-#c=segv: $plain_frames frames, the threshold no longer discriminates"
 
+# Naming no frame at all is the unwinder failing outright, not a weak one, and
+# the floor below would take it for loong64. hts_print_backtrace() says which.
+if grep -q "No stack trace available" <<<"$worker"; then
+    fail "-#c=threadstack: the report carries no stack trace at all"
+fi
+
+# armhf and loong64 cannot unwind past the frame that faulted on the guard page
+# (180 skips on the same floor); the release trace below still judges the fix.
+if [ "$worker_frames" -ge 1 ] && [ "$worker_frames" -lt 6 ]; then
+    echo "-#c=threadstack: $worker_frames frames, too few for this unwinder to" \
+        "show the recursion" >&2
+elif [ "$worker_frames" -lt 100 ]; then
+    fail "-#c=threadstack: $worker_frames frames, expected a runaway recursion"
+fi
+
 # Everything above only proves the stack gets installed: the crashing worker
 # never returns, so the release hook never runs there. -#test=threadwait spawns
 # workers that do return, and the syscall trace says what each did with its

tests/206_install-headers-c99.test

@@ -88,32 +88,36 @@ env MAKEFLAGS= MFLAGS= "$make" -C "$abs_top_builddir/src" install-DevIncludesDAT
     }
 
 headers=("$tmp/include/httrack"/*.h)
-[ "${#headers[@]}" -ge 10 ] || fail "only ${#headers[@]} headers installed, the list cannot be right"
-[ -f "$tmp/include/httrack/htssafe.h" ] || fail "htssafe.h was not installed"
-
-# No libc hands getnameinfo() to a strict-ISO unit, so a consumer of the two
-# socket headers must ask for POSIX. Prefer 2001: 2008 also un-hides strnlen,
-# which would stop those two covering that half of #972.
-posix_argv=(-D_POSIX_C_SOURCE=200112L)
-printf '#include <httrack/htsnet.h>\n' >"$tmp/tu.c"
-if ! "${cc_argv[@]}" "${cpp_argv[@]}" "${posix_argv[@]}" "-std=${stds[0]}" \
-    -fsyntax-only "$tmp/tu.c" 2>/dev/null; then
-    fail "htsnet.h needs POSIX.1-2008, which un-hides strnlen and would leave that half of #972 uncovered"
-fi
+# Derived from DevIncludes_DATA so a new header needs no edit here. A header
+# dropped from that list moves both sides, hence the named checks below.
+declared=$(awk '/^DevIncludes_DATA[[:space:]]*=/ { inlist = 1 }
+    inlist {
+        last = ($0 !~ /\\$/)
+        sub(/^[^=]*=/, "")
+        gsub(/\\/, " ")
+        for (i = 1; i <= NF; i++)
+            if ($i ~ /\.h$/) n++
+        if (last) exit
+    }
+    END { print n + 0 }' "${abs_top_srcdir:?}/src/Makefile.am")
+[ "$declared" -ge 10 ] || fail "read only $declared headers out of DevIncludes_DATA, the parse is wrong"
+[ "${#headers[@]}" -eq "$declared" ] ||
+    fail "${#headers[@]} headers installed, DevIncludes_DATA declares $declared"
+for h in htssafe.h htsnet.h htsopt.h; do
+    [ -f "$tmp/include/httrack/$h" ] || fail "$h was not installed"
+done
 
+# No -D_POSIX_C_SOURCE anywhere below: every installed header, htsnet.h and
+# htsopt.h included, must compile against the strict-ISO libc alone (#1001).
 count=0
 bad=0
 for h in "${headers[@]}"; do
     b=$(basename "$h")
-    argv=("${cpp_argv[@]}")
-    case "$b" in
-    htsnet.h | htsopt.h) argv+=("${posix_argv[@]}") ;;
-    esac
     printf '#include <httrack/%s>\n' "$b" >"$tmp/tu.c"
     for std in "${stds[@]}"; do
         for mode in -UHTS_INTERNAL_BYTECODE -DHTS_INTERNAL_BYTECODE; do
             count=$((count + 1))
-            if ! "${cc_argv[@]}" "${argv[@]}" "-std=$std" "$mode" -fsyntax-only "$tmp/tu.c" 2>"$tmp/cc.log"; then
+            if ! "${cc_argv[@]}" "${cpp_argv[@]}" "-std=$std" "$mode" -fsyntax-only "$tmp/tu.c" 2>"$tmp/cc.log"; then
                 echo "$b does not compile under -std=$std ($mode):" >&2
                 head -5 "$tmp/cc.log" >&2
                 bad=1
@@ -122,6 +126,7 @@ for h in "${headers[@]}"; do
     done
 done
 echo "compiled ${#headers[@]} headers x ${#stds[@]} std x 2 bytecode modes = $count units" >&2
+[ "$count" -eq $((${#headers[@]} * ${#stds[@]} * 2)) ] || fail "the compile loop skipped units"
 [ "$bad" -eq 0 ] || fail "installed headers do not compile in strict-ISO mode"
 
 # A macro body is only compiled where it expands, so the include-only loop above
@@ -178,19 +183,23 @@ int main(void) {
   return 0;
 }
 EOF
+# Vacuous if the gate handed us libc's strnlen. Match that branch, not the
+# loop, or a mutated loop reads as a lost fallback instead. Needs no runnable
+# binary, so it stays outside the gate below.
+for std in "${stds[@]}"; do
+    pp=$("${cc_argv[@]}" "${cpp_argv[@]}" "-std=$std" -E "$tmp/strnlen.c") ||
+        fail "cannot preprocess the htssafe_strnlen_ probe under -std=$std"
+    if grep -q 'return strnlen' <<<"$pp"; then
+        fail "-std=$std left htssafe_strnlen_ on libc, the differential proves nothing"
+    fi
+done
+
 # A noexec TMPDIR or a cross-compiler would red the run for the wrong reason.
 printf 'int main(void) { return 0; }\n' >"$tmp/tu.c"
 if ! ("${cc_argv[@]}" "${cpp_argv[@]}" -o "$tmp/tu" "$tmp/tu.c" 2>/dev/null && "$tmp/tu"); then
     echo "cannot run a binary built in $tmp, skipping the htssafe_strnlen_ differential" >&2
 else
     for std in "${stds[@]}"; do
-        # Vacuous if the gate handed us libc's strnlen. Match that branch, not
-        # the loop, or a mutated loop reads as a lost fallback instead.
-        pp=$("${cc_argv[@]}" "${cpp_argv[@]}" "-std=$std" -E "$tmp/strnlen.c") ||
-            fail "cannot preprocess the htssafe_strnlen_ probe under -std=$std"
-        if grep -q 'return strnlen' <<<"$pp"; then
-            fail "-std=$std left htssafe_strnlen_ on libc, the check below proves nothing"
-        fi
         "${cc_argv[@]}" "${cpp_argv[@]}" "-std=$std" -o "$tmp/strnlen" "$tmp/strnlen.c" 2>"$tmp/cc.log" || {
             head -5 "$tmp/cc.log" >&2
             fail "the htssafe_strnlen_ probe does not build under -std=$std"
@@ -221,9 +230,6 @@ else
     for h in "${headers[@]}"; do
         b=$(basename "$h")
         argv=("${base_cpp_argv[@]}" -Wall -Wextra -Werror)
-        case "$b" in
-        htsnet.h | htsopt.h) argv+=("${posix_argv[@]}") ;;
-        esac
         printf '#include <httrack/%s>\nint main() { return 0; }\n' "$b" >"$tmp/tu.cpp"
         for std in "${cxx_stds[@]}"; do
             for mode in -UHTS_INTERNAL_BYTECODE -DHTS_INTERNAL_BYTECODE; do

tests/210_appstream-metainfo.test

@@ -6,14 +6,34 @@
 set -euo pipefail
 
 srcdir="${abs_top_srcdir:?not run under make check}"
-metainfo="${srcdir}/html/server/div/com.httrack.WebHTTrack.metainfo.xml"
 
-command -v appstreamcli >/dev/null 2>&1 || {
-    echo "appstreamcli not installed; skipping" >&2
+# --no-net/--nonet: validate the file, not the reachability of the screenshot
+# host. Fedora's build gates on appstream-util, stricter about icons and
+# metadata than appstreamcli.
+validate() {
+    case $1 in
+    appstreamcli) appstreamcli validate --no-net --explain "$2" ;;
+    appstream-util) appstream-util validate-relax --nonet "$2" ;;
+    *)
+        echo "FAIL: no validator named $1" >&2
+        exit 1
+        ;;
+    esac
+}
+
+tools=()
+for tool in appstreamcli appstream-util; do
+    command -v "${tool}" >/dev/null 2>&1 && tools+=("${tool}")
+done
+test ${#tools[@]} -gt 0 || {
+    echo "no appstream validator installed; skipping" >&2
     exit 77
 }
-test -f "${metainfo}" || {
-    echo "FAIL: ${metainfo} not found" >&2
+
+# html/Makefile.am installs these by glob; a second one must not ship unchecked.
+metainfos=("${srcdir}"/html/server/div/*.metainfo.xml)
+test -f "${metainfos[0]}" || {
+    echo "FAIL: no metainfo under ${srcdir}/html/server/div" >&2
     exit 1
 }
 
@@ -23,21 +43,18 @@ work=$(mktemp -d "${TMPDIR:-/tmp}/appstream.XXXXXX") || {
 }
 trap 'set +e; rm -rf "${work}"' EXIT
 
-# --no-net: validate the file, not the reachability of the screenshot host.
-validate() { appstreamcli validate --no-net --explain "$1"; }
-
-validate "${metainfo}" || {
-    echo "FAIL: metainfo does not validate" >&2
-    exit 1
-}
-
-# Positive control: an appstreamcli that never fails would pass the above.
-grep -v '<id>com.httrack.WebHTTrack</id>' "${metainfo}" >"${work}/noid.xml"
-if validate "${work}/noid.xml" >"${work}/noid.log" 2>&1; then
-    echo "FAIL: metainfo without <id> validated; the check proves nothing" >&2
-    exit 1
-fi
-grep -q component-id-missing "${work}/noid.log" ||
-    echo "note: id-less copy failed for another reason: $(tail -3 "${work}/noid.log")" >&2
-
-echo "appstream metainfo: valid"
+for metainfo in "${metainfos[@]}"; do
+    # Positive control: a validator that never fails would pass the check below.
+    sed '/<id>/d' "${metainfo}" >"${work}/noid.xml"
+    for tool in "${tools[@]}"; do
+        validate "${tool}" "${metainfo}" || {
+            echo "FAIL: ${metainfo##*/} does not validate under ${tool}" >&2
+            exit 1
+        }
+        if validate "${tool}" "${work}/noid.xml" >"${work}/noid.log" 2>&1; then
+            echo "FAIL: ${tool} validated a metainfo with no <id>; it proves nothing" >&2
+            exit 1
+        fi
+    done
+    echo "appstream metainfo: ${metainfo##*/} valid (${tools[*]})"
+done

tests/224_engine-ftp-cmdlen.test

@@ -0,0 +1,42 @@
+#!/bin/bash
+#
+# A too-long FTP command must be refused, never truncated or aborted (#1019).
+
+set -euo pipefail
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftpcmdlen.XXXXXX") || exit 1
+trap 'set +e; rm -rf "${tmpdir}"' EXIT
+trap 'exit 1' HUP INT QUIT TERM
+
+out=$(httrack -O "${tmpdir}/st" "-#test=ftp-cmdlen" run 2>&1) ||
+    fail "self-test exited non-zero: ${out}"
+grep -q "ftp-cmdlen self-test OK" <<<"${out}" || fail "unexpected output: ${out}"
+
+# The reachable half: the copy into _adr[256] aborted on a long host.
+rep() { awk -v n="$1" 'BEGIN { while (i++ < n) printf "a" }'; }
+
+probe_host() {
+    rm -rf "${tmpdir}/host"
+    httrack "ftp://$1/f.txt" -O "${tmpdir}/host" -q >/dev/null 2>&1 ||
+        fail "a ${#1}-byte FTP host name crashed the engine"
+    cat "${tmpdir}/host/hts-log.txt"
+}
+
+for n in 256 257 300; do
+    log=$(probe_host "$(rep "${n}")")
+    grep -q "Host name too long" <<<"${log}" ||
+        fail "a ${n}-byte host was not rejected: ${log}"
+done
+
+# 255 still fits _adr[256]; 127.0.0.1:1 is the outsider a widened gate swallows.
+log=$(probe_host "$(rep 255)")
+grep -q "Unable to get server's address" <<<"${log}" ||
+    fail "a 255-byte host did not reach the resolver: ${log}"
+log=$(probe_host "127.0.0.1:1")
+grep -q "Unable to connect to the server" <<<"${log}" ||
+    fail "a normal host did not reach the connect: ${log}"

tests/225_install-manifest.test

@@ -0,0 +1,99 @@
+#!/bin/bash
+#
+# Distro packages (Fedora %files, debian/*.files) name every installed file, so a
+# file appearing or vanishing here breaks their build and not ours; it broke
+# Fedora's twice in three weeks.
+
+set -euo pipefail
+
+: "${abs_top_builddir:?not run under make check}"
+manifest="${abs_top_srcdir:?not run under make check}/tests/install-manifest.txt"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+# macOS names the library .dylib, and Windows has no distro packaging to protect.
+test "$(uname -s)" = Linux || {
+    echo "install layout is pinned on Linux only; skipping" >&2
+    exit 77
+}
+test -r "${manifest}" || fail "no ${manifest}"
+
+# The docs are dropped below, so a docdir holding datadir would drop the pinned
+# data files with them and leave nothing to compare.
+docdir=${CONFIGURED_DOCDIR:?not run under make check}
+case "${CONFIGURED_DATADIR:?not run under make check}" in
+"${docdir%/}" | "${docdir%/}"/*)
+    echo "docdir contains datadir; skipping" >&2
+    exit 77
+    ;;
+esac
+
+work=$(mktemp -d "${TMPDIR:-/tmp}/manifest.XXXXXX") || {
+    echo "no tmpdir" >&2
+    exit 1
+}
+trap 'set +e; rm -rf "${work}"' EXIT
+
+stage=${work}/stage
+env -u MAKEFLAGS -u MAKELEVEL "${MAKE:-make}" -C "${abs_top_builddir}" \
+    install DESTDIR="${stage}" >"${work}/install.log" 2>&1 || {
+    cat "${work}/install.log" >&2
+    fail "make install DESTDIR=${stage}"
+}
+
+# Fold the configured directories back to tokens so the pin survives any --prefix
+# or --libdir. Longest path first, so a nested directory beats the one it sits in
+# whatever the layout puts where.
+: >"${work}/dirs"
+for var in DOCDIR HTMLDIR MANDIR INCLUDEDIR LIBDIR BINDIR DATADIR; do
+    name=CONFIGURED_${var}
+    path=${!name:-}
+    test -n "${path}" || fail "${name} not set"
+    path=${path%/}
+    printf '%s\t%s\t%s\n' "${#path}" "${path}" "${var}" >>"${work}/dirs"
+done
+LC_ALL=C sort -rn "${work}/dirs" -o "${work}/dirs"
+
+paths=() tokens=()
+while IFS=$'\t' read -r _ path var; do
+    paths+=("${path}")
+    tokens+=("${var}")
+done <"${work}/dirs"
+
+# Matched literally: a directory holding a regex metacharacter would silently
+# fold nothing if this were a pattern.
+fold() {
+    local line=$1 i
+    for i in "${!paths[@]}"; do
+        case ${line} in
+        "${paths[i]}"/*)
+            printf '@%s@/%s\n' "${tokens[i]}" "${line#"${paths[i]}"/}"
+            return
+            ;;
+        esac
+    done
+    printf '%s\n' "${line}"
+}
+
+# Dropped: the docs, since where they sit under docdir is a --htmldir choice each
+# distro makes its own way and no package names them one by one; the .a/.la
+# libtool bookkeeping no distro ships; and the library revision, which moves
+# every release. The soname digit stays: a bump renames Debian's package.
+(cd "${stage}" && find . \( -type f -o -type l \) | sed 's|^\.||') |
+    while IFS= read -r line; do fold "${line}"; done |
+    sed -e '/^@DOCDIR@\//d' -e '/^@HTMLDIR@\//d' \
+        -e '/\.la$/d' -e '/\.a$/d' \
+        -e 's|\(\.so\.[0-9][0-9]*\)\.[0-9][0-9]*\.[0-9][0-9]*$|\1.@revision@|' |
+    LC_ALL=C sort >"${work}/actual"
+
+# An install that staged nothing matches nothing; say that, not a 300-line diff.
+grep -qx '@BINDIR@/httrack' "${work}/actual" || fail "nothing staged under ${stage}"
+
+LC_ALL=C sort "${manifest}" >"${work}/expected"
+diff -u "${work}/expected" "${work}/actual" ||
+    fail "the installed file set changed; update tests/install-manifest.txt and say so in the release notes -- Fedora's %files and debian/*.files name these files"
+
+echo "install manifest: $(wc -l <"${work}/actual") files"

tests/226_watchdog-native.test

@@ -0,0 +1,726 @@
+#!/bin/bash
+#
+# The off-box suite watchdog (#795): its pure decisions through -SelfTest, what
+# reaches the status API, the process APIs it must never name, and the driver
+# wiring no .ps1 can check for itself.
+
+set -euo pipefail
+
+# Before the driver is sourced: with these set, anything it starts posts a commit
+# status on whatever the surrounding shell was last looking at.
+unset WATCHDOG_TOKEN WATCHDOG_REPO WATCHDOG_SHA WATCHDOG_URL
+export WATCHDOG_CONTEXT='226_watchdog-native (never posted)'
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "$testdir/testlib.sh"
+
+top=${abs_top_srcdir:-$(cd "$testdir/.." && pwd)}
+wdscript="$testdir/ci-windows-watchdog.ps1"
+driver="$testdir/ci-windows-suite.sh"
+workflow="$top/.github/workflows/windows-build.yml"
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_wdnat.XXXXXX")
+sinkpidfile="$tmp/sink.pid"
+: >"$sinkpidfile"
+toppid=$$
+trap 'set +e; stop_server "$(cat "$sinkpidfile" 2>/dev/null)"; rm -rf "$tmp"' EXIT
+trap 'exit 1' TERM
+# Signalled, since a bare exit only ends the subshell a leg runs in.
+fail() {
+    echo "FAIL: $*" >&2
+    kill -TERM "$toppid" 2>/dev/null || true
+    exit 1
+}
+
+test -r "$wdscript" || fail "no $wdscript"
+test -z "${WATCHDOG_TOKEN:-}${WATCHDOG_SHA:-}" || fail "a credential survived the scrub"
+
+# The prose in the script names the very calls the audits forbid. A trailing
+# comment goes only on a quote-free line: cutting at any '#' would hide a call
+# sitting after an issue number inside a string.
+uncommented() { sed -e 's/^[[:space:]]*#.*$//' -e "/['\"]/! s/[[:space:]]#.*$//" "$1"; }
+
+# Telemetry only, so every spawn and kill idiom is out, aliases and short forms
+# included: PowerShell needs neither the System. prefix nor a space after '&'.
+forbidden_calls() {
+    uncommented "$1" | grep -nEi \
+        'Start-Process|Start-Job|Start-ThreadJob|(Start|Stop)-Service|Stop-Process|Invoke-Expression|Invoke-Item|Invoke-Command|Invoke-(Cim|Wmi)Method|Get-WmiObject|Register-Scheduled(Job|Task)|WScript\.Shell|taskkill|cmd\.exe|wmic|schtasks|Diagnostics\.Process|\.Kill\(|(^|[^-a-zA-Z0-9_])(iex|saps|spps|spjb|sajb|spsv|sasv|start|kill)([^-a-zA-Z0-9_]|$)|(^|[^-a-zA-Z0-9_])&[^-a-zA-Z0-9_=&]'
+}
+if hits=$(forbidden_calls "$wdscript"); then
+    fail "the watchdog names a process API: $hits"
+fi
+# One control per idiom: a denylist is worth only the entries that actually fire.
+# shellcheck disable=SC2016 # PowerShell source, not this shell's expansions
+for probe in 'Start-Process notepad.exe' 'Stop-Process -Id 4 -Force' \
+    '[Diagnostics.Process]::Start("x")' '(Get-Process -Name httrack).Kill()' \
+    'Invoke-Expression $cmd' '& "C:\\x.exe" -y' 'Start-Job { 1 }' 'taskkill /F /IM x.exe' \
+    'iex $cmd' '$cmd | iex' '&"C:\\x.exe"' '&$exe' 'saps notepad.exe' 'start notepad.exe' \
+    'spps -Name httrack' 'spjb { 1 }' 'sajb { 1 }' 'kill -Name httrack' \
+    'Start-Service w32time' 'Stop-Service w32time' 'spsv w32time' 'sasv w32time' \
+    'Get-WmiObject Win32_Process' \
+    'Invoke-CimMethod -ClassName Win32_Process -MethodName Create' \
+    'Invoke-WmiMethod -Class Win32_Process -Name Create' \
+    'Invoke-Command -ScriptBlock { 1 }' 'Register-ScheduledJob -Name x -ScriptBlock { 1 }' \
+    'Register-ScheduledTask -TaskName x -Action $a' \
+    '(New-Object -ComObject WScript.Shell).Run("x")' \
+    'Write-Host "#795"; Start-Process x'; do
+    {
+        cat "$wdscript"
+        echo "$probe"
+    } >"$tmp/probe.ps1"
+    if ! forbidden_calls "$tmp/probe.ps1" >/dev/null; then
+        fail "the process-API audit cannot see: $probe"
+    fi
+done
+
+# shellcheck disable=SC2016 # $env: is PowerShell's, not this shell's
+grep -q '\$env:WATCHDOG_TOKEN' "$wdscript" ||
+    fail "the watchdog does not read its token from the environment"
+if sed -n '/^param(/,/^)/p' "$wdscript" | grep -qi 'token'; then
+    fail "the watchdog takes its token as a parameter, where the process list exposes it"
+fi
+grep -q "ApiBase = 'https://api.github.com'" "$wdscript" ||
+    fail "the watchdog's default endpoint is not the GitHub API"
+# One state, so nothing downstream can read a verdict out of a status that only
+# carries telemetry. The job's own conclusion is the verdict.
+test "$(grep -c "state = 'success'" "$wdscript")" -eq 1 ||
+    fail "the watchdog posts something other than the one telemetry state"
+# The token stays in the driver's own shell: exported, every test it runs would
+# inherit a credential that can post statuses. 172 drives that end to end.
+grep -q '^unset WATCHDOG_TOKEN$' "$driver" ||
+    fail "the driver leaves its token in the environment its tests inherit"
+# A background holder of the step's stdout keeps the step open past the suite (#949).
+grep -q '>>watchdog.log 2>&1' "$driver" ||
+    fail "the watchdog's output is not redirected off the step's stdout"
+
+# Sourcing the driver is enough to start an interpreter that can post, so every
+# test that does it has to scrub first.
+scrubs() {
+    grep -q 'ci-windows-suite\.sh' "$1" || return 0
+    grep -qx 'unset WATCHDOG_TOKEN WATCHDOG_REPO WATCHDOG_SHA WATCHDOG_URL' "$1"
+}
+for t in "$testdir"/*.test; do
+    scrubs "$t" || fail "$(basename "$t") can reach the driver without scrubbing its credentials"
+done
+printf '#!/bin/sh\n. ./ci-windows-suite.sh\n' >"$tmp/unscrubbed.test"
+scrubs "$tmp/unscrubbed.test" && fail "the scrub audit passes a test that does not scrub"
+
+# PATH holds MSYS-style entries only, and TMPDIR on Windows is a D:/... path
+# that would split it on the drive colon.
+posixtmp=$tmp
+if is_windows && command -v cygpath >/dev/null 2>&1; then
+    posixtmp=$(cygpath -u "$tmp")
+fi
+bin="$posixtmp/bin"
+mkdir -p "$bin"
+# Renamed into place, so the poll below cannot read a half-written file.
+printf '#!/bin/sh\nprintf "%%s\\n" "$@" >%s/argv.part\nmv %s/argv.part %s/argv\necho "watchdog ready"\n' \
+    "$posixtmp" "$posixtmp" "$posixtmp" >"$bin/argvstub"
+printf '#!/bin/sh\nexit 127\n' >"$bin/deadstub"
+chmod +x "$bin/argvstub" "$bin/deadstub"
+usestub() { cp "$bin/$1" "$bin/pwsh" && cp "$bin/$1" "$bin/powershell.exe"; }
+usestub argvstub
+
+if test -x "$bin/pwsh"; then
+    (
+        cd "$posixtmp"
+        # shellcheck disable=SC2030 # local to this subshell is the point
+        PATH="$bin:$PATH"
+        export PATH
+        # An unshadowed pwsh would drive the real watchdog, and every assertion
+        # below would still read as a plain launch failure.
+        test "$(command -v pwsh)" = "$bin/pwsh" ||
+            fail "the stub does not shadow PowerShell: $(command -v pwsh)"
+        # shellcheck source=tests/ci-windows-suite.sh
+        . "$driver"
+        ci_watchdog_pid=''
+        ci_start_native_watchdog "$posixtmp/progress.log" ||
+            fail "the launch reported no watchdog with a PowerShell on PATH"
+        test -n "$ci_watchdog_pid" || fail "the launch set no pid"
+        # wait only reaps a child, which is what the teardown has to signal.
+        wait "$ci_watchdog_pid" || fail "the watchdog is not a child of the driver"
+
+        # A pid comes from the fork, so a dead interpreter still yields one.
+        usestub deadstub
+        ci_watchdog_pid='dangling'
+        rc=0
+        ci_start_native_watchdog "$posixtmp/progress.log" || rc=$?
+        test "$rc" -ne 0 || fail "an interpreter exiting 127 was reported as launched"
+        test -z "$ci_watchdog_pid" || fail "a failed launch left pid $ci_watchdog_pid behind"
+    )
+    for _ in $(seq 1 100); do
+        test -r "$posixtmp/argv" && break
+        sleep 0.1
+    done
+    test -r "$posixtmp/argv" || fail "the watchdog was never launched"
+    grep -qx -- '-SelfTest' "$posixtmp/argv" && fail "the production launch runs the self-test"
+    # Bypass is load-bearing: Windows PowerShell refuses an unsigned .ps1 without it.
+    for flag in -NoProfile -NonInteractive -ExecutionPolicy Bypass -File; do
+        grep -qx -- "$flag" "$posixtmp/argv" ||
+            fail "the launch dropped $flag: $(tr '\n' ' ' <"$posixtmp/argv")"
+    done
+    grep -qx -- '-ApiBase' "$posixtmp/argv" &&
+        fail "the production launch redirects the status API away from GitHub"
+    grep -qx -- '-NoPost' "$posixtmp/argv" &&
+        fail "the production launch cannot post at all"
+    # Native form, because a non-MSYS interpreter cannot resolve a /d/a/... path.
+    grep -qx -- "$(nativepath "$wdscript")" "$posixtmp/argv" ||
+        fail "the launch did not name the script: $(tr '\n' ' ' <"$posixtmp/argv")"
+    grep -qx -- "$(nativepath "$posixtmp/progress.log")" "$posixtmp/argv" ||
+        fail "the launch did not pass the progress log"
+else
+    echo "note: $tmp is noexec, the driver wiring was not driven"
+fi
+
+py=$(find_python || true)
+if ! test -r "$workflow"; then
+    test -d "$top/.github" && fail "$workflow is gone but $top/.github is not"
+    echo "note: no .github under $top (dist tarball), the workflow audits did not run"
+elif test -z "$py" || ! "$py" -c 'import yaml' 2>/dev/null; then
+    echo "note: no python3 with PyYAML here, the workflow audits did not run"
+else
+    # Parsed, not grepped: every key below has to be read from the block that
+    # owns it, and an ordinary reformatting of the workflow must not move it.
+    cat >"$tmp/wf.py" <<'PY'
+import sys
+
+import yaml
+
+
+def steps_of(wf):
+    return [s for j in (wf.get("jobs") or {}).values() for s in (j.get("steps") or [])]
+
+
+def checkouts(wf):
+    return [s for s in steps_of(wf) if str(s.get("uses", "")).startswith("actions/checkout")]
+
+
+def suite_steps(wf):
+    return [s for s in steps_of(wf) if "ci-windows-suite.sh" in str(s.get("run", ""))]
+
+
+def suite_jobs(wf):
+    return [
+        j
+        for j in (wf.get("jobs") or {}).values()
+        if any("ci-windows-suite.sh" in str(s.get("run", "")) for s in (j.get("steps") or []))
+    ]
+
+
+# A job-level block replaces the workflow's outright, so one naming only contents
+# silently drops statuses: write.
+def perms_of(wf, job):
+    p = job.get("permissions", wf.get("permissions"))
+    if p in ("read-all", "write-all"):
+        w = "read" if p == "read-all" else "write"
+        return {"contents": w, "statuses": w}
+    return p if isinstance(p, dict) else {}
+
+
+WANT_ENV = {
+    "WATCHDOG_TOKEN": "${{ secrets.GITHUB_TOKEN }}",
+    "WATCHDOG_REPO": "${{ github.repository }}",
+    # The merge commit, which no PR checks UI reads.
+    "WATCHDOG_SHA": "${{ github.sha }}",
+}
+
+
+def audit(wf):
+    bad = []
+    if not checkouts(wf):
+        bad.append("no actions/checkout step to audit")
+    for s in checkouts(wf):
+        if (s.get("with") or {}).get("persist-credentials") is not False:
+            bad.append("a checkout persists the job token for the whole job")
+    jobs, found = suite_jobs(wf), suite_steps(wf)
+    if len(jobs) != 1 or len(found) != 1:
+        bad.append("%d jobs and %d steps run the suite, cannot tell which is watched" % (len(jobs), len(found)))
+        return bad
+    perms = perms_of(wf, jobs[0])
+    if perms.get("statuses") != "write":
+        bad.append("the suite job cannot post a commit status: statuses is %r" % perms.get("statuses"))
+    if perms.get("contents") != "read":
+        bad.append("contents: read was dropped from the suite job")
+    env = found[0].get("env") or {}
+    for v, want in WANT_ENV.items():
+        if env.get(v) != want:
+            bad.append("%s is %r, want %r" % (v, env.get(v), want))
+    # Both axes, or the matrix legs overwrite each other's status.
+    for axis in ("matrix.platform", "matrix.configuration"):
+        if axis not in str(env.get("WATCHDOG_CONTEXT") or ""):
+            bad.append("WATCHDOG_CONTEXT does not name %s" % axis)
+    if "github.run_id" not in str(env.get("WATCHDOG_URL") or ""):
+        bad.append("WATCHDOG_URL does not link the run being watched")
+    cap = found[0].get("timeout-minutes")
+    if not isinstance(cap, int):
+        bad.append("the suite step has no timeout-minutes of its own")
+    else:
+        print("timeout=%d" % cap)
+    return bad
+
+
+def mutate(wf, kind):
+    if kind == "perm":
+        wf["permissions"]["statuses"] = "read"
+    elif kind == "jobperm":
+        suite_jobs(wf)[0]["permissions"] = {"contents": "read"}
+    elif kind == "creds":
+        for s in checkouts(wf):
+            (s.get("with") or {}).pop("persist-credentials", None)
+    elif kind == "env":
+        suite_steps(wf)[0]["env"].pop("WATCHDOG_TOKEN")
+    elif kind == "token":
+        suite_steps(wf)[0]["env"]["WATCHDOG_TOKEN"] = "${{ secrets.WATCHDOG_PAT }}"
+    elif kind == "sha":
+        suite_steps(wf)[0]["env"]["WATCHDOG_SHA"] = "${{ github.event.pull_request.head.sha }}"
+    elif kind == "context":
+        suite_steps(wf)[0]["env"]["WATCHDOG_CONTEXT"] = "windows-suite"
+    elif kind == "url":
+        suite_steps(wf)[0]["env"]["WATCHDOG_URL"] = "${{ github.server_url }}"
+    elif kind == "cap":
+        suite_steps(wf)[0].pop("timeout-minutes")
+    elif kind != "none":
+        raise SystemExit("unknown mutation " + kind)
+    return wf
+
+
+wf = yaml.safe_load(open(sys.argv[2]))
+if sys.argv[1] == "mutate":
+    yaml.safe_dump(mutate(wf, sys.argv[3]), sys.stdout)
+else:
+    for b in audit(wf):
+        print("BAD " + b)
+PY
+    said=$("$py" "$tmp/wf.py" audit "$workflow") || fail "the workflow audit did not run: $said"
+    grep -q '^BAD ' <<<"$said" && fail "$(grep '^BAD ' <<<"$said" | sed 's/^BAD //')"
+    cap=$(($(sed -n 's/^timeout=//p' <<<"$said") * 60))
+
+    # One control per key, or an audit that stopped looking at one would ride on
+    # another still firing. "none" round-trips the file through the parser, which
+    # rewrites quoting, order and flow style: no verdict may move.
+    for m in none:'' perm:'cannot post' jobperm:'cannot post' creds:'persists the job token' \
+        env:'WATCHDOG_TOKEN is' token:'WATCHDOG_TOKEN is' sha:'WATCHDOG_SHA is' \
+        context:'does not name matrix.platform' url:'does not link the run' \
+        cap:'no timeout-minutes of its own'; do
+        "$py" "$tmp/wf.py" mutate "$workflow" "${m%%:*}" >"$tmp/wf.yml"
+        said=$("$py" "$tmp/wf.py" audit "$tmp/wf.yml") ||
+            fail "the workflow audit did not run on the ${m%%:*} control: $said"
+        if test -z "${m#*:}"; then
+            grep -q '^BAD ' <<<"$said" && fail "a reformatted workflow fails the audit: $said"
+        else
+            grep -q "^BAD .*${m#*:}" <<<"$said" ||
+                fail "the ${m%%:*} control did not trip the audit: $said"
+        fi
+    done
+
+    # The heartbeat still owns the kill and must beat the step timeout.
+    stuck=$(sed -n 's/^stuck=\([0-9]*\)$/\1/p' "$driver")
+    deadline=$(sed -n 's/^suite_deadline=\([0-9]*\)$/\1/p' "$driver")
+    test -n "$stuck" || fail "cannot read the heartbeat's static window"
+    test -n "$deadline" || fail "cannot read the driver's suite deadline"
+    # A test may start one second under the deadline and never report again.
+    test $((deadline + stuck + 240)) -le "$cap" ||
+        fail "a kill at ${deadline}+${stuck}s leaves under 240s before the ${cap}s step timeout"
+
+    # The watchdog's own stop is its MaxSeconds, which must land inside the step.
+    # shellcheck disable=SC2016 # $MaxSeconds is PowerShell's
+    maxsecs=$(sed -n 's/^[[:space:]]*\[int\]\$MaxSeconds = \([0-9]*\).*$/\1/p' "$wdscript")
+    test -n "$maxsecs" || fail "cannot read the watchdog's own deadline"
+    test "$maxsecs" -le "$cap" || fail "the watchdog runs ${maxsecs}s past the ${cap}s step timeout"
+    # Bounded below too: the reporter has to outlive the heartbeat's own kill.
+    test "$maxsecs" -ge $((deadline + stuck)) ||
+        fail "the watchdog stops at ${maxsecs}s, before the kill at $((deadline + stuck))s"
+fi
+
+psruns=()
+for c in pwsh powershell.exe powershell; do
+    command -v "$c" >/dev/null 2>&1 || continue
+    # powershell and powershell.exe are one interpreter under two names.
+    seen=0
+    for p in ${psruns[@]+"${psruns[@]}"}; do
+        test "${p%.exe}" != "${c%.exe}" || seen=1
+    done
+    test "$seen" -eq 1 || psruns+=("$c")
+done
+if test "${#psruns[@]}" -eq 0; then
+    echo "SKIP: no PowerShell here, the watchdog's own self-test cannot run"
+    exit 77
+fi
+
+# Bounded: a mutant that loops forever would otherwise be caught only by the step
+# timeout this work exists to beat. -NoPost layers over the scrub above.
+run_wd() {
+    local secs=$1
+    shift
+    # -NoPost last: everything after -File <script> belongs to the script.
+    run_with_timeout "$secs" "$psrun" "${psargs[@]}" "$@" -NoPost 2>&1
+}
+
+# The legs below grade what reached the API. Printing and posting are decoupled,
+# so counting what the loop wrote can see neither the request rate nor the backoff.
+sink="$tmp/sink.py"
+cat >"$sink" <<'PY'
+import http.server
+import socketserver
+import sys
+
+codes = [int(c) for c in sys.argv[1].split(",")]
+rec = sys.argv[2]
+seen = []
+
+
+class Handler(http.server.BaseHTTPRequestHandler):
+    def do_POST(self):
+        n = int(self.headers.get("Content-Length") or 0)
+        with open(rec, "ab") as f:
+            f.write(self.rfile.read(n) + b"\n")
+        seen.append(1)
+        self.send_response(codes[(len(seen) - 1) % len(codes)])
+        self.send_header("Content-Length", "2")
+        self.end_headers()
+        self.wfile.write(b"{}")
+
+    def log_message(self, *args):
+        pass
+
+
+class Sink(http.server.HTTPServer):
+    # Skip stock server_bind()'s getfqdn(): it stalls 35s on macOS (#870).
+    def server_bind(self):
+        socketserver.TCPServer.server_bind(self)
+        self.server_name, self.server_port = self.server_address[:2]
+
+
+srv = Sink(("127.0.0.1", 0), Handler)
+# LF, since discover_server_port parses this line: Windows would emit \r\n.
+sys.stdout.reconfigure(newline="\n")
+print("PORT %d" % srv.server_port, flush=True)
+srv.serve_forever()
+PY
+
+posts="$tmp/posts"
+sinklog="$tmp/sink.log"
+legout="$tmp/leg.out"
+port='' sinkpid=''
+sink_start() {
+    : >"$posts"
+    : >"$sinklog"
+    # Its own log: a leg's command substitution would else wait on the pipe the
+    # sink inherited for as long as the sink lives.
+    "$py" "$sink" "$1" "$posts" >"$sinklog" 2>&1 &
+    sinkpid=$!
+    # Where the teardown can find it: sink_start runs in a subshell of its own.
+    echo "$sinkpid" >"$sinkpidfile"
+    port=$(discover_server_port "$sinklog" "$sinkpid") ||
+        fail "the status sink never bound a port: $(cat "$sinklog")"
+}
+sink_stop() {
+    stop_server "$sinkpid"
+    : >"$sinkpidfile"
+    sinkpid=''
+}
+# Ours, so a leg that lost its -ApiBase reaches GitHub with a credential it
+# refuses. Distinctive, since the legs grade the log on never carrying it.
+export WATCHDOG_TOKEN=n0tat0ken WATCHDOG_REPO=octo/nowhere WATCHDOG_SHA=0000000
+
+# The log rides into the job's artifact, which nothing secret-masks.
+token_free() { ! grep -q "$WATCHDOG_TOKEN" "$1"; }
+
+run_posting() {
+    local secs=$1
+    shift
+    run_with_timeout "$secs" "$psrun" "${psargs[@]}" "$@" -ApiBase "http://127.0.0.1:$port"
+}
+# t and q of each posted status, in order and whatever key order the JSON took.
+posted_pairs() {
+    sed -n 's/.*t=\([0-9]*\)s q=\([0-9?]*\)s.*/\1 \2/p' "$posts"
+}
+
+# One property per leg, each returning its diagnosis so a mutant can be required
+# to trip that same check rather than merely to fail.
+cadence_leg() {
+    local n
+    sink_start 201
+    printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
+    run_posting 60 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
+        -IntervalSeconds 100 -PollSeconds 1 -MaxSeconds 4 >"$legout" 2>&1 || {
+        sink_stop
+        echo "the loop exited nonzero: $(cat "$legout")"
+        return 1
+    }
+    sink_stop
+    n=$(grep -c . "$posts" || true)
+    test "$n" -eq 1 || {
+        echo "$n API calls at a 100s cadence over 4s, want 1"
+        return 1
+    }
+    grep -q '36_local-bigcrawl.test' "$posts" || {
+        echo "the posted status does not name the test in flight: $(cat "$posts")"
+        return 1
+    }
+    case $(posted_pairs) in
+    *[0-9]' '[0-9]*) ;;
+    *)
+        echo "a readable log posted no staticness: $(posted_pairs)"
+        return 1
+        ;;
+    esac
+}
+
+# Staticness has to follow the log through movement and then through silence,
+# which is why this leg is the long one.
+staticness_leg() {
+    local moved dq dt legpid stop
+    sink_start 201
+    printf 'RUN 36_local-bigcrawl.test at 0s\n' >"$tmp/progress.log"
+    run_posting 90 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
+        -IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 16 >"$legout" 2>&1 &
+    legpid=$!
+    # Started on the first status that lands rather than on this shell's clock:
+    # an interpreter three seconds into its own startup would see silence only,
+    # and the moving half would go untested.
+    for _ in $(seq 1 300); do
+        test -s "$posts" && break
+        sleep 0.1
+    done
+    stop=$((SECONDS + 3))
+    while test "$SECONDS" -lt "$stop"; do
+        printf 'RUN 36_local-bigcrawl.test at %ss\n' "$SECONDS" >>"$tmp/progress.log"
+        sleep 0.4
+    done
+    wait "$legpid" || {
+        sink_stop
+        echo "the loop exited nonzero: $(cat "$legout")"
+        return 1
+    }
+    sink_stop
+    moved=$(posted_pairs | awk '$2 ~ /^[0-9]+$/ && $1 >= 2 && $2 <= 1 { n++ } END { print n + 0 }')
+    test "$moved" -ge 1 || {
+        echo "no post read the log as moving while it was being written: $(posted_pairs | tr '\n' '/')"
+        return 1
+    }
+    # Once the log stops, staticness is elapsed time minus a constant, so it has
+    # to track it one for one. A bound on it would report every wedge as the cap.
+    dq=$(posted_pairs | awk '$1 >= 6 && $2 ~ /^[0-9]+$/ { if (f == "") f = $2; l = $2 } END { print l - f }')
+    dt=$(posted_pairs | awk '$1 >= 6 && $2 ~ /^[0-9]+$/ { if (f == "") f = $1; l = $1 } END { print l - f }')
+    if test "${dq:-0}" -ne "${dt:-0}" || test "${dq:-0}" -lt 4; then
+        echo "staticness moved ${dq}s over ${dt}s of silence: $(posted_pairs | tr '\n' '/')"
+        return 1
+    fi
+}
+
+# Every due tick logs its line before the throttle decides, so these two counts
+# are a ratio no slow box can move: one to one while nothing is being skipped.
+throttle_leg() {
+    local calls lines
+    sink_start 201
+    printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
+    run_posting 60 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
+        -IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 8 >"$legout" 2>&1 || {
+        sink_stop
+        echo "the loop exited nonzero: $(cat "$legout")"
+        return 1
+    }
+    sink_stop
+    calls=$(grep -c . "$posts" || true)
+    lines=$(grep -c 't=[0-9]*s q=' "$legout" || true)
+    test "$lines" -ge 3 || {
+        echo "$lines due ticks over 8s at a 1s interval: too few to judge a throttle"
+        return 1
+    }
+    test $((calls * 4)) -ge $((lines * 3)) || {
+        echo "$calls of $lines due ticks reached an accepting API: a landed post throttles the next"
+        return 1
+    }
+}
+
+backoff_leg() {
+    local calls lines
+    sink_start 403
+    printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
+    run_posting 60 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
+        -IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 8 >"$legout" 2>&1 || {
+        sink_stop
+        echo "the loop exited nonzero: $(cat "$legout")"
+        return 1
+    }
+    sink_stop
+    calls=$(grep -c . "$posts" || true)
+    lines=$(grep -c 't=[0-9]*s q=' "$legout" || true)
+    test "$calls" -ge 2 || {
+        echo "a rejecting API was called $calls times: the loop stopped trying"
+        return 1
+    }
+    # One diagnosis for both: which one trips first depends on box speed. The
+    # backoff must thin the calls, not the log; a token-less run leaves only the log.
+    if test "$calls" -gt 5 || test "$lines" -lt $((2 * calls)); then
+        echo "$calls calls and $lines log lines against an API rejecting every one: nothing backs off"
+        return 1
+    fi
+}
+
+fullstdout_leg() {
+    local n
+    sink_start 201
+    printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
+    run_posting 60 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
+        -IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 5 >/dev/full 2>&1 || true
+    sink_stop
+    n=$(grep -c . "$posts" || true)
+    test "$n" -ge 2 || {
+        echo "$n posts with a full stdout: a failed log write took the loop with it"
+        return 1
+    }
+}
+
+first=1
+for psrun in "${psruns[@]}"; do
+    psargs=(-NoProfile -NonInteractive)
+    case $psrun in
+    powershell*) psargs+=(-ExecutionPolicy Bypass) ;;
+    esac
+
+    out=$(run_wd 120 -File "$(nativepath "$wdscript")" -SelfTest) ||
+        fail "the $psrun self-test failed: $out"
+    grep -q 'watchdog self-test OK' <<<"$out" || fail "the $psrun self-test said: $out"
+
+    # A mutant the script survives is a property nothing is checking. Graded on
+    # the assertion named: one that merely fails to parse would read as caught.
+    mutate() {
+        local name=$1 expr=$2 want=$3 m="$tmp/$1.ps1" rc=0 said
+        sed "$expr" "$wdscript" >"$m"
+        cmp -s "$wdscript" "$m" && fail "mutant $name changed nothing, so it proves nothing"
+        said=$(run_wd 120 -File "$(nativepath "$m")" -SelfTest) || rc=$?
+        test "$rc" -ne 0 || fail "mutant $name passed the $psrun self-test: $said"
+        grep -q "self-test FAIL: .*$want" <<<"$said" ||
+            fail "mutant $name tripped no assertion of its own: $said"
+    }
+    # shellcheck disable=SC2016 # PowerShell variables, quoted for sed
+    mutate cadence-off-by-one 's/-ge \$Interval/-gt \$Interval/' 'due exactly on the interval'
+    # shellcheck disable=SC2016
+    mutate backoff-uncapped 's/\[Math\]::Min(\$Current \* 2, 32)/$Current * 2/' 'backoff is not capped'
+    # shellcheck disable=SC2016
+    mutate throttle-no-reset 's/if (\$Ok) { return @(0, 0) }/if ($Ok) { return @(0, $Backoff) }/' \
+        'landed post leaves'
+    # shellcheck disable=SC2016
+    mutate status-unclipped '/if (\$s.Length -gt 140)/d' 'was not clipped'
+    # An oversized status is cut from the tail: the head is what a wedge is read for.
+    # shellcheck disable=SC2016
+    mutate status-clipped-head 's/\$s = \$s.Substring(0, 140)/$s = $s.Substring($s.Length - 140)/' \
+        'dropped the leading fields'
+    # shellcheck disable=SC2016
+    mutate tail-always-ok 's/\$r = @{ Ok = \$false;/$r = @{ Ok = $true;/' 'reads as one that was read'
+    mutate counters-empty "s/return (\$f -join ' ')/return ''/" 'counters are not key=value'
+    # The production cadence: no leg below runs without a schedule of its own.
+    # shellcheck disable=SC2016
+    mutate default-cadence 's/\[int\]\$IntervalSeconds = 30/[int]$IntervalSeconds = 3000/' \
+        'default status cadence'
+    # shellcheck disable=SC2016
+    mutate default-poll 's/\[int\]\$PollSeconds = 5/[int]$PollSeconds = 50/' 'default poll'
+
+    # Timed and counted by this shell, never read out of what the watchdog
+    # writes: the cadence and the deadline are both mine to set.
+    printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
+    out=$(run_wd 60 -File "$(nativepath "$wdscript")" \
+        -ProgressLog "$(nativepath "$tmp/progress.log")" \
+        -IntervalSeconds 100 -PollSeconds 1 -MaxSeconds 5) ||
+        fail "the $psrun slow-cadence loop exited nonzero: $out"
+    slow=$(grep -c 't=[0-9]*s q=' <<<"$out" || true)
+    test "$slow" -eq 1 || fail "$slow status lines at a 100s cadence over 5s, want 1: $out"
+
+    began=$SECONDS
+    out=$(run_wd 60 -File "$(nativepath "$wdscript")" \
+        -ProgressLog "$(nativepath "$tmp/progress.log")" \
+        -IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 5) ||
+        fail "the $psrun loop exited nonzero: $out"
+    spent=$((SECONDS - began))
+    test "$spent" -ge 4 || fail "a 5s watchdog returned after ${spent}s: MaxSeconds is not seconds"
+    test "$spent" -le 20 || fail "a 5s watchdog ran ${spent}s: it does not stop on its own"
+    fast=$(grep -c 't=[0-9]*s q=' <<<"$out" || true)
+    # Against the slow leg, not a floor: a probe tick is not instant.
+    test "$fast" -gt "$slow" || fail "$fast status lines at 1s against $slow at 100s: $out"
+    grep -q '36_local-bigcrawl.test' <<<"$out" || fail "the status does not name the test in flight: $out"
+    grep -q '95_local-sitemap' <<<"$out" && fail "the status named a test that was never in flight"
+    # Trips if the scrub and -NoPost were both lost: only a real attempt logs this.
+    grep -q 'status post failed' <<<"$out" && fail "a test leg reached the status API"
+
+    # An unreadable log must not read as a wedged one.
+    out=$(run_wd 60 -File "$(nativepath "$wdscript")" \
+        -ProgressLog "$(nativepath "$tmp/no-such.log")" \
+        -IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 3) ||
+        fail "the $psrun loop exited nonzero on a missing log: $out"
+    grep -q 'q=?s' <<<"$out" || fail "a missing log reported a staticness: $out"
+
+    # The legs below read the sink, not the loop, and they exercise the script's
+    # decisions rather than the interpreter's: one flavour is enough.
+    test "$first" -eq 1 || continue
+    first=0
+    if test -z "$py"; then
+        echo "note: no python3 here, the posting legs did not run"
+        continue
+    fi
+
+    # ENOSPC on stdout, a state the sick runner reaches. Measured, since the leg
+    # rests on it: MSYS hands a native PowerShell a /dev/full it can write to.
+    devfull=0
+    # shellcheck disable=SC2016 # PowerShell source, not this shell's expansions
+    if test -w /dev/full &&
+        ! run_with_timeout 60 "$psrun" "${psargs[@]}" \
+            -Command '$ErrorActionPreference = "Stop"; Write-Host ("x" * 200000)' \
+            >/dev/full 2>/dev/null; then
+        devfull=1
+    fi
+    legs=(cadence throttle staticness backoff)
+    test "$devfull" -eq 0 || legs+=(fullstdout)
+    for leg in "${legs[@]}"; do
+        : >"$legout"
+        said=$("${leg}_leg" "$(nativepath "$wdscript")") ||
+            fail "the $psrun watchdog fails the $leg leg: $said"
+        token_free "$legout" || fail "the $leg leg logged the status token"
+    done
+    # Control: a catch block widened to the whole exception is how it would get out.
+    leaky="$tmp/token-in-log.ps1"
+    # shellcheck disable=SC2016 # PowerShell variables, quoted for sed
+    sed '/status post failed/s/$_.Exception.Message/$script:Token/' "$wdscript" >"$leaky"
+    cmp -s "$wdscript" "$leaky" && fail "the leaky mutant changed nothing, so it proves nothing"
+    backoff_leg "$(nativepath "$leaky")" >/dev/null || true
+    token_free "$legout" && fail "the token audit cannot see a logged token"
+
+    # Same shape as mutate() above, graded by a leg instead of the self-test, and
+    # on the diagnosis rather than the exit status for the same reason.
+    mutate_leg() {
+        local name=$1 expr=$2 leg=$3 want=$4 m="$tmp/$1.ps1" said
+        sed "$expr" "$wdscript" >"$m"
+        cmp -s "$wdscript" "$m" && fail "mutant $name changed nothing, so it proves nothing"
+        said=$("${leg}_leg" "$(nativepath "$m")") &&
+            fail "mutant $name passed the $leg leg"
+        grep -q "$want" <<<"$said" || fail "mutant $name tripped no check of its own: $said"
+    }
+    mutate_leg status-every-poll \
+        "s/if ((Get-WatchdogAction \$now \$postedAt \$IntervalSeconds) -eq 'post')/if (\$true)/" \
+        cadence 'API calls at a 100s cadence'
+    # shellcheck disable=SC2016 # PowerShell variables, quoted for sed
+    mutate_leg static-unknown 's/\$static = \$now - \$movedAt/$static = -1/' \
+        cadence 'posted no staticness'
+    # shellcheck disable=SC2016
+    mutate_leg static-elapsed 's/\$static = \$now - \$movedAt/$static = $now/' \
+        staticness 'read the log as moving'
+    # shellcheck disable=SC2016
+    mutate_leg static-capped 's/if (\$tail.Ok) { \$static = \$now - \$movedAt }/if ($tail.Ok) { $static = [Math]::Min($now - $movedAt, 5) }/' \
+        staticness 'staticness moved'
+    # shellcheck disable=SC2016
+    mutate_leg skip-when-none-owed 's/if (\$skip -gt 0)/if ($skip -ge 0)/' \
+        throttle 'a landed post throttles the next'
+    # shellcheck disable=SC2016
+    mutate_leg success-skips 's/if (\$Ok) { return @(0, 0) }/if ($Ok) { return @(1, 0) }/' \
+        throttle 'a landed post throttles the next'
+    # shellcheck disable=SC2016
+    mutate_leg backoff-never-skips 's/if (\$skip -gt 0)/if ($false)/' backoff 'nothing backs off'
+    if test "$devfull" -eq 1; then
+        mutate_leg log-write-fatal 's/try { \(Write-Host .*\) } catch { }/\1/' \
+            fullstdout 'took the loop with it'
+    fi
+done
+
+echo "native watchdog OK (${psruns[*]})"

tests/227_watchdog-poll.test

@@ -0,0 +1,151 @@
+#!/bin/bash
+#
+# testlib's polling helpers: the deadline tick every watchdog turns on, and the
+# wait for a fixture server's port line. Both used to fork a process per tick,
+# which is what the whole harness is built out of (#795).
+
+set -eu
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_poll.XXXXXX") || exit 1
+# A colon-free root for the PATH shim below: MSYS hands out a drive-letter TMPDIR
+# and a PATH entry carrying a colon is read as two, which left every count at zero.
+shim=$(mktemp -d /tmp/httrack_pollshim.XXXXXX 2>/dev/null) || shim="$tmp/count"
+trap 'set +e; rm -rf "$tmp" "$shim"' EXIT
+trap 'rm -rf "$tmp" "$shim"' HUP INT QUIT PIPE TERM
+
+# --- the tick waits, and costs no process where the fd timer works -----------
+count_sleeps() { # count_sleeps <dir> <log>: a sleep shim recording each call
+    mkdir -p "$1"
+    printf '#!/bin/sh\necho tick >>"%s"\nexec %s "$@"\n' "$2" "$(command -v sleep)" >"$1/sleep"
+    chmod +x "$1/sleep"
+    PATH="$1:$PATH"
+    export PATH
+    # Proved, not assumed: an unreachable shim counts zero sleeps, which reads as
+    # "nothing forked" and would pass the very assertions it silently disarmed.
+    : >"$2"
+    sleep 0
+    test -s "$2" || fail "the sleep shim is not on PATH (entry '$1')"
+}
+ticks() { # ticks <label> <secs> <count> <floor>: polls, counting the sleeps forked
+    local began=$SECONDS
+    local i=0
+    : >"$tmp/sleeps"
+    while test "$i" -lt "$3"; do
+        poll_wait "$2"
+        i=$((i + 1))
+    done
+    # Floored at the whole wait asked for, so a tick running at half speed fails
+    # too, not only one that returns at once and spins every deadline loop.
+    test "$((SECONDS - began))" -ge "$4" || fail "$1: $3 polls of ${2}s took under ${4}s"
+}
+sleeps() { wc -l <"$tmp/sleeps" | tr -d ' '; }
+# Probed, not assumed: a host that cannot make a fifo -- and Windows, which opts
+# out -- is entitled to the forked tick, but one that can and took it anyway has
+# lost the whole point. The override counts as opting out too, or the escape hatch
+# for a host whose fd tick misbehaves would red the suite it exists to rescue.
+canfifo=
+if test -z "${HTTRACK_POLL_SLEEP:-}" && ! is_windows && mkfifo "$tmp/probe" 2>/dev/null; then
+    canfifo=1
+fi
+(
+    count_sleeps "$shim" "$tmp/sleeps"
+    poll_wait 0.1 # settles POLL_STATE before the count
+    echo "poll tick: $POLL_STATE"
+    ticks "$POLL_STATE tick" 0.1 30 3
+    test -z "$canfifo" || test "$POLL_STATE" = fifo ||
+        fail "the tick fell back to sleep on a host with working fifos"
+    if test "$POLL_STATE" = fifo; then
+        if test "${BASH_VERSINFO[0]}" -ge 4; then
+            test "$(sleeps)" -eq 0 || fail "the fd tick forked $(sleeps) sleeps"
+        else
+            # No fractional read -t below bash 4, so the documented fallback is the sleep asked for.
+            test "$(sleeps)" -eq 30 || fail "bash ${BASH_VERSINFO[0]} forked $(sleeps) of 30 fractional ticks"
+        fi
+        # The whole-second tick every watchdog runs on must take the fd on every bash:
+        # where a fraction cannot, it is the only win left. Sized for ~3s either way,
+        # bash 4 shortening its own wait to 0.1s.
+        if test "${BASH_VERSINFO[0]}" -ge 4; then n=30; else n=3; fi
+        ticks "fd 1s tick" 1 "$n" 3
+        test "$(sleeps)" -eq 0 || fail "the 1s fd tick forked $(sleeps) sleeps"
+    fi
+    # Forced back to sleep(1), which is the only tick a shimmed sleep can reach.
+    # POLL_STATE is left as it settled: a run already on the fd must give it up
+    # when a test starves the poll mid-run, so the override is read per tick.
+    export HTTRACK_POLL_SLEEP=1
+    ticks "forced sleep tick" 0.1 30 3
+    test "$(sleeps)" -eq 30 || fail "HTTRACK_POLL_SLEEP did not fork a sleep per tick"
+    # A timer fd that stops timing out must fall back, not spin. Whole seconds: a
+    # fraction never reaches the fd below bash 4.
+    unset HTTRACK_POLL_SLEEP # the fd is what is under test here, not the override
+    POLL_STATE=fifo
+    exec 9<&-
+    : >"$tmp/sleeps"
+    poll_wait 1 2>/dev/null
+    test -s "$tmp/sleeps" || fail "a dead timer fd did not fall back to sleep"
+    test "$POLL_STATE" = forked || fail "a dead timer fd was kept ($POLL_STATE)"
+)
+
+# --- the port wait outlasts a slow announcement ------------------------------
+# A cold Python start lags seconds on a loaded runner, so falling through early
+# would fail every crawl test on exactly the hosts the wait exists for.
+log="$tmp/late.log"
+: >"$log"
+(
+    sleep 2
+    echo "warning: something merged from stderr"
+    echo "PORT 40001"
+    sleep 30
+) >>"$log" 2>&1 &
+writer=$!
+began=$SECONDS
+port=$(discover_server_port "$log" "$writer") || fail "the late port line was missed"
+test "$port" = 40001 || fail "read port '$port', want 40001"
+test "$((SECONDS - began))" -ge 2 || fail "the port arrived before it was written"
+kill "$writer" 2>/dev/null || true
+wait "$writer" 2>/dev/null || true
+
+# --- a half-written line is not a port ---------------------------------------
+# The announcement is one write in practice, but reading it in two would hand the
+# crawl a truncated port number, which resolves to a different server or none.
+log="$tmp/split.log"
+printf 'PORT 40' >"$log"
+(
+    sleep 2
+    printf '002\n'
+    sleep 30
+) >>"$log" 2>&1 &
+writer=$!
+port=$(discover_server_port "$log" "$writer") || fail "the completed port line was missed"
+test "$port" = 40002 || fail "read port '$port' from a split line, want 40002"
+kill "$writer" 2>/dev/null || true
+wait "$writer" 2>/dev/null || true
+
+# --- a server that dies without announcing is reported, not waited out -------
+log="$tmp/dead.log"
+echo "Traceback: address already in use" >"$log"
+(exit 1) &
+writer=$!
+wait "$writer" 2>/dev/null || true
+began=$SECONDS
+rc=0
+err=$(discover_server_port "$log" "$writer" 2>&1 >/dev/null) || rc=$?
+test "$rc" -ne 0 || fail "a dead server yielded a port"
+test "$((SECONDS - began))" -lt 20 || fail "a dead server was waited out ($((SECONDS - began))s)"
+case "$err" in
+*"exited early"*) ;;
+*) fail "a dead server reported '$err'" ;;
+esac
+# 2, not 1: only the deadline is the announce race 72 and 105 skip on, so one
+# status for both had local-crawl.sh skipping a crashed server instead of failing.
+test "$rc" -eq 2 || fail "a dead server exited $rc, want 2"
+
+echo "poll helpers OK"

tests/228_icon-small-flat.test

@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Issue #938: downscaling the vector master to 16 px antialiased every stem to
+# grey. The small raster is drawn on the pixel grid, so it must stay flat.
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+srcdir="${abs_top_srcdir:?not run under make check}"
+div="${srcdir}/html/server/div"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+colors() {
+    "${python}" "${testdir}/png-colors.py" "${div}/$1/httrack.png" ||
+        fail "$1: cannot read the icon"
+}
+
+small=$(colors 16x16)
+n=$(wc -l <<<"${small}")
+test "${n}" -le 4 || fail "16x16 icon has ${n} colours, the stems are antialiased"
+
+# Both flats present: an all-field tile and an all-ink one are flat too.
+for flat in '#9999cc' '#040404'; do
+    grep -q " ${flat}\$" <<<"${small}" || fail "16x16 icon has no ${flat} pixel"
+done
+
+# A hairline and a near-solid block both satisfy every count above.
+ink=$(sed -n "s/ #040404\$//p" <<<"${small}")
+if test "${ink}" -lt 26 || test "${ink}" -gt 128; then
+    fail "16x16 icon inks ${ink} of 256 pixels"
+fi
+
+# Control: a counter blind to antialiasing would pass every assertion above.
+big=$(wc -l <<<"$(colors 48x48)")
+test "${big}" -gt 16 || fail "48x48 icon has ${big} colours, the counter is blind"
+
+echo "16x16 icon: ${n} colours, ${ink}/256 ink; 48x48: ${big} colours"

tests/230_local-ftp-userpass.test

@@ -0,0 +1,94 @@
+#!/bin/bash
+#
+# The credentials an URL carries must reach the control channel whole: a clip
+# logged in as a different account (#1032). The engine test only ever probes a
+# dead port, so nothing there reads what the login actually sends.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+# Starves the x64 CI runner until the whole suite step is lost; Win32 is fine (#1038).
+if is_windows; then
+    echo "windows: wedges the x64 runner (#1038), skipping"
+    exit 77
+fi
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftpuser.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+out="${tmpdir}/crawl"
+cmds="${tmpdir}/cmds"
+mkdir -p "$root" "$out"
+printf 'body\n' >"${root}/f.txt"
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" --require-pass \
+    --log "$(nativepath "$cmds")" >"$serverlog" 2>&1 &
+serverpid=$!
+port=$(discover_server_port "$serverlog" "$serverpid") || exit 1
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+rep() { awk -v n="$2" -v c="$1" 'BEGIN { while (i++ < n) printf "%s", c }'; }
+
+crawl() {
+    : >"$cmds"
+    rm -rf "${out:?}"
+    mkdir -p "$out"
+    run_with_timeout 60 httrack "ftp://$1@127.0.0.1:${port}/f.txt" -O "$out" \
+        --quiet --disable-security-limits --robots=0 --timeout=20 \
+        --max-time=45 --retries=1 -c1 >"${tmpdir}/log" 2>&1
+}
+
+# --- a plain login goes out unchanged ----------------------------------------
+crawl "bob:secret" || fail "the plain-login crawl never finished"
+sent=$(cat "$cmds")
+grep -qxF "USER bob" <<<"$sent" || fail "no USER bob on the wire: ${sent}"
+grep -qxF "PASS secret" <<<"$sent" || fail "no PASS secret on the wire: ${sent}"
+cmp -s "${out}/127.0.0.1_${port}/f.txt" "${root}/f.txt" ||
+    fail "the login did not bring the body back; mirror holds $(find "$out" -type f)"
+ok "bob:secret reaches the server verbatim, and mirrors"
+
+# --- the longest userinfo that fits, byte for byte ---------------------------
+# 255 of each: a 256th byte is refused, so this is what the wire must carry.
+user=$(rep a 255)
+pass=$(rep b 255)
+crawl "${user}:${pass}" || fail "the 255-byte login crawl never finished"
+sent=$(cat "$cmds")
+grep -qxF "USER ${user}" <<<"$sent" ||
+    fail "the wire user is not the 255 bytes the URL named: $(grep -a '^USER' <<<"$sent")"
+grep -qxF "PASS ${pass}" <<<"$sent" ||
+    fail "the wire pass is not the 255 bytes the URL named: $(grep -a '^PASS' <<<"$sent")"
+ok "a 255-byte user and password reach the server unclipped"
+
+# --- over-long userinfo never opens a session --------------------------------
+# The bare name has no ':' to bound it, only the '@', and is the worse branch.
+for u in "$(rep a 256):pw" "$(rep a 256)"; do
+    crawl "$u" || fail "the over-long crawl never finished"
+    grep -aq "FTP user name or password too long" "${out}/hts-log.txt" ||
+        fail "over-long userinfo was not refused: $(cat "${out}/hts-log.txt")"
+    sent=$(cat "$cmds")
+    test -z "$sent" || fail "the engine logged in under a clipped name: ${sent}"
+done
+ok "an over-long name is refused before anything reaches the control channel"

tests/231_tests-list-bijection.test

@@ -0,0 +1,40 @@
+#!/bin/bash
+#
+# Covers the unregistered-file case; a stale registration cannot reach this test,
+# because make check dies first.
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+checker="${testdir}/check-tests-list.sh"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+work=$(mktemp -d "${TMPDIR:-/tmp}/tests_list.XXXXXX") || fail "no tmpdir"
+trap 'set +e; rm -rf "$work"' EXIT
+
+bash "$checker" "$testdir" || fail "the tree's own registrations do not check out"
+
+# Positive controls: every branch of the checker is silent on a clean tree, so
+# one that had stopped comparing would pass the assertion above just as well.
+control() {
+    local name=$1 want=$2 mutate=$3 dir="${work}/$1" out=""
+    mkdir -p "$dir"
+    : >"${dir}/00_kept.test"
+    printf 'TESTS =\nTESTS += 00_kept.test\n' >"${dir}/tests-list.mk"
+    (cd "$dir" && eval "$mutate")
+    if out=$(bash "$checker" "$dir" 2>&1); then
+        fail "$name: the checker accepted a broken list"
+    fi
+    grep -q "$want" <<<"$out" || fail "$name: wrong diagnostic: $out"
+}
+
+control absent 'registered but absent' 'echo "TESTS += 99_gone.test" >>tests-list.mk'
+control unregistered 'present but unregistered' ': >99_ghost.test'
+control duplicate 'registered twice' 'echo "TESTS += 00_kept.test" >>tests-list.mk'
+control malformed 'malformed' 'echo "TESTS += 00_kept.test 99_x.test" >>tests-list.mk'
+
+echo "OK: bijection holds, and the checker still sees each way it can break"

tests/232_online-gate-outoftree.test

@@ -0,0 +1,172 @@
+#!/bin/bash
+#
+# The online crawl gates must resolve their helper scripts against the script's
+# own directory: an out-of-tree build tree holds no copy, so a bare relative
+# path fails to open and the tests skip claiming the network is unavailable
+# (#1016). Everything here runs from an unrelated directory with the flag off,
+# so it never touches the network.
+
+set -euo pipefail
+
+# 226 asks this of every test naming the Windows suite driver, as the sweep's
+# allowlist below does.
+unset WATCHDOG_TOKEN WATCHDOG_REPO WATCHDOG_SHA WATCHDOG_URL
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+
+work=$(mktemp -d)
+cleanup() {
+    rm -rf "$work"
+}
+trap 'set +e; cleanup' EXIT
+trap 'set +e; cleanup; exit 1' HUP INT QUIT TERM
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+## the gated tests must skip for the reason the flag gave, not because the gate
+## script could not be opened
+
+# Discovered rather than listed, so a new online test is covered too. Matches the
+# gate line itself, not the mentions of it in comments.
+gated=$(cd "$testdir" && command grep -lE '^[[:space:]]*[^#[:space:]].*check-network\.sh.*exit 77' -- *.test | sort)
+count=$(printf '%s\n' "$gated" | command grep -c . || true)
+# Floor: the seven of #1016. A silently empty list would pass every check below.
+test "$count" -ge 7 || fail "found only $count tests gating on check-network.sh, expected at least 7"
+
+for t in $gated; do
+    out=$(cd "$work" && ONLINE_UNIT_TESTS=no bash "$testdir/$t" 2>&1) && rc=0 || rc=$?
+    test "$rc" -eq 77 || fail "$t exited $rc from an out-of-tree cwd, expected 77 (skip)"
+    # Only check-network.sh emits this, and only once it has actually run. The
+    # "skipping online unit tests" banner prints on the broken path too, so
+    # matching that instead would assert nothing.
+    command grep -q 'online tests are disabled' <<<"$out" ||
+        fail "$t skipped without the gate's reason, got: $out"
+done
+
+## the flag still decides, from an out-of-tree cwd
+
+out=$(cd "$work" && ONLINE_UNIT_TESTS=no bash "$testdir/check-network.sh" 2>&1) && rc=0 || rc=$?
+test "$rc" -eq 1 || fail "check-network.sh exited $rc with the flag off, expected 1"
+command grep -q 'online tests are disabled' <<<"$out" || fail "check-network.sh gave no reason: $out"
+
+out=$(cd "$work" && ONLINE_UNIT_TESTS=yes bash "$testdir/check-network.sh" 2>&1) && rc=0 || rc=$?
+test "$rc" -eq 0 || fail "check-network.sh exited $rc with the flag on, expected 0: $out"
+
+## no helper anywhere in tests/ is invoked through the caller's cwd
+#
+# The gate above cannot see this class: with the flag off check-network.sh exits
+# before a test's own crawl-test.sh line is ever reached, so the sweep is the
+# only thing standing between #1016 and a rerun of it.
+
+# An invocation is safe when its operand resolves the same from any cwd: an
+# absolute path, or one built from a variable holding a directory. Quotes and a
+# leading "./" change nothing about what the shell opens, so neither is exempt.
+invoke='(^|[[:space:];&|(`])(bash|sh|source|\.)[[:space:]]+'
+quote='["'"'"']?'
+literal_re="$invoke$quote"'(\./)?[A-Za-z0-9_][A-Za-z0-9_./-]*\.sh'
+var_re="$invoke$quote"'\$\{?[A-Za-z_][A-Za-z0-9_]*\}?'"$quote"'([[:space:]]|$)'
+# Anchored at a command position, or the word inside these failure messages
+# would match and this file would pin itself.
+eval_re='(^|[;&|(`])[[:space:]]*eval[[:space:]]'
+
+# The only two call sites that mean the cwd: each cd's into a directory it has
+# just filled with copies, and the copy is the target. Named rather than matched
+# by shape, because a shape exemption cannot tell them from #1016 in disguise.
+allowed='172_ci-windows-driver.test ci-windows-suite.sh
+ci-windows-suite.sh test-timeout.sh'
+
+# Matching lines of the given files as file:line:text, comments dropped:
+# local-crawl.sh documents its own usage with a bare call.
+code_lines() {
+    local re=$1
+    shift
+    command grep -HnE "$re" -- "$@" | command grep -vE '^[^:]*:[0-9]+:[[:space:]]*#' || true
+}
+
+# Prints every invocation resolved against the caller's cwd, and every one whose
+# operand it cannot resolve. Silence means the files are clean.
+sweep() {
+    local hit file text script var val
+
+    while IFS= read -r hit; do
+        test -n "$hit" || continue
+        file=${hit%%:*}
+        text=${hit#*:}
+        text=${text#*:}
+        script=$(command grep -oE '[A-Za-z0-9_][A-Za-z0-9_./-]*\.sh' <<<"$text" | head -1)
+        command grep -qxF "${file##*/} $script" <<<"$allowed" || echo "cwd-relative: $hit"
+    done < <(code_lines "$literal_re" "$@")
+
+    # A variable operand is only as good as what it holds, so resolve it: an
+    # assignment that is not itself anchored, or that cannot be found in the
+    # file at all, is reported rather than assumed innocent.
+    while IFS= read -r hit; do
+        test -n "$hit" || continue
+        file=${hit%%:*}
+        text=${hit#*:}
+        text=${text#*:}
+        var=$(command grep -oE "$var_re" <<<"$text" | head -1 |
+            sed -E 's/.*\$\{?([A-Za-z_][A-Za-z0-9_]*)\}?.*/\1/')
+        val=$(command grep -hE "^[[:space:]]*$var=" "$file" | head -1)
+        val=${val#*=}
+        val=${val#[\"\']}
+        case $val in
+        /* | \$*) ;;
+        *) echo "unresolved \$$var: $hit" ;;
+        esac
+    done < <(code_lines "$var_re" "$@")
+}
+
+# eval hides its operand from any static reading, so the set of them is pinned
+# instead: a new one fails here and gets a human's eye rather than passing unseen.
+scan_eval() {
+    code_lines "$eval_re" "$@" |
+        sed -E 's#^\./##; s/^([^:]*):[0-9]+:[[:space:]]*(.*[[:space:]])?eval[[:space:]]+/\1 /'
+}
+
+# Controls. Each evasion is a printf format here, so the forms under test reach
+# the fixtures without appearing in this file as invocations the sweep would see.
+name=crawl-test.sh
+driver=ci-windows-suite.sh
+ev=$work/evasions
+mkdir -p "$ev"
+printf '#!/bin/bash\nbash %s --errors 0\n' "$name" >"$ev/bare.sh"
+printf '#!/bin/bash\nbash ./%s --errors 0\n' "$name" >"$ev/dotslash.sh"
+printf '#!/bin/bash\nbash "%s" --errors 0\n' "$name" >"$ev/quoted.sh"
+# shellcheck disable=SC2016 # the fixture's text, not this shell's
+printf '#!/bin/bash\nS=%s\nbash $S --errors 0\n' "$name" >"$ev/indirect.sh"
+# shellcheck disable=SC2016 # the fixture's text, not this shell's
+printf '#!/bin/bash\nbash ./%s "$bin"\n' "$driver" >"$ev/impostor.sh"
+# shellcheck disable=SC2016 # the fixture's text, not this shell's
+printf '#!/bin/bash\nG="$testdir/%s"\nbash $G --errors 0\n' "$name" >"$ev/anchored.sh"
+printf '#!/bin/bash\n# bash %s --errors 0\n' "$name" >"$ev/commented.sh"
+
+for f in bare dotslash quoted indirect impostor; do
+    test -n "$(sweep "$ev/$f.sh")" || fail "the sweep let $f through: $(tail -n +2 "$ev/$f.sh")"
+done
+# The other half of the control: a rule that flags everything proves nothing.
+for f in anchored commented; do
+    out=$(sweep "$ev/$f.sh")
+    test -z "$out" || fail "the sweep flagged $f.sh, which is correct code: $out"
+done
+printf '#!/bin/bash\neval "bash %s"\n' "$name" >"$ev/eval.sh"
+test -n "$(scan_eval "$ev/eval.sh")" || fail "the eval scan missed a planted eval"
+
+hits=$(cd "$testdir" && sweep ./*.test ./*.sh)
+test -z "$hits" || fail "helper scripts reachable only from the right cwd:
+$hits"
+
+# Pinned by operand, since none of these builds a command naming a helper today.
+# shellcheck disable=SC2016 # the pinned operands, quoted as they appear
+reviewed='145_webhttrack-datadir.test "${line}"
+231_tests-list-bijection.test "$mutate")
+61_webhttrack-locale.test "${block}"
+61_webhttrack-locale.test "${funcs}"'
+got=$(cd "$testdir" && scan_eval ./*.test ./*.sh | sort)
+test "$got" = "$(sort <<<"$reviewed")" || fail "the set of eval sites changed; each hides its operand from the sweep and needs reading:
+$got"
+
+echo "OK: $count gated tests skip with the flag's reason, and tests/ is free of cwd-relative helper calls"

tests/233_local-ftp-ctrl-timeout.test

@@ -0,0 +1,127 @@
+#!/bin/bash
+#
+# Exercises --timeout on the FTP control and data channels, with two bounds no single
+# hardcoded constant satisfies, plus --timeout=0 meaning unlimited rather than instant.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftptmo.XXXXXX")
+mutepid=
+livepid=
+cleanup() {
+    stop_server "$mutepid"
+    stop_server "$livepid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+cmds="${tmpdir}/cmds"
+mkdir -p "$root"
+printf 'body\n' >"${root}/f.txt"
+# One holds the data connection open and says nothing; the other feeds it in
+# chunks 0.2s apart, so a zero-length wait cannot ride the gaps out.
+awk 'BEGIN { s = sprintf("%0999d", 0); for (i = 0; i < 100; i++) print s }' \
+    >"${root}/stall.bin"
+awk 'BEGIN { s = sprintf("%0999d", 0); for (i = 0; i < 60; i++) print s }' \
+    >"${root}/trick.bin"
+printf '/stall.bin stall\n/trick.bin trickle\n' >"${tmpdir}/modes"
+
+mutelog="${tmpdir}/mute.out"
+"$python" "$server" --root "$(nativepath "$root")" --mute >"$mutelog" 2>&1 &
+mutepid=$!
+muteport=$(discover_server_port "$mutelog" "$mutepid") || exit 1
+
+livelog="${tmpdir}/live.out"
+"$python" "$server" --root "$(nativepath "$root")" \
+    --mode-file "$(nativepath "${tmpdir}/modes")" \
+    --log "$(nativepath "$cmds")" >"$livelog" 2>&1 &
+livepid=$!
+liveport=$(discover_server_port "$livelog" "$livepid") || exit 1
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+
+# No --max-time anywhere below, so --timeout is the only thing that can end a run.
+crawl() {
+    local tag=$1 to=$2 cap=$3 url=$4
+    local out="${tmpdir}/out${tag}" rc=0
+
+    mkdir -p "$out"
+    run_with_timeout "$cap" httrack "$url" -O "$out" --quiet \
+        --disable-security-limits --robots=0 --timeout="$to" -c1 \
+        >"${tmpdir}/log${tag}" 2>&1 || rc=$?
+    # Only the cap and a signal are the crawl's own failure. A nonzero exit against
+    # a deliberately broken server is not, and the assertions below judge the rest.
+    test "$rc" -ne 124 || fail "the ${tag} run was still going at ${cap}s"
+    test "$rc" -lt 128 || fail "the ${tag} run died on signal $((rc - 128))"
+}
+
+# Every case names the timeout it saw, since htsftp.c prints the value actually in
+# force: that kills a hardcoded constant more directly than a timing bound does.
+logged() {
+    grep -aq "$2" "${tmpdir}/out${1}/hts-log.txt" ||
+        fail "no '$2' in the log: $(grep -a Error "${tmpdir}/out${1}/hts-log.txt" || true)"
+}
+
+# --- control channel: a server that connects and never greets -----------------
+# The engine retries once, so a run costs about twice the value asked for.
+start=$SECONDS
+crawl 4 4 60 "ftp://127.0.0.1:${muteport}/f.txt"
+short=$((SECONDS - start))
+logged 4 'read timeout (4)'
+test "$short" -lt 18 || fail "--timeout=4 took ${short}s: the greeting read ignores it"
+ok "a 4s timeout ends the greeting read in ${short}s"
+
+# The floor here sits above the ceiling above, so no one constant satisfies both.
+start=$SECONDS
+crawl 20 20 120 "ftp://127.0.0.1:${muteport}/f.txt"
+long=$((SECONDS - start))
+logged 20 'read timeout (20)'
+test "$long" -ge 20 ||
+    fail "--timeout=20 gave up after ${long}s, short of the value asked for"
+test "$long" -lt 70 || fail "--timeout=20 took ${long}s"
+ok "a 20s timeout waits ${long}s: the wait tracks the value, not a constant"
+
+# --- data channel: logged in, mid-body, and the server goes quiet -------------
+# The control channel is answered here, so only the body-transfer wait can hang.
+: >"$cmds"
+start=$SECONDS
+crawl stall 5 60 "ftp://127.0.0.1:${liveport}/stall.bin"
+stalled=$((SECONDS - start))
+sent=$(cat "$cmds")
+grep -q '^RETR ' <<<"$sent" || fail "the body transfer never started: ${sent}"
+logged stall 'Time out (5)'
+test "$stalled" -lt 40 ||
+    fail "a stalled body transfer took ${stalled}s under --timeout=5"
+ok "a 5s timeout ends the body-transfer wait in ${stalled}s"
+
+# --- --timeout=0 is unlimited, not zero --------------------------------------
+: >"$cmds"
+start=$SECONDS
+crawl zero 0 90 "ftp://127.0.0.1:${liveport}/trick.bin"
+zeroed=$((SECONDS - start))
+got="${tmpdir}/outzero/127.0.0.1_${liveport}/trick.bin"
+test -f "$got" || fail "--timeout=0 fetched nothing; mirror holds $(find \
+    "${tmpdir}/outzero" -type f -not -path '*hts-cache*')"
+want=$(wc -c <"${root}/trick.bin")
+have=$(wc -c <"$got")
+test "$have" -eq "$want" ||
+    fail "--timeout=0 gave up mid-transfer: ${have} of ${want} bytes"
+ok "--timeout=0 rides out a ${zeroed}s trickle and keeps all ${want} bytes"

tests/234_local-ftp-maxtime.test

@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# Exercises --max-time against an in-flight FTP transfer; --timeout is set far above
+# the cap so only the mirror deadline can end the run.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftpmax.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+out="${tmpdir}/crawl"
+cmds="${tmpdir}/cmds"
+mkdir -p "$root" "$out"
+# Big enough that the served prefix leaves the transfer open and mid-body.
+awk 'BEGIN { s = sprintf("%0999d", 0); for (i = 0; i < 100; i++) print s }' \
+    >"${root}/f.bin"
+printf '* stall\n' >"${tmpdir}/modes"
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" \
+    --mode-file "$(nativepath "${tmpdir}/modes")" \
+    --log "$(nativepath "$cmds")" >"$serverlog" 2>&1 &
+serverpid=$!
+port=$(discover_server_port "$serverlog" "$serverpid") || exit 1
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+
+start=$SECONDS
+run_with_timeout 90 httrack "ftp://127.0.0.1:${port}/f.bin" -O "$out" --quiet \
+    --disable-security-limits --robots=0 --timeout=200 --max-time=20 -c1 \
+    >"${tmpdir}/log" 2>&1 || fail "the run did not finish in $((SECONDS - start))s"
+elapsed=$((SECONDS - start))
+
+log="${out}/hts-log.txt"
+# Confirms the transfer had started; otherwise the wait under test never runs.
+sent=$(cat "$cmds")
+grep -q '^RETR ' <<<"$sent" ||
+    fail "the transfer never started, so nothing was in flight: ${sent}"
+ok "the crawl was inside the body transfer when the cap hit"
+
+test "$elapsed" -ge 15 ||
+    fail "the run ended after ${elapsed}s, before --max-time could be reached"
+test "$elapsed" -lt 60 || fail "--max-time=20 left the run going for ${elapsed}s"
+ok "--max-time ends the in-flight transfer in ${elapsed}s"
+
+grep -aq 'More than 20 seconds passed' "$log" ||
+    fail "the mirror deadline is not what ended the run: $(grep -a Error "$log" || true)"
+# Proves the run really did carry a 200s timeout it could have waited out.
+grep -aq 'Time out (200)' "$log" ||
+    fail "--timeout=200 was not the timeout in force: $(grep -a Error "$log" || true)"
+ok "the deadline beat a --timeout ten times its size"

tests/236_local-ftp-teardown.test

@@ -0,0 +1,102 @@
+#!/bin/bash
+#
+# Regression test for #1051: the engine freed FTP transfer slots a worker thread
+# was still writing to.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+size_of() { wc -c <"$1" | tr -d '[:space:]'; }
+
+errlog="${tmpdir}/httrack.err"
+# A sanitized build reports the use-after-free and still exits 1, so the status
+# alone is blind to it. Leaks at exit are out of scope, hence no LeakSanitizer.
+assert_clean_run() {
+    cat "$errlog" >&2 # the redirect must not cost the harness log the crawl's stderr
+    ! grep -Eq 'AddressSanitizer:|MemorySanitizer:|runtime error:' "$errlog" ||
+        fail "$1: the crawl produced a sanitizer report (#1051)"
+}
+
+root="${tmpdir}/root"
+modes="${tmpdir}/modes"
+mkdir -p "$root"
+# Big enough that neither mode can finish before the cap trips.
+"$python" -c 'import sys; sys.stdout.buffer.write(b"F" * 4194304)' >"${root}/f.bin"
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" \
+    --mode-file "$(nativepath "$modes")" >"$serverlog" 2>&1 &
+serverpid=$!
+port=$(discover_server_port "$serverlog" "$serverpid") || exit 1
+
+CAP=100000
+
+# The cap must trip with the transfer still in flight, or nothing here is tested.
+assert_capped() {
+    local out=$1 got
+    got=$(size_of "${out}/127.0.0.1_${port}/f.bin" 2>/dev/null || echo 0)
+    test "$got" -ge "$CAP" ||
+        fail "$2: only ${got} bytes arrived, the ${CAP}-byte cap never tripped"
+}
+
+# --- the crash: cap reached while the FTP thread is still writing to its slot
+echo "* trickle" >"$modes"
+out="${tmpdir}/trickle"
+rc=0
+start=$SECONDS
+run_with_timeout 120 httrack "ftp://127.0.0.1:${port}/f.bin" -O "$out" --quiet \
+    --disable-security-limits --robots=0 --timeout=200 -M"$CAP" -c1 2>"$errlog" || rc=$?
+elapsed=$((SECONDS - start))
+assert_clean_run "trickle"
+test "$rc" -ne 124 || fail "trickle crawl overran its 120s cap"
+# Faults reliably only while the backlog array stays mmap-sized: free() unmaps it.
+test "$rc" -lt 128 || fail "trickle crawl died on signal $((rc - 128)) (#1051)"
+assert_capped "$out" "trickle"
+# A drain that waits the transfer out instead of cancelling it also exits 0, and
+# the trickle runs ~100s against ~3s for a cancel, so budget the clock as well.
+test "$elapsed" -lt 30 ||
+    fail "trickle crawl took ${elapsed}s, teardown waited for the transfer"
+ok "a --max-size cap on a trickling FTP transfer exits cleanly in ${elapsed}s"
+
+# --- the bound: teardown cancels the thread instead of waiting out its read.
+# The FTP data read is a 300s wait, so anything near that means the stop flag
+# was only seen between operations.
+echo "* stall" >"$modes"
+out="${tmpdir}/stall"
+rc=0
+start=$SECONDS
+run_with_timeout 120 httrack "ftp://127.0.0.1:${port}/f.bin" -O "$out" --quiet \
+    --disable-security-limits --robots=0 --timeout=200 -M"$CAP" -c1 2>"$errlog" || rc=$?
+elapsed=$((SECONDS - start))
+assert_clean_run "stall"
+test "$rc" -ne 124 || fail "stalled crawl overran its 120s cap, teardown is unbounded"
+test "$rc" -lt 128 || fail "stalled crawl died on signal $((rc - 128)) (#1051)"
+assert_capped "$out" "stall"
+test "$elapsed" -lt 60 ||
+    fail "stalled crawl took ${elapsed}s, teardown waited out the FTP read"
+ok "teardown of a stalled FTP transfer took ${elapsed}s"

tests/58_watchdog.test

@@ -12,7 +12,10 @@ testdir=$(cd "$(dirname "$0")" && pwd)
 . "${testdir}/testlib.sh"
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_wd.XXXXXX")
-trap 'set +e; rm -rf "$tmp"' EXIT
+# A colon-free root for the PATH shim below: MSYS hands out a drive-letter TMPDIR
+# and a PATH entry carrying a colon is read as two, which leaves it unreachable.
+shim=$(mktemp -d /tmp/httrack_wdshim.XXXXXX 2>/dev/null) || shim="$tmp/starve"
+trap 'set +e; rm -rf "$tmp" "$shim"' EXIT
 fail() {
     echo "FAIL: $*" >&2
     exit 1
@@ -73,7 +76,10 @@ fi
 # Wall clock, not iteration count: under starvation a poll costs more than it asks,
 # so a counted deadline never fires before the CI step it was meant to beat.
 (
-    starve_sleep "$tmp/starve" 4 || fail "could not install the slow sleep"
+    # The forked tick is the only one a shimmed sleep can reach: against the fd
+    # tick nothing here would be starved but the payload, and the case goes inert.
+    export HTTRACK_POLL_SLEEP=1
+    starve_sleep "$shim" 4 || fail "could not install the slow sleep"
     # Control: starvation must not make the watchdog kill work that is inside its
     # budget. The command outlives several polls, so a guard that fires on the
     # first one cannot pass this by exiting before it is ever looked at.

tests/62_lang-integrity.test

@@ -67,6 +67,17 @@ fi
 # htsserver.c decodes the POST body with it, so bytes that are not what it says
 # render and convert wrong (#963).
 c1=$(printf '\302[\200-\237]')
+# A single-byte charset maps all 256 byte values, so the iconv decode above
+# can never fail on it and cannot see a value left as raw UTF-8; a whole line
+# that itself decodes as valid multi-byte UTF-8 is that leak.
+# Scoped to single-byte charsets: BIG5, gb2312 and shift-jis are multi-byte, so
+# the premise fails and their genuine text matches this shape constantly.
+# Under 8 bytes a line can be valid UTF-8 by chance (Ukrainian.txt:686 is a
+# 2-byte word that is), so short mixed-encoding lines go undetected.
+# Raw bytes in an awk ERE, not grep -P: BSD grep has no -P. NUL is left out of
+# the ASCII range because an awk record cannot hold one, and no lang file has any.
+utf8_full_line=$(printf '^([\001-\177]|[\302-\337][\200-\277]|\340[\240-\277][\200-\277]|[\341-\354][\200-\277][\200-\277]|\355[\200-\237][\200-\277]|[\356-\357][\200-\277][\200-\277]|\360[\220-\277][\200-\277][\200-\277]|[\361-\363][\200-\277][\200-\277][\200-\277]|\364[\200-\217][\200-\277][\200-\277])*$')
+high_byte=$(printf '[\200-\377]')
 charset_declared_matches_bytes() {
     f=$1
     cs=$(LC_ALL=C awk '{ sub(/\r$/, "") } $0 == "LANGUAGE_CHARSET" { p = 1; next } p { print; exit }' "$f")
@@ -86,6 +97,16 @@ charset_declared_matches_bytes() {
         echo "${f##*/}: $n lines decode to C1 controls under $cs; the bytes are a Windows codepage"
         return 1
     fi
+    case "$(printf '%s' "$cs" | LC_ALL=C tr '[:upper:]' '[:lower:]')" in
+    iso-8859-* | windows-125*)
+        ln=$(LC_ALL=C awk -v u="$utf8_full_line" -v h="$high_byte" \
+            'length($0) >= 8 && $0 ~ h && $0 ~ u { print NR; exit }' "$f")
+        if [ -n "$ln" ]; then
+            echo "${f##*/}:$ln: line fully decodes as UTF-8 despite declared $cs; mixed encoding"
+            return 1
+        fi
+        ;;
+    esac
 }
 
 if ! command -v iconv >/dev/null 2>&1; then
@@ -95,7 +116,10 @@ else
     # No CR: BSD sed writes a literal r, and the extractor strips CR anyway.
     LC_ALL=C sed '10s/.*/UTF-8/' "$langdir/Francais.txt" >"$tmp/undecodable.txt"
     LC_ALL=C sed '10s/.*/ISO-8859-2/' "$langdir/Croatian.txt" >"$tmp/c1.txt"
-    for probe in undecodable c1; do
+    # A value line left as raw UTF-8 under a declared single-byte charset.
+    LC_ALL=C awk -v repl="$(printf 'Une fuite UTF-8 : caf\xc3\xa9, na\xc3\xafve, t\xc3\xa9l\xc3\xa9phone')" \
+        'NR == 20 { print repl; next } { print }' "$langdir/Francais.txt" >"$tmp/utf8mix.txt"
+    for probe in undecodable c1 utf8mix; do
         if charset_declared_matches_bytes "$tmp/$probe.txt" >/dev/null; then
             echo "the charset check passed a deliberately mislabelled file ($probe)"
             exit 1

tests/94_local-single-file.test

@@ -38,6 +38,14 @@ cat >"${doc}/index.html" <<'EOF'
 <img src="pixel.svg" data-src="lazy.svg" alt="l">
 <script src="app.js"></script>
 <a href="other.html">other</a>
+<p>id=pixel.svg#!0011223344556677.-.9 end</p>
+<link rel="stylesheet" href="style_noext">
+<script src="app_noext"></script>
+<img src="pixel.svg#icon">
+<a href="pixel.svg">raw</a>
+<a href=pixel.svg>unquoted</a>
+<iframe src=pixel.svg></iframe>
+<link href="style_norel.css">
 </body></html>
 EOF
 cat >"${doc}/other.html" <<'EOF'
@@ -51,6 +59,9 @@ cat >"${doc}/nested.css" <<'EOF'
 div { background: url(pixel.svg); }
 EOF
 printf 'var app = 1;\n' >"${doc}/app.js"
+printf 'body { color: red; }\n' >"${doc}/style_noext"
+printf 'body { color: green; }\n' >"${doc}/style_norel.css"
+printf 'var noext = 1;\n' >"${doc}/app_noext"
 printf '<svg xmlns="http://www.w3.org/2000/svg"><rect width="1" height="1"/></svg>\n' \
     >"${doc}/pixel.svg"
 printf '<svg xmlns="http://www.w3.org/2000/svg"><circle r="1"/></svg>\n' \
@@ -171,6 +182,40 @@ test -f "${out}/127.0.0.1_${port}/other.html" || {
 }
 echo OK
 
+printf '[a navigational rel keeps its link] ..\t'
+# The page carries a later rel="stylesheet"; reading rel past its own value let
+# this canonical link inherit it and inline the page it names.
+grep -q 'rel="canonical" href="other.html"' "$page" || {
+    echo "FAIL: rel=canonical was inlined"
+    exit 1
+}
+echo OK
+
+printf '[an unnamed tag cannot skip the deny rules] ..\t'
+# An unquoted value ending at '>' loses the tag name, which used to read as
+# "a CSS body" and took these past every tag-bearing deny rule.
+grep -q '<a href=pixel.svg>' "$page" || {
+    echo "FAIL: an unquoted anchor was inlined"
+    exit 1
+}
+grep -q '<iframe src=pixel.svg>' "$page" || {
+    echo "FAIL: an unquoted iframe was inlined"
+    exit 1
+}
+echo OK
+
+printf '[a rel-less link keeps its link] ..\t'
+# Nothing loads it, so it is not an asset however inlinable the file looks.
+grep -q '<link href="style_norel.css">' "$page" || {
+    echo "FAIL: a <link> with no rel was inlined"
+    exit 1
+}
+test -f "${out}/127.0.0.1_${port}/style_norel.css" || {
+    echo "FAIL: style_norel.css is missing, so the probe proves nothing"
+    exit 1
+}
+echo OK
+
 printf '[over-cap asset stays a link] ..\t'
 grep -q 'src="big.svg"' "$page" || {
     echo "FAIL: the over-cap asset did not keep its link"
@@ -242,12 +287,106 @@ grep -q 'data:image/svg+xml;base64,' "$badpage" || {
     echo "FAIL: a rejected cap did not fall back to the default"
     exit 1
 }
-if grep -q 'src="big.svg"' "$badpage"; then
-    echo "FAIL: the default cap did not cover big.svg"
+echo OK
+
+printf '[a page cannot forge a mark] ..\t'
+# A mark of the right shape but the wrong secret, against a real mirrored
+# asset. Expanding needs the secret, so it is deleted rather than resolved: a
+# truncated secret comparison would put a data: URI here instead. pixel.svg is
+# inlined elsewhere on the page, so the encoder was demonstrably willing.
+forged=$(sed -n 's/.*<p>\(id=[^<]*\)<\/p>.*/\1/p' "$page")
+test "$forged" = 'id=pixel.svg end' || {
+    echo "FAIL: the page was altered: $forged"
+    exit 1
+}
+grep -q 'data:image/svg+xml;base64,' "$page" || {
+    echo "FAIL: nothing was inlined at all, so the probe proves nothing"
+    exit 1
+}
+echo OK
+
+printf '[a mark cannot ride in on a header] ..\t'
+# htsparse echoes the Content-Type charset into a <meta>; a forged mark there
+# bypasses anything that scans the body. Unforgeable means the channel does
+# not matter, so this asserts on the channel the body sweep could never see.
+grep -q 'charset=127' "$page" || true
+if grep -q 'content="text/html;charset=data:' "$page"; then
+    echo "FAIL: a charset-borne mark was expanded"
     exit 1
 fi
 echo OK
 
+printf '[no file keeps a mark] ..\t'
+# Stripping matches the mark's shape, not this run's secret, so nothing in the
+# mirror may carry one -- pages included, and whatever an earlier run left.
+marked=$(grep -rlE '#![0-9a-f]{16}\.[-cj]\.[0-9]+' \
+    "${out}/127.0.0.1_${port}" 2>/dev/null | tr '\n' ' ')
+test -z "$marked" || {
+    echo "FAIL: marks survived in $marked"
+    exit 1
+}
+echo OK
+
+printf '[an unrecognised type still inlines from its context] ..\t'
+# style_noext and app_noext have no usable extension, so only the referencing
+# context says what they are. plain.css/plain.js are the control: they inline
+# either way, so a failure here is the context path and not the encoder.
+grep -q 'href="data:text/css;base64,' "$page" || {
+    echo "FAIL: no stylesheet inlined at all"
+    exit 1
+}
+if grep -q 'href="style_noext"' "$page"; then
+    echo "FAIL: extensionless stylesheet was not inlined"
+    exit 1
+fi
+if grep -q 'src="app_noext"' "$page"; then
+    echo "FAIL: extensionless script was not inlined"
+    exit 1
+fi
+echo OK
+
+printf '[a fragment survives onto the data: URI] ..\t'
+# An SVG fragment selects a view, so dropping it renders the wrong thing (#766).
+grep -q 'data:image/svg+xml;base64,[A-Za-z0-9+/=]*#icon' "$page" || {
+    echo "FAIL: the fragment was dropped"
+    exit 1
+}
+echo OK
+
+printf '[a page link is not inlined even when it names an asset] ..\t'
+# <a href> to an image: only the deny rule stops this, since the MIME gate
+# would happily take it. Without the rule the anchor becomes a data: URI.
+grep -q '<a href="pixel.svg">raw</a>' "$page" || {
+    echo "FAIL: an anchor to an inlinable asset was rewritten"
+    exit 1
+}
+echo OK
+
+printf '[-%%M refuses to pair with --single-file] ..\t'
+both=$(httrack -O "${tmpdir}/both" --quiet --robots=0 --retries=0 -%M \
+    --single-file "${base}/index.html" 2>&1 || true)
+case "$both" in
+*"pick one"*) echo OK ;;
+*)
+    echo "FAIL: the combination was accepted: $both"
+    exit 1
+    ;;
+esac
+
+printf '[--changes and --single-file together] ..\t'
+# Both hook end-of-mirror. Expansion must run first, so the report sees the
+# inlined size rather than the marked intermediate.
+chg="${tmpdir}/chg"
+mkdir "$chg"
+httrack "${common[@]}" --single-file --changes "${base}/index.html" \
+    >"${tmpdir}/log6" 2>&1
+chgpage="${chg}/127.0.0.1_${port}/index.html"
+test ! -f "$chgpage" || grep -q 'base64,' "$chgpage" || {
+    echo "FAIL: --changes suppressed the inlining"
+    exit 1
+}
+echo OK
+
 printf '[the rewritten page keeps the mirror file mode] ..\t'
 # The rewrite spools to a temp and renames, bypassing the engine's chmod.
 if is_windows; then

tests/Makefile.am

@@ -4,18 +4,19 @@ include $(srcdir)/tests-list.mk
 # Committed binary fixture read by 01_zlib-cache-golden.test. List it
 # explicitly: automake does not expand wildcards in EXTRA_DIST, so a glob would
 # silently drop it from the dist tarball and break "make distcheck".
-EXTRA_DIST = $(TESTS) renamefail.c threadattrfail.c nobacktrace.c altstackprobe.c nopieprobe.c crawl-test.sh run-all-tests.sh check-network.sh \
+EXTRA_DIST = $(TESTS) renamefail.c threadattrfail.c nobacktrace.c altstackprobe.c nopieprobe.c crawl-test.sh run-all-tests.sh check-network.sh check-tests-list.sh \
 	proxy-https-server.py socks5-server.py proxy-connect-server.py \
 	proxytestlib.py tls-stall-server.py warc-validate.py wacz-validate.py \
 	header-injection-server.py header-injection-check.py \
-	pty-resize.py test-timeout.sh \
+	pty-resize.py test-timeout.sh png-colors.py \
 	local-crawl.sh local-server.py ftp-server.py testlib.sh \
-	ci-windows-suite.sh \
+	ci-windows-suite.sh ci-windows-watchdog.ps1 \
 	server.crt server.key \
 	server-root/simple/basic.html server-root/simple/link.html \
 	server-root/stripquery/index.html server-root/stripquery/a.html \
 	server-root/fraglink/index.html server-root/fraglink/target.html \
-	fixtures/cache-golden/hts-cache/new.zip
+	fixtures/cache-golden/hts-cache/new.zip \
+	install-manifest.txt
 
 TESTS_ENVIRONMENT =
 TESTS_ENVIRONMENT += PATH=$(top_builddir)/src$(PATH_SEPARATOR)$$PATH
@@ -34,6 +35,10 @@ TESTS_ENVIRONMENT += abs_top_builddir=$(abs_top_builddir)
 TESTS_ENVIRONMENT += CONFIGURED_DATADIR=$(datadir)
 TESTS_ENVIRONMENT += CONFIGURED_LIBDIR=$(libdir)
 TESTS_ENVIRONMENT += CONFIGURED_BINDIR=$(bindir)
+# 225_install-manifest.test folds these back out of the staged paths.
+TESTS_ENVIRONMENT += CONFIGURED_MANDIR=$(mandir)
+TESTS_ENVIRONMENT += CONFIGURED_DOCDIR=$(docdir)
+TESTS_ENVIRONMENT += CONFIGURED_HTMLDIR=$(htmldir)
 TESTS_ENVIRONMENT += ORIGIN_RPATH=$(ORIGIN_RPATH)
 # 222_pkgconfig-consumer.test asserts the staged .pc against these, and both
 # sides of the gate deciding whether it hands a consumer an rpath.

tests/altstackprobe.c

@@ -14,6 +14,11 @@
 
 #define _GNU_SOURCE
 
+/* The largefile redirect renames this file's mmap() to mmap64, colliding with
+   its own mmap64(). _TIME_BITS rides on _FILE_OFFSET_BITS, so it goes too. */
+#undef _FILE_OFFSET_BITS
+#undef _TIME_BITS
+
 #include <dlfcn.h>
 #include <fcntl.h>
 #include <signal.h>
@@ -187,6 +192,19 @@ static void *trace_mapped(void *sp) {
   return sp;
 }
 
+/* The fallback when dlsym() failed, so it has to be right: 32-bit arches have
+   no SYS_mmap and i386's wants an argument block, so prefer SYS_mmap2, whose
+   offset is in 4096-byte units whatever the page size. */
+static void *raw_mmap(void *addr, size_t len, int prot, int flags, int fd,
+                      off_t off) {
+#ifdef SYS_mmap2
+  return (void *) syscall(SYS_mmap2, addr, len, prot, flags, fd,
+                          (unsigned long) (off / 4096));
+#else
+  return (void *) syscall(SYS_mmap, addr, len, prot, flags, fd, off);
+#endif
+}
+
 SHIM_EXPORT void *mmap(void *addr, size_t len, int prot, int flags, int fd,
                        off_t off);
 
@@ -194,10 +212,9 @@ SHIM_EXPORT void *mmap(void *addr, size_t len, int prot, int flags, int fd,
    between, so the thread's last mapping is the whole of the check. */
 SHIM_EXPORT void *mmap(void *addr, size_t len, int prot, int flags, int fd,
                        off_t off) {
-  return trace_mapped(
-      real_mmap != NULL
-          ? real_mmap(addr, len, prot, flags, fd, off)
-          : (void *) syscall(SYS_mmap, addr, len, prot, flags, fd, off));
+  return trace_mapped(real_mmap != NULL
+                          ? real_mmap(addr, len, prot, flags, fd, off)
+                          : raw_mmap(addr, len, prot, flags, fd, off));
 }
 
 /* glibc only, and the name that matters: _FILE_OFFSET_BITS=64 redirects the

tests/check-network.sh

@@ -1,6 +1,8 @@
 #!/bin/bash
 #
 
+testdir=$(cd "$(dirname "$0")" && pwd)
+
 # ensure the httrack unit tests are available so that ut will not break
 # the build in case of network outage
 
@@ -32,7 +34,7 @@ else
         fi
 
     # fetch single file
-    elif bash crawl-test.sh --errors 0 --files 1 httrack --timeout=3 --max-time=3 "$url" 2>/dev/null >/dev/null; then
+    elif bash "$testdir/crawl-test.sh" --errors 0 --files 1 httrack --timeout=3 --max-time=3 "$url" 2>/dev/null >/dev/null; then
         echo "ok" >$cache
         exit 0
     else

tests/check-tests-list.sh

@@ -0,0 +1,52 @@
+#!/bin/bash
+#
+# tests-list.mk and tests/*.test must stay a bijection (#1037): a stale
+# registration kills make check silently, an unregistered file just never runs.
+
+set -euo pipefail
+export LC_ALL=C
+
+testdir=$(cd "${1:-$(dirname "$0")}" && pwd)
+list="${testdir}/tests-list.mk"
+[ -r "$list" ] || {
+    echo "not readable: $list" >&2
+    exit 2
+}
+
+status=0
+bad() {
+    echo "$*" >&2
+    status=1
+}
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/tests-list.XXXXXX")
+trap 'set +e; rm -rf "$tmp"' EXIT
+
+# The sed below assumes one file per "TESTS +=" line; a continuation or a second
+# name on a line would slip past unseen.
+strays=$(grep -nv -e '^$' -e '^#' -e '^TESTS =$' -e '^TESTS += [^ ]*\.test$' "$list" || true)
+[ -z "$strays" ] || bad "malformed tests-list.mk lines, want one 'TESTS += NN_name.test' each:
+$strays"
+
+sed -n 's/^TESTS += //p' "$list" | sort >"$tmp/all"
+# A directory or a dangling symlink named NN_x.test is not a test file, and
+# make check would choke on it; count it as absent.
+for f in "$testdir"/*.test; do
+    if [ -f "$f" ]; then printf '%s\n' "${f##*/}"; fi
+done | sort >"$tmp/present"
+
+# Dedupe before comparing, or a repeated entry also reads as an absent file.
+dup=$(uniq -d "$tmp/all")
+[ -z "$dup" ] || bad "registered twice: $dup"
+uniq "$tmp/all" >"$tmp/registered"
+
+missing=$(comm -23 "$tmp/registered" "$tmp/present")
+[ -z "$missing" ] || bad "registered but absent, so make check dies with no results: $missing"
+
+extra=$(comm -13 "$tmp/registered" "$tmp/present")
+[ -z "$extra" ] || bad "present but unregistered, so never run: $extra"
+
+if [ "$status" -eq 0 ]; then
+    echo "tests-list.mk: $(wc -l <"$tmp/registered") registrations, each matching a file"
+fi
+exit "$status"

tests/ci-windows-suite.sh

@@ -29,6 +29,41 @@ ci_annotate() {
 # schedule the workflow really passes off a virtual clock instead of waiting it out.
 hb_now() { echo "$SECONDS"; }
 
+# Start the off-box telemetry over the progress log $1, setting ci_watchdog_pid;
+# return 1 with no PowerShell available. Forks nothing and kills nothing, so it
+# reports where the heartbeat below cannot (#795).
+ci_start_native_watchdog() {
+    local progress=$1 ps1 c exe='' waited=0
+    ps1="$testdir/ci-windows-watchdog.ps1"
+    test -r "$ps1" || return 1
+    for c in pwsh powershell.exe; do
+        if command -v "$c" >/dev/null 2>&1; then
+            exe=$c
+            break
+        fi
+    done
+    test -n "$exe" || return 1
+    # Never the step's stdout: a background holder of that pipe keeps the step
+    # open past the suite (#949), and tests/*.log reaches the artifact anyway.
+    : >watchdog.log
+    WATCHDOG_TOKEN="${ci_watchdog_token:-}" \
+        "$exe" -NoProfile -NonInteractive -ExecutionPolicy Bypass \
+        -File "$(nativepath "$ps1")" \
+        -ProgressLog "$(nativepath "$progress")" \
+        >>watchdog.log 2>&1 &
+    ci_watchdog_pid=$!
+    # $! comes from the fork, not the exec: wait for it to actually speak.
+    while test "$waited" -lt 30; do
+        grep -q '^watchdog ready$' watchdog.log && return 0
+        kill -0 "$ci_watchdog_pid" 2>/dev/null || break
+        sleep 1
+        waited=$((waited + 1))
+    done
+    kill_pid "$ci_watchdog_pid"
+    ci_watchdog_pid=''
+    return 1
+}
+
 ci_suite_heartbeat() {
     local quiet=$1 every=$2 progress=$3 stuck=$4 main=$5
     local tick=$2 begin now line said moved last=''
@@ -48,6 +83,9 @@ ci_suite_heartbeat() {
         # outlive the step's own timeout before being caught.
         if test $((now - moved)) -ge "$stuck"; then
             ci_annotate error "suite watchdog" "killing the step: $((now - moved))s without progress, in flight: $last"
+            # Ahead of the kill, which runs no EXIT trap: an orphan would outlive the
+            # step and overwrite its last status with a frozen tail.
+            test -z "${watchdog:-}" || kill_pid "$watchdog"
             # Direct first: kill_tree may reap this watchdog before its own root (#953).
             kill_pid "$main"
             kill_tree "$main"
@@ -77,6 +115,11 @@ command -v httrack >/dev/null || {
     exit 1
 }
 
+# Out of the environment before the first child forks below: an inherited token
+# could post a commit status in the name of the run.
+ci_watchdog_token=${WATCHDOG_TOKEN:-}
+unset WATCHDOG_TOKEN
+
 # httrack.exe is native, so MSYS rewrites any argument shaped like a
 # POSIX path, and a URL path is shaped exactly like one: "/a/b.html"
 # reached the engine as "C:/Program Files/Git/a/b.html". Switch that
@@ -119,9 +162,16 @@ export HTTRACK_PROGRESS_LOG="$PWD/$progress"
 # its own terms keeps its log. Quiet past 16 min, clear of the 13 a healthy
 # run measures here, and a kill 900s after the last progress line clears
 # the longest legitimate gap, one $per_test.
-ci_suite_heartbeat 960 360 "$progress" 900 $$ &
+stuck=900
+# Orthogonal to that heartbeat rather than a spare of it: the heartbeat needs
+# 960s of quiet and 900s of static log, and every #795 death measured so far
+# lands inside the first 750s of the step.
+watchdog='' ci_watchdog_pid=''
+ci_start_native_watchdog "$PWD/$progress" && watchdog=$ci_watchdog_pid
+test -n "$watchdog" || echo "no off-box watchdog: no usable PowerShell"
+ci_suite_heartbeat 960 360 "$progress" "$stuck" $$ &
 heartbeat=$!
-trap 'kill "$heartbeat" 2>/dev/null || true' EXIT
+trap 'set +e; kill "$heartbeat" 2>/dev/null; test -z "$watchdog" || kill_pid "$watchdog"' EXIT
 
 pass=0 fail=0 skip=0 failed="" skipped="" deadline=0
 # label:pattern, globbed rather than enumerated so a new NNN_engine-*.test or
@@ -217,7 +267,8 @@ echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
 # datadir-ospath copies the unwrapped binary the automake build leaves in .libs,
 # and needs the loader variable libtool picked, neither of which this job has;
 # link-control-bytes names its fixtures with the raw control bytes the requests
-# decode back to, which NTFS refuses.
+# decode back to, which NTFS refuses;
+# ftp-userpass starves the x64 runner until the step is lost, Win32 passing (#1038).
 expected_skips="01_engine-footer-overflow.test
 100_local-purge-longpath.test
 158_local-link-control-bytes.test
@@ -228,6 +279,7 @@ expected_skips="01_engine-footer-overflow.test
 152_engine-string-oom.test
 153_local-proxytrack-quiet.test
 215_engine-datadir-ospath.test
+230_local-ftp-userpass.test
 48_local-crange-memresume.test
 71_local-crange-repaircache.test
 79_local-proxytrack-webdav-mime.test

tests/ci-windows-watchdog.ps1

@@ -0,0 +1,255 @@
+# Off-box telemetry for the Windows suite step (#795): it spawns and kills
+# nothing, so a wedge cannot disarm it, and reports as a commit status, the only
+# channel that outlives a dead runner. Every status carries the same state.
+param(
+    [string]$ProgressLog = '',
+    [int]$IntervalSeconds = 30,
+    [int]$PollSeconds = 5,
+    # Cannot outlive the step, whatever the caller forgets to kill.
+    [int]$MaxSeconds = 2700,
+    # Seam for the suite's own test, which points it at a sink it can count.
+    [string]$ApiBase = 'https://api.github.com',
+    [switch]$NoPost,
+    [switch]$SelfTest
+)
+
+$ErrorActionPreference = 'Stop'
+# Windows PowerShell renders a progress bar per web call otherwise.
+$ProgressPreference = 'SilentlyContinue'
+
+# --- decisions, kept pure so -SelfTest can drive them by return value ---------
+
+function Get-WatchdogAction {
+    param([int]$Now, [int]$Posted, [int]$Interval)
+    if (($Now - $Posted) -ge $Interval) { return 'post' }
+    return 'wait'
+}
+
+# Posts to skip after a rejected one: a fork PR's token is read-only for the run.
+function Get-NextBackoff {
+    param([int]$Current)
+    if ($Current -lt 1) { return 1 }
+    return [Math]::Min($Current * 2, 32)
+}
+
+# The (skip, backoff) pair after an attempted post, $Ok being whether it landed:
+# a rejection that later resolves has to leave nothing behind.
+function Get-NextThrottle {
+    param([bool]$Ok, [int]$Backoff)
+    if ($Ok) { return @(0, 0) }
+    $b = Get-NextBackoff $Backoff
+    return @($b, $b)
+}
+
+# GitHub truncates a description at 140 chars, so the counters take the cut, not
+# the fields a wedge is read for. $Static below zero is unknown.
+function Format-WatchdogStatus {
+    param([int]$Elapsed, [int]$Static, [string]$InFlight, [string]$Counters)
+    $q = '?'
+    if ($Static -ge 0) { $q = [string]$Static }
+    $t = ($InFlight -replace '\s+', ' ').Trim()
+    if ($t.Length -gt 46) { $t = $t.Substring(0, 46) }
+    $s = 't={0}s q={1}s {2} | {3}' -f $Elapsed, $q, $t, $Counters
+    if ($s.Length -gt 140) { $s = $s.Substring(0, 140) }
+    return $s
+}
+
+# --- probes ------------------------------------------------------------------
+
+# One try/catch per counter: a probe that fails costs its own field, not the loop.
+# In-process only. A CIM query is richer, but its connect to a wedged WMI service
+# is unbounded, and would hang the one reporter still standing.
+function Get-WatchdogCounters {
+    $f = New-Object System.Collections.ArrayList
+    try {
+        $ps = @(Get-Process)
+        [void]$f.Add('p={0}' -f $ps.Count)
+        [void]$f.Add('h={0}' -f (($ps | Measure-Object -Property Handles -Sum).Sum))
+    } catch { [void]$f.Add('p=? h=?') }
+    try {
+        $drive = New-Object System.IO.DriveInfo($env:SystemDrive + '\')
+        [void]$f.Add('d={0}' -f [int]($drive.AvailableFreeSpace / 1MB))
+    } catch { [void]$f.Add('d=?') }
+    return ($f -join ' ')
+}
+
+# Ok separates "nothing moved" from "could not read it", which would otherwise report
+# a wedge for an unreadable file. Share flags: the driver appends as we read.
+function Get-ProgressTail {
+    param([string]$Path)
+    $r = @{ Ok = $false; Signature = ''; Line = '' }
+    if (-not $Path) { return $r }
+    try {
+        $share = [System.IO.FileShare]::ReadWrite -bor [System.IO.FileShare]::Delete
+        $fs = New-Object System.IO.FileStream($Path, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, $share)
+        try {
+            $sr = New-Object System.IO.StreamReader($fs)
+            $text = $sr.ReadToEnd()
+        } finally { $fs.Dispose() }
+        $lines = @($text -split "`r?`n" | Where-Object { $_ -ne '' })
+        if ($lines.Count -gt 0) { $r.Line = $lines[-1] }
+        $r.Signature = '{0}|{1}' -f $text.Length, $r.Line
+        $r.Ok = $true
+    } catch { }
+    return $r
+}
+
+function Write-WatchdogLog {
+    param([string]$Message)
+    # A full disk is a state the sick runner reaches, and a log line lost to it
+    # must not take the loop reporting off-box with it.
+    try { Write-Host ('[watchdog {0:HH:mm:ss}] {1}' -f (Get-Date), $Message) } catch { }
+}
+
+# --- reporting ---------------------------------------------------------------
+
+$script:Repo = $env:WATCHDOG_REPO
+$script:Sha = $env:WATCHDOG_SHA
+$script:Token = $env:WATCHDOG_TOKEN
+$script:TargetUrl = $env:WATCHDOG_URL
+$script:Context = $env:WATCHDOG_CONTEXT
+if (-not $script:Context) { $script:Context = 'windows-suite-watchdog' }
+
+# One state always, so nothing downstream can read a verdict out of telemetry:
+# GitHub records the job's conclusion already. -NoPost is the test's hard stop,
+# whatever the environment holds.
+function Send-WatchdogStatus {
+    param([string]$Description)
+    if ($NoPost -or $SelfTest) { return $false }
+    if (-not $script:Token -or -not $script:Repo -or -not $script:Sha) { return $false }
+    $body = @{ state = 'success'; context = $script:Context; description = $Description }
+    if ($script:TargetUrl) { $body['target_url'] = $script:TargetUrl }
+    try {
+        Invoke-RestMethod -Method Post -TimeoutSec 20 `
+            -Uri ('{0}/repos/{1}/statuses/{2}' -f $ApiBase.TrimEnd('/'), $script:Repo, $script:Sha) `
+            -UserAgent 'httrack-windows-suite-watchdog' `
+            -Headers @{
+            Authorization = ('Bearer {0}' -f $script:Token)
+            Accept        = 'application/vnd.github+json'
+        } -ContentType 'application/json' -Body ($body | ConvertTo-Json -Compress) | Out-Null
+        return $true
+    } catch {
+        Write-WatchdogLog ('status post failed: {0}' -f $_.Exception.Message)
+        return $false
+    }
+}
+
+# --- self-test ----------------------------------------------------------------
+
+function Invoke-WatchdogSelfTest {
+    $bad = New-Object System.Collections.ArrayList
+    function Assert-That($cond, $what) { if (-not $cond) { [void]$bad.Add($what) } }
+
+    Assert-That ((Get-WatchdogAction 30 0 30) -eq 'post') 'a post due exactly on the interval was skipped'
+    Assert-That ((Get-WatchdogAction 29 0 30) -eq 'wait') 'posted ahead of the interval'
+    Assert-That ((Get-WatchdogAction 5000 4990 30) -eq 'wait') 'posted off cadence'
+
+    Assert-That ((Get-NextBackoff 0) -eq 1) 'the first rejection does not back off'
+    Assert-That ((Get-NextBackoff 1) -eq 2) 'the backoff does not grow'
+    Assert-That ((Get-NextBackoff 32) -eq 32) 'the backoff is not capped'
+
+    $ok = Get-NextThrottle $true 8
+    Assert-That ($ok[0] -eq 0 -and $ok[1] -eq 0) 'a landed post leaves the run throttled'
+    $ko = Get-NextThrottle $false 0
+    Assert-That ($ko[0] -eq 1 -and $ko[1] -eq 1) 'a first rejection skips nothing'
+    $ko = Get-NextThrottle $false 4
+    Assert-That ($ko[0] -eq 8 -and $ko[1] -eq 8) 'a repeat rejection does not widen the gap'
+
+    $long = '43_local-update-truncate-with-a-very-long-name-indeed.test'
+    $line = Format-WatchdogStatus 812 41 $long 'p=118 h=41230 d=13210'
+    Assert-That ($line.Length -le 140) ('status description is {0} characters' -f $line.Length)
+    Assert-That ($line -like 't=812s q=41s 43_local-update-truncate*') ('status leads with the wrong fields: {0}' -f $line)
+    Assert-That ($line -like '*d=13210') 'the counters did not survive a long test name'
+    # -match, not -like: '?' is a wildcard there, so q=0s would satisfy it too.
+    Assert-That ((Format-WatchdogStatus 8 -1 'x' 'y') -match '^t=8s q=\?s x \| y$') 'an unknown staticness reads as a number'
+    $clip = Format-WatchdogStatus 1 2 ('x' * 80) 'c'
+    Assert-That ($clip -match '^t=1s q=2s x{46} \| c$') ('the in-flight name was not clipped to 46: {0}' -f $clip)
+    $wide = Format-WatchdogStatus 1 2 ('x' * 300) ('y' * 300)
+    Assert-That ($wide.Length -le 140) ('an oversized status was not clipped: {0}' -f $wide.Length)
+    # Cut from the tail: the head carries the fields a wedge is read for.
+    Assert-That ($wide -like 't=1s q=2s x*') ('clipping dropped the leading fields: {0}' -f $wide)
+
+    $gone = Get-ProgressTail -Path ('no-such-progress-log-{0}.tmp' -f $PID)
+    Assert-That (-not $gone.Ok) 'an unreadable log reads as one that was read'
+    $f = New-Object System.IO.FileInfo([System.IO.Path]::GetTempFileName())
+    try {
+        [System.IO.File]::WriteAllText($f.FullName, "first`nRUN 42_probe.test at 7s`n")
+        $tail = Get-ProgressTail -Path $f.FullName
+        Assert-That ($tail.Ok) 'a readable log reads as unreadable'
+        Assert-That ($tail.Line -eq 'RUN 42_probe.test at 7s') ('the tail is not the last line: {0}' -f $tail.Line)
+    } finally { [System.IO.File]::Delete($f.FullName) }
+
+    # Space-separated key=value: the counters share the 140-char description with
+    # the fields a wedge is read for, and '?' from a failed probe is a value.
+    $c = Get-WatchdogCounters
+    Assert-That ($c -match '^[a-z]+=\S+( [a-z]+=\S+)*$') ('the counters are not key=value pairs: {0}' -f $c)
+    foreach ($k in 'p', 'h', 'd') {
+        Assert-That ($c -match ('(^| ){0}=' -f $k)) ('the counters dropped {0}=: {1}' -f $k, $c)
+    }
+    Assert-That ($c.Length -le 60) ('the counters take {0} of the 140 characters' -f $c.Length)
+
+    # Nothing else reads these: every other leg passes its own schedule.
+    Assert-That ($IntervalSeconds -eq 30) ('the default status cadence is {0}s' -f $IntervalSeconds)
+    Assert-That ($PollSeconds -eq 5) ('the default poll is {0}s' -f $PollSeconds)
+
+    Assert-That (-not (Send-WatchdogStatus 'self-test')) 'the self-test can reach the API'
+
+    if ($bad.Count -gt 0) {
+        foreach ($b in $bad) { Write-Host ('self-test FAIL: {0}' -f $b) }
+        exit 1
+    }
+    Write-Host 'watchdog self-test OK'
+    exit 0
+}
+
+if ($SelfTest) { Invoke-WatchdogSelfTest }
+
+# --- main loop ----------------------------------------------------------------
+
+# Windows PowerShell still defaults below TLS 1.2, which api.github.com refuses.
+try { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 } catch { }
+
+$sw = [System.Diagnostics.Stopwatch]::StartNew()
+$lastSig = ''
+$movedAt = 0
+# Negative, so the first tick posts: an early status is itself a datum.
+$postedAt = -$IntervalSeconds
+$backoff = 0
+$skip = 0
+
+# Guarded like the rest; the launcher waits for this exact line.
+try { Write-Host 'watchdog ready' } catch { }
+Write-WatchdogLog ('watching {0} every {1}s' -f $ProgressLog, $IntervalSeconds)
+
+while ($sw.Elapsed.TotalSeconds -lt $MaxSeconds) {
+    # Measured, never accumulated: starvation is what makes a sleep overshoot.
+    $now = [int]$sw.Elapsed.TotalSeconds
+    try {
+        $tail = Get-ProgressTail -Path $ProgressLog
+        if ($tail.Ok -and $tail.Signature -ne $lastSig) {
+            $lastSig = $tail.Signature
+            $movedAt = $now
+        }
+        if ((Get-WatchdogAction $now $postedAt $IntervalSeconds) -eq 'post') {
+            $postedAt = $now
+            $static = -1
+            if ($tail.Ok) { $static = $now - $movedAt }
+            $desc = Format-WatchdogStatus $now $static $tail.Line (Get-WatchdogCounters)
+            # Logged whatever the backoff decides: it throttles the API, not the
+            # artifact, which is all a run whose token cannot post will leave.
+            Write-WatchdogLog $desc
+            if ($skip -gt 0) {
+                $skip--
+            } else {
+                $next = Get-NextThrottle (Send-WatchdogStatus $desc) $backoff
+                $skip = $next[0]
+                $backoff = $next[1]
+            }
+        }
+    } catch {
+        Write-WatchdogLog ('tick failed: {0}' -f $_.Exception.Message)
+    }
+    Start-Sleep -Seconds $PollSeconds
+}
+
+Write-WatchdogLog ('stopping after {0}s' -f [int]$sw.Elapsed.TotalSeconds)

tests/ftp-server.py

@@ -8,8 +8,13 @@ directory). Each line is "<path> <mode>...", path "*" matching everything:
 
   truncate  send a prefix of the body, then drop the data connection
   empty     open the data connection and send nothing
+  stall     send a prefix, then hold the data connection open and go quiet
+  trickle   send the body in small chunks, slowly enough to stay in flight
   norest    answer REST with 500, so the client re-fetches from scratch
   nomdtm    answer MDTM with 500, like a server predating RFC 3659
+
+--require-pass answers USER with 331, the only way the client's PASS is sent.
+--mute accepts the control connection and never speaks, not even the greeting.
 """
 
 import argparse
@@ -20,17 +25,27 @@ import threading
 import time
 
 
+# Longer than any bound a test asserts, so going quiet can never be what ends
+# the run under test.
+QUIET_SECONDS = 900
+
+TRICKLE_CHUNK = 8192
+TRICKLE_PAUSE = 0.2
+
+
 def reply(conn, text):
     conn.sendall((text + "\r\n").encode("utf-8", "replace"))
 
 
 class Session(threading.Thread):
-    def __init__(self, conn, root, mode_file, log):
+    def __init__(self, conn, root, mode_file, log, require_pass=False, mute=False):
         threading.Thread.__init__(self, daemon=True)
         self.conn = conn
         self.root = root
         self.mode_file = mode_file
         self.log = log
+        self.require_pass = require_pass
+        self.mute = mute
         self.pasv = None
         self.rest = 0
         self.path = "/"  # named by SIZE/RETR; REST carries no path
@@ -88,12 +103,21 @@ class Session(threading.Thread):
                 pass
             elif "truncate" in modes:
                 data.sendall(body[: max(1, len(body) // 8)])
+            elif "stall" in modes:
+                # A fraction, never the whole body: a completed transfer would
+                # end the wait the tests are timing.
+                data.sendall(body[: max(1, len(body) // 8)])
+                time.sleep(QUIET_SECONDS)
+            elif "trickle" in modes:
+                for off in range(0, len(body), TRICKLE_CHUNK):
+                    data.sendall(body[off : off + TRICKLE_CHUNK])
+                    time.sleep(TRICKLE_PAUSE)
             else:
                 data.sendall(body)
         except OSError:
             pass
         data.close()
-        if modes & {"empty", "truncate"}:
+        if modes & {"empty", "truncate", "stall"}:
             reply(self.conn, "426 transfer aborted")
         else:
             reply(self.conn, "226 Transfer complete")
@@ -132,7 +156,9 @@ class Session(threading.Thread):
 
     def dispatch(self, verb, arg):
         conn = self.conn
-        if verb in ("USER", "PASS", "TYPE", "NOOP"):
+        if verb == "USER" and self.require_pass:
+            reply(conn, "331 password required")  # the client sends PASS only on a 3xx
+        elif verb in ("USER", "PASS", "TYPE", "NOOP"):
             reply(conn, "200 ok")
         elif verb == "SYST":
             reply(conn, "215 UNIX Type: L8")
@@ -183,6 +209,9 @@ class Session(threading.Thread):
         conn = self.conn
         buf = b""
         try:
+            if self.mute:
+                time.sleep(QUIET_SECONDS)  # connected, but not even a greeting
+                return
             reply(conn, "220 httrack test ftp")
             while True:
                 while b"\r\n" not in buf:
@@ -209,6 +238,8 @@ def main():
     ap.add_argument("--root", required=True)
     ap.add_argument("--mode-file")
     ap.add_argument("--log")
+    ap.add_argument("--require-pass", action="store_true")
+    ap.add_argument("--mute", action="store_true")
     args = ap.parse_args()
 
     logfp = open(args.log, "a", encoding="utf-8") if args.log else None
@@ -231,7 +262,7 @@ def main():
     root = os.path.abspath(args.root)
     while True:
         conn, _ = srv.accept()
-        Session(conn, root, args.mode_file, log).start()
+        Session(conn, root, args.mode_file, log, args.require_pass, args.mute).start()
 
 
 if __name__ == "__main__":

tests/install-manifest.txt

@@ -0,0 +1,123 @@
+@BINDIR@/htsserver
+@BINDIR@/httrack
+@BINDIR@/proxytrack
+@BINDIR@/webhttrack
+@DATADIR@/applications/WebHTTrack-Websites.desktop
+@DATADIR@/applications/WebHTTrack.desktop
+@DATADIR@/httrack/html
+@DATADIR@/httrack/lang.def
+@DATADIR@/httrack/lang.indexes
+@DATADIR@/httrack/lang/Bulgarian.txt
+@DATADIR@/httrack/lang/Castellano.txt
+@DATADIR@/httrack/lang/Cesky.txt
+@DATADIR@/httrack/lang/Chinese-BIG5.txt
+@DATADIR@/httrack/lang/Chinese-Simplified.txt
+@DATADIR@/httrack/lang/Croatian.txt
+@DATADIR@/httrack/lang/Dansk.txt
+@DATADIR@/httrack/lang/Deutsch.txt
+@DATADIR@/httrack/lang/Eesti.txt
+@DATADIR@/httrack/lang/English.txt
+@DATADIR@/httrack/lang/Finnish.txt
+@DATADIR@/httrack/lang/Francais.txt
+@DATADIR@/httrack/lang/Greek.txt
+@DATADIR@/httrack/lang/Italiano.txt
+@DATADIR@/httrack/lang/Japanese.txt
+@DATADIR@/httrack/lang/Macedonian.txt
+@DATADIR@/httrack/lang/Magyar.txt
+@DATADIR@/httrack/lang/Nederlands.txt
+@DATADIR@/httrack/lang/Norsk.txt
+@DATADIR@/httrack/lang/Polski.txt
+@DATADIR@/httrack/lang/Portugues-Brasil.txt
+@DATADIR@/httrack/lang/Portugues.txt
+@DATADIR@/httrack/lang/Romanian.txt
+@DATADIR@/httrack/lang/Russian.txt
+@DATADIR@/httrack/lang/Slovak.txt
+@DATADIR@/httrack/lang/Slovenian.txt
+@DATADIR@/httrack/lang/Svenska.txt
+@DATADIR@/httrack/lang/Turkish.txt
+@DATADIR@/httrack/lang/Ukrainian.txt
+@DATADIR@/httrack/lang/Uzbek.txt
+@DATADIR@/httrack/libtest/callbacks-example-baselinks.c
+@DATADIR@/httrack/libtest/callbacks-example-changecontent.c
+@DATADIR@/httrack/libtest/callbacks-example-contentfilter.c
+@DATADIR@/httrack/libtest/callbacks-example-displayheader.c
+@DATADIR@/httrack/libtest/callbacks-example-filename.c
+@DATADIR@/httrack/libtest/callbacks-example-filename2.c
+@DATADIR@/httrack/libtest/callbacks-example-filenameiisbug.c
+@DATADIR@/httrack/libtest/callbacks-example-listlinks.c
+@DATADIR@/httrack/libtest/callbacks-example-log.c
+@DATADIR@/httrack/libtest/callbacks-example-simple.c
+@DATADIR@/httrack/libtest/example-main.c
+@DATADIR@/httrack/libtest/example-main.h
+@DATADIR@/httrack/libtest/readme.txt
+@DATADIR@/httrack/templates/index-body.html
+@DATADIR@/httrack/templates/index-footer.html
+@DATADIR@/httrack/templates/index-header.html
+@DATADIR@/httrack/templates/topindex-body.html
+@DATADIR@/httrack/templates/topindex-bodycat.html
+@DATADIR@/httrack/templates/topindex-footer.html
+@DATADIR@/httrack/templates/topindex-header.html
+@DATADIR@/icons/hicolor/128x128/apps/httrack.png
+@DATADIR@/icons/hicolor/16x16/apps/httrack.png
+@DATADIR@/icons/hicolor/256x256/apps/httrack.png
+@DATADIR@/icons/hicolor/32x32/apps/httrack.png
+@DATADIR@/icons/hicolor/48x48/apps/httrack.png
+@DATADIR@/icons/hicolor/64x64/apps/httrack.png
+@DATADIR@/icons/hicolor/scalable/apps/httrack.svg
+@DATADIR@/metainfo/com.httrack.WebHTTrack.metainfo.xml
+@DATADIR@/pixmaps/httrack.xpm
+@DATADIR@/pixmaps/httrack16x16.xpm
+@DATADIR@/pixmaps/httrack32x32.xpm
+@DATADIR@/pixmaps/httrack48x48.xpm
+@INCLUDEDIR@/httrack/config.h
+@INCLUDEDIR@/httrack/htsarrays.h
+@INCLUDEDIR@/httrack/htsbasenet.h
+@INCLUDEDIR@/httrack/htsbauth.h
+@INCLUDEDIR@/httrack/htsconfig.h
+@INCLUDEDIR@/httrack/htsdefines.h
+@INCLUDEDIR@/httrack/htsglobal.h
+@INCLUDEDIR@/httrack/htsmodules.h
+@INCLUDEDIR@/httrack/htsnet.h
+@INCLUDEDIR@/httrack/htsopt.h
+@INCLUDEDIR@/httrack/htssafe.h
+@INCLUDEDIR@/httrack/htsstrings.h
+@INCLUDEDIR@/httrack/htswrap.h
+@INCLUDEDIR@/httrack/httrack-library.h
+@LIBDIR@/httrack/libbaselinks.so
+@LIBDIR@/httrack/libbaselinks.so.1
+@LIBDIR@/httrack/libbaselinks.so.1.@revision@
+@LIBDIR@/httrack/libchangecontent.so
+@LIBDIR@/httrack/libchangecontent.so.1
+@LIBDIR@/httrack/libchangecontent.so.1.@revision@
+@LIBDIR@/httrack/libcontentfilter.so
+@LIBDIR@/httrack/libcontentfilter.so.1
+@LIBDIR@/httrack/libcontentfilter.so.1.@revision@
+@LIBDIR@/httrack/libdisplayheader.so
+@LIBDIR@/httrack/libdisplayheader.so.1
+@LIBDIR@/httrack/libdisplayheader.so.1.@revision@
+@LIBDIR@/httrack/libfilename.so
+@LIBDIR@/httrack/libfilename.so.1
+@LIBDIR@/httrack/libfilename.so.1.@revision@
+@LIBDIR@/httrack/libfilename2.so
+@LIBDIR@/httrack/libfilename2.so.1
+@LIBDIR@/httrack/libfilename2.so.1.@revision@
+@LIBDIR@/httrack/libfilenameiisbug.so
+@LIBDIR@/httrack/libfilenameiisbug.so.1
+@LIBDIR@/httrack/libfilenameiisbug.so.1.@revision@
+@LIBDIR@/httrack/liblistlinks.so
+@LIBDIR@/httrack/liblistlinks.so.1
+@LIBDIR@/httrack/liblistlinks.so.1.@revision@
+@LIBDIR@/httrack/liblog.so
+@LIBDIR@/httrack/liblog.so.1
+@LIBDIR@/httrack/liblog.so.1.@revision@
+@LIBDIR@/httrack/libsimple.so
+@LIBDIR@/httrack/libsimple.so.1
+@LIBDIR@/httrack/libsimple.so.1.@revision@
+@LIBDIR@/libhttrack.so
+@LIBDIR@/libhttrack.so.3
+@LIBDIR@/libhttrack.so.3.@revision@
+@LIBDIR@/pkgconfig/libhttrack.pc
+@MANDIR@/man1/htsserver.1
+@MANDIR@/man1/httrack.1
+@MANDIR@/man1/proxytrack.1
+@MANDIR@/man1/webhttrack.1

tests/local-crawl.sh

@@ -253,14 +253,12 @@ serverpid=$!
 
 # Wait for the "PORT <n>" line (server prints it once bound). A cold Python
 # start under a parallel `make check -jN` can lag past a second on a loaded Windows runner.
-port=
-for _ in $(seq 1 300); do
-    # Match anywhere: a startup warning merged via 2>&1 could precede the PORT line.
-    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
-    kill -0 "$serverpid" 2>/dev/null || die "server exited early: $(cat "$serverlog")"
-    sleep 0.1
-done
-test -n "$port" || die "could not discover server port: $(cat "$serverlog")"
+port=$(discover_server_port "$serverlog" "$serverpid") || {
+    # A server that died is a hard failure; only the deadline is the announce race
+    # 72 and 105 skip on, and they key on this exact wording.
+    test "$?" -ne 2 || die "server exited early"
+    die "could not discover server port"
+}
 debug "server listening on ${scheme}://127.0.0.1:${port}"
 
 baseurl="${scheme}://127.0.0.1:${port}"

tests/png-colors.py

@@ -0,0 +1,107 @@
+#!/usr/bin/env python3
+"""Print a PNG's distinct colours as "count #rrggbb", commonest first.
+
+Usage: png-colors.py FILE
+
+Stdlib only: the suite cannot assume Pillow or ImageMagick on a build host.
+"""
+
+import struct
+import sys
+import zlib
+
+CHANNELS = {0: 1, 2: 3, 3: 1, 4: 2, 6: 4}
+
+
+def chunks(blob):
+    off = 8
+    while off < len(blob):
+        (n,) = struct.unpack(">I", blob[off : off + 4])
+        yield blob[off + 4 : off + 8], blob[off + 8 : off + 8 + n]
+        off += 12 + n
+
+
+def unfilter(raw, width, height, bpp, stride):
+    """Undo the per-scanline filters; returns the concatenated raw scanlines."""
+    out, prev, off = bytearray(), bytearray(stride), 0
+    for _ in range(height):
+        ftype, line = raw[off], bytearray(raw[off + 1 : off + 1 + stride])
+        off += 1 + stride
+        for i in range(stride):
+            a = line[i - bpp] if i >= bpp else 0
+            b = prev[i]
+            c = prev[i - bpp] if i >= bpp else 0
+            if ftype == 1:
+                line[i] = (line[i] + a) & 0xFF
+            elif ftype == 2:
+                line[i] = (line[i] + b) & 0xFF
+            elif ftype == 3:
+                line[i] = (line[i] + (a + b) // 2) & 0xFF
+            elif ftype == 4:
+                p = a + b - c
+                pa, pb, pc = abs(p - a), abs(p - b), abs(p - c)
+                line[i] = (
+                    line[i] + (a if pa <= pb and pa <= pc else b if pb <= pc else c)
+                ) & 0xFF
+            elif ftype != 0:
+                raise SystemExit("unknown filter %d" % ftype)
+        out += line
+        prev = line
+    return out
+
+
+def samples(row, depth, count):
+    """The first `count` samples of one unfiltered scanline."""
+    if depth == 8:
+        return list(row[:count])
+    if depth == 16:
+        return [row[i * 2] << 8 | row[i * 2 + 1] for i in range(count)]
+    per, mask = 8 // depth, (1 << depth) - 1
+    return [(row[i // per] >> (8 - depth * (i % per + 1))) & mask for i in range(count)]
+
+
+def colors(path):
+    blob = open(path, "rb").read()
+    if blob[:8] != b"\x89PNG\r\n\x1a\n":
+        raise SystemExit("%s: not a PNG" % path)
+    plte, idat = b"", b""
+    for kind, data in chunks(blob):
+        if kind == b"IHDR":
+            width, height, depth, ctype, _, _, interlace = struct.unpack(
+                ">IIBBBBB", data
+            )
+        elif kind == b"PLTE":
+            plte = data
+        elif kind == b"IDAT":
+            idat += data
+    if interlace:
+        raise SystemExit("%s: interlaced" % path)
+    nchan = CHANNELS[ctype]
+    stride = (width * nchan * depth + 7) // 8
+    bpp = max(1, nchan * depth // 8)
+    raw = unfilter(zlib.decompress(idat), width, height, bpp, stride)
+
+    tally = {}
+    for y in range(height):
+        row = raw[y * stride : (y + 1) * stride]
+        vals = samples(row, depth, width * nchan)
+        for x in range(width):
+            px = tuple(vals[x * nchan : (x + 1) * nchan])
+            if ctype == 3:
+                px = tuple(plte[px[0] * 3 : px[0] * 3 + 3])
+            elif ctype in (0, 4):
+                px = (px[0], px[0], px[0])
+            tally[px[:3]] = tally.get(px[:3], 0) + 1
+    return tally
+
+
+def main():
+    if len(sys.argv) != 2:
+        raise SystemExit(__doc__)
+    tally = colors(sys.argv[1])
+    for px, n in sorted(tally.items(), key=lambda kv: (-kv[1], kv[0])):
+        print("%d #%02x%02x%02x" % ((n,) + px))
+
+
+if __name__ == "__main__":
+    main()

tests/test-timeout.sh

@@ -54,9 +54,8 @@ test -n "$had_m" || test -n "$windows" || set +m
 
 # Poll, because bash cannot wait with a deadline and a watchdog subshell would
 # have to signal across process groups, which MSYS cannot do. The interval is the
-# latency this adds to every healthy test, so keep it small where fork is cheap;
-# under MSYS fork costs tens of milliseconds, and a second is what the Windows
-# suite already paid before this wrapper existed.
+# latency this adds to every healthy test; the ceiling only matters where poll_wait
+# has to fall back to a forked sleep, which under MSYS costs tens of milliseconds.
 if test -n "$windows"; then
     tick=1
 else
@@ -80,6 +79,6 @@ while kill -0 "$pid" 2>/dev/null; do
         dump_crawl_logs
         exit 124
     fi
-    sleep "$tick"
+    poll_wait "$tick"
 done
 wait "$pid"

tests/testlib.sh

@@ -64,40 +64,117 @@ stage_install_exec() {
     stage_install_target install-exec "$1" "$2"
 }
 
+IS_WINDOWS=
 is_windows() {
-    case "$(uname -s)" in
-    MINGW* | MSYS* | CYGWIN*) return 0 ;;
-    *) return 1 ;;
-    esac
+    if test -z "$IS_WINDOWS"; then
+        case "$(uname -s)" in
+        MINGW* | MSYS* | CYGWIN*) IS_WINDOWS=yes ;;
+        *) IS_WINDOWS=no ;;
+        esac
+    fi
+    test "$IS_WINDOWS" = yes
+}
+
+# Open the timer fd poll_wait reads from: a fifo held open read-write, so there is
+# always a writer and a read blocks to its own timeout instead of seeing EOF. fd 9
+# is the harness's from here on; no test may hold it open across a poll.
+POLL_STATE=
+poll_open() {
+    local f
+    POLL_STATE=forked
+    test -z "${HTTRACK_POLL_SLEEP:-}" || return 0
+    # Not under MSYS: its fifos are emulated, and the leg whose stability #795 is
+    # about is no place to discover how its select() behaves. That tick is a whole
+    # second anyway, so there is little to win.
+    is_windows && return 0
+    # Unique: $$ is the same in every subshell, and the loser of a race on one name
+    # opens a path that is gone.
+    f=$(mktemp -u "${TMPDIR:-/tmp}/.httrack-poll.XXXXXX" 2>/dev/null) || return 0
+    mkfifo "$f" 2>/dev/null || return 0
+    # Braced: "exec 9<>x 2>/dev/null" alone would send this shell's stderr to
+    # /dev/null for good, exec applying its redirections to the shell itself.
+    # Still a fifo, or bash's <> created a regular file on a vanished path, which
+    # reads EOF at once and spins every deadline loop.
+    if { exec 9<>"$f"; } 2>/dev/null && test -p "$f"; then
+        POLL_STATE=fifo
+    else
+        exec 9<&-
+    fi
+    rm -f "$f" 2>/dev/null
+    return 0
+}
+
+# One poll tick of at most $1 seconds, forking nothing where bash can time out a
+# read on that fd (a `sleep` per tick was 22% of every process the suite created,
+# and costs milliseconds apiece under MSYS, #795). $1 is a ceiling: the fd tick
+# returns every 0.1s whatever it is asked, and callers re-check their own deadline.
+# HTTRACK_POLL_SLEEP=1 forces the forked tick back, re-read per tick so a test can
+# starve the poll mid-run.
+poll_wait() {
+    local secs=$1 rc=0
+    test -n "$POLL_STATE" || poll_open
+    if test "$POLL_STATE" = fifo && test -z "${HTTRACK_POLL_SLEEP:-}"; then
+        # bash 4.0 is where -t took a fraction; below it, hand a caller asking for
+        # one the sleep it asked for rather than rounding its tick up to a second.
+        if test "${BASH_VERSINFO[0]}" -ge 4; then
+            secs=0.1
+        else
+            case "$secs" in *.*)
+                sleep "$secs"
+                return 0
+                ;;
+            esac
+        fi
+        read -t "$secs" -r -u 9 _ || rc=$?
+        test "$rc" -le 128 || return 0 # >128 is the timeout we asked for
+        # bash 4.0 is also where a timeout started reporting >128; 3.2 says 1, which is
+        # what a closed fd says everywhere. Ask the fd itself rather than the status,
+        # or macOS retires its timer on the first tick and forks for the whole run.
+        # `true`, not `:`: a redirection error on a POSIX special builtin exits the
+        # shell, which POSIXLY_CORRECT in the environment is enough to turn on.
+        if { true >&9; } 2>/dev/null; then return 0; fi
+        POLL_STATE=forked # not a timer any more, stop trusting it
+        exec 9<&-
+    fi
+    sleep "$1"
 }
 
 # On Windows MSYS can't signal a native python.exe, so kill_tree ends the whole
-# tree (a bare kill -9 leaves children). "|| true" throughout: callers run under
-# set -e and the reap makes wait return 143.
+# tree (a bare kill -9 leaves children). Bounded because this runs from an EXIT
+# trap, where a survivor would turn a passing test into a harness timeout.
 stop_server() {
     test -n "${1:-}" || return 0
     kill "$1" 2>/dev/null || true
     if is_windows; then kill_tree "$1"; fi
-    wait "$1" 2>/dev/null || true
+    reap_bounded "$1" || true
     return 0
 }
 
-# Echo the port local-server.py announces on $1 (its log), $2 being its pid.
-# Waits 30s: a cold Python start under a parallel `make check -jN` lags well
-# past a second, and 5s was tight enough that macos-15 missed it on 15 tests.
-# Matches anywhere, since a warning merged via 2>&1 can precede the line.
+# Echo the port local-server.py announces on $1 (its log), $2 being its pid, or
+# return 2 if the server died and 1 on the deadline: only the deadline is a race a
+# caller may skip on. Matches anywhere, since a warning merged via 2>&1 can precede
+# the line. A full minute of wall clock: a cold Python start under a parallel
+# `make check -jN` lags well past a second, 5s had macos-15 missing it on 15 tests,
+# and the count-of-ticks loop this replaced self-extended under load instead.
 discover_server_port() {
-    local log=$1 pid=$2 line _i
-    for _i in $(seq 1 300); do
-        if line=$(grep -m1 '^PORT ' "$log" 2>/dev/null); then
-            printf '%s\n' "${line#PORT }"
-            return 0
+    local log=$1 pid=$2 line start=$SECONDS
+    while :; do
+        if test -r "$log"; then
+            # Read in the shell: a grep per tick is a process per tick (#795).
+            while read -r line; do
+                case "$line" in 'PORT '*)
+                    printf '%s\n' "${line#PORT }"
+                    return 0
+                    ;;
+                esac
+            done <"$log"
         fi
         kill -0 "$pid" 2>/dev/null || {
             echo "server exited early: $(cat "$log" 2>/dev/null)" >&2
-            return 1
+            return 2
         }
-        sleep 0.1
+        test "$((SECONDS - start))" -lt 60 || break
+        poll_wait 0.1
     done
     echo "could not discover server port: $(cat "$log" 2>/dev/null)" >&2
     return 1
@@ -176,6 +253,70 @@ kill_tree() {
 ENGINE_EXE_RE='^(lt-)?(httrack|proxytrack|htsserver|webhttrack)([.]exe)?$'
 FIXTURE_SERVER_RE='^(local-server|proxy-https-server|proxy-connect-server|socks5-server|tls-stall-server)[.]py$'
 
+# awk prologue for the matchers below: under qemu-user the kernel reports the
+# binfmt interpreter as the command, so the name we match on lands one column
+# right (Debian's hppa buildd emulates).
+# shellcheck disable=SC2016 # awk fields, not shell expansions
+AWK_PROC_NAMES='
+function basen(s) { sub(/.*[\/\\]/, "", s); return s }
+function qemushift(  n) {
+    n = basen($6)
+    # qemu-img and friends take an image, not a program, and this list is fed to
+    # kill: shifting past them would read a disk path as the process name.
+    if (n ~ /^qemu-(img|nbd|io|ga|edid|keymap)$/) return 0
+    return n ~ /^qemu-[[:alnum:]_]+(-static)?$|^[[:alnum:]_]+-binfmt(-[[:upper:]]+)?$/ ? 1 : 0
+}'
+
+# Every process as "PID PPID PGID ELAPSED S COMMAND", header first. A Fedora
+# build root has no procps, and an empty list reads as "nothing running" (#1021).
+ps_snapshot() {
+    local snap
+    # POSIX keywords, so the ps route holds on macOS too; a ps that lists nothing
+    # (hidepid, a locked-down container) is no better than an absent one.
+    if snap=$(ps -A -o pid,ppid,pgid,etime,state,args 2>/dev/null |
+        awk 'NR > 1 { rows++ } { print } END { exit rows ? 0 : 1 }'); then
+        printf '%s\n' "$snap"
+        return 0
+    fi
+    proc_snapshot && return 0
+    # Every consumer drops line 1, so the notice rides in the header's place.
+    printf 'no process list: this host has neither ps nor a readable /proc\n'
+    return 1
+}
+
+# The same six columns out of /proc, in bash alone; elapsed as plain seconds.
+proc_snapshot() {
+    local d stat rest arg args comm hz uptime
+    local -a f
+    local ws # spelled out of line: bash 3.2 fails to parse $'..' inside ${v//p/r}
+    ws=$' \t\n\v\f\r'
+    test -r /proc/self/stat || return 1
+    hz=$(getconf CLK_TCK 2>/dev/null) || hz=
+    # A zero or non-numeric tick would abort the shell in the division below.
+    case "$hz" in '' | 0 | *[!0-9]*) hz=100 ;; esac
+    read -r uptime _ </proc/uptime 2>/dev/null || return 1
+    printf 'PID PPID PGID ELAPSED S COMMAND\n'
+    for d in /proc/[0-9]*; do
+        # Braced: a failed open reports to the caller's stderr, not the redirect.
+        { read -r stat <"$d/stat"; } 2>/dev/null || continue
+        # comm may hold spaces and parens; every field past it is numeric.
+        rest=${stat##*') '}
+        comm=${stat#*(}
+        comm=${comm%)*}
+        read -ra f <<<"$rest" || continue
+        test "${#f[@]}" -ge 20 || continue # short read: the process is going away
+        args=
+        # Whitespace inside an argv would split the row or shift the columns the
+        # consumers match on, which ps avoids by mapping those bytes away.
+        { while IFS= read -r -d '' arg; do
+            args="${args:+$args }${arg//["$ws"]/ }"
+        done <"$d/cmdline"; } 2>/dev/null || true
+        printf '%s %s %s %s %s %s\n' "${d#/proc/}" "${f[1]}" "${f[2]}" \
+            "$(((${uptime%%.*} * hz - f[19]) / hz))" \
+            "${f[0]}" "${args:-[$comm]}"
+    done
+}
+
 # List processes a hung test may have left running, one per line. $1 is the test's
 # process group; $2 selects "group" (that group's members, whatever their name),
 # "others" (engine and fixture processes outside it, which under "make check -j"
@@ -190,15 +331,16 @@ list_stray_processes() {
         test "$mode" != others || return 0
         tasklist 2>/dev/null | grep -Ei 'httrack|proxytrack|htsserver|python' || true
     else
-        # `args` and `state` are POSIX ps keywords, so this holds on macOS too.
         # Fields 6 and 7 are the command and its first argument (the interpreter
         # and its script, for the Python fixtures).
-        ps -A -o pid,ppid,pgid,etime,state,args 2>/dev/null |
-            awk -v pg="$pgid" -v mode="$mode" -v eng="$ENGINE_EXE_RE" -v srv="$FIXTURE_SERVER_RE" '
+        ps_snapshot |
+            awk -v pg="$pgid" -v mode="$mode" -v eng="$ENGINE_EXE_RE" -v srv="$FIXTURE_SERVER_RE" \
+                "$AWK_PROC_NAMES"'
                 NR == 1 { print; next }
                 { ingroup = (pg > 0 && $3 == pg)
-                  c = $6; sub(/.*[\/\\]/, "", c)
-                  s = $7; sub(/.*[\/\\]/, "", s)
+                  q = qemushift()
+                  c = basen($(6 + q))
+                  s = basen($(7 + q))
                   named = (c ~ eng || s ~ srv)
                   if (mode == "group" ? ingroup : \
                       mode == "named" ? named : (named && !ingroup)) print }' || true
@@ -239,9 +381,9 @@ reap_leftover_processes() {
 list_engine_pids() {
     local pgid=${1:-0}
     test "$pgid" -gt 0 2>/dev/null || return 0
-    ps -A -o pid,pgid,args 2>/dev/null |
-        awk -v pg="$pgid" -v eng="$ENGINE_EXE_RE" \
-            'NR > 1 && $2 == pg { c = $3; sub(/.*[\/\\]/, "", c); if (c ~ eng) print $1 }'
+    ps_snapshot |
+        awk -v pg="$pgid" -v eng="$ENGINE_EXE_RE" "$AWK_PROC_NAMES"'
+            NR > 1 && $3 == pg { if (basen($(6 + qemushift())) ~ eng) print $1 }'
 }
 
 # Ask the wedged test's engine processes for a stack. What is obtainable differs
@@ -364,6 +506,9 @@ EOF
     chmod +x "$dir/sleep" || return 1
     PATH="$dir:$PATH"
     export PATH
+    # Proved, not assumed: MSYS hands out a drive-letter TMPDIR and a PATH entry
+    # carrying a colon is read as two, so an unreachable shim starves nothing.
+    test "$(command -v sleep)" = "$dir/sleep" || return 1
 }
 
 # Skip when the next of $1 remaining steps, at 1.5x the $2 seconds the last one
@@ -390,7 +535,7 @@ reap_bounded() {
     local pid=$1 start=$SECONDS
     while kill -0 "$pid" 2>/dev/null; do
         test "$((SECONDS - start))" -le "$REAP_GRACE" || return 1
-        sleep 1
+        poll_wait 1
     done
     wait "$pid" 2>/dev/null || true
     return 0
@@ -417,7 +562,7 @@ run_with_timeout() {
             reap_bounded "$pid" || true
             return 124
         fi
-        sleep 1
+        poll_wait 1
     done
     wait "$pid"
 }
@@ -432,7 +577,7 @@ wait_bounded() {
             reap_bounded "$pid" || true
             return 124
         fi
-        sleep 1
+        poll_wait 1
     done
     wait "$pid"
 }

tests/tests-list.mk

@@ -259,3 +259,14 @@ TESTS += 216_engine-ftp-ctrlchars.test
 TESTS += 221_local-ftp-ctrlchars.test
 TESTS += 218_crash-nopie-frames.test
 TESTS += 222_pkgconfig-consumer.test
+TESTS += 236_local-ftp-teardown.test
+TESTS += 227_watchdog-poll.test
+TESTS += 228_icon-small-flat.test
+TESTS += 226_watchdog-native.test
+TESTS += 224_engine-ftp-cmdlen.test
+TESTS += 230_local-ftp-userpass.test
+TESTS += 225_install-manifest.test
+TESTS += 231_tests-list-bijection.test
+TESTS += 233_local-ftp-ctrl-timeout.test
+TESTS += 234_local-ftp-maxtime.test
+TESTS += 232_online-gate-outoftree.test