Merge origin/master after #788 and #789

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -79,7 +79,10 @@ jobs:
       - name: Build
         run: make -j"$(nproc)"
 
+      # A backstop only: tests/test-timeout.sh bounds each test, and a healthy run
+      # is a minute here, two on macOS. Without it a stall ran to the job's 6h default.
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -119,6 +122,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test without python3
+        timeout-minutes: 20
         run: |
           set -euo pipefail
           # Hide every python3* so `command -v python3` fails like it does in the
@@ -173,6 +177,7 @@ jobs:
           sudo ifconfig lo0 alias 127.0.0.3 up
 
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -245,6 +250,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -302,6 +308,7 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -345,6 +352,7 @@ jobs:
       - name: Test (offline self-tests under MSan)
         env:
           MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
+        timeout-minutes: 20
         run: |
           set -euo pipefail
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
@@ -426,6 +434,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"

.github/workflows/windows-build.yml

@@ -189,25 +189,53 @@ jobs:
           # A wedged crawl must not eat the job's timeout budget. timeout(1)'s
           # signals can't reap a native httrack.exe (MSYS signals don't reach it),
           # so a hang orphaned processes that starved the runner; run_with_timeout
-          # TerminateProcess-es the whole tree. 600s clears the slowest multi-pass
-          # crawl (a few passes at --max-time=120 each).
+          # TerminateProcess-es the whole tree. 600s is unchanged: it clears the
+          # 540s a three-pass crawl may legitimately take under local-crawl.sh's
+          # own watchdogs, against a slowest healthy test here of 39s.
           . ./testlib.sh
+          per_test=600
 
-          pass=0 fail=0 skip=0 failed="" skipped=""
+          # The whole suite must give up before the step timeout above. A cancelled
+          # step keeps neither its log nor the artifacts the later if:always()
+          # steps would upload, so an overrun that ends in a cancel tells us
+          # nothing; failing on our own terms keeps both. Healthy runs take 8-9
+          # min. The check sits between tests, so the step can still reach 25 min
+          # plus one per-test budget, and that worst case stays inside the 45.
+          suite_deadline=1500
+          started=$SECONDS
+
+          # Survives into the artifact even if the tail of the step log does not.
+          progress=suite-progress.log
+          : >"$progress"
+
+          pass=0 fail=0 skip=0 failed="" skipped="" deadline=0
           for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
                    *_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
                    60_crawl-log-salvage.test; do
+            elapsed=$((SECONDS - started))
+            if [ "$elapsed" -ge "$suite_deadline" ]; then
+              echo "::error::suite deadline: ${elapsed}s elapsed, stopping before $t"
+              echo "DEADLINE before $t after ${elapsed}s" >>"$progress"
+              # Per-test start times, so the slow ones are named rather than guessed.
+              sed 's/^/      /' "$progress"
+              deadline=1
+              break
+            fi
+            echo "RUN $t at ${elapsed}s" >>"$progress"
             rc=0
-            run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
+            # Same guard "make check" uses on POSIX, so a wedge is diagnosed the
+            # same way on every platform. It dumps before it kills, which a bare
+            # run_with_timeout cannot: by the time that returns, the tree whose
+            # stack we wanted is already gone.
+            HTTRACK_TEST_TIMEOUT=$per_test bash ./test-timeout.sh "$t" >"$t.log" 2>&1 || rc=$?
             case "$rc" in
               0) pass=$((pass + 1)); echo "PASS $t" ;;
               77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
               124)
                 fail=$((fail + 1)) failed="$failed $t"
+                # test-timeout.sh has already written the process list, the stacks
+                # and the killed crawl's own logs into $t.log.
                 echo "FAIL $t (timed out, tree killed)"
-                # Re-running a wedge traced would just hang again: salvage the
-                # killed crawl's own logs into the artifact instead.
-                dump_crawl_logs >>"$t.log"
                 tail -n 25 "$t.log" | sed 's/^/      /'
                 ;;
               *)
@@ -215,10 +243,15 @@ jobs:
                 echo "FAIL $t (exit $rc)"
                 # These assert with `test "$(...)" == "..." || exit 1`, which
                 # says nothing at all on failure. Re-run traced, still bounded.
-                run_with_timeout 600 bash -x "$t" >>"$t.log" 2>&1 || true
+                run_with_timeout "$per_test" bash -x "$t" >>"$t.log" 2>&1 || true
                 tail -n 25 "$t.log" | sed 's/^/      /'
                 ;;
             esac
+            echo "$rc $t" >>"$progress"
+            # An orphaned native httrack.exe spins and starves the runner, which
+            # is how this job dies with "lost communication" rather than a plain
+            # timeout. Clear them between tests and name whoever leaked them.
+            reap_leftover_processes "$t" | tee -a "$progress"
           done
           echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
             tee -a "$GITHUB_STEP_SUMMARY"
@@ -231,6 +264,8 @@ jobs:
           # badmtime needs a filesystem that stores an mtime past gmtime's range;
           # single-file ends on a GUI half needing htsserver, which this job does not build.
           expected_skips=" 01_engine-footer-overflow.test 100_local-purge-longpath.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
+          # First, or the deadline reads as an unexplained shortfall in the gates below.
+          [ "$deadline" -eq 0 ] || { echo "::error::suite did not finish within ${suite_deadline}s"; exit 1; }
           [ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
           [ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
           [ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }

tests/105_suite-timeout.test

@@ -0,0 +1,140 @@
+#!/bin/bash
+#
+# The harness timeout (tests/test-timeout.sh) must name the test it killed, say
+# what was still running, and exit 124 -- otherwise a wedge only ever shows up as
+# a cancelled CI step, which keeps no log and no artifact.
+
+set -eu
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+driver="${testdir}/test-timeout.sh"
+test -r "$driver" || {
+    echo "no test-timeout.sh next to $0" >&2
+    exit 1
+}
+
+fail() {
+    echo "FAIL: $*" >&2
+    test -z "${out:-}" || sed 's/^/  | /' <"$out" >&2
+    exit 1
+}
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_sutmo.XXXXXX") || exit 1
+trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+out="$tmp/out"
+
+# --- a hanging test is killed, named, and reported 124 ----------------------
+cat >"$tmp/90_wedged.test" <<'EOF'
+echo "marker: the wedged test started"
+sleep 300
+EOF
+
+start=$SECONDS
+rc=0
+HTTRACK_TEST_TIMEOUT=5 bash "$driver" "$tmp/90_wedged.test" >"$out" 2>&1 || rc=$?
+elapsed=$((SECONDS - start))
+
+test "$rc" -eq 124 || fail "wedged test reported $rc, want 124"
+# Never before the budget, or a slow-but-healthy test would be killed too.
+test "$elapsed" -ge 5 || fail "the guard fired early (${elapsed}s of a 5s budget)"
+test "$elapsed" -lt 30 || fail "the guard fired late (${elapsed}s)"
+grep -q 'marker: the wedged test started' "$out" || fail "the test's own output was lost"
+# The header, not a bare name: the process list quotes the test's path too, so a
+# wrapper that named nothing would still match that.
+grep -q '^===== TIMEOUT: 90_wedged.test exceeded' "$out" ||
+    fail "the diagnostics do not name the test"
+grep -q "own process tree" "$out" || fail "no process list in the diagnostics"
+
+# The budget is read, not hard-coded: well under it, the same shape survives.
+printf 'sleep 3\necho "slow but healthy"\n' >"$tmp/92_slow.test"
+rc=0
+HTTRACK_TEST_TIMEOUT=45 bash "$driver" "$tmp/92_slow.test" >"$out" 2>&1 || rc=$?
+test "$rc" -eq 0 || fail "a healthy 3s test under a 45s budget reported $rc"
+grep -q 'slow but healthy' "$out" || fail "healthy test output lost"
+! grep -q 'TIMEOUT' "$out" || fail "the guard fired on a healthy test"
+
+# The killed tree must really be gone, or the next test inherits its ports.
+sleep 1
+! grep -q "$tmp/90_wedged.test" <<<"$(ps -A -o args 2>/dev/null)" ||
+    fail "the wedged test survived the guard"
+
+# --- exit status and output of a healthy test pass straight through ----------
+# 77 is automake's SKIP and 99 its hard error, so both have to survive the wrapper.
+for want in 0 1 77 99; do
+    printf 'echo "ran with %s"\nexit %s\n' "$want" "$want" >"$tmp/91_plain.test"
+    rc=0
+    HTTRACK_TEST_TIMEOUT=60 bash "$driver" "$tmp/91_plain.test" >"$out" 2>&1 || rc=$?
+    test "$rc" -eq "$want" || fail "exit $want came back as $rc"
+    grep -q "ran with $want" "$out" || fail "output lost for exit $want"
+    ! grep -q 'TIMEOUT' "$out" || fail "the guard fired on a test that exited at once"
+done
+
+# A budget of 0 disables the guard: the test still runs, and is not killed. Its
+# own file, so the assertion cannot ride on whatever the loop above left behind.
+printf 'echo "guard off"\nexit 3\n' >"$tmp/93_off.test"
+rc=0
+HTTRACK_TEST_TIMEOUT=0 bash "$driver" "$tmp/93_off.test" >"$out" 2>&1 || rc=$?
+test "$rc" -eq 3 || fail "disabled guard changed the exit status ($rc)"
+grep -q 'guard off' "$out" || fail "output lost with the guard disabled"
+
+# --- a wedged crawl yields a symbolized engine stack ------------------------
+# The whole point of the dump: name the frame the engine is stuck in. Windows has
+# neither half (MSYS signals do not reach a native httrack.exe, and that build has
+# no backtrace()), so only assert it where the machinery exists.
+if is_windows; then
+    echo "suite timeout OK (no stack capture on Windows)"
+    exit 0
+fi
+command -v httrack >/dev/null || {
+    echo "SKIP: no httrack on PATH for the stack half" >&2
+    exit 77
+}
+
+# /watchdog/stall never answers, so the crawl wedges for real. CRAWL_DEADLINE is
+# raised past our budget so the guard under test is what fires, not local-crawl's.
+cat >"$tmp/92_stalled.test" <<EOF
+CRAWL_DEADLINE=600 bash "${testdir}/local-crawl.sh" --errors 0 \\
+    httrack 'BASEURL/watchdog/stall'
+EOF
+
+rc=0
+HTTRACK_TEST_TIMEOUT=20 bash "$driver" "$tmp/92_stalled.test" >"$out" 2>&1 || rc=$?
+
+if grep -qE 'python3 not found|could not discover server port' "$out"; then
+    echo "SKIP: local crawl prerequisites missing" >&2
+    exit 77
+fi
+test "$rc" -eq 124 || fail "stalled crawl reported $rc, want 124"
+
+if grep -q 'No stack trace available on this OS' "$out"; then
+    echo "suite timeout OK (no backtrace() on this platform)"
+    exit 0
+fi
+
+# Each platform's mechanism has its own marker, and the marker is what proves the
+# whole chain connected up rather than the dump merely running.
+case "$(uname -s)" in
+Linux)
+    # sig_fatal writes to the engine's own stderr, so this reaching our log also
+    # proves the crawl-log salvage worked.
+    grep -q '^Caught signal ' "$out" || fail "no engine backtrace was salvaged"
+    ;;
+Darwin)
+    grep -q '^Call graph:' "$out" || fail "sample(1) produced no call graph"
+    ;;
+*)
+    grep -q 'no stack mechanism known' "$out" ||
+        fail "unknown platform reported neither a stack nor its absence"
+    echo "suite timeout OK (no stack mechanism on $(uname -s))"
+    exit 0
+    ;;
+esac
+# A stalled crawl is necessarily inside httpmirror, so these frames must appear.
+# Matching "httrack" alone would pass on the process list above.
+grep -qE '\b(back_wait|httpmirror|hts_main_internal)\b' "$out" ||
+    fail "the stack names no engine frame"
+
+echo "suite timeout OK (named the test, dumped a stack, exited 124)"

tests/Makefile.am

@@ -4,7 +4,7 @@
 EXTRA_DIST = $(TESTS) renamefail.c crawl-test.sh run-all-tests.sh check-network.sh \
 	proxy-https-server.py socks5-server.py proxy-connect-server.py \
 	proxytestlib.py tls-stall-server.py warc-validate.py wacz-validate.py \
-	pty-resize.py \
+	pty-resize.py test-timeout.sh \
 	local-crawl.sh local-server.py ftp-server.py testlib.sh \
 	server.crt server.key \
 	server-root/simple/basic.html server-root/simple/link.html \
@@ -37,7 +37,12 @@ TEST_EXTENSIONS = .test
 # work when the source tree sits on a noexec filesystem (the driver would
 # otherwise fail with "Permission denied"), and removes any reliance on the
 # scripts' executable bit. The scripts are #!/bin/bash and use bash features.
-TEST_LOG_COMPILER = $(BASH)
+#
+# test-timeout.sh wraps that in a wall-clock budget: a wedged test must fail
+# named, with diagnostics, rather than hang the job until CI cancels it -- a
+# cancelled step keeps neither its log nor its artifacts, so today a hang tells
+# us nothing at all. HTTRACK_TEST_TIMEOUT overrides the budget; 0 disables it.
+TEST_LOG_COMPILER = $(BASH) $(srcdir)/test-timeout.sh
 TESTS = \
 	00_runnable.test \
 	01_engine-addlink.test \
@@ -218,6 +223,7 @@ TESTS = \
 	100_local-purge-longpath.test \
 	101_local-update-stale-bak.test \
 	102_local-ftp-refetch.test \
-	104_engine-warc-longurl.test
+	104_engine-warc-longurl.test \
+	105_suite-timeout.test
 
 CLEANFILES = check-network_sh.cache

tests/test-timeout.sh

@@ -0,0 +1,76 @@
+#!/bin/bash
+#
+# automake TEST_LOG_COMPILER: run one .test under a wall-clock budget. A wedged
+# test otherwise hangs the whole job until CI cancels it, and a cancelled step
+# keeps neither its log nor its artifacts -- so nobody ever learns which test
+# hung. On expiry this names the test, dumps what is still running, and exits
+# 124, which makes the step FAIL instead: logs survive a failure.
+#
+# Output goes to the test's own tests/NN_*.log (automake redirects us there) and
+# from there into test-suite.log, which CI prints.
+
+set -u
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+# 600s is what the Windows leg already bounded each test at, and it clears the
+# 540s that local-crawl.sh's own watchdogs tolerate for a three-pass crawl
+# (CRAWL_DEADLINE, 180s a pass) -- budget below that and a slow-but-legitimate
+# run would be killed. The slowest healthy test measures 39s. A non-numeric or
+# absurd value falls back; 0 disables the guard, for use under a debugger.
+budget=${HTTRACK_TEST_TIMEOUT:-600}
+case "$budget" in
+'' | *[!0-9]*) budget=600 ;;
+esac
+test "$budget" -gt 0 || exec "$BASH" "$@"
+
+# The test script is the last argument; automake passes no others today.
+for name in "$@"; do :; done
+name=$(basename "$name")
+
+# Give the test its own TMPDIR, so the hang dump can salvage exactly this test's
+# crawl logs instead of racing (and deleting) a sibling's under "make check -j".
+tmproot=${TMPDIR:-/tmp}
+if mkdir -p "$tmproot/ht.$$" 2>/dev/null; then
+    TMPDIR="$tmproot/ht.$$"
+    export TMPDIR
+    trap 'rm -rf "$TMPDIR" 2>/dev/null || true' EXIT
+fi
+
+windows=
+is_windows && windows=1
+
+had_m=
+case "$-" in *m*) had_m=1 ;; esac
+test -n "$windows" || set -m # own process group, so kill_tree can signal the group
+"$BASH" "$@" &
+pid=$!
+test -n "$had_m" || test -n "$windows" || set +m
+
+# Poll, because bash cannot wait with a deadline and a watchdog subshell would
+# have to signal across process groups, which MSYS cannot do. The interval is the
+# latency this adds to every healthy test, so keep it small where fork is cheap;
+# under MSYS fork costs tens of milliseconds, and a second is what the Windows
+# suite already paid before this wrapper existed.
+if test -n "$windows"; then
+    tick=1 per_sec=1
+else
+    tick=0.1 per_sec=10
+fi
+
+ticks=0
+while kill -0 "$pid" 2>/dev/null; do
+    if test "$((ticks / per_sec))" -ge "$budget"; then
+        dump_hang_diagnostics "$pid" "$name" "$budget"
+        kill_tree "$pid"
+        reap_bounded "$pid" || echo "hang: the tree outlived the kill; see the process list above"
+        # After the kill, so a crawl log holds the backtrace its engine just wrote.
+        dump_crawl_logs
+        exit 124
+    fi
+    sleep "$tick"
+    ticks=$((ticks + 1))
+done
+wait "$pid"

tests/testlib.sh

@@ -83,6 +83,200 @@ kill_tree() {
     kill -9 -"$pid" 2>/dev/null || kill -9 "$pid" 2>/dev/null || true
 }
 
+# Engine and fixture-server processes, matched on the executable basename only.
+# Matching the whole ps line instead would catch every unrelated command whose
+# arguments merely mention a path containing "httrack".
+ENGINE_EXE_RE='^(lt-)?(httrack|proxytrack|htsserver|webhttrack)([.]exe)?$'
+FIXTURE_SERVER_RE='^(local-server|proxy-https-server|proxy-connect-server|socks5-server|tls-stall-server)[.]py$'
+
+# List processes a hung test may have left running, one per line. $1 is the test's
+# process group; $2 selects "group" (that group's members, whatever their name),
+# "others" (engine and fixture processes outside it, which under "make check -j"
+# belong to healthy siblings) or "named" (every engine and fixture process on the
+# host). Read-only: it never signals anything.
+list_stray_processes() {
+    local pgid=${1:-0} mode=${2:-group}
+    if is_windows; then
+        # No process groups here, so every mode gives the same host-wide list. No
+        # slash switches: without MSYS_NO_PATHCONV a /fi would be rewritten to a
+        # path. Plain output is Image Name + PID, which is all we need.
+        test "$mode" != others || return 0
+        tasklist 2>/dev/null | grep -Ei 'httrack|proxytrack|htsserver|python' || true
+    else
+        # `args` and `state` are POSIX ps keywords, so this holds on macOS too.
+        # Fields 6 and 7 are the command and its first argument (the interpreter
+        # and its script, for the Python fixtures).
+        ps -A -o pid,ppid,pgid,etime,state,args 2>/dev/null |
+            awk -v pg="$pgid" -v mode="$mode" -v eng="$ENGINE_EXE_RE" -v srv="$FIXTURE_SERVER_RE" '
+                NR == 1 { print; next }
+                { ingroup = (pg > 0 && $3 == pg)
+                  c = $6; sub(/.*[\/\\]/, "", c)
+                  s = $7; sub(/.*[\/\\]/, "", s)
+                  named = (c ~ eng || s ~ srv)
+                  if (mode == "group" ? ingroup : \
+                      mode == "named" ? named : (named && !ingroup)) print }' || true
+    fi
+}
+
+# Kill engine processes a finished test left behind, and print what was found so
+# the leak is attributed to the test that just ran: an orphaned httrack.exe spins
+# and starves the runner, which is how the Windows job dies of "lost
+# communication" rather than a clean timeout. SERIAL RUNNERS ONLY -- it matches by
+# name host-wide, so under a parallel "make check" it would kill a healthy
+# sibling's engine. Only the engine images: a runner may run python.exe of its
+# own, and tasklist alone cannot tell that one from a leaked fixture server.
+reap_leftover_processes() {
+    local label=${1:-} left
+    if is_windows; then
+        left=$(tasklist 2>/dev/null | grep -Ei 'httrack|proxytrack|htsserver' || true)
+    else
+        left=$(list_stray_processes 0 named | awk 'NR > 1')
+    fi
+    test -n "$left" || return 0
+    printf '::warning::%s left processes behind\n' "$label"
+    printf '%s\n' "$left"
+    if is_windows; then
+        taskkill /F /IM httrack.exe >/dev/null 2>&1 || true
+        taskkill /F /IM proxytrack.exe >/dev/null 2>&1 || true
+    else
+        printf '%s\n' "$left" | awk '{ print $1 }' |
+            while read -r p; do kill -9 "$p" 2>/dev/null || true; done
+    fi
+    return 0
+}
+
+# Pids of engine processes in process group $1. Scoped to the group because the
+# caller signals them, and under "make check -j" a global match would abort a
+# healthy sibling test's engine. Matches the executable basename only, so a
+# harness script whose *path* contains "httrack" is not mistaken for the engine.
+list_engine_pids() {
+    local pgid=${1:-0}
+    test "$pgid" -gt 0 2>/dev/null || return 0
+    ps -A -o pid,pgid,args 2>/dev/null |
+        awk -v pg="$pgid" -v eng="$ENGINE_EXE_RE" \
+            'NR > 1 && $2 == pg { c = $3; sub(/.*[\/\\]/, "", c); if (c ~ eng) print $1 }'
+}
+
+# Ask the wedged test's engine processes for a stack. What is obtainable differs
+# per platform, and each branch says which one it took: a dump that silently
+# produces nothing reads as coverage when it is not.
+#
+#   Linux  httrack's own SIGABRT handler (sig_fatal in httrack.c) symbolizes via
+#          addr2line and writes to the process's OWN stderr, so the trace lands in
+#          whatever log the test gave it -- the caller must salvage those logs.
+#          It walks only the signalled thread, and it aborts the process.
+#   macOS  htsbacktrace.c gates that handler on __linux, so SIGABRT would yield
+#          "No stack trace available on this OS". sample(1) is OS-provided, needs
+#          no root, covers every thread and leaves the process running.
+#
+# gdb -p is not an option on either: it is EPERM from a sibling under yama
+# ptrace_scope=1, which is how a harness watchdog necessarily invokes it.
+request_engine_backtraces() {
+    local p os
+    local sent='' abrt=''
+    os=$(uname -s 2>/dev/null || echo unknown)
+    for p in $(list_engine_pids "$1"); do
+        sent=1
+        test ! -r "/proc/$p/wchan" ||
+            printf 'pid %s blocked in: %s\n' "$p" "$(cat "/proc/$p/wchan")"
+        case "$os" in
+        Linux)
+            kill -ABRT "$p" 2>/dev/null && abrt=1
+            ;;
+        Darwin)
+            if test -x /usr/bin/sample; then
+                # Drop the trailing image map: ~40 lines of load addresses that
+                # say nothing about the hang.
+                /usr/bin/sample "$p" 2 -mayDie -file /dev/stdout 2>&1 |
+                    sed '/^Binary Images:/,$d' ||
+                    printf 'pid %s: sample(1) failed\n' "$p"
+            else
+                printf 'pid %s: no stack, /usr/bin/sample is absent\n' "$p"
+            fi
+            ;;
+        *)
+            printf 'pid %s: no stack mechanism known for %s\n' "$p" "$os"
+            ;;
+        esac
+    done
+    test -n "$sent" || printf 'no engine process left to ask (see the list above)\n'
+    test -z "$abrt" || sleep 3 # let the handlers symbolize and print
+}
+
+# Stack of every native engine process, through cdb. Windows has neither half of
+# the POSIX route: htsbacktrace.c is gated on __linux, and MSYS signals never
+# reach a native httrack.exe. cdb ships with the SDK on the runner image but that
+# is incidental, so probe for it and say so when it is missing. The MSVC build
+# writes PDBs beside the binaries, which the test step already puts on PATH, so
+# frames resolve to names. Serial runners only: it stacks every engine process it
+# finds, having no process group to scope by.
+dump_windows_stacks() {
+    local c p
+    local cdb='' found=''
+    for c in "$(command -v cdb 2>/dev/null)" \
+        "/c/Program Files (x86)/Windows Kits/10/Debuggers/x64/cdb.exe" \
+        "/c/Program Files (x86)/Windows Kits/10/Debuggers/x86/cdb.exe"; do
+        test -n "$c" || continue
+        test -x "$c" || continue
+        cdb=$c
+        break
+    done
+    if test -z "$cdb"; then
+        printf 'no stack: cdb.exe is not in the SDK Debuggers directories or on PATH\n'
+        return 0
+    fi
+    for p in $(tasklist 2>/dev/null | grep -Ei '^(httrack|proxytrack)[.]exe' | awk '{print $2}'); do
+        found=1
+        printf -- '--- cdb stack of pid %s ---\n' "$p"
+        # Bounded, so a debugger that wedges cannot become the new hang. "qd"
+        # detaches and leaves the process for kill_tree.
+        run_with_timeout 60 "$cdb" -p "$p" -c '~*kv; qd' 2>&1 ||
+            printf 'cdb failed or timed out on pid %s\n' "$p"
+    done
+    test -n "$found" || printf 'no engine process left to ask (see the list above)\n'
+}
+
+# Report what a wedged test left behind, into the log the harness keeps: the test
+# that blew its budget, the processes still running, and a stack for each engine
+# process. $1 is the timed-out job's pid (its process group leader on POSIX).
+# Never deletes, so it is safe under a parallel "make check".
+dump_hang_diagnostics() {
+    local pid=$1 label=${2:-?} secs=${3:-?}
+    printf '\n===== TIMEOUT: %s exceeded its %ss budget =====\n' "$label" "$secs"
+    if is_windows; then
+        printf -- '--- still running ---\n'
+        list_stray_processes 0 group
+        printf -- '--- stacks (via cdb) ---\n'
+        dump_windows_stacks
+    else
+        printf -- '--- the test'\''s own process tree (group %s) ---\n' "$pid"
+        list_stray_processes "$pid" group
+        # Under "make check -j" these belong to healthy siblings, so they are
+        # reported but never signalled; a leaked orphan also lands here.
+        printf -- '--- other engine processes on this host ---\n'
+        list_stray_processes "$pid" others
+        printf -- '--- stacks (via the engine SIGABRT handler) ---\n'
+        request_engine_backtraces "$pid"
+    fi
+    printf -- '===== end of diagnostics: %s =====\n' "$label"
+}
+
+# Collect a killed job, giving up after REAP_GRACE seconds. kill_tree can fail to
+# reap a native Windows descendant -- the very case these watchdogs exist for --
+# and a bare `wait` then blocks the watchdog itself forever, so the timeout it was
+# about to report is never printed and the whole suite wedges silently.
+REAP_GRACE=${REAP_GRACE:-10}
+reap_bounded() {
+    local pid=$1 waited=0
+    while kill -0 "$pid" 2>/dev/null; do
+        test "$waited" -lt "$REAP_GRACE" || return 1
+        sleep 1
+        waited=$((waited + 1))
+    done
+    wait "$pid" 2>/dev/null || true
+    return 0
+}
+
 # Run "$@" under a wall-clock deadline of $1 seconds; return its exit status, or
 # 124 if it overran and was killed. timeout(1) is unusable here: it's absent on
 # macOS and its signals can't reap httrack.exe on Windows. We poll and kill_tree.
@@ -99,7 +293,7 @@ run_with_timeout() {
     while kill -0 "$pid" 2>/dev/null; do
         if test "$waited" -ge "$secs"; then
             kill_tree "$pid"
-            wait "$pid" 2>/dev/null || true
+            reap_bounded "$pid" || true
             return 124
         fi
         sleep 1
@@ -115,7 +309,7 @@ wait_bounded() {
     while kill -0 "$pid" 2>/dev/null; do
         if test "$waited" -ge "$secs"; then
             kill_tree "$pid"
-            wait "$pid" 2>/dev/null || true
+            reap_bounded "$pid" || true
             return 124
         fi
         sleep 1