tests: share one fetch-pass counter across the update-refetch routes

The #562 /uptrunc/ routes copied upcodec's per-path body-fetch counter verbatim. Fold both onto one refetch_pass() helper and one dict.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/windows-build.yml

@@ -123,9 +123,12 @@ jobs:
       # is Linux/macOS only. These are the offline ones, driven from Git Bash
       # against the native httrack.exe. They subsume the self-tests this step
       # used to run inline (codecs, cache, fsize).
+      # The *_local-* ones crawl the bundled Python server over loopback: the real
+      # TLS handshake, cache and file writer, which nothing else on Windows covers.
       - name: Run the engine test suite (offline tests)
         shell: bash
         working-directory: tests
+        timeout-minutes: 45
         run: |
           set -u
           bin="$(cygpath -u "$GITHUB_WORKSPACE")/src/${{ matrix.platform }}/${{ matrix.configuration }}"
@@ -141,13 +144,24 @@ jobs:
           TMPDIR="$(cygpath -m "$RUNNER_TEMP")"
           export TMPDIR
 
-          pass=0 fail=0 skip=0 failed=""
-          for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test; do
+          # Mirror what configure hands the suite. LC_ALL sets the codeset MSYS maps
+          # a UTF-8 mirror name onto UTF-16 with, which the intl crawls "test -f".
+          export HTTPS_SUPPORT=yes BROTLI_ENABLED=yes ZSTD_ENABLED=yes
+          export LC_ALL=C.UTF-8
+
+          # A wedged crawl must not eat the job's whole timeout budget.
+          watchdog=""
+          command -v timeout >/dev/null && watchdog="timeout 600"
+
+          pass=0 fail=0 skip=0 failed="" skipped=""
+          for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
+                   *_local-*.test 13_crawl_proxy_https.test; do
             rc=0
-            bash "$t" >"$t.log" 2>&1 || rc=$?
+            # shellcheck disable=SC2086
+            $watchdog bash "$t" >"$t.log" 2>&1 || rc=$?
             case "$rc" in
               0) pass=$((pass + 1)); echo "PASS $t" ;;
-              77) skip=$((skip + 1)); echo "SKIP $t" ;;
+              77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
               *)
                 fail=$((fail + 1)) failed="$failed $t"
                 echo "FAIL $t (exit $rc)"
@@ -161,10 +175,11 @@ jobs:
           echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
             tee -a "$GITHUB_STEP_SUMMARY"
 
-          # Every gate in these scripts exits 77, so a suite that degraded to
-          # all-skipped would report green having tested nothing: assert a floor
-          # on what actually ran, not just the absence of failures.
-          [ "$pass" -ge 45 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
+          # Every gate here exits 77, so an all-skipped suite would report green having
+          # tested nothing: pin the skips, and floor the passes in case the glob empties.
+          expected_skips=" 48_local-crange-memresume.test"  # pending #581
+          [ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
+          [ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
           [ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
 
       - name: Upload the test logs

src/htsback.c

@@ -628,6 +628,10 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                       back[p].r.size, back[p].url_adr, back[p].url_fil);
       }
       back_finalize_backup(opt, &back[p], HTS_FALSE);
+      /* Keep the surviving copy in new.lst, else the update purge drops the
+         file we refused to overwrite with the partial body (#562). */
+      if (fexist_utf8(back[p].url_sav))
+        filenote(&opt->state.strc, back[p].url_sav, NULL);
       return -1;
     }
 

tests/13_crawl_proxy_https.test

@@ -8,22 +8,26 @@ set -euo pipefail
 
 : "${top_srcdir:=..}"
 
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
 if test "${HTTPS_SUPPORT:-}" == "no"; then
     echo "no https support compiled, skipping"
     exit 77
 fi
-if ! command -v python3 >/dev/null 2>&1 || ! command -v openssl >/dev/null 2>&1; then
+python=$(find_python) || python=
+if test -z "$python" || ! command -v openssl >/dev/null 2>&1; then
     echo "python3/openssl missing, skipping"
     exit 77
 fi
 
-server="$top_srcdir/tests/proxy-https-server.py"
+server=$(nativepath "$top_srcdir/tests/proxy-https-server.py")
 tmpdir=$(mktemp -d)
 pids=
 
 cleanup() {
     for pid in $pids; do
-        kill "$pid" 2>/dev/null || true
+        stop_server "$pid"
     done
     rm -rf "$tmpdir"
 }
@@ -38,10 +42,13 @@ cat "$tmpdir/key.pem" "$tmpdir/cert.pem" >"$tmpdir/both.pem"
 # start_server <logdir> <mode>: launches a proxy+origin pair, sets $origin_port
 # and $proxy_port from its announced ephemeral ports.
 start_server() {
-    local dir="$1" mode="$2" ports
+    local dir="$1" mode="$2" ports pem
     mkdir -p "$dir"
     ports="$dir/ports.txt"
-    python3 "$server" "$tmpdir/both.pem" "$dir" "$mode" \
+    pem=$(nativepath "$tmpdir/both.pem")
+    dir_native=$(nativepath "$dir")
+    : >"$ports"
+    "$python" "$server" "$pem" "$dir_native" "$mode" \
         >"$ports" 2>"$dir/server.err" &
     pids="$pids $!"
     for _ in $(seq 1 100); do

tests/19_local-connect-fallback.test

@@ -12,14 +12,17 @@ set -euo pipefail
 
 : "${top_srcdir:=..}"
 
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
 if test "${V6_SUPPORT:-}" == "no"; then
     echo "no IPv6 support (resolver list/override is IPv6-only), skipping"
     exit 77
 fi
-if ! command -v python3 >/dev/null 2>&1; then
+python=$(find_python) || {
     echo "python3 missing, skipping"
     exit 77
-fi
+}
 # The fixture needs a second loopback IP (dead 127.0.0.2 + live 127.0.0.1) for
 # the fallback to have a target; GNU/Hurd has only 127.0.0.1, so skip there.
 case "$(uname -s)" in
@@ -29,23 +32,20 @@ GNU | GNU/*)
     ;;
 esac
 
-server="$top_srcdir/tests/local-server.py"
-root="$top_srcdir/tests/server-root"
+server=$(nativepath "$top_srcdir/tests/local-server.py")
+root=$(nativepath "$top_srcdir/tests/server-root")
 tmpdir=$(mktemp -d)
 serverpid=
 
 cleanup() {
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null || true
-        wait "$serverpid" 2>/dev/null || true
-    fi
+    stop_server "$serverpid"
     rm -rf "$tmpdir"
     return 0
 }
 trap cleanup EXIT
 
 # bind the live server to 127.0.0.1 only, so 127.0.0.2 refuses the connect
-python3 "$server" --root "$root" --bind 127.0.0.1 >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &
+"$python" "$server" --root "$root" --bind 127.0.0.1 >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &
 serverpid=$!
 port=
 for _ in $(seq 1 50); do

tests/20_local-resume-loop.test

@@ -5,19 +5,19 @@ set -u
 
 : "${top_srcdir:=..}"
 testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+server=$(nativepath "${testdir}/local-server.py")
+root=$(nativepath "${testdir}/server-root")
 
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
 
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_206.XXXXXX") || exit 1
 serverpid=
 crawlpid=
 cleanup() {
     test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
+    stop_server "$serverpid"
     rm -rf "$tmpdir"
 }
 trap cleanup EXIT HUP INT QUIT PIPE TERM
@@ -25,8 +25,9 @@ trap cleanup EXIT HUP INT QUIT PIPE TERM
 # --- start the server, discover its ephemeral port --------------------------
 # RESUME_COUNTER gets a byte per /resume/blob.txt request (pass-2 delta bounds re-gets).
 serverlog="${tmpdir}/server.log"
+: >"$serverlog"
 counter="${tmpdir}/blobcount"
-RESUME_COUNTER="$counter" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
+RESUME_COUNTER="$counter" "$python" "$server" --root "$root" >"$serverlog" 2>&1 &
 serverpid=$!
 port=
 for _ in $(seq 1 50); do

tests/24_local-resume-overlap.test

@@ -9,29 +9,30 @@ set -eu
 
 : "${top_srcdir:=..}"
 testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+server=$(nativepath "${testdir}/local-server.py")
+root=$(nativepath "${testdir}/server-root")
 
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
 
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_198.XXXXXX") || exit 1
 serverpid=
 crawlpid=
 cleanup() {
     if test -n "$crawlpid"; then kill -9 "$crawlpid" 2>/dev/null || true; fi
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null || true
-        wait "$serverpid" 2>/dev/null || true
-    fi
+    stop_server "$serverpid"
     rm -rf "$tmpdir"
 }
 trap cleanup EXIT HUP INT QUIT PIPE TERM
 
 # OVERLAP_COUNTER gets a byte per flaky.bin request so pass 1 knows when to interrupt.
 serverlog="${tmpdir}/server.log"
+: >"$serverlog"
 counter="${tmpdir}/hits"
 resumed="${tmpdir}/resumed" # gets a byte when the server serves a resume 206
 OVERLAP_COUNTER="$counter" OVERLAP_RESUMED="$resumed" \
-    python3 "$server" --root "${testdir}/server-root" \
+    "$python" "$server" --root "$root" \
     >"$serverlog" 2>&1 &
 serverpid=$!
 port=

tests/28_local-pause.test

@@ -9,9 +9,12 @@ set -e
 
 : "${top_srcdir:=..}"
 
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
 # python3 runs the local server (mirror local-crawl.sh); skip when absent, else
 # run() swallows its exit-77 and the serverless 0s/0s crawl looks like a fail.
-command -v python3 >/dev/null || {
+find_python >/dev/null || {
     echo "python3 not found; skipping local crawl tests"
     exit 77
 }

tests/46_local-update-304-resume.test

@@ -7,27 +7,28 @@ set -u
 
 : "${top_srcdir:=..}"
 testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+server=$(nativepath "${testdir}/local-server.py")
+root=$(nativepath "${testdir}/server-root")
 
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
 
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_c7.XXXXXX") || exit 1
 serverpid=
 crawlpid=
 cleanup() {
     test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
+    stop_server "$serverpid"
     rm -rf "$tmpdir"
 }
 trap cleanup EXIT HUP INT QUIT PIPE TERM
 
 serverlog="${tmpdir}/server.log"
+: >"$serverlog"
 counter="${tmpdir}/reqcount"
 mark="${tmpdir}/got304"
-RESUME304_COUNTER="$counter" RESUME304_MARK="$mark" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
+RESUME304_COUNTER="$counter" RESUME304_MARK="$mark" "$python" "$server" --root "$root" >"$serverlog" 2>&1 &
 serverpid=$!
 port=
 for _ in $(seq 1 50); do

tests/47_local-crange-overflow.test

@@ -8,25 +8,26 @@ set -u
 
 : "${top_srcdir:=..}"
 testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+server=$(nativepath "${testdir}/local-server.py")
+root=$(nativepath "${testdir}/server-root")
 
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
 
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crange.XXXXXX") || exit 1
 serverpid=
 crawlpid=
 cleanup() {
     test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
+    stop_server "$serverpid"
     rm -rf "$tmpdir"
 }
 trap cleanup EXIT HUP INT QUIT PIPE TERM
 
 serverlog="${tmpdir}/server.log"
-python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
+: >"$serverlog"
+"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
 serverpid=$!
 port=
 for _ in $(seq 1 50); do

tests/48_local-crange-memresume.test

@@ -8,25 +8,34 @@ set -u
 
 : "${top_srcdir:=..}"
 testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+server=$(nativepath "${testdir}/local-server.py")
+root=$(nativepath "${testdir}/server-root")
 
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+
+# The resume needs a graceful pass-1 interrupt (a clean cache), which MSYS can't
+# deliver to a native exe, and the restart-whole path fails on a repaired cache
+# (#581); TODO: drop once that lands.
+if is_windows; then
+    echo "Windows: interrupted-resume restart fails on a repaired cache (#581), skipping"
+    exit 77
+fi
 
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangemem.XXXXXX") || exit 1
 serverpid=
 crawlpid=
 cleanup() {
     test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
+    stop_server "$serverpid"
     rm -rf "$tmpdir"
 }
 trap cleanup EXIT HUP INT QUIT PIPE TERM
 
 serverlog="${tmpdir}/server.log"
-python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
+: >"$serverlog"
+"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
 serverpid=$!
 port=
 for _ in $(seq 1 50); do
@@ -77,7 +86,14 @@ echo "OK (temp-ref present)"
 
 # --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
 printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
-httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1
+rc=0
+httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1 || rc=$?
+# a crash here would otherwise read as a clean "terminated"
+test "$rc" -eq 0 || {
+    echo "FAIL: httrack exited $rc"
+    cat "${tmpdir}/log2" >&2
+    exit 1
+}
 echo "OK (terminated)"
 
 blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)

tests/52_local-socks5.test

@@ -8,11 +8,15 @@ set -euo pipefail
 
 : "${top_srcdir:=..}"
 
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
 if test "${HTTPS_SUPPORT:-}" == "no"; then
     echo "no https support compiled, skipping"
     exit 77
 fi
-if ! command -v python3 >/dev/null 2>&1 || ! command -v openssl >/dev/null 2>&1; then
+python=$(find_python) || python=
+if test -z "$python" || ! command -v openssl >/dev/null 2>&1; then
     echo "python3/openssl missing, skipping"
     exit 77
 fi
@@ -20,13 +24,13 @@ fi
 # a .invalid name never resolves (RFC 6761): reaching the page at all proves the
 # proxy did the DNS
 host="socks-origin.invalid"
-server="$top_srcdir/tests/socks5-server.py"
+server=$(nativepath "$top_srcdir/tests/socks5-server.py")
 tmpdir=$(mktemp -d)
 pids=
 
 cleanup() {
     for pid in $pids; do
-        kill "$pid" 2>/dev/null || true
+        stop_server "$pid"
     done
     rm -rf "$tmpdir"
 }
@@ -39,10 +43,13 @@ cat "$tmpdir/key.pem" "$tmpdir/cert.pem" >"$tmpdir/both.pem"
 
 # start_server <logdir> <mode>: sets $tls_port/$http_port/$socks_port
 start_server() {
-    local dir="$1" mode="$2" ports
+    local dir="$1" mode="$2" ports pem
     mkdir -p "$dir"
     ports="$dir/ports.txt"
-    python3 "$server" "$tmpdir/both.pem" "$dir" "$mode" \
+    pem=$(nativepath "$tmpdir/both.pem")
+    dir_native=$(nativepath "$dir")
+    : >"$ports"
+    "$python" "$server" "$pem" "$dir_native" "$mode" \
         >"$ports" 2>"$dir/server.err" &
     pids="$pids $!"
     for _ in $(seq 1 100); do

tests/54_local-update-truncate-purge.test

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# An --update re-fetch that comes in short must not delete the mirrored file (#562).
+# Pass 1 mirrors page.html (in-memory) and file.bin (direct-to-disk); pass 2 serves
+# both with the full declared Content-Length but half the body, then closes. httrack
+# refuses the partial ("will be retried") — and unfixed, the end-of-update purge then
+# deletes the good pass-1 copy, which was never overwritten. stay.html is the control:
+# fully served both passes, it must update to V2.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update' \
+    --log-found 'incomplete transfer.*uptrunc/page\.html' \
+    --log-found 'incomplete transfer.*uptrunc/file\.bin' \
+    --file-matches 'uptrunc/page.html' 'MIRRORED-PAGE-V1' \
+    --file-matches 'uptrunc/file.bin' 'MIRRORED-BIN-V1' \
+    --file-min-bytes 'uptrunc/file.bin' 32768 \
+    --file-matches 'uptrunc/stay.html' 'STAY-V2' \
+    --file-not-matches 'uptrunc/stay.html' 'STAY-V1' \
+    httrack 'BASEURL/uptrunc/index.html'

tests/Makefile.am

@@ -3,7 +3,7 @@
 # silently drop it from the dist tarball and break "make distcheck".
 EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
 	proxy-https-server.py socks5-server.py \
-	local-crawl.sh local-server.py server.crt server.key \
+	local-crawl.sh local-server.py testlib.sh server.crt server.key \
 	server-root/simple/basic.html server-root/simple/link.html \
 	server-root/stripquery/index.html server-root/stripquery/a.html \
 	server-root/fraglink/index.html server-root/fraglink/target.html \
@@ -130,6 +130,7 @@ TESTS = \
 	50_local-contentcodings.test \
 	51_local-update-codec.test \
 	52_local-socks5.test \
-	53_local-proxytrack-cache-corrupt.test
+	53_local-proxytrack-cache-corrupt.test \
+	54_local-update-truncate-purge.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -38,6 +38,8 @@
 set -u
 
 testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
 server="${testdir}/local-server.py"
 root="${LOCAL_SERVER_ROOT:-${testdir}/server-root}"
 cert="${testdir}/server.crt"
@@ -72,12 +74,8 @@ function cleanup {
         kill -9 "$crawlpid" 2>/dev/null
         crawlpid=
     fi
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        # Reap it so the port is released before we rm the tmpdir/log.
-        wait "$serverpid" 2>/dev/null
-        serverpid=
-    fi
+    stop_server "$serverpid"
+    serverpid=
     if test -n "$tmpdir" && test -d "$tmpdir"; then
         test -n "$nopurge" || rm -rf "$tmpdir"
     fi
@@ -96,7 +94,7 @@ nopurge=
 trap cleanup EXIT HUP INT QUIT PIPE TERM
 
 # python3 is required; mirror check-network.sh's skip-with-77 convention.
-command -v python3 >/dev/null || ! echo "python3 not found; skipping local crawl tests" || exit 77
+python=$(find_python) || ! echo "python3 not found; skipping local crawl tests" >&2 || exit 77
 
 tmptopdir=${TMPDIR:-/tmp}
 test -d "$tmptopdir" || mkdir -p "$tmptopdir" || die "no temporary directory; set TMPDIR"
@@ -158,12 +156,12 @@ done
 # --- start the server --------------------------------------------------------
 test -r "$server" || die "cannot read $server"
 serverlog="${tmpdir}/server.log"
-serverargs=(--root "$root")
+serverargs=(--root "$(nativepath "$root")")
 if test -n "$tls"; then
-    serverargs+=(--tls --cert "$cert" --key "$key")
+    serverargs+=(--tls --cert "$(nativepath "$cert")" --key "$(nativepath "$key")")
 fi
-debug "starting python3 $server ${serverargs[*]}"
-python3 "$server" "${serverargs[@]}" >"$serverlog" 2>&1 &
+debug "starting $python $server ${serverargs[*]}"
+"$python" "$(nativepath "$server")" "${serverargs[@]}" >"$serverlog" 2>&1 &
 serverpid=$!
 
 # Wait for the "PORT <n>" line (server prints it once bound).
@@ -203,7 +201,7 @@ if test "${#cookies[@]}" -gt 0; then
 fi
 
 # --- run httrack -------------------------------------------------------------
-which httrack >/dev/null || die "could not find httrack"
+command -v httrack >/dev/null || die "could not find httrack"
 ver=$(httrack -O /dev/null --version | sed -e 's/HTTrack version //')
 test -n "$ver" || die "could not run httrack"
 
@@ -278,8 +276,7 @@ if test -n "$rerun_dead"; then
     test -s "$zip" || die "no cache was written by the first pass"
     cp "$zip" "${tmpdir}/cache-before.zip"
     cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
-    kill "$serverpid" 2>/dev/null
-    wait "$serverpid" 2>/dev/null
+    stop_server "$serverpid"
     serverpid=
     info "re-running httrack against the stopped server"
     httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
@@ -441,12 +438,18 @@ while test "$i" -lt "${#audit[@]}"; do
     --file-mode)
         path="${audit[$((i + 1))]}"
         i=$((i + 2))
-        mode=$(stat -c '%a' "${hostroot}/${path}" 2>/dev/null ||
-            stat -f '%Lp' "${hostroot}/${path}" 2>/dev/null)
-        info "checking ${path} mode ${mode:-none} is ${audit[$i]}"
-        if test "$mode" = "${audit[$i]}"; then result "OK"; else
-            result "wrong mode"
-            exit 1
+        if is_windows; then
+            # No POSIX modes, and the engine only chmods #ifndef _WIN32.
+            info "checking ${path} mode"
+            result "SKIP (no POSIX modes)"
+        else
+            mode=$(stat -c '%a' "${hostroot}/${path}" 2>/dev/null ||
+                stat -f '%Lp' "${hostroot}/${path}" 2>/dev/null)
+            info "checking ${path} mode ${mode:-none} is ${audit[$i]}"
+            if test "$mode" = "${audit[$i]}"; then result "OK"; else
+                result "wrong mode"
+                exit 1
+            fi
         fi
         ;;
     esac

tests/local-server.py

@@ -18,6 +18,7 @@ import base64
 import gzip
 import hashlib
 import os
+import sys
 import time
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
 from urllib.parse import quote, unquote, urlsplit
@@ -699,15 +700,17 @@ class Handler(SimpleHTTPRequestHandler):
     # Pass 1 mirrors each file from a valid gzip body; pass 2 (--update) serves
     # a body that cannot be decoded. The previously-mirrored copy must survive.
     # fresh.html is the control: its pass-2 body decodes, so it must be updated.
-    UPCODEC_SEEN = {}
+    # Per-path body-fetch counter shared by the update-refetch routes; paths are
+    # distinct so one dict serves all of them.
+    REFETCH_SEEN = {}
 
-    def upcodec_pass(self):
-        """1 on the first body fetch of this path, 2 on the next ones. HEADs
-        don't count, so a stray one can't shift which pass gets the bad body."""
+    def refetch_pass(self):
+        """1 on the first body fetch of this path, N on the Nth. HEADs don't
+        count, so a stray one can't shift which pass gets the special body."""
         if self.command == "HEAD":
             return 1
-        seen = Handler.UPCODEC_SEEN.get(self.path, 0) + 1
-        Handler.UPCODEC_SEEN[self.path] = seen
+        seen = Handler.REFETCH_SEEN.get(self.path, 0) + 1
+        Handler.REFETCH_SEEN[self.path] = seen
         return seen
 
     @staticmethod
@@ -739,36 +742,78 @@ class Handler(SimpleHTTPRequestHandler):
     UNSUP_V1 = b"<html><body><p>MIRRORED-UNSUP-V1</p></body></html>"
 
     def route_upcodec_mem(self):
-        if self.upcodec_pass() == 1:
+        if self.refetch_pass() == 1:
             self.send_coded(self.gzipped(self.MEM_V1), "text/html")
         else:
             self.send_coded(self.bad_gzip(self.MEM_V1), "text/html")
 
     def route_upcodec_disk(self):
-        if self.upcodec_pass() == 1:
+        if self.refetch_pass() == 1:
             self.send_coded(self.gzipped(self.DISK_V1), "application/octet-stream")
         else:
             self.send_coded(self.bad_gzip(self.DISK_V1), "application/octet-stream")
 
     # Pass 2 switches to a coding we have no decoder for.
     def route_upcodec_unsup(self):
-        if self.upcodec_pass() == 1:
+        if self.refetch_pass() == 1:
             self.send_coded(self.gzipped(self.UNSUP_V1), "text/html")
         else:
             self.send_coded(self.UNSUP_V1, "text/html", coding="compress")
 
     def route_upcodec_fresh(self):
-        pass1 = self.upcodec_pass() == 1
+        pass1 = self.refetch_pass() == 1
         body = b"<html><body><p>FRESH-V%d</p></body></html>" % (1 if pass1 else 2)
         self.send_coded(self.gzipped(body), "text/html")
 
     # Same, direct-to-disk: the update pass decodes, so the temp is renamed over
     # an existing mirror file.
     def route_upcodec_freshdisk(self):
-        pass1 = self.upcodec_pass() == 1
+        pass1 = self.refetch_pass() == 1
         body = b"FRESHDISK-V%d\n" % (1 if pass1 else 2) + b"\x03\x02\x01\xfe" * 8192
         self.send_coded(self.gzipped(body), "application/octet-stream")
 
+    # #562: pass 1 mirrors fully; pass 2 (--update) declares the full
+    # Content-Length but delivers half then closes, so httrack refuses the partial.
+    PAGE_V1 = b"<html><body><p>MIRRORED-PAGE-V1</p></body></html>"
+    BIN_V1 = b"MIRRORED-BIN-V1\n" + b"\x00\x01\x02\xff" * 8192
+
+    def route_uptrunc_index(self):
+        self.send_html(
+            '\t<a href="page.html">page</a>\n'
+            '\t<a href="file.bin">file</a>\n'
+            '\t<a href="stay.html">stay</a>\n'
+        )
+
+    def send_truncated(self, body, content_type):
+        self.send_response(200)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            self.wfile.write(body[: len(body) // 2])  # short, then close
+            self.wfile.flush()
+        except OSError:
+            pass
+
+    def route_uptrunc_page(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.PAGE_V1, "text/html")
+        else:
+            self.send_truncated(self.PAGE_V1, "text/html")
+
+    def route_uptrunc_file(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.BIN_V1, "application/octet-stream")
+        else:
+            self.send_truncated(self.BIN_V1, "application/octet-stream")
+
+    # Control: fully served both passes, so a normal --update still lands.
+    def route_uptrunc_stay(self):
+        v = 1 if self.refetch_pass() == 1 else 2
+        self.send_raw(b"<html><body><p>STAY-V%d</p></body></html>" % v, "text/html")
+
     # Echo what httrack advertised, so a crawl can assert the header.
     def route_codec_ae(self):
         self.send_raw(
@@ -1472,6 +1517,10 @@ class Handler(SimpleHTTPRequestHandler):
         "/upcodec/unsup.html": route_upcodec_unsup,
         "/upcodec/fresh.html": route_upcodec_fresh,
         "/upcodec/freshdisk.bin": route_upcodec_freshdisk,
+        "/uptrunc/index.html": route_uptrunc_index,
+        "/uptrunc/page.html": route_uptrunc_page,
+        "/uptrunc/file.bin": route_uptrunc_file,
+        "/uptrunc/stay.html": route_uptrunc_stay,
         "/types/index.html": route_types_index,
         "/types/control.php": route_types,
         "/types/photo.png": route_types,
@@ -1817,7 +1866,8 @@ def main():
         httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
 
     port = httpd.socket.getsockname()[1]
-    # The launcher reads this line to discover the ephemeral port.
+    # Keep the port line the launcher parses LF: Windows would emit \r\n.
+    sys.stdout.reconfigure(newline="\n")
     print(f"PORT {port}", flush=True)
 
     try:

tests/proxy-https-server.py

@@ -141,6 +141,8 @@ def main():
         open(os.path.join(logdir, name), "w").close()
     origin_port = start_origin(certfile, logdir)
     proxy_port = start_proxy(logdir, mode)
+    # Keep the port lines the caller parses LF: Windows would emit \r\n.
+    sys.stdout.reconfigure(newline="\n")
     print("ORIGIN %d" % origin_port, flush=True)
     print("PROXY %d" % proxy_port, flush=True)
     print("ready", flush=True)

tests/socks5-server.py

@@ -217,6 +217,8 @@ def main():
     tls_port = start_origin(logdir, certfile)
     http_port = start_origin(logdir, None)
     socks_port = start_socks(logdir, mode)
+    # Keep the port lines the caller parses LF: Windows would emit \r\n.
+    sys.stdout.reconfigure(newline="\n")
     print("TLS %d" % tls_port, flush=True)
     print("HTTP %d" % http_port, flush=True)
     print("SOCKS %d" % socks_port, flush=True)

tests/testlib.sh

@@ -0,0 +1,45 @@
+#!/bin/bash
+#
+# Helpers shared by the crawl tests. Sourced, not run.
+
+# Python 3 interpreter, or empty: Windows only installs python.exe, and a bare
+# "python" may be 2.x or the Store stub.
+find_python() {
+    local py
+    for py in "${PYTHON:-}" python3 python; do
+        test -n "$py" || continue
+        "$py" -c 'import sys; sys.exit(sys.version_info[0] != 3)' 2>/dev/null || continue
+        printf '%s\n' "$py"
+        return 0
+    done
+    return 1
+}
+
+# Native form of a path: a non-MSYS binary cannot resolve Git Bash's /d/a/... ones.
+nativepath() {
+    if is_windows && command -v cygpath >/dev/null 2>&1; then
+        cygpath -m "$1"
+    else
+        printf '%s\n' "$1"
+    fi
+}
+
+is_windows() {
+    case "$(uname -s)" in
+    MINGW* | MSYS* | CYGWIN*) return 0 ;;
+    *) return 1 ;;
+    esac
+}
+
+# Stop a backgrounded server and reap it; MSYS cannot signal a native python.exe,
+# so only -9 lands. Every step is "|| true": callers run under set -e, and reaping
+# a server we just signalled makes wait return 143.
+stop_server() {
+    test -n "${1:-}" || return 0
+    kill "$1" 2>/dev/null || true
+    if is_windows; then
+        kill -9 "$1" 2>/dev/null || true
+    fi
+    wait "$1" 2>/dev/null || true
+    return 0
+}