Merge origin/master into fix-629-webhttrack-argv-utf8

Resolve tests/Makefile.am: keep 66/67/69 (merged) alongside this branch's 68.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/windows-build.yml

@@ -188,7 +188,7 @@ jobs:
 
           # Every gate here exits 77, so an all-skipped suite would report green having
           # tested nothing: pin the skips, and floor the passes in case the glob empties.
-          expected_skips=" 48_local-crange-memresume.test 71_local-crange-repaircache.test"  # pending #581
+          expected_skips=" 48_local-crange-memresume.test"  # pending #581
           [ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
           [ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
           [ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }

AGENTS.md

@@ -19,13 +19,6 @@ the operational checklist: toolchain, invariants, and how to ship a change.
   (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
   `-c16` bigcrawl the way Python's default backlog of 5 did.
   Or run `sh build.sh` to do bootstrap + configure + make in one shot.
-- A `tests/NN_*.test` runs only if listed in `tests/Makefile.am`'s `TESTS`; an
-  unregistered file is silently skipped.
-- `make check` prepends the build's `src/` to `PATH`, but a hand-run `.test` does
-  not — an installed `/usr/bin/httrack` then shadows your build. Run via `make
-  check`, or `PATH="<bld>/src:$PATH"` for a manual run.
-- Give new `.test` scripts `set -e`: the older ones predate the rule, so several
-  `local-crawl.sh` calls with no `set -e` report PASS on any non-last failure.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every
@@ -38,32 +31,15 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 - **Format only changed lines** with `git clang-format` (clang-format 19). Never
   reformat untouched code: the engine was formatted by an old tool and won't
   round-trip.
-- **Byte-safe edits.** A few tracked files carry raw ISO-8859-1 high bytes
-  (French comments): `src/htsconcat.c`, `lang/*.txt`, `html/contact.html`, and
-  the `fuzz/corpus/*` vectors. Edit those byte-wise (`perl -0pi`, `sed`), not
-  through a tool that re-encodes to UTF-8 and corrupts them. The rest of the tree
-  is UTF-8 and safe to edit normally.
+- **Byte-safe edits.** Files with raw high bytes are ISO-8859-1 (French
+  comments). Edit them byte-wise (`perl -0pi`, `sed`), not through a tool that
+  re-encodes to UTF-8 and corrupts them.
 
 ## Security (HTTrack parses hostile input off the network)
 - Bounds-check every copy. Overflow-safe form: put the untrusted value alone,
   `untrusted < limit - controlled` — never `controlled + untrusted < limit`,
   which can wrap and pass.
 
-## C conventions
-- **Use the `*t` allocator wrappers, never raw libc** (`htssafe.h`):
-  `malloct`/`calloct`/`realloct`/`freet`/`strdupt`, in test and selftest code
-  too. `freet` NULLs its (lvalue) argument and tolerates NULL; `calloct(n, sz)`
-  keeps calloc's arg order. Only exception: storing or calling a libc symbol
-  itself (e.g. a resolver-backend function pointer).
-- **Exported API is `HTSEXT_API`.** Everything else is hidden by
-  `-fvisibility=hidden` and free to change (check with `nm -D --defined-only
-  libhttrack.so`). Touching an installed-header struct (see `DevIncludes_DATA` in
-  `src/Makefile.am`) or an exported signature is an ABI break — flag and discuss,
-  bump the soname, and prefer keeping the old entry point beside a new one.
-- **Windows ABI is free to break, POSIX is not.** The Windows DLL ships next to
-  the exe with no soname contract, so a `_WIN32`-only ABI change needs no
-  deprecation dance; POSIX/ELF keeps the flag-discuss-bump rules.
-
 ## Code & prose
 - Be terse. Comment the why, in English; translate French comments you touch.
 - Strip AI tells from prose (em-dash overuse, rule-of-three, filler, vague

tests/71_local-crange-repaircache.test

@@ -1,134 +0,0 @@
-#!/bin/bash
-# #581: an interrupted mirror can leave a truncated new.zip that pass 2 must
-# repair before resuming. The repaired-cache resume then hits the same hostile
-# 206 as test 48, so restart-whole must still drop the partial and refetch the
-# whole file. Pass 1 leaves a partial + temp-ref; we truncate new.zip past its
-# last local entry (dropping the central directory) so unzOpen fails and the
-# repair path runs; pass 2 resumes, rejects the range, and refetches whole.
-set -u
-
-: "${top_srcdir:=..}"
-testdir=$(cd "$(dirname "$0")" && pwd)
-# shellcheck source=tests/testlib.sh
-. "${testdir}/testlib.sh"
-server=$(nativepath "${testdir}/local-server.py")
-root=$(nativepath "${testdir}/server-root")
-
-python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
-
-# On Windows the pass-1 interrupt is a hard kill (MSYS can't signal a native
-# exe) and the restart-whole path fails on the repaired cache (#581) -- the very
-# bug this exercises; skip until the engine fix lands.
-if is_windows; then
-    echo "Windows: restart-whole fails on a repaired cache (#581), skipping"
-    exit 77
-fi
-
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangerep.XXXXXX") || exit 1
-serverpid=
-crawlpid=
-cleanup() {
-    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    stop_server "$serverpid"
-    rm -rf "$tmpdir"
-}
-trap cleanup EXIT HUP INT QUIT PIPE TERM
-
-serverlog="${tmpdir}/server.log"
-: >"$serverlog"
-"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
-serverpid=$!
-port=
-for _ in $(seq 1 50); do
-    line=$(head -n1 "$serverlog" 2>/dev/null)
-    if test "${line%% *}" == "PORT"; then
-        port="${line#PORT }"
-        break
-    fi
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "server exited early: $(cat "$serverlog")"
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover server port"
-    exit 1
-}
-base="http://127.0.0.1:${port}"
-
-which httrack >/dev/null || {
-    echo "could not find httrack"
-    exit 1
-}
-out="${tmpdir}/crawl"
-mkdir "$out"
-common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
-refdir="${out}/hts-cache/ref"
-newzip="${out}/hts-cache/new.zip"
-
-# --- pass 1: crawl, interrupt once the blob download is underway -------------
-printf '[pass 1: interrupt mid-download] ..\t'
-httrack "${common[@]}" "${base}/crange206mem/index.html" >"${tmpdir}/log1" 2>&1 &
-crawlpid=$!
-for _ in $(seq 1 300); do
-    test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
-    kill -0 "$crawlpid" 2>/dev/null || break
-    sleep 0.1
-done
-sleep 0.3
-kill -TERM "$crawlpid" 2>/dev/null
-wait "$crawlpid" 2>/dev/null
-crawlpid=
-test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
-    echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
-    exit 1
-}
-echo "OK (temp-ref present)"
-
-# --- damage the cache: drop the central directory a hard kill never wrote ----
-printf '[damage new.zip -> forces repair] ..\t'
-"$python" - "$newzip" <<'PY' || exit 1
-import os, sys
-path = sys.argv[1]
-data = open(path, "rb").read()
-cut = data.find(b"PK\x01\x02")  # first central-directory header
-if cut <= 0:
-    sys.exit("no central directory to drop in %s" % path)
-os.truncate(path, cut)
-PY
-echo "OK"
-
-# --- pass 2: --continue -> repair -> resume -> hostile 206 -> refetch whole ---
-printf '[pass 2: repair, reject range, refetch] ..\t'
-rc=0
-httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1 || rc=$?
-test "$rc" -eq 0 || {
-    echo "FAIL: httrack exited $rc"
-    cat "${tmpdir}/log2" >&2
-    exit 1
-}
-echo "OK (terminated)"
-
-# The repair path must actually have run, else this is just a copy of test 48.
-printf '[cache repair fired] ..\t'
-grep -q 'damaged cache' "${out}/hts-log.txt" 2>/dev/null || {
-    echo "FAIL: repair path did not run; damage was ineffective"
-    exit 1
-}
-echo "OK"
-
-blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
-full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
-
-printf '[file recovered whole] ..\t'
-test -s "$blob" || {
-    echo "FAIL: blob.bin missing after the repaired-cache resume"
-    exit 1
-}
-got=$(wc -c <"$blob")
-test "$got" -eq "$full" || {
-    echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
-    exit 1
-}
-echo "OK (${got} bytes)"

tests/Makefile.am

@@ -151,7 +151,6 @@ TESTS = \
 	66_engine-port80-strip.test \
 	67_engine-delayed-truncate.test \
 	68_webhttrack-outdir-charset.test \
-	69_local-intl-logdir.test \
-	71_local-crange-repaircache.test
+	69_local-intl-logdir.test
 
 CLEANFILES = check-network_sh.cache