Test 46: assert the resume's Range actually drew the 304

The size and request-count checks alone could pass a wrong-fix that silently
switched to a fresh re-crawl instead of handling the 304 on the resume path.
Mark when the server 304s a Range request and assert it fired, so recovery is
proven to go through the resume rather than around it.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -55,7 +55,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -99,7 +101,8 @@ jobs:
           sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
             -exec mv {} {}.hidden \;
           ! command -v python3
-          make check
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -133,7 +136,15 @@ jobs:
         run: make -j"$(sysctl -n hw.ncpu)"
 
       - name: Test
-        run: make check
+        # bigcrawl's sustained -c8 crawl drops fetches on macOS's loopback when
+        # it competes with other crawls, flaking its exact file count (the #527
+        # macOS drop). Run everything else in parallel, then bigcrawl alone (its
+        # serial-safe condition). Linux tolerates the full parallel run.
+        run: |
+          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          rest=$(cd tests && ls *.test | grep -v '^36_local-bigcrawl\.test$' | tr '\n' ' ')
+          make check -j"$jobs" TESTS="$rest"
+          make check TESTS=36_local-bigcrawl.test
 
       - name: Print the test log on failure
         if: failure()
@@ -171,7 +182,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -226,7 +239,9 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -272,7 +287,8 @@ jobs:
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
           # skipped here (uninstrumented libz floods MSan with false positives).
           tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
-          make check TESTS="$tests"
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs" TESTS="$tests"
 
       - name: Print the test log on failure
         if: failure()
@@ -347,7 +363,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()

AGENTS.md

@@ -6,8 +6,19 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 ## Build & test
 - Fresh clone first: `git submodule update --init src/coucal`
 - `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
-  automake, libtool), then `bash configure && make && make check`. Or run
-  `sh build.sh` to do bootstrap + configure + make in one shot.
+  automake, libtool), then `bash configure && make -j"$(nproc)" && make check
+  -j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
+  automake's parallel harness and each crawl test binds its own ephemeral-port
+  server, so `-j` never contends and a multi-minute serial run drops to
+  seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
+  automatically; only a test slower than the current longest raises the floor.
+  On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
+  spend much of their wall time asleep (server trickles, httrack self-pacing),
+  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)`. macOS runs
+  36_local-bigcrawl alone in a second pass: its sustained `-c8` crawl overloads
+  the macOS loopback when it competes with other crawls and flakes its exact
+  file count (Linux tolerates the full parallel run).
+  Or run `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

src/htsback.c

@@ -3512,6 +3512,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     continue;
                   }
 
+                  // The server really sent 304 here; the *-hacks below force
+                  // NOT_MODIFIED only after confirming the file is complete.
+                  const hts_boolean server_sent_304 =
+                      (back[i].r.statuscode == HTTP_NOT_MODIFIED);
+
                   /*
                      Solve "false" 416 problems
                    */
@@ -3540,22 +3545,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.statuscode == 406) {    // 'Not Acceptable'
                     back[i].r.statuscode = HTTP_OK;
                   }
-                  // 'do not erase already downloaded file'
-                  // on an updated file
-                  // with an error : consider a 304 error
-                  if (!opt->delete_old) {
-                    if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
-                        && !back[i].testmode) {
-                      if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
-                        hts_log_print(opt, LOG_DEBUG,
-                                      "Error ignored %d (%s) because of 'no purge' option for %s%s",
-                                      back[i].r.statuscode, back[i].r.msg,
-                                      back[i].url_adr, back[i].url_fil);
-                        back[i].r.statuscode = HTTP_NOT_MODIFIED;
-                        deletehttp(&back[i].r);
-                        back[i].r.soc = INVALID_SOCKET;
-                      }
-                    }
+                  // On update, keep the good copy on error (mask as 304); skip
+                  // resume paths so a stale-partial 416 still re-fetches.
+                  if (HTTP_IS_ERROR(back[i].r.statuscode) &&
+                      back[i].is_update && !back[i].testmode &&
+                      back[i].range_req_size == 0 && back[i].url_sav[0] &&
+                      fexist_utf8(back[i].url_sav)) {
+                    hts_log_print(opt, LOG_DEBUG,
+                                  "Error %d (%s) ignored on update, keeping "
+                                  "existing copy: %s%s",
+                                  back[i].r.statuscode, back[i].r.msg,
+                                  back[i].url_adr, back[i].url_fil);
+                    back[i].r.statuscode = HTTP_NOT_MODIFIED;
+                    deletehttp(&back[i].r);
+                    back[i].r.soc = INVALID_SOCKET;
                   }
                   // Various hacks to limit re-transfers when updating a mirror
                   // Force update if same size detected
@@ -3703,6 +3706,21 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     }
                   }
 
+                  // Out-of-protocol 304 to a Range resume: the file is still
+                  // partial, so drop it and refetch instead of trusting it.
+                  if (server_sent_304 && back[i].range_req_size > 0) {
+                    url_savename_refname_remove(opt, back[i].url_adr,
+                                                back[i].url_fil);
+                    UNLINK(back[i].url_sav);
+                    deletehttp(&back[i].r);
+                    back[i].r.soc = INVALID_SOCKET;
+                    back[i].r.statuscode = STATUSCODE_NON_FATAL;
+                    strcpybuff(back[i].r.msg,
+                               "Bogus 304 on resume, restarting");
+                    back[i].status = STATUS_READY;
+                    back_set_finished(sback, i);
+                  }
+
                   /* sinon, continuer */
                   /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok récupérer body? */
                   // head: terminé
@@ -4180,10 +4198,13 @@ static int back_maxtime_grace(const int maxtime) {
    No floor (unlike the time grace): a size overrun should abort promptly. */
 static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
 
-/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481). */
+/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481).
+   -M measures received volume (HTS_TOTAL_RECV), not saved 200-only stat_bytes
+   which undercounts redirect/error-heavy crawls (#520). */
 static hts_mirror_limit back_mirror_limit(httrackp *opt) {
-  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
-      HTS_STAT.stat_bytes - opt->maxsite >= back_maxsize_grace(opt->maxsite))
+  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
+      HTS_STAT.HTS_TOTAL_RECV - opt->maxsite >=
+          back_maxsize_grace(opt->maxsite))
     return HTS_MIRROR_LIMIT_SIZE;
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
@@ -4197,7 +4218,7 @@ static hts_mirror_limit back_mirror_limit(httrackp *opt) {
 
 int back_checkmirror(httrackp *opt) {
   /* request a smooth stop the first time each cap is reached */
-  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
+  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
       !opt->state.stop) {
     hts_log_print(opt, LOG_ERROR,
                   "More than " LLintP

src/htsglobal.h

@@ -56,6 +56,10 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
+// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
+#include <stdint.h>
+#include <inttypes.h>
+
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -111,29 +115,6 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
-#ifndef HTS_LONGLONG
-#ifdef SIZEOF_LONG_LONG
-#if SIZEOF_LONG_LONG == 8
-#define HTS_LONGLONG 1
-#endif
-#endif
-
-#ifndef HTS_LONGLONG
-#ifdef __sun
-#define HTS_LONGLONG 0
-#endif
-#ifdef __osf__
-#define HTS_LONGLONG 0
-#endif
-#ifdef __linux
-#define HTS_LONGLONG 1
-#endif
-#ifdef _WIN32
-#define HTS_LONGLONG 1
-#endif
-#endif
-#endif
-
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -326,59 +307,11 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-#ifndef HTS_LONGLONG
-#ifdef HTS_NO_64_BIT
-#define HTS_LONGLONG 0
-#else
-#define HTS_LONGLONG 1
-#endif
-#endif
-
-/* Wide integer types, chosen per platform.
-   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
-           plain int where 64-bit is unavailable).
-   TStamp: timestamp/duration in the same width (a double in the no-64-bit
-           fallback).
-   LLintP: the printf conversion for an LLint. */
-#if HTS_LONGLONG
-#ifdef LLINT_FORMAT
-typedef LLINT_TYPE LLint;
-
-typedef LLINT_TYPE TStamp;
-
-#define LLintP LLINT_FORMAT
-#else
-
-#ifdef _WIN32
-typedef __int64 LLint;
-
-typedef __int64 TStamp;
-
-#define LLintP "%I64d"
-#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
-       defined(__64BIT__))
-
-typedef long int LLint;
-
-typedef long int TStamp;
-
-#define LLintP "%ld"
-#else
-typedef long long int LLint;
-
-typedef long long int TStamp;
-
-#define LLintP "%lld"
-#endif
-
-#endif /* HTS_LONGLONG */
-
-#else
-typedef int LLint;
-
-#define LLintP "%d"
-typedef double TStamp;
-#endif
+/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
+typedef int64_t LLint;
+typedef int64_t TStamp;
+/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
+#define LLintP "%" PRId64
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain

src/htslib.c

@@ -3038,12 +3038,11 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
 /* See http://physics.nist.gov/cuu/Units/binary.html */
 #define ToLLint(a) ((LLint)(a))
 #define ToLLintKiB (ToLLint(1024))
-#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
-#ifdef HTS_LONGLONG
-#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#endif
+#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
+#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintPiB                                                             \
+  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
 HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
   if (n < ToLLintKiB) {
     sprintf(strc->buff1, "%d", (int) (LLint) n);
@@ -3052,9 +3051,7 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
             (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
     strcpybuff(strc->buff2, "KiB");
-  }
-#ifdef HTS_LONGLONG
-  else if (n < ToLLintGiB) {
+  } else if (n < ToLLintGiB) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
             (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
     strcpybuff(strc->buff2, "MiB");
@@ -3071,13 +3068,6 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
             (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
     strcpybuff(strc->buff2, "PiB");
   }
-#else
-  else {
-    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
-            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
-    strcpybuff(strc->buff2, "MiB");
-  }
-#endif
   strc->buffadr[0] = strc->buff1;
   strc->buffadr[1] = strc->buff2;
   return strc->buffadr;

tests/19_local-connect-fallback.test

@@ -66,9 +66,12 @@ test -n "$port" || {
 }
 
 out="$tmpdir/crawl"
+# macOS has only 127.0.0.1, so the dead 127.0.0.x connects block to the timeout
+# instead of refusing instantly (as on Linux); a small --timeout + --retries=0
+# bound the stall without changing what the fallback exercises.
 HTTRACK_DEBUG_RESOLVE="deadhost:127.0.0.2,127.0.0.1" \
     httrack "http://deadhost:$port/simple/basic.html" -O "$out" \
-    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log" 2>&1
+    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log" 2>&1
 
 log="$out/hts-log.txt"
 
@@ -92,12 +95,12 @@ test -f "$out/deadhost_$port/simple/basic.html" || {
     exit 1
 }
 
-# every address refused: the slot exhausts the list, then errors out (the
-# harness timeout would catch a hang/loop; refused connects are instant)
+# every address dead: the slot exhausts the list, then errors out (the harness
+# timeout would catch a hang/loop)
 out2="$tmpdir/crawl2"
 HTTRACK_DEBUG_RESOLVE="alldead:127.0.0.2,127.0.0.3" \
     httrack "http://alldead:$port/simple/basic.html" -O "$out2" \
-    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log2" 2>&1
+    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log2" 2>&1
 log2="$out2/hts-log.txt"
 
 grep -q "trying next address" "$log2" || {

tests/31_local-javaclass.test

@@ -17,7 +17,10 @@ printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
 printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
     >"$tmproot/javaclass/Foo.class"
 
+# --log-found first: a plugin-load failure (wrong soname, shadowing system
+# libhtsjava) trips here on the banner suffix, not as an opaque "not found".
 bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
+    --log-found '\+libhtsjava' \
     --found 'javaclass/Foo.class' \
     --found 'javaclass/hello.gif' \
     httrack 'BASEURL/javaclass/index.html'

tests/36_local-bigcrawl.test

@@ -9,7 +9,7 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --errors 4 --files 361 \
+    --errors-content 4 --files 361 \
     --found 'big/p/95.html' \
     --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
     --found 'big/a/f2-2x.png' \

tests/44_local-update-errormask.test

@@ -0,0 +1,13 @@
+#!/bin/bash
+# Issue #176: on --update a page that 200'd on the first crawl but now 403s must
+# keep its good local copy, not be overwritten by the error body nor purged.
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --rerun \
+    --found 'errmask/keep.dat' \
+    --file-min-bytes 'errmask/keep.dat' 1024 \
+    --file-matches 'errmask/keep.dat' '^KEEP' \
+    --file-not-matches 'errmask/keep.dat' 'error 403' \
+    httrack 'BASEURL/errmask/index.html'

tests/45_local-maxsize-recv.test

@@ -0,0 +1,16 @@
+#!/bin/bash
+#
+# -M caps received volume, not saved 200-only bytes (#520): links to large 404
+# bodies pump received >> saved; the cap must trip though little lands on disk.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# The mirror bound is the load-bearing half: it fires while <100 KB is saved,
+# so the cap can only have tripped on received volume, not saved bytes (which
+# would exceed the cap and fetch all 16 links).
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'More than 500000 bytes have been transferred.. giving up' \
+    --max-mirror-bytes 100000 \
+    httrack 'BASEURL/maxrecv/index.html' -M500000 -c4

tests/46_local-update-304-resume.test

@@ -0,0 +1,118 @@
+#!/bin/bash
+# C7: a server 304 answering a resume's Range request is out of protocol; httrack
+# must drop the partial and refetch, not finalize it as complete at the partial
+# byte count. Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets a bogus
+# 304, and must recover the whole file.
+set -u
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+server="${testdir}/local-server.py"
+
+command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_c7.XXXXXX") || exit 1
+serverpid=
+crawlpid=
+cleanup() {
+    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
+    if test -n "$serverpid"; then
+        kill "$serverpid" 2>/dev/null
+        wait "$serverpid" 2>/dev/null
+    fi
+    rm -rf "$tmpdir"
+}
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+serverlog="${tmpdir}/server.log"
+counter="${tmpdir}/reqcount"
+mark="${tmpdir}/got304"
+RESUME304_COUNTER="$counter" RESUME304_MARK="$mark" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$serverlog" 2>/dev/null)
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$serverlog")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+base="http://127.0.0.1:${port}"
+
+which httrack >/dev/null || {
+    echo "could not find httrack"
+    exit 1
+}
+out="${tmpdir}/crawl"
+mkdir "$out"
+common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
+refdir="${out}/hts-cache/ref"
+
+# --- pass 1: crawl, interrupt once the blob download is underway -------------
+printf '[pass 1: interrupt mid-download] ..\t'
+httrack "${common[@]}" "${base}/resume304/index.html" >"${tmpdir}/log1" 2>&1 &
+crawlpid=$!
+for _ in $(seq 1 300); do
+    test -s "$counter" && break
+    kill -0 "$crawlpid" 2>/dev/null || break
+    sleep 0.1
+done
+sleep 0.5
+kill -TERM "$crawlpid" 2>/dev/null
+wait "$crawlpid" 2>/dev/null
+crawlpid=
+test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
+    echo "FAIL: no temp-ref survived pass 1; cannot drive C7"
+    exit 1
+}
+echo "OK (temp-ref present)"
+before=$(wc -c <"$counter" 2>/dev/null || echo 0)
+
+# --- pass 2: --continue -> resume Range -> bogus 304 -> drop + refetch --------
+printf '[pass 2: resume gets 304, must refetch] ..\t'
+httrack "${common[@]}" --continue "${base}/resume304/index.html" >"${tmpdir}/log2" 2>&1
+echo "OK (terminated)"
+
+blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
+full=8200 # len(RESUME304_BODY) = len("C7DATA--") + 8192
+
+printf '[file recovered whole] ..\t'
+test -s "$blob" || {
+    echo "FAIL: blob.bin missing after the 304 resume (partial dropped, not refetched)"
+    exit 1
+}
+got=$(wc -c <"$blob")
+test "$got" -eq "$full" || {
+    echo "FAIL: blob.bin is ${got} bytes, expected ${full} (partial finalized as complete)"
+    exit 1
+}
+echo "OK (${got} bytes)"
+
+# Pass 2 must issue at least two requests: the resume (Range -> 304) and the
+# range-less refetch (-> 200). Exactly one means the partial was trusted.
+after=$(wc -c <"$counter" 2>/dev/null || echo 0)
+hits=$((after - before))
+printf '[resume then refetch] ..\t'
+test "$hits" -ge 2 || {
+    echo "FAIL: only ${hits} pass-2 request(s); the 304 was trusted, no refetch"
+    exit 1
+}
+echo "OK (${hits} requests)"
+
+# The recovery must go through the resume path: a Range request that drew the
+# bogus 304 (a fresh re-crawl sends no Range, so this marker stays empty).
+printf '[resume Range drew a 304] ..\t'
+test -s "$mark" || {
+    echo "FAIL: no Range request got a 304; the resume path was not exercised"
+    exit 1
+}
+echo "OK"

tests/Makefile.am

@@ -112,6 +112,9 @@ TESTS = \
 	40_local-why.test \
 	41_local-utf8-link.test \
 	42_local-maxsize-slow.test \
-	43_local-update-truncate.test
+	43_local-update-truncate.test \
+	44_local-update-errormask.test \
+	45_local-maxsize-recv.test \
+	46_local-update-304-resume.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -14,11 +14,13 @@
 # Usage:
 #   bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
 #       [--rerun-args 'ARGS'] \
-#       --errors N --files N --found PATH ... --directory PATH ... \
+#       --errors N --errors-content N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
 #       --file-min-bytes PATH N --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
+# --errors counts every "Error:" log line; --errors-content drops transient
+# network failures (codes -2..-6) that flake on busy loopback under -c8.
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
 # --max/--min-mirror-bytes bound the mirrored content bytes (host root).
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
@@ -131,7 +133,7 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         rerun_args="${args[$pos]}"
         ;;
-    --errors | --files)
+    --errors | --errors-content | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -334,6 +336,14 @@ while test "$i" -lt "${#audit[@]}"; do
         assert_equals "checking errors" "${audit[$i]}" \
             "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
         ;;
+    --errors-content)
+        i=$((i + 1))
+        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
+        # transient network failures (statuscode -2..-6) flake on busy loopback;
+        # the code parens are followed by " at link" or " after N retries at link"
+        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
+        assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
+        ;;
     --files)
         i=$((i + 1))
         nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |

tests/local-server.py

@@ -790,6 +790,56 @@ class Handler(SimpleHTTPRequestHandler):
         self.send_header("Content-Length", "0")
         self.end_headers()
 
+    # C7: stall the first GET (partial + temp-ref), then answer the resume's
+    # Range with a bogus 304; httrack must drop the partial and refetch.
+    RESUME304_BODY = b"C7DATA--" + bytes((i * 7 + 3) % 256 for i in range(8192))
+    _resume304_started = False
+
+    def route_resume304_index(self):
+        self.send_html('\t<a href="blob.bin">blob</a>')
+
+    def route_resume304(self):
+        counter = os.environ.get("RESUME304_COUNTER")
+        if counter:
+            with open(counter, "a") as fp:
+                fp.write("x")
+        rng = self.headers.get("Range")
+        if not Handler._resume304_started:
+            Handler._resume304_started = True
+            self.send_response(200)
+            self.send_header("Content-Type", "application/octet-stream")
+            self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
+            self.send_header("Accept-Ranges", "bytes")
+            self.send_header("Last-Modified", BIG_LASTMOD)
+            self.send_header("ETag", '"c7"')
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(self.RESUME304_BODY[:4096])
+                self.wfile.flush()
+                try:
+                    while True:
+                        time.sleep(3600)
+                except OSError:
+                    pass
+            return
+        if rng is not None:  # resume request: bogus out-of-protocol 304
+            mark = os.environ.get("RESUME304_MARK")
+            if mark:
+                with open(mark, "a") as fp:
+                    fp.write("z")
+            self.send_response(304)
+            self.send_header("ETag", '"c7"')
+            self.send_header("Content-Length", "0")
+            self.end_headers()
+            return
+        # Range-less refetch after the partial is dropped: whole file.
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(self.RESUME304_BODY)
+
     # 206 resume must honor the server's Content-Range, not the offset we asked
     # for (#198): a server resuming a few bytes *before* the request must not
     # leave httrack duplicating the overlap onto the partial. flaky.bin
@@ -934,6 +984,33 @@ class Handler(SimpleHTTPRequestHandler):
     def route_mini304_page(self):
         self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
 
+    # --- /errmask/: issue #176 — a page that 200'd on the first crawl but 403s
+    # on the update fetch must keep its good copy, not be overwritten nor purged.
+    ERRMASK_GOOD = b"KEEP" + b"." * 1020  # 1024 B distinctive non-HTML body
+    ERRMASK_ERR = b"<html><body>error 403</body></html>\n"
+
+    def route_errmask_index(self):
+        self.send_html('\t<a href="keep.dat">keep</a>\n')
+
+    def route_errmask_keep(self):
+        # First crawl (no validator) gets the 1024 B body + Last-Modified; the
+        # update sends a conditional and gets a 403 error page.
+        if self.headers.get("If-Modified-Since") or self.headers.get("If-None-Match"):
+            self.send_response(403, "Forbidden")
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", str(len(self.ERRMASK_ERR)))
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(self.ERRMASK_ERR)
+            return
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Last-Modified", BIG_LASTMOD)
+        self.send_header("Content-Length", str(len(self.ERRMASK_GOOD)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(self.ERRMASK_GOOD)
+
     # --- delayed-type degenerate paths (issues #5/#107) --------------------
     def route_delayed_index(self):
         self.send_html(
@@ -1077,6 +1154,22 @@ class Handler(SimpleHTTPRequestHandler):
         except OSError:
             pass
 
+    # -M received-volume cap (#520): links to large 404 bodies. httrack receives
+    # each (HTS_TOTAL_RECV climbs) but saves none, so saved stays far below -M.
+    def route_maxrecv_index(self):
+        self.send_html(
+            "".join('\t<a href="r%d.bin">r%d</a>\n' % (i, i) for i in range(16))
+        )
+
+    def route_maxrecv_404(self):
+        body = b"x" * self.BIGFILE_BYTES
+        self.send_response(404, "Not Found")
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(body)
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -1106,6 +1199,8 @@ class Handler(SimpleHTTPRequestHandler):
         "/intl/" + INTL_NAME: route_intl_page,
         "/resume/index.html": route_resume_index,
         "/resume/blob.txt": route_resume,
+        "/resume304/index.html": route_resume304_index,
+        "/resume304/blob.bin": route_resume304,
         "/overlap/index.html": route_overlap_index,
         "/overlap/flaky.bin": route_overlap,
         "/overlap/full.bin": route_overlap_full,
@@ -1177,6 +1272,25 @@ class Handler(SimpleHTTPRequestHandler):
         "/redir/target.html": route_redir_target,
         "/mini304/index.html": route_mini304_index,
         "/mini304/page.html": route_mini304_page,
+        "/errmask/index.html": route_errmask_index,
+        "/errmask/keep.dat": route_errmask_keep,
+        "/maxrecv/index.html": route_maxrecv_index,
+        "/maxrecv/r0.bin": route_maxrecv_404,
+        "/maxrecv/r1.bin": route_maxrecv_404,
+        "/maxrecv/r2.bin": route_maxrecv_404,
+        "/maxrecv/r3.bin": route_maxrecv_404,
+        "/maxrecv/r4.bin": route_maxrecv_404,
+        "/maxrecv/r5.bin": route_maxrecv_404,
+        "/maxrecv/r6.bin": route_maxrecv_404,
+        "/maxrecv/r7.bin": route_maxrecv_404,
+        "/maxrecv/r8.bin": route_maxrecv_404,
+        "/maxrecv/r9.bin": route_maxrecv_404,
+        "/maxrecv/r10.bin": route_maxrecv_404,
+        "/maxrecv/r11.bin": route_maxrecv_404,
+        "/maxrecv/r12.bin": route_maxrecv_404,
+        "/maxrecv/r13.bin": route_maxrecv_404,
+        "/maxrecv/r14.bin": route_maxrecv_404,
+        "/maxrecv/r15.bin": route_maxrecv_404,
     }
 
     # --- /big/ seeded pseudo-site ------------------------------------------