Isolate bigcrawl in a second pass on macOS to parallelize the rest

macOS was the CI bottleneck (~460s serial, the ~8min total): its loopback drops
fetches when bigcrawl's sustained -c8 crawl competes with other crawls, so a
parallel make check flaked bigcrawl's exact file count. Rather than keep the
whole macOS suite serial, run everything except bigcrawl in parallel (2x cores,
capped 16), then bigcrawl alone in a second pass, which is the serial-safe
condition green on master. Linux still runs the full suite in parallel.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsback.c

@@ -3512,11 +3512,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     continue;
                   }
 
-                  // The server really sent 304 here; the *-hacks below force
-                  // NOT_MODIFIED only after confirming the file is complete.
-                  const hts_boolean server_sent_304 =
-                      (back[i].r.statuscode == HTTP_NOT_MODIFIED);
-
                   /*
                      Solve "false" 416 problems
                    */
@@ -3706,21 +3701,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     }
                   }
 
-                  // Out-of-protocol 304 to a Range resume: the file is still
-                  // partial, so drop it and refetch instead of trusting it.
-                  if (server_sent_304 && back[i].range_req_size > 0) {
-                    url_savename_refname_remove(opt, back[i].url_adr,
-                                                back[i].url_fil);
-                    UNLINK(back[i].url_sav);
-                    deletehttp(&back[i].r);
-                    back[i].r.soc = INVALID_SOCKET;
-                    back[i].r.statuscode = STATUSCODE_NON_FATAL;
-                    strcpybuff(back[i].r.msg,
-                               "Bogus 304 on resume, restarting");
-                    back[i].status = STATUS_READY;
-                    back_set_finished(sback, i);
-                  }
-
                   /* sinon, continuer */
                   /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok récupérer body? */
                   // head: terminé
@@ -4198,13 +4178,10 @@ static int back_maxtime_grace(const int maxtime) {
    No floor (unlike the time grace): a size overrun should abort promptly. */
 static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
 
-/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481).
-   -M measures received volume (HTS_TOTAL_RECV), not saved 200-only stat_bytes
-   which undercounts redirect/error-heavy crawls (#520). */
+/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481). */
 static hts_mirror_limit back_mirror_limit(httrackp *opt) {
-  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
-      HTS_STAT.HTS_TOTAL_RECV - opt->maxsite >=
-          back_maxsize_grace(opt->maxsite))
+  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
+      HTS_STAT.stat_bytes - opt->maxsite >= back_maxsize_grace(opt->maxsite))
     return HTS_MIRROR_LIMIT_SIZE;
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
@@ -4218,7 +4195,7 @@ static hts_mirror_limit back_mirror_limit(httrackp *opt) {
 
 int back_checkmirror(httrackp *opt) {
   /* request a smooth stop the first time each cap is reached */
-  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
+  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
       !opt->state.stop) {
     hts_log_print(opt, LOG_ERROR,
                   "More than " LLintP

tests/19_local-connect-fallback.test

@@ -66,12 +66,9 @@ test -n "$port" || {
 }
 
 out="$tmpdir/crawl"
-# macOS has only 127.0.0.1, so the dead 127.0.0.x connects block to the timeout
-# instead of refusing instantly (as on Linux); a small --timeout + --retries=0
-# bound the stall without changing what the fallback exercises.
 HTTRACK_DEBUG_RESOLVE="deadhost:127.0.0.2,127.0.0.1" \
     httrack "http://deadhost:$port/simple/basic.html" -O "$out" \
-    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log" 2>&1
+    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log" 2>&1
 
 log="$out/hts-log.txt"
 
@@ -95,12 +92,12 @@ test -f "$out/deadhost_$port/simple/basic.html" || {
     exit 1
 }
 
-# every address dead: the slot exhausts the list, then errors out (the harness
-# timeout would catch a hang/loop)
+# every address refused: the slot exhausts the list, then errors out (the
+# harness timeout would catch a hang/loop; refused connects are instant)
 out2="$tmpdir/crawl2"
 HTTRACK_DEBUG_RESOLVE="alldead:127.0.0.2,127.0.0.3" \
     httrack "http://alldead:$port/simple/basic.html" -O "$out2" \
-    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log2" 2>&1
+    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log2" 2>&1
 log2="$out2/hts-log.txt"
 
 grep -q "trying next address" "$log2" || {

tests/31_local-javaclass.test

@@ -17,10 +17,7 @@ printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
 printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
     >"$tmproot/javaclass/Foo.class"
 
-# --log-found first: a plugin-load failure (wrong soname, shadowing system
-# libhtsjava) trips here on the banner suffix, not as an opaque "not found".
 bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
-    --log-found '\+libhtsjava' \
     --found 'javaclass/Foo.class' \
     --found 'javaclass/hello.gif' \
     httrack 'BASEURL/javaclass/index.html'

tests/45_local-maxsize-recv.test

@@ -1,16 +0,0 @@
-#!/bin/bash
-#
-# -M caps received volume, not saved 200-only bytes (#520): links to large 404
-# bodies pump received >> saved; the cap must trip though little lands on disk.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-# The mirror bound is the load-bearing half: it fires while <100 KB is saved,
-# so the cap can only have tripped on received volume, not saved bytes (which
-# would exceed the cap and fetch all 16 links).
-bash "$top_srcdir/tests/local-crawl.sh" \
-    --log-found 'More than 500000 bytes have been transferred.. giving up' \
-    --max-mirror-bytes 100000 \
-    httrack 'BASEURL/maxrecv/index.html' -M500000 -c4

tests/46_local-update-304-resume.test

@@ -1,118 +0,0 @@
-#!/bin/bash
-# C7: a server 304 answering a resume's Range request is out of protocol; httrack
-# must drop the partial and refetch, not finalize it as complete at the partial
-# byte count. Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets a bogus
-# 304, and must recover the whole file.
-set -u
-
-: "${top_srcdir:=..}"
-testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
-
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
-
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_c7.XXXXXX") || exit 1
-serverpid=
-crawlpid=
-cleanup() {
-    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
-    rm -rf "$tmpdir"
-}
-trap cleanup EXIT HUP INT QUIT PIPE TERM
-
-serverlog="${tmpdir}/server.log"
-counter="${tmpdir}/reqcount"
-mark="${tmpdir}/got304"
-RESUME304_COUNTER="$counter" RESUME304_MARK="$mark" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
-serverpid=$!
-port=
-for _ in $(seq 1 50); do
-    line=$(head -n1 "$serverlog" 2>/dev/null)
-    if test "${line%% *}" == "PORT"; then
-        port="${line#PORT }"
-        break
-    fi
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "server exited early: $(cat "$serverlog")"
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover server port"
-    exit 1
-}
-base="http://127.0.0.1:${port}"
-
-which httrack >/dev/null || {
-    echo "could not find httrack"
-    exit 1
-}
-out="${tmpdir}/crawl"
-mkdir "$out"
-common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
-refdir="${out}/hts-cache/ref"
-
-# --- pass 1: crawl, interrupt once the blob download is underway -------------
-printf '[pass 1: interrupt mid-download] ..\t'
-httrack "${common[@]}" "${base}/resume304/index.html" >"${tmpdir}/log1" 2>&1 &
-crawlpid=$!
-for _ in $(seq 1 300); do
-    test -s "$counter" && break
-    kill -0 "$crawlpid" 2>/dev/null || break
-    sleep 0.1
-done
-sleep 0.5
-kill -TERM "$crawlpid" 2>/dev/null
-wait "$crawlpid" 2>/dev/null
-crawlpid=
-test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
-    echo "FAIL: no temp-ref survived pass 1; cannot drive C7"
-    exit 1
-}
-echo "OK (temp-ref present)"
-before=$(wc -c <"$counter" 2>/dev/null || echo 0)
-
-# --- pass 2: --continue -> resume Range -> bogus 304 -> drop + refetch --------
-printf '[pass 2: resume gets 304, must refetch] ..\t'
-httrack "${common[@]}" --continue "${base}/resume304/index.html" >"${tmpdir}/log2" 2>&1
-echo "OK (terminated)"
-
-blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
-full=8200 # len(RESUME304_BODY) = len("C7DATA--") + 8192
-
-printf '[file recovered whole] ..\t'
-test -s "$blob" || {
-    echo "FAIL: blob.bin missing after the 304 resume (partial dropped, not refetched)"
-    exit 1
-}
-got=$(wc -c <"$blob")
-test "$got" -eq "$full" || {
-    echo "FAIL: blob.bin is ${got} bytes, expected ${full} (partial finalized as complete)"
-    exit 1
-}
-echo "OK (${got} bytes)"
-
-# Pass 2 must issue at least two requests: the resume (Range -> 304) and the
-# range-less refetch (-> 200). Exactly one means the partial was trusted.
-after=$(wc -c <"$counter" 2>/dev/null || echo 0)
-hits=$((after - before))
-printf '[resume then refetch] ..\t'
-test "$hits" -ge 2 || {
-    echo "FAIL: only ${hits} pass-2 request(s); the 304 was trusted, no refetch"
-    exit 1
-}
-echo "OK (${hits} requests)"
-
-# The recovery must go through the resume path: a Range request that drew the
-# bogus 304 (a fresh re-crawl sends no Range, so this marker stays empty).
-printf '[resume Range drew a 304] ..\t'
-test -s "$mark" || {
-    echo "FAIL: no Range request got a 304; the resume path was not exercised"
-    exit 1
-}
-echo "OK"

tests/Makefile.am

@@ -113,8 +113,6 @@ TESTS = \
 	41_local-utf8-link.test \
 	42_local-maxsize-slow.test \
 	43_local-update-truncate.test \
-	44_local-update-errormask.test \
-	45_local-maxsize-recv.test \
-	46_local-update-304-resume.test
+	44_local-update-errormask.test
 
 CLEANFILES = check-network_sh.cache

tests/crawl-test.sh

@@ -105,7 +105,7 @@ function start-crawl {
     log="${tmp}/log"
     debug starting httrack -O "${tmp}" "${moreargs[@]}" "${@:pos}"
     info "running httrack ${*:pos}"
-    httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -mrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
+    httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -omrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
     crawlpid="$!"
     debug "started cralwer on pid $crawlpid"
     wait "$crawlpid"

tests/local-crawl.sh

@@ -213,7 +213,7 @@ mkdir "$out" || die "could not create $out"
 declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
 log="${tmpdir}/log"
 info "running httrack ${hts[*]}"
-httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
+httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
 crawlpid=$!
 wait "$crawlpid"
 crawlres=$?
@@ -230,7 +230,7 @@ grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
 # --- optional second pass: re-mirror into the same dir (cache/update path) ----
 if test -n "$rerun"; then
     info "re-running httrack (update pass)"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -258,7 +258,7 @@ fi
 if test -n "$rerun_args"; then
     read -ra extra <<<"$rerun_args"
     info "re-running httrack with ${rerun_args}"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
         "${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -281,7 +281,7 @@ if test -n "$rerun_dead"; then
     wait "$serverpid" 2>/dev/null
     serverpid=
     info "re-running httrack against the stopped server"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
     crawlpid=$!
     wait "$crawlpid" || true

tests/local-server.py

@@ -790,56 +790,6 @@ class Handler(SimpleHTTPRequestHandler):
         self.send_header("Content-Length", "0")
         self.end_headers()
 
-    # C7: stall the first GET (partial + temp-ref), then answer the resume's
-    # Range with a bogus 304; httrack must drop the partial and refetch.
-    RESUME304_BODY = b"C7DATA--" + bytes((i * 7 + 3) % 256 for i in range(8192))
-    _resume304_started = False
-
-    def route_resume304_index(self):
-        self.send_html('\t<a href="blob.bin">blob</a>')
-
-    def route_resume304(self):
-        counter = os.environ.get("RESUME304_COUNTER")
-        if counter:
-            with open(counter, "a") as fp:
-                fp.write("x")
-        rng = self.headers.get("Range")
-        if not Handler._resume304_started:
-            Handler._resume304_started = True
-            self.send_response(200)
-            self.send_header("Content-Type", "application/octet-stream")
-            self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
-            self.send_header("Accept-Ranges", "bytes")
-            self.send_header("Last-Modified", BIG_LASTMOD)
-            self.send_header("ETag", '"c7"')
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.RESUME304_BODY[:4096])
-                self.wfile.flush()
-                try:
-                    while True:
-                        time.sleep(3600)
-                except OSError:
-                    pass
-            return
-        if rng is not None:  # resume request: bogus out-of-protocol 304
-            mark = os.environ.get("RESUME304_MARK")
-            if mark:
-                with open(mark, "a") as fp:
-                    fp.write("z")
-            self.send_response(304)
-            self.send_header("ETag", '"c7"')
-            self.send_header("Content-Length", "0")
-            self.end_headers()
-            return
-        # Range-less refetch after the partial is dropped: whole file.
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(self.RESUME304_BODY)
-
     # 206 resume must honor the server's Content-Range, not the offset we asked
     # for (#198): a server resuming a few bytes *before* the request must not
     # leave httrack duplicating the overlap onto the partial. flaky.bin
@@ -1154,22 +1104,6 @@ class Handler(SimpleHTTPRequestHandler):
         except OSError:
             pass
 
-    # -M received-volume cap (#520): links to large 404 bodies. httrack receives
-    # each (HTS_TOTAL_RECV climbs) but saves none, so saved stays far below -M.
-    def route_maxrecv_index(self):
-        self.send_html(
-            "".join('\t<a href="r%d.bin">r%d</a>\n' % (i, i) for i in range(16))
-        )
-
-    def route_maxrecv_404(self):
-        body = b"x" * self.BIGFILE_BYTES
-        self.send_response(404, "Not Found")
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(len(body)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(body)
-
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -1199,8 +1133,6 @@ class Handler(SimpleHTTPRequestHandler):
         "/intl/" + INTL_NAME: route_intl_page,
         "/resume/index.html": route_resume_index,
         "/resume/blob.txt": route_resume,
-        "/resume304/index.html": route_resume304_index,
-        "/resume304/blob.bin": route_resume304,
         "/overlap/index.html": route_overlap_index,
         "/overlap/flaky.bin": route_overlap,
         "/overlap/full.bin": route_overlap_full,
@@ -1274,23 +1206,6 @@ class Handler(SimpleHTTPRequestHandler):
         "/mini304/page.html": route_mini304_page,
         "/errmask/index.html": route_errmask_index,
         "/errmask/keep.dat": route_errmask_keep,
-        "/maxrecv/index.html": route_maxrecv_index,
-        "/maxrecv/r0.bin": route_maxrecv_404,
-        "/maxrecv/r1.bin": route_maxrecv_404,
-        "/maxrecv/r2.bin": route_maxrecv_404,
-        "/maxrecv/r3.bin": route_maxrecv_404,
-        "/maxrecv/r4.bin": route_maxrecv_404,
-        "/maxrecv/r5.bin": route_maxrecv_404,
-        "/maxrecv/r6.bin": route_maxrecv_404,
-        "/maxrecv/r7.bin": route_maxrecv_404,
-        "/maxrecv/r8.bin": route_maxrecv_404,
-        "/maxrecv/r9.bin": route_maxrecv_404,
-        "/maxrecv/r10.bin": route_maxrecv_404,
-        "/maxrecv/r11.bin": route_maxrecv_404,
-        "/maxrecv/r12.bin": route_maxrecv_404,
-        "/maxrecv/r13.bin": route_maxrecv_404,
-        "/maxrecv/r14.bin": route_maxrecv_404,
-        "/maxrecv/r15.bin": route_maxrecv_404,
     }
 
     # --- /big/ seeded pseudo-site ------------------------------------------