Keep LLint 64-bit on the x32 ABI

The wide-integer typedef keyed on __x86_64__, which the x32 ABI also
defines even though its long is 32-bit. That silently made LLint -- the
signed 64-bit type for byte counts and file sizes -- 32-bit on x32, so
any value past INT_MAX overflowed (a >2GB size wrapped negative and the
cache dropped the entry). Exclude __ILP32__ so x32 takes the long-long
path, plus a compile-time guard that trips wherever 64-bit support is
claimed but LLint isn't actually 64-bit.

Surfaced by the cache selftests on Debian x32: cache-writefail (oversize)
and cache-corrupt both exercise >2GB entries and failed.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -55,9 +55,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -101,8 +99,7 @@ jobs:
           sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
             -exec mv {} {}.hidden \;
           ! command -v python3
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+          make check
 
       - name: Print the test log on failure
         if: failure()
@@ -136,15 +133,7 @@ jobs:
         run: make -j"$(sysctl -n hw.ncpu)"
 
       - name: Test
-        # bigcrawl's sustained -c8 crawl drops fetches on macOS's loopback when
-        # it competes with other crawls, flaking its exact file count (the #527
-        # macOS drop). Run everything else in parallel, then bigcrawl alone (its
-        # serial-safe condition). Linux tolerates the full parallel run.
-        run: |
-          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          rest=$(cd tests && ls *.test | grep -v '^36_local-bigcrawl\.test$' | tr '\n' ' ')
-          make check -j"$jobs" TESTS="$rest"
-          make check TESTS=36_local-bigcrawl.test
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -182,9 +171,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -239,9 +226,7 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -287,8 +272,7 @@ jobs:
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
           # skipped here (uninstrumented libz floods MSan with false positives).
           tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs" TESTS="$tests"
+          make check TESTS="$tests"
 
       - name: Print the test log on failure
         if: failure()
@@ -363,9 +347,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()

AGENTS.md

@@ -6,19 +6,8 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 ## Build & test
 - Fresh clone first: `git submodule update --init src/coucal`
 - `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
-  automake, libtool), then `bash configure && make -j"$(nproc)" && make check
-  -j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
-  automake's parallel harness and each crawl test binds its own ephemeral-port
-  server, so `-j` never contends and a multi-minute serial run drops to
-  seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
-  automatically; only a test slower than the current longest raises the floor.
-  On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
-  spend much of their wall time asleep (server trickles, httrack self-pacing),
-  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)`. macOS runs
-  36_local-bigcrawl alone in a second pass: its sustained `-c8` crawl overloads
-  the macOS loopback when it competes with other crawls and flakes its exact
-  file count (Linux tolerates the full parallel run).
-  Or run `sh build.sh` to do bootstrap + configure + make in one shot.
+  automake, libtool), then `bash configure && make && make check`. Or run
+  `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

src/htsback.c

@@ -3540,20 +3540,22 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.statuscode == 406) {    // 'Not Acceptable'
                     back[i].r.statuscode = HTTP_OK;
                   }
-                  // On update, keep the good copy on error (mask as 304); skip
-                  // resume paths so a stale-partial 416 still re-fetches.
-                  if (HTTP_IS_ERROR(back[i].r.statuscode) &&
-                      back[i].is_update && !back[i].testmode &&
-                      back[i].range_req_size == 0 && back[i].url_sav[0] &&
-                      fexist_utf8(back[i].url_sav)) {
-                    hts_log_print(opt, LOG_DEBUG,
-                                  "Error %d (%s) ignored on update, keeping "
-                                  "existing copy: %s%s",
-                                  back[i].r.statuscode, back[i].r.msg,
-                                  back[i].url_adr, back[i].url_fil);
-                    back[i].r.statuscode = HTTP_NOT_MODIFIED;
-                    deletehttp(&back[i].r);
-                    back[i].r.soc = INVALID_SOCKET;
+                  // 'do not erase already downloaded file'
+                  // on an updated file
+                  // with an error : consider a 304 error
+                  if (!opt->delete_old) {
+                    if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
+                        && !back[i].testmode) {
+                      if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
+                        hts_log_print(opt, LOG_DEBUG,
+                                      "Error ignored %d (%s) because of 'no purge' option for %s%s",
+                                      back[i].r.statuscode, back[i].r.msg,
+                                      back[i].url_adr, back[i].url_fil);
+                        back[i].r.statuscode = HTTP_NOT_MODIFIED;
+                        deletehttp(&back[i].r);
+                        back[i].r.soc = INVALID_SOCKET;
+                      }
+                    }
                   }
                   // Various hacks to limit re-transfers when updating a mirror
                   // Force update if same size detected

src/htsglobal.h

@@ -56,10 +56,6 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
-// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
-#include <stdint.h>
-#include <inttypes.h>
-
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -115,6 +111,29 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
+#ifndef HTS_LONGLONG
+#ifdef SIZEOF_LONG_LONG
+#if SIZEOF_LONG_LONG == 8
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+#ifndef HTS_LONGLONG
+#ifdef __sun
+#define HTS_LONGLONG 0
+#endif
+#ifdef __osf__
+#define HTS_LONGLONG 0
+#endif
+#ifdef __linux
+#define HTS_LONGLONG 1
+#endif
+#ifdef _WIN32
+#define HTS_LONGLONG 1
+#endif
+#endif
+#endif
+
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -307,11 +326,66 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
-typedef int64_t LLint;
-typedef int64_t TStamp;
-/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
-#define LLintP "%" PRId64
+#ifndef HTS_LONGLONG
+#ifdef HTS_NO_64_BIT
+#define HTS_LONGLONG 0
+#else
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+/* Wide integer types, chosen per platform.
+   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
+           plain int where 64-bit is unavailable).
+   TStamp: timestamp/duration in the same width (a double in the no-64-bit
+           fallback).
+   LLintP: the printf conversion for an LLint. */
+#if HTS_LONGLONG
+#ifdef LLINT_FORMAT
+typedef LLINT_TYPE LLint;
+
+typedef LLINT_TYPE TStamp;
+
+#define LLintP LLINT_FORMAT
+#else
+
+#ifdef _WIN32
+typedef __int64 LLint;
+
+typedef __int64 TStamp;
+
+#define LLintP "%I64d"
+/* x32/ILP32 sets __x86_64__ but long is 32-bit; exclude it so LLint stays
+ * 64-bit. */
+#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
+       defined(__64BIT__)) &&                                                  \
+    !defined(__ILP32__)
+
+typedef long int LLint;
+
+typedef long int TStamp;
+
+#define LLintP "%ld"
+#else
+typedef long long int LLint;
+
+typedef long long int TStamp;
+
+#define LLintP "%lld"
+#endif
+
+#endif /* HTS_LONGLONG */
+
+/* Claiming long-long support must yield a real 64-bit LLint (x32 regressed:
+   __x86_64__ set but long is 32-bit). Compile-time trip, portable to C90. */
+typedef char hts_assert_llint_is_64bit[sizeof(LLint) == 8 ? 1 : -1];
+
+#else
+typedef int LLint;
+
+#define LLintP "%d"
+typedef double TStamp;
+#endif
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain

src/htslib.c

@@ -3038,11 +3038,12 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
 /* See http://physics.nist.gov/cuu/Units/binary.html */
 #define ToLLint(a) ((LLint)(a))
 #define ToLLintKiB (ToLLint(1024))
-#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
-#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#define ToLLintPiB                                                             \
-  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
+#ifdef HTS_LONGLONG
+#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#endif
 HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
   if (n < ToLLintKiB) {
     sprintf(strc->buff1, "%d", (int) (LLint) n);
@@ -3051,7 +3052,9 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
             (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
     strcpybuff(strc->buff2, "KiB");
-  } else if (n < ToLLintGiB) {
+  }
+#ifdef HTS_LONGLONG
+  else if (n < ToLLintGiB) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
             (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
     strcpybuff(strc->buff2, "MiB");
@@ -3068,6 +3071,13 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
             (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
     strcpybuff(strc->buff2, "PiB");
   }
+#else
+  else {
+    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
+            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
+    strcpybuff(strc->buff2, "MiB");
+  }
+#endif
   strc->buffadr[0] = strc->buff1;
   strc->buffadr[1] = strc->buff2;
   return strc->buffadr;

tests/36_local-bigcrawl.test

@@ -9,7 +9,7 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --errors-content 4 --files 361 \
+    --errors 4 --files 361 \
     --found 'big/p/95.html' \
     --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
     --found 'big/a/f2-2x.png' \

tests/44_local-update-errormask.test

@@ -1,13 +0,0 @@
-#!/bin/bash
-# Issue #176: on --update a page that 200'd on the first crawl but now 403s must
-# keep its good local copy, not be overwritten by the error body nor purged.
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --found 'errmask/keep.dat' \
-    --file-min-bytes 'errmask/keep.dat' 1024 \
-    --file-matches 'errmask/keep.dat' '^KEEP' \
-    --file-not-matches 'errmask/keep.dat' 'error 403' \
-    httrack 'BASEURL/errmask/index.html'

tests/Makefile.am

@@ -112,7 +112,6 @@ TESTS = \
 	40_local-why.test \
 	41_local-utf8-link.test \
 	42_local-maxsize-slow.test \
-	43_local-update-truncate.test \
-	44_local-update-errormask.test
+	43_local-update-truncate.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -14,13 +14,11 @@
 # Usage:
 #   bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
 #       [--rerun-args 'ARGS'] \
-#       --errors N --errors-content N --files N --found PATH ... --directory PATH ... \
+#       --errors N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
 #       --file-min-bytes PATH N --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
-# --errors counts every "Error:" log line; --errors-content drops transient
-# network failures (codes -2..-6) that flake on busy loopback under -c8.
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
 # --max/--min-mirror-bytes bound the mirrored content bytes (host root).
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
@@ -133,7 +131,7 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         rerun_args="${args[$pos]}"
         ;;
-    --errors | --errors-content | --files)
+    --errors | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -336,14 +334,6 @@ while test "$i" -lt "${#audit[@]}"; do
         assert_equals "checking errors" "${audit[$i]}" \
             "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
         ;;
-    --errors-content)
-        i=$((i + 1))
-        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
-        # transient network failures (statuscode -2..-6) flake on busy loopback;
-        # the code parens are followed by " at link" or " after N retries at link"
-        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
-        assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
-        ;;
     --files)
         i=$((i + 1))
         nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |

tests/local-server.py

@@ -934,33 +934,6 @@ class Handler(SimpleHTTPRequestHandler):
     def route_mini304_page(self):
         self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
 
-    # --- /errmask/: issue #176 — a page that 200'd on the first crawl but 403s
-    # on the update fetch must keep its good copy, not be overwritten nor purged.
-    ERRMASK_GOOD = b"KEEP" + b"." * 1020  # 1024 B distinctive non-HTML body
-    ERRMASK_ERR = b"<html><body>error 403</body></html>\n"
-
-    def route_errmask_index(self):
-        self.send_html('\t<a href="keep.dat">keep</a>\n')
-
-    def route_errmask_keep(self):
-        # First crawl (no validator) gets the 1024 B body + Last-Modified; the
-        # update sends a conditional and gets a 403 error page.
-        if self.headers.get("If-Modified-Since") or self.headers.get("If-None-Match"):
-            self.send_response(403, "Forbidden")
-            self.send_header("Content-Type", "text/html")
-            self.send_header("Content-Length", str(len(self.ERRMASK_ERR)))
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.ERRMASK_ERR)
-            return
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Last-Modified", BIG_LASTMOD)
-        self.send_header("Content-Length", str(len(self.ERRMASK_GOOD)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(self.ERRMASK_GOOD)
-
     # --- delayed-type degenerate paths (issues #5/#107) --------------------
     def route_delayed_index(self):
         self.send_html(
@@ -1204,8 +1177,6 @@ class Handler(SimpleHTTPRequestHandler):
         "/redir/target.html": route_redir_target,
         "/mini304/index.html": route_mini304_index,
         "/mini304/page.html": route_mini304_page,
-        "/errmask/index.html": route_errmask_index,
-        "/errmask/keep.dat": route_errmask_keep,
     }
 
     # --- /big/ seeded pseudo-site ------------------------------------------