st_cookies: assert the saved jar is non-empty

The mode assertions alone would pass a cookie_save that creates an
empty 0600 file and returns 0; check st_size on both saves (proven by
a negative control that skips the write loop).

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -55,9 +55,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -101,8 +99,7 @@ jobs:
           sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
             -exec mv {} {}.hidden \;
           ! command -v python3
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+          make check
 
       - name: Print the test log on failure
         if: failure()
@@ -136,9 +133,7 @@ jobs:
         run: make -j"$(sysctl -n hw.ncpu)"
 
       - name: Test
-        run: |
-          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -176,9 +171,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -233,9 +226,7 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -281,8 +272,7 @@ jobs:
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
           # skipped here (uninstrumented libz floods MSan with false positives).
           tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs" TESTS="$tests"
+          make check TESTS="$tests"
 
       - name: Print the test log on failure
         if: failure()
@@ -357,9 +347,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()

AGENTS.md

@@ -6,19 +6,8 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 ## Build & test
 - Fresh clone first: `git submodule update --init src/coucal`
 - `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
-  automake, libtool), then `bash configure && make -j"$(nproc)" && make check
-  -j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
-  automake's parallel harness and each crawl test binds its own ephemeral-port
-  server, so `-j` never contends and a multi-minute serial run drops to
-  seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
-  automatically; only a test slower than the current longest raises the floor.
-  On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
-  spend much of their wall time asleep (server trickles, httrack self-pacing),
-  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
-  platform, macOS included: the test server raises its listen backlog
-  (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
-  `-c16` bigcrawl the way Python's default backlog of 5 did.
-  Or run `sh build.sh` to do bootstrap + configure + make in one shot.
+  automake, libtool), then `bash configure && make && make check`. Or run
+  `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,11 +29,11 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
-# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
-# so it stays soname .so.3; bump revision.
+# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
+# installed headers (no struct layout or exported signature changed vs
+# 3.49.10), so it stays soname .so.3; bump revision.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:4:0"
+VERSION_INFO="3:3:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 

debian/changelog

@@ -1,13 +1,3 @@
-httrack (3.49.12-1) unstable; urgency=medium
-
-  * New upstream release: security and crawl-correctness fixes (remote stack
-    overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
-    world-readable cookies.txt, filter-pattern denial of service) plus a new
-    --why filter diagnostic; full list in history.txt.
-  * Build with _FORTIFY_SOURCE and -fstack-protector-strong.
-
- -- Xavier Roche <xavier@debian.org>  Fri, 10 Jul 2026 21:11:09 +0200
-
 httrack (3.49.11-1) unstable; urgency=medium
 
   * New upstream release: crawl correctness and security fixes (network-facing

fuzz/Makefile.am

@@ -36,7 +36,6 @@ EXTRA_DIST = README.md run-fuzzers.sh \
 	corpus/unescape/percent.txt \
 	corpus/filters/filter.bin corpus/filters/filter-size.bin \
 	corpus/filters/regress-empty-subject-unique.bin \
-	corpus/filters/redos-star-classes.bin \
 	corpus/url/http-url.txt corpus/url/relative-path.txt \
 	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
 	corpus/header/full-response.txt corpus/header/redirect.txt \

fuzz/corpus/meta/meta-content-first.html

@@ -1 +0,0 @@
-<meta content="text/html; charset=gb2312" http-equiv="content-type">

fuzz/corpus/meta/meta-truncated.html

@@ -1 +0,0 @@
-<meta charset='utf-8

fuzz/corpus/meta/meta-unquoted.html

@@ -1 +0,0 @@
-<meta   charset = utf-8 />

fuzz/fuzz-cachendx.c

@@ -27,7 +27,7 @@ Please visit our Website: http://www.httrack.com
 
 /* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
    walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
-   Mirrors the -#C cache-listing scan (htscoremain.c). */
+   Mirrors the loader in cache_readex_new (htscache.c). */
 #include "fuzz.h"
 #include "htscache.h"
 #include "htslib.h"
@@ -42,8 +42,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   a += cache_brstr(a, firstline, sizeof(firstline));
   a += cache_brstr(a, firstline, sizeof(firstline));
 
-  /* body: newline-delimited host/file/position triples; the length-prefixed
-     scan must stay inside the buffer */
+  /* body: newline-delimited host/file/position triples. The coucal insert
+     the real loader ends with is a separate library's concern; this harness
+     targets the length-prefixed scan that must stay inside the buffer. */
   while (a != NULL && a < end) {
     char BIGSTK line[HTS_URLMAXSIZE * 2];
     char linepos[256];

fuzz/fuzz-header.c

@@ -50,13 +50,9 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   while (p != NULL && *p != '\0') {
     char *nl = strchr(p, '\n');
     size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
-    size_t i, len = 0;
 
-    /* binput drops every '\r' on the wire; mirror it */
-    for (i = 0; i < n; i++)
-      if (p[i] != '\r')
-        line[len++] = p[i];
-    line[len] = '\0';
+    memcpy(line, p, n);
+    line[n] = '\0';
     if (first) {
       treatfirstline(&r, line);
       first = 0;

fuzz/run-fuzzers.sh

@@ -2,8 +2,8 @@
 # Drive every built harness against its seed corpus.
 #   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
 #   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
-# Replay is crash/leak-only and never mutates; the per-unit -timeout guards
-# both modes against pathological slowdowns (e.g. pre-#501 strjoker).
+# Replay is crash/leak-only and never mutates, so it can't hit strjoker's
+# catastrophic backtracking; the timed mode can, hence the per-unit -timeout.
 set -euo pipefail
 
 srcdir=$(cd "$(dirname "$0")" && pwd)

history.txt

@@ -4,23 +4,6 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
-3.49-12
-+ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
-+ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
-+ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
-+ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
-+ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
-+ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
-+ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
-+ Fixed: cookies.txt was created world-readable (#511)
-+ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
-+ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
-+ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
-+ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
-+ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
-+ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
-+ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
-
 3.49-11
 + New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
 + New: advertise deflate in Accept-Encoding and decode deflate responses (#450)

src/htsback.c

@@ -68,15 +68,6 @@ static int slot_can_be_cached_on_disk(const lien_back * back);
 static int slot_can_be_cleaned(const lien_back * back);
 static int slot_can_be_finalized(httrackp * opt, const lien_back * back);
 
-/* Which hard quota, if any, is currently aborting the mirror. */
-typedef enum {
-  HTS_MIRROR_LIMIT_NONE = 0,
-  HTS_MIRROR_LIMIT_SIZE,
-  HTS_MIRROR_LIMIT_TIME,
-} hts_mirror_limit;
-
-static hts_mirror_limit back_mirror_limit(httrackp *opt);
-
 struct_back *back_new(httrackp *opt, int back_max) {
   int i;
   struct_back *sback = calloct(1, sizeof(struct_back));
@@ -533,22 +524,22 @@ int back_nsoc_overall(const struct_back * sback) {
 
 /* generate temporary file on lien_back */
 /* Note: utf-8 */
-static int create_back_tmpfile(httrackp *opt, lien_back *const back,
-                               const char *ext) {
+static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
   // do not use tempnam() but a regular filename
   back->tmpfile_buffer[0] = '\0';
   if (back->url_sav != NULL && back->url_sav[0] != '\0') {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
-             back->url_sav, ext);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.z", 
+             back->url_sav);
     back->tmpfile = back->tmpfile_buffer;
     if (structcheck(back->tmpfile) != 0) {
-      hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
+      hts_log_print(opt, LOG_WARNING, "can not create directory to %s", 
                     back->tmpfile);
       return -1;
     }
   } else {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
-             StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer),
+             "%s/tmp%d.z", StringBuff(opt->path_html_utf8),
+             opt->state.tmpnameid++);
     back->tmpfile = back->tmpfile_buffer;
   }
   /* OK */
@@ -556,32 +547,6 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
   return 0;
 }
 
-/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
-   existing file moved the good copy to back->tmpfile before truncating url_sav.
-   commit keeps the new file and drops the backup; else restore it so an aborted
-   transfer leaves the previous copy intact. Skips the zlib .z temp. */
-static void back_finalize_backup(httrackp *opt, lien_back *const back,
-                                 const hts_boolean commit) {
-  if (back->tmpfile == NULL || back->r.compressed)
-    return;
-  if (commit) {
-    (void) UNLINK(back->tmpfile); /* new copy is good; drop the backup */
-  } else {
-    if (back->r.out != NULL) {
-      fclose(back->r.out);
-      back->r.out = NULL;
-    }
-    (void) UNLINK(back->url_sav); /* drop the failed partial */
-    /* On rename failure keep the backup: it still holds the good copy, so
-       losing it would be worse than an orphaned temp. */
-    if (RENAME(back->tmpfile, back->url_sav) != 0)
-      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
-                    "could not restore %s; previous copy kept as %s",
-                    back->url_sav, back->tmpfile);
-  }
-  back->tmpfile = NULL;
-}
-
 // objet (lien) téléchargé ou transféré depuis le cache
 //
 // fermer les paramètres de transfert,
@@ -617,7 +582,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                       LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
                       back[p].r.size, back[p].url_adr, back[p].url_fil);
       }
-      back_finalize_backup(opt, &back[p], HTS_FALSE);
       return -1;
     }
 
@@ -636,7 +600,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
             // en mémoire -> passage sur disque
             if (!back[p].r.is_write) {
               // do not use tempnam() but a regular filename
-              if (create_back_tmpfile(opt, &back[p], "z") == 0) {
+              if (create_back_tmpfile(opt, &back[p]) == 0) {
                 assertf(back[p].tmpfile != NULL);
                 /* note: tmpfile is utf-8 */
                 back[p].r.out = FOPEN(back[p].tmpfile, "wb");
@@ -709,9 +673,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
           }
         }
 #endif
-        /* Body fully received: keep the freshly written url_sav, drop the
-           backup of the previous copy. */
-        back_finalize_backup(opt, &back[p], HTS_TRUE);
         /* Write mode to disk */
         if (back[p].r.is_write && back[p].r.adr != NULL) {
           freet(back[p].r.adr);
@@ -940,9 +901,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
       }
     }
   }
-  /* Aborted, error, or not ready: url_sav (if written) is broken; restore the
-     previous copy from the backup. */
-  back_finalize_backup(opt, &back[p], HTS_FALSE);
   return -1;
 }
 
@@ -1688,104 +1646,138 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       }
     }
     // tester cache
-    if ((strcmp(adr, "file://")) /* pas fichier */
-        && ((!test) ||
-            (cache->type == 1)) /* cache prioritaire, laisser passer en test! */
-        &&
-        ((strnotempty(save)) || (strcmp(fil, "/robots.txt") ==
-                                 0))) { // si en test on ne doit pas utiliser le
-                                        // cache sinon telescopage avec le 302..
+    if ((strcmp(adr, "file://"))        /* pas fichier */
+        &&((!test) || (cache->type == 1))       /* cache prioritaire, laisser passer en test! */
+        &&((strnotempty(save)) || (strcmp(fil, "/robots.txt") == 0))) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
+#if HTS_FAST_CACHE
       intptr_t hash_pos;
       int hash_pos_return = 0;
-
+#else
+      char *a = NULL;
+#endif
+#if HTS_FAST_CACHE
       if (cache->hashtable) {
+#else
+      if (cache->use) {
+#endif
         char BIGSTK buff[HTS_URLMAXSIZE * 4];
 
+#if HTS_FAST_CACHE
         strcpybuff(buff, adr);
         strcatbuff(buff, fil);
         hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
+#else
+        buff[0] = '\0';
+        strcatbuff(buff, "\n");
+        strcatbuff(buff, adr);
+        strcatbuff(buff, "\n");
+        strcatbuff(buff, fil);
+        strcatbuff(buff, "\n");
+        a = strstr(cache->use, buff);
+#endif
 
+        // Ok, noté en cache->. mais bien présent dans le cache ou sur disque?
+#if HTS_FAST_CACHE
         // negative values when data is not in cache
         if (hash_pos_return < 0) {
-          if (!test) { // not test mode
-            /* note: no check with IS_DELAYED_EXT() enabled - postcheck by
-             * client please! */
-            if (save[0] != '\0' && !IS_DELAYED_EXT(save) &&
-                fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <=
-                    0) { // final file missing or empty
-              int found = 0;
+#else
+        if (a) {
+#endif
+          if (!test) {          // non mode test
+#if HTS_FAST_CACHE==0
+            int pos = -1;
 
-              /* It is possible that the file has been moved due to changes in
-               * build structure */
-              {
-                char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
-                htsblk r;
+            a += strlen(buff);
+            sscanf(a, "%d", &pos);      // lire position
+#endif
 
-                previous_save[0] = '\0';
-                r = cache_readex(opt, cache, adr, fil, /*head */ NULL,
+#if HTS_FAST_CACHE==0
+            if (pos < 0) {      // pas de mise en cache data, vérifier existence
+#endif
+              /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
+              if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <= 0) {  // fichier final n'existe pas ou est vide!
+                int found = 0;
+
+                /* It is possible that the file has been moved due to changes in build structure */
+                {
+                  char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
+                  htsblk r;
+
+                  previous_save[0] = '\0';
+                  r =
+                    cache_readex(opt, cache, adr, fil, /*head */ NULL,
                                  /*bound to back[p] (temporary) */
                                  back[p].location_buffer, previous_save, /*ro */
                                  1);
-                /* Is supposed to be on disk only */
-                if (r.is_write && previous_save[0] != '\0') {
-                  /* Exists, but with another (old) filename: rename (almost)
-                   * silently */
-                  if (strcmp(previous_save, save) != 0 &&
-                      fexist_utf8(
-                          fconv(catbuff, sizeof(catbuff), previous_save))) {
-                    rename(fconv(catbuff, sizeof(catbuff), previous_save),
-                           fconv(catbuff2, sizeof(catbuff2), save));
-                    if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
-                      found = 1;
-                      hts_log_print(opt, LOG_DEBUG,
-                                    "File '%s' has been renamed since last "
-                                    "mirror to '%s' ; applying changes",
-                                    previous_save, save);
-                    } else {
-                      hts_log_print(opt, LOG_ERROR,
-                                    "Could not rename '%s' to '%s' ; will have "
-                                    "to retransfer it",
-                                    previous_save, save);
+                  /* Is supposed to be on disk only */
+                  if (r.is_write && previous_save[0] != '\0') {
+                    /* Exists, but with another (old) filename: rename (almost) silently */
+                    if (strcmp(previous_save, save) != 0
+                        && fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) {
+                      rename(fconv(catbuff, sizeof(catbuff), previous_save),
+                             fconv(catbuff2, sizeof(catbuff2), save));
+                      if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
+                        found = 1;
+                        hts_log_print(opt, LOG_DEBUG,
+                                      "File '%s' has been renamed since last mirror to '%s' ; applying changes",
+                                      previous_save, save);
+                      } else {
+                        hts_log_print(opt, LOG_ERROR,
+                                      "Could not rename '%s' to '%s' ; will have to retransfer it",
+                                      previous_save, save);
+                      }
                     }
                   }
+                  back[p].location_buffer[0] = '\0';
                 }
-                back[p].location_buffer[0] = '\0';
-              }
 
-              /* Not found ? */
-              if (!found) {
-                // invalidate: gone from disk, force a refetch
-                hash_pos_return = 0;
-                if (opt->norecatch) {
-                  if (!fexist_utf8(fconv(
-                          catbuff, sizeof(catbuff),
-                          save))) { // declared but missing: user erased it
-                    FILE *fp =
-                        FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
+                /* Not found ? */
+                if (!found) {
+#if HTS_FAST_CACHE
+                  hash_pos_return = 0;
+#else
+                  a = NULL;
+#endif
+                  // dévalider car non présent sur disque dans structure originale!!!
+                  // sinon, le fichier est ok à priori, mais on renverra un if-modified-since pour
+                  // en être sûr
+                  if (opt->norecatch) { // tester norecatch
+                    if (!fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {   // fichier existe pas mais déclaré: on l'a effacé
+                      FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
 
-                    if (fp)
-                      fclose(fp);
+                      if (fp)
+                        fclose(fp);
+                      hts_log_print(opt, LOG_WARNING,
+                                    "Previous file '%s' not found (erased by user ?), ignoring: %s%s",
+                                    save, back[p].url_adr, back[p].url_fil);
+                    }
+                  } else {
                     hts_log_print(opt, LOG_WARNING,
-                                  "Previous file '%s' not found (erased by "
-                                  "user ?), ignoring: %s%s",
+                                  "Previous file '%s' not found (erased by user ?), recatching: %s%s",
                                   save, back[p].url_adr, back[p].url_fil);
                   }
-                } else {
-                  hts_log_print(opt, LOG_WARNING,
-                                "Previous file '%s' not found (erased by user "
-                                "?), recatching: %s%s",
-                                save, back[p].url_adr, back[p].url_fil);
                 }
-              }
-            } // fsize() <= 0
+              }                 // fsize() <= 0
+#if HTS_FAST_CACHE==0
+            }
+#endif
           }
         }
         //
       } else {
+#if HTS_FAST_CACHE
         hash_pos_return = 0;
+#else
+        a = NULL;
+#endif
       }
 
-      if (hash_pos_return) { // in cache, with data
+      // Existe pas en cache, ou bien pas de cache présent
+#if HTS_FAST_CACHE
+      if (hash_pos_return) {    // OK existe en cache (et données aussi)!
+#else
+      if (a != NULL) {          // OK existe en cache (et données aussi)!
+#endif
         const int cache_is_prioritary = cache->type == 1
           || opt->state.stop != 0;
         if (cache_is_prioritary) {      // cache prioritaire (pas de test if-modified..)
@@ -1906,8 +1898,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #endif
         }
       }
-      /* Not in cache ; maybe in temporary cache ? Warning: non-movable
-         "url_sav" */
+      /* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */
       else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
         const off_t file_size = fsize_utf8(itemback->url_sav);
 
@@ -2092,9 +2083,13 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
       if (!back_trylive(opt, cache, sback, p)) {
 #if HTS_XGETHOST
-        back[p].status = STATUS_WAIT_DNS; // host name resolution attempt
-        soc = INVALID_SOCKET;             // not opened yet
-        if (host_wait(opt, &back[p])) {   // ready (file, or cached dns)
+#if HDEBUG
+        printf("back_solve..\n");
+#endif
+        back[p].status = STATUS_WAIT_DNS;       // tentative de résolution du nom de host
+        soc = INVALID_SOCKET;   // pas encore ouverte
+        back_solve(opt, &back[p]);      // préparer
+        if (host_wait(opt, &back[p])) { // prêt, par ex fichier ou dispo dans dns
 #if HDEBUG
           printf("ok, dns cache ready..\n");
 #endif
@@ -2237,9 +2232,37 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 }
 
 #if HTS_XGETHOST
-// Resolution is synchronous inside the connect path; no pre-resolve step, so
-// the host is always immediately ready.
-int host_wait(httrackp *opt, lien_back *back) { return 1; }
+// attendre que le host (ou celui du proxy) ait été résolu
+// si c'est un fichier, la résolution est immédiate
+// idem pour ftp://
+void back_solve(httrackp * opt, lien_back * back) {
+  assertf(opt != NULL);
+  assertf(back != NULL);
+  if ((!strfield(back->url_adr, "file://"))
+      && !strfield(back->url_adr, "ftp://")
+    ) {
+    const char *a;
+
+    if (!(back->r.req.proxy.active))
+      a = back->url_adr;
+    else
+      a = back->r.req.proxy.name;
+    assertf(a != NULL);
+    a = jump_protocol_const(a);
+    if (check_hostname_dns(a)) {
+      hts_log_print(opt, LOG_DEBUG, "resolved: %s", a);
+    } else {
+      hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
+    }
+  }
+}
+
+// détermine si le host a pu être résolu
+int host_wait(httrackp * opt, lien_back * back) {
+  // Always synchronous. No more background DNS resolution
+  // (does not really improve performances)
+  return 1;
+}
 #endif
 
 // élimine les fichiers non html en backing (anticipation)
@@ -2426,15 +2449,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
-  /* Time/size limit exceeded past grace: abort in-flight transfers so no wait
-     loop starves (#481, #77). FTP slots stay, their thread owns the socket. */
+  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
+     starves (#481). FTP slots stay, their thread owns the socket. */
   if (!back_checkmirror(opt)) {
-    const hts_mirror_limit limit = back_mirror_limit(opt);
-    const char *const reason =
-        (limit == HTS_MIRROR_LIMIT_SIZE) ? "size limit" : "time limit";
-    const char *const slotmsg = (limit == HTS_MIRROR_LIMIT_SIZE)
-                                    ? "Mirror Size Limit"
-                                    : "Mirror Time Out";
     int aborted = 0;
     unsigned int i;
 
@@ -2448,15 +2465,15 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
           back_delayed_discard(opt, &back[i]);
         back[i].r.statuscode = STATUSCODE_TIMEOUT;
-        strcpybuff(back[i].r.msg, slotmsg);
+        strcpybuff(back[i].r.msg, "Mirror Time Out");
         back[i].status = STATUS_READY;
         back_set_finished(sback, i);
         aborted++;
       }
     }
     if (aborted > 0)
-      hts_log_print(opt, LOG_WARNING, "%s reached, %d transfer(s) aborted",
-                    reason, aborted);
+      hts_log_print(opt, LOG_WARNING,
+                    "time limit reached, %d transfer(s) aborted", aborted);
     return;
   }
 
@@ -2946,8 +2963,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                 strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "gz") == 0
                                 && strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "tgz") == 0
                               ) {
-                              if (create_back_tmpfile(opt, &back[i], "z") ==
-                                  0) {
+                              if (create_back_tmpfile(opt, &back[i]) == 0) {
                                 assertf(back[i].tmpfile != NULL);
                                 /* note: tmpfile is utf-8 */
                                 if ((back[i].r.out =
@@ -2960,20 +2976,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
-                              /* Re-fetch over an existing file (#77 follow-up):
-                                 move the good copy aside before truncating it
-                                 so an aborted transfer can restore it. url_sav
-                                 is still written normally (file list intact).
-                               */
-                              back[i].tmpfile = NULL;
-                              if (fexist_utf8(back[i].url_sav)) {
-                                if (create_back_tmpfile(opt, &back[i], "bak") !=
-                                        0 ||
-                                    RENAME(back[i].url_sav, back[i].tmpfile) !=
-                                        0) {
-                                  back[i].tmpfile = NULL;
-                                }
-                              }
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {
@@ -3512,11 +3514,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     continue;
                   }
 
-                  // The server really sent 304 here; the *-hacks below force
-                  // NOT_MODIFIED only after confirming the file is complete.
-                  const hts_boolean server_sent_304 =
-                      (back[i].r.statuscode == HTTP_NOT_MODIFIED);
-
                   /*
                      Solve "false" 416 problems
                    */
@@ -3545,20 +3542,22 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.statuscode == 406) {    // 'Not Acceptable'
                     back[i].r.statuscode = HTTP_OK;
                   }
-                  // On update, keep the good copy on error (mask as 304); skip
-                  // resume paths so a stale-partial 416 still re-fetches.
-                  if (HTTP_IS_ERROR(back[i].r.statuscode) &&
-                      back[i].is_update && !back[i].testmode &&
-                      back[i].range_req_size == 0 && back[i].url_sav[0] &&
-                      fexist_utf8(back[i].url_sav)) {
-                    hts_log_print(opt, LOG_DEBUG,
-                                  "Error %d (%s) ignored on update, keeping "
-                                  "existing copy: %s%s",
-                                  back[i].r.statuscode, back[i].r.msg,
-                                  back[i].url_adr, back[i].url_fil);
-                    back[i].r.statuscode = HTTP_NOT_MODIFIED;
-                    deletehttp(&back[i].r);
-                    back[i].r.soc = INVALID_SOCKET;
+                  // 'do not erase already downloaded file'
+                  // on an updated file
+                  // with an error : consider a 304 error
+                  if (!opt->delete_old) {
+                    if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
+                        && !back[i].testmode) {
+                      if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
+                        hts_log_print(opt, LOG_DEBUG,
+                                      "Error ignored %d (%s) because of 'no purge' option for %s%s",
+                                      back[i].r.statuscode, back[i].r.msg,
+                                      back[i].url_adr, back[i].url_fil);
+                        back[i].r.statuscode = HTTP_NOT_MODIFIED;
+                        deletehttp(&back[i].r);
+                        back[i].r.soc = INVALID_SOCKET;
+                      }
+                    }
                   }
                   // Various hacks to limit re-transfers when updating a mirror
                   // Force update if same size detected
@@ -3611,7 +3610,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) {       // 'OK'
                         if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) {  // not HTML
                           if (strnotempty(back[i].url_sav)) {   // target found
-                            off_t size = fsize_utf8(back[i].url_sav);
+                            int size = fsize_utf8(back[i].url_sav);     // target size
 
                             if (size >= 0) {
                               if (back[i].r.totalsize == size) {        // same size!
@@ -3706,21 +3705,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     }
                   }
 
-                  // Out-of-protocol 304 to a Range resume: the file is still
-                  // partial, so drop it and refetch instead of trusting it.
-                  if (server_sent_304 && back[i].range_req_size > 0) {
-                    url_savename_refname_remove(opt, back[i].url_adr,
-                                                back[i].url_fil);
-                    UNLINK(back[i].url_sav);
-                    deletehttp(&back[i].r);
-                    back[i].r.soc = INVALID_SOCKET;
-                    back[i].r.statuscode = STATUSCODE_NON_FATAL;
-                    strcpybuff(back[i].r.msg,
-                               "Bogus 304 on resume, restarting");
-                    back[i].status = STATUS_READY;
-                    back_set_finished(sback, i);
-                  }
-
                   /* sinon, continuer */
                   /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok récupérer body? */
                   // head: terminé
@@ -3839,7 +3823,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       const hts_boolean range_ok =
                           back[i].r.crange > 0 && resume >= 0 &&
                           resume <= (LLint) sz &&
-                          back[i].r.crange_end == back[i].r.crange - 1 &&
+                          back[i].r.crange_end + 1 == back[i].r.crange &&
                           (back[i].r.totalsize < 0 ||
                            back[i].r.totalsize ==
                                back[i].r.crange_end - resume + 1);
@@ -3908,20 +3892,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           if (fp) {
                             LLint alloc_mem = resume + 1;
 
-                            // Bound the in-memory buffer to a 32-bit size (real
-                            // in-RAM resources are far smaller); a hostile
-                            // Content-Length that would overflow the add or the
-                            // (size_t) cast is dropped and refetched instead.
-                            if (back[i].r.totalsize > INT32_MAX - alloc_mem) {
-                              url_savename_refname_remove(opt, back[i].url_adr,
-                                                          back[i].url_fil);
-                              UNLINK(back[i].url_sav);
-                              alloc_mem = -1;
-                            } else if (back[i].r.totalsize >= 0)
+                            if (back[i].r.totalsize >= 0)
                               alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
-                            if (alloc_mem >= 0 && deleteaddr(&back[i].r) &&
-                                (back[i].r.adr =
-                                     (char *) malloct((size_t) alloc_mem))) {
+                            if (deleteaddr(&back[i].r)
+                                && (back[i].r.adr =
+                                    (char *) malloct((size_t) alloc_mem))) {
                               back[i].r.size = resume;
                               if (back[i].r.totalsize >= 0)
                                 back[i].r.totalsize += resume; // -> full size
@@ -4203,46 +4178,38 @@ static int back_maxtime_grace(const int maxtime) {
   return maximum(5, minimum(30, maxtime / 10));
 }
 
-/* Bytes the smooth stop may overrun before in-flight transfers are aborted.
-   No floor (unlike the time grace): a size overrun should abort promptly. */
-static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
-
-/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481).
-   -M measures received volume (HTS_TOTAL_RECV), not saved 200-only stat_bytes
-   which undercounts redirect/error-heavy crawls (#520). */
-static hts_mirror_limit back_mirror_limit(httrackp *opt) {
-  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
-      HTS_STAT.HTS_TOTAL_RECV - opt->maxsite >=
-          back_maxsize_grace(opt->maxsite))
-    return HTS_MIRROR_LIMIT_SIZE;
+int back_checkmirror(httrackp * opt) {
+  // Check max size
+  if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
+    if (!opt->state.stop) {     /* not yet stopped */
+      hts_log_print(opt, LOG_ERROR,
+                    "More than " LLintP
+                    " bytes have been transferred.. giving up",
+                    (LLint) opt->maxsite);
+      /* cancel mirror smoothly */
+      hts_request_stop(opt, 0);
+    }
+    return 1;                   /* don'k break mirror too sharply for size limits, but stop requested */
+    /*return 0;
+     */
+  }
+  // Check max time
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
 
-    if (elapsed >= opt->maxtime &&
-        elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
-      return HTS_MIRROR_LIMIT_TIME;
+    if (elapsed >= opt->maxtime) {
+      if (!opt->state.stop) { /* not yet stopped */
+        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                      opt->maxtime);
+        /* cancel mirror smoothly */
+        hts_request_stop(opt, 0);
+      }
+      /* smooth stop starved past the grace period: stop waiting (#481) */
+      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+        return 0;
+    }
   }
-  return HTS_MIRROR_LIMIT_NONE;
-}
-
-int back_checkmirror(httrackp *opt) {
-  /* request a smooth stop the first time each cap is reached */
-  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
-      !opt->state.stop) {
-    hts_log_print(opt, LOG_ERROR,
-                  "More than " LLintP
-                  " bytes have been transferred.. giving up",
-                  (LLint) opt->maxsite);
-    hts_request_stop(opt, 0);
-  }
-  if (opt->maxtime > 0 && !opt->state.stop &&
-      (time_local() - HTS_STAT.stat_timestart) >= opt->maxtime) {
-    hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-                  opt->maxtime);
-    hts_request_stop(opt, 0);
-  }
-  /* hard stop once a cap overruns its grace (callers must stop waiting) */
-  return back_mirror_limit(opt) == HTS_MIRROR_LIMIT_NONE;
+  return 1;                     /* Ok, go on */
 }
 
 // octets transférés + add

src/htsback.h

@@ -138,11 +138,12 @@ LLint back_transferred(LLint add, struct_back * sback);
 
 // hostback
 #if HTS_XGETHOST
+void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
-/* Enforce -M/-E quotas: smooth-stops when reached; returns 0 once the -M cap
-   or -E deadline overruns its grace period (callers must stop waiting). */
+/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
+   the -E deadline overran its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htscache.h

@@ -89,6 +89,11 @@ typedef enum {
    the involved files are absent. */
 void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
 
+int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
+                    const char *str2, char *outbuff, int len);
+int cache_readdata(cache_back * cache, const char *str1, const char *str2,
+                   char **inbuff, int *len);
+
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);

src/htscache_selftest.c

@@ -78,6 +78,14 @@ static void selftest_open_for_read(cache_back *cache, httrackp *opt) {
 }
 
 static void selftest_close(cache_back *cache) {
+  if (cache->dat != NULL) {
+    fclose(cache->dat);
+    cache->dat = NULL;
+  }
+  if (cache->ndx != NULL) {
+    fclose(cache->ndx);
+    cache->ndx = NULL;
+  }
   if (cache->zipOutput != NULL) {
     zipClose(cache->zipOutput,
              "Created by HTTrack Website Copier (cache self-test)");
@@ -981,15 +989,26 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
 
-  /* PROMOTE: the removed pre-3.31 legacy pair is left alone */
+  /* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
+     zip cache existed) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", TINY);
   hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
+
+  /* PROMOTE: a half-written legacy new pair is replaced by the old pair */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
 
   /* INTERRUPTED: no lock file, no action */
   reconcile_wipe(opt);
@@ -1039,7 +1058,7 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
 
-  /* INTERRUPTED: the removed pre-3.31 legacy pair is left alone */
+  /* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-in_progress.lock", 0);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
@@ -1048,13 +1067,9 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
   failures +=
-      reconcile_expect(opt, "hts-cache/new.dat", TINY, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
   failures +=
-      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.dat", SOLID, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.ndx", MID, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
 
   /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
      its only good generation here) */
@@ -1074,18 +1089,17 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
 
-  /* ROLLBACK: sidecars restored, the removed pre-3.31 pair left alone */
+  /* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   reconcile_put(opt, "hts-cache/old.lst", MID);
   reconcile_put(opt, "hts-cache/old.txt", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", TINY, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
 
@@ -1099,101 +1113,6 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   return failures;
 }
 
-/* The pre-3.31 .dat/.ndx import was removed: cache_init must refuse the
-   legacy pair, leave the files in place, and not flag an update run. */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-  int variant;
-
-  selftest_tag = "cache-legacy";
-  golden_setup(opt, dir);
-#ifdef _WIN32
-  mkdir(reconcile_st_path(opt, "hts-cache"));
-#else
-  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
-#endif
-
-  /* variant 0: old.dat/old.ndx (--update layout); 1: new.dat/new.ndx (ro) */
-  for (variant = 0; variant < 2; variant++) {
-    cache_back cache;
-    const char *const dat =
-        variant == 0 ? "hts-cache/old.dat" : "hts-cache/new.dat";
-    const char *const ndx =
-        variant == 0 ? "hts-cache/old.ndx" : "hts-cache/new.ndx";
-
-    reconcile_wipe(opt);
-    reconcile_put(opt, dat, 4096);
-    reconcile_put(opt, ndx, 4096);
-    opt->is_update = 0;
-    selftest_open(&cache, opt, /*ro*/ variant == 1);
-    if (cache_readable(&cache)) {
-      printf("cache-legacy: variant %d imported a removed format\n", variant);
-      failures++;
-    }
-    if (coucal_nitems(cache.hashtable) != 0) { /* no phantom index entries */
-      printf("cache-legacy: variant %d imported index entries\n", variant);
-      failures++;
-    }
-    if (opt->is_update) {
-      printf("cache-legacy: variant %d flagged an update\n", variant);
-      failures++;
-    }
-    if (fsize(reconcile_st_path(opt, dat)) != 4096 ||
-        fsize(reconcile_st_path(opt, ndx)) != 4096) {
-      printf("cache-legacy: variant %d touched the legacy files\n", variant);
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  /* a healthy zip must win over stray legacy files, with no refusal */
-  {
-    cache_back cache;
-    const char *const body = "<html>zip wins</html>";
-
-    reconcile_wipe(opt);
-    selftest_open_for_write(&cache, opt);
-    store_entry(opt, &cache, "example.com", "/", "example.com/index.html", 200,
-                "OK", "text/html", "utf-8", "", "", "", "", body, strlen(body));
-    selftest_close(&cache);
-    reconcile_put(opt, "hts-cache/old.ndx", 4096);
-    reconcile_put(opt, "hts-cache/old.dat", 4096);
-    selftest_open_for_read(&cache, opt);
-    if (!cache_readable(&cache)) {
-      printf("cache-legacy: stray legacy files shadowed the zip cache\n");
-      failures++;
-    }
-    failures +=
-        check_entry(opt, &cache, "example.com", "/", 200, "OK", "text/html",
-                    "utf-8", "", "", "", "", body, strlen(body));
-    if (fsize(reconcile_st_path(opt, "hts-cache/old.ndx")) != 4096 ||
-        fsize(reconcile_st_path(opt, "hts-cache/old.dat")) != 4096) {
-      printf("cache-legacy: zip-wins pass touched the legacy files\n");
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  reconcile_wipe(opt);
-  return failures;
-}
-
 /* --- read-side corruption injection --------------------------------------- */
 
 /* canary read back intact after each corruption; victim gets the byte surgery

src/htscache_selftest.h

@@ -60,10 +60,6 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir);
    under <dir>. Returns the failed-check count. */
 int cache_reconcile_selftest(httrackp *opt, const char *dir);
 
-/* Verify cache_init refuses a pre-3.31 .dat/.ndx cache without touching it.
-   Returns the number of failed checks (0 = pass). */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir);
-
 /* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
    under <dir> and assert every case degrades to STATUSCODE_INVALID without
    tainting a sibling entry. */

src/htscharset.c

@@ -84,7 +84,7 @@ static int hts_equalsAlphanum(const char *a, const char *b) {
 
 /* Copy the memory region [s .. s + size - 1 ] as a \0-terminated string. */
 static char *hts_stringMemCopy(const char *s, size_t size) {
-  char *dest = malloct(size + 1);
+  char *dest = malloc(size + 1);
 
   if (dest != NULL) {
     memcpy(dest, s, size);
@@ -549,6 +549,42 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
 
 #endif
 
+HTS_STATIC char *hts_getCharsetFromContentType(const char *mime) {
+  /* text/html; charset=utf-8 */
+  const char *const charset = "charset";
+  char *pos = strstr(mime, charset);
+
+  if (pos != NULL) {
+    /* Skip spaces */
+    int eq = 0;
+
+    for(pos += strlen(charset);
+        *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\''; pos++) {
+      if (*pos == '=') {
+        eq = 1;
+      }
+    }
+    if (eq == 1) {
+      int len;
+
+      for(len = 0;
+          pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'';
+          pos++) ;
+      if (len != 0) {
+        char *const s = malloc(len + 1);
+        int i;
+
+        for(i = 0; i < len; i++) {
+          s[i] = pos[i];
+        }
+        s[len] = '\0';
+        return s;
+      }
+    }
+  }
+  return NULL;
+}
+
 #ifdef _WIN32
 #define strcasecmp(a,b) stricmp(a,b)
 #define strncasecmp(a,b,n) strnicmp(a,b,n)
@@ -608,106 +644,58 @@ int hts_isCharsetUTF8(const char *charset) {
          || strcasecmp(charset, "utf8") == 0 );
 }
 
-/* Extract X from a "text/html; charset=X" content= attribute value. */
-static char *charset_from_content(const char *s, size_t len) {
-  size_t i;
-
-  for (i = 0; i + 7 < len; i++) {
-    if ((i == 0 || is_space(s[i - 1]) || s[i - 1] == ';') &&
-        strncasecmp(&s[i], "charset", 7) == 0 && is_space_or_equal(s[i + 7])) {
-      size_t j, val;
-
-      for (j = i + 7; j < len && is_space_or_equal_or_quote(s[j]); j++)
-        ;
-      for (val = j; j < len && s[j] != '"' && s[j] != '\'' && s[j] != ';' &&
-                    !is_space(s[j]);
-           j++)
-        ;
-      if (j != val) {
-        return hts_stringMemCopy(&s[val], j - val);
-      }
-    }
-  }
-  return NULL;
-}
-
 char *hts_getCharsetFromMeta(const char *html, size_t size) {
-  size_t i;
+  int i;
 
-  /* HTML5 <meta charset=X> or legacy
-     <meta http-equiv="Content-Type" content="text/html; charset=X">,
-     attributes in any order; first resolvable tag wins. */
-  for (i = 0; i + 6 < size; i++) {
-    size_t j;
-    const char *equiv = NULL, *content = NULL;
-    size_t equiv_len = 0, content_len = 0;
+  /* <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8" > */
+  for(i = 0; i < size; i++) {
+    if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0
+        && is_space(html[i + 5])) {
+      /* Skip spaces */
+      for(i += 5; is_space(html[i]); i++) ;
+      if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0
+          && is_space_or_equal(html[i + 10])) {
+        for(i += 10; is_space_or_equal_or_quote(html[i]); i++) ;
+        if (strncasecmp(&html[i], "CONTENT-TYPE", 12) == 0) {
+          for(i += 12; is_space_or_equal_or_quote(html[i]); i++) ;
+          if (strncasecmp(&html[i], "CONTENT", 7) == 0
+              && is_space_or_equal(html[i + 7])) {
+            for(i += 7; is_space_or_equal_or_quote(html[i]); i++) ;
+            /* Skip content-type */
+            for(;
+                i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'';
+                i++) ;
+            /* Expect charset attribute here */
+            if (html[i] == ';') {
+              for(i++; is_space(html[i]); i++) ;
+              /* Look for charset */
+              if (strncasecmp(&html[i], "charset", 7) == 0
+                  && is_space_or_equal(html[i + 7])) {
+                int len;
 
-    if (html[i] != '<' || strncasecmp(&html[i + 1], "meta", 4) != 0 ||
-        !is_space(html[i + 5])) {
-      continue;
-    }
-    /* Attribute scan, strictly bounded by size. */
-    for (j = i + 6; j < size && html[j] != '>';) {
-      size_t name, name_len, val = 0, val_len = 0;
+                for(i += 7; is_space_or_equal(html[i]) || html[i] == '\'';
+                    i++) ;
+                /* Charset */
+                for(len = 0;
+                    i + len < size && html[i + len] != '"'
+                    && html[i + len] != '\'' && html[i + len] != ' '; len++) ;
+                /* No error ? */
+                if (len != 0 && i < size) {
+                  char *const s = malloc(len + 1);
+                  int j;
 
-      if (is_space(html[j]) || html[j] == '/') {
-        j++;
-        continue;
-      }
-      for (name = j; j < size && html[j] != '=' && html[j] != '>' &&
-                     html[j] != '/' && !is_space(html[j]);
-           j++)
-        ;
-      name_len = j - name;
-      for (; j < size && is_space(html[j]); j++)
-        ;
-      if (j < size && html[j] == '=') {
-        for (j++; j < size && is_space(html[j]); j++)
-          ;
-        if (j < size && (html[j] == '"' || html[j] == '\'')) {
-          const char quote = html[j++];
-
-          for (val = j; j < size && html[j] != quote; j++)
-            ;
-          if (j >= size) /* unterminated quote: drop the tag */
-            break;
-          val_len = j++ - val;
-        } else {
-          /* unquoted: ends at whitespace or '>' only (HTML5 prescan); '/'
-             belongs to the value except as the tail of a self-close */
-          for (val = j; j < size && !is_space(html[j]) && html[j] != '>'; j++)
-            ;
-          val_len = j - val;
-          if (val_len != 0 && j < size && html[j] == '>' &&
-              html[val + val_len - 1] == '/') {
-            val_len--;
+                  for(j = 0; j < len; j++) {
+                    s[j] = html[i + j];
+                  }
+                  s[len] = '\0';
+                  return s;
+                }
+              }
+            }
           }
         }
       }
-      /* first occurrence of each attribute wins, as in browsers */
-      if (val_len != 0) {
-        if (name_len == 7 && strncasecmp(&html[name], "charset", 7) == 0) {
-          return hts_stringMemCopy(&html[val], val_len);
-        } else if (equiv == NULL && name_len == 10 &&
-                   strncasecmp(&html[name], "http-equiv", 10) == 0) {
-          equiv = &html[val];
-          equiv_len = val_len;
-        } else if (content == NULL && name_len == 7 &&
-                   strncasecmp(&html[name], "content", 7) == 0) {
-          content = &html[val];
-          content_len = val_len;
-        }
-      }
     }
-    if (equiv_len == 12 && strncasecmp(equiv, "content-type", 12) == 0 &&
-        content != NULL) {
-      char *const s = charset_from_content(content, content_len);
-
-      if (s != NULL) {
-        return s;
-      }
-    }
-    i = j;
   }
   return NULL;
 }
@@ -1152,45 +1140,17 @@ int hts_isStringUTF8(const char *s, size_t size) {
   const unsigned char *const data = (const unsigned char*) s;
   size_t i;
 
-  /* Strict RFC 3629: reject overlongs, surrogates, > U+10FFFF, 5/6-byte. */
   for(i = 0 ; i < size ; ) {
-    const unsigned char c = data[i];
-    size_t len, k;
-    hts_UCS4 uc, min;
+    /* Reader: can read bytes up to j */
+#define RD ( i < size ? data[i++] : -1 )
 
-    if (c < 0x80) {
-      i++;
-      continue;
-    } else if (c >= 0xc2 && c <= 0xdf) {
-      len = 2;
-      min = 0x80;
-      uc = c & 0x1f;
-    } else if (c >= 0xe0 && c <= 0xef) {
-      len = 3;
-      min = 0x800;
-      uc = c & 0x0f;
-    } else if (c >= 0xf0 && c <= 0xf4) {
-      len = 4;
-      min = 0x10000;
-      uc = c & 0x07;
-    } else { /* continuation byte, overlong C0/C1, or F5..FF lead */
-      return 0;
-    }
-    if (size - i < len) {
-      return 0;
-    }
-    for (k = 1; k < len; k++) {
-      const unsigned char cc = data[i + k];
+    /* Writer: a malformed sequence means the string is not UTF-8 (return 0) */
+#define WR(C) if ((C) == -1) { return 0; }
 
-      if ((cc & 0xc0) != 0x80) {
-        return 0;
-      }
-      uc = (uc << 6) | (cc & 0x3f);
-    }
-    if (uc < min || uc > 0x10FFFF || (uc >= 0xD800 && uc <= 0xDFFF)) {
-      return 0;
-    }
-    i += len;
+    /* Read Unicode character. */
+    READ_UNICODE(RD, WR);
+#undef RD
+#undef WR
   }
 
   return 1;

src/htscharset.h

@@ -76,8 +76,7 @@ extern char *hts_convertStringIDNAToUTF8(const char *s, size_t size);
 extern int hts_isStringIDNA(const char *s, size_t size);
 
 /**
- * Extract the <meta> charset from the HTML buffer "html" (HTML5 charset= or
- * legacy http-equiv form). Returns a malloc'ed string, or NULL if none.
+ * Extract the charset from the HTML buffer "html"
  **/
 extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 
@@ -87,8 +86,7 @@ extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 extern int hts_isStringAscii(const char *s, size_t size);
 
 /**
- * Is the given string valid UTF-8 ? Strict RFC 3629: overlong forms,
- * surrogates, codepoints above U+10FFFF and 5/6-byte sequences are rejected.
+ * Is the given string an UTF-8 string ?
  **/
 extern int hts_isStringUTF8(const char *s, size_t size);
 

src/htsconfig.h

@@ -76,6 +76,9 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
+// fast cache (build hash table)
+#define HTS_FAST_CACHE 1
+
 // le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
 // valide)
 #define GT_ENDS_COMMENT 1

src/htscore.c

@@ -134,6 +134,18 @@ RUN_CALLBACK0(opt, end); \
     back_delete_all(opt, &cache, sback);                                       \
     back_free(&sback);                                                         \
     checkrobots_free(&robots);                                                 \
+    if (cache.use) {                                                           \
+      freet(cache.use);                                                        \
+      cache.use = NULL;                                                        \
+    }                                                                          \
+    if (cache.dat) {                                                           \
+      fclose(cache.dat);                                                       \
+      cache.dat = NULL;                                                        \
+    }                                                                          \
+    if (cache.ndx) {                                                           \
+      fclose(cache.ndx);                                                       \
+      cache.ndx = NULL;                                                        \
+    }                                                                          \
     if (cache.zipOutput) {                                                     \
       zipClose(cache.zipOutput,                                                \
                "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \
@@ -143,6 +155,10 @@ RUN_CALLBACK0(opt, end); \
       unzClose(cache.zipInput);                                                \
       cache.zipInput = NULL;                                                   \
     }                                                                          \
+    if (cache.olddat) {                                                        \
+      fclose(cache.olddat);                                                    \
+      cache.olddat = NULL;                                                     \
+    }                                                                          \
     if (cache.lst) {                                                           \
       fclose(cache.lst);                                                       \
       cache.lst = opt->state.strc.lst = NULL;                                  \
@@ -472,11 +488,32 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
   *makeindex_done = 1;
 }
 
-/* Flush the parsed HTML output buffer to disk. */
+/* Flush the parsed HTML output buffer to disk, skipping the rewrite when the
+ * on-disk MD5 is unchanged. */
 void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
                           FILE **fp, const char *ht_buff, size_t ht_len,
                           const char *adr, const char *fil, const char *save) {
-  {
+  char digest[32 + 2];
+  off_t fsize_old =
+      fsize(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), save));
+  int ok = 0;
+
+  digest[0] = '\0';
+  domd5mem(ht_buff, ht_len, digest, 1);
+  if (fsize_old == (off_t) ht_len) {
+    int mlen = 0;
+    char *mbuff;
+
+    cache_readdata(cache, "//[HTML-MD5]//", save, &mbuff, &mlen);
+    if (mlen)
+      mbuff[mlen] = '\0';
+    if ((mlen == 32) && (strcmp(((mbuff != NULL) ? mbuff : ""), digest) == 0)) {
+      ok = 1;
+      hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s", save);
+    }
+    freet(mbuff);
+  }
+  if (!ok) {
     file_notify(opt, adr, fil, save, 1, 1, r->notmodified);
     *fp = filecreate(&opt->state.strc, save);
     if (*fp) {
@@ -508,7 +545,13 @@ void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
       if (fcheck)
         hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
     }
+  } else {
+    file_notify(opt, adr, fil, save, 0, 0, r->notmodified);
+    filenote(&opt->state.strc, save, NULL);
   }
+  if (cache->ndx)
+    cache_writedata(cache->ndx, cache->dat, "//[HTML-MD5]//", save, digest,
+                    (int) strlen(digest));
 }
 
 /* does it look like XML ? (SVG et al.) */
@@ -1904,11 +1947,8 @@ int httpmirror(char *url1, httrackp * opt) {
             }
           }
 
-          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
-          // An empty body must not overwrite the file when the transfer failed
-          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
-          // copy to 0 (#77 follow-up).
-          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
+          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!          
+          if (r.adr != NULL || r.size == 0) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {
@@ -1929,6 +1969,7 @@ int httpmirror(char *url1, httrackp * opt) {
                */
             }
           }
+
         }
 
         /* Parsing of other media types (java, ram..) */

src/htscore.h

@@ -193,7 +193,8 @@ struct cache_back {
   /* */
   int type;
   int ro;                   /**< read-only: no new cache is written */
-
+  FILE *dat, *ndx, *olddat; /**< new data, new index, old data files */
+  char *use;                /**< in-memory list of cached adr+fil keys */
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
   char lastmodified[256];
@@ -287,12 +288,12 @@ struct filecreate_params {
 
 /* True if a new cache is being written (plain or zip backend). */
 HTS_STATIC int cache_writable(cache_back * cache) {
-  return (cache != NULL && cache->zipOutput != NULL);
+  return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL));
 }
 
 /* True if an old cache is available to read (plain or zip backend). */
 HTS_STATIC int cache_readable(cache_back * cache) {
-  return (cache != NULL && cache->zipInput != NULL);
+  return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL));
 }
 
 #endif

src/htsdns_selftest.c

@@ -297,26 +297,6 @@ int dns_selftests(httrackp *opt) {
     IPV6_resolver = 0;
   }
 
-  /* "host:port" resolves as the bare host: the cache/connect path strips
-     ":port" before both the backend and the cache key (#181). */
-  mock_reset_calls();
-  {
-    SOCaddr a;
-    char ip[64];
-    const char *err = NULL;
-
-    /* cache miss: the backend is hit with the stripped host, not "host:port" */
-    CHECK(hts_dns_resolve2(opt, "v6only.test:8080", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* the bare host shares that one cache entry (stripped key) */
-    CHECK(hts_dns_resolve2(opt, "v6only.test", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* a port variant of an already-cached host also hits, right address */
-    CHECK(hts_dns_resolve2(opt, "v4only.test:1234", &a, &err) != NULL);
-    SOCaddr_inetntoa(ip, sizeof(ip), a);
-    CHECK(strcmp(ip, "1.2.3.4") == 0);
-  }
-
   /* newhttp_addr() must connect to the addr_index-th address, not always the
      first: this is what back_connect_next relies on to reach the fallback. */
   {

src/htsfilters.c

@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
 #include "htsbase.h"
 #include "htslib.h"
 #include <ctype.h>
-#include <stdint.h>
 /* END specific definitions */
 
 // à partir d'un tableau de {"+*.toto","-*.zip","+*.tata"} définit si nom est autorisé
@@ -121,81 +120,16 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
   return use1 ? jok1 : jok2;
 }
 
-/* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
-   pair keeps star-heavy patterns polynomial instead of exponential (#501). */
-typedef struct strjoker_memo {
-  const char *chaine0, *joker0; /* offsets are relative to these bases */
-  size_t stride;                /* strlen(joker0) + 1 */
-  unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
-} strjoker_memo;
-
-static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
-                                 const char *joker, LLint *size,
-                                 int *size_flag);
-
-/* A pair that failed once fails forever (*size is only written on the success
-   path), so record NULL results and cut the re-exploration. */
-static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
-                                const char *joker, LLint *size,
-                                int *size_flag) {
-  size_t bit = 0;
-  const char *adr;
-
-  if (memo->failed) {
-    bit = (size_t) (chaine - memo->chaine0) * memo->stride +
-          (size_t) (joker - memo->joker0);
-    if (memo->failed[bit >> 3] & (unsigned char) (1u << (bit & 7)))
-      return NULL;
-  }
-  adr = strjoker_impl(memo, chaine, joker, size, size_flag);
-  if (adr == NULL && memo->failed)
-    memo->failed[bit >> 3] |= (unsigned char) (1u << (bit & 7));
-  return adr;
-}
-
-// wildcard comparator: match chaine against joker (pattern), case-insensitive
-// returns the address of the first matched letter past any leading joker
-// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
+// supercomparateur joker (tm)
+// compare a et b (b=avec joker dedans), case insensitive [voir CI]
+// renvoi l'adresse de la première lettre de la chaine
+// (càd *[..]toto.. renvoi adresse de toto dans la chaine)
+// accepte les délires du genre www.*.*/ * / * truc*.*
+// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
+// * = toute lettre
+// --?-- : spécifique à HTTrack et aux ?
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
                                 LLint *size, int *size_flag) {
-  strjoker_memo memo = {chaine, joker, 0, NULL};
-  unsigned char stackbits[2048];
-  hts_boolean onheap = HTS_FALSE;
-  const char *adr;
-
-  if (chaine != NULL && joker != NULL) {
-    const size_t n1 = strlen(chaine) + 1;
-
-    memo.stride = strlen(joker) + 1;
-    if (n1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
-      const size_t bytes = (n1 * memo.stride + 7) / 8;
-
-      if (bytes <= sizeof(stackbits)) {
-        memset(stackbits, 0, bytes);
-        memo.failed = stackbits;
-      } else {
-        memo.failed = (unsigned char *) calloct(bytes, 1);
-        onheap = memo.failed != NULL ? HTS_TRUE : HTS_FALSE;
-      }
-    }
-  }
-  adr = strjoker_rec(&memo, chaine, joker, size, size_flag);
-  if (onheap)
-    freet(memo.failed);
-  return adr;
-}
-
-/* Test-only oracle: the same matcher without the failure memo. */
-const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
-                            int *size_flag) {
-  const strjoker_memo memo = {chaine, joker, 0, NULL};
-
-  return strjoker_rec(&memo, chaine, joker, size, size_flag);
-}
-
-static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
-                                 const char *joker, LLint *size,
-                                 int *size_flag) {
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -369,7 +303,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 
       // tester sans le joker (pas ()+ mais ()*)
       if (!unique) {
-        if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag))) {
+        if ((adr = strjoker(chaine, joker + jmp, size, size_flag))) {
           return adr;
         }
       }
@@ -381,8 +315,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
         max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
-          if ((adr = strjoker_rec(memo, chaine + i + 1, joker + jmp, size,
-                                  size_flag))) {
+          if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
             return adr;
           }
           i++;
@@ -392,8 +325,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 
       // tester chaîne vide
       if (i != max + 2)         // avant c'est ok
-        if ((adr = strjoker_rec(memo, chaine + max, joker + jmp, size,
-                                size_flag)))
+        if ((adr = strjoker(chaine + max, joker + jmp, size, size_flag)))
           return adr;
 
       return NULL;              // perdu
@@ -415,7 +347,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       // comparaison ok?
       if (ok) {
         // continuer la comparaison.
-        if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag))
+        if (strjoker(chaine + jmp, joker + jmp, size, size_flag))
           return chaine;        // retourner 1e lettre
       }
 

src/htsfilters.h

@@ -48,10 +48,6 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
                      const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
-/* strjoker() without the failure memo (exponential worst case); test-only
-   oracle for the memoized matcher. */
-const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
-                            int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);
 #endif
 

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-12"
-#define HTTRACK_VERSIONID "3.49.12"
+#define HTTRACK_VERSION "3.49-11"
+#define HTTRACK_VERSIONID "3.49.11"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 
@@ -56,10 +56,6 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
-// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
-#include <stdint.h>
-#include <inttypes.h>
-
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -115,6 +111,29 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
+#ifndef HTS_LONGLONG
+#ifdef SIZEOF_LONG_LONG
+#if SIZEOF_LONG_LONG == 8
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+#ifndef HTS_LONGLONG
+#ifdef __sun
+#define HTS_LONGLONG 0
+#endif
+#ifdef __osf__
+#define HTS_LONGLONG 0
+#endif
+#ifdef __linux
+#define HTS_LONGLONG 1
+#endif
+#ifdef _WIN32
+#define HTS_LONGLONG 1
+#endif
+#endif
+#endif
+
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -307,11 +326,59 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
-typedef int64_t LLint;
-typedef int64_t TStamp;
-/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
-#define LLintP "%" PRId64
+#ifndef HTS_LONGLONG
+#ifdef HTS_NO_64_BIT
+#define HTS_LONGLONG 0
+#else
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+/* Wide integer types, chosen per platform.
+   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
+           plain int where 64-bit is unavailable).
+   TStamp: timestamp/duration in the same width (a double in the no-64-bit
+           fallback).
+   LLintP: the printf conversion for an LLint. */
+#if HTS_LONGLONG
+#ifdef LLINT_FORMAT
+typedef LLINT_TYPE LLint;
+
+typedef LLINT_TYPE TStamp;
+
+#define LLintP LLINT_FORMAT
+#else
+
+#ifdef _WIN32
+typedef __int64 LLint;
+
+typedef __int64 TStamp;
+
+#define LLintP "%I64d"
+#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
+       defined(__64BIT__))
+
+typedef long int LLint;
+
+typedef long int TStamp;
+
+#define LLintP "%ld"
+#else
+typedef long long int LLint;
+
+typedef long long int TStamp;
+
+#define LLintP "%lld"
+#endif
+
+#endif /* HTS_LONGLONG */
+
+#else
+typedef int LLint;
+
+#define LLintP "%d"
+typedef double TStamp;
+#endif
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain

src/htslib.c

@@ -1059,10 +1059,10 @@ static int append_cookie_header(buff_struct *bstr, t_cookie *cookie,
   return cook;
 }
 
-/* Build the request Cookie line for domain/path into dst (always
-   NUL-terminated). Returns the number of cookies emitted. */
-int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
-                       char *dst, size_t dst_size) {
+/* Self-test entry for append_cookie_header(): build the request Cookie line
+   into dst (always NUL-terminated). Returns the number of cookies emitted. */
+int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
+                                const char *path, char *dst, size_t dst_size) {
   buff_struct bstr = {dst, dst_size, 0};
 
   assertf(dst != NULL && dst_size > 0);
@@ -1586,7 +1586,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a = strchr(rcvd + p, '/');
         if (a != NULL) {
           a++;
-          if (sscanf(a, LLintP, &retour->crange) == 1 && retour->crange >= 0) {
+          if (sscanf(a, LLintP, &retour->crange) == 1) {
             retour->crange_start = 0;
             retour->crange_end = retour->crange - 1;
           } else {
@@ -1594,12 +1594,6 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           }
         }
       }
-      // A valid Content-Range has no negative field; reject hostile values so
-      // the crange +/- 1 arithmetic downstream cannot sign-overflow (UB).
-      if (retour->crange_start < 0 || retour->crange_end < 0 ||
-          retour->crange < 0) {
-        retour->crange_start = retour->crange_end = retour->crange = 0;
-      }
     }
   } else if ((p = strfield(rcvd, "Connection:")) != 0) {
     char *a = rcvd + p;
@@ -2004,15 +1998,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
 
   if (bufl > 0) {
     if (!r->is_write) {         // stocker en mémoire
-      // In-memory content must fit a 32-bit index (allocs below add 1, reads
-      // use int offsets): reject a hostile Content-Length or endless stream.
-      const LLint inmem_want =
-          (r->totalsize >= 0) ? r->totalsize : (r->size + bufl);
-      if (inmem_want >= INT32_MAX) {
-        r->statuscode = STATUSCODE_INVALID;
-        strcpybuff(r->msg, "In-memory content too large");
-        return READ_ERROR;
-      }
       if (r->totalsize >= 0) {  // totalsize déterminé ET ALLOUE
         if (r->adr == NULL) {
           r->adr = (char *) malloct((size_t) r->totalsize + 1);
@@ -2575,11 +2560,6 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
     // recopier adrfil->adresse www..
     strncatbuff(adrfil->adr, p, ((int) (q - p)));
     // *( adrfil->adr+( ((int) q) - ((int) p) ) )=0;  // faut arrêter la fumette!
-
-    // fil[] holds the path plus a possible leading '/' the top strlen() misses.
-    if (strlen(q) >= sizeof(adrfil->fil) - (q[0] != '/' ? 1 : 0))
-      return -1;
-
     // recopier chemin /pub/..
     if (q[0] != '/')            // page par défaut (/)
       strcatbuff(adrfil->fil, "/");
@@ -3058,11 +3038,12 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
 /* See http://physics.nist.gov/cuu/Units/binary.html */
 #define ToLLint(a) ((LLint)(a))
 #define ToLLintKiB (ToLLint(1024))
-#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
-#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#define ToLLintPiB                                                             \
-  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
+#ifdef HTS_LONGLONG
+#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#endif
 HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
   if (n < ToLLintKiB) {
     sprintf(strc->buff1, "%d", (int) (LLint) n);
@@ -3071,7 +3052,9 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
             (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
     strcpybuff(strc->buff2, "KiB");
-  } else if (n < ToLLintGiB) {
+  }
+#ifdef HTS_LONGLONG
+  else if (n < ToLLintGiB) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
             (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
     strcpybuff(strc->buff2, "MiB");
@@ -3088,6 +3071,13 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
             (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
     strcpybuff(strc->buff2, "PiB");
   }
+#else
+  else {
+    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
+            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
+    strcpybuff(strc->buff2, "MiB");
+  }
+#endif
   strc->buffadr[0] = strc->buff1;
   strc->buffadr[1] = strc->buff2;
   return strc->buffadr;

src/htslib.h

@@ -184,10 +184,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
                   const char *referer_adr, const char *referer_fil,
                   htsblk * retour);
 /* Build the request "Cookie:" header line for stored cookies matching
-   domain/path into dst (NUL-terminated), wrapping the logic http_sendhead()
-   uses. Returns cookies emitted. */
-int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
-                       char *dst, size_t dst_size);
+   domain/path into dst (NUL-terminated). Exposed for the -#Q self-test;
+   wraps the same logic http_sendhead() uses. Returns cookies emitted. */
+int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
+                                const char *path, char *dst, size_t dst_size);
 
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
               int waitconnect);

src/htsparse.c

@@ -2007,7 +2007,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                 /* Unescape/escape %20 and other   */
                 {
-                  // NULL when UTF-8 conversion is off (-%T0)
+                  // Note: always true (iso-8859-1 as default)
                   const char *const charset = str->page_charset_;
                   const int hasCharset = charset != NULL 
                     && *charset != '\0';
@@ -2030,12 +2030,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                   // Force to encode non-printable chars (should never happend)
                   escape_remove_control(lien);
-
-                  // charset conversion for the URI filename (not the query
-                  // string), unless the bytes already are valid UTF-8:
-                  // converting those would double-encode them (#180)
-                  if (hasCharset && !hts_isCharsetUTF8(charset) &&
-                      !hts_isStringUTF8(lien, strlen(lien))) {
+                  
+                  // charset conversion for the URI filename, 
+                  // and not already UTF-8
+                  // (note: not for the query string!)
+                  if (hasCharset && !hts_isCharsetUTF8(charset)) {
                     char *const s = hts_convertStringToUTF8(lien, strlen(lien), charset);
                     if (s != NULL) {
                       hts_log_print(opt, LOG_DEBUG,
@@ -4311,8 +4310,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
       *forbidden_url == 0 && IS_DELAYED_EXT(afs->save) && !opt->state.stop) {
     int loops;
     int continue_loop;
-    char BIGSTK
-        cookie_before[16384]; /* #15: this URL's Cookie header, pre-request */
 
     hts_log_print(opt, LOG_DEBUG, "Waiting for type to be known: %s%s", afs->af.adr,
                   afs->af.fil);
@@ -4321,12 +4318,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
     for(loops = 0, continue_loop = 1;
         IS_DELAYED_EXT(afs->save) && continue_loop && loops < 7; loops++) {
       continue_loop = 0;
-      /* #15: snapshot the Cookie header THIS url would send, so only its own
-         self-redirect Set-Cookie trips the retry, not a concurrent slot's. */
-      cookie_before[0] = '\0';
-      if (opt->accept_cookie && opt->cookie != NULL)
-        http_cookie_header(opt->cookie, jump_identification_const(afs->af.adr),
-                           afs->af.fil, cookie_before, sizeof(cookie_before));
 
       /* Wait for an available slot */
       if (!hts_wait_available_socket(sback, opt, cache, ptr))
@@ -4524,17 +4515,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           /* Moved! */
           else if (HTTP_IS_REDIRECT(back[b].r.statuscode)) {
             char BIGSTK mov_url[HTS_URLMAXSIZE * 2];
-            char BIGSTK cookie_after[16384];
-            hts_boolean cookies_changed;
-
-            /* #15: cookie-wall signal: this URL's own Cookie header changed
-               across its self-redirect (a Set-Cookie it just set). */
-            cookie_after[0] = '\0';
-            if (opt->accept_cookie && opt->cookie != NULL)
-              http_cookie_header(
-                  opt->cookie, jump_identification_const(afs->af.adr),
-                  afs->af.fil, cookie_after, sizeof(cookie_after));
-            cookies_changed = strcmp(cookie_before, cookie_after) != 0;
 
             mov_url[0] = '\0';
             strcpybuff(mov_url, back[b].r.location);    // copier URL
@@ -4604,24 +4584,11 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
                   url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, 
                                opt, sback, cache, hash, ptr, numero_passe,
                                &delayed_back);
-                } else if (cookies_changed) {
-                  // #15: cookie-wall self-redirect; evict the cached
-                  // fast-header so the re-issue refetches with the cookie.
-                  if (cache->cached_tests != NULL)
-                    coucal_remove(cache->cached_tests,
-                                  concat(OPT_GET_BUFF(opt),
-                                         OPT_GET_BUFF_SIZE(opt), afs->af.adr,
-                                         afs->af.fil));
-                  afs->save[0] = '\0';
-                  url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, opt,
-                               sback, cache, hash, ptr, numero_passe,
-                               &delayed_back);
-                  continue_loop = 1;
                 } else {
                   hts_log_print(opt, LOG_WARNING,
                                 "Unable to test %s%s (loop to same filename)",
                                 afs->af.adr, afs->af.fil);
-                } // loop to same location
+                }               // loop to same location
               }                 // ident_url_relatif()
             }                   // location
           }                     // redirect

src/htsselftest.c

@@ -621,105 +621,6 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* Mime-type filter verdict via fa_strjoker(type=1): only mime: rules apply. */
-static int st_filtermime(httrackp *opt, int argc, char **argv) {
-  int verdict;
-
-  (void) opt;
-  if (argc < 2) {
-    fprintf(stderr, "filtermime: needs <mime> <filter> [filter...]\n");
-    return 1;
-  }
-  verdict = fa_strjoker(1, &argv[1], argc - 1, argv[0], NULL, NULL, NULL);
-  printf("verdict=%s\n", verdict > 0   ? "allowed"
-                         : verdict < 0 ? "forbidden"
-                                       : "unknown");
-  return 0;
-}
-
-/* SplitMix64: deterministic, platform-independent case generator. */
-static uint64_t st_mix64(uint64_t *state) {
-  uint64_t z = (*state += UINT64_C(0x9E3779B97F4A7C15));
-
-  z = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4B5B9);
-  z = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB);
-  return z ^ (z >> 31);
-}
-
-/* Differential test: memoized strjoker vs the no-memo oracle on seeded random
-   pattern/subject/size cases must agree on result, *size and *size_flag. */
-static int st_filtermemo(httrackp *opt, int argc, char **argv) {
-  static const char *const pieces[] = {
-      "a",       "b",        "A",    "c",      ".",       "/",
-      "?",       "*",        "*[a]", "*[ab]",  "*[a-c]",  "*[A-Z]",
-      "*[<5]",   "*[>5]",    "*(a)", "*(a,b)", "*[file]", "*[path]",
-      "*[name]", "*[param]", "*[]",  "*[\\a]", "*[a-"};
-  static const char subject_chars[] = "abAc./?";
-  uint64_t rng = UINT64_C(0x501);
-  int iters = 20000, matched = 0, unmatched = 0;
-  int it;
-
-  (void) opt;
-  if (argc >= 1)
-    sscanf(argv[0], "%d", &iters);
-  for (it = 0; it < iters; it++) {
-    char pat[64], str[16];
-    size_t pl = 0;
-    const int npieces = (int) (st_mix64(&rng) % 5);
-    const int slen = (int) (st_mix64(&rng) % 13);
-    const int szsel = (int) (st_mix64(&rng) % 4); /* none/unknown/3/20 */
-    LLint sz1, sz2, off1, off2;
-    int i, flag1 = 0, flag2 = 0;
-    char *hpat, *hstr;
-    const char *adr;
-
-    for (i = 0; i < npieces; i++) {
-      const char *p =
-          pieces[st_mix64(&rng) % (sizeof(pieces) / sizeof(pieces[0]))];
-      const size_t len = strlen(p);
-
-      if (len >= sizeof(pat) - pl)
-        break;
-      memcpy(pat + pl, p, len);
-      pl += len;
-    }
-    pat[pl] = '\0';
-    for (i = 0; i < slen; i++)
-      str[i] = subject_chars[st_mix64(&rng) % (sizeof(subject_chars) - 1)];
-    str[slen] = '\0';
-    sz1 = sz2 = (szsel <= 1) ? -1 : (szsel == 2) ? 3 : 20;
-    /* exact-size heap copies per run so a sanitizer traps any over-read */
-    hpat = strdupt(pat);
-    hstr = strdupt(str);
-    adr = strjoker(hstr, hpat, szsel ? &sz1 : NULL, szsel ? &flag1 : NULL);
-    off1 = adr != NULL ? (LLint) (adr - hstr) : -1;
-    freet(hpat);
-    freet(hstr);
-    hpat = strdupt(pat);
-    hstr = strdupt(str);
-    adr =
-        strjoker_nomemo(hstr, hpat, szsel ? &sz2 : NULL, szsel ? &flag2 : NULL);
-    off2 = adr != NULL ? (LLint) (adr - hstr) : -1;
-    freet(hpat);
-    freet(hstr);
-    if (off1 != off2 || sz1 != sz2 || flag1 != flag2) {
-      printf("filtermemo MISMATCH pat=[%s] str=[%s] szsel=%d: "
-             "off %d/%d size %d/%d flag %d/%d\n",
-             pat, str, szsel, (int) off1, (int) off2, (int) sz1, (int) sz2,
-             flag1, flag2);
-      return 1;
-    }
-    if (off1 >= 0)
-      matched++;
-    else
-      unmatched++;
-  }
-  /* both polarities must actually occur or the test proves nothing */
-  assertf(matched > 0 && unmatched > 0);
-  printf("filtermemo: %d cases OK\n", iters);
-  return 0;
-}
-
 /* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
 static int st_filterdual(httrackp *opt, int argc, char **argv) {
   int depth = -1, verdict;
@@ -788,30 +689,6 @@ static int st_charset(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-static int st_metacharset(httrackp *opt, int argc, char **argv) {
-  char *s;
-
-  (void) opt;
-  if (argc < 1) {
-    fprintf(stderr, "metacharset: needs an html string\n");
-    return 1;
-  }
-  s = hts_getCharsetFromMeta(argv[0], strlen(argv[0]));
-  printf("%s\n", s != NULL ? s : "(none)");
-  freet(s);
-  return 0;
-}
-
-static int st_isutf8(httrackp *opt, int argc, char **argv) {
-  (void) opt;
-  if (argc < 1) {
-    fprintf(stderr, "isutf8: needs a string\n");
-    return 1;
-  }
-  printf("%d\n", hts_isStringUTF8(argv[0], strlen(argv[0])) ? 1 : 0);
-  return 0;
-}
-
 static int st_idna_encode(httrackp *opt, int argc, char **argv) {
   char *s;
 
@@ -1266,30 +1143,6 @@ static int st_identurl(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* Regression for the one-byte fil[] overflow: a 2047-byte hostless "?"-URL used
-   to abort in strncat_safe_ when the missing leading '/' pushed fil to 2048. */
-static int st_identabs(httrackp *opt, int argc, char **argv) {
-  lien_adrfil af;
-  const size_t len =
-      sizeof(af.fil) - 1; /* 2047: max URL the top guard admits */
-  char *url = malloct(len + 1);
-
-  (void) opt;
-  (void) argc;
-  (void) argv;
-  url[0] = '?';
-  memset(url + 1, 'a', len - 1);
-  url[len] = '\0';
-  assertf(ident_url_absolute(url, &af) == -1);
-  freet(url);
-  /* valid URLs still parse, so the guard is not over-rejecting */
-  assertf(ident_url_absolute("http://www.example.com/a/b/c.html?x=1", &af) ==
-          0);
-  assertf(ident_url_absolute("www.foo.com?bar=1", &af) == 0);
-  printf("identabs self-test OK\n");
-  return 0;
-}
-
 /* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
    no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
    registering an already-crawled link (dedup/collision paths). */
@@ -1334,82 +1187,6 @@ static int st_headerlong(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* http_xfread1 must refuse an in-memory buffer whose size would exceed a 32-bit
-   index (hostile Content-Length or endless stream) rather than allocate it.
-   The guard returns before any socket read, so no real connection is needed. */
-static int st_xfread_limit(httrackp *opt, int argc, char **argv) {
-  htsblk r;
-
-  (void) opt;
-  (void) argc;
-  (void) argv;
-
-  // Content-Length just over 2 GiB.
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = (LLint) INT32_MAX + 1;
-  printf("bylen: refused=%d adr=%s msg=%s\n",
-         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
-         r.msg);
-  if (r.adr != NULL)
-    freet(r.adr);
-
-  // Unknown length, buffer already at the limit: the next read would exceed it.
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = -1;
-  r.size = (LLint) INT32_MAX;
-  printf("bygrow: refused=%d adr=%s msg=%s\n",
-         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
-         r.msg);
-  if (r.adr != NULL)
-    freet(r.adr);
-
-  // Exactly at the 2 GiB index (size + bufl == INT32_MAX): must also be
-  // refused, since the reallocs below add 1 (a `> INT32_MAX` check would let
-  // this through and overflow the int realloc size).
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = -1;
-  r.size = (LLint) INT32_MAX - 8192;
-  http_xfread1(&r, 8192);
-  printf("boundary: msg=%s\n", r.msg);
-
-  // A legitimate small size must NOT be refused by the guard (the read then
-  // fails on the invalid socket, but the size-too-large msg must not be set).
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = 1000;
-  http_xfread1(&r, 8192);
-  printf("accept: msg=%s\n", r.msg);
-  if (r.adr != NULL)
-    freet(r.adr);
-  return 0;
-}
-
-/* Parse a Content-Range header and print the sanitized triple. A hostile value
-   (negative or INT64 extreme) must clamp to 0 without signed-overflow UB. */
-static int st_crange(httrackp *opt, int argc, char **argv) {
-  int i;
-
-  (void) opt;
-  if (argc < 1) {
-    fprintf(stderr, "crange: needs at least one raw Content-Range line\n");
-    return 1;
-  }
-  for (i = 0; i < argc; i++) {
-    htsblk r;
-    char BIGSTK line[HTS_URLMAXSIZE * 2];
-
-    memset(&r, 0, sizeof(r));
-    strcpybuff(line, argv[i]);
-    treathead(NULL, "www.example.com", "/", &r, line);
-    printf("crange_start=" LLintP " crange_end=" LLintP " crange=" LLintP "\n",
-           (LLint) r.crange_start, (LLint) r.crange_end, (LLint) r.crange);
-  }
-  return 0;
-}
-
 /* Decode a body argument ("hex:FFD8.." or literal text) into buf. */
 static size_t st_decode_body(const char *arg, char *buf, size_t size) {
   size_t n = 0;
@@ -1723,18 +1500,6 @@ static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
-static int st_cache_legacy(httrackp *opt, int argc, char **argv) {
-  int err;
-
-  if (argc < 1) {
-    fprintf(stderr, "cache-legacy: needs a directory\n");
-    return 1;
-  }
-  err = cache_legacy_refused_selftest(opt, argv[0]);
-  printf("cache-legacy: %s\n", err ? "FAIL" : "OK");
-  return err;
-}
-
 static int st_reconcile(httrackp *opt, int argc, char **argv) {
   int err;
 
@@ -1779,7 +1544,7 @@ static int st_cookies(httrackp *opt, int argc, char **argv) {
     return 1;
   }
 
-  http_cookie_header(&cookie, dom, "/", hdr, sizeof(hdr));
+  http_cookie_header_selftest(&cookie, dom, "/", hdr, sizeof(hdr));
   if (strcmp(hdr, expected) != 0)
     err = 1;
   if (strstr(hdr, "$Version") != NULL || strstr(hdr, "$Path") != NULL)
@@ -2247,19 +2012,6 @@ static int ae_write_collision(const char *path, const unsigned char *src,
   return ok ? 0 : 1;
 }
 
-/* Write src[0..len) to path as-is; 0 on success. */
-static int ae_write_raw(const char *path, const unsigned char *src,
-                        size_t len) {
-  FILE *const f = FOPEN(path, "wb");
-  int ok;
-
-  if (f == NULL)
-    return 1;
-  ok = fwrite(src, 1, len, f) == len;
-  fclose(f);
-  return ok ? 0 : 1;
-}
-
 /* Compare path's bytes to expect[0..len); 0 if equal. Streams (large files). */
 static int ae_check_decoded(const char *path, const unsigned char *expect,
                             size_t len) {
@@ -2327,46 +2079,6 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
     assertf(ae_write_collision(inpath, body, 64) == 0);
     assertf(hts_zunpack(inpath, outpath) == 64);
     assertf(ae_check_decoded(outpath, body, 64) == 0);
-    /* Identity fallback (#47): a plain body mislabeled as compressed is kept
-       verbatim, small and multi-chunk (> one 8 KiB fread). */
-    assertf(ae_write_raw(inpath, small, slen) == 0);
-    assertf(hts_zunpack(inpath, outpath) == (int) slen);
-    assertf(ae_check_decoded(outpath, small, slen) == 0);
-    {
-      const size_t ilen = 16 * 1024;
-      unsigned char *idbody = malloct(ilen);
-
-      assertf(idbody != NULL);
-      for (i = 0; i < ilen; i++)
-        idbody[i] = small[i % slen];
-      assertf(ae_write_raw(inpath, idbody, ilen) == 0);
-      assertf(hts_zunpack(inpath, outpath) == (int) ilen);
-      assertf(ae_check_decoded(outpath, idbody, ilen) == 0);
-      freet(idbody);
-    }
-    /* Truncated gzip (CRC+ISIZE cut), zlib (ADLER32 cut) and raw deflate
-       must all still fail, not fall back to a verbatim copy. */
-    {
-      static const struct {
-        int wb;
-        size_t cut;
-      } tr[] = {{16 + MAX_WBITS, 8}, {MAX_WBITS, 4}, {-MAX_WBITS, 5}};
-
-      for (i = 0; i < sizeof(tr) / sizeof(tr[0]); i++) {
-        unsigned char z[512];
-        size_t zlen;
-        FILE *f;
-
-        assertf(ae_write_packed(inpath, tr[i].wb, small, slen) == 0);
-        f = FOPEN(inpath, "rb");
-        assertf(f != NULL);
-        zlen = fread(z, 1, sizeof(z), f);
-        fclose(f);
-        assertf(zlen > tr[i].cut && zlen < sizeof(z));
-        assertf(ae_write_raw(inpath, z, zlen - tr[i].cut) == 0);
-        assertf(hts_zunpack(inpath, outpath) < 0);
-      }
-    }
     freet(body);
   }
 #else
@@ -2555,10 +2267,6 @@ static const struct selftest_entry {
     {"filtersize", "<size> <string> <filter>...",
      "size-aware filter verdict (negative size = unknown/scan time)",
      st_filtersize},
-    {"filtermime", "<mime> <filter>...",
-     "mime-type filter verdict (fa_strjoker type=1)", st_filtermime},
-    {"filtermemo", "[iterations]",
-     "memoized vs unmemoized matcher differential", st_filtermemo},
     {"filterdual", "<string1> <string2> <filter>...",
      "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
@@ -2571,9 +2279,6 @@ static const struct selftest_entry {
     {"mime", "<filename>", "MIME type for a filename", st_mime},
     {"charset", "<charset> <string>",
      "convert a string to UTF-8 from a charset", st_charset},
-    {"metacharset", "<html>", "extract the <meta> charset from an HTML page",
-     st_metacharset},
-    {"isutf8", "<string>", "is the string valid UTF-8 (1/0)", st_isutf8},
     {"idna-encode", "<host>", "encode a hostname to IDNA/punycode",
      st_idna_encode},
     {"idna-decode", "<host>", "decode an IDNA/punycode hostname",
@@ -2591,17 +2296,11 @@ static const struct selftest_entry {
     {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
      st_resolve},
     {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
-    {"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
-     st_identabs},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
     {"headerlong", "[header-name:]",
      "over-long header value must not overflow the parse scratch",
      st_headerlong},
-    {"crange", "<raw-content-range-line> ...",
-     "Content-Range parse integer safety", st_crange},
-    {"xfread-limit", "", "in-memory receive buffer size bound",
-     st_xfread_limit},
     {"savename", "<fil> <content-type> [key=value ...]",
      "local save-name for a URL", st_savename},
     {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
@@ -2615,8 +2314,6 @@ static const struct selftest_entry {
      st_cache_writefail},
     {"reconcile", "<dir>", "cache generation reconcile policy self-test",
      st_reconcile},
-    {"cache-legacy", "<dir>", "pre-3.31 legacy cache refusal self-test",
-     st_cache_legacy},
     {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
      st_cache_corrupt},
     {"dns", "", "DNS resolver/cache self-test", st_dns},

src/htszlib.c

@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
 
 /*
   Unpack file into a new file (gzip, zlib RFC1950 or raw deflate RFC1951).
-  A body provably in no deflate framing is copied verbatim (identity).
   Return value: size of the new file, or -1 if an error occurred
 */
 /* Note: utf-8 */
@@ -69,10 +68,6 @@ int hts_zunpack(char *filename, char *newfile) {
             ((inbuf[0] & 0x0f) == Z_DEFLATED &&
              (((unsigned) inbuf[0] << 8 | inbuf[1]) % 31) == 0)));
       int attempt;
-      /* not_deflate: the raw attempt hit a data error, so the body is in no
-         deflate framing at all. env_error: local I/O or memory failure. */
-      hts_boolean not_deflate = HTS_FALSE;
-      hts_boolean env_error = HTS_FALSE;
 
       /* deflate is ambiguous; on failure retry with the other windowBits */
       for (attempt = 0; attempt < 2 && ret < 0; attempt++) {
@@ -83,20 +78,15 @@ int hts_zunpack(char *filename, char *newfile) {
 
         if (attempt > 0) {
           /* rewind input; reopening fpout "wb" discards the partial output */
-          if (fseek(in, 0, SEEK_SET) != 0) {
-            env_error = HTS_TRUE;
+          if (fseek(in, 0, SEEK_SET) != 0)
             break;
-          }
           navail = fread(inbuf, 1, sizeof(inbuf), in);
         }
         fpout = FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
-        if (fpout == NULL) {
-          env_error = HTS_TRUE;
+        if (fpout == NULL)
           break;
-        }
         memset(&strm, 0, sizeof(strm));
         if (inflateInit2(&strm, windowBits) != Z_OK) {
-          env_error = HTS_TRUE;
           fclose(fpout);
           break;
         }
@@ -118,17 +108,12 @@ int hts_zunpack(char *filename, char *newfile) {
               zerr = inflate(&strm, Z_NO_FLUSH);
               if (zerr == Z_NEED_DICT || zerr == Z_DATA_ERROR ||
                   zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR) {
-                if (zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR)
-                  env_error = HTS_TRUE;
-                else if (windowBits < 0)
-                  not_deflate = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
               produced = sizeof(outbuf) - strm.avail_out;
               if (produced > 0 &&
                   fwrite(outbuf, 1, produced, fpout) != produced) {
-                env_error = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
@@ -144,28 +129,6 @@ int hts_zunpack(char *filename, char *newfile) {
         inflateEnd(&strm);
         fclose(fpout);
       }
-      /* keep a mislabeled identity body verbatim only when provably not
-         deflate; truncation or a local failure must keep failing (#47) */
-      if (ret < 0 && !wrapped && not_deflate && !env_error && !ferror(in)) {
-        FILE *const fpout =
-            FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
-
-        if (fpout != NULL && fseek(in, 0, SEEK_SET) == 0) {
-          int size = 0;
-
-          while ((navail = fread(inbuf, 1, sizeof(inbuf), in)) > 0) {
-            if (fwrite(inbuf, 1, navail, fpout) != navail) {
-              size = -1;
-              break;
-            }
-            size += (int) navail;
-          }
-          if (size >= 0 && !ferror(in))
-            ret = size;
-        }
-        if (fpout != NULL)
-          fclose(fpout);
-      }
       fclose(in);
     }
   }

tests/01_engine-charset.test

@@ -36,64 +36,3 @@ test "$(httrack -O /dev/null -#test=charset 'no-such-charset-xyz' 'café' 2>/dev
 # malformed UTF-8 (lone continuation byte, truncated lead byte) must not crash
 runs 'utf-8' $'\x80'
 runs 'utf-8' $'\xc3'
-
-# hts_getCharsetFromMeta: <meta> charset extraction, HTML5 and legacy forms.
-# -#test=metacharset <html> prints the charset, or "(none)".
-meta() {
-    test "$(httrack -O /dev/null -#test=metacharset "$1")" == "$2" || exit 1
-}
-
-# HTML5 form, quoting/spacing flavors
-meta '<meta charset="utf-8">' 'utf-8'
-meta "<meta charset='utf-8'>" 'utf-8'
-meta '<meta charset=utf-8>' 'utf-8'
-meta '<meta charset=utf-8 />' 'utf-8'
-meta '<meta charset=utf-8/>' 'utf-8'
-# charset labels are case-insensitive downstream; only require case-insensitive match
-test "$(httrack -O /dev/null -#test=metacharset '<META CHARSET=UTF-8>' | tr '[:upper:]' '[:lower:]')" == 'utf-8' || exit 1
-meta '<html><head><meta   charset = "utf-8"  ></head>' 'utf-8'
-
-# legacy http-equiv form, attributes in any order
-meta '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">' 'iso-8859-1'
-meta '<meta content="text/html; charset=gb2312" http-equiv="content-type">' 'gb2312'
-meta '<meta http-equiv=Content-Type content="text/html;charset=euc-jp">' 'euc-jp'
-meta '<meta http-equiv="Content-Type" content="text/html; charset = koi8-r ">' 'koi8-r'
-meta '<meta http-equiv="Content-Type" content="text/html; charset=utf-8;">' 'utf-8'
-meta '<meta http-equiv=content-type content=text/html;charset=utf-8 />' 'utf-8'
-meta '<meta http-equiv=content-type content=text/html;charset=utf-8>' 'utf-8'
-
-# first resolvable meta wins; unrelated metas are skipped
-meta '<meta name="viewport" content="width=1"><meta charset="koi8-r">' 'koi8-r'
-meta '<meta charset="utf-8"><meta http-equiv="Content-Type" content="text/html; charset=gb2312">' 'utf-8'
-
-# duplicate attribute: first wins, as in browsers
-meta '<meta http-equiv="Content-Type" content="text/html; charset=big5" http-equiv="refresh">' 'big5'
-
-# negatives: wrong tag/attribute, empty value
-meta '<p>x</p>' '(none)'
-meta '<meta name="desc" content="charset=bogus">' '(none)'
-meta '<meta http-equiv="refresh" content="0; url=x.html">' '(none)'
-meta '<body charset="utf-8">' '(none)'
-meta '<meta charset="">' '(none)'
-
-# hostile truncations: no crash, no match
-meta '<meta charset="utf-8' '(none)'
-meta '<meta charset=' '(none)'
-meta '<meta ' '(none)'
-meta '<meta' '(none)'
-
-# hts_isStringUTF8: valid-UTF-8 probe (guards the #180 link conversion).
-utf8() {
-    test "$(httrack -O /dev/null -#test=isutf8 "$1")" == "$2" || exit 1
-}
-utf8 'plain' '1'
-utf8 'café' '1'
-utf8 '统计' '1'
-utf8 "$(printf 'caf\xe9')" '0'              # latin-1 accent
-utf8 "$(printf '\xc3')" '0'                 # truncated lead byte
-utf8 "$(printf '\xa9')" '0'                 # lone continuation byte
-utf8 "$(printf '\xf0\x9f\x98\x80')" '1'     # 4-byte emoji
-utf8 "$(printf '\xc0\xaf')" '0'             # overlong (latin-1 "À¯" must stay convertible)
-utf8 "$(printf '\xed\xa0\x80')" '0'         # UTF-16 surrogate
-utf8 "$(printf '\xf4\x90\x80\x80')" '0'     # above U+10FFFF
-utf8 "$(printf '\xf8\x88\x80\x80\x80')" '0' # legacy 5-byte form

tests/01_engine-crange.test

@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# Content-Range parse (treathead via -#test=crange). Hostile values must clamp
-# to 0, and the crange +/- 1 arithmetic must not sign-overflow (UBSan aborts on
-# the pre-fix binary at the INT64_MIN cases).
-
-cr() {
-    local want="$1"
-    shift
-    out="$(httrack -O /dev/null -#test=crange "$@")"
-    test "$out" == "$want" || {
-        echo "FAIL: $* -> '$out' (want '$want')"
-        exit 1
-    }
-}
-
-# Normal range and the '*/N' fallback both parse through unchanged.
-cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes 0-70870/70871'
-cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes */70871'
-
-# INT64_MIN total in the fallback would make crange-1 overflow: clamp to 0.
-cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes */-9223372036854775808'
-
-# Any negative field in the 3-field form clamps the whole triple to 0 (each
-# field independently: a leading negative start/end alone still zeroes all).
-cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -1--1/-1'
-cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -5-10/20'
-
-# INT64_MAX is non-negative: it passes through, the parse must not mangle it.
-cr 'crange_start=0 crange_end=9223372036854775807 crange=9223372036854775807' \
-    'Content-Range: bytes 0-9223372036854775807/9223372036854775807'

tests/01_engine-filter.test

@@ -131,64 +131,3 @@ match '*.aspx' 'page.aspx?y=2'
 # empty subject against a unique-match allow-all pattern (heap over-read regress)
 nomatch_empty '*(('
 nomatch_empty '**(('
-
-# near-miss negatives: word order, exact tails, range boundaries, class case
-nomatch 'foo*bar' 'foobaz'
-nomatch '*foo*bar' 'barfoo' # both words present, wrong order
-nomatch '*.gif' 'a.gifx'    # trailing junk after the extension
-nomatch '*.gif' 'a.gi'
-nomatch '*[b-y]' 'a'
-nomatch '*[b-y]' 'z'
-nomatch '*[a-z].txt' 'B.txt' # classes are case-sensitive...
-match 'FoO*' 'foobar'        # ...literals are not
-
-# a star may match an empty run, but never skip a required literal
-match '*[0-9].gif' '.gif'
-nomatch '*[0-9].gif' 'x.gif'
-
-# a descending range *[c-a] is an empty class, not c..a reversed
-match '*[c-a]x' 'x'
-nomatch '*[c-a]x' 'cx'
-
-# *[param] is all-or-nothing on '?': spans everything after one, else nothing
-match 'a*[param]' 'a?x=1'
-nomatch 'a*[param]' 'ab?x=1'
-
-# the latest matching rule wins, regardless of sign
-fsize 'verdict=forbidden size_flag=0' -1 a.gif '+*.gif' '-*.gif'
-fsize 'verdict=allowed size_flag=0' -1 a.gif '-*.gif' '+*.gif'
-
-# a failed size test sets size_flag but leaves the rule unmatched
-fsize 'verdict=unknown size_flag=1' 5 foo.jpg '-*.jpg*[>10]'
-
-# mime filters (-#test=filtermime = fa_strjoker type=1): only mime: rules
-# apply to a mime type, and mime: rules never apply to a URL
-fmime() {
-    local want="$1"
-    shift
-    test "$(httrack -O /dev/null -#test=filtermime "$@")" == "verdict=$want" || exit 1
-}
-fmime allowed 'image/png' '+mime:image/*'
-fmime unknown 'text/html' '+mime:image/*'
-fmime forbidden 'image/png' '+mime:image/*' '-mime:image/png' # last wins
-fmime allowed 'image/png' '-mime:image/png' '+mime:image/*'
-fmime unknown 'image/png' '+image/*'                               # URL rule: skipped
-fsize 'verdict=unknown size_flag=0' -1 'image/png' '+mime:image/*' # and vice versa
-
-# memoized vs no-memo differential over seeded random patterns and subjects,
-# matching and non-matching (the self-test asserts both polarities occurred)
-test "$(httrack -O /dev/null -#test=filtermemo)" == "filtermemo: 20000 cases OK" || exit 1
-
-# star-heavy non-match must stay polynomial (#501); 400 chars also exercises
-# the heap-allocated memo. timeout (where available) turns a hang into a FAIL.
-with_timeout() {
-    if command -v timeout >/dev/null; then timeout 60 "$@"; else "$@"; fi
-}
-pat=$(printf '*[a]%.0s' $(seq 1 12))b
-subj=$(printf 'a%.0s' $(seq 1 400))
-test "$(with_timeout httrack -O /dev/null -#test=filter "$pat" "$subj")" == "$subj does NOT match $pat" || exit 1
-
-# a later star's class dead-end must re-extend the earlier star, not give up
-# (guards against a single-backtrack-point matcher rewrite)
-match '*[aX]X*[a]Y' 'aXaXaY'
-nomatch '*[aX]X*[a]Y' 'aXbXaY'

tests/01_engine-identabs.test

@@ -1,8 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# A 2047-byte hostless "?"-URL used to overflow fil[] by one byte and abort in
-# strncat_safe_; the guard now returns -1 while valid URLs still parse.
-test "$(httrack -O /dev/null -#test=identabs)" == "identabs self-test OK" || exit 1

tests/01_engine-xfread.test

@@ -1,29 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# http_xfread1 must refuse an in-memory receive buffer that would exceed a
-# 32-bit index (hostile Content-Length, or an endless stream) rather than
-# allocate it, while still accepting a normal small size. -#test=xfread-limit
-# drives both refusal paths and the accept path.
-
-out="$(httrack -O /dev/null -#test=xfread-limit)"
-for case in bylen bygrow; do
-    echo "$out" | grep -q "${case}: refused=1 adr=null msg=In-memory content too large" || {
-        echo "FAIL ${case}: $out"
-        exit 1
-    }
-done
-
-# Exactly INT32_MAX must be refused too (the reallocs add 1).
-echo "$out" | grep -q 'boundary: msg=In-memory content too large' || {
-    echo "FAIL boundary: $out"
-    exit 1
-}
-
-# The guard must NOT fire for a legitimate small size.
-if echo "$out" | grep -q 'accept: msg=In-memory content too large'; then
-    echo "FAIL accept (guard fired on a legit size): $out"
-    exit 1
-fi

tests/01_zlib-cache-legacy.test

@@ -1,23 +0,0 @@
-#!/bin/bash
-#
-# The pre-3.31 .dat/.ndx cache import was removed: cache_init must refuse the
-# legacy pair and leave the files untouched (httrack -#test=cache-legacy <dir>).
-
-set -eu
-
-dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
-
-# the refusal errors land on stdout (no log file); pin them and the verdict
-out=$(httrack -#test=cache-legacy "$dir" 2>/dev/null)
-cnt=$(printf '%s\n' "$out" | grep -c 'no longer supported' || true)
-test "$cnt" = 2 || {
-    echo "expected 2 refusal messages, got $cnt" >&2
-    exit 1
-}
-out=$(printf '%s\n' "$out" | tail -n1)
-
-test "$out" = "cache-legacy: OK" || {
-    echo "expected 'cache-legacy: OK', got: $out" >&2
-    exit 1
-}

tests/11_crawl-international.test

@@ -22,9 +22,9 @@ bash crawl-test.sh \
     httrack http://ut.httrack.com/unicode-links/utf8.html
 
 bash crawl-test.sh \
-    --errors 2 --files 9 \
-    --found ut.httrack.com/unicode-links/café3860.html \
-    --found ut.httrack.com/unicode-links/café9fa8.html \
+    --errors 4 --files 7 \
+    --found ut.httrack.com/unicode-links/café3860.html \
+    --found ut.httrack.com/unicode-links/café9fa8.html \
     --found ut.httrack.com/unicode-links/café30f4.html \
     --found ut.httrack.com/unicode-links/café5e1f.html \
     --found ut.httrack.com/unicode-links/café7b30.html \

tests/19_local-connect-fallback.test

@@ -66,12 +66,9 @@ test -n "$port" || {
 }
 
 out="$tmpdir/crawl"
-# macOS has only 127.0.0.1, so the dead 127.0.0.x connects block to the timeout
-# instead of refusing instantly (as on Linux); a small --timeout + --retries=0
-# bound the stall without changing what the fallback exercises.
 HTTRACK_DEBUG_RESOLVE="deadhost:127.0.0.2,127.0.0.1" \
     httrack "http://deadhost:$port/simple/basic.html" -O "$out" \
-    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log" 2>&1
+    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log" 2>&1
 
 log="$out/hts-log.txt"
 
@@ -95,12 +92,12 @@ test -f "$out/deadhost_$port/simple/basic.html" || {
     exit 1
 }
 
-# every address dead: the slot exhausts the list, then errors out (the harness
-# timeout would catch a hang/loop)
+# every address refused: the slot exhausts the list, then errors out (the
+# harness timeout would catch a hang/loop; refused connects are instant)
 out2="$tmpdir/crawl2"
 HTTRACK_DEBUG_RESOLVE="alldead:127.0.0.2,127.0.0.3" \
     httrack "http://alldead:$port/simple/basic.html" -O "$out2" \
-    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log2" 2>&1
+    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log2" 2>&1
 log2="$out2/hts-log.txt"
 
 grep -q "trying next address" "$log2" || {

tests/31_local-javaclass.test

@@ -17,10 +17,7 @@ printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
 printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
     >"$tmproot/javaclass/Foo.class"
 
-# --log-found first: a plugin-load failure (wrong soname, shadowing system
-# libhtsjava) trips here on the banner suffix, not as an opaque "not found".
 bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
-    --log-found '\+libhtsjava' \
     --found 'javaclass/Foo.class' \
     --found 'javaclass/hello.gif' \
     httrack 'BASEURL/javaclass/index.html'

tests/36_local-bigcrawl.test

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 # Diverse seeded /big/ crawl: 12 pattern families, decoy absence, update pass
-# must 304-revalidate. 361 = 1 index + 96 pages + 192 imgs + 5 shared + 61
+# must 304-revalidate. 360 = 1 index + 96 pages + 192 imgs + 5 shared + 60
 # family + 6 singles; the 4 planted errors write -o1 pages, not counted.
 
 set -euo pipefail
@@ -9,7 +9,7 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --errors-content 4 --files 361 \
+    --errors 4 --files 360 \
     --found 'big/p/95.html' \
     --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
     --found 'big/a/f2-2x.png' \
@@ -40,7 +40,6 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --not-found 'big/x/bj.jar' \
     --not-found 'big/x/is1.png' \
     --not-found 'big/x/concat.html' \
-    --file-matches 'big/f1/gzid.html' '<p>gzid</p>' \
     --file-matches 'big/p/2.html' 'srcset="\.\./a/f2-1x\.png 1x, \.\./a/f2-2x\.png 2x"' \
     --file-matches 'big/a/blk2.css' 'url\(blk2-bg\.png\)' \
     --file-matches 'big/p/5.html' "document\\.write\\('<a href=\"\\.\\./f5/dw\\.html\"" \
@@ -53,4 +52,4 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --log-found ', no files updated' \
     --max-mirror-bytes 700000 \
     --min-mirror-bytes 500000 \
-    httrack 'BASEURL/big/index.html' --retries=0 -c16 -%c100 -A100000000
+    httrack 'BASEURL/big/index.html' --retries=0 -c8 -%c100 -A100000000

tests/41_local-utf8-link.test

@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# #180: raw non-ASCII hrefs must reach the wire UTF-8 percent-encoded whatever
-# the declared page charset; each PDF exists only at its exact UTF-8 path.
-
-set -euo pipefail
-: "${top_srcdir:=..}"
-
-bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
-    --found 'charset/header/统计大数据服务平台.pdf' \
-    --found 'charset/meta5/统计大数据服务平台.pdf' \
-    --found 'charset/metaeq/统计大数据服务平台.pdf' \
-    --found 'charset/none/统计大数据服务平台.pdf' \
-    --found 'charset/latin1hdr/统计大数据服务平台.pdf' \
-    --found 'charset/latin1real/café.pdf' \
-    --found 'charset/metalatin1/déjà.pdf' \
-    --found 'charset/latin1ovl/À¡x.pdf' \
-    --found 'charset/priority/nuée.pdf' \
-    --found 'charset/preenc/统计大数据服务平台.pdf' \
-    --found 'charset/bom/统计大数据服务平台.pdf' \
-    httrack 'BASEURL/charset/index.html'

tests/42_local-maxsize-slow.test

@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# -M byte cap under a slow server (#77): p0 overruns -M at once while p1..p3
-# trickle for a minute; the cap must abort the in-flight transfers, not wait
-# them out. Unfixed, the smooth stop drains the trickle at server pace.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-start=$(date +%s)
-bash "$top_srcdir/tests/local-crawl.sh" \
-    --log-found 'More than 400000 bytes have been transferred.. giving up' \
-    httrack 'BASEURL/bigtrickle/index.html' -M400000 -c4
-wall=$(($(date +%s) - start))
-# trickle runs 60s; a smooth-only stop waits it out, the hard stop lands near 8s.
-# Wall clock is the discriminating check: the log line above fires either way.
-if [ "$wall" -ge 30 ]; then
-    echo "crawl took ${wall}s, -M hard stop did not engage" >&2
-    exit 1
-fi

tests/43_local-update-truncate.test

@@ -1,24 +0,0 @@
-#!/bin/bash
-#
-# -M hard-abort must not destroy already-complete files (#77 follow-up).
-# Pass 1 mirrors bigtrunc fully. Pass 2 re-fetches under --update -M: fast.bin
-# overruns the cap, its abort tears down slow.bin's stalled re-fetch. Both were
-# complete on disk from pass 1, and both must survive the aborted update:
-#   - slow.bin's re-fetch is incomplete; the direct-to-disk write must not be
-#     left truncated (the previous copy is restored).
-#   - fast.bin fully arrives but is torn down; the aborted slot must not overwrite
-#     it with an empty body.
-# Unfixed, slow.bin is truncated and fast.bin is zeroed while the cache still
-# records them complete.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-bash "$top_srcdir/tests/local-crawl.sh" \
-    --rerun-args '--update -M400000' \
-    --log-found 'More than 400000 bytes have been transferred.. giving up' \
-    --found bigtrunc/slow.bin \
-    --file-min-bytes bigtrunc/slow.bin 655360 \
-    --file-min-bytes bigtrunc/fast.bin 655360 \
-    httrack 'BASEURL/bigtrunc/index.html' -c2

tests/44_local-update-errormask.test

@@ -1,13 +0,0 @@
-#!/bin/bash
-# Issue #176: on --update a page that 200'd on the first crawl but now 403s must
-# keep its good local copy, not be overwritten by the error body nor purged.
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --found 'errmask/keep.dat' \
-    --file-min-bytes 'errmask/keep.dat' 1024 \
-    --file-matches 'errmask/keep.dat' '^KEEP' \
-    --file-not-matches 'errmask/keep.dat' 'error 403' \
-    httrack 'BASEURL/errmask/index.html'

tests/45_local-maxsize-recv.test

@@ -1,16 +0,0 @@
-#!/bin/bash
-#
-# -M caps received volume, not saved 200-only bytes (#520): links to large 404
-# bodies pump received >> saved; the cap must trip though little lands on disk.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-# The mirror bound is the load-bearing half: it fires while <100 KB is saved,
-# so the cap can only have tripped on received volume, not saved bytes (which
-# would exceed the cap and fetch all 16 links).
-bash "$top_srcdir/tests/local-crawl.sh" \
-    --log-found 'More than 500000 bytes have been transferred.. giving up' \
-    --max-mirror-bytes 100000 \
-    httrack 'BASEURL/maxrecv/index.html' -M500000 -c4

tests/46_local-update-304-resume.test

@@ -1,118 +0,0 @@
-#!/bin/bash
-# C7: a server 304 answering a resume's Range request is out of protocol; httrack
-# must drop the partial and refetch, not finalize it as complete at the partial
-# byte count. Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets a bogus
-# 304, and must recover the whole file.
-set -u
-
-: "${top_srcdir:=..}"
-testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
-
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
-
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_c7.XXXXXX") || exit 1
-serverpid=
-crawlpid=
-cleanup() {
-    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
-    rm -rf "$tmpdir"
-}
-trap cleanup EXIT HUP INT QUIT PIPE TERM
-
-serverlog="${tmpdir}/server.log"
-counter="${tmpdir}/reqcount"
-mark="${tmpdir}/got304"
-RESUME304_COUNTER="$counter" RESUME304_MARK="$mark" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
-serverpid=$!
-port=
-for _ in $(seq 1 50); do
-    line=$(head -n1 "$serverlog" 2>/dev/null)
-    if test "${line%% *}" == "PORT"; then
-        port="${line#PORT }"
-        break
-    fi
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "server exited early: $(cat "$serverlog")"
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover server port"
-    exit 1
-}
-base="http://127.0.0.1:${port}"
-
-which httrack >/dev/null || {
-    echo "could not find httrack"
-    exit 1
-}
-out="${tmpdir}/crawl"
-mkdir "$out"
-common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
-refdir="${out}/hts-cache/ref"
-
-# --- pass 1: crawl, interrupt once the blob download is underway -------------
-printf '[pass 1: interrupt mid-download] ..\t'
-httrack "${common[@]}" "${base}/resume304/index.html" >"${tmpdir}/log1" 2>&1 &
-crawlpid=$!
-for _ in $(seq 1 300); do
-    test -s "$counter" && break
-    kill -0 "$crawlpid" 2>/dev/null || break
-    sleep 0.1
-done
-sleep 0.5
-kill -TERM "$crawlpid" 2>/dev/null
-wait "$crawlpid" 2>/dev/null
-crawlpid=
-test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
-    echo "FAIL: no temp-ref survived pass 1; cannot drive C7"
-    exit 1
-}
-echo "OK (temp-ref present)"
-before=$(wc -c <"$counter" 2>/dev/null || echo 0)
-
-# --- pass 2: --continue -> resume Range -> bogus 304 -> drop + refetch --------
-printf '[pass 2: resume gets 304, must refetch] ..\t'
-httrack "${common[@]}" --continue "${base}/resume304/index.html" >"${tmpdir}/log2" 2>&1
-echo "OK (terminated)"
-
-blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
-full=8200 # len(RESUME304_BODY) = len("C7DATA--") + 8192
-
-printf '[file recovered whole] ..\t'
-test -s "$blob" || {
-    echo "FAIL: blob.bin missing after the 304 resume (partial dropped, not refetched)"
-    exit 1
-}
-got=$(wc -c <"$blob")
-test "$got" -eq "$full" || {
-    echo "FAIL: blob.bin is ${got} bytes, expected ${full} (partial finalized as complete)"
-    exit 1
-}
-echo "OK (${got} bytes)"
-
-# Pass 2 must issue at least two requests: the resume (Range -> 304) and the
-# range-less refetch (-> 200). Exactly one means the partial was trusted.
-after=$(wc -c <"$counter" 2>/dev/null || echo 0)
-hits=$((after - before))
-printf '[resume then refetch] ..\t'
-test "$hits" -ge 2 || {
-    echo "FAIL: only ${hits} pass-2 request(s); the 304 was trusted, no refetch"
-    exit 1
-}
-echo "OK (${hits} requests)"
-
-# The recovery must go through the resume path: a Range request that drew the
-# bogus 304 (a fresh re-crawl sends no Range, so this marker stays empty).
-printf '[resume Range drew a 304] ..\t'
-test -s "$mark" || {
-    echo "FAIL: no Range request got a 304; the resume path was not exercised"
-    exit 1
-}
-echo "OK"

tests/47_local-crange-overflow.test

@@ -1,96 +0,0 @@
-#!/bin/bash
-# C2: a resume whose 206 carries a Content-Range end of INT64_MAX must not
-# sign-overflow the crange+1 range check (UBSan aborts on the pre-fix binary).
-# Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets the hostile 206, and
-# must reject the range and refetch the whole file rather than overflow. Teeth
-# are UBSan-only: a normal build wraps to the same reject+restart.
-set -u
-
-: "${top_srcdir:=..}"
-testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
-
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
-
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crange.XXXXXX") || exit 1
-serverpid=
-crawlpid=
-cleanup() {
-    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
-    rm -rf "$tmpdir"
-}
-trap cleanup EXIT HUP INT QUIT PIPE TERM
-
-serverlog="${tmpdir}/server.log"
-python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
-serverpid=$!
-port=
-for _ in $(seq 1 50); do
-    line=$(head -n1 "$serverlog" 2>/dev/null)
-    if test "${line%% *}" == "PORT"; then
-        port="${line#PORT }"
-        break
-    fi
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "server exited early: $(cat "$serverlog")"
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover server port"
-    exit 1
-}
-base="http://127.0.0.1:${port}"
-
-which httrack >/dev/null || {
-    echo "could not find httrack"
-    exit 1
-}
-out="${tmpdir}/crawl"
-mkdir "$out"
-common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
-refdir="${out}/hts-cache/ref"
-
-# --- pass 1: crawl, interrupt once the blob download is underway -------------
-printf '[pass 1: interrupt mid-download] ..\t'
-httrack "${common[@]}" "${base}/crange206/index.html" >"${tmpdir}/log1" 2>&1 &
-crawlpid=$!
-for _ in $(seq 1 300); do
-    test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
-    kill -0 "$crawlpid" 2>/dev/null || break
-    sleep 0.1
-done
-sleep 0.3
-kill -TERM "$crawlpid" 2>/dev/null
-wait "$crawlpid" 2>/dev/null
-crawlpid=
-test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
-    echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
-    exit 1
-}
-echo "OK (temp-ref present)"
-
-# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
-printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
-httrack "${common[@]}" --continue "${base}/crange206/index.html" >"${tmpdir}/log2" 2>&1
-echo "OK (terminated)"
-
-blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
-full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
-
-printf '[file recovered whole] ..\t'
-test -s "$blob" || {
-    echo "FAIL: blob.bin missing after the hostile 206"
-    exit 1
-}
-got=$(wc -c <"$blob")
-test "$got" -eq "$full" || {
-    echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
-    exit 1
-}
-echo "OK (${got} bytes)"

tests/48_local-crange-memresume.test

@@ -1,96 +0,0 @@
-#!/bin/bash
-# C2 (memory branch): a resume whose 206 lies text/html with a matching
-# INT64_MAX Content-Length must not overflow the resume buffer-size add
-# (UBSan aborts on the pre-fix binary at alloc_mem += totalsize). Pass 1 leaves
-# a partial + temp-ref; pass 2 resumes and must reject and refetch the whole
-# file. Teeth are UBSan-only: a normal build wraps to the same reject+restart.
-set -u
-
-: "${top_srcdir:=..}"
-testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
-
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
-
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangemem.XXXXXX") || exit 1
-serverpid=
-crawlpid=
-cleanup() {
-    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
-    rm -rf "$tmpdir"
-}
-trap cleanup EXIT HUP INT QUIT PIPE TERM
-
-serverlog="${tmpdir}/server.log"
-python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
-serverpid=$!
-port=
-for _ in $(seq 1 50); do
-    line=$(head -n1 "$serverlog" 2>/dev/null)
-    if test "${line%% *}" == "PORT"; then
-        port="${line#PORT }"
-        break
-    fi
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "server exited early: $(cat "$serverlog")"
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover server port"
-    exit 1
-}
-base="http://127.0.0.1:${port}"
-
-which httrack >/dev/null || {
-    echo "could not find httrack"
-    exit 1
-}
-out="${tmpdir}/crawl"
-mkdir "$out"
-common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
-refdir="${out}/hts-cache/ref"
-
-# --- pass 1: crawl, interrupt once the blob download is underway -------------
-printf '[pass 1: interrupt mid-download] ..\t'
-httrack "${common[@]}" "${base}/crange206mem/index.html" >"${tmpdir}/log1" 2>&1 &
-crawlpid=$!
-for _ in $(seq 1 300); do
-    test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
-    kill -0 "$crawlpid" 2>/dev/null || break
-    sleep 0.1
-done
-sleep 0.3
-kill -TERM "$crawlpid" 2>/dev/null
-wait "$crawlpid" 2>/dev/null
-crawlpid=
-test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
-    echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
-    exit 1
-}
-echo "OK (temp-ref present)"
-
-# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
-printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
-httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1
-echo "OK (terminated)"
-
-blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
-full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
-
-printf '[file recovered whole] ..\t'
-test -s "$blob" || {
-    echo "FAIL: blob.bin missing after the hostile 206"
-    exit 1
-}
-got=$(wc -c <"$blob")
-test "$got" -eq "$full" || {
-    echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
-    exit 1
-}
-echo "OK (${got} bytes)"

tests/49_local-cookiewall.test

@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Cookie-wall self-redirect (#15): httrack replays a Set-Cookie to mirror the
-# real page, gives up when nothing changes, and stays bounded when it never does.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-# Unknown-ext (wall.php) and known-ext (wall.html): the wall sets the cookie once,
-# then serves real content; httrack must replay it and mirror that content.
-bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
-    --found 'cookiewall/wall.html' \
-    --file-matches 'cookiewall/wall.html' 'REAL-CONTENT-BEHIND-COOKIE-WALL' \
-    httrack 'BASEURL/cookiewall/index.html'
-
-bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
-    --found 'cookiewall2/wall.html' \
-    --file-matches 'cookiewall2/wall.html' 'REAL-CONTENT-BEHIND-COOKIE-WALL' \
-    httrack 'BASEURL/cookiewall2/index.html'
-
-# No Set-Cookie: the jar never changes, so httrack must give up at once (retry
-# stays gated) and not mirror the wall. A dropped gate would skip this log line.
-bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
-    --not-found 'cookiewall3/wall.html' \
-    --log-found 'loop to same filename' \
-    httrack 'BASEURL/cookiewall3/index.html'
-
-# Ever-changing cookie never satisfies the wall: httrack must stop at the retry
-# cap (bounded, no early give-up) and not mirror it.
-bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
-    --not-found 'cookiewall4/wall.html' \
-    --log-not-found 'loop to same filename' \
-    httrack 'BASEURL/cookiewall4/index.html'

tests/Makefile.am

@@ -30,7 +30,6 @@ TESTS = \
 	01_engine-cmdline.test \
 	01_engine-cookies.test \
 	01_engine-copyopt.test \
-	01_engine-crange.test \
 	01_engine-dns.test \
 	01_engine-doitlog.test \
 	01_engine-entities.test \
@@ -44,7 +43,6 @@ TESTS = \
 	01_engine-header.test \
 	01_engine-idna.test \
 	01_engine-identurl.test \
-	01_engine-identabs.test \
 	01_engine-escape-room.test \
 	01_engine-inplace-escape.test \
 	01_engine-java.test \
@@ -67,11 +65,9 @@ TESTS = \
 	01_engine-urlhack.test \
 	01_engine-unescape-bounds.test \
 	01_engine-useragent.test \
-	01_engine-xfread.test \
 	01_zlib-acceptencoding.test \
 	01_zlib-cache.test \
 	01_zlib-cache-corrupt.test \
-	01_zlib-cache-legacy.test \
 	01_zlib-cache-golden.test \
 	01_zlib-cache-writefail.test \
 	01_zlib-savename-cached.test \
@@ -112,15 +108,6 @@ TESTS = \
 	37_local-cache-outage.test \
 	38_local-update-304.test \
 	39_local-delayed-cancel.test \
-	40_local-why.test \
-	41_local-utf8-link.test \
-	42_local-maxsize-slow.test \
-	43_local-update-truncate.test \
-	44_local-update-errormask.test \
-	45_local-maxsize-recv.test \
-	46_local-update-304-resume.test \
-	47_local-crange-overflow.test \
-	48_local-crange-memresume.test \
-	49_local-cookiewall.test
+	40_local-why.test
 
 CLEANFILES = check-network_sh.cache

tests/crawl-test.sh

@@ -105,7 +105,7 @@ function start-crawl {
     log="${tmp}/log"
     debug starting httrack -O "${tmp}" "${moreargs[@]}" "${@:pos}"
     info "running httrack ${*:pos}"
-    httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -mrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
+    httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -omrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
     crawlpid="$!"
     debug "started cralwer on pid $crawlpid"
     wait "$crawlpid"

tests/local-crawl.sh

@@ -13,21 +13,15 @@
 #
 # Usage:
 #   bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
-#       [--rerun-args 'ARGS'] \
-#       --errors N --errors-content N --files N --found PATH ... --directory PATH ... \
+#       --errors N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
-#       --file-min-bytes PATH N --max-mirror-bytes N \
+#       --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
-# --errors counts every "Error:" log line; --errors-content drops transient
-# network failures (codes -2..-6) that flake on busy loopback under -c8.
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
 # --max/--min-mirror-bytes bound the mirrored content bytes (host root).
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
-# --file-min-bytes asserts a mirrored file (PATH) is at least N bytes.
-# --rerun-args runs a second pass (same server and mirror dir) with the given
-# extra httrack args appended, e.g. an --update run under a cap.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
 # which the ephemeral port forces into the cookie domain) and passes it to
 # httrack via --cookies-file, to exercise preloaded cookies.
@@ -45,7 +39,6 @@ key="${testdir}/server.key"
 tls=
 verbose=
 rerun=
-rerun_args=
 rerun_dead=
 tmpdir=
 serverpid=
@@ -129,11 +122,7 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         cookies+=("${args[$pos]}")
         ;;
-    --rerun-args)
-        pos=$((pos + 1))
-        rerun_args="${args[$pos]}"
-        ;;
-    --errors | --errors-content | --files)
+    --errors | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -141,7 +130,7 @@ while test "$pos" -lt "$nargs"; do
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --file-matches | --file-not-matches | --file-min-bytes)
+    --file-matches | --file-not-matches)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}" "${args[$((pos + 2))]}")
         pos=$((pos + 2))
         ;;
@@ -213,7 +202,7 @@ mkdir "$out" || die "could not create $out"
 declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
 log="${tmpdir}/log"
 info "running httrack ${hts[*]}"
-httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
+httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
 crawlpid=$!
 wait "$crawlpid"
 crawlres=$?
@@ -230,7 +219,7 @@ grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
 # --- optional second pass: re-mirror into the same dir (cache/update path) ----
 if test -n "$rerun"; then
     info "re-running httrack (update pass)"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -252,25 +241,6 @@ if test -n "$rerun"; then
     fi
 fi
 
-# --- optional second pass with extra args (e.g. an --update run under a cap) ---
-# Same server and mirror dir as the first pass, so the second pass sees the
-# cache the first pass wrote. Used to exercise re-fetch/update behaviour.
-if test -n "$rerun_args"; then
-    read -ra extra <<<"$rerun_args"
-    info "re-running httrack with ${rerun_args}"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
-        "${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
-    crawlpid=$!
-    wait "$crawlpid"
-    crawlres=$?
-    crawlpid=
-    test "$crawlres" -eq 0 || ! result "second pass exited $crawlres" || {
-        cat "${log}.2" >&2
-        exit 1
-    }
-    result "OK (second pass)"
-fi
-
 # --- optional dead pass: server stopped, the cache must survive the rollback --
 if test -n "$rerun_dead"; then
     zip="${out}/hts-cache/new.zip"
@@ -281,7 +251,7 @@ if test -n "$rerun_dead"; then
     wait "$serverpid" 2>/dev/null
     serverpid=
     info "re-running httrack against the stopped server"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
     crawlpid=$!
     wait "$crawlpid" || true
@@ -336,14 +306,6 @@ while test "$i" -lt "${#audit[@]}"; do
         assert_equals "checking errors" "${audit[$i]}" \
             "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
         ;;
-    --errors-content)
-        i=$((i + 1))
-        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
-        # transient network failures (statuscode -2..-6) flake on busy loopback;
-        # the code parens are followed by " at link" or " after N retries at link"
-        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
-        assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
-        ;;
     --files)
         i=$((i + 1))
         nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
@@ -426,16 +388,6 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         else result "OK"; fi
         ;;
-    --file-min-bytes)
-        path="${audit[$((i + 1))]}"
-        i=$((i + 2))
-        sz=$(wc -c <"${hostroot}/${path}" 2>/dev/null | tr -d '[:space:]')
-        info "checking ${path} size ${sz:-0} >= ${audit[$i]} bytes"
-        if test -n "$sz" && test "$sz" -ge "${audit[$i]}"; then result "OK"; else
-            result "file too small (or missing)"
-            exit 1
-        fi
-        ;;
     esac
     i=$((i + 1))
 done

tests/local-server.py

@@ -374,7 +374,6 @@ def big_index(port):
         '<img src="/big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png">'
         '<a href="/big/f1/long.html?x=%s">long</a>'
         '<a href="/big/f1/gzok.html">gzok</a>'
-        '<a href="/big/f1/gzid.html">gzid</a>'
         '<a href="//127.0.0.1:%d/big/f1/protorel.html">protorel</a>'
         '<a href="http://127.0.0.1:%d/big/f1/abshost.html">abshost</a>'
         '<a href="/big/e/404.html">e404</a>'
@@ -400,7 +399,6 @@ BIG_SIMPLE_PAGES = {
     "/big/f1/dir/": "dir index",
     "/big/f1/long.html": "long",
     "/big/f1/gzok.html": "gzok",
-    "/big/f1/gzid.html": "gzid",
     "/big/f1/protorel.html": "protorel",
     "/big/f1/abshost.html": "abshost",
     "/big/f5/dw.html": "dw target",
@@ -662,97 +660,6 @@ class Handler(SimpleHTTPRequestHandler):
     def route_intl_page(self):
         self.send_raw(b"<html><body>accented page</body></html>\n", "text/html")
 
-    # Raw non-ASCII href matrix (#180): each variant declares the page charset
-    # differently; the PDF exists only at its exact UTF-8 path, so a
-    # mis-decoded link 404s.
-    CHARSET_CJK = "统计大数据服务平台.pdf"
-    # variant -> (index Content-Type, <head> bytes, href bytes, pdf name)
-    CHARSET_VARIANTS = {
-        "header": ("text/html; charset=utf-8", b"", CHARSET_CJK.encode(), CHARSET_CJK),
-        "meta5": (
-            "text/html",
-            b'<meta charset="utf-8">',
-            CHARSET_CJK.encode(),
-            CHARSET_CJK,
-        ),
-        "metaeq": (
-            "text/html",
-            b'<meta http-equiv="Content-Type" content="text/html; charset=utf-8">',
-            CHARSET_CJK.encode(),
-            CHARSET_CJK,
-        ),
-        "none": ("text/html", b"", CHARSET_CJK.encode(), CHARSET_CJK),
-        "latin1hdr": (
-            "text/html; charset=iso-8859-1",
-            b"",
-            CHARSET_CJK.encode(),
-            CHARSET_CJK,
-        ),
-        # genuine latin-1 href: the charset conversion must still apply
-        "latin1real": (
-            "text/html; charset=iso-8859-1",
-            b"",
-            "café.pdf".encode("latin-1"),
-            "café.pdf",
-        ),
-        # latin-1 declared by META only: the meta parser is load-bearing
-        "metalatin1": (
-            "text/html",
-            b'<meta charset="iso-8859-1">',
-            "déjà.pdf".encode("latin-1"),
-            "déjà.pdf",
-        ),
-        # latin-1 bytes that form an overlong UTF-8 shape: strict validation
-        # must still convert them
-        "latin1ovl": (
-            "text/html; charset=iso-8859-1",
-            b"",
-            "À¡x.pdf".encode("latin-1"),
-            "À¡x.pdf",
-        ),
-        # header wins over meta: latin-1 href only resolves if iso-8859-1 is kept
-        "priority": (
-            "text/html; charset=iso-8859-1",
-            b'<meta charset="utf-8">',
-            "nuée.pdf".encode("latin-1"),
-            "nuée.pdf",
-        ),
-        "preenc": ("text/html", b"", quote(CHARSET_CJK).encode(), CHARSET_CJK),
-        "bom": ("text/html", b"", CHARSET_CJK.encode(), CHARSET_CJK),
-    }
-
-    def route_charset(self):
-        path = unquote(urlsplit(self.path).path)
-        parts = path.split("/")
-        if path == "/charset/index.html":
-            self.send_html(
-                "".join(
-                    '\t<a href="%s/index.html">%s</a>\n' % (v, v)
-                    for v in self.CHARSET_VARIANTS
-                )
-            )
-            return
-        if len(parts) == 4 and parts[2] in self.CHARSET_VARIANTS:
-            ctype, head, href, pdf = self.CHARSET_VARIANTS[parts[2]]
-            if parts[3] == "index.html":
-                body = (
-                    b"<html><head>"
-                    + head
-                    + b'</head><body><a href="'
-                    + href
-                    + b'">doc</a></body></html>'
-                )
-                if parts[2] == "bom":
-                    body = b"\xef\xbb\xbf" + body
-                self.send_raw(body, ctype)
-                return
-            if parts[3] == pdf:
-                self.send_raw(self.FAKE_PDF, "application/pdf")
-                return
-        self.send_response(404)
-        self.send_header("Content-Length", "0")
-        self.end_headers()
-
     # resume / 416 loop (#206): the first GET stalls after a prefix so the crawl
     # can be interrupted (partial + temp-ref); every later request is 416.
     RESUME_PREFIX = b"PARTIAL-" + b"x" * 4096  # flushed before the stall
@@ -790,56 +697,6 @@ class Handler(SimpleHTTPRequestHandler):
         self.send_header("Content-Length", "0")
         self.end_headers()
 
-    # C7: stall the first GET (partial + temp-ref), then answer the resume's
-    # Range with a bogus 304; httrack must drop the partial and refetch.
-    RESUME304_BODY = b"C7DATA--" + bytes((i * 7 + 3) % 256 for i in range(8192))
-    _resume304_started = False
-
-    def route_resume304_index(self):
-        self.send_html('\t<a href="blob.bin">blob</a>')
-
-    def route_resume304(self):
-        counter = os.environ.get("RESUME304_COUNTER")
-        if counter:
-            with open(counter, "a") as fp:
-                fp.write("x")
-        rng = self.headers.get("Range")
-        if not Handler._resume304_started:
-            Handler._resume304_started = True
-            self.send_response(200)
-            self.send_header("Content-Type", "application/octet-stream")
-            self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
-            self.send_header("Accept-Ranges", "bytes")
-            self.send_header("Last-Modified", BIG_LASTMOD)
-            self.send_header("ETag", '"c7"')
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.RESUME304_BODY[:4096])
-                self.wfile.flush()
-                try:
-                    while True:
-                        time.sleep(3600)
-                except OSError:
-                    pass
-            return
-        if rng is not None:  # resume request: bogus out-of-protocol 304
-            mark = os.environ.get("RESUME304_MARK")
-            if mark:
-                with open(mark, "a") as fp:
-                    fp.write("z")
-            self.send_response(304)
-            self.send_header("ETag", '"c7"')
-            self.send_header("Content-Length", "0")
-            self.end_headers()
-            return
-        # Range-less refetch after the partial is dropped: whole file.
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(self.RESUME304_BODY)
-
     # 206 resume must honor the server's Content-Range, not the offset we asked
     # for (#198): a server resuming a few bytes *before* the request must not
     # leave httrack duplicating the overlap onto the partial. flaky.bin
@@ -905,94 +762,6 @@ class Handler(SimpleHTTPRequestHandler):
         if self.command != "HEAD":
             self.wfile.write(part)
 
-    # C2: a resume answered with a 206 whose Content-Range end is INT64_MAX would
-    # sign-overflow the crange+1 range check (UBSan abort). Stall first (partial +
-    # ref), then answer the resume Range with that hostile 206; httrack must reject
-    # the range and refetch, never overflow.
-    CRANGE206_BODY = b"CR206DAT" + bytes((i * 5 + 1) % 256 for i in range(6000))
-    _crange206_started = False
-
-    def route_crange206_index(self):
-        self.send_html('\t<a href="blob.bin">blob</a>')
-
-    def route_crange206(self):
-        rng = self.headers.get("Range")
-        if not Handler._crange206_started:
-            Handler._crange206_started = True
-            self.send_response(200)
-            self.send_header("Content-Type", "application/octet-stream")
-            self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
-            self.send_header("Accept-Ranges", "bytes")
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.CRANGE206_BODY[:3000])
-                self.wfile.flush()
-                try:
-                    while True:
-                        time.sleep(3600)
-                except OSError:
-                    pass
-            return
-        if rng is not None:  # resume: hostile 206, Content-Range end = INT64_MAX
-            self.send_response(206, "Partial Content")
-            self.send_header("Content-Type", "application/octet-stream")
-            self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
-            self.send_header("Content-Range", "bytes 0-9223372036854775807/1")
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.CRANGE206_BODY)
-            return
-        # range-less refetch after the bad range is rejected: whole file.
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(self.CRANGE206_BODY)
-
-    # C2 memory branch: a resume answered with a 206 that lies text/html (so the
-    # resume buffers in memory) plus a matching INT64_MAX Content-Length would
-    # overflow the buffer-size add. Stall first, then send that hostile 206.
-    _crange206mem_started = False
-
-    def route_crange206mem_index(self):
-        self.send_html('\t<a href="blob.bin">blob</a>')
-
-    def route_crange206mem(self):
-        rng = self.headers.get("Range")
-        if not Handler._crange206mem_started:
-            Handler._crange206mem_started = True
-            self.send_response(200)
-            self.send_header("Content-Type", "application/octet-stream")
-            self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
-            self.send_header("Accept-Ranges", "bytes")
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.CRANGE206_BODY[:3000])
-                self.wfile.flush()
-                try:
-                    while True:
-                        time.sleep(3600)
-                except OSError:
-                    pass
-            return
-        if rng is not None:  # resume: text/html + matching INT64_MAX Content-Length
-            self.send_response(206, "Partial Content")
-            self.send_header("Content-Type", "text/html")
-            self.send_header("Content-Length", "9223372036854775807")
-            self.send_header(
-                "Content-Range", "bytes 0-9223372036854775806/9223372036854775807"
-            )
-            self.end_headers()
-            return  # the overflow is computed before any body read
-        # range-less refetch after the resume is rejected: whole file.
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(self.CRANGE206_BODY)
-
     # error pages / 0-byte files (#17): -o0 ("no error pages") must keep 4xx/5xx
     # bodies off disk; a genuine 0-byte 200 is a valid file and stays.
     def route_errpage_index(self):
@@ -1072,33 +841,6 @@ class Handler(SimpleHTTPRequestHandler):
     def route_mini304_page(self):
         self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
 
-    # --- /errmask/: issue #176 — a page that 200'd on the first crawl but 403s
-    # on the update fetch must keep its good copy, not be overwritten nor purged.
-    ERRMASK_GOOD = b"KEEP" + b"." * 1020  # 1024 B distinctive non-HTML body
-    ERRMASK_ERR = b"<html><body>error 403</body></html>\n"
-
-    def route_errmask_index(self):
-        self.send_html('\t<a href="keep.dat">keep</a>\n')
-
-    def route_errmask_keep(self):
-        # First crawl (no validator) gets the 1024 B body + Last-Modified; the
-        # update sends a conditional and gets a 403 error page.
-        if self.headers.get("If-Modified-Since") or self.headers.get("If-None-Match"):
-            self.send_response(403, "Forbidden")
-            self.send_header("Content-Type", "text/html")
-            self.send_header("Content-Length", str(len(self.ERRMASK_ERR)))
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.ERRMASK_ERR)
-            return
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Last-Modified", BIG_LASTMOD)
-        self.send_header("Content-Length", str(len(self.ERRMASK_GOOD)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(self.ERRMASK_GOOD)
-
     # --- delayed-type degenerate paths (issues #5/#107) --------------------
     def route_delayed_index(self):
         self.send_html(
@@ -1143,56 +885,6 @@ class Handler(SimpleHTTPRequestHandler):
     def route_delayed_empty(self):
         self.send_raw(b"", "text/html")  # 200 + Content-Length: 0
 
-    # --- /cookiewall/ (#15): a self-redirect that only sets a cookie is a
-    # consent wall; httrack must replay the cookie and fetch the real page.
-    WALL_MARK = b"REAL-CONTENT-BEHIND-COOKIE-WALL"
-
-    def route_cookiewall_index(self):
-        self.send_html('\t<a href="wall.php">wall</a>')
-
-    def route_cookiewall_wall(self):
-        self._cookiewall_reply("wall.php")
-
-    # Known-extension twin: .html so the type is not delayed-resolved.
-    def route_cookiewall2_index(self):
-        self.send_html('\t<a href="wall.html">wall</a>')
-
-    def route_cookiewall2_wall(self):
-        self._cookiewall_reply("wall.html")
-
-    def _cookiewall_reply(self, location):
-        if self.request_cookies().get("gate") == "1":
-            self.send_raw(
-                b"<html><body>" + self.WALL_MARK + b"</body></html>\n", "text/html"
-            )
-        else:
-            self._wall_redirect(location, "gate=1; Path=/")
-
-    # No-cookie self-redirect: the jar never changes, so httrack must give up at
-    # once rather than re-fetch (proves the cookie-wall retry stays gated on #15).
-    def route_cookiewall3_index(self):
-        self.send_html('\t<a href="wall.php">wall</a>')
-
-    def route_cookiewall3_wall(self):
-        self._wall_redirect("wall.php", None)
-
-    # Ever-changing cookie: every hit sets a fresh value, so the jar keeps
-    # changing; httrack must stop at the loops<7 cap, not spin forever.
-    def route_cookiewall4_index(self):
-        self.send_html('\t<a href="wall.php">wall</a>')
-
-    def route_cookiewall4_wall(self):
-        nonce = int(self.request_cookies().get("gate", "0")) + 1
-        self._wall_redirect("wall.php", f"gate={nonce}; Path=/")
-
-    def _wall_redirect(self, location, set_cookie):
-        self.send_response(302, "Found")
-        self.send_header("Location", location)
-        if set_cookie is not None:
-            self.send_header("Set-Cookie", set_cookie)
-        self.send_header("Content-Length", "0")
-        self.end_headers()
-
     # -E time-limit (#481): pages that trickle far longer than any -E budget,
     # so only an engine-side abort can end the crawl.
     TRICKLE_SECONDS = 60
@@ -1250,64 +942,6 @@ class Handler(SimpleHTTPRequestHandler):
     def route_bigfile(self):
         self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
 
-    # -M under a slow server (#77): p0 is a fast 640KB file that alone overruns
-    # -M; p1..p3 trickle for a minute. The cap must abort those in-flight
-    # transfers, not wait them out.
-    def route_bigtrickle_index(self):
-        self.send_html(
-            "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(4))
-        )
-
-    # -M hard-abort must not destroy an already-complete file (#77 follow-up).
-    # "fast.bin" alone overruns -M and completes; "slow.bin" transfers fully on
-    # its first fetch (initial mirror) but stalls on every later fetch (the
-    # --update re-fetch), so the -M abort tears it down mid-body. An engine that
-    # truncates the good local copy on the aborted re-fetch loses data.
-    slow_seen = 0
-
-    def route_bigtrunc_index(self):
-        self.send_html('\t<a href="fast.bin">fast</a>\n\t<a href="slow.bin">slow</a>\n')
-
-    def route_bigtrunc_slow(self):
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(self.BIGFILE_BYTES))
-        self.end_headers()
-        if self.command == "HEAD":
-            return
-        # Count body fetches only, so a stray HEAD can't shift which pass stalls.
-        Handler.slow_seen += 1
-        first = Handler.slow_seen == 1
-        try:
-            if first:
-                self.wfile.write(b"x" * self.BIGFILE_BYTES)
-                self.wfile.flush()
-            else:
-                self.wfile.write(b"x" * 4096)
-                self.wfile.flush()
-                for _ in range(120):
-                    self.wfile.write(b"x")
-                    self.wfile.flush()
-                    time.sleep(1.0)
-        except OSError:
-            pass
-
-    # -M received-volume cap (#520): links to large 404 bodies. httrack receives
-    # each (HTS_TOTAL_RECV climbs) but saves none, so saved stays far below -M.
-    def route_maxrecv_index(self):
-        self.send_html(
-            "".join('\t<a href="r%d.bin">r%d</a>\n' % (i, i) for i in range(16))
-        )
-
-    def route_maxrecv_404(self):
-        body = b"x" * self.BIGFILE_BYTES
-        self.send_response(404, "Not Found")
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(len(body)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(body)
-
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -1337,15 +971,9 @@ class Handler(SimpleHTTPRequestHandler):
         "/intl/" + INTL_NAME: route_intl_page,
         "/resume/index.html": route_resume_index,
         "/resume/blob.txt": route_resume,
-        "/resume304/index.html": route_resume304_index,
-        "/resume304/blob.bin": route_resume304,
         "/overlap/index.html": route_overlap_index,
         "/overlap/flaky.bin": route_overlap,
         "/overlap/full.bin": route_overlap_full,
-        "/crange206/index.html": route_crange206_index,
-        "/crange206/blob.bin": route_crange206,
-        "/crange206mem/index.html": route_crange206mem_index,
-        "/crange206mem/blob.bin": route_crange206mem,
         "/size/index.html": route_size_index,
         "/size/oversize.bin": route_size_oversize,
         "/errpage/index.html": route_errpage_index,
@@ -1386,14 +1014,6 @@ class Handler(SimpleHTTPRequestHandler):
         "/bigfiles/p5.bin": route_bigfile,
         "/bigfiles/p6.bin": route_bigfile,
         "/bigfiles/p7.bin": route_bigfile,
-        "/bigtrickle/index.html": route_bigtrickle_index,
-        "/bigtrickle/p0.bin": route_bigfile,
-        "/bigtrickle/p1.bin": route_trickle_page,
-        "/bigtrickle/p2.bin": route_trickle_page,
-        "/bigtrickle/p3.bin": route_trickle_page,
-        "/bigtrunc/index.html": route_bigtrunc_index,
-        "/bigtrunc/fast.bin": route_bigfile,
-        "/bigtrunc/slow.bin": route_bigtrunc_slow,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,
@@ -1409,38 +1029,11 @@ class Handler(SimpleHTTPRequestHandler):
         "/delayed/chain7.php": route_delayed_chain,
         "/delayed/chain8.php": route_delayed_chain,
         "/delayed/chain9.php": route_delayed_chain,
-        "/cookiewall/index.html": route_cookiewall_index,
-        "/cookiewall/wall.php": route_cookiewall_wall,
-        "/cookiewall2/index.html": route_cookiewall2_index,
-        "/cookiewall2/wall.html": route_cookiewall2_wall,
-        "/cookiewall3/index.html": route_cookiewall3_index,
-        "/cookiewall3/wall.php": route_cookiewall3_wall,
-        "/cookiewall4/index.html": route_cookiewall4_index,
-        "/cookiewall4/wall.php": route_cookiewall4_wall,
         "/redir/index.html": route_redir_index,
         "/redir/go.php": route_redir_go,
         "/redir/target.html": route_redir_target,
         "/mini304/index.html": route_mini304_index,
         "/mini304/page.html": route_mini304_page,
-        "/errmask/index.html": route_errmask_index,
-        "/errmask/keep.dat": route_errmask_keep,
-        "/maxrecv/index.html": route_maxrecv_index,
-        "/maxrecv/r0.bin": route_maxrecv_404,
-        "/maxrecv/r1.bin": route_maxrecv_404,
-        "/maxrecv/r2.bin": route_maxrecv_404,
-        "/maxrecv/r3.bin": route_maxrecv_404,
-        "/maxrecv/r4.bin": route_maxrecv_404,
-        "/maxrecv/r5.bin": route_maxrecv_404,
-        "/maxrecv/r6.bin": route_maxrecv_404,
-        "/maxrecv/r7.bin": route_maxrecv_404,
-        "/maxrecv/r8.bin": route_maxrecv_404,
-        "/maxrecv/r9.bin": route_maxrecv_404,
-        "/maxrecv/r10.bin": route_maxrecv_404,
-        "/maxrecv/r11.bin": route_maxrecv_404,
-        "/maxrecv/r12.bin": route_maxrecv_404,
-        "/maxrecv/r13.bin": route_maxrecv_404,
-        "/maxrecv/r14.bin": route_maxrecv_404,
-        "/maxrecv/r15.bin": route_maxrecv_404,
     }
 
     # --- /big/ seeded pseudo-site ------------------------------------------
@@ -1486,13 +1079,6 @@ class Handler(SimpleHTTPRequestHandler):
                     "text/html",
                     extra=[("Content-Encoding", "gzip")],
                 )
-            elif path == "/big/f1/gzid.html":
-                # Plain body mislabeled as gzip: identity fallback keeps it (#47)
-                self.big_send(
-                    body,
-                    "text/html",
-                    extra=[("Content-Encoding", "gzip")],
-                )
             else:
                 self.big_send(body, "text/html")
         elif path == "/big/f1/list.html":
@@ -1608,9 +1194,6 @@ class Handler(SimpleHTTPRequestHandler):
         if path.startswith("/big/"):
             self.route_big()
             return True
-        if path.startswith("/charset/"):
-            self.route_charset()
-            return True
         # Match percent-encoded paths (accented #157 route) by their decoded form.
         handler = self.ROUTES.get(path) or self.ROUTES.get(unquote(path))
         if handler is not None:
@@ -1645,12 +1228,7 @@ def main():
     def factory(*a, **kw):
         return Handler(*a, directory=root, **kw)
 
-    # macOS/BSD drop SYNs when the listen backlog overflows (Linux is lenient);
-    # raise it from Python's default 5 so a busy -c8 crawl can't lose fetches.
-    class BacklogHTTPServer(ThreadingHTTPServer):
-        request_queue_size = 128
-
-    httpd = BacklogHTTPServer((args.bind, 0), factory)
+    httpd = ThreadingHTTPServer((args.bind, 0), factory)
 
     if args.tls:
         import ssl