htszlib: fall back to identity only on a deflate data error

Adversarial review found two over-broad paths to the verbatim copy: a
truncated raw deflate stream (no header, exhausts input without a data
error) and any local failure (fwrite, inflateInit2, fseek, FOPEN) also
reached it, saving compressed garbage as the document with a success
return. Gate the fallback on the raw-deflate attempt hitting a data
error with no environmental failure. Selftest grows a multi-chunk
identity body and truncated gzip/zlib/raw-deflate negatives; the /big/
crawl now serves a mislabeled plain page end-to-end.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/codeql.yml

@@ -43,6 +43,12 @@ jobs:
           languages: c-cpp
           build-mode: manual
           queries: security-extended
+          # fopen's umask-controlled 0666 is intended for mirror/cache/log
+          # output; the one credential file (cookies.txt) is kept 0600 on Unix.
+          config: |
+            query-filters:
+              - exclude:
+                  id: cpp/world-writable-file-creation
 
       # Manual build: CodeQL traces the compiler, so build exactly what ships.
       - name: Build

fuzz/Makefile.am

@@ -36,6 +36,7 @@ EXTRA_DIST = README.md run-fuzzers.sh \
 	corpus/unescape/percent.txt \
 	corpus/filters/filter.bin corpus/filters/filter-size.bin \
 	corpus/filters/regress-empty-subject-unique.bin \
+	corpus/filters/redos-star-classes.bin \
 	corpus/url/http-url.txt corpus/url/relative-path.txt \
 	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
 	corpus/header/full-response.txt corpus/header/redirect.txt \

fuzz/fuzz-cachendx.c

@@ -27,7 +27,7 @@ Please visit our Website: http://www.httrack.com
 
 /* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
    walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
-   Mirrors the loader in cache_readex_new (htscache.c). */
+   Mirrors the -#C cache-listing scan (htscoremain.c). */
 #include "fuzz.h"
 #include "htscache.h"
 #include "htslib.h"
@@ -42,9 +42,8 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   a += cache_brstr(a, firstline, sizeof(firstline));
   a += cache_brstr(a, firstline, sizeof(firstline));
 
-  /* body: newline-delimited host/file/position triples. The coucal insert
-     the real loader ends with is a separate library's concern; this harness
-     targets the length-prefixed scan that must stay inside the buffer. */
+  /* body: newline-delimited host/file/position triples; the length-prefixed
+     scan must stay inside the buffer */
   while (a != NULL && a < end) {
     char BIGSTK line[HTS_URLMAXSIZE * 2];
     char linepos[256];

fuzz/fuzz-header.c

@@ -50,9 +50,13 @@ int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
   while (p != NULL && *p != '\0') {
     char *nl = strchr(p, '\n');
     size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
+    size_t i, len = 0;
 
-    memcpy(line, p, n);
-    line[n] = '\0';
+    /* binput drops every '\r' on the wire; mirror it */
+    for (i = 0; i < n; i++)
+      if (p[i] != '\r')
+        line[len++] = p[i];
+    line[len] = '\0';
     if (first) {
       treatfirstline(&r, line);
       first = 0;

fuzz/run-fuzzers.sh

@@ -2,8 +2,8 @@
 # Drive every built harness against its seed corpus.
 #   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
 #   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
-# Replay is crash/leak-only and never mutates, so it can't hit strjoker's
-# catastrophic backtracking; the timed mode can, hence the per-unit -timeout.
+# Replay is crash/leak-only and never mutates; the per-unit -timeout guards
+# both modes against pathological slowdowns (e.g. pre-#501 strjoker).
 set -euo pipefail
 
 srcdir=$(cd "$(dirname "$0")" && pwd)

src/htsback.c

@@ -1646,138 +1646,104 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       }
     }
     // tester cache
-    if ((strcmp(adr, "file://"))        /* pas fichier */
-        &&((!test) || (cache->type == 1))       /* cache prioritaire, laisser passer en test! */
-        &&((strnotempty(save)) || (strcmp(fil, "/robots.txt") == 0))) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
-#if HTS_FAST_CACHE
+    if ((strcmp(adr, "file://")) /* pas fichier */
+        && ((!test) ||
+            (cache->type == 1)) /* cache prioritaire, laisser passer en test! */
+        &&
+        ((strnotempty(save)) || (strcmp(fil, "/robots.txt") ==
+                                 0))) { // si en test on ne doit pas utiliser le
+                                        // cache sinon telescopage avec le 302..
       intptr_t hash_pos;
       int hash_pos_return = 0;
-#else
-      char *a = NULL;
-#endif
-#if HTS_FAST_CACHE
+
       if (cache->hashtable) {
-#else
-      if (cache->use) {
-#endif
         char BIGSTK buff[HTS_URLMAXSIZE * 4];
 
-#if HTS_FAST_CACHE
         strcpybuff(buff, adr);
         strcatbuff(buff, fil);
         hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
-#else
-        buff[0] = '\0';
-        strcatbuff(buff, "\n");
-        strcatbuff(buff, adr);
-        strcatbuff(buff, "\n");
-        strcatbuff(buff, fil);
-        strcatbuff(buff, "\n");
-        a = strstr(cache->use, buff);
-#endif
 
-        // Ok, noté en cache->. mais bien présent dans le cache ou sur disque?
-#if HTS_FAST_CACHE
         // negative values when data is not in cache
         if (hash_pos_return < 0) {
-#else
-        if (a) {
-#endif
-          if (!test) {          // non mode test
-#if HTS_FAST_CACHE==0
-            int pos = -1;
+          if (!test) { // not test mode
+            /* note: no check with IS_DELAYED_EXT() enabled - postcheck by
+             * client please! */
+            if (save[0] != '\0' && !IS_DELAYED_EXT(save) &&
+                fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <=
+                    0) { // final file missing or empty
+              int found = 0;
 
-            a += strlen(buff);
-            sscanf(a, "%d", &pos);      // lire position
-#endif
+              /* It is possible that the file has been moved due to changes in
+               * build structure */
+              {
+                char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
+                htsblk r;
 
-#if HTS_FAST_CACHE==0
-            if (pos < 0) {      // pas de mise en cache data, vérifier existence
-#endif
-              /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
-              if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <= 0) {  // fichier final n'existe pas ou est vide!
-                int found = 0;
-
-                /* It is possible that the file has been moved due to changes in build structure */
-                {
-                  char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
-                  htsblk r;
-
-                  previous_save[0] = '\0';
-                  r =
-                    cache_readex(opt, cache, adr, fil, /*head */ NULL,
+                previous_save[0] = '\0';
+                r = cache_readex(opt, cache, adr, fil, /*head */ NULL,
                                  /*bound to back[p] (temporary) */
                                  back[p].location_buffer, previous_save, /*ro */
                                  1);
-                  /* Is supposed to be on disk only */
-                  if (r.is_write && previous_save[0] != '\0') {
-                    /* Exists, but with another (old) filename: rename (almost) silently */
-                    if (strcmp(previous_save, save) != 0
-                        && fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) {
-                      rename(fconv(catbuff, sizeof(catbuff), previous_save),
-                             fconv(catbuff2, sizeof(catbuff2), save));
-                      if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
-                        found = 1;
-                        hts_log_print(opt, LOG_DEBUG,
-                                      "File '%s' has been renamed since last mirror to '%s' ; applying changes",
-                                      previous_save, save);
-                      } else {
-                        hts_log_print(opt, LOG_ERROR,
-                                      "Could not rename '%s' to '%s' ; will have to retransfer it",
-                                      previous_save, save);
-                      }
+                /* Is supposed to be on disk only */
+                if (r.is_write && previous_save[0] != '\0') {
+                  /* Exists, but with another (old) filename: rename (almost)
+                   * silently */
+                  if (strcmp(previous_save, save) != 0 &&
+                      fexist_utf8(
+                          fconv(catbuff, sizeof(catbuff), previous_save))) {
+                    rename(fconv(catbuff, sizeof(catbuff), previous_save),
+                           fconv(catbuff2, sizeof(catbuff2), save));
+                    if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
+                      found = 1;
+                      hts_log_print(opt, LOG_DEBUG,
+                                    "File '%s' has been renamed since last "
+                                    "mirror to '%s' ; applying changes",
+                                    previous_save, save);
+                    } else {
+                      hts_log_print(opt, LOG_ERROR,
+                                    "Could not rename '%s' to '%s' ; will have "
+                                    "to retransfer it",
+                                    previous_save, save);
                     }
                   }
-                  back[p].location_buffer[0] = '\0';
                 }
+                back[p].location_buffer[0] = '\0';
+              }
 
-                /* Not found ? */
-                if (!found) {
-#if HTS_FAST_CACHE
-                  hash_pos_return = 0;
-#else
-                  a = NULL;
-#endif
-                  // dévalider car non présent sur disque dans structure originale!!!
-                  // sinon, le fichier est ok à priori, mais on renverra un if-modified-since pour
-                  // en être sûr
-                  if (opt->norecatch) { // tester norecatch
-                    if (!fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {   // fichier existe pas mais déclaré: on l'a effacé
-                      FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
+              /* Not found ? */
+              if (!found) {
+                // invalidate: gone from disk, force a refetch
+                hash_pos_return = 0;
+                if (opt->norecatch) {
+                  if (!fexist_utf8(fconv(
+                          catbuff, sizeof(catbuff),
+                          save))) { // declared but missing: user erased it
+                    FILE *fp =
+                        FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
 
-                      if (fp)
-                        fclose(fp);
-                      hts_log_print(opt, LOG_WARNING,
-                                    "Previous file '%s' not found (erased by user ?), ignoring: %s%s",
-                                    save, back[p].url_adr, back[p].url_fil);
-                    }
-                  } else {
+                    if (fp)
+                      fclose(fp);
                     hts_log_print(opt, LOG_WARNING,
-                                  "Previous file '%s' not found (erased by user ?), recatching: %s%s",
+                                  "Previous file '%s' not found (erased by "
+                                  "user ?), ignoring: %s%s",
                                   save, back[p].url_adr, back[p].url_fil);
                   }
+                } else {
+                  hts_log_print(opt, LOG_WARNING,
+                                "Previous file '%s' not found (erased by user "
+                                "?), recatching: %s%s",
+                                save, back[p].url_adr, back[p].url_fil);
                 }
-              }                 // fsize() <= 0
-#if HTS_FAST_CACHE==0
-            }
-#endif
+              }
+            } // fsize() <= 0
           }
         }
         //
       } else {
-#if HTS_FAST_CACHE
         hash_pos_return = 0;
-#else
-        a = NULL;
-#endif
       }
 
-      // Existe pas en cache, ou bien pas de cache présent
-#if HTS_FAST_CACHE
-      if (hash_pos_return) {    // OK existe en cache (et données aussi)!
-#else
-      if (a != NULL) {          // OK existe en cache (et données aussi)!
-#endif
+      if (hash_pos_return) { // in cache, with data
         const int cache_is_prioritary = cache->type == 1
           || opt->state.stop != 0;
         if (cache_is_prioritary) {      // cache prioritaire (pas de test if-modified..)
@@ -1898,7 +1864,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #endif
         }
       }
-      /* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */
+      /* Not in cache ; maybe in temporary cache ? Warning: non-movable
+         "url_sav" */
       else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
         const off_t file_size = fsize_utf8(itemback->url_sav);
 

src/htsbauth.c

@@ -307,14 +307,25 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   return -1;
 }
 
-// écrire cookies.txt
-// !=0 : erreur
+/* Write cookies.txt; returns 0 on success. The jar holds live session
+   cookies, so keep it owner-only on Unix (Windows inherits folder ACLs). */
 int cookie_save(t_cookie * cookie, const char *name) {
   char catbuff[CATBUFF_SIZE];
 
   if (strnotempty(cookie->data)) {
     char BIGSTK line[8192];
+#ifdef _WIN32
     FILE *fp = fopen(fconv(catbuff, sizeof(catbuff), name), "wb");
+#else
+    const int fd = open(fconv(catbuff, sizeof(catbuff), name),
+                        O_WRONLY | O_CREAT | O_TRUNC, HTS_PROTECT_FILE);
+    FILE *fp = fd != -1 ? fdopen(fd, "wb") : NULL;
+
+    if (fd != -1 && fp == NULL)
+      close(fd);    /* fdopen failed: don't leak the descriptor */
+    if (fp != NULL) /* O_CREAT's mode skips pre-existing jars: tighten those */
+      (void) fchmod(fd, HTS_PROTECT_FILE);
+#endif
 
     if (fp) {
       char *a = cookie->data;

src/htscache.h

@@ -89,11 +89,6 @@ typedef enum {
    the involved files are absent. */
 void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
 
-int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
-                    const char *str2, char *outbuff, int len);
-int cache_readdata(cache_back * cache, const char *str1, const char *str2,
-                   char **inbuff, int *len);
-
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);

src/htscache_selftest.c

@@ -78,14 +78,6 @@ static void selftest_open_for_read(cache_back *cache, httrackp *opt) {
 }
 
 static void selftest_close(cache_back *cache) {
-  if (cache->dat != NULL) {
-    fclose(cache->dat);
-    cache->dat = NULL;
-  }
-  if (cache->ndx != NULL) {
-    fclose(cache->ndx);
-    cache->ndx = NULL;
-  }
   if (cache->zipOutput != NULL) {
     zipClose(cache->zipOutput,
              "Created by HTTrack Website Copier (cache self-test)");
@@ -989,26 +981,15 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
 
-  /* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
-     zip cache existed) */
+  /* PROMOTE: the removed pre-3.31 legacy pair is left alone */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", TINY);
   hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
-
-  /* PROMOTE: a half-written legacy new pair is replaced by the old pair */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.dat", TINY);
-  reconcile_put(opt, "hts-cache/old.dat", SOLID);
-  reconcile_put(opt, "hts-cache/old.ndx", TINY);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", -1, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.ndx", TINY, "promote-dat");
 
   /* INTERRUPTED: no lock file, no action */
   reconcile_wipe(opt);
@@ -1058,7 +1039,7 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
 
-  /* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
+  /* INTERRUPTED: the removed pre-3.31 legacy pair is left alone */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-in_progress.lock", 0);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
@@ -1067,9 +1048,13 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
   failures +=
-      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.dat", TINY, "interrupted-dat");
   failures +=
-      reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.dat", SOLID, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.ndx", MID, "interrupted-dat");
 
   /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
      its only good generation here) */
@@ -1089,17 +1074,18 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
 
-  /* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
+  /* ROLLBACK: sidecars restored, the removed pre-3.31 pair left alone */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
-  reconcile_put(opt, "hts-cache/new.ndx", TINY);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   reconcile_put(opt, "hts-cache/old.lst", MID);
   reconcile_put(opt, "hts-cache/old.txt", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", TINY, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.ndx", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
 
@@ -1113,6 +1099,101 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   return failures;
 }
 
+/* The pre-3.31 .dat/.ndx import was removed: cache_init must refuse the
+   legacy pair, leave the files in place, and not flag an update run. */
+int cache_legacy_refused_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+  int variant;
+
+  selftest_tag = "cache-legacy";
+  golden_setup(opt, dir);
+#ifdef _WIN32
+  mkdir(reconcile_st_path(opt, "hts-cache"));
+#else
+  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
+#endif
+
+  /* variant 0: old.dat/old.ndx (--update layout); 1: new.dat/new.ndx (ro) */
+  for (variant = 0; variant < 2; variant++) {
+    cache_back cache;
+    const char *const dat =
+        variant == 0 ? "hts-cache/old.dat" : "hts-cache/new.dat";
+    const char *const ndx =
+        variant == 0 ? "hts-cache/old.ndx" : "hts-cache/new.ndx";
+
+    reconcile_wipe(opt);
+    reconcile_put(opt, dat, 4096);
+    reconcile_put(opt, ndx, 4096);
+    opt->is_update = 0;
+    selftest_open(&cache, opt, /*ro*/ variant == 1);
+    if (cache_readable(&cache)) {
+      printf("cache-legacy: variant %d imported a removed format\n", variant);
+      failures++;
+    }
+    if (coucal_nitems(cache.hashtable) != 0) { /* no phantom index entries */
+      printf("cache-legacy: variant %d imported index entries\n", variant);
+      failures++;
+    }
+    if (opt->is_update) {
+      printf("cache-legacy: variant %d flagged an update\n", variant);
+      failures++;
+    }
+    if (fsize(reconcile_st_path(opt, dat)) != 4096 ||
+        fsize(reconcile_st_path(opt, ndx)) != 4096) {
+      printf("cache-legacy: variant %d touched the legacy files\n", variant);
+      failures++;
+    }
+    if (cache.lst != NULL) {
+      fclose(cache.lst);
+      cache.lst = NULL;
+    }
+    if (cache.txt != NULL) {
+      fclose(cache.txt);
+      cache.txt = NULL;
+    }
+    selftest_close(&cache);
+  }
+
+  /* a healthy zip must win over stray legacy files, with no refusal */
+  {
+    cache_back cache;
+    const char *const body = "<html>zip wins</html>";
+
+    reconcile_wipe(opt);
+    selftest_open_for_write(&cache, opt);
+    store_entry(opt, &cache, "example.com", "/", "example.com/index.html", 200,
+                "OK", "text/html", "utf-8", "", "", "", "", body, strlen(body));
+    selftest_close(&cache);
+    reconcile_put(opt, "hts-cache/old.ndx", 4096);
+    reconcile_put(opt, "hts-cache/old.dat", 4096);
+    selftest_open_for_read(&cache, opt);
+    if (!cache_readable(&cache)) {
+      printf("cache-legacy: stray legacy files shadowed the zip cache\n");
+      failures++;
+    }
+    failures +=
+        check_entry(opt, &cache, "example.com", "/", 200, "OK", "text/html",
+                    "utf-8", "", "", "", "", body, strlen(body));
+    if (fsize(reconcile_st_path(opt, "hts-cache/old.ndx")) != 4096 ||
+        fsize(reconcile_st_path(opt, "hts-cache/old.dat")) != 4096) {
+      printf("cache-legacy: zip-wins pass touched the legacy files\n");
+      failures++;
+    }
+    if (cache.lst != NULL) {
+      fclose(cache.lst);
+      cache.lst = NULL;
+    }
+    if (cache.txt != NULL) {
+      fclose(cache.txt);
+      cache.txt = NULL;
+    }
+    selftest_close(&cache);
+  }
+
+  reconcile_wipe(opt);
+  return failures;
+}
+
 /* --- read-side corruption injection --------------------------------------- */
 
 /* canary read back intact after each corruption; victim gets the byte surgery

src/htscache_selftest.h

@@ -60,6 +60,10 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir);
    under <dir>. Returns the failed-check count. */
 int cache_reconcile_selftest(httrackp *opt, const char *dir);
 
+/* Verify cache_init refuses a pre-3.31 .dat/.ndx cache without touching it.
+   Returns the number of failed checks (0 = pass). */
+int cache_legacy_refused_selftest(httrackp *opt, const char *dir);
+
 /* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
    under <dir> and assert every case degrades to STATUSCODE_INVALID without
    tainting a sibling entry. */

src/htsconfig.h

@@ -76,9 +76,6 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
-// fast cache (build hash table)
-#define HTS_FAST_CACHE 1
-
 // le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
 // valide)
 #define GT_ENDS_COMMENT 1

src/htscore.c

@@ -134,18 +134,6 @@ RUN_CALLBACK0(opt, end); \
     back_delete_all(opt, &cache, sback);                                       \
     back_free(&sback);                                                         \
     checkrobots_free(&robots);                                                 \
-    if (cache.use) {                                                           \
-      freet(cache.use);                                                        \
-      cache.use = NULL;                                                        \
-    }                                                                          \
-    if (cache.dat) {                                                           \
-      fclose(cache.dat);                                                       \
-      cache.dat = NULL;                                                        \
-    }                                                                          \
-    if (cache.ndx) {                                                           \
-      fclose(cache.ndx);                                                       \
-      cache.ndx = NULL;                                                        \
-    }                                                                          \
     if (cache.zipOutput) {                                                     \
       zipClose(cache.zipOutput,                                                \
                "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \
@@ -155,10 +143,6 @@ RUN_CALLBACK0(opt, end); \
       unzClose(cache.zipInput);                                                \
       cache.zipInput = NULL;                                                   \
     }                                                                          \
-    if (cache.olddat) {                                                        \
-      fclose(cache.olddat);                                                    \
-      cache.olddat = NULL;                                                     \
-    }                                                                          \
     if (cache.lst) {                                                           \
       fclose(cache.lst);                                                       \
       cache.lst = opt->state.strc.lst = NULL;                                  \
@@ -488,32 +472,11 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
   *makeindex_done = 1;
 }
 
-/* Flush the parsed HTML output buffer to disk, skipping the rewrite when the
- * on-disk MD5 is unchanged. */
+/* Flush the parsed HTML output buffer to disk. */
 void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
                           FILE **fp, const char *ht_buff, size_t ht_len,
                           const char *adr, const char *fil, const char *save) {
-  char digest[32 + 2];
-  off_t fsize_old =
-      fsize(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), save));
-  int ok = 0;
-
-  digest[0] = '\0';
-  domd5mem(ht_buff, ht_len, digest, 1);
-  if (fsize_old == (off_t) ht_len) {
-    int mlen = 0;
-    char *mbuff;
-
-    cache_readdata(cache, "//[HTML-MD5]//", save, &mbuff, &mlen);
-    if (mlen)
-      mbuff[mlen] = '\0';
-    if ((mlen == 32) && (strcmp(((mbuff != NULL) ? mbuff : ""), digest) == 0)) {
-      ok = 1;
-      hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s", save);
-    }
-    freet(mbuff);
-  }
-  if (!ok) {
+  {
     file_notify(opt, adr, fil, save, 1, 1, r->notmodified);
     *fp = filecreate(&opt->state.strc, save);
     if (*fp) {
@@ -545,13 +508,7 @@ void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
       if (fcheck)
         hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
     }
-  } else {
-    file_notify(opt, adr, fil, save, 0, 0, r->notmodified);
-    filenote(&opt->state.strc, save, NULL);
   }
-  if (cache->ndx)
-    cache_writedata(cache->ndx, cache->dat, "//[HTML-MD5]//", save, digest,
-                    (int) strlen(digest));
 }
 
 /* does it look like XML ? (SVG et al.) */

src/htscore.h

@@ -193,8 +193,7 @@ struct cache_back {
   /* */
   int type;
   int ro;                   /**< read-only: no new cache is written */
-  FILE *dat, *ndx, *olddat; /**< new data, new index, old data files */
-  char *use;                /**< in-memory list of cached adr+fil keys */
+
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
   char lastmodified[256];
@@ -288,12 +287,12 @@ struct filecreate_params {
 
 /* True if a new cache is being written (plain or zip backend). */
 HTS_STATIC int cache_writable(cache_back * cache) {
-  return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL));
+  return (cache != NULL && cache->zipOutput != NULL);
 }
 
 /* True if an old cache is available to read (plain or zip backend). */
 HTS_STATIC int cache_readable(cache_back * cache) {
-  return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL));
+  return (cache != NULL && cache->zipInput != NULL);
 }
 
 #endif

src/htsfilters.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsbase.h"
 #include "htslib.h"
 #include <ctype.h>
+#include <stdint.h>
 /* END specific definitions */
 
 // à partir d'un tableau de {"+*.toto","-*.zip","+*.tata"} définit si nom est autorisé
@@ -120,16 +121,81 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
   return use1 ? jok1 : jok2;
 }
 
-// supercomparateur joker (tm)
-// compare a et b (b=avec joker dedans), case insensitive [voir CI]
-// renvoi l'adresse de la première lettre de la chaine
-// (càd *[..]toto.. renvoi adresse de toto dans la chaine)
-// accepte les délires du genre www.*.*/ * / * truc*.*
-// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
-// * = toute lettre
-// --?-- : spécifique à HTTrack et aux ?
+/* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
+   pair keeps star-heavy patterns polynomial instead of exponential (#501). */
+typedef struct strjoker_memo {
+  const char *chaine0, *joker0; /* offsets are relative to these bases */
+  size_t stride;                /* strlen(joker0) + 1 */
+  unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
+} strjoker_memo;
+
+static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
+                                 const char *joker, LLint *size,
+                                 int *size_flag);
+
+/* A pair that failed once fails forever (*size is only written on the success
+   path), so record NULL results and cut the re-exploration. */
+static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
+                                const char *joker, LLint *size,
+                                int *size_flag) {
+  size_t bit = 0;
+  const char *adr;
+
+  if (memo->failed) {
+    bit = (size_t) (chaine - memo->chaine0) * memo->stride +
+          (size_t) (joker - memo->joker0);
+    if (memo->failed[bit >> 3] & (unsigned char) (1u << (bit & 7)))
+      return NULL;
+  }
+  adr = strjoker_impl(memo, chaine, joker, size, size_flag);
+  if (adr == NULL && memo->failed)
+    memo->failed[bit >> 3] |= (unsigned char) (1u << (bit & 7));
+  return adr;
+}
+
+// wildcard comparator: match chaine against joker (pattern), case-insensitive
+// returns the address of the first matched letter past any leading joker
+// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
                                 LLint *size, int *size_flag) {
+  strjoker_memo memo = {chaine, joker, 0, NULL};
+  unsigned char stackbits[2048];
+  hts_boolean onheap = HTS_FALSE;
+  const char *adr;
+
+  if (chaine != NULL && joker != NULL) {
+    const size_t n1 = strlen(chaine) + 1;
+
+    memo.stride = strlen(joker) + 1;
+    if (n1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
+      const size_t bytes = (n1 * memo.stride + 7) / 8;
+
+      if (bytes <= sizeof(stackbits)) {
+        memset(stackbits, 0, bytes);
+        memo.failed = stackbits;
+      } else {
+        memo.failed = (unsigned char *) calloct(bytes, 1);
+        onheap = memo.failed != NULL ? HTS_TRUE : HTS_FALSE;
+      }
+    }
+  }
+  adr = strjoker_rec(&memo, chaine, joker, size, size_flag);
+  if (onheap)
+    freet(memo.failed);
+  return adr;
+}
+
+/* Test-only oracle: the same matcher without the failure memo. */
+const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
+                            int *size_flag) {
+  const strjoker_memo memo = {chaine, joker, 0, NULL};
+
+  return strjoker_rec(&memo, chaine, joker, size, size_flag);
+}
+
+static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
+                                 const char *joker, LLint *size,
+                                 int *size_flag) {
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -303,7 +369,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
 
       // tester sans le joker (pas ()+ mais ()*)
       if (!unique) {
-        if ((adr = strjoker(chaine, joker + jmp, size, size_flag))) {
+        if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag))) {
           return adr;
         }
       }
@@ -315,7 +381,8 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
         max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
-          if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
+          if ((adr = strjoker_rec(memo, chaine + i + 1, joker + jmp, size,
+                                  size_flag))) {
             return adr;
           }
           i++;
@@ -325,7 +392,8 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
 
       // tester chaîne vide
       if (i != max + 2)         // avant c'est ok
-        if ((adr = strjoker(chaine + max, joker + jmp, size, size_flag)))
+        if ((adr = strjoker_rec(memo, chaine + max, joker + jmp, size,
+                                size_flag)))
           return adr;
 
       return NULL;              // perdu
@@ -347,7 +415,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
       // comparaison ok?
       if (ok) {
         // continuer la comparaison.
-        if (strjoker(chaine + jmp, joker + jmp, size, size_flag))
+        if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag))
           return chaine;        // retourner 1e lettre
       }
 

src/htsfilters.h

@@ -48,6 +48,10 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
                      const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
+/* strjoker() without the failure memo (exponential worst case); test-only
+   oracle for the memoized matcher. */
+const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
+                            int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);
 #endif
 

src/htsglobal.h

@@ -422,9 +422,10 @@ typedef int T_SOC;
 #endif
 
 /* Permission bits for created folders and files (mkdir and chmod).
-   PROTECT_FOLDER is owner-only. With HTS_ACCESS set (the default) the ACCESS_
-   modes also grant group/other read; otherwise they stay owner-only. */
+   PROTECT_FOLDER/FILE are owner-only. With HTS_ACCESS set (the default) the
+   ACCESS_ modes also grant group/other read; otherwise they stay owner-only. */
 #define HTS_PROTECT_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
+#define HTS_PROTECT_FILE (S_IRUSR | S_IWUSR)
 
 #if HTS_ACCESS
 #define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)

src/htsselftest.c

@@ -621,6 +621,105 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Mime-type filter verdict via fa_strjoker(type=1): only mime: rules apply. */
+static int st_filtermime(httrackp *opt, int argc, char **argv) {
+  int verdict;
+
+  (void) opt;
+  if (argc < 2) {
+    fprintf(stderr, "filtermime: needs <mime> <filter> [filter...]\n");
+    return 1;
+  }
+  verdict = fa_strjoker(1, &argv[1], argc - 1, argv[0], NULL, NULL, NULL);
+  printf("verdict=%s\n", verdict > 0   ? "allowed"
+                         : verdict < 0 ? "forbidden"
+                                       : "unknown");
+  return 0;
+}
+
+/* SplitMix64: deterministic, platform-independent case generator. */
+static uint64_t st_mix64(uint64_t *state) {
+  uint64_t z = (*state += UINT64_C(0x9E3779B97F4A7C15));
+
+  z = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4B5B9);
+  z = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB);
+  return z ^ (z >> 31);
+}
+
+/* Differential test: memoized strjoker vs the no-memo oracle on seeded random
+   pattern/subject/size cases must agree on result, *size and *size_flag. */
+static int st_filtermemo(httrackp *opt, int argc, char **argv) {
+  static const char *const pieces[] = {
+      "a",       "b",        "A",    "c",      ".",       "/",
+      "?",       "*",        "*[a]", "*[ab]",  "*[a-c]",  "*[A-Z]",
+      "*[<5]",   "*[>5]",    "*(a)", "*(a,b)", "*[file]", "*[path]",
+      "*[name]", "*[param]", "*[]",  "*[\\a]", "*[a-"};
+  static const char subject_chars[] = "abAc./?";
+  uint64_t rng = UINT64_C(0x501);
+  int iters = 20000, matched = 0, unmatched = 0;
+  int it;
+
+  (void) opt;
+  if (argc >= 1)
+    sscanf(argv[0], "%d", &iters);
+  for (it = 0; it < iters; it++) {
+    char pat[64], str[16];
+    size_t pl = 0;
+    const int npieces = (int) (st_mix64(&rng) % 5);
+    const int slen = (int) (st_mix64(&rng) % 13);
+    const int szsel = (int) (st_mix64(&rng) % 4); /* none/unknown/3/20 */
+    LLint sz1, sz2, off1, off2;
+    int i, flag1 = 0, flag2 = 0;
+    char *hpat, *hstr;
+    const char *adr;
+
+    for (i = 0; i < npieces; i++) {
+      const char *p =
+          pieces[st_mix64(&rng) % (sizeof(pieces) / sizeof(pieces[0]))];
+      const size_t len = strlen(p);
+
+      if (len >= sizeof(pat) - pl)
+        break;
+      memcpy(pat + pl, p, len);
+      pl += len;
+    }
+    pat[pl] = '\0';
+    for (i = 0; i < slen; i++)
+      str[i] = subject_chars[st_mix64(&rng) % (sizeof(subject_chars) - 1)];
+    str[slen] = '\0';
+    sz1 = sz2 = (szsel <= 1) ? -1 : (szsel == 2) ? 3 : 20;
+    /* exact-size heap copies per run so a sanitizer traps any over-read */
+    hpat = strdupt(pat);
+    hstr = strdupt(str);
+    adr = strjoker(hstr, hpat, szsel ? &sz1 : NULL, szsel ? &flag1 : NULL);
+    off1 = adr != NULL ? (LLint) (adr - hstr) : -1;
+    freet(hpat);
+    freet(hstr);
+    hpat = strdupt(pat);
+    hstr = strdupt(str);
+    adr =
+        strjoker_nomemo(hstr, hpat, szsel ? &sz2 : NULL, szsel ? &flag2 : NULL);
+    off2 = adr != NULL ? (LLint) (adr - hstr) : -1;
+    freet(hpat);
+    freet(hstr);
+    if (off1 != off2 || sz1 != sz2 || flag1 != flag2) {
+      printf("filtermemo MISMATCH pat=[%s] str=[%s] szsel=%d: "
+             "off %d/%d size %d/%d flag %d/%d\n",
+             pat, str, szsel, (int) off1, (int) off2, (int) sz1, (int) sz2,
+             flag1, flag2);
+      return 1;
+    }
+    if (off1 >= 0)
+      matched++;
+    else
+      unmatched++;
+  }
+  /* both polarities must actually occur or the test proves nothing */
+  assertf(matched > 0 && unmatched > 0);
+  printf("filtermemo: %d cases OK\n", iters);
+  return 0;
+}
+
 /* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
 static int st_filterdual(httrackp *opt, int argc, char **argv) {
   int depth = -1, verdict;
@@ -1500,6 +1599,18 @@ static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+static int st_cache_legacy(httrackp *opt, int argc, char **argv) {
+  int err;
+
+  if (argc < 1) {
+    fprintf(stderr, "cache-legacy: needs a directory\n");
+    return 1;
+  }
+  err = cache_legacy_refused_selftest(opt, argv[0]);
+  printf("cache-legacy: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 static int st_reconcile(httrackp *opt, int argc, char **argv) {
   int err;
 
@@ -1551,6 +1662,26 @@ static int st_cookies(httrackp *opt, int argc, char **argv) {
     err = 1;
   if (strstr(hdr, "junk") != NULL) // wrong-domain cookie leaked
     err = 1;
+#ifndef _WIN32
+  /* the jar holds live session cookies: cookie_save must keep it 0600 */
+  {
+    const char *jar = "st-cookies-jar.txt";
+    struct stat st;
+
+    (void) UNLINK(jar);
+    assertf(cookie_save(&cookie, jar) == 0);
+    assertf(stat(jar, &st) == 0);
+    assertf((st.st_mode & 07777) == HTS_PROTECT_FILE);
+    assertf(st.st_size > 0); /* mode-only checks would pass an empty jar */
+    /* a pre-existing world-readable jar must be tightened, not kept */
+    assertf(chmod(jar, 0644) == 0);
+    assertf(cookie_save(&cookie, jar) == 0);
+    assertf(stat(jar, &st) == 0);
+    assertf((st.st_mode & 07777) == HTS_PROTECT_FILE);
+    assertf(st.st_size > 0);
+    (void) UNLINK(jar);
+  }
+#endif
   printf("cookie-header: %s\n", err ? "FAIL" : "OK");
   if (err)
     printf("  got: %s\n", hdr);
@@ -1992,6 +2123,19 @@ static int ae_write_collision(const char *path, const unsigned char *src,
   return ok ? 0 : 1;
 }
 
+/* Write src[0..len) to path as-is; 0 on success. */
+static int ae_write_raw(const char *path, const unsigned char *src,
+                        size_t len) {
+  FILE *const f = FOPEN(path, "wb");
+  int ok;
+
+  if (f == NULL)
+    return 1;
+  ok = fwrite(src, 1, len, f) == len;
+  fclose(f);
+  return ok ? 0 : 1;
+}
+
 /* Compare path's bytes to expect[0..len); 0 if equal. Streams (large files). */
 static int ae_check_decoded(const char *path, const unsigned char *expect,
                             size_t len) {
@@ -2059,6 +2203,46 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
     assertf(ae_write_collision(inpath, body, 64) == 0);
     assertf(hts_zunpack(inpath, outpath) == 64);
     assertf(ae_check_decoded(outpath, body, 64) == 0);
+    /* Identity fallback (#47): a plain body mislabeled as compressed is kept
+       verbatim, small and multi-chunk (> one 8 KiB fread). */
+    assertf(ae_write_raw(inpath, small, slen) == 0);
+    assertf(hts_zunpack(inpath, outpath) == (int) slen);
+    assertf(ae_check_decoded(outpath, small, slen) == 0);
+    {
+      const size_t ilen = 16 * 1024;
+      unsigned char *idbody = malloct(ilen);
+
+      assertf(idbody != NULL);
+      for (i = 0; i < ilen; i++)
+        idbody[i] = small[i % slen];
+      assertf(ae_write_raw(inpath, idbody, ilen) == 0);
+      assertf(hts_zunpack(inpath, outpath) == (int) ilen);
+      assertf(ae_check_decoded(outpath, idbody, ilen) == 0);
+      freet(idbody);
+    }
+    /* Truncated gzip (CRC+ISIZE cut), zlib (ADLER32 cut) and raw deflate
+       must all still fail, not fall back to a verbatim copy. */
+    {
+      static const struct {
+        int wb;
+        size_t cut;
+      } tr[] = {{16 + MAX_WBITS, 8}, {MAX_WBITS, 4}, {-MAX_WBITS, 5}};
+
+      for (i = 0; i < sizeof(tr) / sizeof(tr[0]); i++) {
+        unsigned char z[512];
+        size_t zlen;
+        FILE *f;
+
+        assertf(ae_write_packed(inpath, tr[i].wb, small, slen) == 0);
+        f = FOPEN(inpath, "rb");
+        assertf(f != NULL);
+        zlen = fread(z, 1, sizeof(z), f);
+        fclose(f);
+        assertf(zlen > tr[i].cut && zlen < sizeof(z));
+        assertf(ae_write_raw(inpath, z, zlen - tr[i].cut) == 0);
+        assertf(hts_zunpack(inpath, outpath) < 0);
+      }
+    }
     freet(body);
   }
 #else
@@ -2247,6 +2431,10 @@ static const struct selftest_entry {
     {"filtersize", "<size> <string> <filter>...",
      "size-aware filter verdict (negative size = unknown/scan time)",
      st_filtersize},
+    {"filtermime", "<mime> <filter>...",
+     "mime-type filter verdict (fa_strjoker type=1)", st_filtermime},
+    {"filtermemo", "[iterations]",
+     "memoized vs unmemoized matcher differential", st_filtermemo},
     {"filterdual", "<string1> <string2> <filter>...",
      "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
@@ -2294,6 +2482,8 @@ static const struct selftest_entry {
      st_cache_writefail},
     {"reconcile", "<dir>", "cache generation reconcile policy self-test",
      st_reconcile},
+    {"cache-legacy", "<dir>", "pre-3.31 legacy cache refusal self-test",
+     st_cache_legacy},
     {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
      st_cache_corrupt},
     {"dns", "", "DNS resolver/cache self-test", st_dns},

src/htszlib.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 
 /*
   Unpack file into a new file (gzip, zlib RFC1950 or raw deflate RFC1951).
+  A body provably in no deflate framing is copied verbatim (identity).
   Return value: size of the new file, or -1 if an error occurred
 */
 /* Note: utf-8 */
@@ -68,6 +69,10 @@ int hts_zunpack(char *filename, char *newfile) {
             ((inbuf[0] & 0x0f) == Z_DEFLATED &&
              (((unsigned) inbuf[0] << 8 | inbuf[1]) % 31) == 0)));
       int attempt;
+      /* not_deflate: the raw attempt hit a data error, so the body is in no
+         deflate framing at all. env_error: local I/O or memory failure. */
+      hts_boolean not_deflate = HTS_FALSE;
+      hts_boolean env_error = HTS_FALSE;
 
       /* deflate is ambiguous; on failure retry with the other windowBits */
       for (attempt = 0; attempt < 2 && ret < 0; attempt++) {
@@ -78,15 +83,20 @@ int hts_zunpack(char *filename, char *newfile) {
 
         if (attempt > 0) {
           /* rewind input; reopening fpout "wb" discards the partial output */
-          if (fseek(in, 0, SEEK_SET) != 0)
+          if (fseek(in, 0, SEEK_SET) != 0) {
+            env_error = HTS_TRUE;
             break;
+          }
           navail = fread(inbuf, 1, sizeof(inbuf), in);
         }
         fpout = FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
-        if (fpout == NULL)
+        if (fpout == NULL) {
+          env_error = HTS_TRUE;
           break;
+        }
         memset(&strm, 0, sizeof(strm));
         if (inflateInit2(&strm, windowBits) != Z_OK) {
+          env_error = HTS_TRUE;
           fclose(fpout);
           break;
         }
@@ -108,12 +118,17 @@ int hts_zunpack(char *filename, char *newfile) {
               zerr = inflate(&strm, Z_NO_FLUSH);
               if (zerr == Z_NEED_DICT || zerr == Z_DATA_ERROR ||
                   zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR) {
+                if (zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR)
+                  env_error = HTS_TRUE;
+                else if (windowBits < 0)
+                  not_deflate = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
               produced = sizeof(outbuf) - strm.avail_out;
               if (produced > 0 &&
                   fwrite(outbuf, 1, produced, fpout) != produced) {
+                env_error = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
@@ -129,6 +144,28 @@ int hts_zunpack(char *filename, char *newfile) {
         inflateEnd(&strm);
         fclose(fpout);
       }
+      /* keep a mislabeled identity body verbatim only when provably not
+         deflate; truncation or a local failure must keep failing (#47) */
+      if (ret < 0 && !wrapped && not_deflate && !env_error && !ferror(in)) {
+        FILE *const fpout =
+            FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
+
+        if (fpout != NULL && fseek(in, 0, SEEK_SET) == 0) {
+          int size = 0;
+
+          while ((navail = fread(inbuf, 1, sizeof(inbuf), in)) > 0) {
+            if (fwrite(inbuf, 1, navail, fpout) != navail) {
+              size = -1;
+              break;
+            }
+            size += (int) navail;
+          }
+          if (size >= 0 && !ferror(in))
+            ret = size;
+        }
+        if (fpout != NULL)
+          fclose(fpout);
+      }
       fclose(in);
     }
   }

tests/01_engine-filter.test

@@ -131,3 +131,64 @@ match '*.aspx' 'page.aspx?y=2'
 # empty subject against a unique-match allow-all pattern (heap over-read regress)
 nomatch_empty '*(('
 nomatch_empty '**(('
+
+# near-miss negatives: word order, exact tails, range boundaries, class case
+nomatch 'foo*bar' 'foobaz'
+nomatch '*foo*bar' 'barfoo' # both words present, wrong order
+nomatch '*.gif' 'a.gifx'    # trailing junk after the extension
+nomatch '*.gif' 'a.gi'
+nomatch '*[b-y]' 'a'
+nomatch '*[b-y]' 'z'
+nomatch '*[a-z].txt' 'B.txt' # classes are case-sensitive...
+match 'FoO*' 'foobar'        # ...literals are not
+
+# a star may match an empty run, but never skip a required literal
+match '*[0-9].gif' '.gif'
+nomatch '*[0-9].gif' 'x.gif'
+
+# a descending range *[c-a] is an empty class, not c..a reversed
+match '*[c-a]x' 'x'
+nomatch '*[c-a]x' 'cx'
+
+# *[param] is all-or-nothing on '?': spans everything after one, else nothing
+match 'a*[param]' 'a?x=1'
+nomatch 'a*[param]' 'ab?x=1'
+
+# the latest matching rule wins, regardless of sign
+fsize 'verdict=forbidden size_flag=0' -1 a.gif '+*.gif' '-*.gif'
+fsize 'verdict=allowed size_flag=0' -1 a.gif '-*.gif' '+*.gif'
+
+# a failed size test sets size_flag but leaves the rule unmatched
+fsize 'verdict=unknown size_flag=1' 5 foo.jpg '-*.jpg*[>10]'
+
+# mime filters (-#test=filtermime = fa_strjoker type=1): only mime: rules
+# apply to a mime type, and mime: rules never apply to a URL
+fmime() {
+    local want="$1"
+    shift
+    test "$(httrack -O /dev/null -#test=filtermime "$@")" == "verdict=$want" || exit 1
+}
+fmime allowed 'image/png' '+mime:image/*'
+fmime unknown 'text/html' '+mime:image/*'
+fmime forbidden 'image/png' '+mime:image/*' '-mime:image/png' # last wins
+fmime allowed 'image/png' '-mime:image/png' '+mime:image/*'
+fmime unknown 'image/png' '+image/*'                               # URL rule: skipped
+fsize 'verdict=unknown size_flag=0' -1 'image/png' '+mime:image/*' # and vice versa
+
+# memoized vs no-memo differential over seeded random patterns and subjects,
+# matching and non-matching (the self-test asserts both polarities occurred)
+test "$(httrack -O /dev/null -#test=filtermemo)" == "filtermemo: 20000 cases OK" || exit 1
+
+# star-heavy non-match must stay polynomial (#501); 400 chars also exercises
+# the heap-allocated memo. timeout (where available) turns a hang into a FAIL.
+with_timeout() {
+    if command -v timeout >/dev/null; then timeout 60 "$@"; else "$@"; fi
+}
+pat=$(printf '*[a]%.0s' $(seq 1 12))b
+subj=$(printf 'a%.0s' $(seq 1 400))
+test "$(with_timeout httrack -O /dev/null -#test=filter "$pat" "$subj")" == "$subj does NOT match $pat" || exit 1
+
+# a later star's class dead-end must re-extend the earlier star, not give up
+# (guards against a single-backtrack-point matcher rewrite)
+match '*[aX]X*[a]Y' 'aXaXaY'
+nomatch '*[aX]X*[a]Y' 'aXbXaY'

tests/01_zlib-cache-legacy.test

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# The pre-3.31 .dat/.ndx cache import was removed: cache_init must refuse the
+# legacy pair and leave the files untouched (httrack -#test=cache-legacy <dir>).
+
+set -eu
+
+dir=$(mktemp -d)
+trap 'rm -rf "$dir"' EXIT
+
+# the refusal errors land on stdout (no log file); pin them and the verdict
+out=$(httrack -#test=cache-legacy "$dir" 2>/dev/null)
+cnt=$(printf '%s\n' "$out" | grep -c 'no longer supported' || true)
+test "$cnt" = 2 || {
+    echo "expected 2 refusal messages, got $cnt" >&2
+    exit 1
+}
+out=$(printf '%s\n' "$out" | tail -n1)
+
+test "$out" = "cache-legacy: OK" || {
+    echo "expected 'cache-legacy: OK', got: $out" >&2
+    exit 1
+}

tests/36_local-bigcrawl.test

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 # Diverse seeded /big/ crawl: 12 pattern families, decoy absence, update pass
-# must 304-revalidate. 360 = 1 index + 96 pages + 192 imgs + 5 shared + 60
+# must 304-revalidate. 361 = 1 index + 96 pages + 192 imgs + 5 shared + 61
 # family + 6 singles; the 4 planted errors write -o1 pages, not counted.
 
 set -euo pipefail
@@ -9,7 +9,7 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --errors 4 --files 360 \
+    --errors 4 --files 361 \
     --found 'big/p/95.html' \
     --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
     --found 'big/a/f2-2x.png' \
@@ -40,6 +40,7 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --not-found 'big/x/bj.jar' \
     --not-found 'big/x/is1.png' \
     --not-found 'big/x/concat.html' \
+    --file-matches 'big/f1/gzid.html' '<p>gzid</p>' \
     --file-matches 'big/p/2.html' 'srcset="\.\./a/f2-1x\.png 1x, \.\./a/f2-2x\.png 2x"' \
     --file-matches 'big/a/blk2.css' 'url\(blk2-bg\.png\)' \
     --file-matches 'big/p/5.html' "document\\.write\\('<a href=\"\\.\\./f5/dw\\.html\"" \

tests/Makefile.am

@@ -68,6 +68,7 @@ TESTS = \
 	01_zlib-acceptencoding.test \
 	01_zlib-cache.test \
 	01_zlib-cache-corrupt.test \
+	01_zlib-cache-legacy.test \
 	01_zlib-cache-golden.test \
 	01_zlib-cache-writefail.test \
 	01_zlib-savename-cached.test \

tests/local-server.py

@@ -374,6 +374,7 @@ def big_index(port):
         '<img src="/big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png">'
         '<a href="/big/f1/long.html?x=%s">long</a>'
         '<a href="/big/f1/gzok.html">gzok</a>'
+        '<a href="/big/f1/gzid.html">gzid</a>'
         '<a href="//127.0.0.1:%d/big/f1/protorel.html">protorel</a>'
         '<a href="http://127.0.0.1:%d/big/f1/abshost.html">abshost</a>'
         '<a href="/big/e/404.html">e404</a>'
@@ -399,6 +400,7 @@ BIG_SIMPLE_PAGES = {
     "/big/f1/dir/": "dir index",
     "/big/f1/long.html": "long",
     "/big/f1/gzok.html": "gzok",
+    "/big/f1/gzid.html": "gzid",
     "/big/f1/protorel.html": "protorel",
     "/big/f1/abshost.html": "abshost",
     "/big/f5/dw.html": "dw target",
@@ -1079,6 +1081,13 @@ class Handler(SimpleHTTPRequestHandler):
                     "text/html",
                     extra=[("Content-Encoding", "gzip")],
                 )
+            elif path == "/big/f1/gzid.html":
+                # Plain body mislabeled as gzip: identity fallback keeps it (#47)
+                self.big_send(
+                    body,
+                    "text/html",
+                    extra=[("Content-Encoding", "gzip")],
+                )
             else:
                 self.big_send(body, "text/html")
         elif path == "/big/f1/list.html":