htszlib: fall back to identity only on a deflate data error

Adversarial review found two over-broad paths to the verbatim copy: a
truncated raw deflate stream (no header, exhausts input without a data
error) and any local failure (fwrite, inflateInit2, fseek, FOPEN) also
reached it, saving compressed garbage as the document with a success
return. Gate the fallback on the raw-deflate attempt hitting a data
error with no environmental failure. Selftest grows a multi-chunk
identity body and truncated gzip/zlib/raw-deflate negatives; the /big/
crawl now serves a mislabeled plain page end-to-end.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsback.c

@@ -1646,138 +1646,104 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       }
     }
     // tester cache
-    if ((strcmp(adr, "file://"))        /* pas fichier */
-        &&((!test) || (cache->type == 1))       /* cache prioritaire, laisser passer en test! */
-        &&((strnotempty(save)) || (strcmp(fil, "/robots.txt") == 0))) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
-#if HTS_FAST_CACHE
+    if ((strcmp(adr, "file://")) /* pas fichier */
+        && ((!test) ||
+            (cache->type == 1)) /* cache prioritaire, laisser passer en test! */
+        &&
+        ((strnotempty(save)) || (strcmp(fil, "/robots.txt") ==
+                                 0))) { // si en test on ne doit pas utiliser le
+                                        // cache sinon telescopage avec le 302..
       intptr_t hash_pos;
       int hash_pos_return = 0;
-#else
-      char *a = NULL;
-#endif
-#if HTS_FAST_CACHE
+
       if (cache->hashtable) {
-#else
-      if (cache->use) {
-#endif
         char BIGSTK buff[HTS_URLMAXSIZE * 4];
 
-#if HTS_FAST_CACHE
         strcpybuff(buff, adr);
         strcatbuff(buff, fil);
         hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
-#else
-        buff[0] = '\0';
-        strcatbuff(buff, "\n");
-        strcatbuff(buff, adr);
-        strcatbuff(buff, "\n");
-        strcatbuff(buff, fil);
-        strcatbuff(buff, "\n");
-        a = strstr(cache->use, buff);
-#endif
 
-        // Ok, noté en cache->. mais bien présent dans le cache ou sur disque?
-#if HTS_FAST_CACHE
         // negative values when data is not in cache
         if (hash_pos_return < 0) {
-#else
-        if (a) {
-#endif
-          if (!test) {          // non mode test
-#if HTS_FAST_CACHE==0
-            int pos = -1;
+          if (!test) { // not test mode
+            /* note: no check with IS_DELAYED_EXT() enabled - postcheck by
+             * client please! */
+            if (save[0] != '\0' && !IS_DELAYED_EXT(save) &&
+                fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <=
+                    0) { // final file missing or empty
+              int found = 0;
 
-            a += strlen(buff);
-            sscanf(a, "%d", &pos);      // lire position
-#endif
+              /* It is possible that the file has been moved due to changes in
+               * build structure */
+              {
+                char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
+                htsblk r;
 
-#if HTS_FAST_CACHE==0
-            if (pos < 0) {      // pas de mise en cache data, vérifier existence
-#endif
-              /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
-              if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <= 0) {  // fichier final n'existe pas ou est vide!
-                int found = 0;
-
-                /* It is possible that the file has been moved due to changes in build structure */
-                {
-                  char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
-                  htsblk r;
-
-                  previous_save[0] = '\0';
-                  r =
-                    cache_readex(opt, cache, adr, fil, /*head */ NULL,
+                previous_save[0] = '\0';
+                r = cache_readex(opt, cache, adr, fil, /*head */ NULL,
                                  /*bound to back[p] (temporary) */
                                  back[p].location_buffer, previous_save, /*ro */
                                  1);
-                  /* Is supposed to be on disk only */
-                  if (r.is_write && previous_save[0] != '\0') {
-                    /* Exists, but with another (old) filename: rename (almost) silently */
-                    if (strcmp(previous_save, save) != 0
-                        && fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) {
-                      rename(fconv(catbuff, sizeof(catbuff), previous_save),
-                             fconv(catbuff2, sizeof(catbuff2), save));
-                      if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
-                        found = 1;
-                        hts_log_print(opt, LOG_DEBUG,
-                                      "File '%s' has been renamed since last mirror to '%s' ; applying changes",
-                                      previous_save, save);
-                      } else {
-                        hts_log_print(opt, LOG_ERROR,
-                                      "Could not rename '%s' to '%s' ; will have to retransfer it",
-                                      previous_save, save);
-                      }
+                /* Is supposed to be on disk only */
+                if (r.is_write && previous_save[0] != '\0') {
+                  /* Exists, but with another (old) filename: rename (almost)
+                   * silently */
+                  if (strcmp(previous_save, save) != 0 &&
+                      fexist_utf8(
+                          fconv(catbuff, sizeof(catbuff), previous_save))) {
+                    rename(fconv(catbuff, sizeof(catbuff), previous_save),
+                           fconv(catbuff2, sizeof(catbuff2), save));
+                    if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
+                      found = 1;
+                      hts_log_print(opt, LOG_DEBUG,
+                                    "File '%s' has been renamed since last "
+                                    "mirror to '%s' ; applying changes",
+                                    previous_save, save);
+                    } else {
+                      hts_log_print(opt, LOG_ERROR,
+                                    "Could not rename '%s' to '%s' ; will have "
+                                    "to retransfer it",
+                                    previous_save, save);
                     }
                   }
-                  back[p].location_buffer[0] = '\0';
                 }
+                back[p].location_buffer[0] = '\0';
+              }
 
-                /* Not found ? */
-                if (!found) {
-#if HTS_FAST_CACHE
-                  hash_pos_return = 0;
-#else
-                  a = NULL;
-#endif
-                  // dévalider car non présent sur disque dans structure originale!!!
-                  // sinon, le fichier est ok à priori, mais on renverra un if-modified-since pour
-                  // en être sûr
-                  if (opt->norecatch) { // tester norecatch
-                    if (!fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {   // fichier existe pas mais déclaré: on l'a effacé
-                      FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
+              /* Not found ? */
+              if (!found) {
+                // invalidate: gone from disk, force a refetch
+                hash_pos_return = 0;
+                if (opt->norecatch) {
+                  if (!fexist_utf8(fconv(
+                          catbuff, sizeof(catbuff),
+                          save))) { // declared but missing: user erased it
+                    FILE *fp =
+                        FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
 
-                      if (fp)
-                        fclose(fp);
-                      hts_log_print(opt, LOG_WARNING,
-                                    "Previous file '%s' not found (erased by user ?), ignoring: %s%s",
-                                    save, back[p].url_adr, back[p].url_fil);
-                    }
-                  } else {
+                    if (fp)
+                      fclose(fp);
                     hts_log_print(opt, LOG_WARNING,
-                                  "Previous file '%s' not found (erased by user ?), recatching: %s%s",
+                                  "Previous file '%s' not found (erased by "
+                                  "user ?), ignoring: %s%s",
                                   save, back[p].url_adr, back[p].url_fil);
                   }
+                } else {
+                  hts_log_print(opt, LOG_WARNING,
+                                "Previous file '%s' not found (erased by user "
+                                "?), recatching: %s%s",
+                                save, back[p].url_adr, back[p].url_fil);
                 }
-              }                 // fsize() <= 0
-#if HTS_FAST_CACHE==0
-            }
-#endif
+              }
+            } // fsize() <= 0
           }
         }
         //
       } else {
-#if HTS_FAST_CACHE
         hash_pos_return = 0;
-#else
-        a = NULL;
-#endif
       }
 
-      // Existe pas en cache, ou bien pas de cache présent
-#if HTS_FAST_CACHE
-      if (hash_pos_return) {    // OK existe en cache (et données aussi)!
-#else
-      if (a != NULL) {          // OK existe en cache (et données aussi)!
-#endif
+      if (hash_pos_return) { // in cache, with data
         const int cache_is_prioritary = cache->type == 1
           || opt->state.stop != 0;
         if (cache_is_prioritary) {      // cache prioritaire (pas de test if-modified..)
@@ -1898,7 +1864,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #endif
         }
       }
-      /* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */
+      /* Not in cache ; maybe in temporary cache ? Warning: non-movable
+         "url_sav" */
       else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
         const off_t file_size = fsize_utf8(itemback->url_sav);
 

src/htscache.c

@@ -491,11 +491,7 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
   hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
   /* avoid errors on data entries */
   if (adr[0] == '/' && adr[1] == '/' && adr[2] == '[') {
-#if HTS_FAST_CACHE
     hash_pos_return = 0;
-#else
-    a = NULL;
-#endif
   }
 
   if (hash_pos_return != 0) {

src/htsconfig.h

@@ -76,9 +76,6 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
-// fast cache (build hash table)
-#define HTS_FAST_CACHE 1
-
 // le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
 // valide)
 #define GT_ENDS_COMMENT 1

src/htscore.c

@@ -134,10 +134,6 @@ RUN_CALLBACK0(opt, end); \
     back_delete_all(opt, &cache, sback);                                       \
     back_free(&sback);                                                         \
     checkrobots_free(&robots);                                                 \
-    if (cache.use) {                                                           \
-      freet(cache.use);                                                        \
-      cache.use = NULL;                                                        \
-    }                                                                          \
     if (cache.zipOutput) {                                                     \
       zipClose(cache.zipOutput,                                                \
                "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \

src/htscore.h

@@ -194,7 +194,6 @@ struct cache_back {
   int type;
   int ro;                   /**< read-only: no new cache is written */
 
-  char *use;                /**< in-memory list of cached adr+fil keys */
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
   char lastmodified[256];

src/htsselftest.c

@@ -2123,6 +2123,19 @@ static int ae_write_collision(const char *path, const unsigned char *src,
   return ok ? 0 : 1;
 }
 
+/* Write src[0..len) to path as-is; 0 on success. */
+static int ae_write_raw(const char *path, const unsigned char *src,
+                        size_t len) {
+  FILE *const f = FOPEN(path, "wb");
+  int ok;
+
+  if (f == NULL)
+    return 1;
+  ok = fwrite(src, 1, len, f) == len;
+  fclose(f);
+  return ok ? 0 : 1;
+}
+
 /* Compare path's bytes to expect[0..len); 0 if equal. Streams (large files). */
 static int ae_check_decoded(const char *path, const unsigned char *expect,
                             size_t len) {
@@ -2190,6 +2203,46 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
     assertf(ae_write_collision(inpath, body, 64) == 0);
     assertf(hts_zunpack(inpath, outpath) == 64);
     assertf(ae_check_decoded(outpath, body, 64) == 0);
+    /* Identity fallback (#47): a plain body mislabeled as compressed is kept
+       verbatim, small and multi-chunk (> one 8 KiB fread). */
+    assertf(ae_write_raw(inpath, small, slen) == 0);
+    assertf(hts_zunpack(inpath, outpath) == (int) slen);
+    assertf(ae_check_decoded(outpath, small, slen) == 0);
+    {
+      const size_t ilen = 16 * 1024;
+      unsigned char *idbody = malloct(ilen);
+
+      assertf(idbody != NULL);
+      for (i = 0; i < ilen; i++)
+        idbody[i] = small[i % slen];
+      assertf(ae_write_raw(inpath, idbody, ilen) == 0);
+      assertf(hts_zunpack(inpath, outpath) == (int) ilen);
+      assertf(ae_check_decoded(outpath, idbody, ilen) == 0);
+      freet(idbody);
+    }
+    /* Truncated gzip (CRC+ISIZE cut), zlib (ADLER32 cut) and raw deflate
+       must all still fail, not fall back to a verbatim copy. */
+    {
+      static const struct {
+        int wb;
+        size_t cut;
+      } tr[] = {{16 + MAX_WBITS, 8}, {MAX_WBITS, 4}, {-MAX_WBITS, 5}};
+
+      for (i = 0; i < sizeof(tr) / sizeof(tr[0]); i++) {
+        unsigned char z[512];
+        size_t zlen;
+        FILE *f;
+
+        assertf(ae_write_packed(inpath, tr[i].wb, small, slen) == 0);
+        f = FOPEN(inpath, "rb");
+        assertf(f != NULL);
+        zlen = fread(z, 1, sizeof(z), f);
+        fclose(f);
+        assertf(zlen > tr[i].cut && zlen < sizeof(z));
+        assertf(ae_write_raw(inpath, z, zlen - tr[i].cut) == 0);
+        assertf(hts_zunpack(inpath, outpath) < 0);
+      }
+    }
     freet(body);
   }
 #else

src/htszlib.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 
 /*
   Unpack file into a new file (gzip, zlib RFC1950 or raw deflate RFC1951).
+  A body provably in no deflate framing is copied verbatim (identity).
   Return value: size of the new file, or -1 if an error occurred
 */
 /* Note: utf-8 */
@@ -68,6 +69,10 @@ int hts_zunpack(char *filename, char *newfile) {
             ((inbuf[0] & 0x0f) == Z_DEFLATED &&
              (((unsigned) inbuf[0] << 8 | inbuf[1]) % 31) == 0)));
       int attempt;
+      /* not_deflate: the raw attempt hit a data error, so the body is in no
+         deflate framing at all. env_error: local I/O or memory failure. */
+      hts_boolean not_deflate = HTS_FALSE;
+      hts_boolean env_error = HTS_FALSE;
 
       /* deflate is ambiguous; on failure retry with the other windowBits */
       for (attempt = 0; attempt < 2 && ret < 0; attempt++) {
@@ -78,15 +83,20 @@ int hts_zunpack(char *filename, char *newfile) {
 
         if (attempt > 0) {
           /* rewind input; reopening fpout "wb" discards the partial output */
-          if (fseek(in, 0, SEEK_SET) != 0)
+          if (fseek(in, 0, SEEK_SET) != 0) {
+            env_error = HTS_TRUE;
             break;
+          }
           navail = fread(inbuf, 1, sizeof(inbuf), in);
         }
         fpout = FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
-        if (fpout == NULL)
+        if (fpout == NULL) {
+          env_error = HTS_TRUE;
           break;
+        }
         memset(&strm, 0, sizeof(strm));
         if (inflateInit2(&strm, windowBits) != Z_OK) {
+          env_error = HTS_TRUE;
           fclose(fpout);
           break;
         }
@@ -108,12 +118,17 @@ int hts_zunpack(char *filename, char *newfile) {
               zerr = inflate(&strm, Z_NO_FLUSH);
               if (zerr == Z_NEED_DICT || zerr == Z_DATA_ERROR ||
                   zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR) {
+                if (zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR)
+                  env_error = HTS_TRUE;
+                else if (windowBits < 0)
+                  not_deflate = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
               produced = sizeof(outbuf) - strm.avail_out;
               if (produced > 0 &&
                   fwrite(outbuf, 1, produced, fpout) != produced) {
+                env_error = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
@@ -129,6 +144,28 @@ int hts_zunpack(char *filename, char *newfile) {
         inflateEnd(&strm);
         fclose(fpout);
       }
+      /* keep a mislabeled identity body verbatim only when provably not
+         deflate; truncation or a local failure must keep failing (#47) */
+      if (ret < 0 && !wrapped && not_deflate && !env_error && !ferror(in)) {
+        FILE *const fpout =
+            FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
+
+        if (fpout != NULL && fseek(in, 0, SEEK_SET) == 0) {
+          int size = 0;
+
+          while ((navail = fread(inbuf, 1, sizeof(inbuf), in)) > 0) {
+            if (fwrite(inbuf, 1, navail, fpout) != navail) {
+              size = -1;
+              break;
+            }
+            size += (int) navail;
+          }
+          if (size >= 0 && !ferror(in))
+            ret = size;
+        }
+        if (fpout != NULL)
+          fclose(fpout);
+      }
       fclose(in);
     }
   }

tests/36_local-bigcrawl.test

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 # Diverse seeded /big/ crawl: 12 pattern families, decoy absence, update pass
-# must 304-revalidate. 360 = 1 index + 96 pages + 192 imgs + 5 shared + 60
+# must 304-revalidate. 361 = 1 index + 96 pages + 192 imgs + 5 shared + 61
 # family + 6 singles; the 4 planted errors write -o1 pages, not counted.
 
 set -euo pipefail
@@ -9,7 +9,7 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --errors 4 --files 360 \
+    --errors 4 --files 361 \
     --found 'big/p/95.html' \
     --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
     --found 'big/a/f2-2x.png' \
@@ -40,6 +40,7 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --not-found 'big/x/bj.jar' \
     --not-found 'big/x/is1.png' \
     --not-found 'big/x/concat.html' \
+    --file-matches 'big/f1/gzid.html' '<p>gzid</p>' \
     --file-matches 'big/p/2.html' 'srcset="\.\./a/f2-1x\.png 1x, \.\./a/f2-2x\.png 2x"' \
     --file-matches 'big/a/blk2.css' 'url\(blk2-bg\.png\)' \
     --file-matches 'big/p/5.html' "document\\.write\\('<a href=\"\\.\\./f5/dw\\.html\"" \

tests/local-server.py

@@ -374,6 +374,7 @@ def big_index(port):
         '<img src="/big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png">'
         '<a href="/big/f1/long.html?x=%s">long</a>'
         '<a href="/big/f1/gzok.html">gzok</a>'
+        '<a href="/big/f1/gzid.html">gzid</a>'
         '<a href="//127.0.0.1:%d/big/f1/protorel.html">protorel</a>'
         '<a href="http://127.0.0.1:%d/big/f1/abshost.html">abshost</a>'
         '<a href="/big/e/404.html">e404</a>'
@@ -399,6 +400,7 @@ BIG_SIMPLE_PAGES = {
     "/big/f1/dir/": "dir index",
     "/big/f1/long.html": "long",
     "/big/f1/gzok.html": "gzok",
+    "/big/f1/gzid.html": "gzid",
     "/big/f1/protorel.html": "protorel",
     "/big/f1/abshost.html": "abshost",
     "/big/f5/dw.html": "dw target",
@@ -1079,6 +1081,13 @@ class Handler(SimpleHTTPRequestHandler):
                     "text/html",
                     extra=[("Content-Encoding", "gzip")],
                 )
+            elif path == "/big/f1/gzid.html":
+                # Plain body mislabeled as gzip: identity fallback keeps it (#47)
+                self.big_send(
+                    body,
+                    "text/html",
+                    extra=[("Content-Encoding", "gzip")],
+                )
             else:
                 self.big_send(body, "text/html")
         elif path == "/big/f1/list.html":