tests: international crawl tracked the double-encoded behavior

The default.html sub-crawl asserted the pre-fix output: two mojibake
café*.html filenames from double-encoded fetches and 4 errors where
the fix leaves 2 (the caf%e9.html Latin-1 escapes the server really
lacks). Update to the corrected names and counts; the other three
sub-crawls are unchanged. Only runs where ut.httrack.com is reachable
(in-tree builds), which is why just the deb CI job caught it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,11 +29,11 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
-# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
-# so it stays soname .so.3; bump revision.
+# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
+# installed headers (no struct layout or exported signature changed vs
+# 3.49.10), so it stays soname .so.3; bump revision.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:4:0"
+VERSION_INFO="3:3:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 

debian/changelog

@@ -1,13 +1,3 @@
-httrack (3.49.12-1) unstable; urgency=medium
-
-  * New upstream release: security and crawl-correctness fixes (remote stack
-    overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
-    world-readable cookies.txt, filter-pattern denial of service) plus a new
-    --why filter diagnostic; full list in history.txt.
-  * Build with _FORTIFY_SOURCE and -fstack-protector-strong.
-
- -- Xavier Roche <xavier@debian.org>  Fri, 10 Jul 2026 21:11:09 +0200
-
 httrack (3.49.11-1) unstable; urgency=medium
 
   * New upstream release: crawl correctness and security fixes (network-facing

history.txt

@@ -4,23 +4,6 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
-3.49-12
-+ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
-+ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
-+ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
-+ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
-+ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
-+ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
-+ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
-+ Fixed: cookies.txt was created world-readable (#511)
-+ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
-+ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
-+ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
-+ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
-+ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
-+ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
-+ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
-
 3.49-11
 + New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
 + New: advertise deflate in Accept-Encoding and decode deflate responses (#450)

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-12"
-#define HTTRACK_VERSIONID "3.49.12"
+#define HTTRACK_VERSION "3.49-11"
+#define HTTRACK_VERSIONID "3.49.11"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 

src/htsselftest.c

@@ -2147,19 +2147,6 @@ static int ae_write_collision(const char *path, const unsigned char *src,
   return ok ? 0 : 1;
 }
 
-/* Write src[0..len) to path as-is; 0 on success. */
-static int ae_write_raw(const char *path, const unsigned char *src,
-                        size_t len) {
-  FILE *const f = FOPEN(path, "wb");
-  int ok;
-
-  if (f == NULL)
-    return 1;
-  ok = fwrite(src, 1, len, f) == len;
-  fclose(f);
-  return ok ? 0 : 1;
-}
-
 /* Compare path's bytes to expect[0..len); 0 if equal. Streams (large files). */
 static int ae_check_decoded(const char *path, const unsigned char *expect,
                             size_t len) {
@@ -2227,46 +2214,6 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
     assertf(ae_write_collision(inpath, body, 64) == 0);
     assertf(hts_zunpack(inpath, outpath) == 64);
     assertf(ae_check_decoded(outpath, body, 64) == 0);
-    /* Identity fallback (#47): a plain body mislabeled as compressed is kept
-       verbatim, small and multi-chunk (> one 8 KiB fread). */
-    assertf(ae_write_raw(inpath, small, slen) == 0);
-    assertf(hts_zunpack(inpath, outpath) == (int) slen);
-    assertf(ae_check_decoded(outpath, small, slen) == 0);
-    {
-      const size_t ilen = 16 * 1024;
-      unsigned char *idbody = malloct(ilen);
-
-      assertf(idbody != NULL);
-      for (i = 0; i < ilen; i++)
-        idbody[i] = small[i % slen];
-      assertf(ae_write_raw(inpath, idbody, ilen) == 0);
-      assertf(hts_zunpack(inpath, outpath) == (int) ilen);
-      assertf(ae_check_decoded(outpath, idbody, ilen) == 0);
-      freet(idbody);
-    }
-    /* Truncated gzip (CRC+ISIZE cut), zlib (ADLER32 cut) and raw deflate
-       must all still fail, not fall back to a verbatim copy. */
-    {
-      static const struct {
-        int wb;
-        size_t cut;
-      } tr[] = {{16 + MAX_WBITS, 8}, {MAX_WBITS, 4}, {-MAX_WBITS, 5}};
-
-      for (i = 0; i < sizeof(tr) / sizeof(tr[0]); i++) {
-        unsigned char z[512];
-        size_t zlen;
-        FILE *f;
-
-        assertf(ae_write_packed(inpath, tr[i].wb, small, slen) == 0);
-        f = FOPEN(inpath, "rb");
-        assertf(f != NULL);
-        zlen = fread(z, 1, sizeof(z), f);
-        fclose(f);
-        assertf(zlen > tr[i].cut && zlen < sizeof(z));
-        assertf(ae_write_raw(inpath, z, zlen - tr[i].cut) == 0);
-        assertf(hts_zunpack(inpath, outpath) < 0);
-      }
-    }
     freet(body);
   }
 #else

src/htszlib.c

@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
 
 /*
   Unpack file into a new file (gzip, zlib RFC1950 or raw deflate RFC1951).
-  A body provably in no deflate framing is copied verbatim (identity).
   Return value: size of the new file, or -1 if an error occurred
 */
 /* Note: utf-8 */
@@ -69,10 +68,6 @@ int hts_zunpack(char *filename, char *newfile) {
             ((inbuf[0] & 0x0f) == Z_DEFLATED &&
              (((unsigned) inbuf[0] << 8 | inbuf[1]) % 31) == 0)));
       int attempt;
-      /* not_deflate: the raw attempt hit a data error, so the body is in no
-         deflate framing at all. env_error: local I/O or memory failure. */
-      hts_boolean not_deflate = HTS_FALSE;
-      hts_boolean env_error = HTS_FALSE;
 
       /* deflate is ambiguous; on failure retry with the other windowBits */
       for (attempt = 0; attempt < 2 && ret < 0; attempt++) {
@@ -83,20 +78,15 @@ int hts_zunpack(char *filename, char *newfile) {
 
         if (attempt > 0) {
           /* rewind input; reopening fpout "wb" discards the partial output */
-          if (fseek(in, 0, SEEK_SET) != 0) {
-            env_error = HTS_TRUE;
+          if (fseek(in, 0, SEEK_SET) != 0)
             break;
-          }
           navail = fread(inbuf, 1, sizeof(inbuf), in);
         }
         fpout = FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
-        if (fpout == NULL) {
-          env_error = HTS_TRUE;
+        if (fpout == NULL)
           break;
-        }
         memset(&strm, 0, sizeof(strm));
         if (inflateInit2(&strm, windowBits) != Z_OK) {
-          env_error = HTS_TRUE;
           fclose(fpout);
           break;
         }
@@ -118,17 +108,12 @@ int hts_zunpack(char *filename, char *newfile) {
               zerr = inflate(&strm, Z_NO_FLUSH);
               if (zerr == Z_NEED_DICT || zerr == Z_DATA_ERROR ||
                   zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR) {
-                if (zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR)
-                  env_error = HTS_TRUE;
-                else if (windowBits < 0)
-                  not_deflate = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
               produced = sizeof(outbuf) - strm.avail_out;
               if (produced > 0 &&
                   fwrite(outbuf, 1, produced, fpout) != produced) {
-                env_error = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
@@ -144,28 +129,6 @@ int hts_zunpack(char *filename, char *newfile) {
         inflateEnd(&strm);
         fclose(fpout);
       }
-      /* keep a mislabeled identity body verbatim only when provably not
-         deflate; truncation or a local failure must keep failing (#47) */
-      if (ret < 0 && !wrapped && not_deflate && !env_error && !ferror(in)) {
-        FILE *const fpout =
-            FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
-
-        if (fpout != NULL && fseek(in, 0, SEEK_SET) == 0) {
-          int size = 0;
-
-          while ((navail = fread(inbuf, 1, sizeof(inbuf), in)) > 0) {
-            if (fwrite(inbuf, 1, navail, fpout) != navail) {
-              size = -1;
-              break;
-            }
-            size += (int) navail;
-          }
-          if (size >= 0 && !ferror(in))
-            ret = size;
-        }
-        if (fpout != NULL)
-          fclose(fpout);
-      }
       fclose(in);
     }
   }

tests/36_local-bigcrawl.test

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 # Diverse seeded /big/ crawl: 12 pattern families, decoy absence, update pass
-# must 304-revalidate. 361 = 1 index + 96 pages + 192 imgs + 5 shared + 61
+# must 304-revalidate. 360 = 1 index + 96 pages + 192 imgs + 5 shared + 60
 # family + 6 singles; the 4 planted errors write -o1 pages, not counted.
 
 set -euo pipefail
@@ -9,7 +9,7 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --errors 4 --files 361 \
+    --errors 4 --files 360 \
     --found 'big/p/95.html' \
     --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
     --found 'big/a/f2-2x.png' \
@@ -40,7 +40,6 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --not-found 'big/x/bj.jar' \
     --not-found 'big/x/is1.png' \
     --not-found 'big/x/concat.html' \
-    --file-matches 'big/f1/gzid.html' '<p>gzid</p>' \
     --file-matches 'big/p/2.html' 'srcset="\.\./a/f2-1x\.png 1x, \.\./a/f2-2x\.png 2x"' \
     --file-matches 'big/a/blk2.css' 'url\(blk2-bg\.png\)' \
     --file-matches 'big/p/5.html' "document\\.write\\('<a href=\"\\.\\./f5/dw\\.html\"" \

tests/local-server.py

@@ -374,7 +374,6 @@ def big_index(port):
         '<img src="/big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png">'
         '<a href="/big/f1/long.html?x=%s">long</a>'
         '<a href="/big/f1/gzok.html">gzok</a>'
-        '<a href="/big/f1/gzid.html">gzid</a>'
         '<a href="//127.0.0.1:%d/big/f1/protorel.html">protorel</a>'
         '<a href="http://127.0.0.1:%d/big/f1/abshost.html">abshost</a>'
         '<a href="/big/e/404.html">e404</a>'
@@ -400,7 +399,6 @@ BIG_SIMPLE_PAGES = {
     "/big/f1/dir/": "dir index",
     "/big/f1/long.html": "long",
     "/big/f1/gzok.html": "gzok",
-    "/big/f1/gzid.html": "gzid",
     "/big/f1/protorel.html": "protorel",
     "/big/f1/abshost.html": "abshost",
     "/big/f5/dw.html": "dw target",
@@ -1172,13 +1170,6 @@ class Handler(SimpleHTTPRequestHandler):
                     "text/html",
                     extra=[("Content-Encoding", "gzip")],
                 )
-            elif path == "/big/f1/gzid.html":
-                # Plain body mislabeled as gzip: identity fallback keeps it (#47)
-                self.big_send(
-                    body,
-                    "text/html",
-                    extra=[("Content-Encoding", "gzip")],
-                )
             else:
                 self.big_send(body, "text/html")
         elif path == "/big/f1/list.html":