Add --why to explain which filter rule accepts or rejects a URL

The top user-confusion class is filters silently not doing what their
author expects. --why <url>, given the rest of the crawl command line,
prints the deciding rule ("rejected by rule #2 (-*.zip)") or reports
that no rule matches, then exits without crawling. It reuses the real
option parsing and filter seeding end to end, and evaluates through the
new fa_strjoker_dual(), which also replaces the two hand-rolled copies
of the dual-form verdict merge in htswizard.c.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/codeql.yml

@@ -1,57 +0,0 @@
-# CodeQL static analysis (C). The security-extended suite covers the classes
-# this codebase actually fights: overflows, tainted-size allocs, format bugs.
-name: CodeQL
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-  schedule:
-    # Weekly re-scan of master so new/updated queries land without a push.
-    - cron: "17 4 * * 1"
-
-permissions:
-  contents: read
-
-concurrency:
-  group: codeql-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: analyze (c-cpp)
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: read
-      # Upload findings to the repo's code-scanning dashboard.
-      security-events: write
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev
-
-      - uses: github/codeql-action/init@v4
-        with:
-          languages: c-cpp
-          build-mode: manual
-          queries: security-extended
-
-      # Manual build: CodeQL traces the compiler, so build exactly what ships.
-      - name: Build
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure
-          make -j"$(nproc)"
-
-      - uses: github/codeql-action/analyze@v4
-        with:
-          category: "/language:c-cpp"

.gitignore

@@ -34,9 +34,8 @@ Makefile
 *.so.*
 *.a
 
-# make dist output; dist/ holds httrack-gh-release staging artifacts.
+# make dist output.
 /httrack-*.tar.gz
-/dist/
 
 # Editor / autotools backup files.
 *~

configure.ac

@@ -102,8 +102,7 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
 AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
 AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
 AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
-AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
-	[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
+AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
 AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
 AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
 AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -111,13 +110,6 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
 AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
 AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
 
-# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
-# sanitizer builds, whose interceptors want the unfortified calls.
-case "$CFLAGS" in
-*-fsanitize=*) ;;
-*) AX_ADD_FORTIFY_SOURCE ;;
-esac
-
 # Force libc back into DT_NEEDED for libraries that reach it only through
 # libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
 # linker; Apple ld rejects these flags and links libSystem unconditionally.

fuzz/Makefile.am

@@ -1,7 +1,7 @@
 # libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
 if FUZZERS
 noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
-	fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx
+	fuzz-unescape fuzz-filters fuzz-url
 endif
 
 AM_CPPFLAGS = \
@@ -23,8 +23,6 @@ fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
 fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
 fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
 fuzz_url_SOURCES = fuzz-url.c fuzz.h
-fuzz_header_SOURCES = fuzz-header.c fuzz.h
-fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
 
 # List corpus files explicitly: automake does not expand EXTRA_DIST globs.
 EXTRA_DIST = README.md run-fuzzers.sh \
@@ -37,8 +35,4 @@ EXTRA_DIST = README.md run-fuzzers.sh \
 	corpus/filters/filter.bin corpus/filters/filter-size.bin \
 	corpus/filters/regress-empty-subject-unique.bin \
 	corpus/url/http-url.txt corpus/url/relative-path.txt \
-	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
-	corpus/header/full-response.txt corpus/header/redirect.txt \
-	corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
-	corpus/cachendx/regress-overadvance.bin \
-	corpus/cachendx/regress-truncated-entry.bin
+	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt

fuzz/corpus/cachendx/new-format.txt

@@ -1,7 +0,0 @@
-8
-CACHE-1.1
-28
-Mon, 01 Jan 2024 00:00:00 GMT
-www.example.com
-/index.html
-123

fuzz/corpus/cachendx/old-format.txt

@@ -1,5 +0,0 @@
-3
-1.0
-www.example.com
-/page
-5

fuzz/corpus/cachendx/regress-overadvance.bin

@@ -1,2 +0,0 @@
-32768
-CACHE-1.1

fuzz/corpus/cachendx/regress-truncated-entry.bin

@@ -1,5 +0,0 @@
-8
-CACHE-1.1
-1
-x
-www.example.com

fuzz/corpus/header/full-response.txt

@@ -1,10 +0,0 @@
-HTTP/1.1 200 OK
-Content-Type: text/html; charset=utf-8
-Content-Length: 1234
-Content-Encoding: gzip
-Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
-Etag: "abc"
-Location: http://example.com/x
-Set-Cookie: ID=42; path=/; domain=.example.com
-Content-Range: bytes 0-99/100
-Transfer-Encoding: chunked

fuzz/corpus/header/redirect.txt

@@ -1,3 +0,0 @@
-HTTP/1.0 301 Moved
-Location: /elsewhere
-Content-Disposition: attachment; filename="a.pdf"

fuzz/fuzz-cachendx.c

@@ -1,65 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
-   walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
-   Mirrors the -#C cache-listing scan (htscoremain.c). */
-#include "fuzz.h"
-#include "htscache.h"
-#include "htslib.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  const char *const end = buf + size;
-  char firstline[256];
-  char *a = buf;
-
-  /* header: two length-prefixed fields (version, last-modified) */
-  a += cache_brstr(a, firstline, sizeof(firstline));
-  a += cache_brstr(a, firstline, sizeof(firstline));
-
-  /* body: newline-delimited host/file/position triples; the length-prefixed
-     scan must stay inside the buffer */
-  while (a != NULL && a < end) {
-    char BIGSTK line[HTS_URLMAXSIZE * 2];
-    char linepos[256];
-    int pos;
-
-    a = strchr(a + 1, '\n');
-    if (a == NULL)
-      break;
-    a++;
-    a += cache_binput(a, end, line, HTS_URLMAXSIZE);
-    a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
-    a += cache_binput(a, end, linepos, 200);
-    sscanf(linepos, "%d", &pos);
-    (void) pos;
-  }
-
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-header.c

@@ -1,70 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
-   status line, treathead on each following header. Both consume raw bytes
-   off the wire and copy fields into fixed htsblk buffers; treathead also
-   mutates its line in place and drives cookie parsing. */
-#include "fuzz.h"
-#include "htslib.h"
-#include "htsbauth.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  htsblk r;
-  t_cookie *cookie = calloct(1, sizeof(*cookie));
-  char *line = malloct(size + 1);
-  char *p = buf;
-  int first = 1;
-
-  memset(&r, 0, sizeof(r));
-  cookie->max_len = (int) sizeof(cookie->data);
-  r.location = malloct(HTS_URLMAXSIZE * 2);
-  r.location[0] = '\0';
-
-  /* feed one header line at a time, as the receive loop does */
-  while (p != NULL && *p != '\0') {
-    char *nl = strchr(p, '\n');
-    size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
-
-    memcpy(line, p, n);
-    line[n] = '\0';
-    if (first) {
-      treatfirstline(&r, line);
-      first = 0;
-    } else {
-      treathead(cookie, "www.example.com", "/", &r, line);
-    }
-    p = (nl != NULL) ? nl + 1 : NULL;
-  }
-
-  freet(r.location);
-  freet(line);
-  freet(cookie);
-  freet(buf);
-  return 0;
-}

m4/ax_add_fortify_source.m4

@@ -1,119 +0,0 @@
-# ===========================================================================
-#  https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_ADD_FORTIFY_SOURCE
-#
-# DESCRIPTION
-#
-#   Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
-#   redefinition warnings, other cpp warnings or linker. Some distributions
-#   (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
-#   their compilers, leading to unnecessary warnings in the form of
-#
-#     <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
-#     <built-in>: note: this is the location of the previous definition
-#
-#   which is a problem if -Werror is enabled. This macro checks whether
-#   _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
-#   to CPPFLAGS.
-#
-#   Newer mingw-w64 msys2 package comes with a bug in
-#   headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
-#   and would need -lssp or -fstack-protector.  See
-#   https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
-#   link it.
-#
-# LICENSE
-#
-#   Copyright (c) 2017 David Seifert <soap@gentoo.org>
-#   Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved.  This file is offered as-is, without any
-#   warranty.
-
-#serial 10
-
-AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
-    ac_save_cflags=$CFLAGS
-    ac_cwerror_flag=yes
-    AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
-    ax_add_fortify_3_failed=
-    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],
-            [[
-                #ifndef _FORTIFY_SOURCE
-                    return 0;
-                #else
-                    _FORTIFY_SOURCE_already_defined;
-                #endif
-            ]]
-        )],
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([[
-                #define _FORTIFY_SOURCE 3
-                #include <string.h>
-                int main(void) {
-                    char *s = " ";
-                    strcpy(s, "x");
-                    return strlen(s)-1;
-                }
-              ]]
-            )],
-            [
-              AC_MSG_RESULT([yes])
-              CFLAGS=$ac_save_cflags
-              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
-            ], [
-              AC_MSG_RESULT([no])
-              ax_add_fortify_3_failed=1
-            ],
-        ),
-        [
-          AC_MSG_RESULT([no])
-          ax_add_fortify_3_failed=1
-        ])
-    if test -n "$ax_add_fortify_3_failed"
-    then
-    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],
-            [[
-                #ifndef _FORTIFY_SOURCE
-                    return 0;
-                #else
-                    _FORTIFY_SOURCE_already_defined;
-                #endif
-            ]]
-        )],
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([[
-                #define _FORTIFY_SOURCE 2
-                #include <string.h>
-                int main(void) {
-                    char *s = " ";
-                    strcpy(s, "x");
-                    return strlen(s)-1;
-                }
-              ]]
-            )],
-            [
-              AC_MSG_RESULT([yes])
-              CFLAGS=$ac_save_cflags
-              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
-            ], [
-              AC_MSG_RESULT([no])
-              CFLAGS=$ac_save_cflags
-            ],
-        ),
-        [
-          AC_MSG_RESULT([no])
-          CFLAGS=$ac_save_cflags
-        ])
-    fi
-])

src/htscache.h

@@ -89,11 +89,14 @@ typedef enum {
    the involved files are absent. */
 void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
 
+int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
+                    const char *str2, char *outbuff, int len);
+int cache_readdata(cache_back * cache, const char *str1, const char *str2,
+                   char **inbuff, int *len);
+
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);
-/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
-int cache_binput(char *adr, const char *end, char *s, int max);
 int cache_brint(char *adr, int *i);
 void cache_rint(FILE * fp, int *i);
 void cache_rLLint(FILE * fp, LLint * i);

src/htscache_selftest.c

@@ -78,6 +78,14 @@ static void selftest_open_for_read(cache_back *cache, httrackp *opt) {
 }
 
 static void selftest_close(cache_back *cache) {
+  if (cache->dat != NULL) {
+    fclose(cache->dat);
+    cache->dat = NULL;
+  }
+  if (cache->ndx != NULL) {
+    fclose(cache->ndx);
+    cache->ndx = NULL;
+  }
   if (cache->zipOutput != NULL) {
     zipClose(cache->zipOutput,
              "Created by HTTrack Website Copier (cache self-test)");
@@ -981,15 +989,26 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
 
-  /* PROMOTE: the removed pre-3.31 legacy pair is left alone */
+  /* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
+     zip cache existed) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", TINY);
   hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
+
+  /* PROMOTE: a half-written legacy new pair is replaced by the old pair */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
 
   /* INTERRUPTED: no lock file, no action */
   reconcile_wipe(opt);
@@ -1039,7 +1058,7 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
 
-  /* INTERRUPTED: the removed pre-3.31 legacy pair is left alone */
+  /* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-in_progress.lock", 0);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
@@ -1048,13 +1067,9 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
   failures +=
-      reconcile_expect(opt, "hts-cache/new.dat", TINY, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
   failures +=
-      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.dat", SOLID, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.ndx", MID, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
 
   /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
      its only good generation here) */
@@ -1074,18 +1089,17 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
 
-  /* ROLLBACK: sidecars restored, the removed pre-3.31 pair left alone */
+  /* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   reconcile_put(opt, "hts-cache/old.lst", MID);
   reconcile_put(opt, "hts-cache/old.txt", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", TINY, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
 
@@ -1099,101 +1113,6 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   return failures;
 }
 
-/* The pre-3.31 .dat/.ndx import was removed: cache_init must refuse the
-   legacy pair, leave the files in place, and not flag an update run. */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-  int variant;
-
-  selftest_tag = "cache-legacy";
-  golden_setup(opt, dir);
-#ifdef _WIN32
-  mkdir(reconcile_st_path(opt, "hts-cache"));
-#else
-  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
-#endif
-
-  /* variant 0: old.dat/old.ndx (--update layout); 1: new.dat/new.ndx (ro) */
-  for (variant = 0; variant < 2; variant++) {
-    cache_back cache;
-    const char *const dat =
-        variant == 0 ? "hts-cache/old.dat" : "hts-cache/new.dat";
-    const char *const ndx =
-        variant == 0 ? "hts-cache/old.ndx" : "hts-cache/new.ndx";
-
-    reconcile_wipe(opt);
-    reconcile_put(opt, dat, 4096);
-    reconcile_put(opt, ndx, 4096);
-    opt->is_update = 0;
-    selftest_open(&cache, opt, /*ro*/ variant == 1);
-    if (cache_readable(&cache)) {
-      printf("cache-legacy: variant %d imported a removed format\n", variant);
-      failures++;
-    }
-    if (coucal_nitems(cache.hashtable) != 0) { /* no phantom index entries */
-      printf("cache-legacy: variant %d imported index entries\n", variant);
-      failures++;
-    }
-    if (opt->is_update) {
-      printf("cache-legacy: variant %d flagged an update\n", variant);
-      failures++;
-    }
-    if (fsize(reconcile_st_path(opt, dat)) != 4096 ||
-        fsize(reconcile_st_path(opt, ndx)) != 4096) {
-      printf("cache-legacy: variant %d touched the legacy files\n", variant);
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  /* a healthy zip must win over stray legacy files, with no refusal */
-  {
-    cache_back cache;
-    const char *const body = "<html>zip wins</html>";
-
-    reconcile_wipe(opt);
-    selftest_open_for_write(&cache, opt);
-    store_entry(opt, &cache, "example.com", "/", "example.com/index.html", 200,
-                "OK", "text/html", "utf-8", "", "", "", "", body, strlen(body));
-    selftest_close(&cache);
-    reconcile_put(opt, "hts-cache/old.ndx", 4096);
-    reconcile_put(opt, "hts-cache/old.dat", 4096);
-    selftest_open_for_read(&cache, opt);
-    if (!cache_readable(&cache)) {
-      printf("cache-legacy: stray legacy files shadowed the zip cache\n");
-      failures++;
-    }
-    failures +=
-        check_entry(opt, &cache, "example.com", "/", 200, "OK", "text/html",
-                    "utf-8", "", "", "", "", body, strlen(body));
-    if (fsize(reconcile_st_path(opt, "hts-cache/old.ndx")) != 4096 ||
-        fsize(reconcile_st_path(opt, "hts-cache/old.dat")) != 4096) {
-      printf("cache-legacy: zip-wins pass touched the legacy files\n");
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  reconcile_wipe(opt);
-  return failures;
-}
-
 /* --- read-side corruption injection --------------------------------------- */
 
 /* canary read back intact after each corruption; victim gets the byte surgery

src/htscache_selftest.h

@@ -60,10 +60,6 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir);
    under <dir>. Returns the failed-check count. */
 int cache_reconcile_selftest(httrackp *opt, const char *dir);
 
-/* Verify cache_init refuses a pre-3.31 .dat/.ndx cache without touching it.
-   Returns the number of failed checks (0 = pass). */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir);
-
 /* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
    under <dir> and assert every case degrades to STATUSCODE_INVALID without
    tainting a sibling entry. */

src/htscore.c

@@ -138,6 +138,14 @@ RUN_CALLBACK0(opt, end); \
       freet(cache.use);                                                        \
       cache.use = NULL;                                                        \
     }                                                                          \
+    if (cache.dat) {                                                           \
+      fclose(cache.dat);                                                       \
+      cache.dat = NULL;                                                        \
+    }                                                                          \
+    if (cache.ndx) {                                                           \
+      fclose(cache.ndx);                                                       \
+      cache.ndx = NULL;                                                        \
+    }                                                                          \
     if (cache.zipOutput) {                                                     \
       zipClose(cache.zipOutput,                                                \
                "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \
@@ -147,6 +155,10 @@ RUN_CALLBACK0(opt, end); \
       unzClose(cache.zipInput);                                                \
       cache.zipInput = NULL;                                                   \
     }                                                                          \
+    if (cache.olddat) {                                                        \
+      fclose(cache.olddat);                                                    \
+      cache.olddat = NULL;                                                     \
+    }                                                                          \
     if (cache.lst) {                                                           \
       fclose(cache.lst);                                                       \
       cache.lst = opt->state.strc.lst = NULL;                                  \
@@ -476,11 +488,32 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
   *makeindex_done = 1;
 }
 
-/* Flush the parsed HTML output buffer to disk. */
+/* Flush the parsed HTML output buffer to disk, skipping the rewrite when the
+ * on-disk MD5 is unchanged. */
 void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
                           FILE **fp, const char *ht_buff, size_t ht_len,
                           const char *adr, const char *fil, const char *save) {
-  {
+  char digest[32 + 2];
+  off_t fsize_old =
+      fsize(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), save));
+  int ok = 0;
+
+  digest[0] = '\0';
+  domd5mem(ht_buff, ht_len, digest, 1);
+  if (fsize_old == (off_t) ht_len) {
+    int mlen = 0;
+    char *mbuff;
+
+    cache_readdata(cache, "//[HTML-MD5]//", save, &mbuff, &mlen);
+    if (mlen)
+      mbuff[mlen] = '\0';
+    if ((mlen == 32) && (strcmp(((mbuff != NULL) ? mbuff : ""), digest) == 0)) {
+      ok = 1;
+      hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s", save);
+    }
+    freet(mbuff);
+  }
+  if (!ok) {
     file_notify(opt, adr, fil, save, 1, 1, r->notmodified);
     *fp = filecreate(&opt->state.strc, save);
     if (*fp) {
@@ -512,7 +545,13 @@ void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
       if (fcheck)
         hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
     }
+  } else {
+    file_notify(opt, adr, fil, save, 0, 0, r->notmodified);
+    filenote(&opt->state.strc, save, NULL);
   }
+  if (cache->ndx)
+    cache_writedata(cache->ndx, cache->dat, "//[HTML-MD5]//", save, digest,
+                    (int) strlen(digest));
 }
 
 /* does it look like XML ? (SVG et al.) */

src/htscore.h

@@ -193,7 +193,7 @@ struct cache_back {
   /* */
   int type;
   int ro;                   /**< read-only: no new cache is written */
-
+  FILE *dat, *ndx, *olddat; /**< new data, new index, old data files */
   char *use;                /**< in-memory list of cached adr+fil keys */
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
@@ -288,12 +288,12 @@ struct filecreate_params {
 
 /* True if a new cache is being written (plain or zip backend). */
 HTS_STATIC int cache_writable(cache_back * cache) {
-  return (cache != NULL && cache->zipOutput != NULL);
+  return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL));
 }
 
 /* True if an old cache is available to read (plain or zip backend). */
 HTS_STATIC int cache_readable(cache_back * cache) {
-  return (cache != NULL && cache->zipInput != NULL);
+  return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL));
 }
 
 #endif

src/htscoremain.c

@@ -1953,20 +1953,18 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     char BIGSTK url[HTS_URLMAXSIZE * 2];
                     char linepos[256];
                     int pos;
-                    LLint cacheNdxLen = 0;
-                    char *cacheNdx = readfile2(
-                        fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                StringBuff(opt->path_log), "hts-cache/new.ndx"),
-                        &cacheNdxLen);
+                    char *cacheNdx =
+                      readfile(fconcat
+                               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                                "hts-cache/new.ndx"));
                     cache_init(&cache, opt);    /* load cache */
                     if (cacheNdx != NULL) {
                       char firstline[256];
                       char *a = cacheNdx;
-                      const char *const end = cacheNdx + cacheNdxLen;
 
                       a += cache_brstr(a, firstline, sizeof(firstline));
                       a += cache_brstr(a, firstline, sizeof(firstline));
-                      while (a != NULL && a < end) {
+                      while(a != NULL) {
                         a = strchr(a + 1, '\n');        /* start of line */
                         if (a) {
                           htsblk r;
@@ -1974,15 +1972,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                           /* */
                           a++;
                           /* read "host/file" */
-                          a += cache_binput(a, end, afs.af.adr, HTS_URLMAXSIZE);
-                          a += cache_binput(a, end, afs.af.fil, HTS_URLMAXSIZE);
+                          a += binput(a, afs.af.adr, HTS_URLMAXSIZE);
+                          a += binput(a, afs.af.fil, HTS_URLMAXSIZE);
                           url[0] = '\0';
                           if (!link_has_authority(afs.af.adr))
                             strcatbuff(url, "http://");
                           strcatbuff(url, afs.af.adr);
                           strcatbuff(url, afs.af.fil);
                           /* read position */
-                          a += cache_binput(a, end, linepos, 200);
+                          a += binput(a, linepos, 200);
                           sscanf(linepos, "%d", &pos);
                           if (!hasFilter
                               || (strjoker(url, filter, NULL, NULL) != NULL)

src/htslib.c

@@ -1562,7 +1562,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       }
       // An empty/whitespace Content-Type value yields no token: keep the
       // sentinel default rather than reading an uninitialized tempo.
-      if (sscanf(rcvd + p, "%1099s", tempo) == 1) { // tempo[1100], server data
+      if (sscanf(rcvd + p, "%s", tempo) == 1) {
         if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
           strcpybuff(retour->contenttype, tempo);
         else
@@ -1657,11 +1657,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         if (a)
           *a = '\0';
       }
-      // Bound to tempo[1100]; no token => leave empty, don't read uninit tempo.
-      if (sscanf(a, "%1099s", tempo) == 1 && strlen(tempo) < 64)
+      sscanf(a, "%s", tempo);
+      if (strlen(tempo) < 64)   // pas trop long!!
         strcpybuff(retour->contentencoding, tempo);
       else
-        retour->contentencoding[0] = '\0';
+        retour->contentencoding[0] = '\0';      // erreur
 #if HTS_USEZLIB
       /* Check known encodings */
       if (retour->contentencoding[0]) {

src/htsselftest.c

@@ -46,7 +46,6 @@ Please visit our Website: http://www.httrack.com
 #include "htsdefines.h"
 #include "htslib.h"
 #include "htsparse.h"
-#include "htscache.h"
 #include "htscache_selftest.h"
 #include "htsdns_selftest.h"
 #include "htscharset.h"
@@ -1164,26 +1163,6 @@ static int st_header(httrackp *opt, int argc, char **argv) {
     treathead(NULL, "www.example.com", "/", &r, line);
   }
   printf("contenttype=%s cdispo=%s\n", r.contenttype, r.cdispo);
-  printf("contentencoding=%s\n", r.contentencoding);
-  return 0;
-}
-
-/* An over-long header value must not overflow treathead's tempo[1100]. */
-static int st_headerlong(httrackp *opt, int argc, char **argv) {
-  htsblk r;
-  char BIGSTK line[HTS_URLMAXSIZE * 2];
-  const char *const name = argc >= 1 ? argv[0] : "Content-Type:";
-  const int pad = 1500; /* > tempo[1100] */
-  size_t n;
-
-  (void) opt;
-  memset(&r, 0, sizeof(r));
-  n = (size_t) snprintf(line, sizeof(line), "%s ", name);
-  memset(line + n, 'a', pad);
-  line[n + pad] = '\0';
-  treathead(NULL, "www.example.com", "/", &r, line);
-  printf("contenttype_len=%d contentencoding_len=%d\n",
-         (int) strlen(r.contenttype), (int) strlen(r.contentencoding));
   return 0;
 }
 
@@ -1391,78 +1370,6 @@ static int st_cache(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
-/* A corrupt cache index (.ndx) must not walk the length-prefixed scan past
-   the buffer. Checks the two primitives the loader is built from. */
-static int st_cacheindex(httrackp *opt, int argc, char **argv) {
-  int fail = 0;
-
-  (void) opt;
-  (void) argc;
-  (void) argv;
-
-  /* A length prefix that overstates the bytes present must bound the advance
-     to the buffer, not trust the declared length. */
-  {
-    static const char src[] = "32768\nCACHE-1.1";
-    const size_t len = sizeof(src) - 1;
-    char *buf = malloct(len + 1);
-    char s[256];
-    int off;
-
-    memcpy(buf, src, len + 1);
-    off = cache_brstr(buf, s, sizeof(s));
-    if (off > (int) len) {
-      printf("cacheindex: over-advance off=%d len=%d\n", off, (int) len);
-      fail = 1;
-    }
-    if (strcmp(s, "CACHE-1.1") != 0) {
-      printf("cacheindex: value=%s\n", s);
-      fail = 1;
-    }
-    freet(buf);
-  }
-
-  /* cache_binput reads a field while in bounds, but refuses one starting at
-     or past end-of-buffer. */
-  {
-    char buf[8] = "ab\ncd";
-    const char *const end = buf + 5;
-    char s[16];
-
-    if (cache_binput(buf, end, s, sizeof(s)) != 3 || strcmp(s, "ab") != 0)
-      fail = 1; /* normal read: "ab" then the '\n', 3 bytes consumed */
-    if (cache_binput(end, end, s, sizeof(s)) != 0 || s[0] != '\0')
-      fail = 1;
-  }
-
-  /* Drive the full loader scan over a truncated index: a declared length that
-     overshoots plus a half-written entry. ASan aborts here on the pre-fix
-     scan; the cursor must never leave the buffer. */
-  {
-    static const char src[] = "9\nCACHE-1.1\n99\nwww.example.com\n/a";
-    const size_t len = sizeof(src) - 1;
-    char *buf = malloct(len + 1);
-    const char *const end = buf + len;
-    char line[256];
-    char *a = buf;
-
-    memcpy(buf, src, len + 1);
-    a += cache_brstr(a, line, sizeof(line));
-    a += cache_brstr(a, line, sizeof(line));
-    while (a != NULL && a < end) {
-      a = strchr(a + 1, '\n');
-      if (a == NULL)
-        break;
-      a++;
-      a += cache_binput(a, end, line, sizeof(line));
-    }
-    freet(buf);
-  }
-
-  printf("cacheindex: %s\n", fail ? "FAIL" : "OK");
-  return fail;
-}
-
 static int st_cache_golden(httrackp *opt, int argc, char **argv) {
   int regen, err;
 
@@ -1500,18 +1407,6 @@ static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
-static int st_cache_legacy(httrackp *opt, int argc, char **argv) {
-  int err;
-
-  if (argc < 1) {
-    fprintf(stderr, "cache-legacy: needs a directory\n");
-    return 1;
-  }
-  err = cache_legacy_refused_selftest(opt, argv[0]);
-  printf("cache-legacy: %s\n", err ? "FAIL" : "OK");
-  return err;
-}
-
 static int st_reconcile(httrackp *opt, int argc, char **argv) {
   int err;
 
@@ -2290,24 +2185,17 @@ static const struct selftest_entry {
     {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
-    {"headerlong", "[header-name:]",
-     "over-long header value must not overflow the parse scratch",
-     st_headerlong},
     {"savename", "<fil> <content-type> [key=value ...]",
      "local save-name for a URL", st_savename},
     {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
      st_sniff},
     {"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
-    {"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
-     st_cacheindex},
     {"cache-golden", "<dir> [regen]", "frozen cache-format read self-test",
      st_cache_golden},
     {"cache-writefail", "<dir>", "cache write-failure handling self-test",
      st_cache_writefail},
     {"reconcile", "<dir>", "cache generation reconcile policy self-test",
      st_reconcile},
-    {"cache-legacy", "<dir>", "pre-3.31 legacy cache refusal self-test",
-     st_cache_legacy},
     {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
      st_cache_corrupt},
     {"dns", "", "DNS resolver/cache self-test", st_dns},

tests/01_engine-cacheindex.test

@@ -1,10 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# Cache-index (.ndx) parse must stay inside the buffer on a corrupt length
-# prefix (ASan aborts here on the pre-fix binary; the OK check gates the
-# non-sanitized build too).
-
-test "$(httrack -O /dev/null -#test=cacheindex)" == 'cacheindex: OK'

tests/01_engine-header.test

@@ -27,16 +27,3 @@ hdr 'contenttype= cdispo=report.pdf' \
 # Path components in the filename are dropped on the wire (RFC 2616).
 hdr 'contenttype= cdispo=evil.pdf' \
     'Content-Disposition: attachment; filename="../../evil.pdf"'
-
-# An over-long header value must not overflow the parse scratch (ASan aborts
-# here on the pre-fix binary).
-test "$(httrack -O /dev/null -#test=headerlong 'Content-Type:')" \
-    == 'contenttype_len=32 contentencoding_len=0'
-test "$(httrack -O /dev/null -#test=headerlong 'Content-Encoding:')" \
-    == 'contenttype_len=0 contentencoding_len=0'
-
-# An empty Content-Encoding yields no token: leave it empty, don't read the
-# uninitialized scratch (MSan aborts here on the pre-fix binary).
-enc() { httrack -O /dev/null -#test=header "$1" | grep '^contentencoding='; }
-test "$(enc 'Content-Encoding:')" == 'contentencoding='
-test "$(enc 'Content-Encoding: gzip')" == 'contentencoding=gzip'

tests/01_zlib-cache-legacy.test

@@ -1,23 +0,0 @@
-#!/bin/bash
-#
-# The pre-3.31 .dat/.ndx cache import was removed: cache_init must refuse the
-# legacy pair and leave the files untouched (httrack -#test=cache-legacy <dir>).
-
-set -eu
-
-dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
-
-# the refusal errors land on stdout (no log file); pin them and the verdict
-out=$(httrack -#test=cache-legacy "$dir" 2>/dev/null)
-cnt=$(printf '%s\n' "$out" | grep -c 'no longer supported' || true)
-test "$cnt" = 2 || {
-    echo "expected 2 refusal messages, got $cnt" >&2
-    exit 1
-}
-out=$(printf '%s\n' "$out" | tail -n1)
-
-test "$out" = "cache-legacy: OK" || {
-    echo "expected 'cache-legacy: OK', got: $out" >&2
-    exit 1
-}

tests/Makefile.am

@@ -38,7 +38,6 @@ TESTS = \
 	01_engine-filterdual.test \
 	01_engine-ftp-line.test \
 	01_engine-ftp-userpass.test \
-	01_engine-cacheindex.test \
 	01_engine-hashtable.test \
 	01_engine-header.test \
 	01_engine-idna.test \
@@ -68,7 +67,6 @@ TESTS = \
 	01_zlib-acceptencoding.test \
 	01_zlib-cache.test \
 	01_zlib-cache-corrupt.test \
-	01_zlib-cache-legacy.test \
 	01_zlib-cache-golden.test \
 	01_zlib-cache-writefail.test \
 	01_zlib-savename-cached.test \