Add a CodeQL C analysis workflow

Scan on push to master, PRs, and a weekly cron, with the security-extended
query suite; findings land in the code-scanning dashboard. Manual build mode
so CodeQL traces the real autotools build.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.gitignore

@@ -34,9 +34,8 @@ Makefile
 *.so.*
 *.a
 
-# make dist output; dist/ holds httrack-gh-release staging artifacts.
+# make dist output.
 /httrack-*.tar.gz
-/dist/
 
 # Editor / autotools backup files.
 *~

configure.ac

@@ -102,8 +102,7 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
 AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
 AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
 AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
-AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
-	[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
+AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
 AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
 AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
 AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -111,13 +110,6 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
 AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
 AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
 
-# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
-# sanitizer builds, whose interceptors want the unfortified calls.
-case "$CFLAGS" in
-*-fsanitize=*) ;;
-*) AX_ADD_FORTIFY_SOURCE ;;
-esac
-
 # Force libc back into DT_NEEDED for libraries that reach it only through
 # libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
 # linker; Apple ld rejects these flags and links libSystem unconditionally.

fuzz/Makefile.am

@@ -1,7 +1,7 @@
 # libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
 if FUZZERS
 noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
-	fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx
+	fuzz-unescape fuzz-filters fuzz-url
 endif
 
 AM_CPPFLAGS = \
@@ -23,8 +23,6 @@ fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
 fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
 fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
 fuzz_url_SOURCES = fuzz-url.c fuzz.h
-fuzz_header_SOURCES = fuzz-header.c fuzz.h
-fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
 
 # List corpus files explicitly: automake does not expand EXTRA_DIST globs.
 EXTRA_DIST = README.md run-fuzzers.sh \
@@ -37,8 +35,4 @@ EXTRA_DIST = README.md run-fuzzers.sh \
 	corpus/filters/filter.bin corpus/filters/filter-size.bin \
 	corpus/filters/regress-empty-subject-unique.bin \
 	corpus/url/http-url.txt corpus/url/relative-path.txt \
-	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
-	corpus/header/full-response.txt corpus/header/redirect.txt \
-	corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
-	corpus/cachendx/regress-overadvance.bin \
-	corpus/cachendx/regress-append-overflow.bin
+	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt

fuzz/corpus/cachendx/new-format.txt

@@ -1,7 +0,0 @@
-8
-CACHE-1.1
-28
-Mon, 01 Jan 2024 00:00:00 GMT
-www.example.com
-/index.html
-123

fuzz/corpus/cachendx/old-format.txt

@@ -1,5 +0,0 @@
-3
-1.0
-www.example.com
-/page
-5

fuzz/corpus/cachendx/regress-append-overflow.bin

@@ -1,5 +0,0 @@
-9
-CACHE-1.129
-Wed, 01 Jan 2020 00:00:00 GMT2101
-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA
-42

fuzz/corpus/cachendx/regress-overadvance.bin

@@ -1,2 +0,0 @@
-32768
-CACHE-1.1

fuzz/corpus/header/full-response.txt

@@ -1,10 +0,0 @@
-HTTP/1.1 200 OK
-Content-Type: text/html; charset=utf-8
-Content-Length: 1234
-Content-Encoding: gzip
-Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
-Etag: "abc"
-Location: http://example.com/x
-Set-Cookie: ID=42; path=/; domain=.example.com
-Content-Range: bytes 0-99/100
-Transfer-Encoding: chunked

fuzz/corpus/header/redirect.txt

@@ -1,3 +0,0 @@
-HTTP/1.0 301 Moved
-Location: /elsewhere
-Content-Disposition: attachment; filename="a.pdf"

fuzz/fuzz-cachendx.c

@@ -1,71 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
-   walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
-   Mirrors the loader in cache_readex_new (htscache.c). */
-#include "fuzz.h"
-#include "htscache.h"
-#include "htslib.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  const char *const end = buf + size;
-  char firstline[256];
-  char *a = buf;
-
-  /* header: two length-prefixed fields (version, last-modified) */
-  a += cache_brstr(a, firstline, sizeof(firstline));
-  a += cache_brstr(a, firstline, sizeof(firstline));
-
-  /* body: newline-delimited host/file/position triples. The coucal insert
-     the real loader ends with is a separate library's concern; this harness
-     targets the length-prefixed scan that must stay inside the buffer. */
-  while (a != NULL && a < end) {
-    char BIGSTK line[HTS_URLMAXSIZE * 2];
-    char linepos[256];
-    int pos;
-    size_t used, avail;
-
-    a = strchr(a + 1, '\n');
-    if (a == NULL)
-      break;
-    a++;
-    a += cache_binput(a, end, line, HTS_URLMAXSIZE);
-    /* cap the append: binput's NUL can land at s[max] (mirrors cache_init) */
-    used = strlen(line);
-    avail = sizeof(line) - used - 1;
-    a += cache_binput(a, end, line + used,
-                      avail < HTS_URLMAXSIZE ? (int) avail : HTS_URLMAXSIZE);
-    a += cache_binput(a, end, linepos, 200);
-    sscanf(linepos, "%d", &pos);
-    (void) pos;
-  }
-
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-header.c

@@ -1,74 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
-   status line, treathead on each following header. Both consume raw bytes
-   off the wire and copy fields into fixed htsblk buffers; treathead also
-   mutates its line in place and drives cookie parsing. */
-#include "fuzz.h"
-#include "htslib.h"
-#include "htsbauth.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  htsblk r;
-  t_cookie *cookie = calloct(1, sizeof(*cookie));
-  char *line = malloct(size + 1);
-  char *p = buf;
-  int first = 1;
-
-  memset(&r, 0, sizeof(r));
-  cookie->max_len = (int) sizeof(cookie->data);
-  r.location = malloct(HTS_URLMAXSIZE * 2);
-  r.location[0] = '\0';
-
-  /* feed one header line at a time, as the receive loop does */
-  while (p != NULL && *p != '\0') {
-    char *nl = strchr(p, '\n');
-    size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
-    size_t i, len = 0;
-
-    /* binput drops every '\r' on the wire; mirror it */
-    for (i = 0; i < n; i++)
-      if (p[i] != '\r')
-        line[len++] = p[i];
-    line[len] = '\0';
-    if (first) {
-      treatfirstline(&r, line);
-      first = 0;
-    } else {
-      treathead(cookie, "www.example.com", "/", &r, line);
-    }
-    p = (nl != NULL) ? nl + 1 : NULL;
-  }
-
-  freet(r.location);
-  freet(line);
-  freet(cookie);
-  freet(buf);
-  return 0;
-}

m4/ax_add_fortify_source.m4

@@ -1,119 +0,0 @@
-# ===========================================================================
-#  https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_ADD_FORTIFY_SOURCE
-#
-# DESCRIPTION
-#
-#   Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
-#   redefinition warnings, other cpp warnings or linker. Some distributions
-#   (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
-#   their compilers, leading to unnecessary warnings in the form of
-#
-#     <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
-#     <built-in>: note: this is the location of the previous definition
-#
-#   which is a problem if -Werror is enabled. This macro checks whether
-#   _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
-#   to CPPFLAGS.
-#
-#   Newer mingw-w64 msys2 package comes with a bug in
-#   headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
-#   and would need -lssp or -fstack-protector.  See
-#   https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
-#   link it.
-#
-# LICENSE
-#
-#   Copyright (c) 2017 David Seifert <soap@gentoo.org>
-#   Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved.  This file is offered as-is, without any
-#   warranty.
-
-#serial 10
-
-AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
-    ac_save_cflags=$CFLAGS
-    ac_cwerror_flag=yes
-    AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
-    ax_add_fortify_3_failed=
-    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],
-            [[
-                #ifndef _FORTIFY_SOURCE
-                    return 0;
-                #else
-                    _FORTIFY_SOURCE_already_defined;
-                #endif
-            ]]
-        )],
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([[
-                #define _FORTIFY_SOURCE 3
-                #include <string.h>
-                int main(void) {
-                    char *s = " ";
-                    strcpy(s, "x");
-                    return strlen(s)-1;
-                }
-              ]]
-            )],
-            [
-              AC_MSG_RESULT([yes])
-              CFLAGS=$ac_save_cflags
-              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
-            ], [
-              AC_MSG_RESULT([no])
-              ax_add_fortify_3_failed=1
-            ],
-        ),
-        [
-          AC_MSG_RESULT([no])
-          ax_add_fortify_3_failed=1
-        ])
-    if test -n "$ax_add_fortify_3_failed"
-    then
-    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],
-            [[
-                #ifndef _FORTIFY_SOURCE
-                    return 0;
-                #else
-                    _FORTIFY_SOURCE_already_defined;
-                #endif
-            ]]
-        )],
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([[
-                #define _FORTIFY_SOURCE 2
-                #include <string.h>
-                int main(void) {
-                    char *s = " ";
-                    strcpy(s, "x");
-                    return strlen(s)-1;
-                }
-              ]]
-            )],
-            [
-              AC_MSG_RESULT([yes])
-              CFLAGS=$ac_save_cflags
-              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
-            ], [
-              AC_MSG_RESULT([no])
-              CFLAGS=$ac_save_cflags
-            ],
-        ),
-        [
-          AC_MSG_RESULT([no])
-          CFLAGS=$ac_save_cflags
-        ])
-    fi
-])

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "07 July 2026" "httrack website copier"
+.TH httrack 1 "27 June 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -51,7 +51,6 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
 [ \fB\-%K, \-\-cookies\-file\fR ]
-[ \fB\-%Y, \-\-why\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -219,8 +218,6 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
 .IP \-%K
 load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
-.IP \-%Y
-explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j

src/htsalias.c

@@ -114,8 +114,6 @@ const char *hts_optalias[][4] = {
    "strip [host/pattern=]key1,key2,... from URLs"},
   {"cookies-file", "-%K", "param1",
    "load extra cookies from a Netscape cookies.txt"},
-  {"why", "-%Y", "param1",
-   "explain which filter rule accepts or rejects a URL, then exit"},
   {"pause", "-%G", "param1",
    "random pause of MIN[:MAX] seconds between files"},
   {"generate-errors", "-o", "single", ""},

src/htscache.c

@@ -1937,24 +1937,16 @@ void cache_init(cache_back * cache, httrackp * opt) {
             char BIGSTK line[HTS_URLMAXSIZE * 2];
             char linepos[256];
             int pos;
-            size_t used, avail;
 
-            const char *const end = cache->use + buffl;
-
-            while ((a != NULL) && (a < end)) {
+            while((a != NULL) && (a < (cache->use + buffl))) {
               a = strchr(a + 1, '\n');  /* start of line */
               if (a) {
                 a++;
                 /* read "host/file" */
-                a += cache_binput(a, end, line, HTS_URLMAXSIZE);
-                /* cap the append: binput's NUL can land at s[max] */
-                used = strlen(line);
-                avail = sizeof(line) - used - 1;
-                a += cache_binput(a, end, line + used,
-                                  avail < HTS_URLMAXSIZE ? (int) avail
-                                                         : HTS_URLMAXSIZE);
+                a += binput(a, line, HTS_URLMAXSIZE);
+                a += binput(a, line + strlen(line), HTS_URLMAXSIZE);
                 /* read position */
-                a += cache_binput(a, end, linepos, 200);
+                a += binput(a, linepos, 200);
                 sscanf(linepos, "%d", &pos);
                 coucal_add(cache->hashtable, line, pos);
               }
@@ -2299,40 +2291,23 @@ int cache_brstr(char *adr, char *s, size_t s_size) {
   char buff[256 + 4];
 
   off = binput(adr, buff, 256);
-  /* binput stops at the buffer's terminating NUL; a value can only follow a
-     real line terminator, so never step past end-of-buffer. */
-  if (adr[off - 1] == '\0') {
-    s[0] = '\0';
-    return off - 1;
-  }
-  /* an empty/non-numeric field leaves i unset: treat as length 0 */
-  if (sscanf(buff, "%d", &i) != 1 || i < 0 || i > 32768)
+  adr += off;
+  sscanf(buff, "%d", &i);
+  if (i < 0 || i > 32768) /* guard a corrupt length */
     i = 0;
   if (i > 0) {
-    /* A corrupt/truncated cache may declare a length past the buffer end;
-       bound both the copy and the advance to the bytes actually present. */
-    const size_t avail = strnlen(adr + off, (size_t) i);
-    const size_t store = avail < s_size ? avail : s_size - 1;
+    /* copy at most s_size-1 bytes; advance past the full field regardless */
+    const size_t store = (size_t) i < s_size ? (size_t) i : s_size - 1;
 
-    memcpy(s, adr + off, store);
+    strncpy(s, adr, store);
     s[store] = '\0';
-    off += (int) avail;
   } else {
     s[0] = '\0';
   }
+  off += i;
   return off;
 }
 
-/* binput bounded to a NUL-terminated buffer: refuse to start a read at or
-   past `end`, so a prior over-advance can't walk a cache-index parse OOB. */
-int cache_binput(char *adr, const char *end, char *s, int max) {
-  if (adr >= end) {
-    s[0] = '\0';
-    return 0;
-  }
-  return binput(adr, s, max);
-}
-
 /* idem, mais en int */
 int cache_brint(char *adr, int *i) {
   char s[256];

src/htscache.h

@@ -97,8 +97,6 @@ int cache_readdata(cache_back * cache, const char *str1, const char *str2,
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);
-/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
-int cache_binput(char *adr, const char *end, char *s, int max);
 int cache_brint(char *adr, int *i);
 void cache_rint(FILE * fp, int *i);
 void cache_rLLint(FILE * fp, LLint * i);

src/htscache_selftest.c

@@ -1400,55 +1400,3 @@ int cache_corruption_selftest(httrackp *opt, const char *dir) {
 
   return failures;
 }
-
-/* Old-format (.ndx) import: cache_init scans host+file pairs into one line[]
-   accumulator; a record longer than it must not write past it (ASan gates). */
-int cache_oldndx_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-  cache_back cache;
-  FILE *fp;
-  intptr_t pos = 0;
-
-  selftest_tag = "cache-oldndx";
-  golden_setup(opt, dir);
-#ifdef _WIN32
-  mkdir(reconcile_st_path(opt, "hts-cache"));
-#else
-  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
-#endif
-
-  fp = fopen(reconcile_st_path(opt, "hts-cache/new.ndx"), "wb");
-  assertf(fp != NULL);
-  assertf(cache_wstr(fp, "CACHE-1.1") == 0);
-  assertf(cache_wstr(fp, "Wed, 01 Jan 2020 00:00:00 GMT") == 0);
-  /* a normal record, then one whose host saturates both line[] reads; the
-     oversized one stays last (the scan does not resync after it) */
-  assertf(cache_wstr(fp, "www.example.com\n/index.html\n") == 0);
-  assertf(fwrite("42\n", 1, 3, fp) == 3);
-  {
-    const size_t hostlen = HTS_URLMAXSIZE * 2 + 16;
-    char *rec = malloct(hostlen + sizeof("\n/x\n"));
-
-    memset(rec, 'A', hostlen);
-    memcpy(rec + hostlen, "\n/x\n", sizeof("\n/x\n"));
-    assertf(cache_wstr(fp, rec) == 0);
-    assertf(fwrite("7\n", 1, 2, fp) == 2);
-    freet(rec);
-  }
-  fclose(fp);
-  reconcile_put(opt, "hts-cache/new.dat", 0); /* .ndx alone is not loaded */
-
-  selftest_open_for_read(&cache, opt);
-  if (!coucal_read(cache.hashtable, "www.example.com/index.html", &pos) ||
-      pos != 42) {
-    fprintf(stderr, "cache-oldndx: normal record not indexed (pos=%d)\n",
-            (int) pos);
-    failures++;
-  }
-  if (cache.olddat != NULL) {
-    fclose(cache.olddat);
-    cache.olddat = NULL;
-  }
-  selftest_close(&cache);
-  return failures;
-}

src/htscache_selftest.h

@@ -65,11 +65,6 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir);
    tainting a sibling entry. */
 int cache_corruption_selftest(httrackp *opt, const char *dir);
 
-/* Drive the old-format (.ndx) cache import under <dir> with a record longer
-   than the loader's line accumulator. Returns the failed-check count; the
-   overflow itself is caught by the sanitizer builds. */
-int cache_oldndx_selftest(httrackp *opt, const char *dir);
-
 #endif
 
 #endif

src/htscore.c

@@ -846,52 +846,6 @@ int httpmirror(char *url1, httrackp * opt) {
       }
     }                           // while
 
-    /* --why: print which filter rule decides for this URL, then stop */
-    if (StringNotEmpty(opt->why_url)) {
-      char BIGSTK url[HTS_URLMAXSIZE * 2];
-      lien_adrfil af;
-
-      if (strstr(StringBuff(opt->why_url), ":/") == NULL)
-        snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
-      else
-        strcpybuff(url, StringBuff(opt->why_url));
-      if (ident_url_absolute(url, &af) < 0) {
-        printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
-      } else {
-        char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
-        int jok, jokDepth = 0;
-
-        /* the two forms the wizard tests */
-        strcpybuff(l, jump_identification_const(af.adr));
-        if (*af.fil != '/')
-          strcatbuff(l, "/");
-        strcatbuff(l, af.fil);
-        if (!link_has_authority(af.adr))
-          strcpybuff(lfull, "http://");
-        else
-          lfull[0] = '\0';
-        strcatbuff(lfull, af.adr);
-        if (*af.fil != '/')
-          strcatbuff(lfull, "/");
-        strcatbuff(lfull, af.fil);
-        jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
-                               NULL, &jokDepth);
-        if (jok > 0)
-          printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
-                 filters[jokDepth]);
-        else if (jok < 0)
-          printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
-                 filters[jokDepth]);
-        else
-          printf("%s: no filter rule matches; the wizard decides "
-                 "(same-site links are followed by default)" LF,
-                 url);
-      }
-      freet(primary);
-      XH_extuninit;
-      return 1;
-    }
-
     /* load URL file list */
     /* OPTIMIZED for fast load */
     if (StringNotEmpty(opt->filelist)) {

src/htscoremain.c

@@ -1803,23 +1803,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   StringCopy(opt->cookies_file, argv[na]);
                 }
                 break;
-              case 'Y': // why: explain the filter verdict for a URL, no crawl
-                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
-                  HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
-                  printf(
-                      "Example: --why \"http://www.example.com/file.zip\"\n");
-                  htsmain_free();
-                  return -1;
-                } else {
-                  na++;
-                  if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
-                    HTS_PANIC_PRINTF("why URL too long");
-                    htsmain_free();
-                    return -1;
-                  }
-                  StringCopy(opt->why_url, argv[na]);
-                }
-                break;
               case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF("Option pause needs a blank space and a "
@@ -1953,20 +1936,18 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     char BIGSTK url[HTS_URLMAXSIZE * 2];
                     char linepos[256];
                     int pos;
-                    LLint cacheNdxLen = 0;
-                    char *cacheNdx = readfile2(
-                        fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                StringBuff(opt->path_log), "hts-cache/new.ndx"),
-                        &cacheNdxLen);
+                    char *cacheNdx =
+                      readfile(fconcat
+                               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                                "hts-cache/new.ndx"));
                     cache_init(&cache, opt);    /* load cache */
                     if (cacheNdx != NULL) {
                       char firstline[256];
                       char *a = cacheNdx;
-                      const char *const end = cacheNdx + cacheNdxLen;
 
                       a += cache_brstr(a, firstline, sizeof(firstline));
                       a += cache_brstr(a, firstline, sizeof(firstline));
-                      while (a != NULL && a < end) {
+                      while(a != NULL) {
                         a = strchr(a + 1, '\n');        /* start of line */
                         if (a) {
                           htsblk r;
@@ -1974,15 +1955,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                           /* */
                           a++;
                           /* read "host/file" */
-                          a += cache_binput(a, end, afs.af.adr, HTS_URLMAXSIZE);
-                          a += cache_binput(a, end, afs.af.fil, HTS_URLMAXSIZE);
+                          a += binput(a, afs.af.adr, HTS_URLMAXSIZE);
+                          a += binput(a, afs.af.fil, HTS_URLMAXSIZE);
                           url[0] = '\0';
                           if (!link_has_authority(afs.af.adr))
                             strcatbuff(url, "http://");
                           strcatbuff(url, afs.af.adr);
                           strcatbuff(url, afs.af.fil);
                           /* read position */
-                          a += cache_binput(a, end, linepos, 200);
+                          a += binput(a, linepos, 200);
                           sscanf(linepos, "%d", &pos);
                           if (!hasFilter
                               || (strjoker(url, filter, NULL, NULL) != NULL)

src/htsfilters.c

@@ -94,32 +94,6 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
   return verdict;
 }
 
-int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
-                     const char *nom2, LLint *size, int *size_flag,
-                     int *depth) {
-  int depth1 = 0, depth2 = 0;
-  int flag1 = 0, flag2 = 0;
-  LLint sz1 = 0, sz2 = 0;
-  int jok1, jok2, use1;
-
-  if (size) {
-    sz1 = *size;
-    sz2 = *size;
-  }
-  jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
-                     size_flag ? &flag1 : NULL, &depth1);
-  jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
-                     size_flag ? &flag2 : NULL, &depth2);
-  use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
-  if (size)
-    *size = use1 ? sz1 : sz2;
-  if (size_flag)
-    *size_flag = use1 ? flag1 : flag2;
-  if (depth)
-    *depth = use1 ? depth1 : depth2;
-  return use1 ? jok1 : jok2;
-}
-
 // supercomparateur joker (tm)
 // compare a et b (b=avec joker dedans), case insensitive [voir CI]
 // renvoi l'adresse de la première lettre de la chaine

src/htsfilters.h

@@ -41,11 +41,6 @@ Please visit our Website: http://www.httrack.com
 
 int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
                 int *size_flag, int *depth);
-/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
-   scheme); the match latest in the list wins, a "don't know" verdict defers.
-   Returns the merged verdict; the out-params carry the winner's values. */
-int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
-                     const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);

src/htshelp.c

@@ -543,7 +543,6 @@ void help(const char *app, int more) {
   infomsg("Spider options:");
   infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
   infomsg(" %K  load extra cookies from a Netscape cookies.txt");
-  infomsg(" %Y  explain which filter rule accepts or rejects a URL, then exit");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
   infomsg

src/htslib.c

@@ -1562,7 +1562,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       }
       // An empty/whitespace Content-Type value yields no token: keep the
       // sentinel default rather than reading an uninitialized tempo.
-      if (sscanf(rcvd + p, "%1099s", tempo) == 1) { // tempo[1100], server data
+      if (sscanf(rcvd + p, "%s", tempo) == 1) {
         if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
           strcpybuff(retour->contenttype, tempo);
         else
@@ -1657,11 +1657,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         if (a)
           *a = '\0';
       }
-      // Bound to tempo[1100]; no token => leave empty, don't read uninit tempo.
-      if (sscanf(a, "%1099s", tempo) == 1 && strlen(tempo) < 64)
+      sscanf(a, "%s", tempo);
+      if (strlen(tempo) < 64)   // pas trop long!!
         strcpybuff(retour->contentencoding, tempo);
       else
-        retour->contentencoding[0] = '\0';
+        retour->contentencoding[0] = '\0';      // erreur
 #if HTS_USEZLIB
       /* Check known encodings */
       if (retour->contentencoding[0]) {
@@ -6006,7 +6006,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
   StringCopy(opt->strip_query, "");
   StringCopy(opt->cookies_file, "");
-  StringCopy(opt->why_url, "");
   opt->pause_min_ms = 0;
   opt->pause_max_ms = 0;
   opt->ftp_proxy = HTS_TRUE;
@@ -6154,7 +6153,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
     StringFree(opt->mod_blacklist);
     StringFree(opt->strip_query);
     StringFree(opt->cookies_file);
-    StringFree(opt->why_url);
 
     StringFree(opt->path_html);
     StringFree(opt->path_html_utf8);

src/htsopt.h

@@ -536,8 +536,6 @@ struct httrackp {
                                  (--cookies-file) */
   int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
   int pause_max_ms; /**< inter-file pause upper bound, ms */
-  String why_url;   /**< URL to diagnose (--why): print the deciding filter rule
-                         and exit without crawling */
 };
 
 /* Running statistics for a mirror. */

src/htsselftest.c

@@ -46,7 +46,6 @@ Please visit our Website: http://www.httrack.com
 #include "htsdefines.h"
 #include "htslib.h"
 #include "htsparse.h"
-#include "htscache.h"
 #include "htscache_selftest.h"
 #include "htsdns_selftest.h"
 #include "htscharset.h"
@@ -621,26 +620,6 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
-static int st_filterdual(httrackp *opt, int argc, char **argv) {
-  int depth = -1, verdict;
-
-  (void) opt;
-  if (argc < 3) {
-    fprintf(stderr,
-            "filterdual: needs <string1> <string2> <filter> [filter...]\n");
-    return 1;
-  }
-  verdict = fa_strjoker_dual(0, &argv[2], argc - 2, argv[0], argv[1], NULL,
-                             NULL, &depth);
-  printf("verdict=%s rule=%d\n",
-         verdict > 0   ? "allowed"
-         : verdict < 0 ? "forbidden"
-                       : "unknown",
-         depth);
-  return 0;
-}
-
 static int st_simplify(httrackp *opt, int argc, char **argv) {
   (void) opt;
   if (argc < 1) {
@@ -1164,26 +1143,6 @@ static int st_header(httrackp *opt, int argc, char **argv) {
     treathead(NULL, "www.example.com", "/", &r, line);
   }
   printf("contenttype=%s cdispo=%s\n", r.contenttype, r.cdispo);
-  printf("contentencoding=%s\n", r.contentencoding);
-  return 0;
-}
-
-/* An over-long header value must not overflow treathead's tempo[1100]. */
-static int st_headerlong(httrackp *opt, int argc, char **argv) {
-  htsblk r;
-  char BIGSTK line[HTS_URLMAXSIZE * 2];
-  const char *const name = argc >= 1 ? argv[0] : "Content-Type:";
-  const int pad = 1500; /* > tempo[1100] */
-  size_t n;
-
-  (void) opt;
-  memset(&r, 0, sizeof(r));
-  n = (size_t) snprintf(line, sizeof(line), "%s ", name);
-  memset(line + n, 'a', pad);
-  line[n + pad] = '\0';
-  treathead(NULL, "www.example.com", "/", &r, line);
-  printf("contenttype_len=%d contentencoding_len=%d\n",
-         (int) strlen(r.contenttype), (int) strlen(r.contentencoding));
   return 0;
 }
 
@@ -1391,83 +1350,6 @@ static int st_cache(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
-/* A corrupt cache index (.ndx) must not walk the length-prefixed scan past
-   the buffer. Checks the two primitives the loader is built from. */
-static int st_cacheindex(httrackp *opt, int argc, char **argv) {
-  int fail = 0;
-
-  if (argc < 1) {
-    fprintf(stderr, "cacheindex: needs a directory\n");
-    return 1;
-  }
-
-  /* A length prefix that overstates the bytes present must bound the advance
-     to the buffer, not trust the declared length. */
-  {
-    static const char src[] = "32768\nCACHE-1.1";
-    const size_t len = sizeof(src) - 1;
-    char *buf = malloct(len + 1);
-    char s[256];
-    int off;
-
-    memcpy(buf, src, len + 1);
-    off = cache_brstr(buf, s, sizeof(s));
-    if (off > (int) len) {
-      printf("cacheindex: over-advance off=%d len=%d\n", off, (int) len);
-      fail = 1;
-    }
-    if (strcmp(s, "CACHE-1.1") != 0) {
-      printf("cacheindex: value=%s\n", s);
-      fail = 1;
-    }
-    freet(buf);
-  }
-
-  /* cache_binput reads a field while in bounds, but refuses one starting at
-     or past end-of-buffer. */
-  {
-    char buf[8] = "ab\ncd";
-    const char *const end = buf + 5;
-    char s[16];
-
-    if (cache_binput(buf, end, s, sizeof(s)) != 3 || strcmp(s, "ab") != 0)
-      fail = 1; /* normal read: "ab" then the '\n', 3 bytes consumed */
-    if (cache_binput(end, end, s, sizeof(s)) != 0 || s[0] != '\0')
-      fail = 1;
-  }
-
-  /* Drive the full loader scan over a truncated index: a declared length that
-     overshoots plus a half-written entry. ASan aborts here on the pre-fix
-     scan; the cursor must never leave the buffer. */
-  {
-    static const char src[] = "9\nCACHE-1.1\n99\nwww.example.com\n/a";
-    const size_t len = sizeof(src) - 1;
-    char *buf = malloct(len + 1);
-    const char *const end = buf + len;
-    char line[256];
-    char *a = buf;
-
-    memcpy(buf, src, len + 1);
-    a += cache_brstr(a, line, sizeof(line));
-    a += cache_brstr(a, line, sizeof(line));
-    while (a != NULL && a < end) {
-      a = strchr(a + 1, '\n');
-      if (a == NULL)
-        break;
-      a++;
-      a += cache_binput(a, end, line, sizeof(line));
-    }
-    freet(buf);
-  }
-
-  /* the real loader loop, over a crafted old-format index */
-  if (cache_oldndx_selftest(opt, argv[0]) != 0)
-    fail = 1;
-
-  printf("cacheindex: %s\n", fail ? "FAIL" : "OK");
-  return fail;
-}
-
 static int st_cache_golden(httrackp *opt, int argc, char **argv) {
   int regen, err;
 
@@ -2252,8 +2134,6 @@ static const struct selftest_entry {
     {"filtersize", "<size> <string> <filter>...",
      "size-aware filter verdict (negative size = unknown/scan time)",
      st_filtersize},
-    {"filterdual", "<string1> <string2> <filter>...",
-     "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
@@ -2283,16 +2163,11 @@ static const struct selftest_entry {
     {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
-    {"headerlong", "[header-name:]",
-     "over-long header value must not overflow the parse scratch",
-     st_headerlong},
     {"savename", "<fil> <content-type> [key=value ...]",
      "local save-name for a URL", st_savename},
     {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
      st_sniff},
     {"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
-    {"cacheindex", "<dir>", "cache-index (.ndx) parse must stay in bounds",
-     st_cacheindex},
     {"cache-golden", "<dir> [regen]", "frozen cache-format read self-test",
      st_cache_golden},
     {"cache-writefail", "<dir>", "cache write-failure handling self-test",

src/htswizard.c

@@ -524,11 +524,28 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
 
         // filters, 0=sait pas 1=ok -1=interdit
         {
-          int jokDepth = 0;
+          int jokDepth1 = 0, jokDepth2 = 0;
+          int jok1 = 0, jok2 = 0;
 
-          jok = fa_strjoker_dual(/*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, l,
-                                 NULL, NULL, &jokDepth);
-          mdepth = _FILTERS[jokDepth];
+          jok1 =
+            fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, NULL, NULL,
+                        &jokDepth1);
+          jok2 =
+            fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, l, NULL, NULL,
+                        &jokDepth2);
+          if (jok2 == 0) {      // #2 doesn't know
+            jok = jok1;         // then, use #1
+            mdepth = _FILTERS[jokDepth1];
+          } else if (jok1 == 0) {       // #1 doesn't know
+            jok = jok2;         // then, use #2
+            mdepth = _FILTERS[jokDepth2];
+          } else if (jokDepth1 >= jokDepth2) {  // #1 matching rule is "after" #2, then it is prioritary
+            jok = jok1;
+            mdepth = _FILTERS[jokDepth1];
+          } else {              // #2 matching rule is "after" #1, then it is prioritary
+            jok = jok2;
+            mdepth = _FILTERS[jokDepth2];
+          }
         }
 
         if (jok == 1) {         // autorisé
@@ -948,10 +965,34 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz
 
       // filters, 0=sait pas 1=ok -1=interdit
       {
-        sz = size;
-        jok = fa_strjoker_dual(/*url */ 0, *opt->filters.filters,
-                               *opt->filters.filptr, lfull, l, &sz, &size_flag,
-                               NULL);
+        int jokDepth1 = 0, jokDepth2 = 0;
+        int jok1 = 0, jok2 = 0;
+        LLint sz1 = size, sz2 = size;
+        int size_flag1 = 0, size_flag2 = 0;
+
+        jok1 =
+          fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
+                      lfull, &sz1, &size_flag1, &jokDepth1);
+        jok2 =
+          fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
+                      l, &sz2, &size_flag2, &jokDepth2);
+        if (jok2 == 0) {        // #2 doesn't know
+          jok = jok1;           // then, use #1
+          sz = sz1;
+          size_flag = size_flag1;
+        } else if (jok1 == 0) { // #1 doesn't know
+          jok = jok2;           // then, use #2
+          sz = sz2;
+          size_flag = size_flag2;
+        } else if (jokDepth1 >= jokDepth2) {    // #1 matching rule is "after" #2, then it is prioritary
+          jok = jok1;
+          sz = sz1;
+          size_flag = size_flag1;
+        } else {                // #2 matching rule is "after" #1, then it is prioritary
+          jok = jok2;
+          sz = sz2;
+          size_flag = size_flag2;
+        }
       }
 
       // log

tests/01_engine-cacheindex.test

@@ -1,13 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# Cache-index (.ndx) parse must stay inside the buffer on a corrupt length
-# prefix or an oversized record (ASan aborts here on the pre-fix binary; the
-# OK check gates the non-sanitized build too).
-
-dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
-
-test "$(httrack "-#test=cacheindex" "$dir")" == 'cacheindex: OK'

tests/01_engine-filterdual.test

@@ -1,20 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# fa_strjoker_dual: merged verdict on the two URL forms the engine tests;
-# the match latest in the filter list wins, an unknown verdict defers.
-
-fdual() {
-    local want="$1"
-    shift
-    test "$(httrack -O /dev/null -#test=filterdual "$@")" == "$want" || exit 1
-}
-
-fdual 'verdict=unknown rule=0' zzz yyy '+a*'     # neither form matches
-fdual 'verdict=forbidden rule=0' aaa zzz '-a*'   # only form 1 matches
-fdual 'verdict=forbidden rule=0' zzz aaa '-a*'   # only form 2 matches
-fdual 'verdict=forbidden rule=1' a b '+a*' '-b*' # later rule wins (form 2)
-fdual 'verdict=forbidden rule=1' b a '+a*' '-b*' # later rule wins (form 1)
-fdual 'verdict=allowed rule=0' a aa '+a*'        # tie: form 1 wins

tests/01_engine-header.test

@@ -27,16 +27,3 @@ hdr 'contenttype= cdispo=report.pdf' \
 # Path components in the filename are dropped on the wire (RFC 2616).
 hdr 'contenttype= cdispo=evil.pdf' \
     'Content-Disposition: attachment; filename="../../evil.pdf"'
-
-# An over-long header value must not overflow the parse scratch (ASan aborts
-# here on the pre-fix binary).
-test "$(httrack -O /dev/null -#test=headerlong 'Content-Type:')" \
-    == 'contenttype_len=32 contentencoding_len=0'
-test "$(httrack -O /dev/null -#test=headerlong 'Content-Encoding:')" \
-    == 'contenttype_len=0 contentencoding_len=0'
-
-# An empty Content-Encoding yields no token: leave it empty, don't read the
-# uninitialized scratch (MSan aborts here on the pre-fix binary).
-enc() { httrack -O /dev/null -#test=header "$1" | grep '^contentencoding='; }
-test "$(enc 'Content-Encoding:')" == 'contentencoding='
-test "$(enc 'Content-Encoding: gzip')" == 'contentencoding=gzip'

tests/40_local-why.test

@@ -1,35 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# --why: report which +/- filter rule decides for a URL, without crawling.
-
-tmpdir="$(mktemp -d)"
-trap 'rm -rf "$tmpdir"' EXIT
-
-why() {
-    local want="$1" out
-    shift
-    out="$(httrack -O "$tmpdir/p" -q "$@" |
-        grep -E 'accepted|rejected|no filter|unable')"
-    test "$out" == "$want" || {
-        echo "want: $want"
-        echo "got : $out"
-        exit 1
-    }
-}
-
-base=http://www.example.com
-why "$base/x.zip: rejected by rule #2 (-*.zip)" \
-    --why "$base/x.zip" "$base/" '+*.png' '-*.zip'
-why "$base/a.png: accepted by rule #1 (+*.png)" \
-    --why "$base/a.png" "$base/" '+*.png' '-*.zip'
-why "$base/i.html: no filter rule matches; the wizard decides (same-site links are followed by default)" \
-    --why "$base/i.html" "$base/" '+*.png' '-*.zip'
-# scheme-less input gets the same http:// default as primary URLs
-why "$base/x.zip: rejected by rule #2 (-*.zip)" \
-    --why "www.example.com/x.zip" "$base/" '+*.png' '-*.zip'
-# the rule latest in the list decides
-why "$base/x.zip: accepted by rule #3 (+*x.zip)" \
-    --why "$base/x.zip" "$base/" '+*.png' '-*.zip' '+*x.zip'

tests/Makefile.am

@@ -35,10 +35,8 @@ TESTS = \
 	01_engine-entities.test \
 	01_engine-filelist.test \
 	01_engine-filter.test \
-	01_engine-filterdual.test \
 	01_engine-ftp-line.test \
 	01_engine-ftp-userpass.test \
-	01_engine-cacheindex.test \
 	01_engine-hashtable.test \
 	01_engine-header.test \
 	01_engine-idna.test \
@@ -107,7 +105,6 @@ TESTS = \
 	36_local-bigcrawl.test \
 	37_local-cache-outage.test \
 	38_local-update-304.test \
-	39_local-delayed-cancel.test \
-	40_local-why.test
+	39_local-delayed-cancel.test
 
 CLEANFILES = check-network_sh.cache