Fix remote stack overflow and uninit read in Content-Type/-Encoding parsing (#506)

treathead() parsed the Content-Type and Content-Encoding response-header
values with a width-less sscanf("%s") into a 1100-byte stack scratch,
bounds-checking the result only after the copy. The receive path in
back_wait hands treathead a line buffer of up to 2000 bytes (htsback.c
rcvd[2048], binput cap 2000), so a hostile server sending a header value
longer than 1099 non-whitespace bytes overflowed the stack. Bound both
scans to the buffer (%1099s), matching the existing %31s/%255s idiom in
the request-line parsers.

The Content-Encoding branch also ignored the sscanf return, so an empty
value (e.g. "Content-Encoding:") left tempo uninitialized and then ran
strlen() over it: an uninitialized read that can also over-read past the
buffer. The Content-Type branch already guarded this; mirror it.

Found via the P3-3 static-analysis baseline. New -#test=headerlong plus
empty-value cases in 01_engine-header.test build the hostile inputs in
the handler (the CLI caps argument length); they abort on the pre-fix
binary under ASan (overflow) and MSan (uninitialized read).

Signed-off-by: Xavier Roche <roche@httrack.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

.github/workflows/codeql.yml

@@ -0,0 +1,57 @@
+# CodeQL static analysis (C). The security-extended suite covers the classes
+# this codebase actually fights: overflows, tainted-size allocs, format bugs.
+name: CodeQL
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+  schedule:
+    # Weekly re-scan of master so new/updated queries land without a push.
+    - cron: "17 4 * * 1"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: codeql-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: analyze (c-cpp)
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      # Upload findings to the repo's code-scanning dashboard.
+      security-events: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - uses: github/codeql-action/init@v4
+        with:
+          languages: c-cpp
+          build-mode: manual
+          queries: security-extended
+
+      # Manual build: CodeQL traces the compiler, so build exactly what ships.
+      - name: Build
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure
+          make -j"$(nproc)"
+
+      - uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:c-cpp"

configure.ac

@@ -102,7 +102,8 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
 AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
 AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
 AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
-AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
+AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
+	[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
 AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
 AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
 AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -110,6 +111,13 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
 AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
 AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
 
+# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
+# sanitizer builds, whose interceptors want the unfortified calls.
+case "$CFLAGS" in
+*-fsanitize=*) ;;
+*) AX_ADD_FORTIFY_SOURCE ;;
+esac
+
 # Force libc back into DT_NEEDED for libraries that reach it only through
 # libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
 # linker; Apple ld rejects these flags and links libSystem unconditionally.

m4/ax_add_fortify_source.m4

@@ -0,0 +1,119 @@
+# ===========================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_ADD_FORTIFY_SOURCE
+#
+# DESCRIPTION
+#
+#   Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
+#   redefinition warnings, other cpp warnings or linker. Some distributions
+#   (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
+#   their compilers, leading to unnecessary warnings in the form of
+#
+#     <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
+#     <built-in>: note: this is the location of the previous definition
+#
+#   which is a problem if -Werror is enabled. This macro checks whether
+#   _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
+#   to CPPFLAGS.
+#
+#   Newer mingw-w64 msys2 package comes with a bug in
+#   headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
+#   and would need -lssp or -fstack-protector.  See
+#   https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
+#   link it.
+#
+# LICENSE
+#
+#   Copyright (c) 2017 David Seifert <soap@gentoo.org>
+#   Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#serial 10
+
+AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
+    ac_save_cflags=$CFLAGS
+    ac_cwerror_flag=yes
+    AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
+    ax_add_fortify_3_failed=
+    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
+    AC_LINK_IFELSE([
+        AC_LANG_PROGRAM([],
+            [[
+                #ifndef _FORTIFY_SOURCE
+                    return 0;
+                #else
+                    _FORTIFY_SOURCE_already_defined;
+                #endif
+            ]]
+        )],
+        AC_LINK_IFELSE([
+            AC_LANG_SOURCE([[
+                #define _FORTIFY_SOURCE 3
+                #include <string.h>
+                int main(void) {
+                    char *s = " ";
+                    strcpy(s, "x");
+                    return strlen(s)-1;
+                }
+              ]]
+            )],
+            [
+              AC_MSG_RESULT([yes])
+              CFLAGS=$ac_save_cflags
+              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
+            ], [
+              AC_MSG_RESULT([no])
+              ax_add_fortify_3_failed=1
+            ],
+        ),
+        [
+          AC_MSG_RESULT([no])
+          ax_add_fortify_3_failed=1
+        ])
+    if test -n "$ax_add_fortify_3_failed"
+    then
+    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
+    AC_LINK_IFELSE([
+        AC_LANG_PROGRAM([],
+            [[
+                #ifndef _FORTIFY_SOURCE
+                    return 0;
+                #else
+                    _FORTIFY_SOURCE_already_defined;
+                #endif
+            ]]
+        )],
+        AC_LINK_IFELSE([
+            AC_LANG_SOURCE([[
+                #define _FORTIFY_SOURCE 2
+                #include <string.h>
+                int main(void) {
+                    char *s = " ";
+                    strcpy(s, "x");
+                    return strlen(s)-1;
+                }
+              ]]
+            )],
+            [
+              AC_MSG_RESULT([yes])
+              CFLAGS=$ac_save_cflags
+              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
+            ], [
+              AC_MSG_RESULT([no])
+              CFLAGS=$ac_save_cflags
+            ],
+        ),
+        [
+          AC_MSG_RESULT([no])
+          CFLAGS=$ac_save_cflags
+        ])
+    fi
+])

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "27 June 2026" "httrack website copier"
+.TH httrack 1 "07 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -51,6 +51,7 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
 [ \fB\-%K, \-\-cookies\-file\fR ]
+[ \fB\-%Y, \-\-why\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -218,6 +219,8 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
 .IP \-%K
 load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
+.IP \-%Y
+explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j

src/htsalias.c

@@ -114,6 +114,8 @@ const char *hts_optalias[][4] = {
    "strip [host/pattern=]key1,key2,... from URLs"},
   {"cookies-file", "-%K", "param1",
    "load extra cookies from a Netscape cookies.txt"},
+  {"why", "-%Y", "param1",
+   "explain which filter rule accepts or rejects a URL, then exit"},
   {"pause", "-%G", "param1",
    "random pause of MIN[:MAX] seconds between files"},
   {"generate-errors", "-o", "single", ""},

src/htscore.c

@@ -846,6 +846,52 @@ int httpmirror(char *url1, httrackp * opt) {
       }
     }                           // while
 
+    /* --why: print which filter rule decides for this URL, then stop */
+    if (StringNotEmpty(opt->why_url)) {
+      char BIGSTK url[HTS_URLMAXSIZE * 2];
+      lien_adrfil af;
+
+      if (strstr(StringBuff(opt->why_url), ":/") == NULL)
+        snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
+      else
+        strcpybuff(url, StringBuff(opt->why_url));
+      if (ident_url_absolute(url, &af) < 0) {
+        printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
+      } else {
+        char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
+        int jok, jokDepth = 0;
+
+        /* the two forms the wizard tests */
+        strcpybuff(l, jump_identification_const(af.adr));
+        if (*af.fil != '/')
+          strcatbuff(l, "/");
+        strcatbuff(l, af.fil);
+        if (!link_has_authority(af.adr))
+          strcpybuff(lfull, "http://");
+        else
+          lfull[0] = '\0';
+        strcatbuff(lfull, af.adr);
+        if (*af.fil != '/')
+          strcatbuff(lfull, "/");
+        strcatbuff(lfull, af.fil);
+        jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
+                               NULL, &jokDepth);
+        if (jok > 0)
+          printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
+                 filters[jokDepth]);
+        else if (jok < 0)
+          printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
+                 filters[jokDepth]);
+        else
+          printf("%s: no filter rule matches; the wizard decides "
+                 "(same-site links are followed by default)" LF,
+                 url);
+      }
+      freet(primary);
+      XH_extuninit;
+      return 1;
+    }
+
     /* load URL file list */
     /* OPTIMIZED for fast load */
     if (StringNotEmpty(opt->filelist)) {

src/htscoremain.c

@@ -1803,6 +1803,23 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   StringCopy(opt->cookies_file, argv[na]);
                 }
                 break;
+              case 'Y': // why: explain the filter verdict for a URL, no crawl
+                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
+                  HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
+                  printf(
+                      "Example: --why \"http://www.example.com/file.zip\"\n");
+                  htsmain_free();
+                  return -1;
+                } else {
+                  na++;
+                  if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
+                    HTS_PANIC_PRINTF("why URL too long");
+                    htsmain_free();
+                    return -1;
+                  }
+                  StringCopy(opt->why_url, argv[na]);
+                }
+                break;
               case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF("Option pause needs a blank space and a "

src/htsfilters.c

@@ -94,6 +94,32 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
   return verdict;
 }
 
+int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
+                     const char *nom2, LLint *size, int *size_flag,
+                     int *depth) {
+  int depth1 = 0, depth2 = 0;
+  int flag1 = 0, flag2 = 0;
+  LLint sz1 = 0, sz2 = 0;
+  int jok1, jok2, use1;
+
+  if (size) {
+    sz1 = *size;
+    sz2 = *size;
+  }
+  jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
+                     size_flag ? &flag1 : NULL, &depth1);
+  jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
+                     size_flag ? &flag2 : NULL, &depth2);
+  use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
+  if (size)
+    *size = use1 ? sz1 : sz2;
+  if (size_flag)
+    *size_flag = use1 ? flag1 : flag2;
+  if (depth)
+    *depth = use1 ? depth1 : depth2;
+  return use1 ? jok1 : jok2;
+}
+
 // supercomparateur joker (tm)
 // compare a et b (b=avec joker dedans), case insensitive [voir CI]
 // renvoi l'adresse de la première lettre de la chaine

src/htsfilters.h

@@ -41,6 +41,11 @@ Please visit our Website: http://www.httrack.com
 
 int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
                 int *size_flag, int *depth);
+/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
+   scheme); the match latest in the list wins, a "don't know" verdict defers.
+   Returns the merged verdict; the out-params carry the winner's values. */
+int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
+                     const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);

src/htshelp.c

@@ -543,6 +543,7 @@ void help(const char *app, int more) {
   infomsg("Spider options:");
   infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
   infomsg(" %K  load extra cookies from a Netscape cookies.txt");
+  infomsg(" %Y  explain which filter rule accepts or rejects a URL, then exit");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
   infomsg

src/htslib.c

@@ -1562,7 +1562,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       }
       // An empty/whitespace Content-Type value yields no token: keep the
       // sentinel default rather than reading an uninitialized tempo.
-      if (sscanf(rcvd + p, "%s", tempo) == 1) {
+      if (sscanf(rcvd + p, "%1099s", tempo) == 1) { // tempo[1100], server data
         if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
           strcpybuff(retour->contenttype, tempo);
         else
@@ -1657,11 +1657,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         if (a)
           *a = '\0';
       }
-      sscanf(a, "%s", tempo);
-      if (strlen(tempo) < 64)   // pas trop long!!
+      // Bound to tempo[1100]; no token => leave empty, don't read uninit tempo.
+      if (sscanf(a, "%1099s", tempo) == 1 && strlen(tempo) < 64)
         strcpybuff(retour->contentencoding, tempo);
       else
-        retour->contentencoding[0] = '\0';      // erreur
+        retour->contentencoding[0] = '\0';
 #if HTS_USEZLIB
       /* Check known encodings */
       if (retour->contentencoding[0]) {
@@ -6006,6 +6006,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
   StringCopy(opt->strip_query, "");
   StringCopy(opt->cookies_file, "");
+  StringCopy(opt->why_url, "");
   opt->pause_min_ms = 0;
   opt->pause_max_ms = 0;
   opt->ftp_proxy = HTS_TRUE;
@@ -6153,6 +6154,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
     StringFree(opt->mod_blacklist);
     StringFree(opt->strip_query);
     StringFree(opt->cookies_file);
+    StringFree(opt->why_url);
 
     StringFree(opt->path_html);
     StringFree(opt->path_html_utf8);

src/htsopt.h

@@ -536,6 +536,8 @@ struct httrackp {
                                  (--cookies-file) */
   int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
   int pause_max_ms; /**< inter-file pause upper bound, ms */
+  String why_url;   /**< URL to diagnose (--why): print the deciding filter rule
+                         and exit without crawling */
 };
 
 /* Running statistics for a mirror. */

src/htsselftest.c

@@ -620,6 +620,26 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
+static int st_filterdual(httrackp *opt, int argc, char **argv) {
+  int depth = -1, verdict;
+
+  (void) opt;
+  if (argc < 3) {
+    fprintf(stderr,
+            "filterdual: needs <string1> <string2> <filter> [filter...]\n");
+    return 1;
+  }
+  verdict = fa_strjoker_dual(0, &argv[2], argc - 2, argv[0], argv[1], NULL,
+                             NULL, &depth);
+  printf("verdict=%s rule=%d\n",
+         verdict > 0   ? "allowed"
+         : verdict < 0 ? "forbidden"
+                       : "unknown",
+         depth);
+  return 0;
+}
+
 static int st_simplify(httrackp *opt, int argc, char **argv) {
   (void) opt;
   if (argc < 1) {
@@ -1143,6 +1163,26 @@ static int st_header(httrackp *opt, int argc, char **argv) {
     treathead(NULL, "www.example.com", "/", &r, line);
   }
   printf("contenttype=%s cdispo=%s\n", r.contenttype, r.cdispo);
+  printf("contentencoding=%s\n", r.contentencoding);
+  return 0;
+}
+
+/* An over-long header value must not overflow treathead's tempo[1100]. */
+static int st_headerlong(httrackp *opt, int argc, char **argv) {
+  htsblk r;
+  char BIGSTK line[HTS_URLMAXSIZE * 2];
+  const char *const name = argc >= 1 ? argv[0] : "Content-Type:";
+  const int pad = 1500; /* > tempo[1100] */
+  size_t n;
+
+  (void) opt;
+  memset(&r, 0, sizeof(r));
+  n = (size_t) snprintf(line, sizeof(line), "%s ", name);
+  memset(line + n, 'a', pad);
+  line[n + pad] = '\0';
+  treathead(NULL, "www.example.com", "/", &r, line);
+  printf("contenttype_len=%d contentencoding_len=%d\n",
+         (int) strlen(r.contenttype), (int) strlen(r.contentencoding));
   return 0;
 }
 
@@ -2134,6 +2174,8 @@ static const struct selftest_entry {
     {"filtersize", "<size> <string> <filter>...",
      "size-aware filter verdict (negative size = unknown/scan time)",
      st_filtersize},
+    {"filterdual", "<string1> <string2> <filter>...",
+     "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
@@ -2163,6 +2205,9 @@ static const struct selftest_entry {
     {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
+    {"headerlong", "[header-name:]",
+     "over-long header value must not overflow the parse scratch",
+     st_headerlong},
     {"savename", "<fil> <content-type> [key=value ...]",
      "local save-name for a URL", st_savename},
     {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",

src/htswizard.c

@@ -524,28 +524,11 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
 
         // filters, 0=sait pas 1=ok -1=interdit
         {
-          int jokDepth1 = 0, jokDepth2 = 0;
-          int jok1 = 0, jok2 = 0;
+          int jokDepth = 0;
 
-          jok1 =
-            fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, NULL, NULL,
-                        &jokDepth1);
-          jok2 =
-            fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, l, NULL, NULL,
-                        &jokDepth2);
-          if (jok2 == 0) {      // #2 doesn't know
-            jok = jok1;         // then, use #1
-            mdepth = _FILTERS[jokDepth1];
-          } else if (jok1 == 0) {       // #1 doesn't know
-            jok = jok2;         // then, use #2
-            mdepth = _FILTERS[jokDepth2];
-          } else if (jokDepth1 >= jokDepth2) {  // #1 matching rule is "after" #2, then it is prioritary
-            jok = jok1;
-            mdepth = _FILTERS[jokDepth1];
-          } else {              // #2 matching rule is "after" #1, then it is prioritary
-            jok = jok2;
-            mdepth = _FILTERS[jokDepth2];
-          }
+          jok = fa_strjoker_dual(/*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, l,
+                                 NULL, NULL, &jokDepth);
+          mdepth = _FILTERS[jokDepth];
         }
 
         if (jok == 1) {         // autorisé
@@ -965,34 +948,10 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz
 
       // filters, 0=sait pas 1=ok -1=interdit
       {
-        int jokDepth1 = 0, jokDepth2 = 0;
-        int jok1 = 0, jok2 = 0;
-        LLint sz1 = size, sz2 = size;
-        int size_flag1 = 0, size_flag2 = 0;
-
-        jok1 =
-          fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
-                      lfull, &sz1, &size_flag1, &jokDepth1);
-        jok2 =
-          fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
-                      l, &sz2, &size_flag2, &jokDepth2);
-        if (jok2 == 0) {        // #2 doesn't know
-          jok = jok1;           // then, use #1
-          sz = sz1;
-          size_flag = size_flag1;
-        } else if (jok1 == 0) { // #1 doesn't know
-          jok = jok2;           // then, use #2
-          sz = sz2;
-          size_flag = size_flag2;
-        } else if (jokDepth1 >= jokDepth2) {    // #1 matching rule is "after" #2, then it is prioritary
-          jok = jok1;
-          sz = sz1;
-          size_flag = size_flag1;
-        } else {                // #2 matching rule is "after" #1, then it is prioritary
-          jok = jok2;
-          sz = sz2;
-          size_flag = size_flag2;
-        }
+        sz = size;
+        jok = fa_strjoker_dual(/*url */ 0, *opt->filters.filters,
+                               *opt->filters.filptr, lfull, l, &sz, &size_flag,
+                               NULL);
       }
 
       // log

tests/01_engine-filterdual.test

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# fa_strjoker_dual: merged verdict on the two URL forms the engine tests;
+# the match latest in the filter list wins, an unknown verdict defers.
+
+fdual() {
+    local want="$1"
+    shift
+    test "$(httrack -O /dev/null -#test=filterdual "$@")" == "$want" || exit 1
+}
+
+fdual 'verdict=unknown rule=0' zzz yyy '+a*'     # neither form matches
+fdual 'verdict=forbidden rule=0' aaa zzz '-a*'   # only form 1 matches
+fdual 'verdict=forbidden rule=0' zzz aaa '-a*'   # only form 2 matches
+fdual 'verdict=forbidden rule=1' a b '+a*' '-b*' # later rule wins (form 2)
+fdual 'verdict=forbidden rule=1' b a '+a*' '-b*' # later rule wins (form 1)
+fdual 'verdict=allowed rule=0' a aa '+a*'        # tie: form 1 wins

tests/01_engine-header.test

@@ -27,3 +27,16 @@ hdr 'contenttype= cdispo=report.pdf' \
 # Path components in the filename are dropped on the wire (RFC 2616).
 hdr 'contenttype= cdispo=evil.pdf' \
     'Content-Disposition: attachment; filename="../../evil.pdf"'
+
+# An over-long header value must not overflow the parse scratch (ASan aborts
+# here on the pre-fix binary).
+test "$(httrack -O /dev/null -#test=headerlong 'Content-Type:')" \
+    == 'contenttype_len=32 contentencoding_len=0'
+test "$(httrack -O /dev/null -#test=headerlong 'Content-Encoding:')" \
+    == 'contenttype_len=0 contentencoding_len=0'
+
+# An empty Content-Encoding yields no token: leave it empty, don't read the
+# uninitialized scratch (MSan aborts here on the pre-fix binary).
+enc() { httrack -O /dev/null -#test=header "$1" | grep '^contentencoding='; }
+test "$(enc 'Content-Encoding:')" == 'contentencoding='
+test "$(enc 'Content-Encoding: gzip')" == 'contentencoding=gzip'

tests/40_local-why.test

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# --why: report which +/- filter rule decides for a URL, without crawling.
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' EXIT
+
+why() {
+    local want="$1" out
+    shift
+    out="$(httrack -O "$tmpdir/p" -q "$@" |
+        grep -E 'accepted|rejected|no filter|unable')"
+    test "$out" == "$want" || {
+        echo "want: $want"
+        echo "got : $out"
+        exit 1
+    }
+}
+
+base=http://www.example.com
+why "$base/x.zip: rejected by rule #2 (-*.zip)" \
+    --why "$base/x.zip" "$base/" '+*.png' '-*.zip'
+why "$base/a.png: accepted by rule #1 (+*.png)" \
+    --why "$base/a.png" "$base/" '+*.png' '-*.zip'
+why "$base/i.html: no filter rule matches; the wizard decides (same-site links are followed by default)" \
+    --why "$base/i.html" "$base/" '+*.png' '-*.zip'
+# scheme-less input gets the same http:// default as primary URLs
+why "$base/x.zip: rejected by rule #2 (-*.zip)" \
+    --why "www.example.com/x.zip" "$base/" '+*.png' '-*.zip'
+# the rule latest in the list decides
+why "$base/x.zip: accepted by rule #3 (+*x.zip)" \
+    --why "$base/x.zip" "$base/" '+*.png' '-*.zip' '+*x.zip'

tests/Makefile.am

@@ -35,6 +35,7 @@ TESTS = \
 	01_engine-entities.test \
 	01_engine-filelist.test \
 	01_engine-filter.test \
+	01_engine-filterdual.test \
 	01_engine-ftp-line.test \
 	01_engine-ftp-userpass.test \
 	01_engine-hashtable.test \
@@ -105,6 +106,7 @@ TESTS = \
 	36_local-bigcrawl.test \
 	37_local-cache-outage.test \
 	38_local-update-304.test \
-	39_local-delayed-cancel.test
+	39_local-delayed-cancel.test \
+	40_local-why.test
 
 CLEANFILES = check-network_sh.cache