Add libFuzzer harnesses for the HTTP header and cache-index parsers

P3-5 fuzz Tier-2. Two more harnesses over hostile-input parsers that read
structured bytes into fixed buffers: fuzz-header drives treatfirstline plus
treathead on each response-header line (the Content-Type/-Encoding path hardened
in #506, and the cookie/Location/Content-Range fields); fuzz-cachendx drives the
hts-cache/*.ndx length-prefixed scan that cache_readex_new loads on --update.

fuzz-cachendx found the over-advance fixed in the parent commit; its seed corpus
carries the two crash reproducers as replay regressions. The cache harness stops
at the scan rather than the trailing coucal insert, whose murmur hash trips a
separate pointer-overflow the .ndx parser does not own.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -278,6 +278,45 @@ jobs:
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
+  # parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
+  # (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
+  # catastrophic backtracking -- and pins the regression seeds for the bugs the
+  # fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
+  fuzz:
+    name: fuzz (libFuzzer corpus replay, clang)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential clang autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - name: Configure (fuzzers, static)
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure CC=clang \
+            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
+            LDFLAGS="-fsanitize=address,undefined" \
+            --enable-fuzzers --disable-shared
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Replay corpora
+        env:
+          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
+          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
+        run: bash fuzz/run-fuzzers.sh fuzz check
+
   # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
   # the configuration users on minimal systems build, and one libssl is not even
   # installed here so configure cannot silently re-enable it. The matrix above

.github/workflows/codeql.yml

@@ -0,0 +1,57 @@
+# CodeQL static analysis (C). The security-extended suite covers the classes
+# this codebase actually fights: overflows, tainted-size allocs, format bugs.
+name: CodeQL
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+  schedule:
+    # Weekly re-scan of master so new/updated queries land without a push.
+    - cron: "17 4 * * 1"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: codeql-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: analyze (c-cpp)
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      # Upload findings to the repo's code-scanning dashboard.
+      security-events: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - uses: github/codeql-action/init@v4
+        with:
+          languages: c-cpp
+          build-mode: manual
+          queries: security-extended
+
+      # Manual build: CodeQL traces the compiler, so build exactly what ships.
+      - name: Build
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure
+          make -j"$(nproc)"
+
+      - uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:c-cpp"

.gitignore

@@ -34,8 +34,9 @@ Makefile
 *.so.*
 *.a
 
-# make dist output.
+# make dist output; dist/ holds httrack-gh-release staging artifacts.
 /httrack-*.tar.gz
+/dist/
 
 # Editor / autotools backup files.
 *~

Makefile.am

@@ -1,5 +1,5 @@
 
-SUBDIRS = src man m4 libtest templates lang html tests
+SUBDIRS = src man m4 libtest templates lang html tests fuzz
 
 ACLOCAL_AMFLAGS = -I m4
 

configure.ac

@@ -102,7 +102,8 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
 AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
 AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
 AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
-AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
+AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
+	[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
 AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
 AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
 AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -110,6 +111,13 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
 AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
 AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
 
+# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
+# sanitizer builds, whose interceptors want the unfortified calls.
+case "$CFLAGS" in
+*-fsanitize=*) ;;
+*) AX_ADD_FORTIFY_SOURCE ;;
+esac
+
 # Force libc back into DT_NEEDED for libraries that reach it only through
 # libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
 # linker; Apple ld rejects these flags and links libSystem unconditionally.
@@ -310,6 +318,37 @@ AC_ARG_ENABLE([online-unit-tests],
 	])
 AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
 
+## libFuzzer harnesses (fuzz/); requires clang
+AC_MSG_CHECKING(whether to build fuzzers)
+AC_ARG_ENABLE([fuzzers],
+	[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
+	[
+	case "${enableval}" in
+	no|yes)
+		fuzzers=$enableval
+		AC_MSG_RESULT($enableval)
+		;;
+	*)
+		AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
+		;;
+	esac
+	],
+	[
+	fuzzers=no
+	AC_MSG_RESULT(no)
+	])
+if test x"$fuzzers" = x"yes"; then
+	# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
+	AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
+		[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
+		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
+	# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
+	if test x"$enable_shared" != x"no"; then
+		AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
+	fi
+fi
+AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
+
 # Final output
 AC_CONFIG_FILES([
 Makefile
@@ -321,5 +360,6 @@ lang/Makefile
 html/Makefile
 libtest/Makefile
 tests/Makefile
+fuzz/Makefile
 ])
 AC_OUTPUT

fuzz/Makefile.am

@@ -0,0 +1,44 @@
+# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
+if FUZZERS
+noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
+	fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx
+endif
+
+AM_CPPFLAGS = \
+	@DEFAULT_CFLAGS@ \
+	@THREADS_CFLAGS@ \
+	@V6_FLAG@ \
+	@LFS_FLAG@ \
+	-I$(top_srcdir)/src \
+	-I$(top_srcdir)/src/coucal
+
+# Static-link libhttrack.la: the internal symbols are hidden in the .so.
+AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
+LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
+
+fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
+fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
+fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
+fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
+fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
+fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
+fuzz_url_SOURCES = fuzz-url.c fuzz.h
+fuzz_header_SOURCES = fuzz-header.c fuzz.h
+fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
+
+# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
+EXTRA_DIST = README.md run-fuzzers.sh \
+	corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
+	corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
+	corpus/idna/idna.txt corpus/idna/unicode.txt \
+	corpus/idna/regress-multilabel-leak.txt \
+	corpus/entities/entities.txt \
+	corpus/unescape/percent.txt \
+	corpus/filters/filter.bin corpus/filters/filter-size.bin \
+	corpus/filters/regress-empty-subject-unique.bin \
+	corpus/url/http-url.txt corpus/url/relative-path.txt \
+	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
+	corpus/header/full-response.txt corpus/header/redirect.txt \
+	corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
+	corpus/cachendx/regress-overadvance.bin \
+	corpus/cachendx/regress-truncated-entry.bin

fuzz/README.md

@@ -0,0 +1,15 @@
+# Fuzzing httrack
+
+libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
+
+```sh
+./bootstrap
+mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
+CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
+  LDFLAGS="-fsanitize=address,undefined" \
+  bash /path/to/httrack/configure --enable-fuzzers --disable-shared
+make
+bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60   # 60s per target
+```
+
+Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.

fuzz/corpus/cachendx/new-format.txt

@@ -0,0 +1,7 @@
+8
+CACHE-1.1
+28
+Mon, 01 Jan 2024 00:00:00 GMT
+www.example.com
+/index.html
+123

fuzz/corpus/cachendx/old-format.txt

@@ -0,0 +1,5 @@
+3
+1.0
+www.example.com
+/page
+5

fuzz/corpus/cachendx/regress-overadvance.bin

@@ -0,0 +1,2 @@
+32768
+CACHE-1.1

fuzz/corpus/cachendx/regress-truncated-entry.bin

@@ -0,0 +1,5 @@
+8
+CACHE-1.1
+1
+x
+www.example.com

fuzz/corpus/charset/latin1.txt

@@ -0,0 +1 @@
+
café naďve ¤

fuzz/corpus/charset/sjis.txt

@@ -0,0 +1 @@
+コンピュータ

fuzz/corpus/entities/entities.txt

@@ -0,0 +1 @@
+&<>A☃é&notarealentity;�

fuzz/corpus/filters/regress-empty-subject-unique.bin

@@ -0,0 +1 @@
+**((

fuzz/corpus/header/full-response.txt

@@ -0,0 +1,10 @@
+HTTP/1.1 200 OK
+Content-Type: text/html; charset=utf-8
+Content-Length: 1234
+Content-Encoding: gzip
+Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
+Etag: "abc"
+Location: http://example.com/x
+Set-Cookie: ID=42; path=/; domain=.example.com
+Content-Range: bytes 0-99/100
+Transfer-Encoding: chunked

fuzz/corpus/header/redirect.txt

@@ -0,0 +1,3 @@
+HTTP/1.0 301 Moved
+Location: /elsewhere
+Content-Disposition: attachment; filename="a.pdf"

fuzz/corpus/idna/idna.txt

@@ -0,0 +1 @@
+xn--bcher-kva.example.com

fuzz/corpus/idna/regress-multilabel-leak.txt

@@ -0,0 +1 @@
+büchev.例å­bücheple

fuzz/corpus/idna/unicode.txt

@@ -0,0 +1 @@
+bücher.例子.example

fuzz/corpus/meta/meta-charset.html

@@ -0,0 +1 @@
+<html><head><meta charset="utf-8"></head><body>x</body></html>

fuzz/corpus/meta/meta-http-equiv.html

@@ -0,0 +1 @@
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">

fuzz/corpus/unescape/percent.txt

@@ -0,0 +1 @@
+%41%zz%%20%c3%a9+%2e%2e%2f

fuzz/corpus/url/http-url.txt

@@ -0,0 +1 @@
+http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag

fuzz/corpus/url/regress-file-empty-path.txt

@@ -0,0 +1 @@
+file://

fuzz/corpus/url/regress-long-path-abort.txt

@@ -0,0 +1 @@
+ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../

fuzz/corpus/url/relative-path.txt

@@ -0,0 +1 @@
+ftp://ftp.example.com/pub/../file.txt

fuzz/fuzz-cachendx.c

@@ -0,0 +1,66 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
+   walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
+   Mirrors the loader in cache_readex_new (htscache.c). */
+#include "fuzz.h"
+#include "htscache.h"
+#include "htslib.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  const char *const end = buf + size;
+  char firstline[256];
+  char *a = buf;
+
+  /* header: two length-prefixed fields (version, last-modified) */
+  a += cache_brstr(a, firstline, sizeof(firstline));
+  a += cache_brstr(a, firstline, sizeof(firstline));
+
+  /* body: newline-delimited host/file/position triples. The coucal insert
+     the real loader ends with is a separate library's concern; this harness
+     targets the length-prefixed scan that must stay inside the buffer. */
+  while (a != NULL && a < end) {
+    char BIGSTK line[HTS_URLMAXSIZE * 2];
+    char linepos[256];
+    int pos;
+
+    a = strchr(a + 1, '\n');
+    if (a == NULL)
+      break;
+    a++;
+    a += cache_binput(a, end, line, HTS_URLMAXSIZE);
+    a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
+    a += cache_binput(a, end, linepos, 200);
+    sscanf(linepos, "%d", &pos);
+    (void) pos;
+  }
+
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-charset.c

@@ -0,0 +1,84 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
+   UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+static const char *const charsets[] = {
+    "utf-8",    "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
+    "us-ascii", "shift_jis",  "euc-jp",     "iso-2022-jp", "gb2312",
+    "big5",     "euc-kr",     "koi8-r",     "utf-16",      "unknown-charset",
+};
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  const char *charset;
+  char *s;
+
+  if (size == 0)
+    return 0;
+  charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+
+  {
+    char *utf8 = hts_convertStringToUTF8(s, size, charset);
+    freet(utf8);
+  }
+  {
+    char *enc = hts_convertStringFromUTF8(s, size, charset);
+    freet(enc);
+  }
+  {
+    size_t nChars = 0;
+    hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
+
+    if (ucs != NULL) {
+      char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
+      freet(back);
+      freet(ucs);
+    }
+  }
+  {
+    size_t i = 0;
+
+    while (i < size) {
+      hts_UCS4 uc = 0;
+      const size_t nr = hts_readUTF8(s + i, size - i, &uc);
+      char out[8];
+
+      if (nr == 0)
+        break;
+      hts_writeUTF8(uc, out, sizeof(out));
+      i += nr;
+    }
+  }
+
+  freet(s);
+  return 0;
+}

fuzz/fuzz-entities.c

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
+   destination size, so truncation bounds get exercised too. */
+#include "fuzz.h"
+#include "htsencoding.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t dsizes[] = {1, 2, 8, 64, 4096};
+  size_t dsize;
+  char *src, *dest;
+
+  if (size == 0)
+    return 0;
+  dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
+  data++, size--;
+  src = fuzz_strdup(data, size);
+  dest = malloct(dsize);
+
+  (void) hts_unescapeEntities(src, dest, dsize);
+  (void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
+
+  freet(dest);
+  freet(src);
+  return 0;
+}

fuzz/fuzz-filters.c

@@ -0,0 +1,65 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
+   here). Input splits on the first NUL: pattern, then subject string. */
+#include "fuzz.h"
+#include "htsfilters.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  const char *joker = buf;
+  const uint8_t *sep = memchr(data, '\0', size);
+  /* subject in its own allocation so ASan bounds it apart from the pattern */
+  char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
+                          : fuzz_strdup(data + size, 0);
+
+  (void) strjoker(nom, joker, NULL, NULL);
+  {
+    LLint sz = (LLint) size;
+    int size_flag = 0;
+
+    (void) strjoker(nom, joker, &sz, &size_flag);
+  }
+  (void) strjokerfind(nom, joker);
+  {
+    char *filter = malloct(strlen(joker) + 2);
+    char *filters[1];
+    LLint sz = (LLint) size;
+    int size_flag = 0, depth = 0;
+
+    filter[0] = '-';
+    memcpy(filter + 1, joker, strlen(joker) + 1);
+    filters[0] = filter;
+    (void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
+    freet(filter);
+  }
+
+  freet(nom);
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-header.c

@@ -0,0 +1,70 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
+   status line, treathead on each following header. Both consume raw bytes
+   off the wire and copy fields into fixed htsblk buffers; treathead also
+   mutates its line in place and drives cookie parsing. */
+#include "fuzz.h"
+#include "htslib.h"
+#include "htsbauth.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  htsblk r;
+  t_cookie *cookie = calloct(1, sizeof(*cookie));
+  char *line = malloct(size + 1);
+  char *p = buf;
+  int first = 1;
+
+  memset(&r, 0, sizeof(r));
+  cookie->max_len = (int) sizeof(cookie->data);
+  r.location = malloct(HTS_URLMAXSIZE * 2);
+  r.location[0] = '\0';
+
+  /* feed one header line at a time, as the receive loop does */
+  while (p != NULL && *p != '\0') {
+    char *nl = strchr(p, '\n');
+    size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
+
+    memcpy(line, p, n);
+    line[n] = '\0';
+    if (first) {
+      treatfirstline(&r, line);
+      first = 0;
+    } else {
+      treathead(cookie, "www.example.com", "/", &r, line);
+    }
+    p = (nl != NULL) ? nl + 1 : NULL;
+  }
+
+  freet(r.location);
+  freet(line);
+  freet(cookie);
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-idna.c

@@ -0,0 +1,47 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+
+  {
+    char *idna = hts_convertStringUTF8ToIDNA(s, size);
+    freet(idna);
+  }
+  {
+    char *utf8 = hts_convertStringIDNAToUTF8(s, size);
+    freet(utf8);
+  }
+  (void) hts_isStringIDNA(s, size);
+
+  freet(s);
+  return 0;
+}

fuzz/fuzz-meta.c

@@ -0,0 +1,40 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
+   <meta> charset declaration. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *html = fuzz_strdup(data, size);
+  char *charset = hts_getCharsetFromMeta(html, size);
+
+  freet(charset);
+  freet(html);
+  return 0;
+}

fuzz/fuzz-unescape.c

@@ -0,0 +1,53 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
+   output buffer size, so the bounded-copy contract is exercised. */
+#include "fuzz.h"
+#include "httrack-library.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t bsizes[] = {1, 2, 16, 256, 8192};
+  size_t bsize;
+  char *s, *catbuff;
+
+  if (size == 0)
+    return 0;
+  bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+  catbuff = malloct(bsize);
+
+  (void) unescape_http(catbuff, bsize, s);
+  (void) unescape_http_unharm(catbuff, bsize, s, 0);
+  (void) unescape_http_unharm(catbuff, bsize, s, 1);
+  unescape_amp(s);
+
+  freet(catbuff);
+  freet(s);
+  return 0;
+}

fuzz/fuzz-url.c

@@ -0,0 +1,49 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
+   is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
+   in place. */
+#include "fuzz.h"
+#include "htscore.h"
+#include "htslib.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+  lien_adrfil *af = calloct(1, sizeof(*af));
+  /* fil_simplifie rewrites in place and may grow an empty path to "./" */
+  char *path = malloct(size + 3);
+
+  (void) ident_url_absolute(s, af);
+  memcpy(path, s, size + 1);
+  fil_simplifie(path);
+
+  freet(path);
+  freet(af);
+  freet(s);
+  return 0;
+}

fuzz/fuzz.h

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Shared helpers for the libFuzzer harnesses. */
+#ifndef FUZZ_H
+#define FUZZ_H
+
+#define HTS_INTERNAL_BYTECODE
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "htsbase.h"
+
+/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
+static char *fuzz_strdup(const uint8_t *data, size_t size) {
+  char *s = malloct(size + 1);
+
+  memcpy(s, data, size);
+  s[size] = '\0';
+  return s;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
+
+#endif

fuzz/run-fuzzers.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Drive every built harness against its seed corpus.
+#   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
+#   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
+# Replay is crash/leak-only and never mutates, so it can't hit strjoker's
+# catastrophic backtracking; the timed mode can, hence the per-unit -timeout.
+set -euo pipefail
+
+srcdir=$(cd "$(dirname "$0")" && pwd)
+bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
+mode=${2:-20}
+
+status=0
+for f in "$bld"/fuzz-*; do
+    if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
+    case "$f" in *.o | *.c | *.dSYM) continue ;; esac
+    name=$(basename "$f")
+    corpus="$srcdir/corpus/${name#fuzz-}"
+    if [ "$mode" = "check" ]; then
+        echo "=== $name (replay) ==="
+        [ -d "$corpus" ] || continue
+        if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
+            echo "*** $name FAILED on its corpus" >&2
+            status=1
+        fi
+        continue
+    fi
+    work=$(mktemp -d)
+    args=("$work")
+    [ -d "$corpus" ] && args+=("$corpus")
+    echo "=== $name (${mode}s) ==="
+    if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
+        -artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
+        echo "*** $name FAILED; artifacts:" >&2
+        ls -l "$work" >&2
+        status=1
+    else
+        rm -rf "$work"
+    fi
+done
+exit $status

m4/ax_add_fortify_source.m4

@@ -0,0 +1,119 @@
+# ===========================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_ADD_FORTIFY_SOURCE
+#
+# DESCRIPTION
+#
+#   Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
+#   redefinition warnings, other cpp warnings or linker. Some distributions
+#   (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
+#   their compilers, leading to unnecessary warnings in the form of
+#
+#     <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
+#     <built-in>: note: this is the location of the previous definition
+#
+#   which is a problem if -Werror is enabled. This macro checks whether
+#   _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
+#   to CPPFLAGS.
+#
+#   Newer mingw-w64 msys2 package comes with a bug in
+#   headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
+#   and would need -lssp or -fstack-protector.  See
+#   https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
+#   link it.
+#
+# LICENSE
+#
+#   Copyright (c) 2017 David Seifert <soap@gentoo.org>
+#   Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#serial 10
+
+AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
+    ac_save_cflags=$CFLAGS
+    ac_cwerror_flag=yes
+    AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
+    ax_add_fortify_3_failed=
+    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
+    AC_LINK_IFELSE([
+        AC_LANG_PROGRAM([],
+            [[
+                #ifndef _FORTIFY_SOURCE
+                    return 0;
+                #else
+                    _FORTIFY_SOURCE_already_defined;
+                #endif
+            ]]
+        )],
+        AC_LINK_IFELSE([
+            AC_LANG_SOURCE([[
+                #define _FORTIFY_SOURCE 3
+                #include <string.h>
+                int main(void) {
+                    char *s = " ";
+                    strcpy(s, "x");
+                    return strlen(s)-1;
+                }
+              ]]
+            )],
+            [
+              AC_MSG_RESULT([yes])
+              CFLAGS=$ac_save_cflags
+              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
+            ], [
+              AC_MSG_RESULT([no])
+              ax_add_fortify_3_failed=1
+            ],
+        ),
+        [
+          AC_MSG_RESULT([no])
+          ax_add_fortify_3_failed=1
+        ])
+    if test -n "$ax_add_fortify_3_failed"
+    then
+    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
+    AC_LINK_IFELSE([
+        AC_LANG_PROGRAM([],
+            [[
+                #ifndef _FORTIFY_SOURCE
+                    return 0;
+                #else
+                    _FORTIFY_SOURCE_already_defined;
+                #endif
+            ]]
+        )],
+        AC_LINK_IFELSE([
+            AC_LANG_SOURCE([[
+                #define _FORTIFY_SOURCE 2
+                #include <string.h>
+                int main(void) {
+                    char *s = " ";
+                    strcpy(s, "x");
+                    return strlen(s)-1;
+                }
+              ]]
+            )],
+            [
+              AC_MSG_RESULT([yes])
+              CFLAGS=$ac_save_cflags
+              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
+            ], [
+              AC_MSG_RESULT([no])
+              CFLAGS=$ac_save_cflags
+            ],
+        ),
+        [
+          AC_MSG_RESULT([no])
+          CFLAGS=$ac_save_cflags
+        ])
+    fi
+])

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "27 June 2026" "httrack website copier"
+.TH httrack 1 "07 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -51,6 +51,7 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
 [ \fB\-%K, \-\-cookies\-file\fR ]
+[ \fB\-%Y, \-\-why\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -218,6 +219,8 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
 .IP \-%K
 load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
+.IP \-%Y
+explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j

src/htsalias.c

@@ -114,6 +114,8 @@ const char *hts_optalias[][4] = {
    "strip [host/pattern=]key1,key2,... from URLs"},
   {"cookies-file", "-%K", "param1",
    "load extra cookies from a Netscape cookies.txt"},
+  {"why", "-%Y", "param1",
+   "explain which filter rule accepts or rejects a URL, then exit"},
   {"pause", "-%G", "param1",
    "random pause of MIN[:MAX] seconds between files"},
   {"generate-errors", "-o", "single", ""},

src/htscache.c

@@ -1938,15 +1938,17 @@ void cache_init(cache_back * cache, httrackp * opt) {
             char linepos[256];
             int pos;
 
-            while((a != NULL) && (a < (cache->use + buffl))) {
+            const char *const end = cache->use + buffl;
+
+            while ((a != NULL) && (a < end)) {
               a = strchr(a + 1, '\n');  /* start of line */
               if (a) {
                 a++;
                 /* read "host/file" */
-                a += binput(a, line, HTS_URLMAXSIZE);
-                a += binput(a, line + strlen(line), HTS_URLMAXSIZE);
+                a += cache_binput(a, end, line, HTS_URLMAXSIZE);
+                a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
                 /* read position */
-                a += binput(a, linepos, 200);
+                a += cache_binput(a, end, linepos, 200);
                 sscanf(linepos, "%d", &pos);
                 coucal_add(cache->hashtable, line, pos);
               }
@@ -2291,23 +2293,40 @@ int cache_brstr(char *adr, char *s, size_t s_size) {
   char buff[256 + 4];
 
   off = binput(adr, buff, 256);
-  adr += off;
-  sscanf(buff, "%d", &i);
-  if (i < 0 || i > 32768) /* guard a corrupt length */
+  /* binput stops at the buffer's terminating NUL; a value can only follow a
+     real line terminator, so never step past end-of-buffer. */
+  if (adr[off - 1] == '\0') {
+    s[0] = '\0';
+    return off - 1;
+  }
+  /* an empty/non-numeric field leaves i unset: treat as length 0 */
+  if (sscanf(buff, "%d", &i) != 1 || i < 0 || i > 32768)
     i = 0;
   if (i > 0) {
-    /* copy at most s_size-1 bytes; advance past the full field regardless */
-    const size_t store = (size_t) i < s_size ? (size_t) i : s_size - 1;
+    /* A corrupt/truncated cache may declare a length past the buffer end;
+       bound both the copy and the advance to the bytes actually present. */
+    const size_t avail = strnlen(adr + off, (size_t) i);
+    const size_t store = avail < s_size ? avail : s_size - 1;
 
-    strncpy(s, adr, store);
+    memcpy(s, adr + off, store);
     s[store] = '\0';
+    off += (int) avail;
   } else {
     s[0] = '\0';
   }
-  off += i;
   return off;
 }
 
+/* binput bounded to a NUL-terminated buffer: refuse to start a read at or
+   past `end`, so a prior over-advance can't walk a cache-index parse OOB. */
+int cache_binput(char *adr, const char *end, char *s, int max) {
+  if (adr >= end) {
+    s[0] = '\0';
+    return 0;
+  }
+  return binput(adr, s, max);
+}
+
 /* idem, mais en int */
 int cache_brint(char *adr, int *i) {
   char s[256];

src/htscache.h

@@ -97,6 +97,8 @@ int cache_readdata(cache_back * cache, const char *str1, const char *str2,
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);
+/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
+int cache_binput(char *adr, const char *end, char *s, int max);
 int cache_brint(char *adr, int *i);
 void cache_rint(FILE * fp, int *i);
 void cache_rLLint(FILE * fp, LLint * i);

src/htscharset.c

@@ -910,19 +910,19 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
                 /* Reader: can read bytes up to j */
 #define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
 
-                /* Writer: upon error, return FFFD (replacement character) */
-#define WR(C) do { \
-  if ((C) != -1) { \
-    /* copy character */ \
-    assertf(segOutputSize < segSize); \
-    segInt[segOutputSize++] = (C); \
-  } \
-  /* In case of error, abort. */ \
-  else { \
-    FREE_BUFFER(); \
-    return NULL; \
-  } \
-} while(0)
+                /* Writer: a malformed sequence abandons the encode (NULL) */
+#define WR(C)                                                                  \
+  do {                                                                         \
+    if ((C) != -1) {                                                           \
+      /* copy character */                                                     \
+      assertf(segOutputSize < segSize);                                        \
+      segInt[segOutputSize++] = (C);                                           \
+    } else {                                                                   \
+      freet(segInt);                                                           \
+      FREE_BUFFER();                                                           \
+      return NULL;                                                             \
+    }                                                                          \
+  } while (0)
 
                 /* Read/Write Unicode character. */
                 READ_UNICODE(RD, WR);
@@ -1144,7 +1144,7 @@ int hts_isStringUTF8(const char *s, size_t size) {
     /* Reader: can read bytes up to j */
 #define RD ( i < size ? data[i++] : -1 )
 
-    /* Writer: upon error, return FFFD (replacement character) */
+    /* Writer: a malformed sequence means the string is not UTF-8 (return 0) */
 #define WR(C) if ((C) == -1) { return 0; }
 
     /* Read Unicode character. */

src/htscore.c

@@ -846,6 +846,52 @@ int httpmirror(char *url1, httrackp * opt) {
       }
     }                           // while
 
+    /* --why: print which filter rule decides for this URL, then stop */
+    if (StringNotEmpty(opt->why_url)) {
+      char BIGSTK url[HTS_URLMAXSIZE * 2];
+      lien_adrfil af;
+
+      if (strstr(StringBuff(opt->why_url), ":/") == NULL)
+        snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
+      else
+        strcpybuff(url, StringBuff(opt->why_url));
+      if (ident_url_absolute(url, &af) < 0) {
+        printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
+      } else {
+        char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
+        int jok, jokDepth = 0;
+
+        /* the two forms the wizard tests */
+        strcpybuff(l, jump_identification_const(af.adr));
+        if (*af.fil != '/')
+          strcatbuff(l, "/");
+        strcatbuff(l, af.fil);
+        if (!link_has_authority(af.adr))
+          strcpybuff(lfull, "http://");
+        else
+          lfull[0] = '\0';
+        strcatbuff(lfull, af.adr);
+        if (*af.fil != '/')
+          strcatbuff(lfull, "/");
+        strcatbuff(lfull, af.fil);
+        jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
+                               NULL, &jokDepth);
+        if (jok > 0)
+          printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
+                 filters[jokDepth]);
+        else if (jok < 0)
+          printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
+                 filters[jokDepth]);
+        else
+          printf("%s: no filter rule matches; the wizard decides "
+                 "(same-site links are followed by default)" LF,
+                 url);
+      }
+      freet(primary);
+      XH_extuninit;
+      return 1;
+    }
+
     /* load URL file list */
     /* OPTIMIZED for fast load */
     if (StringNotEmpty(opt->filelist)) {

src/htscoremain.c

@@ -1803,6 +1803,23 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   StringCopy(opt->cookies_file, argv[na]);
                 }
                 break;
+              case 'Y': // why: explain the filter verdict for a URL, no crawl
+                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
+                  HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
+                  printf(
+                      "Example: --why \"http://www.example.com/file.zip\"\n");
+                  htsmain_free();
+                  return -1;
+                } else {
+                  na++;
+                  if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
+                    HTS_PANIC_PRINTF("why URL too long");
+                    htsmain_free();
+                    return -1;
+                  }
+                  StringCopy(opt->why_url, argv[na]);
+                }
+                break;
               case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF("Option pause needs a blank space and a "
@@ -1936,18 +1953,20 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     char BIGSTK url[HTS_URLMAXSIZE * 2];
                     char linepos[256];
                     int pos;
-                    char *cacheNdx =
-                      readfile(fconcat
-                               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                                "hts-cache/new.ndx"));
+                    LLint cacheNdxLen = 0;
+                    char *cacheNdx = readfile2(
+                        fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                StringBuff(opt->path_log), "hts-cache/new.ndx"),
+                        &cacheNdxLen);
                     cache_init(&cache, opt);    /* load cache */
                     if (cacheNdx != NULL) {
                       char firstline[256];
                       char *a = cacheNdx;
+                      const char *const end = cacheNdx + cacheNdxLen;
 
                       a += cache_brstr(a, firstline, sizeof(firstline));
                       a += cache_brstr(a, firstline, sizeof(firstline));
-                      while(a != NULL) {
+                      while (a != NULL && a < end) {
                         a = strchr(a + 1, '\n');        /* start of line */
                         if (a) {
                           htsblk r;
@@ -1955,15 +1974,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                           /* */
                           a++;
                           /* read "host/file" */
-                          a += binput(a, afs.af.adr, HTS_URLMAXSIZE);
-                          a += binput(a, afs.af.fil, HTS_URLMAXSIZE);
+                          a += cache_binput(a, end, afs.af.adr, HTS_URLMAXSIZE);
+                          a += cache_binput(a, end, afs.af.fil, HTS_URLMAXSIZE);
                           url[0] = '\0';
                           if (!link_has_authority(afs.af.adr))
                             strcatbuff(url, "http://");
                           strcatbuff(url, afs.af.adr);
                           strcatbuff(url, afs.af.fil);
                           /* read position */
-                          a += binput(a, linepos, 200);
+                          a += cache_binput(a, end, linepos, 200);
                           sscanf(linepos, "%d", &pos);
                           if (!hasFilter
                               || (strjoker(url, filter, NULL, NULL) != NULL)

src/htsfilters.c

@@ -94,6 +94,32 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
   return verdict;
 }
 
+int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
+                     const char *nom2, LLint *size, int *size_flag,
+                     int *depth) {
+  int depth1 = 0, depth2 = 0;
+  int flag1 = 0, flag2 = 0;
+  LLint sz1 = 0, sz2 = 0;
+  int jok1, jok2, use1;
+
+  if (size) {
+    sz1 = *size;
+    sz2 = *size;
+  }
+  jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
+                     size_flag ? &flag1 : NULL, &depth1);
+  jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
+                     size_flag ? &flag2 : NULL, &depth2);
+  use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
+  if (size)
+    *size = use1 ? sz1 : sz2;
+  if (size_flag)
+    *size_flag = use1 ? flag1 : flag2;
+  if (depth)
+    *depth = use1 ? depth1 : depth2;
+  return use1 ? jok1 : jok2;
+}
+
 // supercomparateur joker (tm)
 // compare a et b (b=avec joker dedans), case insensitive [voir CI]
 // renvoi l'adresse de la première lettre de la chaine
@@ -286,7 +312,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
       if (!unique)
         max = (int) strlen(chaine);
       else                      /* *(a) only match a (not aaaaa) */
-        max = 1;
+        max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
           if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {

src/htsfilters.h

@@ -41,6 +41,11 @@ Please visit our Website: http://www.httrack.com
 
 int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
                 int *size_flag, int *depth);
+/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
+   scheme); the match latest in the list wins, a "don't know" verdict defers.
+   Returns the merged verdict; the out-params carry the winner's values. */
+int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
+                     const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);

src/htshelp.c

@@ -543,6 +543,7 @@ void help(const char *app, int more) {
   infomsg("Spider options:");
   infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
   infomsg(" %K  load extra cookies from a Netscape cookies.txt");
+  infomsg(" %Y  explain which filter rule accepts or rejects a URL, then exit");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
   infomsg

src/htslib.c

@@ -1562,7 +1562,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       }
       // An empty/whitespace Content-Type value yields no token: keep the
       // sentinel default rather than reading an uninitialized tempo.
-      if (sscanf(rcvd + p, "%s", tempo) == 1) {
+      if (sscanf(rcvd + p, "%1099s", tempo) == 1) { // tempo[1100], server data
         if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
           strcpybuff(retour->contenttype, tempo);
         else
@@ -1657,11 +1657,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         if (a)
           *a = '\0';
       }
-      sscanf(a, "%s", tempo);
-      if (strlen(tempo) < 64)   // pas trop long!!
+      // Bound to tempo[1100]; no token => leave empty, don't read uninit tempo.
+      if (sscanf(a, "%1099s", tempo) == 1 && strlen(tempo) < 64)
         strcpybuff(retour->contentencoding, tempo);
       else
-        retour->contentencoding[0] = '\0';      // erreur
+        retour->contentencoding[0] = '\0';
 #if HTS_USEZLIB
       /* Check known encodings */
       if (retour->contentencoding[0]) {
@@ -2497,6 +2497,10 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
   // effacer adrfil->adr et adrfil->fil
   adrfil->adr[0] = adrfil->fil[0] = '\0';
 
+  // Reject an over-long URL: a root-relative path is copied whole into fil[].
+  if (strlen(url) >= HTS_URLMAXSIZE * 2)
+    return -1;
+
 #if HDEBUG
   printf("protocol: %s\n", url);
 #endif
@@ -2572,7 +2576,7 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
     if (*p == '/' || *p == '\\') {      /* adrfil->file:///.. */
       strcatbuff(adrfil->fil, p);       // fichier local ; adrfil->adr="#"
     } else {
-      if (p[1] != ':') {
+      if (*p == '\0' || p[1] != ':') {  /* empty path: not a DOS drive letter */
         strcatbuff(adrfil->fil, "//");  /* adrfil->file://server/foo */
         strcatbuff(adrfil->fil, p);
       } else {
@@ -6002,6 +6006,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
   StringCopy(opt->strip_query, "");
   StringCopy(opt->cookies_file, "");
+  StringCopy(opt->why_url, "");
   opt->pause_min_ms = 0;
   opt->pause_max_ms = 0;
   opt->ftp_proxy = HTS_TRUE;
@@ -6149,6 +6154,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
     StringFree(opt->mod_blacklist);
     StringFree(opt->strip_query);
     StringFree(opt->cookies_file);
+    StringFree(opt->why_url);
 
     StringFree(opt->path_html);
     StringFree(opt->path_html_utf8);

src/htsopt.h

@@ -536,6 +536,8 @@ struct httrackp {
                                  (--cookies-file) */
   int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
   int pause_max_ms; /**< inter-file pause upper bound, ms */
+  String why_url;   /**< URL to diagnose (--why): print the deciding filter rule
+                         and exit without crawling */
 };
 
 /* Running statistics for a mirror. */

src/htsselftest.c

@@ -46,6 +46,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsdefines.h"
 #include "htslib.h"
 #include "htsparse.h"
+#include "htscache.h"
 #include "htscache_selftest.h"
 #include "htsdns_selftest.h"
 #include "htscharset.h"
@@ -579,15 +580,16 @@ static int st_filter(httrackp *opt, int argc, char **argv) {
   int matched;
 
   (void) opt;
-  if (argc < 2) {
+  if (argc < 1) {
     fprintf(stderr, "filter: needs a filter pattern and a string\n");
     return 1;
   }
-  /* exact-size heap copies so a sanitizer traps any over-read of the pattern */
-  str = strdupt(argv[1]);
+  /* exact-size heap copies so a sanitizer traps an over-read; a missing
+     subject means "" (not reachable as a CLI arg) */
+  str = strdupt(argc >= 2 ? argv[1] : "");
   pat = strdupt(argv[0]);
   matched = strjoker(str, pat, NULL, NULL) != NULL;
-  printf("%s does %s %s\n", argv[1], matched ? "match" : "NOT match", argv[0]);
+  printf("%s does %s %s\n", str, matched ? "match" : "NOT match", argv[0]);
   freet(str);
   freet(pat);
   return 0;
@@ -619,6 +621,26 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
+static int st_filterdual(httrackp *opt, int argc, char **argv) {
+  int depth = -1, verdict;
+
+  (void) opt;
+  if (argc < 3) {
+    fprintf(stderr,
+            "filterdual: needs <string1> <string2> <filter> [filter...]\n");
+    return 1;
+  }
+  verdict = fa_strjoker_dual(0, &argv[2], argc - 2, argv[0], argv[1], NULL,
+                             NULL, &depth);
+  printf("verdict=%s rule=%d\n",
+         verdict > 0   ? "allowed"
+         : verdict < 0 ? "forbidden"
+                       : "unknown",
+         depth);
+  return 0;
+}
+
 static int st_simplify(httrackp *opt, int argc, char **argv) {
   (void) opt;
   if (argc < 1) {
@@ -1094,6 +1116,33 @@ static int st_resolve(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Split a URL into (adr, fil), or print "error" if rejected. A second arg pads
+   the URL with that many 'a's to reach lengths a CLI arg can't. */
+static int st_identurl(httrackp *opt, int argc, char **argv) {
+  lien_adrfil af;
+  char *url;
+  size_t len, pad = 0;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "identurl: needs a URL\n");
+    return 1;
+  }
+  if (argc >= 2)
+    pad = (size_t) atoi(argv[1]);
+  len = strlen(argv[0]);
+  url = malloct(len + pad + 1);
+  memcpy(url, argv[0], len);
+  memset(url + len, 'a', pad);
+  url[len + pad] = '\0';
+  if (ident_url_absolute(url, &af) >= 0)
+    printf("adr=%s fil=%s\n", af.adr, af.fil);
+  else
+    printf("error\n");
+  freet(url);
+  return 0;
+}
+
 /* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
    no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
    registering an already-crawled link (dedup/collision paths). */
@@ -1115,6 +1164,26 @@ static int st_header(httrackp *opt, int argc, char **argv) {
     treathead(NULL, "www.example.com", "/", &r, line);
   }
   printf("contenttype=%s cdispo=%s\n", r.contenttype, r.cdispo);
+  printf("contentencoding=%s\n", r.contentencoding);
+  return 0;
+}
+
+/* An over-long header value must not overflow treathead's tempo[1100]. */
+static int st_headerlong(httrackp *opt, int argc, char **argv) {
+  htsblk r;
+  char BIGSTK line[HTS_URLMAXSIZE * 2];
+  const char *const name = argc >= 1 ? argv[0] : "Content-Type:";
+  const int pad = 1500; /* > tempo[1100] */
+  size_t n;
+
+  (void) opt;
+  memset(&r, 0, sizeof(r));
+  n = (size_t) snprintf(line, sizeof(line), "%s ", name);
+  memset(line + n, 'a', pad);
+  line[n + pad] = '\0';
+  treathead(NULL, "www.example.com", "/", &r, line);
+  printf("contenttype_len=%d contentencoding_len=%d\n",
+         (int) strlen(r.contenttype), (int) strlen(r.contentencoding));
   return 0;
 }
 
@@ -1322,6 +1391,78 @@ static int st_cache(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* A corrupt cache index (.ndx) must not walk the length-prefixed scan past
+   the buffer. Checks the two primitives the loader is built from. */
+static int st_cacheindex(httrackp *opt, int argc, char **argv) {
+  int fail = 0;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+
+  /* A length prefix that overstates the bytes present must bound the advance
+     to the buffer, not trust the declared length. */
+  {
+    static const char src[] = "32768\nCACHE-1.1";
+    const size_t len = sizeof(src) - 1;
+    char *buf = malloct(len + 1);
+    char s[256];
+    int off;
+
+    memcpy(buf, src, len + 1);
+    off = cache_brstr(buf, s, sizeof(s));
+    if (off > (int) len) {
+      printf("cacheindex: over-advance off=%d len=%d\n", off, (int) len);
+      fail = 1;
+    }
+    if (strcmp(s, "CACHE-1.1") != 0) {
+      printf("cacheindex: value=%s\n", s);
+      fail = 1;
+    }
+    freet(buf);
+  }
+
+  /* cache_binput reads a field while in bounds, but refuses one starting at
+     or past end-of-buffer. */
+  {
+    char buf[8] = "ab\ncd";
+    const char *const end = buf + 5;
+    char s[16];
+
+    if (cache_binput(buf, end, s, sizeof(s)) != 3 || strcmp(s, "ab") != 0)
+      fail = 1; /* normal read: "ab" then the '\n', 3 bytes consumed */
+    if (cache_binput(end, end, s, sizeof(s)) != 0 || s[0] != '\0')
+      fail = 1;
+  }
+
+  /* Drive the full loader scan over a truncated index: a declared length that
+     overshoots plus a half-written entry. ASan aborts here on the pre-fix
+     scan; the cursor must never leave the buffer. */
+  {
+    static const char src[] = "9\nCACHE-1.1\n99\nwww.example.com\n/a";
+    const size_t len = sizeof(src) - 1;
+    char *buf = malloct(len + 1);
+    const char *const end = buf + len;
+    char line[256];
+    char *a = buf;
+
+    memcpy(buf, src, len + 1);
+    a += cache_brstr(a, line, sizeof(line));
+    a += cache_brstr(a, line, sizeof(line));
+    while (a != NULL && a < end) {
+      a = strchr(a + 1, '\n');
+      if (a == NULL)
+        break;
+      a++;
+      a += cache_binput(a, end, line, sizeof(line));
+    }
+    freet(buf);
+  }
+
+  printf("cacheindex: %s\n", fail ? "FAIL" : "OK");
+  return fail;
+}
+
 static int st_cache_golden(httrackp *opt, int argc, char **argv) {
   int regen, err;
 
@@ -2106,6 +2247,8 @@ static const struct selftest_entry {
     {"filtersize", "<size> <string> <filter>...",
      "size-aware filter verdict (negative size = unknown/scan time)",
      st_filtersize},
+    {"filterdual", "<string1> <string2> <filter>...",
+     "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
@@ -2132,13 +2275,19 @@ static const struct selftest_entry {
      st_relative},
     {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
      st_resolve},
+    {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
+    {"headerlong", "[header-name:]",
+     "over-long header value must not overflow the parse scratch",
+     st_headerlong},
     {"savename", "<fil> <content-type> [key=value ...]",
      "local save-name for a URL", st_savename},
     {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
      st_sniff},
     {"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
+    {"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
+     st_cacheindex},
     {"cache-golden", "<dir> [regen]", "frozen cache-format read self-test",
      st_cache_golden},
     {"cache-writefail", "<dir>", "cache write-failure handling self-test",

src/htswizard.c

@@ -524,28 +524,11 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
 
         // filters, 0=sait pas 1=ok -1=interdit
         {
-          int jokDepth1 = 0, jokDepth2 = 0;
-          int jok1 = 0, jok2 = 0;
+          int jokDepth = 0;
 
-          jok1 =
-            fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, NULL, NULL,
-                        &jokDepth1);
-          jok2 =
-            fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, l, NULL, NULL,
-                        &jokDepth2);
-          if (jok2 == 0) {      // #2 doesn't know
-            jok = jok1;         // then, use #1
-            mdepth = _FILTERS[jokDepth1];
-          } else if (jok1 == 0) {       // #1 doesn't know
-            jok = jok2;         // then, use #2
-            mdepth = _FILTERS[jokDepth2];
-          } else if (jokDepth1 >= jokDepth2) {  // #1 matching rule is "after" #2, then it is prioritary
-            jok = jok1;
-            mdepth = _FILTERS[jokDepth1];
-          } else {              // #2 matching rule is "after" #1, then it is prioritary
-            jok = jok2;
-            mdepth = _FILTERS[jokDepth2];
-          }
+          jok = fa_strjoker_dual(/*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, l,
+                                 NULL, NULL, &jokDepth);
+          mdepth = _FILTERS[jokDepth];
         }
 
         if (jok == 1) {         // autorisé
@@ -965,34 +948,10 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz
 
       // filters, 0=sait pas 1=ok -1=interdit
       {
-        int jokDepth1 = 0, jokDepth2 = 0;
-        int jok1 = 0, jok2 = 0;
-        LLint sz1 = size, sz2 = size;
-        int size_flag1 = 0, size_flag2 = 0;
-
-        jok1 =
-          fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
-                      lfull, &sz1, &size_flag1, &jokDepth1);
-        jok2 =
-          fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
-                      l, &sz2, &size_flag2, &jokDepth2);
-        if (jok2 == 0) {        // #2 doesn't know
-          jok = jok1;           // then, use #1
-          sz = sz1;
-          size_flag = size_flag1;
-        } else if (jok1 == 0) { // #1 doesn't know
-          jok = jok2;           // then, use #2
-          sz = sz2;
-          size_flag = size_flag2;
-        } else if (jokDepth1 >= jokDepth2) {    // #1 matching rule is "after" #2, then it is prioritary
-          jok = jok1;
-          sz = sz1;
-          size_flag = size_flag1;
-        } else {                // #2 matching rule is "after" #1, then it is prioritary
-          jok = jok2;
-          sz = sz2;
-          size_flag = size_flag2;
-        }
+        sz = size;
+        jok = fa_strjoker_dual(/*url */ 0, *opt->filters.filters,
+                               *opt->filters.filptr, lfull, l, &sz, &size_flag,
+                               NULL);
       }
 
       // log

tests/01_engine-cacheindex.test

@@ -0,0 +1,10 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Cache-index (.ndx) parse must stay inside the buffer on a corrupt length
+# prefix (ASan aborts here on the pre-fix binary; the OK check gates the
+# non-sanitized build too).
+
+test "$(httrack -O /dev/null -#test=cacheindex)" == 'cacheindex: OK'

tests/01_engine-filter.test

@@ -12,6 +12,11 @@ match() {
 nomatch() {
     test "$(httrack -O /dev/null -#test=filter "$1" "$2")" == "$2 does NOT match $1" || exit 1
 }
+# single-arg call means an empty subject (unreachable as a CLI arg); '*(' used
+# to force max=1 and over-read past it.
+nomatch_empty() {
+    test "$(httrack -O /dev/null -#test=filter "$1")" == " does NOT match $1" || exit 1
+}
 
 # bare star matches everything
 match '*' 'anything/at/all'
@@ -122,3 +127,7 @@ nomatch '*[file]end' 'foo?xend'
 nomatch '*[name]X' 'abc?X'
 match '*[file]' 'foo?x=1' # trailing query: tolerated, as for *.aspx
 match '*.aspx' 'page.aspx?y=2'
+
+# empty subject against a unique-match allow-all pattern (heap over-read regress)
+nomatch_empty '*(('
+nomatch_empty '**(('

tests/01_engine-filterdual.test

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# fa_strjoker_dual: merged verdict on the two URL forms the engine tests;
+# the match latest in the filter list wins, an unknown verdict defers.
+
+fdual() {
+    local want="$1"
+    shift
+    test "$(httrack -O /dev/null -#test=filterdual "$@")" == "$want" || exit 1
+}
+
+fdual 'verdict=unknown rule=0' zzz yyy '+a*'     # neither form matches
+fdual 'verdict=forbidden rule=0' aaa zzz '-a*'   # only form 1 matches
+fdual 'verdict=forbidden rule=0' zzz aaa '-a*'   # only form 2 matches
+fdual 'verdict=forbidden rule=1' a b '+a*' '-b*' # later rule wins (form 2)
+fdual 'verdict=forbidden rule=1' b a '+a*' '-b*' # later rule wins (form 1)
+fdual 'verdict=allowed rule=0' a aa '+a*'        # tie: form 1 wins

tests/01_engine-header.test

@@ -27,3 +27,16 @@ hdr 'contenttype= cdispo=report.pdf' \
 # Path components in the filename are dropped on the wire (RFC 2616).
 hdr 'contenttype= cdispo=evil.pdf' \
     'Content-Disposition: attachment; filename="../../evil.pdf"'
+
+# An over-long header value must not overflow the parse scratch (ASan aborts
+# here on the pre-fix binary).
+test "$(httrack -O /dev/null -#test=headerlong 'Content-Type:')" \
+    == 'contenttype_len=32 contentencoding_len=0'
+test "$(httrack -O /dev/null -#test=headerlong 'Content-Encoding:')" \
+    == 'contenttype_len=0 contentencoding_len=0'
+
+# An empty Content-Encoding yields no token: leave it empty, don't read the
+# uninitialized scratch (MSan aborts here on the pre-fix binary).
+enc() { httrack -O /dev/null -#test=header "$1" | grep '^contentencoding='; }
+test "$(enc 'Content-Encoding:')" == 'contentencoding='
+test "$(enc 'Content-Encoding: gzip')" == 'contentencoding=gzip'

tests/01_engine-identurl.test

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# ident_url_absolute splits a raw URL into (adr, fil), the first parser to touch
+# it. -#test=identurl <url> [pad] prints "adr=.. fil=.." or "error"; pad appends
+# N 'a's, reaching lengths a CLI arg can't.
+
+split() {
+    test "$(httrack -O /dev/null -#test=identurl "$1")" == "$2" || exit 1
+}
+
+split 'http://www.example.com/a/b/../c.html' 'adr=www.example.com fil=/a/c.html'
+split 'ftp://ftp.example.com/pub/file.txt' 'adr=ftp://ftp.example.com fil=/pub/file.txt'
+split 'file:///etc/hostname' 'adr=file:// fil=/etc/hostname'
+
+# empty-path file:// used to read one byte past the URL (p[1] with *p=0)
+split 'file://' 'adr=file:// fil=//'
+
+# a root-relative path is copied whole into fil[HTS_URLMAXSIZE*2], so a URL at
+# the buffer size is the tightest overflow; it must be rejected, not abort.
+split_pad() {
+    test "$(httrack -O /dev/null -#test=identurl "$1" "$2")" == "$3" || exit 1
+}
+split_pad '/' 2047 'error' # 1 + 2047 = 2048 = HTS_URLMAXSIZE*2
+split_pad 'http://h/' 3000 'error'

tests/01_engine-idna.test

@@ -8,8 +8,8 @@ set -euo pipefail
 
 enc() { test "$(httrack -O /dev/null -#test=idna-encode "$1")" == "$2" || exit 1; }
 dec() { test "$(httrack -O /dev/null -#test=idna-decode "$1")" == "$2" || exit 1; }
-# crash probe: malformed ACE input must exit cleanly, not abort.
-runs() { httrack -O /dev/null -#test=idna-decode "$1" >/dev/null 2>&1 || exit 1; }
+# a malformed encode input must be rejected (empty output), not encoded or leaked.
+enc_bad() { test -z "$(httrack -O /dev/null -#test=idna-encode "$1" 2>/dev/null)" || exit 1; }
 
 # encode
 enc 'www.café.com' 'www.xn--caf-dma.com'
@@ -33,6 +33,9 @@ enc 'xn--already-encoded.com' 'xn--already-encoded.com'
 # an empty punycode payload decodes back to the bare xn-- label
 dec 'xn--' 'xn--'
 
-# malformed ACE payloads (invalid base-36, garbage) must not crash
-runs 'xn--!!!'
-runs 'xn--already-encoded.com'
+# malformed ACE payloads decode to a stable (garbage) string, not a crash
+dec 'xn--!!!' '㚯'
+dec 'xn--already-encoded.com' 'ǮalǭǮreaǭǫdy.com'
+
+# second label has a truncated UTF-8 sequence: used to leak the segment buffer.
+enc_bad "$(printf 'b\xc3\xbcchev.\xe4\xbe\x8b\xe5\xad\x62\xc3\xbccheple')"

tests/40_local-why.test

@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# --why: report which +/- filter rule decides for a URL, without crawling.
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' EXIT
+
+why() {
+    local want="$1" out
+    shift
+    out="$(httrack -O "$tmpdir/p" -q "$@" |
+        grep -E 'accepted|rejected|no filter|unable')"
+    test "$out" == "$want" || {
+        echo "want: $want"
+        echo "got : $out"
+        exit 1
+    }
+}
+
+base=http://www.example.com
+why "$base/x.zip: rejected by rule #2 (-*.zip)" \
+    --why "$base/x.zip" "$base/" '+*.png' '-*.zip'
+why "$base/a.png: accepted by rule #1 (+*.png)" \
+    --why "$base/a.png" "$base/" '+*.png' '-*.zip'
+why "$base/i.html: no filter rule matches; the wizard decides (same-site links are followed by default)" \
+    --why "$base/i.html" "$base/" '+*.png' '-*.zip'
+# scheme-less input gets the same http:// default as primary URLs
+why "$base/x.zip: rejected by rule #2 (-*.zip)" \
+    --why "www.example.com/x.zip" "$base/" '+*.png' '-*.zip'
+# the rule latest in the list decides
+why "$base/x.zip: accepted by rule #3 (+*x.zip)" \
+    --why "$base/x.zip" "$base/" '+*.png' '-*.zip' '+*x.zip'

tests/Makefile.am

@@ -35,11 +35,14 @@ TESTS = \
 	01_engine-entities.test \
 	01_engine-filelist.test \
 	01_engine-filter.test \
+	01_engine-filterdual.test \
 	01_engine-ftp-line.test \
 	01_engine-ftp-userpass.test \
+	01_engine-cacheindex.test \
 	01_engine-hashtable.test \
 	01_engine-header.test \
 	01_engine-idna.test \
+	01_engine-identurl.test \
 	01_engine-escape-room.test \
 	01_engine-inplace-escape.test \
 	01_engine-java.test \
@@ -104,6 +107,7 @@ TESTS = \
 	36_local-bigcrawl.test \
 	37_local-cache-outage.test \
 	38_local-update-304.test \
-	39_local-delayed-cancel.test
+	39_local-delayed-cancel.test \
+	40_local-why.test
 
 CLEANFILES = check-network_sh.cache