ci: alias 127.0.0.2/.3 on macOS lo0 for the connect-fallback test

macOS configures only 127.0.0.1 as loopback, so 19_local-connect-fallback's
dead 127.0.0.2/.3 addresses have no fast-refuse path: the connect intermittently
stalls to the timeout and logs "Connect Time Out", which the zero-errors
assertion rejects (flaked ~1/6 on the macOS runner). #531's --timeout/--retries
bound the stall but not the spurious error. Aliasing the addresses onto lo0
makes them refuse instantly like Linux, so the fallback runs cleanly.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -55,7 +55,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -99,7 +101,8 @@ jobs:
           sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
             -exec mv {} {}.hidden \;
           ! command -v python3
-          make check
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -132,8 +135,18 @@ jobs:
       - name: Build
         run: make -j"$(sysctl -n hw.ncpu)"
 
+      - name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
+        # 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
+        # instantly like Linux; alias them onto lo0 so they don't stall to timeout.
+        run: |
+          set -euo pipefail
+          sudo ifconfig lo0 alias 127.0.0.2 up
+          sudo ifconfig lo0 alias 127.0.0.3 up
+
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -171,7 +184,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -226,7 +241,9 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -272,12 +289,52 @@ jobs:
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
           # skipped here (uninstrumented libz floods MSan with false positives).
           tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
-          make check TESTS="$tests"
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs" TESTS="$tests"
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
+  # parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
+  # (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
+  # catastrophic backtracking -- and pins the regression seeds for the bugs the
+  # fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
+  fuzz:
+    name: fuzz (libFuzzer corpus replay, clang)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential clang autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - name: Configure (fuzzers, static)
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure CC=clang \
+            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
+            LDFLAGS="-fsanitize=address,undefined" \
+            --enable-fuzzers --disable-shared
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Replay corpora
+        env:
+          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
+          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
+        run: bash fuzz/run-fuzzers.sh fuzz check
+
   # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
   # the configuration users on minimal systems build, and one libssl is not even
   # installed here so configure cannot silently re-enable it. The matrix above
@@ -308,7 +365,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()

.github/workflows/codeql.yml

@@ -0,0 +1,63 @@
+# CodeQL static analysis (C). The security-extended suite covers the classes
+# this codebase actually fights: overflows, tainted-size allocs, format bugs.
+name: CodeQL
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+  schedule:
+    # Weekly re-scan of master so new/updated queries land without a push.
+    - cron: "17 4 * * 1"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: codeql-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: analyze (c-cpp)
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      # Upload findings to the repo's code-scanning dashboard.
+      security-events: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - uses: github/codeql-action/init@v4
+        with:
+          languages: c-cpp
+          build-mode: manual
+          queries: security-extended
+          # fopen's umask-controlled 0666 is intended for mirror/cache/log
+          # output; the one credential file (cookies.txt) is kept 0600 on Unix.
+          config: |
+            query-filters:
+              - exclude:
+                  id: cpp/world-writable-file-creation
+
+      # Manual build: CodeQL traces the compiler, so build exactly what ships.
+      - name: Build
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure
+          make -j"$(nproc)"
+
+      - uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:c-cpp"

.gitignore

@@ -34,8 +34,9 @@ Makefile
 *.so.*
 *.a
 
-# make dist output.
+# make dist output; dist/ holds httrack-gh-release staging artifacts.
 /httrack-*.tar.gz
+/dist/
 
 # Editor / autotools backup files.
 *~

AGENTS.md

@@ -6,8 +6,19 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 ## Build & test
 - Fresh clone first: `git submodule update --init src/coucal`
 - `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
-  automake, libtool), then `bash configure && make && make check`. Or run
+  automake, libtool), then `bash configure && make -j"$(nproc)" && make check
-  `sh build.sh` to do bootstrap + configure + make in one shot.
+  -j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
+  automake's parallel harness and each crawl test binds its own ephemeral-port
+  server, so `-j` never contends and a multi-minute serial run drops to
+  seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
+  automatically; only a test slower than the current longest raises the floor.
+  On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
+  spend much of their wall time asleep (server trickles, httrack self-pacing),
+  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
+  platform, macOS included: the test server raises its listen backlog
+  (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
+  `-c16` bigcrawl the way Python's default backlog of 5 did.
+  Or run `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

Makefile.am

@@ -1,5 +1,5 @@
 
-SUBDIRS = src man m4 libtest templates lang html tests
+SUBDIRS = src man m4 libtest templates lang html tests fuzz
 
 ACLOCAL_AMFLAGS = -I m4
 

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,11 +29,11 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
+# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
-# installed headers (no struct layout or exported signature changed vs
+# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
-# 3.49.10), so it stays soname .so.3; bump revision.
+# so it stays soname .so.3; bump revision.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:3:0"
+VERSION_INFO="3:4:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 
@@ -102,7 +102,8 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
 AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
 AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
 AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
-AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
+AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
+	[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
 AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
 AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
 AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -110,6 +111,13 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
 AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
 AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
 
+# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
+# sanitizer builds, whose interceptors want the unfortified calls.
+case "$CFLAGS" in
+*-fsanitize=*) ;;
+*) AX_ADD_FORTIFY_SOURCE ;;
+esac
+
 # Force libc back into DT_NEEDED for libraries that reach it only through
 # libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
 # linker; Apple ld rejects these flags and links libSystem unconditionally.
@@ -310,6 +318,37 @@ AC_ARG_ENABLE([online-unit-tests],
 	])
 AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
 
+## libFuzzer harnesses (fuzz/); requires clang
+AC_MSG_CHECKING(whether to build fuzzers)
+AC_ARG_ENABLE([fuzzers],
+	[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
+	[
+	case "${enableval}" in
+	no|yes)
+		fuzzers=$enableval
+		AC_MSG_RESULT($enableval)
+		;;
+	*)
+		AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
+		;;
+	esac
+	],
+	[
+	fuzzers=no
+	AC_MSG_RESULT(no)
+	])
+if test x"$fuzzers" = x"yes"; then
+	# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
+	AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
+		[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
+		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
+	# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
+	if test x"$enable_shared" != x"no"; then
+		AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
+	fi
+fi
+AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
+
 # Final output
 AC_CONFIG_FILES([
 Makefile
@@ -321,5 +360,6 @@ lang/Makefile
 html/Makefile
 libtest/Makefile
 tests/Makefile
+fuzz/Makefile
 ])
 AC_OUTPUT

debian/changelog

@@ -1,3 +1,13 @@
+httrack (3.49.12-1) unstable; urgency=medium
+
+  * New upstream release: security and crawl-correctness fixes (remote stack
+    overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
+    world-readable cookies.txt, filter-pattern denial of service) plus a new
+    --why filter diagnostic; full list in history.txt.
+  * Build with _FORTIFY_SOURCE and -fstack-protector-strong.
+
+ -- Xavier Roche <xavier@debian.org>  Fri, 10 Jul 2026 21:11:09 +0200
+
 httrack (3.49.11-1) unstable; urgency=medium
 
   * New upstream release: crawl correctness and security fixes (network-facing

fuzz/Makefile.am

@@ -0,0 +1,45 @@
+# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
+if FUZZERS
+noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
+	fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx
+endif
+
+AM_CPPFLAGS = \
+	@DEFAULT_CFLAGS@ \
+	@THREADS_CFLAGS@ \
+	@V6_FLAG@ \
+	@LFS_FLAG@ \
+	-I$(top_srcdir)/src \
+	-I$(top_srcdir)/src/coucal
+
+# Static-link libhttrack.la: the internal symbols are hidden in the .so.
+AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
+LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
+
+fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
+fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
+fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
+fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
+fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
+fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
+fuzz_url_SOURCES = fuzz-url.c fuzz.h
+fuzz_header_SOURCES = fuzz-header.c fuzz.h
+fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
+
+# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
+EXTRA_DIST = README.md run-fuzzers.sh \
+	corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
+	corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
+	corpus/idna/idna.txt corpus/idna/unicode.txt \
+	corpus/idna/regress-multilabel-leak.txt \
+	corpus/entities/entities.txt \
+	corpus/unescape/percent.txt \
+	corpus/filters/filter.bin corpus/filters/filter-size.bin \
+	corpus/filters/regress-empty-subject-unique.bin \
+	corpus/filters/redos-star-classes.bin \
+	corpus/url/http-url.txt corpus/url/relative-path.txt \
+	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
+	corpus/header/full-response.txt corpus/header/redirect.txt \
+	corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
+	corpus/cachendx/regress-overadvance.bin \
+	corpus/cachendx/regress-truncated-entry.bin

fuzz/README.md

@@ -0,0 +1,15 @@
+# Fuzzing httrack
+
+libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
+
+```sh
+./bootstrap
+mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
+CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
+  LDFLAGS="-fsanitize=address,undefined" \
+  bash /path/to/httrack/configure --enable-fuzzers --disable-shared
+make
+bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60   # 60s per target
+```
+
+Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.

fuzz/corpus/cachendx/new-format.txt

@@ -0,0 +1,7 @@
+8
+CACHE-1.1
+28
+Mon, 01 Jan 2024 00:00:00 GMT
+www.example.com
+/index.html
+123

fuzz/corpus/cachendx/old-format.txt

@@ -0,0 +1,5 @@
+3
+1.0
+www.example.com
+/page
+5

fuzz/corpus/cachendx/regress-overadvance.bin

@@ -0,0 +1,2 @@
+32768
+CACHE-1.1

fuzz/corpus/cachendx/regress-truncated-entry.bin

@@ -0,0 +1,5 @@
+8
+CACHE-1.1
+1
+x
+www.example.com

fuzz/corpus/charset/latin1.txt

@@ -0,0 +1 @@
+
café naďve ¤

fuzz/corpus/charset/sjis.txt

@@ -0,0 +1 @@
+コンピュータ

fuzz/corpus/entities/entities.txt

@@ -0,0 +1 @@
+&<>A☃é&notarealentity;�

fuzz/corpus/filters/regress-empty-subject-unique.bin

@@ -0,0 +1 @@
+**((

fuzz/corpus/header/full-response.txt

@@ -0,0 +1,10 @@
+HTTP/1.1 200 OK
+Content-Type: text/html; charset=utf-8
+Content-Length: 1234
+Content-Encoding: gzip
+Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
+Etag: "abc"
+Location: http://example.com/x
+Set-Cookie: ID=42; path=/; domain=.example.com
+Content-Range: bytes 0-99/100
+Transfer-Encoding: chunked

fuzz/corpus/header/redirect.txt

@@ -0,0 +1,3 @@
+HTTP/1.0 301 Moved
+Location: /elsewhere
+Content-Disposition: attachment; filename="a.pdf"

fuzz/corpus/idna/idna.txt

@@ -0,0 +1 @@
+xn--bcher-kva.example.com

fuzz/corpus/idna/regress-multilabel-leak.txt

@@ -0,0 +1 @@
+büchev.例å­bücheple

fuzz/corpus/idna/unicode.txt

@@ -0,0 +1 @@
+bücher.例子.example

fuzz/corpus/meta/meta-charset.html

@@ -0,0 +1 @@
+<html><head><meta charset="utf-8"></head><body>x</body></html>

fuzz/corpus/meta/meta-content-first.html

@@ -0,0 +1 @@
+<meta content="text/html; charset=gb2312" http-equiv="content-type">

fuzz/corpus/meta/meta-http-equiv.html

@@ -0,0 +1 @@
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">

fuzz/corpus/meta/meta-truncated.html

@@ -0,0 +1 @@
+<meta charset='utf-8

fuzz/corpus/meta/meta-unquoted.html

@@ -0,0 +1 @@
+<meta   charset = utf-8 />

fuzz/corpus/unescape/percent.txt

@@ -0,0 +1 @@
+%41%zz%%20%c3%a9+%2e%2e%2f

fuzz/corpus/url/http-url.txt

@@ -0,0 +1 @@
+http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag

fuzz/corpus/url/regress-file-empty-path.txt

@@ -0,0 +1 @@
+file://

fuzz/corpus/url/regress-long-path-abort.txt

@@ -0,0 +1 @@
+ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../

fuzz/corpus/url/relative-path.txt

@@ -0,0 +1 @@
+ftp://ftp.example.com/pub/../file.txt

fuzz/fuzz-cachendx.c

@@ -0,0 +1,65 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
+   walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
+   Mirrors the -#C cache-listing scan (htscoremain.c). */
+#include "fuzz.h"
+#include "htscache.h"
+#include "htslib.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  const char *const end = buf + size;
+  char firstline[256];
+  char *a = buf;
+
+  /* header: two length-prefixed fields (version, last-modified) */
+  a += cache_brstr(a, firstline, sizeof(firstline));
+  a += cache_brstr(a, firstline, sizeof(firstline));
+
+  /* body: newline-delimited host/file/position triples; the length-prefixed
+     scan must stay inside the buffer */
+  while (a != NULL && a < end) {
+    char BIGSTK line[HTS_URLMAXSIZE * 2];
+    char linepos[256];
+    int pos;
+
+    a = strchr(a + 1, '\n');
+    if (a == NULL)
+      break;
+    a++;
+    a += cache_binput(a, end, line, HTS_URLMAXSIZE);
+    a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
+    a += cache_binput(a, end, linepos, 200);
+    sscanf(linepos, "%d", &pos);
+    (void) pos;
+  }
+
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-charset.c

@@ -0,0 +1,84 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
+   UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+static const char *const charsets[] = {
+    "utf-8",    "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
+    "us-ascii", "shift_jis",  "euc-jp",     "iso-2022-jp", "gb2312",
+    "big5",     "euc-kr",     "koi8-r",     "utf-16",      "unknown-charset",
+};
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  const char *charset;
+  char *s;
+
+  if (size == 0)
+    return 0;
+  charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+
+  {
+    char *utf8 = hts_convertStringToUTF8(s, size, charset);
+    freet(utf8);
+  }
+  {
+    char *enc = hts_convertStringFromUTF8(s, size, charset);
+    freet(enc);
+  }
+  {
+    size_t nChars = 0;
+    hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
+
+    if (ucs != NULL) {
+      char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
+      freet(back);
+      freet(ucs);
+    }
+  }
+  {
+    size_t i = 0;
+
+    while (i < size) {
+      hts_UCS4 uc = 0;
+      const size_t nr = hts_readUTF8(s + i, size - i, &uc);
+      char out[8];
+
+      if (nr == 0)
+        break;
+      hts_writeUTF8(uc, out, sizeof(out));
+      i += nr;
+    }
+  }
+
+  freet(s);
+  return 0;
+}

fuzz/fuzz-entities.c

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
+   destination size, so truncation bounds get exercised too. */
+#include "fuzz.h"
+#include "htsencoding.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t dsizes[] = {1, 2, 8, 64, 4096};
+  size_t dsize;
+  char *src, *dest;
+
+  if (size == 0)
+    return 0;
+  dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
+  data++, size--;
+  src = fuzz_strdup(data, size);
+  dest = malloct(dsize);
+
+  (void) hts_unescapeEntities(src, dest, dsize);
+  (void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
+
+  freet(dest);
+  freet(src);
+  return 0;
+}

fuzz/fuzz-filters.c

@@ -0,0 +1,65 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
+   here). Input splits on the first NUL: pattern, then subject string. */
+#include "fuzz.h"
+#include "htsfilters.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  const char *joker = buf;
+  const uint8_t *sep = memchr(data, '\0', size);
+  /* subject in its own allocation so ASan bounds it apart from the pattern */
+  char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
+                          : fuzz_strdup(data + size, 0);
+
+  (void) strjoker(nom, joker, NULL, NULL);
+  {
+    LLint sz = (LLint) size;
+    int size_flag = 0;
+
+    (void) strjoker(nom, joker, &sz, &size_flag);
+  }
+  (void) strjokerfind(nom, joker);
+  {
+    char *filter = malloct(strlen(joker) + 2);
+    char *filters[1];
+    LLint sz = (LLint) size;
+    int size_flag = 0, depth = 0;
+
+    filter[0] = '-';
+    memcpy(filter + 1, joker, strlen(joker) + 1);
+    filters[0] = filter;
+    (void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
+    freet(filter);
+  }
+
+  freet(nom);
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-header.c

@@ -0,0 +1,74 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
+   status line, treathead on each following header. Both consume raw bytes
+   off the wire and copy fields into fixed htsblk buffers; treathead also
+   mutates its line in place and drives cookie parsing. */
+#include "fuzz.h"
+#include "htslib.h"
+#include "htsbauth.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  htsblk r;
+  t_cookie *cookie = calloct(1, sizeof(*cookie));
+  char *line = malloct(size + 1);
+  char *p = buf;
+  int first = 1;
+
+  memset(&r, 0, sizeof(r));
+  cookie->max_len = (int) sizeof(cookie->data);
+  r.location = malloct(HTS_URLMAXSIZE * 2);
+  r.location[0] = '\0';
+
+  /* feed one header line at a time, as the receive loop does */
+  while (p != NULL && *p != '\0') {
+    char *nl = strchr(p, '\n');
+    size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
+    size_t i, len = 0;
+
+    /* binput drops every '\r' on the wire; mirror it */
+    for (i = 0; i < n; i++)
+      if (p[i] != '\r')
+        line[len++] = p[i];
+    line[len] = '\0';
+    if (first) {
+      treatfirstline(&r, line);
+      first = 0;
+    } else {
+      treathead(cookie, "www.example.com", "/", &r, line);
+    }
+    p = (nl != NULL) ? nl + 1 : NULL;
+  }
+
+  freet(r.location);
+  freet(line);
+  freet(cookie);
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-idna.c

@@ -0,0 +1,47 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+
+  {
+    char *idna = hts_convertStringUTF8ToIDNA(s, size);
+    freet(idna);
+  }
+  {
+    char *utf8 = hts_convertStringIDNAToUTF8(s, size);
+    freet(utf8);
+  }
+  (void) hts_isStringIDNA(s, size);
+
+  freet(s);
+  return 0;
+}

fuzz/fuzz-meta.c

@@ -0,0 +1,40 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
+   <meta> charset declaration. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *html = fuzz_strdup(data, size);
+  char *charset = hts_getCharsetFromMeta(html, size);
+
+  freet(charset);
+  freet(html);
+  return 0;
+}

fuzz/fuzz-unescape.c

@@ -0,0 +1,53 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
+   output buffer size, so the bounded-copy contract is exercised. */
+#include "fuzz.h"
+#include "httrack-library.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t bsizes[] = {1, 2, 16, 256, 8192};
+  size_t bsize;
+  char *s, *catbuff;
+
+  if (size == 0)
+    return 0;
+  bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+  catbuff = malloct(bsize);
+
+  (void) unescape_http(catbuff, bsize, s);
+  (void) unescape_http_unharm(catbuff, bsize, s, 0);
+  (void) unescape_http_unharm(catbuff, bsize, s, 1);
+  unescape_amp(s);
+
+  freet(catbuff);
+  freet(s);
+  return 0;
+}

fuzz/fuzz-url.c

@@ -0,0 +1,49 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
+   is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
+   in place. */
+#include "fuzz.h"
+#include "htscore.h"
+#include "htslib.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+  lien_adrfil *af = calloct(1, sizeof(*af));
+  /* fil_simplifie rewrites in place and may grow an empty path to "./" */
+  char *path = malloct(size + 3);
+
+  (void) ident_url_absolute(s, af);
+  memcpy(path, s, size + 1);
+  fil_simplifie(path);
+
+  freet(path);
+  freet(af);
+  freet(s);
+  return 0;
+}

fuzz/fuzz.h

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Shared helpers for the libFuzzer harnesses. */
+#ifndef FUZZ_H
+#define FUZZ_H
+
+#define HTS_INTERNAL_BYTECODE
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "htsbase.h"
+
+/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
+static char *fuzz_strdup(const uint8_t *data, size_t size) {
+  char *s = malloct(size + 1);
+
+  memcpy(s, data, size);
+  s[size] = '\0';
+  return s;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
+
+#endif

fuzz/run-fuzzers.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Drive every built harness against its seed corpus.
+#   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
+#   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
+# Replay is crash/leak-only and never mutates; the per-unit -timeout guards
+# both modes against pathological slowdowns (e.g. pre-#501 strjoker).
+set -euo pipefail
+
+srcdir=$(cd "$(dirname "$0")" && pwd)
+bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
+mode=${2:-20}
+
+status=0
+for f in "$bld"/fuzz-*; do
+    if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
+    case "$f" in *.o | *.c | *.dSYM) continue ;; esac
+    name=$(basename "$f")
+    corpus="$srcdir/corpus/${name#fuzz-}"
+    if [ "$mode" = "check" ]; then
+        echo "=== $name (replay) ==="
+        [ -d "$corpus" ] || continue
+        if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
+            echo "*** $name FAILED on its corpus" >&2
+            status=1
+        fi
+        continue
+    fi
+    work=$(mktemp -d)
+    args=("$work")
+    [ -d "$corpus" ] && args+=("$corpus")
+    echo "=== $name (${mode}s) ==="
+    if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
+        -artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
+        echo "*** $name FAILED; artifacts:" >&2
+        ls -l "$work" >&2
+        status=1
+    else
+        rm -rf "$work"
+    fi
+done
+exit $status

history.txt

@@ -4,6 +4,23 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
+3.49-12
++ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
++ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
++ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
++ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
++ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
++ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
++ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
++ Fixed: cookies.txt was created world-readable (#511)
++ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
++ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
++ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
++ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
++ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
++ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
++ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
+
 3.49-11
 + New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
 + New: advertise deflate in Accept-Encoding and decode deflate responses (#450)

m4/ax_add_fortify_source.m4

@@ -0,0 +1,119 @@
+# ===========================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_ADD_FORTIFY_SOURCE
+#
+# DESCRIPTION
+#
+#   Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
+#   redefinition warnings, other cpp warnings or linker. Some distributions
+#   (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
+#   their compilers, leading to unnecessary warnings in the form of
+#
+#     <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
+#     <built-in>: note: this is the location of the previous definition
+#
+#   which is a problem if -Werror is enabled. This macro checks whether
+#   _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
+#   to CPPFLAGS.
+#
+#   Newer mingw-w64 msys2 package comes with a bug in
+#   headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
+#   and would need -lssp or -fstack-protector.  See
+#   https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
+#   link it.
+#
+# LICENSE
+#
+#   Copyright (c) 2017 David Seifert <soap@gentoo.org>
+#   Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#serial 10
+
+AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
+    ac_save_cflags=$CFLAGS
+    ac_cwerror_flag=yes
+    AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
+    ax_add_fortify_3_failed=
+    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
+    AC_LINK_IFELSE([
+        AC_LANG_PROGRAM([],
+            [[
+                #ifndef _FORTIFY_SOURCE
+                    return 0;
+                #else
+                    _FORTIFY_SOURCE_already_defined;
+                #endif
+            ]]
+        )],
+        AC_LINK_IFELSE([
+            AC_LANG_SOURCE([[
+                #define _FORTIFY_SOURCE 3
+                #include <string.h>
+                int main(void) {
+                    char *s = " ";
+                    strcpy(s, "x");
+                    return strlen(s)-1;
+                }
+              ]]
+            )],
+            [
+              AC_MSG_RESULT([yes])
+              CFLAGS=$ac_save_cflags
+              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
+            ], [
+              AC_MSG_RESULT([no])
+              ax_add_fortify_3_failed=1
+            ],
+        ),
+        [
+          AC_MSG_RESULT([no])
+          ax_add_fortify_3_failed=1
+        ])
+    if test -n "$ax_add_fortify_3_failed"
+    then
+    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
+    AC_LINK_IFELSE([
+        AC_LANG_PROGRAM([],
+            [[
+                #ifndef _FORTIFY_SOURCE
+                    return 0;
+                #else
+                    _FORTIFY_SOURCE_already_defined;
+                #endif
+            ]]
+        )],
+        AC_LINK_IFELSE([
+            AC_LANG_SOURCE([[
+                #define _FORTIFY_SOURCE 2
+                #include <string.h>
+                int main(void) {
+                    char *s = " ";
+                    strcpy(s, "x");
+                    return strlen(s)-1;
+                }
+              ]]
+            )],
+            [
+              AC_MSG_RESULT([yes])
+              CFLAGS=$ac_save_cflags
+              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
+            ], [
+              AC_MSG_RESULT([no])
+              CFLAGS=$ac_save_cflags
+            ],
+        ),
+        [
+          AC_MSG_RESULT([no])
+          CFLAGS=$ac_save_cflags
+        ])
+    fi
+])

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "27 June 2026" "httrack website copier"
+.TH httrack 1 "07 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -51,6 +51,7 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
 [ \fB\-%K, \-\-cookies\-file\fR ]
+[ \fB\-%Y, \-\-why\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -218,6 +219,8 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
 .IP \-%K
 load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
+.IP \-%Y
+explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j

src/htsalias.c

@@ -114,6 +114,8 @@ const char *hts_optalias[][4] = {
    "strip [host/pattern=]key1,key2,... from URLs"},
   {"cookies-file", "-%K", "param1",
    "load extra cookies from a Netscape cookies.txt"},
+  {"why", "-%Y", "param1",
+   "explain which filter rule accepts or rejects a URL, then exit"},
   {"pause", "-%G", "param1",
    "random pause of MIN[:MAX] seconds between files"},
   {"generate-errors", "-o", "single", ""},
@@ -283,7 +285,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
       char *position;
       int need_param = 1;
 
-      //int return_param=0;
       int pos;
 
       command[0] = param[0] = addcommand[0] = '\0';
@@ -361,7 +362,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
               strlcatbuff(return_argv[0], "0", return_argv_size);
             else if (strcmp(param, "on") == 0) {
               // on is the default
-              // strcatbuff(return_argv[0],"1");
             } else
               strlcatbuff(return_argv[0], param, return_argv_size);
           }

src/htsback.c

@@ -43,14 +43,12 @@ Please visit our Website: http://www.httrack.com
 
 #include "htsback.h"
 
-//#ifdef _WIN32
 #include "htsftp.h"
 #if HTS_USEZLIB
 #include "htszlib.h"
 #else
 #error HTS_USEZLIB not defined
 #endif
-//#endif
 
 #ifdef _WIN32
 #ifndef __cplusplus
@@ -70,6 +68,15 @@ static int slot_can_be_cached_on_disk(const lien_back * back);
 static int slot_can_be_cleaned(const lien_back * back);
 static int slot_can_be_finalized(httrackp * opt, const lien_back * back);
 
+/* Which hard quota, if any, is currently aborting the mirror. */
+typedef enum {
+  HTS_MIRROR_LIMIT_NONE = 0,
+  HTS_MIRROR_LIMIT_SIZE,
+  HTS_MIRROR_LIMIT_TIME,
+} hts_mirror_limit;
+
+static hts_mirror_limit back_mirror_limit(httrackp *opt);
+
 struct_back *back_new(httrackp *opt, int back_max) {
   int i;
   struct_back *sback = calloct(1, sizeof(struct_back));
@@ -526,22 +533,22 @@ int back_nsoc_overall(const struct_back * sback) {
 
 /* generate temporary file on lien_back */
 /* Note: utf-8 */
-static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
+static int create_back_tmpfile(httrackp *opt, lien_back *const back,
+                               const char *ext) {
   // do not use tempnam() but a regular filename
   back->tmpfile_buffer[0] = '\0';
   if (back->url_sav != NULL && back->url_sav[0] != '\0') {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.z", 
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
-             back->url_sav);
+             back->url_sav, ext);
     back->tmpfile = back->tmpfile_buffer;
     if (structcheck(back->tmpfile) != 0) {
-      hts_log_print(opt, LOG_WARNING, "can not create directory to %s", 
+      hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
                     back->tmpfile);
       return -1;
     }
   } else {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer),
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
-             "%s/tmp%d.z", StringBuff(opt->path_html_utf8),
+             StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
-             opt->state.tmpnameid++);
     back->tmpfile = back->tmpfile_buffer;
   }
   /* OK */
@@ -549,6 +556,32 @@ static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
   return 0;
 }
 
+/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
+   existing file moved the good copy to back->tmpfile before truncating url_sav.
+   commit keeps the new file and drops the backup; else restore it so an aborted
+   transfer leaves the previous copy intact. Skips the zlib .z temp. */
+static void back_finalize_backup(httrackp *opt, lien_back *const back,
+                                 const hts_boolean commit) {
+  if (back->tmpfile == NULL || back->r.compressed)
+    return;
+  if (commit) {
+    (void) UNLINK(back->tmpfile); /* new copy is good; drop the backup */
+  } else {
+    if (back->r.out != NULL) {
+      fclose(back->r.out);
+      back->r.out = NULL;
+    }
+    (void) UNLINK(back->url_sav); /* drop the failed partial */
+    /* On rename failure keep the backup: it still holds the good copy, so
+       losing it would be worse than an orphaned temp. */
+    if (RENAME(back->tmpfile, back->url_sav) != 0)
+      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
+                    "could not restore %s; previous copy kept as %s",
+                    back->url_sav, back->tmpfile);
+  }
+  back->tmpfile = NULL;
+}
+
 // objet (lien) téléchargé ou transféré depuis le cache
 //
 // fermer les paramètres de transfert,
@@ -584,6 +617,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                       LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
                       back[p].r.size, back[p].url_adr, back[p].url_fil);
       }
+      back_finalize_backup(opt, &back[p], HTS_FALSE);
       return -1;
     }
 
@@ -597,13 +631,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
 #if HTS_USEZLIB
         if (back[p].r.compressed) {
           if (back[p].r.size > 0) {
-            //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
             // stats
             back[p].compressed_size = back[p].r.size;
             // en mémoire -> passage sur disque
             if (!back[p].r.is_write) {
               // do not use tempnam() but a regular filename
-              if (create_back_tmpfile(opt, &back[p]) == 0) {
+              if (create_back_tmpfile(opt, &back[p], "z") == 0) {
                 assertf(back[p].tmpfile != NULL);
                 /* note: tmpfile is utf-8 */
                 back[p].r.out = FOPEN(back[p].tmpfile, "wb");
@@ -676,6 +709,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
           }
         }
 #endif
+        /* Body fully received: keep the freshly written url_sav, drop the
+           backup of the previous copy. */
+        back_finalize_backup(opt, &back[p], HTS_TRUE);
         /* Write mode to disk */
         if (back[p].r.is_write && back[p].r.adr != NULL) {
           freet(back[p].r.adr);
@@ -904,6 +940,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
       }
     }
   }
+  /* Aborted, error, or not ready: url_sav (if written) is broken; restore the
+     previous copy from the backup. */
+  back_finalize_backup(opt, &back[p], HTS_FALSE);
   return -1;
 }
 
@@ -927,7 +966,6 @@ int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
     /* clear everything but connection: switch, close, and reswitch */
     back_connxfr(src, &tmp);
     back_delete(opt, cache, sback, p);
-    //deletehttp(src);
     back_connxfr(&tmp, src);
     src->req.flush_garbage = 1; /* ignore CRLF garbage */
     return 1;
@@ -1437,7 +1475,6 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                       back[p].url_adr, back[p].url_fil, back[p].url_sav);
       }
       if (cache != NULL) {
-        //hts_log_print(opt, LOG_TRACE, "finalizing from back_delete");
         back_finalize(opt, cache, sback, p);
       }
     }
@@ -1578,7 +1615,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     if (back[p].r.soc != INVALID_SOCKET) {      /* we never know */
       deletehttp(&back[p].r);
     }
-    //memset(&(back[p].r), 0, sizeof(htsblk)); 
     hts_init_htsblk(&back[p].r);
     back[p].r.location = back[p].location_buffer;
 
@@ -1586,7 +1622,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     strcpybuff(back[p].url_adr, adr);
     strcpybuff(back[p].url_fil, fil);
     strcpybuff(back[p].url_sav, save);
-    //back[p].links_index = links_index;
     // copier referer si besoin
     strcpybuff(back[p].referer_adr, "");
     strcpybuff(back[p].referer_fil, "");
@@ -1653,138 +1688,104 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       }
     }
     // tester cache
-    if ((strcmp(adr, "file://"))        /* pas fichier */
+    if ((strcmp(adr, "file://")) /* pas fichier */
-        &&((!test) || (cache->type == 1))       /* cache prioritaire, laisser passer en test! */
+        && ((!test) ||
-        &&((strnotempty(save)) || (strcmp(fil, "/robots.txt") == 0))) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
+            (cache->type == 1)) /* cache prioritaire, laisser passer en test! */
-#if HTS_FAST_CACHE
+        &&
+        ((strnotempty(save)) || (strcmp(fil, "/robots.txt") ==
+                                 0))) { // si en test on ne doit pas utiliser le
+                                        // cache sinon telescopage avec le 302..
       intptr_t hash_pos;
       int hash_pos_return = 0;
-#else
+
-      char *a = NULL;
-#endif
-#if HTS_FAST_CACHE
       if (cache->hashtable) {
-#else
-      if (cache->use) {
-#endif
         char BIGSTK buff[HTS_URLMAXSIZE * 4];
 
-#if HTS_FAST_CACHE
         strcpybuff(buff, adr);
         strcatbuff(buff, fil);
         hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
-#else
-        buff[0] = '\0';
-        strcatbuff(buff, "\n");
-        strcatbuff(buff, adr);
-        strcatbuff(buff, "\n");
-        strcatbuff(buff, fil);
-        strcatbuff(buff, "\n");
-        a = strstr(cache->use, buff);
-#endif
 
-        // Ok, noté en cache->. mais bien présent dans le cache ou sur disque?
-#if HTS_FAST_CACHE
         // negative values when data is not in cache
         if (hash_pos_return < 0) {
-#else
+          if (!test) { // not test mode
-        if (a) {
+            /* note: no check with IS_DELAYED_EXT() enabled - postcheck by
-#endif
+             * client please! */
-          if (!test) {          // non mode test
+            if (save[0] != '\0' && !IS_DELAYED_EXT(save) &&
-#if HTS_FAST_CACHE==0
+                fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <=
-            int pos = -1;
+                    0) { // final file missing or empty
+              int found = 0;
 
-            a += strlen(buff);
+              /* It is possible that the file has been moved due to changes in
-            sscanf(a, "%d", &pos);      // lire position
+               * build structure */
-#endif
+              {
+                char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
+                htsblk r;
 
-#if HTS_FAST_CACHE==0
+                previous_save[0] = '\0';
-            if (pos < 0) {      // pas de mise en cache data, vérifier existence
+                r = cache_readex(opt, cache, adr, fil, /*head */ NULL,
-#endif
-              /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
-              if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <= 0) {  // fichier final n'existe pas ou est vide!
-                int found = 0;
-
-                /* It is possible that the file has been moved due to changes in build structure */
-                {
-                  char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
-                  htsblk r;
-
-                  previous_save[0] = '\0';
-                  r =
-                    cache_readex(opt, cache, adr, fil, /*head */ NULL,
                                  /*bound to back[p] (temporary) */
                                  back[p].location_buffer, previous_save, /*ro */
                                  1);
-                  /* Is supposed to be on disk only */
+                /* Is supposed to be on disk only */
-                  if (r.is_write && previous_save[0] != '\0') {
+                if (r.is_write && previous_save[0] != '\0') {
-                    /* Exists, but with another (old) filename: rename (almost) silently */
+                  /* Exists, but with another (old) filename: rename (almost)
-                    if (strcmp(previous_save, save) != 0
+                   * silently */
-                        && fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) {
+                  if (strcmp(previous_save, save) != 0 &&
-                      rename(fconv(catbuff, sizeof(catbuff), previous_save),
+                      fexist_utf8(
-                             fconv(catbuff2, sizeof(catbuff2), save));
+                          fconv(catbuff, sizeof(catbuff), previous_save))) {
-                      if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
+                    rename(fconv(catbuff, sizeof(catbuff), previous_save),
-                        found = 1;
+                           fconv(catbuff2, sizeof(catbuff2), save));
-                        hts_log_print(opt, LOG_DEBUG,
+                    if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
-                                      "File '%s' has been renamed since last mirror to '%s' ; applying changes",
+                      found = 1;
-                                      previous_save, save);
+                      hts_log_print(opt, LOG_DEBUG,
-                      } else {
+                                    "File '%s' has been renamed since last "
-                        hts_log_print(opt, LOG_ERROR,
+                                    "mirror to '%s' ; applying changes",
-                                      "Could not rename '%s' to '%s' ; will have to retransfer it",
+                                    previous_save, save);
-                                      previous_save, save);
+                    } else {
-                      }
+                      hts_log_print(opt, LOG_ERROR,
+                                    "Could not rename '%s' to '%s' ; will have "
+                                    "to retransfer it",
+                                    previous_save, save);
                     }
                   }
-                  back[p].location_buffer[0] = '\0';
                 }
+                back[p].location_buffer[0] = '\0';
+              }
 
-                /* Not found ? */
+              /* Not found ? */
-                if (!found) {
+              if (!found) {
-#if HTS_FAST_CACHE
+                // invalidate: gone from disk, force a refetch
-                  hash_pos_return = 0;
+                hash_pos_return = 0;
-#else
+                if (opt->norecatch) {
-                  a = NULL;
+                  if (!fexist_utf8(fconv(
-#endif
+                          catbuff, sizeof(catbuff),
-                  // dévalider car non présent sur disque dans structure originale!!!
+                          save))) { // declared but missing: user erased it
-                  // sinon, le fichier est ok à priori, mais on renverra un if-modified-since pour
+                    FILE *fp =
-                  // en être sûr
+                        FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
-                  if (opt->norecatch) { // tester norecatch
-                    if (!fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {   // fichier existe pas mais déclaré: on l'a effacé
-                      FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
 
-                      if (fp)
+                    if (fp)
-                        fclose(fp);
+                      fclose(fp);
-                      hts_log_print(opt, LOG_WARNING,
-                                    "Previous file '%s' not found (erased by user ?), ignoring: %s%s",
-                                    save, back[p].url_adr, back[p].url_fil);
-                    }
-                  } else {
                     hts_log_print(opt, LOG_WARNING,
-                                  "Previous file '%s' not found (erased by user ?), recatching: %s%s",
+                                  "Previous file '%s' not found (erased by "
+                                  "user ?), ignoring: %s%s",
                                   save, back[p].url_adr, back[p].url_fil);
                   }
+                } else {
+                  hts_log_print(opt, LOG_WARNING,
+                                "Previous file '%s' not found (erased by user "
+                                "?), recatching: %s%s",
+                                save, back[p].url_adr, back[p].url_fil);
                 }
-              }                 // fsize() <= 0
+              }
-#if HTS_FAST_CACHE==0
+            } // fsize() <= 0
-            }
-#endif
           }
         }
         //
       } else {
-#if HTS_FAST_CACHE
         hash_pos_return = 0;
-#else
-        a = NULL;
-#endif
       }
 
-      // Existe pas en cache, ou bien pas de cache présent
+      if (hash_pos_return) { // in cache, with data
-#if HTS_FAST_CACHE
-      if (hash_pos_return) {    // OK existe en cache (et données aussi)!
-#else
-      if (a != NULL) {          // OK existe en cache (et données aussi)!
-#endif
         const int cache_is_prioritary = cache->type == 1
           || opt->state.stop != 0;
         if (cache_is_prioritary) {      // cache prioritaire (pas de test if-modified..)
@@ -1825,8 +1826,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
                             back[p].url_adr, back[p].url_fil);
             }
             back[p].r.notmodified = 1;  // fichier non modifié
-            back[p].status = STATUS_READY;      // OK prêt
+            back[p].status = STATUS_READY; // OK prêt
-            //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);        // not modified
             back_set_finished(sback, p);
 
             // finalize transfer
@@ -1841,7 +1841,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
           } else {              // erreur
             // effacer r
             hts_init_htsblk(&back[p].r);
-            //memset(&(back[p].r), 0, sizeof(htsblk)); 
             back[p].r.location = back[p].location_buffer;
             // et continuer (chercher le fichier)
           }
@@ -1907,7 +1906,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #endif
         }
       }
-      /* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */
+      /* Not in cache ; maybe in temporary cache ? Warning: non-movable
+         "url_sav" */
       else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
         const off_t file_size = fsize_utf8(itemback->url_sav);
 
@@ -2036,7 +2036,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       // ouvrir liaison, envoyer requète
       // ne pas traiter ou recevoir l'en tête immédiatement
       hts_init_htsblk(&back[p].r);
-      // memset(&(back[p].r), 0, sizeof(htsblk));
       back[p].r.location = back[p].location_buffer;
       // fresh connect: address list not yet probed, start at the first
       sback->connect_fallback[p].addr_index = 0;
@@ -2087,20 +2086,15 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #if HTS_USEOPENSSL
       else if (strfield(back[p].url_adr, "https://")) {     // let's rock
         back[p].r.ssl = 1;
-        // back[p].r.ssl_soc = NULL;
         back[p].r.ssl_con = NULL;
       }
 #endif
 
       if (!back_trylive(opt, cache, sback, p)) {
 #if HTS_XGETHOST
-#if HDEBUG
+        back[p].status = STATUS_WAIT_DNS; // host name resolution attempt
-        printf("back_solve..\n");
+        soc = INVALID_SOCKET;             // not opened yet
-#endif
+        if (host_wait(opt, &back[p])) {   // ready (file, or cached dns)
-        back[p].status = STATUS_WAIT_DNS;       // tentative de résolution du nom de host
-        soc = INVALID_SOCKET;   // pas encore ouverte
-        back_solve(opt, &back[p]);      // préparer
-        if (host_wait(opt, &back[p])) { // prêt, par ex fichier ou dispo dans dns
 #if HDEBUG
           printf("ok, dns cache ready..\n");
 #endif
@@ -2153,9 +2147,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
         back[p].rateout = -1;   // pas de gestion (default)
       }
 
-      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est
+      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où
-      // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
+      // cela est rattrapable (exemple: 301,302 moved xxx -> refresh sur la
-      //if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
+      // page!)
 
 #if CNXDEBUG
       printf("Xfopen ok, poll..\n");
@@ -2173,7 +2167,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       if (soc == INVALID_SOCKET) {      // erreur socket
         back[p].status = STATUS_READY;  // FINI
         back_set_finished(sback, p);
-        //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
         back[p].r.soc = INVALID_SOCKET;
       } else {
         if (!back[p].r.is_file)
@@ -2194,8 +2187,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
     // note: si il y a erreur (404,etc) status=2 (terminé/échec) mais
     // le lien est considéré comme traité
-    //if (back[p].soc<0)  // erreur
-    //  return -1;
 
     return 0;
   } else {
@@ -2246,40 +2237,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 }
 
 #if HTS_XGETHOST
-// attendre que le host (ou celui du proxy) ait été résolu
+// Resolution is synchronous inside the connect path; no pre-resolve step, so
-// si c'est un fichier, la résolution est immédiate
+// the host is always immediately ready.
-// idem pour ftp://
+int host_wait(httrackp *opt, lien_back *back) { return 1; }
-void back_solve(httrackp * opt, lien_back * back) {
-  assertf(opt != NULL);
-  assertf(back != NULL);
-  if ((!strfield(back->url_adr, "file://"))
-      && !strfield(back->url_adr, "ftp://")
-    ) {
-    const char *a;
-
-    if (!(back->r.req.proxy.active))
-      a = back->url_adr;
-    else
-      a = back->r.req.proxy.name;
-    assertf(a != NULL);
-    a = jump_protocol_const(a);
-    if (check_hostname_dns(a)) {
-      hts_log_print(opt, LOG_DEBUG, "resolved: %s", a);
-    } else {
-      hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
-    }
-    //if (hts_dnstest(opt, a, 1) == 2) { // non encore testé!..
-    //  hts_log_print(opt, LOG_DEBUG, "resolving in background: %s", a);
-    //}
-  }
-}
-
-// détermine si le host a pu être résolu
-int host_wait(httrackp * opt, lien_back * back) {
-  // Always synchronous. No more background DNS resolution
-  // (does not really improve performances)
-  return 1;
-}
 #endif
 
 // élimine les fichiers non html en backing (anticipation)
@@ -2319,11 +2279,6 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) {
         (void) back_flush_output(opt, cache, sback, i); // flush output buffers
         usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr,
                     back[i].url_fil);
-        //if (back[i].links_index >= 0) {
-        //  assertf(back[i].links_index < opt->hash->max_lien);
-        //  opt->hash->liens[back[i].links_index]->pass2 = -1;
-        //  // *back[i].pass2_ptr=-1;  // Done!
-        //}
         /* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
         {
           int index = hash_read(opt->hash, back[i].url_sav, NULL, HASH_STRUCT_FILENAME );       // lecture type 0 (sav)
@@ -2471,9 +2426,15 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
-  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
+  /* Time/size limit exceeded past grace: abort in-flight transfers so no wait
-     starves (#481). FTP slots stay, their thread owns the socket. */
+     loop starves (#481, #77). FTP slots stay, their thread owns the socket. */
   if (!back_checkmirror(opt)) {
+    const hts_mirror_limit limit = back_mirror_limit(opt);
+    const char *const reason =
+        (limit == HTS_MIRROR_LIMIT_SIZE) ? "size limit" : "time limit";
+    const char *const slotmsg = (limit == HTS_MIRROR_LIMIT_SIZE)
+                                    ? "Mirror Size Limit"
+                                    : "Mirror Time Out";
     int aborted = 0;
     unsigned int i;
 
@@ -2487,15 +2448,15 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
           back_delayed_discard(opt, &back[i]);
         back[i].r.statuscode = STATUSCODE_TIMEOUT;
-        strcpybuff(back[i].r.msg, "Mirror Time Out");
+        strcpybuff(back[i].r.msg, slotmsg);
         back[i].status = STATUS_READY;
         back_set_finished(sback, i);
         aborted++;
       }
     }
     if (aborted > 0)
-      hts_log_print(opt, LOG_WARNING,
+      hts_log_print(opt, LOG_WARNING, "%s reached, %d transfer(s) aborted",
-                    "time limit reached, %d transfer(s) aborted", aborted);
+                    reason, aborted);
     return;
   }
 
@@ -2507,9 +2468,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
     busy_state = busy_recv = 0;
 
-#if 0
-    check_rate(stat_timestart, opt->maxrate);   // vérifier taux de transfert
-#endif
     // inscrire les sockets actuelles, et rechercher l'ID la plus élevée
     FD_ZERO(&fds);
     FD_ZERO(&fds_c);
@@ -2519,8 +2477,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
     nfds = INVALID_SOCKET;
 
     max_c = 1;
-    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-      // for(i=0;i<back_max;i++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       // en cas de gestion du connect préemptif
@@ -2557,8 +2514,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         // poll pour la lecture sur les sockets
       if ((back[i].status > 0) && (back[i].status < 100)) {     // en réception http
 
-#if BDEBUG==1
+#if BDEBUG == 1
-        //printf("....socket in progress: %d\n",back[i].r.soc);
 #endif
         // non local et non ftp
         if (!back[i].r.is_file) {
@@ -2644,8 +2600,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       busy_recv = 0;
 
     // recevoir les données arrivées
-    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-      // for(i=0;i<back_max;i++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       if (back[i].status > 0) {
@@ -2785,7 +2740,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back[i].rateout_time = back[i].ka_time_start;
             }
             // envoyer header
-            //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
             HTS_STAT.stat_nrequests++;
             if (!back[i].head_request)
               http_sendhead(opt, opt->cookie, 0, back[i].send_too,
@@ -2852,7 +2806,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 #if HTS_XGETHOST
       else if (back[i].status == STATUS_WAIT_DNS) {     // attendre gethostbyname
 #if DEBUGDNS
-        //printf("status 101 for %s\n",back[i].url_adr);
 #endif
 
         if (!gestion_timeout)
@@ -2993,7 +2946,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                 strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "gz") == 0
                                 && strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "tgz") == 0
                               ) {
-                              if (create_back_tmpfile(opt, &back[i]) == 0) {
+                              if (create_back_tmpfile(opt, &back[i], "z") ==
+                                  0) {
                                 assertf(back[i].tmpfile != NULL);
                                 /* note: tmpfile is utf-8 */
                                 if ((back[i].r.out =
@@ -3006,6 +2960,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
+                              /* Re-fetch over an existing file (#77 follow-up):
+                                 move the good copy aside before truncating it
+                                 so an aborted transfer can restore it. url_sav
+                                 is still written normally (file list intact).
+                               */
+                              back[i].tmpfile = NULL;
+                              if (fexist_utf8(back[i].url_sav)) {
+                                if (create_back_tmpfile(opt, &back[i], "bak") !=
+                                        0 ||
+                                    RENAME(back[i].url_sav, back[i].tmpfile) !=
+                                        0) {
+                                  back[i].tmpfile = NULL;
+                                }
+                              }
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {
@@ -3125,8 +3093,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
           // Si réception chunk, tester si on est pas à la fin!
           if (back[i].status == 1) {
             if (back[i].is_chunk) {     // attendre prochain chunk
-              if (back[i].r.size == back[i].r.totalsize) {      // fin chunk!
+              if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
-                //printf("chunk end at %d\n",back[i].r.size);
                 back[i].status = STATUS_CHUNK_CR;       /* fetch ending CRLF */
                 if (back[i].chunk_adr != NULL) {
                   freet(back[i].chunk_adr);
@@ -3189,11 +3156,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back_finalize(opt, cache, sback, i);
             }
 
-            if (back[i].r.totalsize >= 0) {     // tester totalsize
+            if (back[i].r.totalsize >= 0) { // tester totalsize
-              //if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) {    // tester totalsize
               if (back[i].r.totalsize != back[i].r.size) {      // pas la même!
                 if (!opt->tolerant) {
-                  //#if HTS_CL_IS_FATAL
                   deleteaddr(&back[i].r);
                   if (back[i].r.size < back[i].r.totalsize)
                     back[i].r.statuscode = STATUSCODE_CONNERROR;        // recatch
@@ -3202,14 +3167,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           " expected)", (LLint) back[i].r.size,
                           (LLint) back[i].r.totalsize);
                 } else {
-                  //#else
                   // Un warning suffira..
                   hts_log_print(opt, LOG_WARNING,
                                 "Incorrect length (" LLintP "!=" LLintP
-                                " expected) for %s%s", (LLint) back[i].r.size,
+                                " expected) for %s%s",
+                                (LLint) back[i].r.size,
                                 (LLint) back[i].r.totalsize, back[i].url_adr,
                                 back[i].url_fil);
-                  //#endif
                 }
               }
             }
@@ -3336,10 +3300,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                          back[i].r.size);
 #endif
                   /* End */
-                  //if (back[i].status==STATUS_CHUNK_CR) {
+                  back[i].status = STATUS_READY; // fin
-                  back[i].status = STATUS_READY;        // fin  
                   back_set_finished(sback, i);
-                  //}
 
                   // finalize transfer if not temporary
                   if (!IS_DELAYED_EXT(back[i].url_sav)) {
@@ -3550,6 +3512,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     continue;
                   }
 
+                  // The server really sent 304 here; the *-hacks below force
+                  // NOT_MODIFIED only after confirming the file is complete.
+                  const hts_boolean server_sent_304 =
+                      (back[i].r.statuscode == HTTP_NOT_MODIFIED);
+
                   /*
                      Solve "false" 416 problems
                    */
@@ -3578,22 +3545,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.statuscode == 406) {    // 'Not Acceptable'
                     back[i].r.statuscode = HTTP_OK;
                   }
-                  // 'do not erase already downloaded file'
+                  // On update, keep the good copy on error (mask as 304); skip
-                  // on an updated file
+                  // resume paths so a stale-partial 416 still re-fetches.
-                  // with an error : consider a 304 error
+                  if (HTTP_IS_ERROR(back[i].r.statuscode) &&
-                  if (!opt->delete_old) {
+                      back[i].is_update && !back[i].testmode &&
-                    if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
+                      back[i].range_req_size == 0 && back[i].url_sav[0] &&
-                        && !back[i].testmode) {
+                      fexist_utf8(back[i].url_sav)) {
-                      if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
+                    hts_log_print(opt, LOG_DEBUG,
-                        hts_log_print(opt, LOG_DEBUG,
+                                  "Error %d (%s) ignored on update, keeping "
-                                      "Error ignored %d (%s) because of 'no purge' option for %s%s",
+                                  "existing copy: %s%s",
-                                      back[i].r.statuscode, back[i].r.msg,
+                                  back[i].r.statuscode, back[i].r.msg,
-                                      back[i].url_adr, back[i].url_fil);
+                                  back[i].url_adr, back[i].url_fil);
-                        back[i].r.statuscode = HTTP_NOT_MODIFIED;
+                    back[i].r.statuscode = HTTP_NOT_MODIFIED;
-                        deletehttp(&back[i].r);
+                    deletehttp(&back[i].r);
-                        back[i].r.soc = INVALID_SOCKET;
+                    back[i].r.soc = INVALID_SOCKET;
-                      }
-                    }
                   }
                   // Various hacks to limit re-transfers when updating a mirror
                   // Force update if same size detected
@@ -3646,7 +3611,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) {       // 'OK'
                         if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) {  // not HTML
                           if (strnotempty(back[i].url_sav)) {   // target found
-                            int size = fsize_utf8(back[i].url_sav);     // target size
+                            off_t size = fsize_utf8(back[i].url_sav);
 
                             if (size >= 0) {
                               if (back[i].r.totalsize == size) {        // same size!
@@ -3741,6 +3706,21 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     }
                   }
 
+                  // Out-of-protocol 304 to a Range resume: the file is still
+                  // partial, so drop it and refetch instead of trusting it.
+                  if (server_sent_304 && back[i].range_req_size > 0) {
+                    url_savename_refname_remove(opt, back[i].url_adr,
+                                                back[i].url_fil);
+                    UNLINK(back[i].url_sav);
+                    deletehttp(&back[i].r);
+                    back[i].r.soc = INVALID_SOCKET;
+                    back[i].r.statuscode = STATUSCODE_NON_FATAL;
+                    strcpybuff(back[i].r.msg,
+                               "Bogus 304 on resume, restarting");
+                    back[i].status = STATUS_READY;
+                    back_set_finished(sback, i);
+                  }
+
                   /* sinon, continuer */
                   /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok récupérer body? */
                   // head: terminé
@@ -3806,7 +3786,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                     back[i].url_adr, back[i].url_fil);
 
                       // finalize
-                      //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);        // not modified
                       if (back[i].r.statuscode > 0) {
                         hts_log_print(opt, LOG_TRACE, "finalizing after cache load");
                         back_finalize(opt, cache, sback, i);
@@ -3816,37 +3795,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                              back[i].url_adr, back[i].url_fil);
 #endif
 
-                      //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                     } else {    // erreur
                       back[i].status = STATUS_READY;    // terminé
                       back_set_finished(sback, i);
-                      //printf("erreur cache\n");
-
                     }
 
-/********** NO - must complete the body! ********** */
-#if 0
-                  } else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
-                             || (back[i].r.statuscode == 412)
-                             || (back[i].r.statuscode == 416)
-                    ) {         // Ne pas prendre le html, erreurs connues et gérées
-#if HTS_DEBUG_CLOSESOCK
-                    DEBUG_W
-                      ("back_wait(301,302,303,307,412,416..): deletehttp\n");
-#endif
-                    // Couper connexion
-                    /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
-                    back_maydeletehttp(opt, cache, sback, i);
-
-                    back[i].status = STATUS_READY;      // terminé
-                    back_set_finished(sback, i);
-                    // finalize
-                    if (back[i].r.statuscode > 0) {
-                      hts_log_print(opt, LOG_TRACE, "finalizing redirect & 4xx");
-                      back_finalize(opt, cache, sback, i);
-                    }
-#endif
-/********** **************************** ********** */
                   }
                   // MIME type excluded by a -mime: filter: abort, don't fetch
                   // the body (#58)
@@ -3886,7 +3839,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       const hts_boolean range_ok =
                           back[i].r.crange > 0 && resume >= 0 &&
                           resume <= (LLint) sz &&
-                          back[i].r.crange_end + 1 == back[i].r.crange &&
+                          back[i].r.crange_end == back[i].r.crange - 1 &&
                           (back[i].r.totalsize < 0 ||
                            back[i].r.totalsize ==
                                back[i].r.crange_end - resume + 1);
@@ -3955,11 +3908,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           if (fp) {
                             LLint alloc_mem = resume + 1;
 
-                            if (back[i].r.totalsize >= 0)
+                            // Bound the in-memory buffer to a 32-bit size (real
+                            // in-RAM resources are far smaller); a hostile
+                            // Content-Length that would overflow the add or the
+                            // (size_t) cast is dropped and refetched instead.
+                            if (back[i].r.totalsize > INT32_MAX - alloc_mem) {
+                              url_savename_refname_remove(opt, back[i].url_adr,
+                                                          back[i].url_fil);
+                              UNLINK(back[i].url_sav);
+                              alloc_mem = -1;
+                            } else if (back[i].r.totalsize >= 0)
                               alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
-                            if (deleteaddr(&back[i].r)
+                            if (alloc_mem >= 0 && deleteaddr(&back[i].r) &&
-                                && (back[i].r.adr =
+                                (back[i].r.adr =
-                                    (char *) malloct((size_t) alloc_mem))) {
+                                     (char *) malloct((size_t) alloc_mem))) {
                               back[i].r.size = resume;
                               if (back[i].r.totalsize >= 0)
                                 back[i].r.totalsize += resume; // -> full size
@@ -4020,7 +3982,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           deletehttp(&back[i].r);
                         }
                         back[i].r.soc = INVALID_SOCKET;
-                        //back[i].r.statuscode=206;  ????????
                         back[i].r.statuscode = STATUSCODE_NON_FATAL;
                         if (strnotempty(back[i].r.msg))
                           strcpybuff(back[i].r.msg,
@@ -4047,8 +4008,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                             hts_log_print(opt, LOG_TRACE, "finalizing empty");
                             back_finalize(opt, cache, sback, i);
                           }
-                        } else if (!back[i].r.is_chunk) {       // pas de chunk
+                        } else if (!back[i].r.is_chunk) { // pas de chunk
-                          //if (back[i].r.http11!=2) {    // pas de chunk
                           back[i].is_chunk = 0;
                           back[i].status = 1;   // start body
                         } else {
@@ -4109,14 +4069,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].status < 0) {
           if (!back[i].testmode) {      // pas en test
             UNLINK(back[i].url_sav);    // éliminer fichier (endommagé)
-            //printf("&& %s\n",back[i].url_sav);
           }
         }
 #endif
 
         /* funny log for commandline users */
-        //if (!opt->quiet) {  
-        // petite animation
         if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
           if (back[i].status == STATUS_READY) {
             if (back[i].r.statuscode == HTTP_OK)
@@ -4129,18 +4086,16 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
             fflush(stdout);
           }
         }
-        //}
 
       }                         // status>0
-    }                           // for
+    } // for
 
     // vérifier timeouts
     if (gestion_timeout) {
       TStamp act;
 
       act = time_local();       // temps en secondes
-      for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+      for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-        // for(i=0;i<back_max;i++) {
         unsigned int i = (i_mod + mod_random) % (back_max);
 
         if (back[i].status > 0) {       // réception/connexion/..
@@ -4168,7 +4123,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 continue;
               }
             }
-            //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
             if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) {
               hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr,
                 back[i].url_fil);
@@ -4249,38 +4203,46 @@ static int back_maxtime_grace(const int maxtime) {
   return maximum(5, minimum(30, maxtime / 10));
 }
 
-int back_checkmirror(httrackp * opt) {
+/* Bytes the smooth stop may overrun before in-flight transfers are aborted.
-  // Check max size
+   No floor (unlike the time grace): a size overrun should abort promptly. */
-  if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
+static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
-    if (!opt->state.stop) {     /* not yet stopped */
+
-      hts_log_print(opt, LOG_ERROR,
+/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481).
-                    "More than " LLintP
+   -M measures received volume (HTS_TOTAL_RECV), not saved 200-only stat_bytes
-                    " bytes have been transferred.. giving up",
+   which undercounts redirect/error-heavy crawls (#520). */
-                    (LLint) opt->maxsite);
+static hts_mirror_limit back_mirror_limit(httrackp *opt) {
-      /* cancel mirror smoothly */
+  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
-      hts_request_stop(opt, 0);
+      HTS_STAT.HTS_TOTAL_RECV - opt->maxsite >=
-    }
+          back_maxsize_grace(opt->maxsite))
-    return 1;                   /* don'k break mirror too sharply for size limits, but stop requested */
+    return HTS_MIRROR_LIMIT_SIZE;
-    /*return 0;
-     */
-  }
-  // Check max time
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
 
-    if (elapsed >= opt->maxtime) {
+    if (elapsed >= opt->maxtime &&
-      if (!opt->state.stop) { /* not yet stopped */
+        elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
-        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+      return HTS_MIRROR_LIMIT_TIME;
-                      opt->maxtime);
-        /* cancel mirror smoothly */
-        hts_request_stop(opt, 0);
-      }
-      /* smooth stop starved past the grace period: stop waiting (#481) */
-      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
-        return 0;
-    }
   }
-  return 1;                     /* Ok, go on */
+  return HTS_MIRROR_LIMIT_NONE;
+}
+
+int back_checkmirror(httrackp *opt) {
+  /* request a smooth stop the first time each cap is reached */
+  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
+      !opt->state.stop) {
+    hts_log_print(opt, LOG_ERROR,
+                  "More than " LLintP
+                  " bytes have been transferred.. giving up",
+                  (LLint) opt->maxsite);
+    hts_request_stop(opt, 0);
+  }
+  if (opt->maxtime > 0 && !opt->state.stop &&
+      (time_local() - HTS_STAT.stat_timestart) >= opt->maxtime) {
+    hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                  opt->maxtime);
+    hts_request_stop(opt, 0);
+  }
+  /* hard stop once a cap overruns its grace (callers must stop waiting) */
+  return back_mirror_limit(opt) == HTS_MIRROR_LIMIT_NONE;
 }
 
 // octets transférés + add

src/htsback.h

@@ -138,12 +138,11 @@ LLint back_transferred(LLint add, struct_back * sback);
 
 // hostback
 #if HTS_XGETHOST
-void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
-/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
+/* Enforce -M/-E quotas: smooth-stops when reached; returns 0 once the -M cap
-   the -E deadline overran its grace period (callers must stop waiting). */
+   or -E deadline overruns its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htsbasenet.h

@@ -55,8 +55,6 @@ Please visit our Website: http://www.httrack.com
 // inline function in Wspiapi.h.
 #include <ws2tcpip.h>
 #include <Wspiapi.h>
-// #include <winsock2.h>
-// #include <tpipv6.h>
 
 #endif
 

src/htsbauth.c

@@ -107,7 +107,6 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
 #if DEBUG_COOK
   printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
          cook_name, cook_value, domain, path);
-  //printf(">>>cook: %s<<<\n",cookie->data);
 #endif
   return 0;
 }
@@ -208,8 +207,6 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   char catbuff[CATBUFF_SIZE];
   char buffer[8192];
 
-  //  cookie->data[0]='\0';
-
   // Fusionner d'abord les éventuels cookies IE
 #ifdef _WIN32
   {
@@ -310,14 +307,25 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   return -1;
 }
 
-// écrire cookies.txt
+/* Write cookies.txt; returns 0 on success. The jar holds live session
-// !=0 : erreur
+   cookies, so keep it owner-only on Unix (Windows inherits folder ACLs). */
 int cookie_save(t_cookie * cookie, const char *name) {
   char catbuff[CATBUFF_SIZE];
 
   if (strnotempty(cookie->data)) {
     char BIGSTK line[8192];
+#ifdef _WIN32
     FILE *fp = fopen(fconv(catbuff, sizeof(catbuff), name), "wb");
+#else
+    const int fd = open(fconv(catbuff, sizeof(catbuff), name),
+                        O_WRONLY | O_CREAT | O_TRUNC, HTS_PROTECT_FILE);
+    FILE *fp = fd != -1 ? fdopen(fd, "wb") : NULL;
+
+    if (fd != -1 && fp == NULL)
+      close(fd);    /* fdopen failed: don't leak the descriptor */
+    if (fp != NULL) /* O_CREAT's mode skips pre-existing jars: tighten those */
+      (void) fchmod(fd, HTS_PROTECT_FILE);
+#endif
 
     if (fp) {
       char *a = cookie->data;

src/htscache.h

@@ -89,14 +89,11 @@ typedef enum {
    the involved files are absent. */
 void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
 
-int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
-                    const char *str2, char *outbuff, int len);
-int cache_readdata(cache_back * cache, const char *str1, const char *str2,
-                   char **inbuff, int *len);
-
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);
+/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
+int cache_binput(char *adr, const char *end, char *s, int max);
 int cache_brint(char *adr, int *i);
 void cache_rint(FILE * fp, int *i);
 void cache_rLLint(FILE * fp, LLint * i);

src/htscache_selftest.c

@@ -78,14 +78,6 @@ static void selftest_open_for_read(cache_back *cache, httrackp *opt) {
 }
 
 static void selftest_close(cache_back *cache) {
-  if (cache->dat != NULL) {
-    fclose(cache->dat);
-    cache->dat = NULL;
-  }
-  if (cache->ndx != NULL) {
-    fclose(cache->ndx);
-    cache->ndx = NULL;
-  }
   if (cache->zipOutput != NULL) {
     zipClose(cache->zipOutput,
              "Created by HTTrack Website Copier (cache self-test)");
@@ -989,26 +981,15 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
 
-  /* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
+  /* PROMOTE: the removed pre-3.31 legacy pair is left alone */
-     zip cache existed) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", TINY);
   hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "promote-dat");
-
+  failures += reconcile_expect(opt, "hts-cache/old.ndx", TINY, "promote-dat");
-  /* PROMOTE: a half-written legacy new pair is replaced by the old pair */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.dat", TINY);
-  reconcile_put(opt, "hts-cache/old.dat", SOLID);
-  reconcile_put(opt, "hts-cache/old.ndx", TINY);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
 
   /* INTERRUPTED: no lock file, no action */
   reconcile_wipe(opt);
@@ -1058,7 +1039,7 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
 
-  /* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
+  /* INTERRUPTED: the removed pre-3.31 legacy pair is left alone */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-in_progress.lock", 0);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
@@ -1067,9 +1048,13 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
   failures +=
-      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.dat", TINY, "interrupted-dat");
   failures +=
-      reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.dat", SOLID, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.ndx", MID, "interrupted-dat");
 
   /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
      its only good generation here) */
@@ -1089,17 +1074,18 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
 
-  /* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
+  /* ROLLBACK: sidecars restored, the removed pre-3.31 pair left alone */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
-  reconcile_put(opt, "hts-cache/new.ndx", TINY);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   reconcile_put(opt, "hts-cache/old.lst", MID);
   reconcile_put(opt, "hts-cache/old.txt", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", TINY, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.ndx", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
 
@@ -1113,6 +1099,101 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   return failures;
 }
 
+/* The pre-3.31 .dat/.ndx import was removed: cache_init must refuse the
+   legacy pair, leave the files in place, and not flag an update run. */
+int cache_legacy_refused_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+  int variant;
+
+  selftest_tag = "cache-legacy";
+  golden_setup(opt, dir);
+#ifdef _WIN32
+  mkdir(reconcile_st_path(opt, "hts-cache"));
+#else
+  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
+#endif
+
+  /* variant 0: old.dat/old.ndx (--update layout); 1: new.dat/new.ndx (ro) */
+  for (variant = 0; variant < 2; variant++) {
+    cache_back cache;
+    const char *const dat =
+        variant == 0 ? "hts-cache/old.dat" : "hts-cache/new.dat";
+    const char *const ndx =
+        variant == 0 ? "hts-cache/old.ndx" : "hts-cache/new.ndx";
+
+    reconcile_wipe(opt);
+    reconcile_put(opt, dat, 4096);
+    reconcile_put(opt, ndx, 4096);
+    opt->is_update = 0;
+    selftest_open(&cache, opt, /*ro*/ variant == 1);
+    if (cache_readable(&cache)) {
+      printf("cache-legacy: variant %d imported a removed format\n", variant);
+      failures++;
+    }
+    if (coucal_nitems(cache.hashtable) != 0) { /* no phantom index entries */
+      printf("cache-legacy: variant %d imported index entries\n", variant);
+      failures++;
+    }
+    if (opt->is_update) {
+      printf("cache-legacy: variant %d flagged an update\n", variant);
+      failures++;
+    }
+    if (fsize(reconcile_st_path(opt, dat)) != 4096 ||
+        fsize(reconcile_st_path(opt, ndx)) != 4096) {
+      printf("cache-legacy: variant %d touched the legacy files\n", variant);
+      failures++;
+    }
+    if (cache.lst != NULL) {
+      fclose(cache.lst);
+      cache.lst = NULL;
+    }
+    if (cache.txt != NULL) {
+      fclose(cache.txt);
+      cache.txt = NULL;
+    }
+    selftest_close(&cache);
+  }
+
+  /* a healthy zip must win over stray legacy files, with no refusal */
+  {
+    cache_back cache;
+    const char *const body = "<html>zip wins</html>";
+
+    reconcile_wipe(opt);
+    selftest_open_for_write(&cache, opt);
+    store_entry(opt, &cache, "example.com", "/", "example.com/index.html", 200,
+                "OK", "text/html", "utf-8", "", "", "", "", body, strlen(body));
+    selftest_close(&cache);
+    reconcile_put(opt, "hts-cache/old.ndx", 4096);
+    reconcile_put(opt, "hts-cache/old.dat", 4096);
+    selftest_open_for_read(&cache, opt);
+    if (!cache_readable(&cache)) {
+      printf("cache-legacy: stray legacy files shadowed the zip cache\n");
+      failures++;
+    }
+    failures +=
+        check_entry(opt, &cache, "example.com", "/", 200, "OK", "text/html",
+                    "utf-8", "", "", "", "", body, strlen(body));
+    if (fsize(reconcile_st_path(opt, "hts-cache/old.ndx")) != 4096 ||
+        fsize(reconcile_st_path(opt, "hts-cache/old.dat")) != 4096) {
+      printf("cache-legacy: zip-wins pass touched the legacy files\n");
+      failures++;
+    }
+    if (cache.lst != NULL) {
+      fclose(cache.lst);
+      cache.lst = NULL;
+    }
+    if (cache.txt != NULL) {
+      fclose(cache.txt);
+      cache.txt = NULL;
+    }
+    selftest_close(&cache);
+  }
+
+  reconcile_wipe(opt);
+  return failures;
+}
+
 /* --- read-side corruption injection --------------------------------------- */
 
 /* canary read back intact after each corruption; victim gets the byte surgery

src/htscache_selftest.h

@@ -60,6 +60,10 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir);
    under <dir>. Returns the failed-check count. */
 int cache_reconcile_selftest(httrackp *opt, const char *dir);
 
+/* Verify cache_init refuses a pre-3.31 .dat/.ndx cache without touching it.
+   Returns the number of failed checks (0 = pass). */
+int cache_legacy_refused_selftest(httrackp *opt, const char *dir);
+
 /* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
    under <dir> and assert every case degrades to STATUSCODE_INVALID without
    tainting a sibling entry. */

src/htscatchurl.c

@@ -197,7 +197,6 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
             htsblk blkretour;
 
             hts_init_htsblk(&blkretour);
-            //memset(&blkretour, 0, sizeof(htsblk));    // effacer
             blkretour.location = loc;   // si non nul, contiendra l'adresse véritable en cas de moved xx
             // Lire en têtes restants
             sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
@@ -207,8 +206,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
               strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
               strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
             }
-            // CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
+            // CR/LF final de l'en tête inutile car déja placé via la ligne vide
-            //strcatbuff(data,"\r\n");
+            // juste au dessus
             if (blkretour.totalsize > 0) {
               int len = (int) min(blkretour.totalsize, 32000);
               int pos = (int) strlen(data);

src/htscharset.c

@@ -84,7 +84,7 @@ static int hts_equalsAlphanum(const char *a, const char *b) {
 
 /* Copy the memory region [s .. s + size - 1 ] as a \0-terminated string. */
 static char *hts_stringMemCopy(const char *s, size_t size) {
-  char *dest = malloc(size + 1);
+  char *dest = malloct(size + 1);
 
   if (dest != NULL) {
     memcpy(dest, s, size);
@@ -549,42 +549,6 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
 
 #endif
 
-HTS_STATIC char *hts_getCharsetFromContentType(const char *mime) {
-  /* text/html; charset=utf-8 */
-  const char *const charset = "charset";
-  char *pos = strstr(mime, charset);
-
-  if (pos != NULL) {
-    /* Skip spaces */
-    int eq = 0;
-
-    for(pos += strlen(charset);
-        *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\''; pos++) {
-      if (*pos == '=') {
-        eq = 1;
-      }
-    }
-    if (eq == 1) {
-      int len;
-
-      for(len = 0;
-          pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'';
-          pos++) ;
-      if (len != 0) {
-        char *const s = malloc(len + 1);
-        int i;
-
-        for(i = 0; i < len; i++) {
-          s[i] = pos[i];
-        }
-        s[len] = '\0';
-        return s;
-      }
-    }
-  }
-  return NULL;
-}
-
 #ifdef _WIN32
 #define strcasecmp(a,b) stricmp(a,b)
 #define strncasecmp(a,b,n) strnicmp(a,b,n)
@@ -644,58 +608,106 @@ int hts_isCharsetUTF8(const char *charset) {
          || strcasecmp(charset, "utf8") == 0 );
 }
 
+/* Extract X from a "text/html; charset=X" content= attribute value. */
+static char *charset_from_content(const char *s, size_t len) {
+  size_t i;
+
+  for (i = 0; i + 7 < len; i++) {
+    if ((i == 0 || is_space(s[i - 1]) || s[i - 1] == ';') &&
+        strncasecmp(&s[i], "charset", 7) == 0 && is_space_or_equal(s[i + 7])) {
+      size_t j, val;
+
+      for (j = i + 7; j < len && is_space_or_equal_or_quote(s[j]); j++)
+        ;
+      for (val = j; j < len && s[j] != '"' && s[j] != '\'' && s[j] != ';' &&
+                    !is_space(s[j]);
+           j++)
+        ;
+      if (j != val) {
+        return hts_stringMemCopy(&s[val], j - val);
+      }
+    }
+  }
+  return NULL;
+}
+
 char *hts_getCharsetFromMeta(const char *html, size_t size) {
-  int i;
+  size_t i;
 
-  /* <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8" > */
+  /* HTML5 <meta charset=X> or legacy
-  for(i = 0; i < size; i++) {
+     <meta http-equiv="Content-Type" content="text/html; charset=X">,
-    if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0
+     attributes in any order; first resolvable tag wins. */
-        && is_space(html[i + 5])) {
+  for (i = 0; i + 6 < size; i++) {
-      /* Skip spaces */
+    size_t j;
-      for(i += 5; is_space(html[i]); i++) ;
+    const char *equiv = NULL, *content = NULL;
-      if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0
+    size_t equiv_len = 0, content_len = 0;
-          && is_space_or_equal(html[i + 10])) {
-        for(i += 10; is_space_or_equal_or_quote(html[i]); i++) ;
-        if (strncasecmp(&html[i], "CONTENT-TYPE", 12) == 0) {
-          for(i += 12; is_space_or_equal_or_quote(html[i]); i++) ;
-          if (strncasecmp(&html[i], "CONTENT", 7) == 0
-              && is_space_or_equal(html[i + 7])) {
-            for(i += 7; is_space_or_equal_or_quote(html[i]); i++) ;
-            /* Skip content-type */
-            for(;
-                i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'';
-                i++) ;
-            /* Expect charset attribute here */
-            if (html[i] == ';') {
-              for(i++; is_space(html[i]); i++) ;
-              /* Look for charset */
-              if (strncasecmp(&html[i], "charset", 7) == 0
-                  && is_space_or_equal(html[i + 7])) {
-                int len;
 
-                for(i += 7; is_space_or_equal(html[i]) || html[i] == '\'';
+    if (html[i] != '<' || strncasecmp(&html[i + 1], "meta", 4) != 0 ||
-                    i++) ;
+        !is_space(html[i + 5])) {
-                /* Charset */
+      continue;
-                for(len = 0;
+    }
-                    i + len < size && html[i + len] != '"'
+    /* Attribute scan, strictly bounded by size. */
-                    && html[i + len] != '\'' && html[i + len] != ' '; len++) ;
+    for (j = i + 6; j < size && html[j] != '>';) {
-                /* No error ? */
+      size_t name, name_len, val = 0, val_len = 0;
-                if (len != 0 && i < size) {
-                  char *const s = malloc(len + 1);
-                  int j;
 
-                  for(j = 0; j < len; j++) {
+      if (is_space(html[j]) || html[j] == '/') {
-                    s[j] = html[i + j];
+        j++;
-                  }
+        continue;
-                  s[len] = '\0';
+      }
-                  return s;
+      for (name = j; j < size && html[j] != '=' && html[j] != '>' &&
-                }
+                     html[j] != '/' && !is_space(html[j]);
-              }
+           j++)
-            }
+        ;
+      name_len = j - name;
+      for (; j < size && is_space(html[j]); j++)
+        ;
+      if (j < size && html[j] == '=') {
+        for (j++; j < size && is_space(html[j]); j++)
+          ;
+        if (j < size && (html[j] == '"' || html[j] == '\'')) {
+          const char quote = html[j++];
+
+          for (val = j; j < size && html[j] != quote; j++)
+            ;
+          if (j >= size) /* unterminated quote: drop the tag */
+            break;
+          val_len = j++ - val;
+        } else {
+          /* unquoted: ends at whitespace or '>' only (HTML5 prescan); '/'
+             belongs to the value except as the tail of a self-close */
+          for (val = j; j < size && !is_space(html[j]) && html[j] != '>'; j++)
+            ;
+          val_len = j - val;
+          if (val_len != 0 && j < size && html[j] == '>' &&
+              html[val + val_len - 1] == '/') {
+            val_len--;
           }
         }
       }
+      /* first occurrence of each attribute wins, as in browsers */
+      if (val_len != 0) {
+        if (name_len == 7 && strncasecmp(&html[name], "charset", 7) == 0) {
+          return hts_stringMemCopy(&html[val], val_len);
+        } else if (equiv == NULL && name_len == 10 &&
+                   strncasecmp(&html[name], "http-equiv", 10) == 0) {
+          equiv = &html[val];
+          equiv_len = val_len;
+        } else if (content == NULL && name_len == 7 &&
+                   strncasecmp(&html[name], "content", 7) == 0) {
+          content = &html[val];
+          content_len = val_len;
+        }
+      }
     }
+    if (equiv_len == 12 && strncasecmp(equiv, "content-type", 12) == 0 &&
+        content != NULL) {
+      char *const s = charset_from_content(content, content_len);
+
+      if (s != NULL) {
+        return s;
+      }
+    }
+    i = j;
   }
   return NULL;
 }
@@ -841,71 +853,6 @@ static unsigned int nlz8(unsigned char x) {
     }                                           \
   } while(0)
 
-/* Sample. */
-#if 0
-
-int main(int argc, char **argv) {
-  int i;
-  int hex = 0;
-  
-#define READ_INT(DEST)                                  \
-  ( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1)      \
-    || (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
-  
-  for(i = 1 ; i < argc ; i++) {
-    unsigned int uc, from, to;
-    
-    if (strcmp(argv[i], "--hex") == 0) {
-      hex = 1;
-    }
-    else if (strcmp(argv[i], "--decimal") == 0) {
-      hex = 0;
-    }
-    else if (strcmp(argv[i], "--decode") == 0) {
-#define RD fgetc_unlocked(stdin)
-#define WR(C) do {                              \
-        if (C != -1) {                          \
-          printf("%04x\n", C);                  \
-        } else if (!feof(stdin)) {              \
-          fprintf(stderr, "read error\n");      \
-          exit(EXIT_FAILURE);                   \
-        }                                       \
-      } while(0)
-      while(!feof(stdin)) {
-        READ_UNICODE(RD, WR);
-      }
-#undef RD
-#undef WR
-    }
-    else if (strcmp(argv[i], "-range") == 0
-             && i + 2 < argc
-             && (++i, 1)
-             && READ_INT(from)
-             && (++i, 1)
-             && READ_INT(to)
-             ) {
-      unsigned int i;
-      for(i = from ; i < to ; i++) {
-#define EM(C) fputc_unlocked(C, stdout)
-        EMIT_UNICODE(i, EM);
-#undef EM
-      }
-    }
-    else if (READ_INT(uc)) {
-#define EM(C) fputc_unlocked(C, stdout)
-      EMIT_UNICODE(uc, EM);
-#undef EM
-    }
-    else {
-      return EXIT_FAILURE;
-    }
-  }
-  
-  return EXIT_SUCCESS;
-}
-
-#endif
-
 /* IDNA helpers. */
 #undef ADD_BYTE
 #undef INCREASE_CAPA
@@ -975,19 +922,19 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
                 /* Reader: can read bytes up to j */
 #define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
 
-                /* Writer: upon error, return FFFD (replacement character) */
+                /* Writer: a malformed sequence abandons the encode (NULL) */
-#define WR(C) do { \
+#define WR(C)                                                                  \
-  if ((C) != -1) { \
+  do {                                                                         \
-    /* copy character */ \
+    if ((C) != -1) {                                                           \
-    assertf(segOutputSize < segSize); \
+      /* copy character */                                                     \
-    segInt[segOutputSize++] = (C); \
+      assertf(segOutputSize < segSize);                                        \
-  } \
+      segInt[segOutputSize++] = (C);                                           \
-  /* In case of error, abort. */ \
+    } else {                                                                   \
-  else { \
+      freet(segInt);                                                           \
-    FREE_BUFFER(); \
+      FREE_BUFFER();                                                           \
-    return NULL; \
+      return NULL;                                                             \
-  } \
+    }                                                                          \
-} while(0)
+  } while (0)
 
                 /* Read/Write Unicode character. */
                 READ_UNICODE(RD, WR);
@@ -1205,17 +1152,45 @@ int hts_isStringUTF8(const char *s, size_t size) {
   const unsigned char *const data = (const unsigned char*) s;
   size_t i;
 
+  /* Strict RFC 3629: reject overlongs, surrogates, > U+10FFFF, 5/6-byte. */
   for(i = 0 ; i < size ; ) {
-    /* Reader: can read bytes up to j */
+    const unsigned char c = data[i];
-#define RD ( i < size ? data[i++] : -1 )
+    size_t len, k;
+    hts_UCS4 uc, min;
 
-    /* Writer: upon error, return FFFD (replacement character) */
+    if (c < 0x80) {
-#define WR(C) if ((C) == -1) { return 0; }
+      i++;
+      continue;
+    } else if (c >= 0xc2 && c <= 0xdf) {
+      len = 2;
+      min = 0x80;
+      uc = c & 0x1f;
+    } else if (c >= 0xe0 && c <= 0xef) {
+      len = 3;
+      min = 0x800;
+      uc = c & 0x0f;
+    } else if (c >= 0xf0 && c <= 0xf4) {
+      len = 4;
+      min = 0x10000;
+      uc = c & 0x07;
+    } else { /* continuation byte, overlong C0/C1, or F5..FF lead */
+      return 0;
+    }
+    if (size - i < len) {
+      return 0;
+    }
+    for (k = 1; k < len; k++) {
+      const unsigned char cc = data[i + k];
 
-    /* Read Unicode character. */
+      if ((cc & 0xc0) != 0x80) {
-    READ_UNICODE(RD, WR);
+        return 0;
-#undef RD
+      }
-#undef WR
+      uc = (uc << 6) | (cc & 0x3f);
+    }
+    if (uc < min || uc > 0x10FFFF || (uc >= 0xD800 && uc <= 0xDFFF)) {
+      return 0;
+    }
+    i += len;
   }
 
   return 1;

src/htscharset.h

@@ -76,7 +76,8 @@ extern char *hts_convertStringIDNAToUTF8(const char *s, size_t size);
 extern int hts_isStringIDNA(const char *s, size_t size);
 
 /**
- * Extract the charset from the HTML buffer "html"
+ * Extract the <meta> charset from the HTML buffer "html" (HTML5 charset= or
+ * legacy http-equiv form). Returns a malloc'ed string, or NULL if none.
  **/
 extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 
@@ -86,7 +87,8 @@ extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 extern int hts_isStringAscii(const char *s, size_t size);
 
 /**
- * Is the given string an UTF-8 string ?
+ * Is the given string valid UTF-8 ? Strict RFC 3629: overlong forms,
+ * surrogates, codepoints above U+10FFFF and 5/6-byte sequences are rejected.
  **/
 extern int hts_isStringUTF8(const char *s, size_t size);
 

src/htsconfig.h

@@ -54,10 +54,6 @@ Please visit our Website: http://www.httrack.com
 // extension par défaut pour fichiers n'en ayant pas
 #define DEFAULT_EXT ".html"
 #define DEFAULT_EXT_SHORT ".htm"
-// #define DEFAULT_BIN_EXT   ".bin"
-// #define DEFAULT_BIN_EXT_SHORT ".bin"
-// #define DEFAULT_EXT       ".txt"
-// #define DEFAULT_EXT_SHORT ".txt"
 
 // éviter les /nul, /con..
 #define HTS_OVERRIDE_DOS_FOLDERS 1
@@ -80,13 +76,6 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
-// gérer une table de hachage?
-// REMOVED
-// #define HTS_HASH 1
-
-// fast cache (build hash table)
-#define HTS_FAST_CACHE 1
-
 // le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
 // valide)
 #define GT_ENDS_COMMENT 1
@@ -97,27 +86,10 @@ Please visit our Website: http://www.httrack.com
 // always transform a '//' into a sigle '/'
 #define HTS_STRIP_DOUBLE_SLASH 0
 
-// case-sensitive pour les dossiers et fichiers (0/1)
-// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est
-// pas très utile..
-// ..et pas bcp respecté]
-// REMOVED
-// #define HTS_CASSE 0
-
-// Un fichier ayant une taille différente du content-length doit il être annulé?
-// SEE opt.tolerant and opt.http10
-// #define HTS_CL_IS_FATAL 0
-
 // une erreur supprime le fichier sur disque
 // (non fixé pour cause de retry)
 #define HTS_REMOVE_BAD_FILES 0
 
-// en cas de Range: xx- donnant un Content-length: xx
-// alors skipper le fichier, considéré comme transmis
-// #define HTS_SKIP_FULL_RANGE 1
-
-// nombre max de filtres que l'utilisateur peut fixer
-// #define HTS_FILTERSMAX 10000
 #define HTS_FILTERSINC 1000
 
 // connect non bloquant? (poll sur write)

src/htscore.c

@@ -119,46 +119,77 @@ hts_log_print(opt, LOG_INFO, "engine: end"); \
 RUN_CALLBACK0(opt, end); \
 }
 
-#define XH_extuninit do { \
+#define XH_extuninit                                                           \
-  HTMLCHECK_UNINIT \
+  do {                                                                         \
-  hts_record_free(opt); \
+    HTMLCHECK_UNINIT                                                           \
-  if (filters && filters[0]) { \
+    hts_record_free(opt);                                                      \
-  freet(filters[0]); filters[0]=NULL; \
+    if (filters && filters[0]) {                                               \
-  } \
+      freet(filters[0]);                                                       \
-  if (filters) { \
+      filters[0] = NULL;                                                       \
-  freet(filters); filters=NULL; \
+    }                                                                          \
-  } \
+    if (filters) {                                                             \
-  back_delete_all(opt,&cache,sback); \
+      freet(filters);                                                          \
-  back_free(&sback); \
+      filters = NULL;                                                          \
-  checkrobots_free(&robots);\
+    }                                                                          \
-  if (cache.use) { freet(cache.use); cache.use=NULL; } \
+    back_delete_all(opt, &cache, sback);                                       \
-  if (cache.dat) { fclose(cache.dat); cache.dat=NULL; }  \
+    back_free(&sback);                                                         \
-  if (cache.ndx) { fclose(cache.ndx); cache.ndx=NULL; } \
+    checkrobots_free(&robots);                                                 \
-  if (cache.zipOutput) { \
+    if (cache.zipOutput) {                                                     \
-    zipClose(cache.zipOutput, "Created by HTTrack Website Copier/"HTTRACK_VERSION); \
+      zipClose(cache.zipOutput,                                                \
-    cache.zipOutput = NULL; \
+               "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \
-  } \
+      cache.zipOutput = NULL;                                                  \
-  if (cache.zipInput) { \
+    }                                                                          \
-    unzClose(cache.zipInput); \
+    if (cache.zipInput) {                                                      \
-    cache.zipInput = NULL; \
+      unzClose(cache.zipInput);                                                \
-  } \
+      cache.zipInput = NULL;                                                   \
-  if (cache.olddat) { fclose(cache.olddat); cache.olddat=NULL; } \
+    }                                                                          \
-  if (cache.lst) { fclose(cache.lst); cache.lst=opt->state.strc.lst=NULL; } \
+    if (cache.lst) {                                                           \
-  if (cache.txt) { fclose(cache.txt); cache.txt=NULL; } \
+      fclose(cache.lst);                                                       \
-  if (opt->log != NULL) fflush(opt->log); \
+      cache.lst = opt->state.strc.lst = NULL;                                  \
-  if (makestat_fp) { fclose(makestat_fp); makestat_fp=NULL; } \
+    }                                                                          \
-  if (maketrack_fp){ fclose(maketrack_fp); maketrack_fp=NULL; } \
+    if (cache.txt) {                                                           \
-  if (opt->accept_cookie) cookie_save(opt->cookie,fconcat(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),StringBuff(opt->path_log),"cookies.txt")); \
+      fclose(cache.txt);                                                       \
-  if (makeindex_fp) { fclose(makeindex_fp); makeindex_fp=NULL; } \
+      cache.txt = NULL;                                                        \
-  if (cache_hashtable) { coucal_delete(&cache_hashtable); } \
+    }                                                                          \
-  if (cache_tests)     { coucal_delete(&cache_tests); } \
+    if (opt->log != NULL)                                                      \
-  if (template_header) { freet(template_header); template_header=NULL; } \
+      fflush(opt->log);                                                        \
-  if (template_body)   { freet(template_body); template_body=NULL; } \
+    if (makestat_fp) {                                                         \
-  if (template_footer) { freet(template_footer); template_footer=NULL; } \
+      fclose(makestat_fp);                                                     \
-  hash_free(&hash); \
+      makestat_fp = NULL;                                                      \
-  clearCallbacks(&opt->state.callbacks); \
+    }                                                                          \
-  /*structcheck_init(-1);*/ \
+    if (maketrack_fp) {                                                        \
-} while(0)
+      fclose(maketrack_fp);                                                    \
+      maketrack_fp = NULL;                                                     \
+    }                                                                          \
+    if (opt->accept_cookie)                                                    \
+      cookie_save(opt->cookie,                                                 \
+                  fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),           \
+                          StringBuff(opt->path_log), "cookies.txt"));          \
+    if (makeindex_fp) {                                                        \
+      fclose(makeindex_fp);                                                    \
+      makeindex_fp = NULL;                                                     \
+    }                                                                          \
+    if (cache_hashtable) {                                                     \
+      coucal_delete(&cache_hashtable);                                         \
+    }                                                                          \
+    if (cache_tests) {                                                         \
+      coucal_delete(&cache_tests);                                             \
+    }                                                                          \
+    if (template_header) {                                                     \
+      freet(template_header);                                                  \
+      template_header = NULL;                                                  \
+    }                                                                          \
+    if (template_body) {                                                       \
+      freet(template_body);                                                    \
+      template_body = NULL;                                                    \
+    }                                                                          \
+    if (template_footer) {                                                     \
+      freet(template_footer);                                                  \
+      template_footer = NULL;                                                  \
+    }                                                                          \
+    hash_free(&hash);                                                          \
+    clearCallbacks(&opt->state.callbacks);                                     \
+  } while (0)
 #define XH_uninit do { XH_extuninit; if (r.adr) { freet(r.adr); r.adr=NULL; } } while(0)
 
 struct lien_buffers {
@@ -441,32 +472,11 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
   *makeindex_done = 1;
 }
 
-/* Flush the parsed HTML output buffer to disk, skipping the rewrite when the
+/* Flush the parsed HTML output buffer to disk. */
- * on-disk MD5 is unchanged. */
 void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
                           FILE **fp, const char *ht_buff, size_t ht_len,
                           const char *adr, const char *fil, const char *save) {
-  char digest[32 + 2];
+  {
-  off_t fsize_old =
-      fsize(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), save));
-  int ok = 0;
-
-  digest[0] = '\0';
-  domd5mem(ht_buff, ht_len, digest, 1);
-  if (fsize_old == (off_t) ht_len) {
-    int mlen = 0;
-    char *mbuff;
-
-    cache_readdata(cache, "//[HTML-MD5]//", save, &mbuff, &mlen);
-    if (mlen)
-      mbuff[mlen] = '\0';
-    if ((mlen == 32) && (strcmp(((mbuff != NULL) ? mbuff : ""), digest) == 0)) {
-      ok = 1;
-      hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s", save);
-    }
-    freet(mbuff);
-  }
-  if (!ok) {
     file_notify(opt, adr, fil, save, 1, 1, r->notmodified);
     *fp = filecreate(&opt->state.strc, save);
     if (*fp) {
@@ -498,13 +508,7 @@ void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
       if (fcheck)
         hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
     }
-  } else {
-    file_notify(opt, adr, fil, save, 0, 0, r->notmodified);
-    filenote(&opt->state.strc, save, NULL);
   }
-  if (cache->ndx)
-    cache_writedata(cache->ndx, cache->dat, "//[HTML-MD5]//", save, digest,
-                    (int) strlen(digest));
 }
 
 /* does it look like XML ? (SVG et al.) */
@@ -523,7 +527,6 @@ int httpmirror(char *url1, httrackp * opt) {
   hash_struct *const hashptr = &hash;
   t_cookie BIGSTK cookie;       // gestion des cookies
 
-  //char* tab_alloc=NULL;
   int ptr;                      // pointeur actuel sur les liens
 
   //
@@ -534,15 +537,12 @@ int httpmirror(char *url1, httrackp * opt) {
   // pour les stats, nombre de fichiers & octets écrits
   LLint stat_fragment = 0;      // pour la fragmentation
 
-  //TStamp istat_timestart;   // départ pour calcul instantanné
-  //
   TStamp last_info_shell = 0;
   int info_shell = 0;
 
   // filtres
   char **filters = NULL;
 
-  //int filter_max=0;
   int filptr = 0;
 
   //
@@ -582,7 +582,6 @@ int httpmirror(char *url1, httrackp * opt) {
   // noter heure actuelle de départ en secondes
   memset(&HTS_STAT, 0, sizeof(HTS_STAT));
   HTS_STAT.stat_timestart = time_local();
-  //istat_timestart=stat_timestart;
   HTS_STAT.istat_timestart[0] = HTS_STAT.istat_timestart[1] = mtime_local();
   /* reset stats */
   HTS_STAT.HTS_TOTAL_RECV = 0;
@@ -616,9 +615,6 @@ int httpmirror(char *url1, httrackp * opt) {
   // initialiser usercommand
   usercommand(opt, opt->sys_com_exec, StringBuff(opt->sys_com), "", "", "");
 
-  // initialiser structcheck
-  // structcheck_init(1);
-
   // initialiser verif_backblue
   verif_backblue(opt, NULL);
   verif_external(opt, 0, 0);
@@ -638,9 +634,6 @@ int httpmirror(char *url1, httrackp * opt) {
                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_bin),
                  "templates/index-footer.html"), HTS_INDEX_FOOTER);
 
-  // initialiser mimedefs
-  //get_userhttptype(opt,1,StringBuff(opt->mimedefs),NULL);
-
   // Initialiser indexation
   if (opt->kindex)
     index_init(StringBuff(opt->path_html));
@@ -684,7 +677,6 @@ int httpmirror(char *url1, httrackp * opt) {
   opt->filters.filters = &filters;
   //
   opt->filters.filptr = &filptr;
-  //opt->filters.filter_max=&filter_max;
 
   // hash table
   opt->hash = &hash;
@@ -731,8 +723,7 @@ int httpmirror(char *url1, httrackp * opt) {
       else if (*a == '-')
         joker = 1;
 
-      if (joker) {              // joker ou filters
+      if (joker) { // joker ou filters
-        //char* p;
         char BIGSTK tempo[HTS_URLMAXSIZE * 2];
         int type;
         int plus = 0;
@@ -787,12 +778,11 @@ int httpmirror(char *url1, httrackp * opt) {
               XH_extuninit;
               return 0;
             }
-            //opt->filters.filters=filters;
           }
 
         }
 
-      } else {                  // adresse normale
+      } else { // adresse normale
         char BIGSTK url[HTS_URLMAXSIZE * 2];
 
         // prochaine adresse
@@ -813,6 +803,52 @@ int httpmirror(char *url1, httrackp * opt) {
       }
     }                           // while
 
+    /* --why: print which filter rule decides for this URL, then stop */
+    if (StringNotEmpty(opt->why_url)) {
+      char BIGSTK url[HTS_URLMAXSIZE * 2];
+      lien_adrfil af;
+
+      if (strstr(StringBuff(opt->why_url), ":/") == NULL)
+        snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
+      else
+        strcpybuff(url, StringBuff(opt->why_url));
+      if (ident_url_absolute(url, &af) < 0) {
+        printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
+      } else {
+        char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
+        int jok, jokDepth = 0;
+
+        /* the two forms the wizard tests */
+        strcpybuff(l, jump_identification_const(af.adr));
+        if (*af.fil != '/')
+          strcatbuff(l, "/");
+        strcatbuff(l, af.fil);
+        if (!link_has_authority(af.adr))
+          strcpybuff(lfull, "http://");
+        else
+          lfull[0] = '\0';
+        strcatbuff(lfull, af.adr);
+        if (*af.fil != '/')
+          strcatbuff(lfull, "/");
+        strcatbuff(lfull, af.fil);
+        jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
+                               NULL, &jokDepth);
+        if (jok > 0)
+          printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
+                 filters[jokDepth]);
+        else if (jok < 0)
+          printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
+                 filters[jokDepth]);
+        else
+          printf("%s: no filter rule matches; the wizard decides "
+                 "(same-site links are followed by default)" LF,
+                 url);
+      }
+      freet(primary);
+      XH_extuninit;
+      return 1;
+    }
+
     /* load URL file list */
     /* OPTIMIZED for fast load */
     if (StringNotEmpty(opt->filelist)) {
@@ -867,7 +903,6 @@ int httpmirror(char *url1, httrackp * opt) {
             htsbuff_cat(&primarybuff, "\n");
           }
         }
-        // fclose(fp);
         hts_log_print(opt, LOG_NOTICE, "%d links added from %s", n,
                       StringBuff(opt->filelist));
 
@@ -922,7 +957,6 @@ int httpmirror(char *url1, httrackp * opt) {
 #endif
 
   // backing
-  //soc_max=opt->maxsoc;
   if (opt->maxsoc > 0) {
 #if BDEBUG==2
     _CLRSCR;
@@ -1236,15 +1270,10 @@ int httpmirror(char *url1, httrackp * opt) {
       // tester r.adr
       if (!error) {
         // erreur, pas de fichier chargé:
-        if ((!r.adr) && (r.is_write == 0)
+        if ((!r.adr) && (r.is_write == 0) && (r.statuscode != 301) &&
-            && (r.statuscode != 301)
+            (r.statuscode != 302) && (r.statuscode != 303) &&
-            && (r.statuscode != 302)
+            (r.statuscode != 307) && (r.statuscode != 412) &&
-            && (r.statuscode != 303)
+            (r.statuscode != 416)) {
-            && (r.statuscode != 307)
-            && (r.statuscode != 412)
-            && (r.statuscode != 416)
-          ) {
-          // error=1;
 
           // peut être que le fichier était trop gros?
           if ((istoobig
@@ -1326,30 +1355,8 @@ int httpmirror(char *url1, httrackp * opt) {
                                          ||may_be_hypertext_mime(opt, r.contenttype, urlfil()))   /* Is real media, .. */
           ) {
 
-          /* Convert charset to UTF-8 - NOT! (what about links ? remote server side will have troubles with converted names) */
+          /* Convert charset to UTF-8 - NOT! (what about links ? remote server
-          //if (r.adr != NULL && r.size != 0 && opt->convert_utf8) {
+           * side will have troubles with converted names) */
-          //  char *charset;
-          //  char *pos;
-          //  if (r.charset[0] != '\0') {
-          //    charset = strdup(r.charset);
-          //  } else {
-          //    charset = hts_getCharsetFromMeta(r.adr, r.size);
-          //  }
-          //  if (charset != NULL) {
-          //    char *const utf8 = hts_convertStringToUTF8(r.adr, r.size, charset);
-          //    /* Use new buffer */
-          //    if (utf8 != NULL) {
-          //      freet(r.adr);
-          //      r.size = strlen(utf8);
-          //      r.adr = utf8;
-          //      /* New UTF-8 charset */
-          //      r.charset[0] = '\0';
-          //      strcpy(r.charset, "utf-8");
-          //    }
-          //    /* Free charset */
-          //    free(charset);
-          //  }
-          //}
 
           /* Check bogus chars */
           if ((r.adr) && (r.size)) {
@@ -1490,50 +1497,6 @@ int httpmirror(char *url1, httrackp * opt) {
 
         }
       }
-      // MOVED IN back_finalize()
-      //
-      // -------------------- 
-      // REAL MEDIA HACK
-      // Check if we have to load locally the file
-      // --------------------
-      //if (!error) {
-      //  if (r.statuscode == HTTP_OK) {    // OK (ou 304 en backing)
-      //    if (r.adr==NULL) {    // Written file
-      //      if (may_be_hypertext_mime(r.contenttype, urlfil())) {   // to parse!
-      //        LLint sz;
-      //        sz=fsize_utf8(savename());
-      //        if (sz>0) {   // ok, exists!
-      //          if (sz < 8192) {   // ok, small file --> to parse!
-      //            FILE* fp=FOPEN(savename(),"rb");
-      //            if (fp) {
-      //              r.adr=malloct(sz + 1);
-      //              if (r.adr) {
-      //                if (fread(r.adr,1,sz,fp) == sz) {
-      //                  r.size=sz;
-      //                                                r.adr[sz] = '\0';
-      //                                                r.is_write = 0;
-      //                } else {
-      //                  freet(r.adr);
-      //                  r.size=0;
-      //                  r.adr = NULL;
-      //                  r.statuscode=STATUSCODE_INVALID;
-      //                  strcpybuff(r.msg, ".RAM read error");
-      //                }
-      //                fclose(fp);
-      //                fp=NULL;
-      //                // remove (temporary) file!
-      //                remove(savename());
-      //              }
-      //              if (fp)
-      //                fclose(fp);
-      //            }
-      //          }
-      //        }
-      //      }
-      //    }
-      //  }
-      //}
-      // EN OF REAL MEDIA HACK
 
       // ---stockage en cache---
       // stocker dans le cache?
@@ -1654,24 +1617,6 @@ int httpmirror(char *url1, httrackp * opt) {
       }
 #endif
 
-      /* info: updated */
-      /*
-         if (ptr>0) {
-         // "mis à jour"
-         if ((!r.notmodified) && (opt->is_update) && (!store_errpage)) {    // page modifiée
-         if (strnotempty(savename())) {
-         HTS_STAT.stat_updated_files++;
-         //if ((opt->debug>0) && (opt->log!=NULL)) {
-         hts_log_print(opt, LOG_INFO, "File updated: %s%s",urladr(),urlfil());
-         }
-         } else {
-         if (!store_errpage) {
-         hts_log_print(opt, LOG_INFO, "File recorded: %s%s",urladr(),urlfil());
-         }
-         }
-         }
-       */
-
       // ------------------------------------------------------
       // traitement (parsing)
       // ------------------------------------------------------
@@ -1717,13 +1662,8 @@ int httpmirror(char *url1, httrackp * opt) {
                 free(charset);
             }
             /* Could not detect charset: could it be UTF-8 ? */
-            /* No, we can not do that: browsers do not do it 
+            /* No, we can not do that: browsers do not do it
                (and it would break links). */
-            //if (page_charset[0] == '\0') {
-            //  if (is_unicode_utf8(r.adr, r.size)) {
-            //    strcpy(page_charset, "utf-8");
-            //  }
-            //}
             /* Could not detect charset */
             if (page_charset[0] == '\0') {
               hts_log_print(opt, LOG_INFO,
@@ -1801,10 +1741,6 @@ int httpmirror(char *url1, httrackp * opt) {
               XH_uninit;
               return -1;
             }
-
-            // I'll have to segment this part
-// #include "htsparse.c"
-
           }
         }
         // Fin parsing HTML
@@ -1900,7 +1836,6 @@ int httpmirror(char *url1, httrackp * opt) {
              HTS_STAT.stat_files++;
              HTS_STAT.stat_bytes+=r.size;
            */
-          //printf("ok......\n");
         } else {
           // Si on doit sauver une page HTML sans la scanner, cela signifie que le niveau de
           // récursion nous en empêche
@@ -1969,10 +1904,11 @@ int httpmirror(char *url1, httrackp * opt) {
             }
           }
 
-          //printf("extern=%s\n",r.contenttype);
+          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
-
+          // An empty body must not overwrite the file when the transfer failed
-          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!          
+          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
-          if (r.adr != NULL || r.size == 0) {
+          // copy to 0 (#77 follow-up).
+          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {
@@ -1993,7 +1929,6 @@ int httpmirror(char *url1, httrackp * opt) {
                */
             }
           }
-
         }
 
         /* Parsing of other media types (java, ram..) */
@@ -2092,16 +2027,9 @@ int httpmirror(char *url1, httrackp * opt) {
           // prochain pass2
           while((ptr < opt->lien_tot) ? (!heap(ptr)->pass2) : 0)
             ptr++;
-
-          //printf("first link==%d\n");
-
         }
       }
     }
-    // copy abort state if necessary from outside
-    //if (!exit_xh && opt->state.exit_xh) {
-    //  exit_xh=opt->state.exit_xh;
-    //}
     // a-t-on dépassé le quota?
     if (!back_checkmirror(opt)) {
       ptr = opt->lien_tot;
@@ -2258,13 +2186,11 @@ int httpmirror(char *url1, httrackp * opt) {
     char BIGSTK htstime[256];
     char BIGSTK infoupdated[256];
 
-    // int n=(int) (stat_loaded/(time_local()-HTS_STAT.stat_timestart));
     LLint n =
       (LLint) (HTS_STAT.HTS_TOTAL_RECV /
                (max(1, time_local() - HTS_STAT.stat_timestart)));
 
     sec2str(htstime, time_local() - HTS_STAT.stat_timestart);
-    //sprintf(finalInfo + strlen(finalInfo),LF"HTS-mirror complete in %s : %d links scanned, %d files written (%d bytes overall) [%d bytes received at %d bytes/sec]"LF,htstime,lien_tot-1,HTS_STAT.stat_files,stat_bytes,stat_loaded,n);
     infoupdated[0] = '\0';
     if (opt->is_update) {
       if (HTS_STAT.stat_updated_files > 0) {
@@ -2347,12 +2273,6 @@ int httpmirror(char *url1, httrackp * opt) {
 
 */
 int engine_stats(void) {
-#if 0
-  static FILE *debug_fp = NULL; /* ok */
-
-  if (!debug_fp)
-    debug_fp = fopen("esstat.txt", "wb");
-#endif
   HTS_STAT.stat_nsocket = HTS_STAT.stat_errors = HTS_STAT.nbk = 0;
   HTS_STAT.nb = 0;
   if (HTS_STAT.HTS_TOTAL_RECV > 2048) {
@@ -2363,10 +2283,6 @@ int engine_stats(void) {
       if ((cdif - HTS_STAT.istat_timestart[i]) >= 2000) {
         TStamp dif;
 
-#if 0
-        fprintf(debug_fp, "set timer %d\n", i);
-        fflush(debug_fp);
-#endif
         dif = cdif - HTS_STAT.istat_timestart[i];
         if ((TStamp) (dif / 1000) > 0) {
           LLint byt = (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[i]);
@@ -2385,10 +2301,6 @@ int engine_stats(void) {
     // timer #0 resync timer #1 when reaching 1 second limit
     if (HTS_STAT.istat_reference01 != HTS_STAT.istat_timestart[0]) {
       if ((cdif - HTS_STAT.istat_timestart[0]) >= 1000) {
-#if 0
-        fprintf(debug_fp, "resync timer 1\n");
-        fflush(debug_fp);
-#endif
         HTS_STAT.istat_bytes[1] = HTS_STAT.HTS_TOTAL_RECV;
         HTS_STAT.istat_timestart[1] = cdif;
         HTS_STAT.istat_reference01 = HTS_STAT.istat_timestart[0];
@@ -2409,7 +2321,6 @@ void host_ban(httrackp * opt, int ptr,
   lien_back *const back = sback->lnk;
   const int back_max = sback->count;
 
-  //int l;
   int i;
 
   if (host[0] == '!')
@@ -2466,9 +2377,7 @@ void host_ban(httrackp * opt, int ptr,
   }
 
   // effacer liens
-  //l=strlen(host);
+  for (i = 0; i < opt->lien_tot; i++) {
-  for(i = 0; i < opt->lien_tot; i++) {
-    //if (heap(i)->adr_len==l) {    // même taille de chaîne
     // Calcul de taille sécurisée
     if (heap(i)) {
       if (heap(i)->adr) {
@@ -2507,7 +2416,6 @@ void host_ban(httrackp * opt, int ptr,
                     "WARNING! HostCancel detected memory leaks [null at %d]",
                     i);
     }
-    //}
   }
 }
 
@@ -2924,7 +2832,6 @@ FILE *fileappend(filenote_strc * strc, const char *s) {
   // noter lst
   filenote(strc, s, NULL);
 
-  // if (*s=='/') strcpybuff(fname,s+1); else strcpybuff(fname,s);    // pas de / (root!!) // ** SIIIIIII!!! à cause de -O <path>
   strcpybuff(fname, s);
 
 #if HTS_DOSNAME
@@ -2968,7 +2875,6 @@ int filecreateempty(filenote_strc * strc, const char *filename) {
 int filenote(filenote_strc * strc, const char *s, filecreate_params * params) {
   // gestion du fichier liste liste
   if (params) {
-    //filecreate_params* p = (filecreate_params*) params;
     strcpybuff(strc->path, params->path);
     strc->lst = params->lst;
     return 0;
@@ -3048,9 +2954,8 @@ void usercommand_exe(const char *cmd, const char *file) {
   }
 }
 
-static void postprocess_file(httrackp * opt, const char *save, const char *adr,
+static void postprocess_file(httrackp *opt, const char *save, const char *adr,
                              const char *fil) {
-  //int first = 0;
   /* MIME-html archive to build */
   if (opt != NULL && opt->mimehtml) {
     if (adr != NULL && strcmp(adr, "primary") == 0) {
@@ -3078,7 +2983,6 @@ static void postprocess_file(httrackp * opt, const char *save, const char *adr,
       }
 
       if (!opt->state.mimehtml_created) {
-        //first = 1;
         opt->state.mimefp =
           fopen(fconcat
                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
@@ -3222,28 +3126,6 @@ int fspc(httrackp * opt, FILE * fp, const char *type) {
   return 0;
 }
 
-// vérifier taux de transfert
-#if 0
-void check_rate(TStamp stat_timestart, int maxrate) {
-  // vérifier taux de transfert (pas trop grand?)
-  /*
-     if (maxrate>0) {
-     int r = (int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-stat_timestart));    // taux actuel de transfert
-     HTS_STAT.HTS_TOTAL_RECV_STATE=0;
-     if (r>maxrate) {    // taux>taux autorisé
-     int taux = (int) (((TStamp) (r - maxrate) * 100) / (TStamp) maxrate);
-     if (taux<15)
-     HTS_STAT.HTS_TOTAL_RECV_STATE=1;   // ralentir un peu (<15% dépassement)
-     else if (taux<50)
-     HTS_STAT.HTS_TOTAL_RECV_STATE=2;   // beaucoup (<50% dépassement)
-     else
-     HTS_STAT.HTS_TOTAL_RECV_STATE=3;   // énormément (>50% dépassement)
-     }
-     }
-   */
-}
-#endif
-
 // ---
 // sous routines liées au moteur et au backing
 
@@ -3253,21 +3135,8 @@ int backlinks_done(const struct_back * sback,
                    int ptr) {
   int n = 0;
 
-#if 0
-  int i;
-
-  //Links done and stored in cache
-  for(i = ptr + 1; i < lien_tot; i++) {
-    if (heap(i)) {
-      if (heap(i)->pass2 == -1) {
-        n++;
-      }
-    }
-  }
-#else
   // finalized in background
   n += HTS_STAT.stat_background;
-#endif
   n += back_done_incache(sback);
   return n;
 }
@@ -3404,8 +3273,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
     /* on a déja parcouru */
     if (p < cache->ptr_ant)
       p = cache->ptr_ant;
-    while(p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
+    while (p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
-      //while((p<lien_tot) && (n>0) && (p < ptr+opt->maxcache_anticipate)) {
       int ok = 1;
 
       // on ne met pas le fichier en backing si il doit être traité après ou s'il a déja été traité
@@ -3417,12 +3285,6 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
           ok = 0;
       }
 
-      // Why in hell did I do that ?
-      //if (ok && heap(p)->sav != NULL && heap(p)->sav[0] != '\0' 
-      //  && hash_read(opt->hash,heap(p)->sav,NULL,HASH_STRUCT_FILENAME ) >= 0)     // lookup in liens_record
-      //{
-      //  ok = 0;
-      //}
       if (heap(p)->sav == NULL || heap(p)->sav[0] == '\0'
           || hash_read(opt->hash, heap(p)->sav, NULL, HASH_STRUCT_FILENAME ) < 0) {
         ok = 0;
@@ -3451,7 +3313,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
         }
       }
       p++;
-    }                           // while
+    } // while
     /* sauver position dernière anticipation */
     cache->ptr_ant = p;
     cache->ptr_last = ptr;
@@ -3713,20 +3575,7 @@ HTSEXT_API hts_boolean hts_has_stopped(httrackp *opt) {
   return ended;
 }
 
-// régler en cours de route les paramètres réglables..
-// -1 : erreur
-//HTSEXT_API int hts_setopt(httrackp* set_opt) {
-//  if (set_opt) {
-//    httrackp* engine_opt=hts_declareoptbuffer(NULL);
-//    if (engine_opt) {
-//      //_hts_setopt=opt;
-//      copy_htsopt(set_opt,engine_opt);
-//    }
-//  }
-//  return 0;
-//}
 // ajout d'URL
-// -1 : erreur
 HTSEXT_API hts_boolean hts_addurl(httrackp *opt, char **url) {
   if (url)
     opt->state._hts_addurl = url;
@@ -3987,10 +3836,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
               heap_top()->link_import = 0; // pas mode import
 
               // écrire autres paramètres de la structure-lien
-              //if (meme_adresse)                                 
               heap_top()->premier = heap(ptr)->premier;
-              //else    // sinon l'objet père est le précédent lui même
-              //  heap_top()->premier=ptr;
 
               heap_top()->precedent = ptr;
               // noter la priorité
@@ -4001,9 +3847,6 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
               heap_top()->pass2 = max(pass_fix, numero_passe);
               heap_top()->retry = opt->retry;
 
-              //strcpybuff(heap_top()->adr,adr);
-              //strcpybuff(heap_top()->fil,fil);
-              //strcpybuff(heap_top()->sav,save); 
               hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s",
                             heap_top()->adr, heap_top()->fil,
                             heap_top()->sav);

src/htscore.h

@@ -193,8 +193,7 @@ struct cache_back {
   /* */
   int type;
   int ro;                   /**< read-only: no new cache is written */
-  FILE *dat, *ndx, *olddat; /**< new data, new index, old data files */
+
-  char *use;                /**< in-memory list of cached adr+fil keys */
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
   char lastmodified[256];
@@ -288,12 +287,12 @@ struct filecreate_params {
 
 /* True if a new cache is being written (plain or zip backend). */
 HTS_STATIC int cache_writable(cache_back * cache) {
-  return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL));
+  return (cache != NULL && cache->zipOutput != NULL);
 }
 
 /* True if an old cache is available to read (plain or zip backend). */
 HTS_STATIC int cache_readable(cache_back * cache) {
-  return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL));
+  return (cache != NULL && cache->zipInput != NULL);
 }
 
 #endif
@@ -412,10 +411,6 @@ char *readfile_utf8(const char *fil);
 
 char *readfile_or(const char *fil, const char *defaultdata);
 
-#if 0
-void check_rate(TStamp stat_timestart, int maxrate);
-#endif
-
 /* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
    Not thread-safe; call from the single crawl loop. */
 
@@ -467,11 +462,6 @@ int ask_continue(httrackp * opt);
 /* Number of decimal digits in n. */
 int nombre_digit(int n);
 
-// Java
-#if 0
-int hts_add_file(char *file, int file_position);
-#endif
-
 // Polling
 #if HTS_POLL
 int check_flot(T_SOC s);

src/htscoremain.c

@@ -145,7 +145,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   char *url = NULL;             // URLS séparées par un espace
   int url_sz = 65535;
 
-  //char url[65536];         // URLS séparées par un espace
   // the parametres
   int httrack_logmode = 3;      // ONE log file
 
@@ -290,12 +289,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         if (tmp_argc == 1) {    /* pas -P & co */
           if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */
             if (argv_url < 0)
-              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
+              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
-            //if (argv_url>=0) {
+                            // previous options
             argv_url++;
             if (!argv_firsturl)
               argv_firsturl = x_argv[x_argc - 1];
-            //}
           } else {
             if (strcmp(tmp_argv[0], "-h") == 0) {
               help(argv[0], !opt->quiet);
@@ -324,8 +322,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         } else if (tmp_argc == 2) {
           if ((strcmp(tmp_argv[0], "-%L") == 0)) {      // liste d'URLs
             if (argv_url < 0)
-              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
+              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
-            //if (argv_url>=0)
+                            // previous options
             argv_url++;         /* forcer */
           }
         }
@@ -403,15 +401,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     else
                       inQuote = 1;
                   } else if (!inQuote && !noDbl && argv[na][i] == ',') {
-                    //StringAddchar(path, '\0');
-                    //j = 0;
                     path = &opt->path_log;
                   } else {
                     StringAddchar(*path, argv[na][i]);
-                    //path[j++] = argv[na][i];
                   }
                 }
-                //path[j++] = '\0';
                 if (StringLength(opt->path_log) == 0) {
                   StringCopyS(opt->path_log, opt->path_html);
                 }
@@ -420,7 +414,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if (check_path(&opt->path_html, argv_firsturl)) {
                   opt->dir_topindex = 1;        // rebuilt top index
                 }
-                //printf("-->%s\n%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log));                
               }
               break;
             }                   // switch
@@ -539,8 +532,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             insert_after++;
           }
         }
-      } while(lastp != NULL);
+      } while (lastp != NULL);
-      //fclose(fp);
     }
   }
 
@@ -716,24 +708,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             HTS_PANIC_PRINTF(s);
             htsmain_free();
             return -1;
-#else
-#if 0
-            char _args[8][256];
-            char *args[8];
-
-            printf("Cheking for updates...\n");
-            strcpybuff(_args[0], argv[0]);
-            strcpybuff(_args[1], "--get");
-            sprintf(_args[2], HTS_UPDATE_WEBSITE, 0, "");
-            strcpybuff(_args[3], "--quickinfo");
-            args[0] = _args[0];
-            args[1] = _args[1];
-            args[2] = _args[2];
-            args[3] = _args[3];
-            args[4] = NULL;
-            if (execvp(args[0], args) == -1) {
-            }
-#endif
 #endif
           }
           //
@@ -751,103 +725,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     }
   }
 
-  // Compter urls/jokers
-  /*
-     if (argv_url<=0) { 
-     int na;
-     argv_url=0;
-     for(na=1;na<argc;na++) {
-     if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) {
-     na++;    // sauter nom de proxy
-     } else if (!cmdl_opt(argv[na])) { 
-     argv_url++;   // un de plus       
-     } else if (strcmp(argv[na],"-h")==0) {
-     help(argv[0],!opt->quiet);
-     htsmain_free();
-     return 0;
-     } else {
-     if ((strchr(argv[na],'q')!=NULL))
-     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
-     if ((strchr(argv[na],'i')!=NULL)) {  // doit.log!
-     argv_url=0;
-     na=argc;
-     }
-     }
-     }
-     }  
-   */
-
-  // Ici on ajoute les arguments qui ont été appelés avant au cas où on récupère une session
-  // Exemple: httrack www.truc.fr -L0 puis ^C puis httrack sans URL : ajouter URL précédente
-  /*
-     if (argv_url==0) {
-     //if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) {  // il existe déja un cache précédent.. renommer
-     if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) {    // un cache est présent
-
-     x_argvblk=(char*) calloct(32768,1);
-
-     if (x_argvblk!=NULL) {
-     FILE* fp;
-     int x_argc;
-
-     //strcpybuff(x_argvblk,"httrack ");
-     fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb");
-     if (fp) {
-     linput(fp,x_argvblk+strlen(x_argvblk),8192);
-     fclose(fp); fp=NULL;
-     }
-
-     // calculer arguments selon derniers arguments
-     x_argv[0]=argv[0];
-     x_argc=1;
-     {
-     char* p=x_argvblk;
-     do {
-     x_argv[x_argc++]=p;
-     //p=strstr(p," ");
-     // exemple de chaine: "echo \"test\"" c:\a "\$0"
-     p=next_token(p,1);    // prochain token
-     if (p) {
-     *p=0;    // octet nul (tableau)
-     p++;
-     }            
-     } while(p!=NULL);
-     }
-     // recopier arguments actuels (pointeurs uniquement)
-     {
-     int na;
-     for(na=1;na<argc;na++) {
-     if (strcmp(argv[na],"-O") != 0)    // SAUF le path!
-     x_argv[x_argc++]=argv[na];
-     else
-     na++;
-     }
-     }
-     argc=x_argc;      // nouvel argc
-     argv=x_argv;      // nouvel argv
-     }
-
-     }
-     //}
-     }
-   */
-
-  // Vérifier quiet
-  /*
-     { 
-     int na;    
-     for(na=1;na<argc;na++) {
-     if (!cmdl_opt(argv[na])) { 
-     if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0))
-     na++;    // sauter nom de proxy
-     } else {
-     if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL))
-     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
-     }
-     }
-     }
-   */
-
   /* Engine self-tests: -#test lists them, -#test=NAME [args] runs one. Handled
      here, ahead of the no-URL usage gate below, so they need no dummy URL. */
   {
@@ -954,8 +831,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   } else {                      // plus de 2 paramètres
     // un fichier log existe?
     if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-      StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // fichier lock?
+                       StringBuff(opt->path_log),
-      //char s[32];
+                       "hts-in_progress.lock"))) { // fichier lock?
 
       opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
       if (opt->quiet == 0) {
@@ -986,11 +863,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           }
         }
       }
-    } else
+    } else if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-      if (fexist
+                              StringBuff(opt->path_html), "index.html"))) {
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "index.html"))) {
-      //char s[32];
       opt->cache = HTS_CACHE_TEST_UPDATE;
       if (opt->quiet == 0) {
         if ((fexist
@@ -1085,7 +959,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             //
           case 'g':            // récupérer un (ou plusieurs) fichiers isolés
             opt->wizard = HTS_WIZARD_AUTO;
-            //opt->wizard=0;             // pas de wizard
             opt->cache = HTS_CACHE_NONE; // ni de cache
             opt->makeindex = 0; // ni d'index
             httrack_logmode = 1;        // erreurs à l'écran
@@ -1185,9 +1058,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             while(isdigit((unsigned char) *(com + 1)))
               com++;
             break;
-            //
-            //case 'A': opt->urlmode=1; break;
-            //case 'R': opt->urlmode=2; break;
           case 'K':
             opt->urlmode = HTS_URLMODE_ABSOLUTE;
             if (isdigit((unsigned char) *(com + 1))) {
@@ -1199,8 +1069,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               while(isdigit((unsigned char) *(com + 1)))
                 com++;
             }
-            //if (*(com+1)=='0') { opt->urlmode=2; com++; } break;
-            //
           case 'c':
             if (isdigit((unsigned char) *(com + 1))) {
               sscanf(com + 1, "%d", &opt->maxsoc);
@@ -1688,7 +1556,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   } else {
                     char *a;
 
-                    //char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs);
                     for(a = argv[na]; *a != '\0'; a++) {
                       if (*a == ';') {  /* next one */
                         StringAddchar(opt->mimedefs, '\n');
@@ -1936,6 +1803,23 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   StringCopy(opt->cookies_file, argv[na]);
                 }
                 break;
+              case 'Y': // why: explain the filter verdict for a URL, no crawl
+                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
+                  HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
+                  printf(
+                      "Example: --why \"http://www.example.com/file.zip\"\n");
+                  htsmain_free();
+                  return -1;
+                } else {
+                  na++;
+                  if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
+                    HTS_PANIC_PRINTF("why URL too long");
+                    htsmain_free();
+                    return -1;
+                  }
+                  StringCopy(opt->why_url, argv[na]);
+                }
+                break;
               case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF("Option pause needs a blank space and a "
@@ -2027,10 +1911,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   HTS_PANIC_PRINTF(s);
                   htsmain_free();
                   return -1;
-                }
+              } break;
-                break;
-
-                //case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break;
               }
             }
             break;
@@ -2072,18 +1953,20 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     char BIGSTK url[HTS_URLMAXSIZE * 2];
                     char linepos[256];
                     int pos;
-                    char *cacheNdx =
+                    LLint cacheNdxLen = 0;
-                      readfile(fconcat
+                    char *cacheNdx = readfile2(
-                               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                "hts-cache/new.ndx"));
+                                StringBuff(opt->path_log), "hts-cache/new.ndx"),
+                        &cacheNdxLen);
                     cache_init(&cache, opt);    /* load cache */
                     if (cacheNdx != NULL) {
                       char firstline[256];
                       char *a = cacheNdx;
+                      const char *const end = cacheNdx + cacheNdxLen;
 
                       a += cache_brstr(a, firstline, sizeof(firstline));
                       a += cache_brstr(a, firstline, sizeof(firstline));
-                      while(a != NULL) {
+                      while (a != NULL && a < end) {
                         a = strchr(a + 1, '\n');        /* start of line */
                         if (a) {
                           htsblk r;
@@ -2091,15 +1974,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                           /* */
                           a++;
                           /* read "host/file" */
-                          a += binput(a, afs.af.adr, HTS_URLMAXSIZE);
+                          a += cache_binput(a, end, afs.af.adr, HTS_URLMAXSIZE);
-                          a += binput(a, afs.af.fil, HTS_URLMAXSIZE);
+                          a += cache_binput(a, end, afs.af.fil, HTS_URLMAXSIZE);
                           url[0] = '\0';
                           if (!link_has_authority(afs.af.adr))
                             strcatbuff(url, "http://");
                           strcatbuff(url, afs.af.adr);
                           strcatbuff(url, afs.af.fil);
                           /* read position */
-                          a += binput(a, linepos, 200);
+                          a += cache_binput(a, end, linepos, 200);
                           sscanf(linepos, "%d", &pos);
                           if (!hasFilter
                               || (strjoker(url, filter, NULL, NULL) != NULL)
@@ -2274,15 +2157,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 return 0;
                 break;
               case '~':        /* internal lib test */
-                HTS_PANIC_PRINTF
+                HTS_PANIC_PRINTF("Option #~ is disabled for security reasons");
-                  ("Option #~ is disabled for security reasons");
-                //Disabled because choke on GCC 4.3 (toni from links2linux.de)
-                //{
-                //  char thisIsATestYouShouldSeeAnError[12];
-                //  const char *const bufferOverflowTest = "0123456789012345678901234567890123456789";
-                //  strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest);
-                //  return 0;
-                //}
                 break;
               case 'f':
                 opt->flush = 1;
@@ -2384,7 +2259,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               opt->proxy.active = 1;
               // Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
               a = argv[na] + strlen(argv[na]) - 1;
-              // a=strstr(argv[na],":");  // port
               while((a > argv[na]) && (*a != ':') && (*a != '@'))
                 a--;
               if (*a == ':') {  // un port est présent, <proxy>:port
@@ -2489,7 +2363,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
 #endif
 #endif
 
-  //printf("WARNING! This is *only* a beta-release of HTTrack\n");
   io_flush;
 
 #if DEBUG_STEPS
@@ -2573,7 +2446,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     {
       FILE *fp = NULL;
 
-      //int n=0;
       char t[256];
 
       time_local_rfc822(t);     // faut bien que ca serve quelque part l'heure RFC1945 arf'
@@ -2607,9 +2479,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       }
 
       strcpybuff(n_lock,
-             fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                     "hts-in_progress.lock"));
+                         StringBuff(opt->path_log), "hts-in_progress.lock"));
-      //sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
       /*do {
          if (!n)
          sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
@@ -2685,8 +2556,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           fprintf(fp, LF);
           fclose(fp);
           fp = NULL;
-          //} else if (opt->debug>1) {
-          //  printf("! FileOpen error, \"%s\"\n",strerror(errno));
         }
       }
       // petit message dans le lock

src/htsdns_selftest.c

@@ -297,6 +297,26 @@ int dns_selftests(httrackp *opt) {
     IPV6_resolver = 0;
   }
 
+  /* "host:port" resolves as the bare host: the cache/connect path strips
+     ":port" before both the backend and the cache key (#181). */
+  mock_reset_calls();
+  {
+    SOCaddr a;
+    char ip[64];
+    const char *err = NULL;
+
+    /* cache miss: the backend is hit with the stripped host, not "host:port" */
+    CHECK(hts_dns_resolve2(opt, "v6only.test:8080", &a, &err) != NULL);
+    CHECK(mock_find("v6only.test")->calls == 1);
+    /* the bare host shares that one cache entry (stripped key) */
+    CHECK(hts_dns_resolve2(opt, "v6only.test", &a, &err) != NULL);
+    CHECK(mock_find("v6only.test")->calls == 1);
+    /* a port variant of an already-cached host also hits, right address */
+    CHECK(hts_dns_resolve2(opt, "v4only.test:1234", &a, &err) != NULL);
+    SOCaddr_inetntoa(ip, sizeof(ip), a);
+    CHECK(strcmp(ip, "1.2.3.4") == 0);
+  }
+
   /* newhttp_addr() must connect to the addr_index-th address, not always the
      first: this is what back_connect_next relies on to reach the fallback. */
   {

src/htsfilters.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsbase.h"
 #include "htslib.h"
 #include <ctype.h>
+#include <stdint.h>
 /* END specific definitions */
 
 // à partir d'un tableau de {"+*.toto","-*.zip","+*.tata"} définit si nom est autorisé
@@ -94,17 +95,107 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
   return verdict;
 }
 
-// supercomparateur joker (tm)
+int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
-// compare a et b (b=avec joker dedans), case insensitive [voir CI]
+                     const char *nom2, LLint *size, int *size_flag,
-// renvoi l'adresse de la première lettre de la chaine
+                     int *depth) {
-// (càd *[..]toto.. renvoi adresse de toto dans la chaine)
+  int depth1 = 0, depth2 = 0;
-// accepte les délires du genre www.*.*/ * / * truc*.*
+  int flag1 = 0, flag2 = 0;
-// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
+  LLint sz1 = 0, sz2 = 0;
-// * = toute lettre
+  int jok1, jok2, use1;
-// --?-- : spécifique à HTTrack et aux ?
+
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
+  if (size) {
-                          int *size_flag) {
+    sz1 = *size;
-  //int err=0;
+    sz2 = *size;
+  }
+  jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
+                     size_flag ? &flag1 : NULL, &depth1);
+  jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
+                     size_flag ? &flag2 : NULL, &depth2);
+  use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
+  if (size)
+    *size = use1 ? sz1 : sz2;
+  if (size_flag)
+    *size_flag = use1 ? flag1 : flag2;
+  if (depth)
+    *depth = use1 ? depth1 : depth2;
+  return use1 ? jok1 : jok2;
+}
+
+/* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
+   pair keeps star-heavy patterns polynomial instead of exponential (#501). */
+typedef struct strjoker_memo {
+  const char *chaine0, *joker0; /* offsets are relative to these bases */
+  size_t stride;                /* strlen(joker0) + 1 */
+  unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
+} strjoker_memo;
+
+static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
+                                 const char *joker, LLint *size,
+                                 int *size_flag);
+
+/* A pair that failed once fails forever (*size is only written on the success
+   path), so record NULL results and cut the re-exploration. */
+static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
+                                const char *joker, LLint *size,
+                                int *size_flag) {
+  size_t bit = 0;
+  const char *adr;
+
+  if (memo->failed) {
+    bit = (size_t) (chaine - memo->chaine0) * memo->stride +
+          (size_t) (joker - memo->joker0);
+    if (memo->failed[bit >> 3] & (unsigned char) (1u << (bit & 7)))
+      return NULL;
+  }
+  adr = strjoker_impl(memo, chaine, joker, size, size_flag);
+  if (adr == NULL && memo->failed)
+    memo->failed[bit >> 3] |= (unsigned char) (1u << (bit & 7));
+  return adr;
+}
+
+// wildcard comparator: match chaine against joker (pattern), case-insensitive
+// returns the address of the first matched letter past any leading joker
+// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
+                                LLint *size, int *size_flag) {
+  strjoker_memo memo = {chaine, joker, 0, NULL};
+  unsigned char stackbits[2048];
+  hts_boolean onheap = HTS_FALSE;
+  const char *adr;
+
+  if (chaine != NULL && joker != NULL) {
+    const size_t n1 = strlen(chaine) + 1;
+
+    memo.stride = strlen(joker) + 1;
+    if (n1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
+      const size_t bytes = (n1 * memo.stride + 7) / 8;
+
+      if (bytes <= sizeof(stackbits)) {
+        memset(stackbits, 0, bytes);
+        memo.failed = stackbits;
+      } else {
+        memo.failed = (unsigned char *) calloct(bytes, 1);
+        onheap = memo.failed != NULL ? HTS_TRUE : HTS_FALSE;
+      }
+    }
+  }
+  adr = strjoker_rec(&memo, chaine, joker, size, size_flag);
+  if (onheap)
+    freet(memo.failed);
+  return adr;
+}
+
+/* Test-only oracle: the same matcher without the failure memo. */
+const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
+                            int *size_flag) {
+  const strjoker_memo memo = {chaine, joker, 0, NULL};
+
+  return strjoker_rec(&memo, chaine, joker, size, size_flag);
+}
+
+static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
+                                 const char *joker, LLint *size,
+                                 int *size_flag) {
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -236,9 +327,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
                 for(j = (int) (unsigned char) joker[i];
                     j <= (int) (unsigned char) joker[i + 2]; j++)
                   pass[j] = 1;
-
               }
-              // else err=1;
               i += 3;
             } else { // 1 car, ex: *[ ]
               pass[(int) (unsigned char) joker[i]] = 1;
@@ -262,7 +351,6 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       for(i = 0; i < 256; i++)
         pass[i] = 1;            // tout autoriser
       jmp = 1;
-      ////if (joker[2]==LEFT) jmp=3;        // permet de recher *<crochet ouvrant>
     }
 
     {
@@ -281,7 +369,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
 
       // tester sans le joker (pas ()+ mais ()*)
       if (!unique) {
-        if ((adr = strjoker(chaine, joker + jmp, size, size_flag))) {
+        if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag))) {
           return adr;
         }
       }
@@ -290,10 +378,11 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       if (!unique)
         max = (int) strlen(chaine);
       else                      /* *(a) only match a (not aaaaa) */
-        max = 1;
+        max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
-          if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
+          if ((adr = strjoker_rec(memo, chaine + i + 1, joker + jmp, size,
+                                  size_flag))) {
             return adr;
           }
           i++;
@@ -303,7 +392,8 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
 
       // tester chaîne vide
       if (i != max + 2)         // avant c'est ok
-        if ((adr = strjoker(chaine + max, joker + jmp, size, size_flag)))
+        if ((adr = strjoker_rec(memo, chaine + max, joker + jmp, size,
+                                size_flag)))
           return adr;
 
       return NULL;              // perdu
@@ -325,7 +415,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       // comparaison ok?
       if (ok) {
         // continuer la comparaison.
-        if (strjoker(chaine + jmp, joker + jmp, size, size_flag))
+        if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag))
           return chaine;        // retourner 1e lettre
       }
 

src/htsfilters.h

@@ -41,8 +41,17 @@ Please visit our Website: http://www.httrack.com
 
 int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
                 int *size_flag, int *depth);
+/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
+   scheme); the match latest in the list wins, a "don't know" verdict defers.
+   Returns the merged verdict; the out-params carry the winner's values. */
+int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
+                     const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
+/* strjoker() without the failure memo (exponential worst case); test-only
+   oracle for the memoized matcher. */
+const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
+                            int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);
 #endif
 

src/htsftp.c

@@ -54,15 +54,9 @@ Please visit our Website: http://www.httrack.com
 #endif
 
 // ftp mode passif
-// #if HTS_INET6==0
 #define FTP_PASV 1
-// #else
-// no passive mode for v6
-// #define FTP_PASV 0
-// #endif
 
 #define FTP_DEBUG 0
-//#define FORK_DEBUG 0
 
 #if USE_BEGINTHREAD
 
@@ -206,20 +200,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   {
     char *a;
 
-#if 0
-    a = back->url_fil + strlen(back->url_fil) - 1;
-    while((a > back->url_fil) && (*a != '/'))
-      a--;
-    if (*a != '/') {
-      a = NULL;
-    }
-#else
     a = back->url_fil;
-#endif
     if (a != NULL && *a != '\0') {
-#if 0
-      a++;                      // sauter /
-#endif
       ftp_filename = a;
       if (strnotempty(a)) {
         char catbuff[CATBUFF_SIZE];
@@ -243,7 +225,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
       }
     } else {
       strcpybuff(back->r.msg, "Unexpected PORT error");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
     }
   }
@@ -277,7 +258,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
       snprintf(back->r.msg, sizeof(back->r.msg),
                "Unable to get server's address: %s", error);
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_NON_FATAL;
       _HALT_FTP return 0;
     }
@@ -290,20 +270,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
     if (soc_ctl == INVALID_SOCKET) {
       strcpybuff(back->r.msg, "Unable to create a socket");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
     }
 
     SOCaddr_initport(server, port);
-    // server.sin_port = htons((unsigned short int) port);
 
     // connexion (bloquante, on est en thread)
     strcpybuff(back->info, "connect");
 
     if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
       strcpybuff(back->r.msg, "Unable to connect to the server");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
 #ifdef _WIN32
@@ -345,64 +322,22 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               // ok
             } else {
               strcpybuff(back->r.msg, "TYPE I error");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
-#if 0
-            // --CWD--
-            char *a;
-
-            a = back->url_fil + strlen(back->url_fil) - 1;
-            while((a > back->url_fil) && (*a != '/'))
-              a--;
-            if (*a == '/') {    // ok repéré
-              char BIGSTK target[1024];
-
-              target[0] = '\0';
-              strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
-              if (strnotempty(target) == 0)
-                strcatbuff(target, "/");
-              strcpybuff(back->info, "cwd");
-              snprintf(line, sizeof(line), "CWD %s", target);
-              send_line(soc_ctl, line);
-              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-              _CHECK_HALT_FTP;
-              if (line[0] == '2') {
-                send_line(soc_ctl, "TYPE I");
-                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-                _CHECK_HALT_FTP;
-                if (line[0] == '2') {
-                  // ok..
-                } else {
-                  strcpybuff(back->r.msg, "TYPE I error");
-                  // back->status=STATUS_FTP_READY;    // fini
-                  back->r.statuscode = STATUSCODE_INVALID;
-                }
-              } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
-                // back->status=STATUS_FTP_READY;    // fini
-                back->r.statuscode = STATUSCODE_INVALID;
-              }                 // sinon on est prêts
-            } else {
-              strcpybuff(back->r.msg, "Unexpected ftp error");
-              // back->status=STATUS_FTP_READY;    // fini
-              back->r.statuscode = STATUSCODE_INVALID;
-            }
-#endif
 
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
+          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
-          // back->status=STATUS_FTP_READY;    // fini
+                   linejmp(line));
           back->r.statuscode = STATUSCODE_INVALID;
         }
       } else {
-        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
+        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
-        // back->status=STATUS_FTP_READY;    // fini
+                 linejmp(line));
         back->r.statuscode = STATUSCODE_INVALID;
       }
 
@@ -469,8 +404,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
             // -- fin analyse de l'adresse IP et du port --
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
@@ -501,13 +436,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 }
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
+              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
-              // back->status=STATUS_FTP_READY;    // fini
+                       linejmp(line));
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         }
@@ -613,9 +548,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   deletesoc(soc_dat);
                   soc_dat = INVALID_SOCKET;
                   //
-                  snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
+                  snprintf(back->r.msg, sizeof(back->r.msg),
-                          linejmp(line));
+                           "RETR command error: %s", linejmp(line));
-                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = STATUSCODE_INVALID;
                 }               // sinon on est prêts
               } else {
@@ -626,23 +560,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 soc_dat = INVALID_SOCKET;
                 //
                 strcpybuff(back->r.msg, "Unable to connect");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }                 // sinon on est prêts
             } else {
               strcpybuff(back->r.msg, "Unable to create a socket");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }                   // sinon on est prêts
           } else {
             snprintf(back->r.msg, sizeof(back->r.msg),
                      "Unable to resolve IP %s: %s", adr_ip, error);
-            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
+          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
-          // back->status=STATUS_FTP_READY;    // fini
+                   linejmp(line));
           back->r.statuscode = STATUSCODE_INVALID;
         }                       // sinon on est prêts
 #else
@@ -663,17 +594,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               //T_SOC soc_dat;
               if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
                 strcpybuff(back->r.msg, "Unable to accept connection");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
+              snprintf(back->r.msg, sizeof(back->r.msg),
-              // back->status=STATUS_FTP_READY;    // fini
+                       "RETR command error: %s", linejmp(line));
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -683,7 +613,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 #endif
         } else {
           strcpybuff(back->r.msg, "Unable to listen to a port");
-          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
 #endif
@@ -707,21 +636,18 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             int len = 1;
             int read_len = 1024;
 
-            //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
-
             while((len > 0) && (!stop_ftp(back))) {
               // attendre les données
               len = 1;          // pas d'erreur pour le moment
               switch (wait_socket_receive(soc_dat, timeout)) {
               case -1:
                 strcpybuff(back->r.msg, "FTP read error");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
               case 0:
-                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
+                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
-                // back->status=STATUS_FTP_READY;    // fini
+                         timeout);
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
@@ -743,17 +669,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                          }
                        */
                       strcpybuff(back->r.msg, "Write error");
-                      // back->status=STATUS_FTP_READY;    // fini
                       back->r.statuscode = STATUSCODE_INVALID;
                       len = 0;  // error
                     }
                   } else {
                     strcpybuff(back->r.msg, "Unexpected write error");
-                    // back->status=STATUS_FTP_READY;    // fini
                     back->r.statuscode = STATUSCODE_INVALID;
                   }
-                } else {        // Erreur ou terminé
+                } else { // Erreur ou terminé
-                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = 0;
                   if (back->r.totalsize > 0
                       && back->r.size != back->r.totalsize) {
@@ -762,7 +685,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   }
                 }
                 read_len = 1024;
-                //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
               }
             }
             if (back->r.fp) {
@@ -771,7 +693,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
           } else {
             strcpybuff(back->r.msg, "Unable to write file");
-            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -787,16 +708,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               get_ftp_line(soc_ctl, line, sizeof(line), timeout);
               if (line[0] == '2') {     // OK
                 strcpybuff(back->r.msg, "OK");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = HTTP_OK;
               } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
+                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
-                // back->status=STATUS_FTP_READY;    // fini
+                         linejmp(line));
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
               strcpybuff(back->r.msg, "FTP read error");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           }
@@ -822,7 +741,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     back->r.statuscode = HTTP_OK;
     strcpybuff(back->r.msg, "OK");
   }
-  // back->status=STATUS_FTP_READY;    // fini
   return 0;
 }
 
@@ -989,9 +907,7 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
       break;
     }
 
-    //HTS_TOTAL_RECV_CHECK(dummy);     // Diminuer au besoin si trop de données reçues
     switch (recv(soc, &b, 1, 0)) {
-      //case 0: break;    // pas encore --> erreur (on attend)!
     case 1:
       HTS_STAT.HTS_TOTAL_RECV += 1;     // compter flux entrant
       if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
@@ -1118,7 +1034,6 @@ int wait_socket_receive(T_SOC soc, int timeout) {
 int stop_ftp(lien_back * back) {
   if (back->stop_ftp) {
     strcpybuff(back->r.msg, "Cancelled by User");
-    // back->status=STATUS_FTP_READY;    // fini
     back->r.statuscode = STATUSCODE_INVALID;
     return 1;
   }

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-11"
+#define HTTRACK_VERSION "3.49-12"
-#define HTTRACK_VERSIONID "3.49.11"
+#define HTTRACK_VERSIONID "3.49.12"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 
@@ -56,6 +56,10 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
+// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
+#include <stdint.h>
+#include <inttypes.h>
+
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -111,29 +115,6 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
-#ifndef HTS_LONGLONG
-#ifdef SIZEOF_LONG_LONG
-#if SIZEOF_LONG_LONG == 8
-#define HTS_LONGLONG 1
-#endif
-#endif
-
-#ifndef HTS_LONGLONG
-#ifdef __sun
-#define HTS_LONGLONG 0
-#endif
-#ifdef __osf__
-#define HTS_LONGLONG 0
-#endif
-#ifdef __linux
-#define HTS_LONGLONG 1
-#endif
-#ifdef _WIN32
-#define HTS_LONGLONG 1
-#endif
-#endif
-#endif
-
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -326,59 +307,11 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-#ifndef HTS_LONGLONG
+/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
-#ifdef HTS_NO_64_BIT
+typedef int64_t LLint;
-#define HTS_LONGLONG 0
+typedef int64_t TStamp;
-#else
+/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
-#define HTS_LONGLONG 1
+#define LLintP "%" PRId64
-#endif
-#endif
-
-/* Wide integer types, chosen per platform.
-   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
-           plain int where 64-bit is unavailable).
-   TStamp: timestamp/duration in the same width (a double in the no-64-bit
-           fallback).
-   LLintP: the printf conversion for an LLint. */
-#if HTS_LONGLONG
-#ifdef LLINT_FORMAT
-typedef LLINT_TYPE LLint;
-
-typedef LLINT_TYPE TStamp;
-
-#define LLintP LLINT_FORMAT
-#else
-
-#ifdef _WIN32
-typedef __int64 LLint;
-
-typedef __int64 TStamp;
-
-#define LLintP "%I64d"
-#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
-       defined(__64BIT__))
-
-typedef long int LLint;
-
-typedef long int TStamp;
-
-#define LLintP "%ld"
-#else
-typedef long long int LLint;
-
-typedef long long int TStamp;
-
-#define LLintP "%lld"
-#endif
-
-#endif /* HTS_LONGLONG */
-
-#else
-typedef int LLint;
-
-#define LLintP "%d"
-typedef double TStamp;
-#endif
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain
@@ -422,9 +355,10 @@ typedef int T_SOC;
 #endif
 
 /* Permission bits for created folders and files (mkdir and chmod).
-   PROTECT_FOLDER is owner-only. With HTS_ACCESS set (the default) the ACCESS_
+   PROTECT_FOLDER/FILE are owner-only. With HTS_ACCESS set (the default) the
-   modes also grant group/other read; otherwise they stay owner-only. */
+   ACCESS_ modes also grant group/other read; otherwise they stay owner-only. */
 #define HTS_PROTECT_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
+#define HTS_PROTECT_FILE (S_IRUSR | S_IWUSR)
 
 #if HTS_ACCESS
 #define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)

src/htshelp.c

@@ -152,16 +152,6 @@ void help_wizard(httrackp * opt) {
 #define str (buffers->str)
 #define argv (buffers->argv)
 
-  //char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
-  //char *mainpath = (char *) malloct(256);
-  //char *projname = (char *) malloct(256);
-  //char *stropt = (char *) malloct(2048);        // options
-  //char *stropt2 = (char *) malloct(2048);       // options longues
-  //char *strwild = (char *) malloct(2048);       // wildcards
-  //char *cmd = (char *) malloct(4096);
-  //char *str = (char *) malloct(256);
-  //char **argv = (char **) malloct(256 * sizeof(char *));
-
   //
   char *a;
 
@@ -227,8 +217,6 @@ void help_wizard(httrackp * opt) {
   strcatbuff(stropt2, str);
   strcatbuff(stropt2, projname);
   strcatbuff(stropt2, "\" ");
-  // Créer si ce n'est fait un index.html 1er niveau
-  make_empty_index(str);
   //
   printf("\n");
   printf("Enter URLs (separated by commas or blank spaces) :");
@@ -349,8 +337,6 @@ void help_wizard(httrackp * opt) {
       }
       hts_main(argc, argv);
     }
-    //} else {
-    //  help("httrack",1);
   }
 
   /* Free buffers */
@@ -455,23 +441,6 @@ void help_catchurl(const char *dest_path) {
     printf("Unable to create a temporary proxy (no remaining port)\n");
 }
 
-// Créer un index.html vide
-void make_empty_index(const char *str) {
-#if 0
-  if (!fexist(fconcat(str, "index.html"))) {
-    FILE *fp = fopen(fconcat(str, "index.html"), "wb");
-
-    if (fp) {
-      fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
-      fprintf(fp,
-              "<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
-              CRLF);
-      fclose(fp);
-    }
-  }
-#endif
-}
-
 // mini-aide  (h: help)
 //           y
 void help(const char *app, int more) {
@@ -574,6 +543,7 @@ void help(const char *app, int more) {
   infomsg("Spider options:");
   infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
   infomsg(" %K  load extra cookies from a Netscape cookies.txt");
+  infomsg(" %Y  explain which filter rule accepts or rejects a URL, then exit");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
   infomsg
@@ -808,7 +778,4 @@ void help(const char *app, int more) {
   infomsg("[compiled: " HTS_PLATFORM_NAME "]");
 #endif
   infomsg(NULL);
-
-//  infomsg("  R  *relative links (e.g ../link)\n");
-//  infomsg("  A   absolute links (e.g /www.adr/link)\n");
 }

src/htshelp.h

@@ -45,7 +45,6 @@ typedef struct httrackp httrackp;
 
 void infomsg(const char *msg);
 void help(const char *app, int more);
-void make_empty_index(const char *str);
 void help_wizard(httrackp * opt);
 int help_query(const char *list, int def);
 void help_catchurl(const char *dest_path);

src/htsindex.c

@@ -142,8 +142,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
   char keyword[KEYW_LEN + 32];
   int i = 0;
 
-  //
-  //int WordIndexSize = 1024;
   coucal WordIndexHash = NULL;
   FILE *tmpfp = NULL;
 
@@ -178,8 +176,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
              || (strfield2(mime, "text/css"))
     ) {
     inscript = 1;
-    //} else if (strfield2(mime, "text/vnd.wap.wml")) {   // humm won't work in many cases
-    //  inscript=0;
   } else
     return 0;
 
@@ -275,8 +271,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
 
   // Process indexing for this page
   {
-    //FILE* fp=NULL;
-    //fp=fopen(concat(indexpath,"index.txt"),"ab");
     if (fp_tmpproject) {
       while(!feof(tmpfp)) {
         char line[KEYW_LEN + 32];
@@ -286,7 +280,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
           intptr_t e = 0;
 
           if (coucal_read(WordIndexHash, line, &e)) {
-            //if (e) {
             char BIGSTK savelst[HTS_URLMAXSIZE * 2];
 
             e++;                /* 0 means "once" */
@@ -300,11 +293,9 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
             fprintf(fp_tmpproject, "%s %d %s\n", line,
                     (int) (KEYW_SORT_MAXCOUNT - e), savelst);
             hts_primindex_size++;
-            //}
           }
         }
       }
-      //fclose(fp);
     }
   }
 
@@ -330,7 +321,6 @@ void index_finish(const char *indexpath, int mode) {
   off_t size = fpsize(fp_tmpproject);
 
   if (size > 0) {
-    //FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
     if (fp_tmpproject) {
       tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
       if (tab) {
@@ -398,8 +388,6 @@ void index_finish(const char *indexpath, int mode) {
                       if (total_hit) {
                         if (mode == 1)  // TEXT
                           fprintf(fp, "\t=%d\r\n", total_hit);
-                        //else                // HTML
-                        //  fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
                         if ((((total_hit * 1000) / hts_primindex_words) >=
                              KEYW_USELESS1K)
                             || (((total_line * 1000) / index) >=
@@ -416,8 +404,6 @@ void index_finish(const char *indexpath, int mode) {
                           if (mode == 1)        // TEXT
                             fprintf(fp, "\t(%d)\r\n",
                                     ((total_hit * 1000) / hts_primindex_words));
-                          //else                // HTML
-                          //  fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
                         }
                       }
                       if (mode == 1)    // TEXT

src/htsjava.c

@@ -175,15 +175,14 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
 #if JAVADEBUG
       printf("fopen\n");
 #endif
-      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
+      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) ==
-        //fprintf(stderr, "Cannot open input file.\n");
+          NULL) {
         sprintf(str->err_msg, "Unable to open file %s", file);
         return 0;               // une erreur..
       }
 #if JAVADEBUG
       printf("fread\n");
 #endif
-      //if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) {   // pas complet..
       if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
         fclose(fpout);
         sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
@@ -257,12 +256,10 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
       printf("addfiles\n");
 #endif
       {
-        //unsigned int acess;
         unsigned int Class;
         unsigned int SClass;
         int i;
 
-        //acess = readshort(fpout);
         Class = readshort(fpout);
         SClass = readshort(fpout);
 
@@ -407,11 +404,9 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
 
       p = &buffer[0];
 
-      //fflush(fp); 
       trans.file_position = ftell(fp);
       length = readshort(fp);
       if (length < HTS_URLMAXSIZE) {
-        // while ((length > 0) && (length<500)) {
         while(length > 0) {
           *p++ = fgetc(fp);
 
@@ -419,17 +414,13 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
         }
         *p = '\0';
 
-        //#if JDEBUG
-        //      if(tris(buffer)==1) printf("%s\n ",buffer);
-        //      if(tris(buffer)==2)  printf("%s\n ",printname(buffer));
-        //#endif
         if (tris(str->opt, buffer) == 1)
           str->addLink(str, buffer);    /* trans.file_position */
         else if (tris(str->opt, buffer) == 2)
           str->addLink(str, printname(rname, buffer));
 
         strcpy(trans.name, buffer);
-      } else {                  // gros pb
+      } else { // gros pb
         while((length > 0) && (!feof(fp))) {
           fgetc(fp);
           length--;
@@ -445,7 +436,6 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
     }
     break;
   default:
-    // printf("Type inconnue\n");
     // on arrête tout 
     sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
     *error = 1;
@@ -508,7 +498,6 @@ static char *printname(char rname[1024], char name[1024]) {
     return rname;  // ""
   }
   p += 2;
-  //rname=(char*)calloct(strlen(name)+8,sizeof(char));
   p1 = rname;
   for(j = 0; name[j] != '\0'; j++, p++) {
     if (*p == '/')

src/htslib.c

@@ -598,56 +598,27 @@ static const char *hts_mime_modern[][2] = {
 
 // Reserved (RFC2396)
 #define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
-#define CHAR_RESERVED(c)  ( CIS(c,';') \
+#define CHAR_RESERVED(c)                                                       \
-                         || CIS(c,'/') \
+  (CIS(c, ';') || CIS(c, '/') || CIS(c, '?') || CIS(c, ':') || CIS(c, '@') ||  \
-                         || CIS(c,'?') \
+   CIS(c, '&') || CIS(c, '=') || CIS(c, '+') || CIS(c, '$') || CIS(c, ','))
-                         || CIS(c,':') \
-                         || CIS(c,'@') \
-                         || CIS(c,'&') \
-                         || CIS(c,'=') \
-                         || CIS(c,'+') \
-                         || CIS(c,'$') \
-                         || CIS(c,',') )
-//#define CHAR_RESERVED(c)  ( strchr(";/?:@&=+$,",(unsigned char)(c)) != 0 )
 // Delimiters (RFC2396)
-#define CHAR_DELIM(c)     ( CIS(c,'<') \
+#define CHAR_DELIM(c)                                                          \
-                         || CIS(c,'>') \
+  (CIS(c, '<') || CIS(c, '>') || CIS(c, '#') || CIS(c, '%') || CIS(c, '\"'))
-                         || CIS(c,'#') \
-                         || CIS(c,'%') \
-                         || CIS(c,'\"') )
-//#define CHAR_DELIM(c)     ( strchr("<>#%\"",(unsigned char)(c)) != 0 )
 // Unwise (RFC2396)
-#define CHAR_UNWISE(c)    ( CIS(c,'{') \
+#define CHAR_UNWISE(c)                                                         \
-                         || CIS(c,'}') \
+  (CIS(c, '{') || CIS(c, '}') || CIS(c, '|') || CIS(c, '\\') || CIS(c, '^') || \
-                         || CIS(c,'|') \
+   CIS(c, '[') || CIS(c, ']') || CIS(c, '`'))
-                         || CIS(c,'\\') \
-                         || CIS(c,'^') \
-                         || CIS(c,'[') \
-                         || CIS(c,']') \
-                         || CIS(c,'`') )
-//#define CHAR_UNWISE(c)    ( strchr("{}|\\^[]`",(unsigned char)(c)) != 0 )
 // Special (escape chars) (RFC2396 + >127 )
 #define CHAR_LOW(c)       ( ((unsigned char)(c) <= 31) )
 #define CHAR_HIG(c)       ( ((unsigned char)(c) >= 127) )
 #define CHAR_SPECIAL(c)   ( CHAR_LOW(c) || CHAR_HIG(c) )
 // We try to avoid them and encode them instead
-#define CHAR_XXAVOID(c)   ( CIS(c,' ') \
+#define CHAR_XXAVOID(c)                                                        \
-                         || CIS(c,'*') \
+  (CIS(c, ' ') || CIS(c, '*') || CIS(c, '\'') || CIS(c, '\"') ||               \
-                         || CIS(c,'\'') \
+   CIS(c, '&') || CIS(c, '!'))
-                         || CIS(c,'\"') \
+#define CHAR_MARK(c)                                                           \
-                         || CIS(c,'&') \
+  (CIS(c, '-') || CIS(c, '_') || CIS(c, '.') || CIS(c, '!') || CIS(c, '~') ||  \
-                         || CIS(c,'!') )
+   CIS(c, '*') || CIS(c, '\'') || CIS(c, '(') || CIS(c, ')'))
-//#define CHAR_XXAVOID(c)   ( strchr(" *'\"!",(unsigned char)(c)) != 0 )
-#define CHAR_MARK(c)      ( CIS(c,'-') \
-                         || CIS(c,'_') \
-                         || CIS(c,'.') \
-                         || CIS(c,'!') \
-                         || CIS(c,'~') \
-                         || CIS(c,'*') \
-                         || CIS(c,'\'') \
-                         || CIS(c,'(') \
-                         || CIS(c,')') )
-//#define CHAR_MARK(c)      ( strchr("-_.!~*'()",(unsigned char)(c)) != 0 )
 
 // conversion éventuelle / vers antislash
 #ifdef _WIN32
@@ -841,15 +812,12 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
 // treat: traiter header?
 // waitconnect: attendre le connect()
 // note: dans retour, on met les params du proxy
-T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
+T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
-                  const char *xsend, const char *adr, const char *fil, htsblk * retour) {
+                  const char *xsend, const char *adr, const char *fil,
-  //htsblk retour;
+                  htsblk *retour) {
-  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
   T_SOC soc = INVALID_SOCKET;
   char BIGSTK tempo_fil[HTS_URLMAXSIZE * 2];
 
-  //char *p,*q;
-
   // retour prédéfini: erreur
   if (retour) {
     retour->adr = NULL;
@@ -934,7 +902,6 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
         strcpybuff(retour->msg, "Unable to open local file");
       else {
         // Note: On passe par un FILE* (plus propre)
-        //soc=open(fil,O_RDONLY,0);    // en lecture seule!
         retour->fp = FOPEN(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), 
           unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil)), "rb");      // ouvrir
         if (retour->fp == NULL)
@@ -1011,17 +978,10 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
 
         } while(strnotempty(rcvd));
 
-        //rcvsize=-1;    // forCER CHARGEMENT INCONNU
+      } else { // si GET, on recevra l'en tête APRES
-
-        //if (retour)
-        //  retour->totalsize=rcvsize;
-
-      } else {                  // si GET, on recevra l'en tête APRES
-        //rcvsize=-1;    // on ne connait pas la taille de l'en-tête
         if (retour)
           retour->totalsize = -1;
       }
-
     }
 
   }
@@ -1099,10 +1059,10 @@ static int append_cookie_header(buff_struct *bstr, t_cookie *cookie,
   return cook;
 }
 
-/* Self-test entry for append_cookie_header(): build the request Cookie line
+/* Build the request Cookie line for domain/path into dst (always
-   into dst (always NUL-terminated). Returns the number of cookies emitted. */
+   NUL-terminated). Returns the number of cookies emitted. */
-int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
+int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
-                                const char *path, char *dst, size_t dst_size) {
+                       char *dst, size_t dst_size) {
   buff_struct bstr = {dst, dst_size, 0};
 
   assertf(dst != NULL && dst_size > 0);
@@ -1118,19 +1078,12 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
   char BIGSTK buffer_head_request[16384];
   buff_struct bstr = { buffer_head_request, sizeof(buffer_head_request), 0 };
 
-  //int use_11=0;     // HTTP 1.1 utilisé
   int direct_url = 0;           // ne pas analyser l'url (exemple: ftp://)
   const char *search_tag = NULL;
 
   // Initialize buffer
   buffer_head_request[0] = '\0';
 
-  // header Date
-  //strcatbuff(buff,"Date: ");
-  //time_gmt_rfc822(buff);    // obtenir l'heure au format rfc822
-  //sendc("\n");
-  //strcatbuff(buff,buff);
-
   // possibilité non documentée: >post: et >postfile:
   // si présence d'un tag >post: alors executer un POST
   // exemple: http://www.example.com/test.cgi?foo>post:posteddata=10&foo=5
@@ -1224,11 +1177,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
     }
 
     // protocole
-    if (!retour->req.http11) {  // forcer HTTP/1.0
+    if (!retour->req.http11) { // forcer HTTP/1.0
-      //use_11=0;
       print_buffer(&bstr, " HTTP/1.0\x0d\x0a");
-    } else {                    // Requète 1.1
+    } else { // Requète 1.1
-      //use_11=1;
       print_buffer(&bstr, " HTTP/1.1\x0d\x0a");
     }
 
@@ -1364,9 +1315,7 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
           print_buffer(&bstr, "Authorization: Basic %s"H_CRLF, autorisation);
         }
       }
-
     }
-    //strcatbuff(buff,"Accept-Charset: iso-8859-1,*,utf-8\n");
 
     // Custom header(s)
     if (strnotempty(retour->req.headers)) {
@@ -1520,14 +1469,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a += strlen("filename=");
         while(is_space(*a))
           a++;
-        //a=strchr(a,'"');
         if (a) {
           char *c = NULL;
 
-          //a++;      /* jump " */
           while((c = strchr(a, '/')))   /* skip all / (see RFC2616) */
             a = c + 1;
-          //b=strchr(a+1,'"');
           b = a + strlen(a) - 1;
           while(is_space(*b))
             b--;
@@ -1544,16 +1490,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
   } else if ((p = strfield(rcvd, "Last-Modified:")) != 0) {
     while(is_realspace(*(rcvd + p)))
       p++;                      // sauter espaces
-    if ((int) strlen(rcvd + p) < 64) {  // pas trop long?
+    if ((int) strlen(rcvd + p) < 64) { // pas trop long?
-      //struct tm* tm_time=convert_time_rfc822(rcvd+p);
       strcpybuff(retour->lastmodified, rcvd + p);
     }
   } else if ((p = strfield(rcvd, "Date:")) != 0) {
     if (strnotempty(retour->lastmodified) == 0) {       /* pas encore de last-modified */
       while(is_realspace(*(rcvd + p)))
         p++;                    // sauter espaces
-      if ((int) strlen(rcvd + p) < 64) {        // pas trop long?
+      if ((int) strlen(rcvd + p) < 64) { // pas trop long?
-        //struct tm* tm_time=convert_time_rfc822(rcvd+p);
         strcpybuff(retour->lastmodified, rcvd + p);
       }
     }
@@ -1566,14 +1510,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       else                      // erreur.. ignorer
         retour->etag[0] = '\0';
     }
-  }
+  } else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
-  // else if ((p=strfield(rcvd,"Transfer-Encoding: chunked"))!=0) {  // chunk!
-  else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) {   // chunk!
     while(is_realspace(*(rcvd + p)))
       p++;                      // sauter espaces
     if (strfield(rcvd + p, "chunked")) {
-      retour->is_chunk = 1;     // chunked
+      retour->is_chunk = 1; // chunked
-      //retour->http11=2;     // chunked
 #if HDEBUG
       printf("ok, Transfer-Encoding: détecté\n");
 #endif
@@ -1621,7 +1562,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       }
       // An empty/whitespace Content-Type value yields no token: keep the
       // sentinel default rather than reading an uninitialized tempo.
-      if (sscanf(rcvd + p, "%s", tempo) == 1) {
+      if (sscanf(rcvd + p, "%1099s", tempo) == 1) { // tempo[1100], server data
         if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
           strcpybuff(retour->contenttype, tempo);
         else
@@ -1645,7 +1586,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a = strchr(rcvd + p, '/');
         if (a != NULL) {
           a++;
-          if (sscanf(a, LLintP, &retour->crange) == 1) {
+          if (sscanf(a, LLintP, &retour->crange) == 1 && retour->crange >= 0) {
             retour->crange_start = 0;
             retour->crange_end = retour->crange - 1;
           } else {
@@ -1653,6 +1594,12 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           }
         }
       }
+      // A valid Content-Range has no negative field; reject hostile values so
+      // the crange +/- 1 arithmetic downstream cannot sign-overflow (UB).
+      if (retour->crange_start < 0 || retour->crange_end < 0 ||
+          retour->crange < 0) {
+        retour->crange_start = retour->crange_end = retour->crange = 0;
+      }
     }
   } else if ((p = strfield(rcvd, "Connection:")) != 0) {
     char *a = rcvd + p;
@@ -1716,11 +1663,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         if (a)
           *a = '\0';
       }
-      sscanf(a, "%s", tempo);
+      // Bound to tempo[1100]; no token => leave empty, don't read uninit tempo.
-      if (strlen(tempo) < 64)   // pas trop long!!
+      if (sscanf(a, "%1099s", tempo) == 1 && strlen(tempo) < 64)
         strcpybuff(retour->contentencoding, tempo);
       else
-        retour->contentencoding[0] = '\0';      // erreur
+        retour->contentencoding[0] = '\0';
 #if HTS_USEZLIB
       /* Check known encodings */
       if (retour->contentencoding[0]) {
@@ -1750,7 +1697,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           retour->location[0] = '\0';
       }
     }
-  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) && (cookie)) {  // ohh un cookie
+  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) &&
+             (cookie)) {        // ohh un cookie
     char *a = rcvd + p;         // pointeur
     char domain[256];           // domaine cookie (.netscape.com)
     char path[256];             // chemin (/)
@@ -1794,10 +1742,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
             a++;                // sauter espaces
           value_st = a;
           while((*a != ';') && (*a))
-            a++;                // prochain ;
+            a++; // prochain ;
-          //while( ((*a!='"') || (*(a-1)=='\\')) && (*a)) a++;    // prochain " (et pas \")
           value_end = a;
-          //if (*a==';') {  // finit par un ;
           // vérifier débordements
           if ((((int) (token_end - token_st)) < 200)
               && (((int) (value_end - value_st)) < 8000)
@@ -2058,6 +2004,15 @@ LLint http_xfread1(htsblk * r, int bufl) {
 
   if (bufl > 0) {
     if (!r->is_write) {         // stocker en mémoire
+      // In-memory content must fit a 32-bit index (allocs below add 1, reads
+      // use int offsets): reject a hostile Content-Length or endless stream.
+      const LLint inmem_want =
+          (r->totalsize >= 0) ? r->totalsize : (r->size + bufl);
+      if (inmem_want >= INT32_MAX) {
+        r->statuscode = STATUSCODE_INVALID;
+        strcpybuff(r->msg, "In-memory content too large");
+        return READ_ERROR;
+      }
       if (r->totalsize >= 0) {  // totalsize déterminé ET ALLOUE
         if (r->adr == NULL) {
           r->adr = (char *) malloct((size_t) r->totalsize + 1);
@@ -2139,8 +2094,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
             nl = READ_ERROR;
           }
         }
-        //if ((nl < 0) || ((r->totalsize>0) && (r->size >= r->totalsize)))
-        //  nl=-1;  // break
 
         // libérer bloc tempo
         freet(buff);
@@ -2228,9 +2181,6 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
   T_SOC soc;
   htsblk retour;
 
-  //int rcvsize=-1;
-  //char* rcv=NULL;    // adresse de retour
-  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
   TStamp tl;
   int timeout = 30;             // timeout pour un check (arbitraire) // **
 
@@ -2239,11 +2189,8 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
 
   loc[0] = '\0';
   hts_init_htsblk(&retour);
-  //memset(&retour, 0, sizeof(htsblk));    // effacer
   retour.location = loc;        // si non nul, contiendra l'adresse véritable en cas de moved xx
 
-  //soc=http_fopen(adr,fil,&retour,NULL);  // ouvrir, + header
-
   // on ouvre en head, et on traite l'en tête
   soc = http_xfopen(opt, 1, 0, 1, NULL, adr, fil, &retour);     // ouvrir HEAD, + envoi header
 
@@ -2565,6 +2512,10 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
   // effacer adrfil->adr et adrfil->fil
   adrfil->adr[0] = adrfil->fil[0] = '\0';
 
+  // Reject an over-long URL: a root-relative path is copied whole into fil[].
+  if (strlen(url) >= HTS_URLMAXSIZE * 2)
+    return -1;
+
 #if HDEBUG
   printf("protocol: %s\n", url);
 #endif
@@ -2619,7 +2570,6 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
 
     // chemin www... trop long!!
     if ((((int) (q - p))) > HTS_URLMAXSIZE) {
-      //strcpybuff(retour.msg,"Path too long");
       return -1;                // erreur
     }
     // recopier adrfil->adresse www..
@@ -2641,7 +2591,7 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
     if (*p == '/' || *p == '\\') {      /* adrfil->file:///.. */
       strcatbuff(adrfil->fil, p);       // fichier local ; adrfil->adr="#"
     } else {
-      if (p[1] != ':') {
+      if (*p == '\0' || p[1] != ':') {  /* empty path: not a DOS drive letter */
         strcatbuff(adrfil->fil, "//");  /* adrfil->file://server/foo */
         strcatbuff(adrfil->fil, p);
       } else {
@@ -2805,7 +2755,6 @@ void deletesoc_r(htsblk * r) {
 #if HTS_USEOPENSSL
   if (r->ssl_con) {
     SSL_shutdown(r->ssl_con);
-    // SSL_CTX_set_quiet_shutdown(r->ssl_con->ctx, 1);
     SSL_free(r->ssl_con);
     r->ssl_con = NULL;
   }
@@ -3104,12 +3053,11 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
 /* See http://physics.nist.gov/cuu/Units/binary.html */
 #define ToLLint(a) ((LLint)(a))
 #define ToLLintKiB (ToLLint(1024))
-#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
+#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
-#ifdef HTS_LONGLONG
+#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#define ToLLintPiB                                                             \
-#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
+  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#endif
 HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
   if (n < ToLLintKiB) {
     sprintf(strc->buff1, "%d", (int) (LLint) n);
@@ -3118,9 +3066,7 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
             (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
     strcpybuff(strc->buff2, "KiB");
-  }
+  } else if (n < ToLLintGiB) {
-#ifdef HTS_LONGLONG
-  else if (n < ToLLintGiB) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
             (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
     strcpybuff(strc->buff2, "MiB");
@@ -3137,13 +3083,6 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
             (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
     strcpybuff(strc->buff2, "PiB");
   }
-#else
-  else {
-    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
-            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
-    strcpybuff(strc->buff2, "MiB");
-  }
-#endif
   strc->buffadr[0] = strc->buff1;
   strc->buffadr[1] = strc->buff2;
   return strc->buffadr;
@@ -3194,7 +3133,6 @@ int finput(T_SOC fd, char *s, int max) {
   int j = 0;
 
   do {
-    //c=fgetc(fp);
     if (read((int) fd, &c, 1) <= 0) {
       c = 0;
     }
@@ -3210,7 +3148,7 @@ int finput(T_SOC fd, char *s, int max) {
         break;
       }
     }
-  } while((c != 0) && (j < max - 1));
+  } while ((c != 0) && (j < max - 1));
   s[j] = '\0';
   return j;
 }
@@ -4869,7 +4807,6 @@ LLint check_downloadable_bytes(int rate) {
 
     time_now = mtime_local();
     elapsed_useconds = time_now - HTS_STAT.istat_timestart[id_timer];
-    // NO totally stupid - elapsed_useconds+=1000;      // for the next second, too
     bytes_transferred_during_period =
       (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[id_timer]);
 
@@ -4882,32 +4819,6 @@ LLint check_downloadable_bytes(int rate) {
     return TAILLE_BUFFER;
 }
 
-//
-// 0 : OK
-// 1 : slow down
-#if 0
-int HTS_TOTAL_RECV_CHECK(int var) {
-  if (HTS_STAT.HTS_TOTAL_RECV_STATE)
-    return 1;
-  /*
-     {
-     if (HTS_STAT.HTS_TOTAL_RECV_STATE==3) { 
-     var = min(var,32); 
-     Sleep(250); 
-     } else if (HTS_STAT.HTS_TOTAL_RECV_STATE==2) { 
-     var = min(var,256); 
-     Sleep(100); 
-     } else { 
-     var/=2; 
-     if (var<=0) var=1; 
-     Sleep(50); 
-     } 
-     }
-   */
-  return 0;
-}
-#endif
-
 // Lecture dans buff de size octets au maximum en utilisant la socket r (structure htsblk)
 // returns: 
 // >0 : data received
@@ -4916,7 +4827,6 @@ int HTS_TOTAL_RECV_CHECK(int var) {
 int hts_read(htsblk * r, char *buff, int size) {
   int retour;
 
-  //  return read(soc,buff,size);
   if (r->is_file) {
 #if HTS_WIDE_DEBUG
     DEBUG_W("read(%p, %d, %d)\n" _(void *)buff _(int) size _(int) r->fp);
@@ -4934,7 +4844,6 @@ int hts_read(htsblk * r, char *buff, int size) {
     if (r->soc == INVALID_SOCKET)
       printf("!!WIDE_DEBUG ERROR, soc==INVALID hts_read\n");
 #endif
-    //HTS_TOTAL_RECV_CHECK(size);         // Diminuer au besoin si trop de données reçues
 #if HTS_USEOPENSSL
     if (r->ssl) {
       retour = SSL_read(r->ssl_con, buff, size);
@@ -5369,18 +5278,13 @@ SOCaddr* hts_dns_resolve(httrackp * opt, const char *_iadr, SOCaddr *const addr)
 
 // --- Tracage des mallocs() ---
 #ifdef HTS_TRACE_MALLOC
-//#define htsLocker(A, N) htsLocker(A, N)
 #define htsLocker(A, N) do {} while(0)
 static mlink trmalloc = { NULL, 0, 0, NULL };
 
 static int trmalloc_id = 0;
 static htsmutex *mallocMutex = NULL;
-static void hts_meminit(void) {
+
-  //if (mallocMutex == NULL) {
+static void hts_meminit(void) {}
-  //  mallocMutex = calloc(sizeof(*mallocMutex), 1);
-  //  htsLocker(mallocMutex, -999);
-  //}
-}
 void *hts_malloc(size_t len) {
   void *adr;
 
@@ -5464,7 +5368,6 @@ void hts_free(void *adr) {
               htsboundary);
       lnk->next = lnk->next->next;
       free((void *) blk_free);
-      //blk_free->id=-1;
       free((char *) adr - bsize);
       htsLocker(mallocMutex, 0);
       return;
@@ -5533,7 +5436,6 @@ mlink *hts_find(char *adr) {
 
     if (depl < 0)
       depl = -depl;
-    //assertf(depl < 512000);   /* near the stack frame.. doesn't look like malloc but stack variable */
     return NULL;
   }
 }
@@ -5589,8 +5491,7 @@ int ftp_available(void) {
 }
 #else
 int ftp_available(void) {
-  return 1;                     // ok!
+  return 1; // ok!
-  //return 0;   // SOUS UNIX, PROBLEMESs
 }
 #endif
 
@@ -5804,7 +5705,6 @@ HTSEXT_API int hts_init(void) {
       assertf("OpenSSL version seems vulnerable to heartbleed bug (CVE-2014-0160)" == NULL);
     }
 
-    // OpenSSL_add_all_algorithms();
     openssl_ctx = SSL_CTX_new(method);
     if (!openssl_ctx) {
       fprintf(stderr, "fatal: unable to initialize TLS: SSL_CTX_new()\n");
@@ -6111,6 +6011,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
   StringCopy(opt->strip_query, "");
   StringCopy(opt->cookies_file, "");
+  StringCopy(opt->why_url, "");
   opt->pause_min_ms = 0;
   opt->pause_max_ms = 0;
   opt->ftp_proxy = HTS_TRUE;
@@ -6126,7 +6027,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->log = stdout;
   opt->errlog = stderr;
   opt->flush = HTS_TRUE;
-  // opt->aff_progress=0;
   opt->keyboard = HTS_FALSE;
   //
   StringCopy(opt->path_html, "");
@@ -6136,8 +6036,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   //
   opt->maxlink = 100000;        // 100,000 liens max par défaut
   opt->maxfilter = 200;         // 200 filtres max par défaut
-  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT --
+  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON
-  //opt->maxcache_anticipate=256;  // maximum de liens à anticiper
+                                // PARAMETRABLE POUR L'INSTANT --
   opt->maxtime = -1;            // temps max en secondes
   opt->maxrate = 100000;        // taux maxi
   opt->maxconn = 5.0;           // nombre connexions/s
@@ -6259,6 +6159,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
     StringFree(opt->mod_blacklist);
     StringFree(opt->strip_query);
     StringFree(opt->cookies_file);
+    StringFree(opt->why_url);
 
     StringFree(opt->path_html);
     StringFree(opt->path_html_utf8);
@@ -6290,11 +6191,9 @@ const hts_stat_struct* hts_get_stats(httrackp * opt) {
 }
 
 // defaut wrappers
-static void __cdecl htsdefault_init(t_hts_callbackarg * carg) {
+static void __cdecl htsdefault_init(t_hts_callbackarg *carg) {}
-}
+
-static void __cdecl htsdefault_uninit(t_hts_callbackarg * carg) {
+static void __cdecl htsdefault_uninit(t_hts_callbackarg *carg) {}
-  // hts_freevar();
-}
 static int __cdecl htsdefault_start(t_hts_callbackarg * carg, httrackp * opt) {
   return 1;
 }

src/htslib.h

@@ -171,7 +171,6 @@ HTSEXT_API const char* hts_version(void);
 // fonctions unix/winsock
 int hts_read(htsblk * r, char *buff, int size);
 
-//int HTS_TOTAL_RECV_CHECK(int var);
 LLint check_downloadable_bytes(int rate);
 
 HTSEXT_API int hts_uninit_module(void);
@@ -185,12 +184,11 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
                   const char *referer_adr, const char *referer_fil,
                   htsblk * retour);
 /* Build the request "Cookie:" header line for stored cookies matching
-   domain/path into dst (NUL-terminated). Exposed for the -#Q self-test;
+   domain/path into dst (NUL-terminated), wrapping the logic http_sendhead()
-   wraps the same logic http_sendhead() uses. Returns cookies emitted. */
+   uses. Returns cookies emitted. */
-int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
+int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
-                                const char *path, char *dst, size_t dst_size);
+                       char *dst, size_t dst_size);
 
-//int newhttp(char* iadr,char* err=NULL);
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
               int waitconnect);
 /* Like newhttp(), but connect to the addr_index-th resolved address of the host

src/htsmodules.c

@@ -55,12 +55,6 @@ extern int fspc(httrackp * opt, FILE * fp, const char *type);
 
 /* >>> Put all modules variables here */
 
-#if 0
-t_gzopen gzopen = NULL;
-t_gzread gzread = NULL;
-t_gzclose gzclose = NULL;
-#endif
-
 int V6_is_available = HTS_INET6;
 
 static char WHAT_is_available[64] = "";
@@ -112,18 +106,6 @@ int hts_parse_externals(htsmoduleStruct * str) {
   return -1;
 }
 
-//static void addCallback(htscallbacks* chain, void* moduleHandle, htscallbacksfncptr exitFnc) {
-//  while(chain->next != NULL) {
-//    chain = chain->next;
-//  }
-//  chain->next = calloct(1, sizeof(htscallbacks));
-//  assertf(chain->next != NULL);
-//  chain = chain->next;
-//  memset(chain, 0, sizeof(*chain));
-//  chain->exitFnc = exitFnc;
-//  chain->moduleHandle = moduleHandle;
-//}
-
 void clearCallbacks(htscallbacks * chain_);
 void clearCallbacks(htscallbacks * chain_) {
   htscallbacks *chain;

src/htsname.c

@@ -1332,8 +1332,7 @@ int url_savename(lien_adrfilsave *const afs,
 
     while((a > afs->save) && (*a != '.') && (*a != '/'))
       a--;
-    if (*a != '.') {            // agh pas de point
+    if (*a != '.') {                         // agh pas de point
-      //strcatbuff(save,".none");                 // a éviter
       strcatbuff(afs->save, ".html");        // préférable!
       hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",
                     adr_complete, fil_complete, afs->save);
@@ -1471,17 +1470,6 @@ int url_savename(lien_adrfilsave *const afs,
   fil_simplifie(afs->save);
 
   /* convert name to UTF-8 ? Note: already done while parsing. */
-  //if (charset != NULL && charset[0] != '\0') {
-  //  char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
-
-  //  if (s != NULL) {
-  //    hts_log_print(opt, LOG_DEBUG,
-  //                  "engine: save-name: charset conversion from '%s' to '%s' using charset '%s'",
-  //                  save, s, charset);
-  //    strcpybuff(save, s);
-  //    free(s);
-  //  }
-  //}
 
   /* callback */
   RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr,
@@ -1629,10 +1617,6 @@ int url_savename(lien_adrfilsave *const afs,
         int sameAdr = (strfield2(heap(i)->adr, normadr) != 0);
         int sameFil;
 
-        // NO - URL hack is only for stripping // and www.
-        //if (opt->urlhack != 0)
-        //  sameFil = ( strfield2(heap(i)->fil, normfil) != 0);
-        //else
         sameFil = (strcmp(heap(i)->fil, normfil) == 0);
         if (sameAdr && sameFil) {       // ok c'est le même lien, adresse déja définie
           /* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */
@@ -1698,17 +1682,13 @@ int url_savename(lien_adrfilsave *const afs,
 
           strcpybuff(afs->save, tempo);
 
-          //printf("switched: %s\n",save);
-
         }                       // if
       }
 #if DEBUG_SAVENAME
       printf("\nEnd search, %s\n", fil_complete);
 #endif
-    } while(!nom_ok);
+    } while (!nom_ok);
-
   }
-  //printf("'%s' %s %s\n",save,adr,fil);
 
   return 0;
 }

src/htsopt.h

@@ -105,7 +105,6 @@ typedef struct htsfilters htsfilters;
 struct htsfilters {
   char ***filters; /**< pointer to the +/-pattern filter array */
   int *filptr;     /**< pointer to the current filter count */
-  // int*    filter_max;
 };
 
 /* User callbacks chain */
@@ -440,7 +439,6 @@ struct httrackp {
   float maxconn;          /**< max connections per second */
   int waittime;           /**< scheduled start time (wall-clock seconds) */
   hts_cachemode cache;    /**< cache generation mode */
-  // int aff_progress;     // progress bar
   hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
   t_proxy proxy;     /**< proxy configuration */
   hts_savename_83
@@ -492,7 +490,6 @@ struct httrackp {
   hts_verbosedisplay verbosedisplay; /**< animated text progress display */
   String footer; /**< footer/info line injected into pages */
   int maxcache;  /**< in-memory cache backing limit (bytes) */
-  // int maxcache_anticipate; // maximum links to anticipate (upper bound)
   hts_boolean ftp_proxy;    /**< use the HTTP proxy for FTP too */
   String filelist;          /**< file listing URLs to include */
   String urllist;           /**< file listing filters to include */
@@ -539,6 +536,8 @@ struct httrackp {
                                  (--cookies-file) */
   int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
   int pause_max_ms; /**< inter-file pause upper bound, ms */
+  String why_url;   /**< URL to diagnose (--why): print the deciding filter rule
+                         and exit without crawling */
 };
 
 /* Running statistics for a mirror. */
@@ -549,8 +548,6 @@ typedef struct hts_stat_struct hts_stat_struct;
 struct hts_stat_struct {
   LLint HTS_TOTAL_RECV; /**< total bytes received from the network */
   LLint stat_bytes;     /**< total bytes written to disk */
-  // int HTS_TOTAL_RECV_STATE;  // status: 0 ok 1: slow down a little 2: slow
-  // down 3: a lot
   TStamp stat_timestart; /**< mirror start time */
   //
   LLint total_packed;    /**< compressed bytes received (on the wire) */
@@ -682,7 +679,6 @@ struct lien_url {
   int depth;        /**< remaining allowed depth; >0 strong, 0 weak */
   int pass2;        /**< second-pass marker; -1 means handled in background */
   char link_import; /**< imported after a move; skip the usual up/down rules */
-  // int moved;          // pointer to moved
   int retry;    /**< remaining retries */
   int testmode; /**< test only: send just a HEAD */
 };
@@ -727,9 +723,6 @@ struct lien_back {
   LLint chunk_size; /**< size of the chunk being loaded */
   LLint chunk_blocksize; /**< data size declared by the chunk */
   LLint compressed_size; /**< compressed size (stats only) */
-  //
-  // int links_index;        // to access liens[links_index]
-  //
   char info[256]; /**< status text, e.g. for FTP */
   int stop_ftp;   /**< stop flag for FTP */
   int finalized;  /**< finalized (memory optimization) */

src/htsparse.c

@@ -395,16 +395,13 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
       // terminaison (" ou ') du "<body onLoad=.."
       int inscriptgen = 0;      // on est dans un code générant, ex après obj.write("..
 
-      //int inscript_check_comments=0, inscript_in_comments=0;    // javascript comments
       char scriptgen_q = '\0';  // caractère faisant office de guillemet (' ou ")
 
-      //int no_esc_utf=0;      // ne pas echapper chars > 127
       int nofollow = 0;         // ne pas scanner
 
       //
       int parseall_lastc = '\0';        // dernier caractère parsé pour parseall
 
-      //int parseall_incomment=0;   // dans un /* */ (exemple: a = /* URL */ "img.gif";)
       //
       const char *intag_start = html;
       const char *intag_name = NULL;
@@ -517,12 +514,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           lastsaved = html;
         }
       }
-      // Detect UTF8 format
-      //if (is_unicode_utf8(r->adr, (unsigned int) r->size) == 1) {
-      //  no_esc_utf=1;
-      //} else {
-      //  no_esc_utf=0;
-      //}
 
       // Hack to prevent any problems with ram files of other files
       *(r->adr + r->size) = '\0';
@@ -687,8 +678,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           ) {
           intag = 1;
           intag_ctype = 0;
-          //parseall_incomment=0;
-          //inquote=0;  // effacer quote
           intag_start = html;
           for(intag_name = html + 1; is_realspace(*intag_name); intag_name++) ;
           intag_start_valid = 1;
@@ -706,8 +695,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               if (len > 0) {
                 if (strfield(token, "content-type")) {
                   intag_ctype = 1;
-                  //NOPE-we do not convert the whole page actually
-                  //intag_start[1] = 'X';
                 } else if (strfield(token, "refresh")) {
                   intag_ctype = 2;
                 }
@@ -768,7 +755,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           html_inline_safe(urlfil(), safe_fil, sizeof(safe_fil)), gmttime,
                           HTTRACK_VERSIONID, /* EOF */ NULL);
                   strcatbuff(tempo, eol);
-                  //fwrite(tempo,1,strlen(tempo),fp);
                   HT_ADD(tempo);
                 }
                 // Emit charset ?
@@ -809,8 +795,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             intag = 0;          //inquote=0;
 
             // entrée dans du javascript?
-            // on parse ICI car il se peut qu'on ait eu a parser les src=.. dedans
+            // on parse ICI car il se peut qu'on ait eu a parser les src=..
-            //if (!inscript) {  // sinon on est dans un obj.write("..
+            // dedans
             if ((intag_start_valid) && (check_tag(intag_start, "script")
                                         || check_tag(intag_start, "style")
                 )
@@ -856,12 +842,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             }
 #endif
           }
-          //}
+        } else if (intag || inscript ||
-        }
+                   in_media) { // nous sommes dans un tag/commentaire, tester si
-        //else if (*adr==34) {
+                               // on recoit un tag
-        //  inquote=(inquote?0:1);
-        //}
-        else if (intag || inscript || in_media) {       // nous sommes dans un tag/commentaire, tester si on recoit un tag
           int p_type = 0;
           int p_nocatch = 0;
           int p_searchMETAURL = 0;      // chercher ..URL=<url>
@@ -1100,27 +1083,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       if (p) {
                         if (intag_ctype == 1) {
                           p = 0;
-#if 0
-                          //if ((pos=rech_tageq(html, "content"))) {
-                          char temp[256];
-                          char *token = NULL;
-                          int len = rech_endtoken(html + pos, &token);
-
-                          if (len > 0 && len < sizeof(temp) - 2) {
-                            char *chpos;
-
-                            temp[0] = '\0';
-                            strncat(temp, token, len);
-                            if ((chpos = strstr(temp, "charset"))
-                                && (chpos = strchr(chpos, '='))
-                              ) {
-                              chpos++;
-                              while(is_space(*chpos))
-                                chpod++;
-                              //chpos
-                            }
-                          }
-#endif
                         }
                         // <META HTTP-EQUIV="Refresh" CONTENT="3;URL=http://www.example.com">
                         else if (intag_ctype == 2) {
@@ -1305,21 +1267,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   }
                 } else if (inscript) {
 
-#if 0
-                  /* Check // javascript comments */
-                  if (*html == 10 || *html == 13) {
-                    inscript_check_comments = 1;
-                    inscript_in_comments = 0;
-                  } else if (inscript_check_comments) {
-                    if (!is_realspace(*html)) {
-                      inscript_check_comments = 0;
-                      if (html[0] == '/' && html[1] == '/') {
-                        inscript_in_comments = 1;
-                      }
-                    }
-                  }
-#endif
-
                   /* Parse */
                   assertf(inscript_name != NULL);
                   if (*html == '/'
@@ -1332,7 +1279,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       && inscript_locked == 0) {
                     const char *a = html;
 
-                    //while(is_realspace(*(--a)));
                     while(is_realspace(*a))
                       a--;
                     a--;
@@ -1343,8 +1289,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                         HT_ADD("<@@ /inscript @@>");
                       }
                     }
-                  } else if (inscript_state_pos ==
+                  } else if (inscript_state_pos == INSCRIPT_START) {
-                             INSCRIPT_START /*!inscript_in_comments */ ) {
                     /*
                        Script Analyzing - different types supported:
                        foo="url"
@@ -1533,12 +1478,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               }
 
             } else {            // ptr == 0
-              //p=rech_tageq(adr,"primary");    // lien primaire, yeah
               p = 0;            // No stupid tag anymore, raw link
               valid_p = 1;      // Valid even if p==0
               while((html[p] == '\r') || (html[p] == '\n'))
                 p++;
-              //can_avoid_quotes=1;
               ending_p = '\r';
             }
 
@@ -1551,25 +1494,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           // risque: générer de faux fichiers parazites
           // fix: ne parse plus dans les commentaires
           // ------------------------------------------------------------
-          if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 && (ptr > 0) && (!in_media) /* && (!inscript_in_comments) */ ) {  // option parsing "brut"
+          if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 &&
-            //int incomment_justquit=0;
+              (ptr > 0) && (!in_media)) { // option parsing "brut"
             if (!is_realspace(*html)) {
               int noparse = 0;
 
-              // Gestion des /* */
-#if 0
-              if (inscript) {
-                if (parseall_incomment) {
-                  if ((*html == '/') && (*(html - 1) == '*'))
-                    parseall_incomment = 0;
-                  incomment_justquit = 1;       // ne pas noter dernier caractère
-                } else {
-                  if ((*html == '/') && (*(html + 1) == '*'))
-                    parseall_incomment = 1;
-                }
-              } else
-                parseall_incomment = 0;
-#endif
               /* ensure automate state  0 (not in comments, quotes..) */
               if (inscript
                   && (inscript_state_pos != INSCRIPT_INQUOTE
@@ -1583,7 +1512,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
               // recherche d'URLs
               if (!noparse) {
-                //if ((!parseall_incomment) && (!noparse)) {
                 if (!p) {       // non déja trouvé
                   if (html != r->adr) {  // >1 caractère
                     // scanner les chaines
@@ -1607,9 +1535,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             && (count > 0)) {
                           char c;
 
-                          //char* aend;
-                          //
-                          //aend=a;     // sauver début
                           a++;
                           while(is_taborspace(*a))
                             a++;
@@ -1664,10 +1589,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                               if (!invalid_url) {
                                 // Un plus à la fin? Alors ne pas prendre sauf si extension ("/toto.html#"+tag)
                                 if (c != '+') { // PAS de plus à la fin
-#if 0
+                                  // "Comparisons of scheme names MUST be
-                                  char *a;
+                                  // case-insensitive" (RFC2616)
-#endif
-                                  // "Comparisons of scheme names MUST be case-insensitive" (RFC2616)                                  
                                   if ((strfield(tempo, "http:"))
                                       || (strfield(tempo, "ftp:"))
 #if HTS_USEOPENSSL
@@ -1680,15 +1603,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                     if (inscript)       // sinon si pas javascript, méfiance (répertoire style base?)
                                       url_ok = 1;
                                   }
-#if 0
-                                  else if ((a = strchr(tempo, '/'))) {  // un slash: ok..
-                                    if (inscript) {     // sinon si pas javascript, méfiance (style "text/css")
-                                      if (strchr(a + 1, '/'))   // un seul / : abandon (STYLE type='text/css')
-                                        if (!strchr(tempo, ' '))        // avoid spaces (too dangerous for comments)
-                                          url_ok = 1;
-                                    }
-                                  }
-#endif
                                 }
                                 // Prendre si extension reconnue
                                 if (!url_ok) {
@@ -1767,9 +1681,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       }
                     }
                   }
-                }               // p == 0               
+                } // p == 0
 
-              }                 // not in comment
+              } // not in comment
 
               // plus dans un commentaire
               if (inscript_state_pos == INSCRIPT_START
@@ -1778,17 +1692,15 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               }
 
             }                   // if realspace
-          }                     // if parseall
+          } // if parseall
 
           // ------------------------------------------------------------
           // p!=0 : on a repéré un éventuel lien
           // ------------------------------------------------------------
           //
-          if ((p > 0) || (valid_p)) {   // on a repéré un lien
+          if ((p > 0) || (valid_p)) { // on a repéré un lien
-            //int lien_valide=0;
             const char *eadr = NULL;  /* fin de l'URL */
 
-            //char* quote_adr=NULL;     /* adresse du ? dans l'adresse */
             int ok = 1;
             char quote = '\0';
             int quoteinscript = 0;
@@ -1811,7 +1723,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               lastsaved = html;  // dernier écrit+1
             }
             // sauter espaces
-            // adr+=p;
             hts_automate_increment(&saut, p);
             while((is_space(*html)
                    || (inscriptgen && html[0] == '\\' && is_space(html[1])
@@ -1840,8 +1751,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             // sauter éventuel \" ou \' javascript
             if (inscript) {     // on est dans un obj.write("..
               if (*html == '\\') {
-                if ((*(html + 1) == '\'') || (*(html + 1) == '"')) {      // \" ou \'
+                if ((*(html + 1) == '\'') || (*(html + 1) == '"')) { // \" ou \'
-                  // html+=2;    // sauter
                   hts_automate_increment(&saut, 2);
                 }
               }
@@ -1916,7 +1826,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   ok = -1;      // ne pas traiter ce lien
 
                 if (ok > 0) {
-                  //if (*eadr!=' ') {  
                   if (is_space(*eadr)) {        // guillemets,CR, etc
                     if ((*eadr == quote && (!quoteinscript || *(eadr - 1) == '\\'))     // end quote
                         || (noquote && (*eadr == '\"' || *eadr == '\''))        // end at any quote
@@ -1962,7 +1871,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       break;
                     }
                   }
-                  //}
                 }
                 eadr++;
               } while(ok == 1);
@@ -1991,14 +1899,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               char BIGSTK lien[HTS_URLMAXSIZE * 2];
               int meme_adresse = 0;     // 0 par défaut pour primary
 
-              //char *copie_de_adr=html;
-              //char* p;
-
               // construire lien (découpage)
               if (eadr - html - 1 < HTS_URLMAXSIZE) {        // pas trop long?
                 strncpy(lien, html, eadr - html - 1);
                 lien[eadr - html - 1] = '\0';
-                //printf("link: %s\n",lien);          
                 // supprimer les espaces
                 while((lien[strlen(lien) - 1] == ' ') && (strnotempty(lien)))
                   lien[strlen(lien) - 1] = '\0';
@@ -2103,7 +2007,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                 /* Unescape/escape %20 and other &nbsp; */
                 {
-                  // Note: always true (iso-8859-1 as default)
+                  // NULL when UTF-8 conversion is off (-%T0)
                   const char *const charset = str->page_charset_;
                   const int hasCharset = charset != NULL 
                     && *charset != '\0';
@@ -2126,11 +2030,12 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                   // Force to encode non-printable chars (should never happend)
                   escape_remove_control(lien);
-                  
+
-                  // charset conversion for the URI filename, 
+                  // charset conversion for the URI filename (not the query
-                  // and not already UTF-8
+                  // string), unless the bytes already are valid UTF-8:
-                  // (note: not for the query string!)
+                  // converting those would double-encode them (#180)
-                  if (hasCharset && !hts_isCharsetUTF8(charset)) {
+                  if (hasCharset && !hts_isCharsetUTF8(charset) &&
+                      !hts_isStringUTF8(lien, strlen(lien))) {
                     char *const s = hts_convertStringToUTF8(lien, strlen(lien), charset);
                     if (s != NULL) {
                       hts_log_print(opt, LOG_DEBUG,
@@ -2242,7 +2147,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     a += 2;
                   else
                     a = lien;
-                  // while((*a) && (*a!='/') && (*a!=':')) a++;
                   a = jump_toport(a);
                   if (a) {      // port
                     int port = 0;
@@ -2250,9 +2154,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     char *b = a + 1;
 
 #if HTS_USEOPENSSL
-                    // FIXME
-                    //if (strfield(adr, "https:")) {
-                    //}
 #endif
                     while(isdigit((unsigned char) *b)) {
                       port *= 10;
@@ -2343,13 +2244,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             *(a + 1) = '\0';
                           }
                         }
-                        //char BIGSTK tempo[HTS_URLMAXSIZE*2];
-                        //strcpybuff(tempo,"http://");
-                        //strcatbuff(tempo,urladr());    // host
-                        //if (*lien!='/')
-                        //  strcatbuff(tempo,"/");
-                        //strcatbuff(tempo,lien);
-                        //strcpybuff(lien,tempo);
                       }
                     }
 
@@ -2378,22 +2272,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       }
                       // stocker base ou codebase?
                       switch (p_type) {
-                      case 2:{
+                      case 2: {
-                          //if (*lien!='/') strcatbuff(base,"/");
+                        strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
-                          strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
+                      } break; // base
-                        }
+                      case -2: {
-                        break;  // base
+                        strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
-                      case -2:{
+                      } break; // base
-                          //if (*lien!='/') strcatbuff(codebase,"/");
-                          strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
-                        }
-                        break;  // base
                       }
 
                       hts_log_print(opt, LOG_DEBUG,
                                     "code/codebase link %s base %s", lien,
                                     base);
-                      //printf("base code: %s - %s\n",lien,base);
                     }
 
                   } else {
@@ -2506,7 +2395,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 // Tester si un lien doit être accepté ou refusé (wizard)
                 // forbidden_url=1 : lien refusé
                 // forbidden_url=0 : lien accepté
-                //if ((ptr>0) && (p_type!=2) && (p_type!=-2)) {    // tester autorisations?
                 if ((p_type != 2) && (p_type != -2)) {  // tester autorisations?
                   if (!p_nocatch) {
                     if (afs.af.adr[0] != '\0') {
@@ -2556,9 +2444,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                       /* Calc */
                       last_adr[0] = '\0';
-                      //char last_fil[HTS_URLMAXSIZE*2]="";
+                      strcpybuff(last_adr, afs.af.adr); // ancienne adresse
-                      strcpybuff(last_adr, afs.af.adr);        // ancienne adresse
-                      //strcpybuff(last_fil,fil);    // ancien chemin
                       r_sv =
                         url_savename(&afs, &former, heap(ptr)->adr, heap(ptr)->fil, opt,
                                      sback, cache, hash, ptr,
@@ -2589,7 +2475,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             }
                           }
                         }
-                        //import_done=1;    // c'est un import!
                         meme_adresse = 0;       // on a changé
                       }
                     } else {
@@ -2787,11 +2672,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                               if (lienrelatif(tempo, sizeof(tempo), save,
                                               relativesavename()) == 0) {
                                 /* Never escape high-chars (we don't know the encoding!!) */
-                                inplace_escape_uri_utf(tempo, sizeof(tempo));  // escape with %xx
+                                inplace_escape_uri_utf(
-                                //if (!no_esc_utf)
+                                    tempo, sizeof(tempo)); // escape with %xx
-                                //  escape_uri(tempo);     // escape with %xx
-                                //else
-                                //  escape_uri_utf(tempo);     // escape with %xx
                                 HT_ADD_HTMLESCAPED(tempo);      // page externe
                                 if (add_url) {
                                   HT_ADD("?link=");     // page externe
@@ -3028,16 +2910,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           // unquoted url() (CSS/JS): keep parens escaped
                           if (ending_p == ')')
                             escape_url_parens(tempo, sizeof(tempo));
-
-                          //if (!no_esc_utf)
-                          //  escape_uri(tempo);     // escape with %xx
-                          //else {
-                          //  /* No escaping at all - remaining upper chars will be escaped below */
-                          //  /* FIXME - Should be done in all local cases */
-                          //  //x_escape_html(tempo);
-                          //  //escape_uri_utf(tempo);     // FIXME - escape with %xx
-                          //  //escape_uri(tempo);     // escape with %xx
-                          //}
                         }
                         hts_log_print(opt, LOG_DEBUG,
                                       "relative link at %s build with %s and %s: %s",
@@ -3095,22 +2967,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                               HT_ADD(tempo4);   // refresh code="
                             }
                           }
-                          //lastsaved=adr;    // dernier écrit+1
                         }
 
                         if ((opt->getmode & HTS_GETMODE_HTML) && (ptr > 0)) {
-                          // convert to local codepage - NOT, already converted into %NN, and passed to the remote server so we do not have anything to do
-                          //if (str->page_charset_ != NULL && *str->page_charset_ != '\0') {
-                          //  char *const local_save = hts_convertStringFromUTF8(tempo, strlen(tempo), str->page_charset_);
-                          //  if (local_save != NULL) {
-                          //    strcpybuff(tempo, local_save);
-                          //    free(local_save);
-                          //  } else {
-                          //    if ((opt->debug>1) && (opt->log!=NULL)) {
-                          //      fprintf(opt->log, "Warning: could not build local charset representation of '%s' in '%s'"LF, tempo, str->page_charset_);
-                          //    }
-                          //  }
-                          //}
 
                           // écrire le lien modifié, relatif
                           // Note: escape all chars, even >127 (no UTF)
@@ -3134,16 +2993,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       }
                     } // sinon le lien sera écrit normalement
 
-#if 0
-                    if (fexist(save)) { // le fichier existe..
-                      adr[0] = '\0';
-                      //if ((opt->debug>0) && (opt->log!=NULL)) {
-                      hts_log_print(opt, LOG_WARNING,
-                                    "Link has already been written on disk, cancelled: %s",
-                                    save);
-                    }
-#endif
-
                     /* Security check */
                     if (strlen(afs.save) >= HTS_URLMAXSIZE) {
                       afs.af.adr[0] = '\0';
@@ -3185,9 +3034,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                         /*
                            if (strnotempty(save)) {
                            if (ishtml(opt,save) == 1) {
-                           // descore_prio = 2;
                            } else {
-                           // descore_prio = 1;
                            }
                            }
                          */
@@ -3223,8 +3070,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           // >>>> CREER LE LIEN <<<<
                           //
                           // enregistrer lien à charger
-                          //heap_top()->adr[0]=heap_top()->fil[0]=heap_top()->sav[0]='\0';
-                          // même adresse: l'objet père est l'objet père de l'actuel
 
                           // DEBUT ROBOTS.TXT AJOUT
                           if (!just_test_it) {
@@ -3303,7 +3148,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             heap_top()->premier = heap(ptr)->premier;
                           else  // sinon l'objet père est le précédent lui même
                             heap_top()->premier = heap_top_index();
-                          // heap_top()->premier=ptr;
 
                           heap_top()->precedent = ptr;
                           // noter la priorité
@@ -3315,9 +3159,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           heap_top()->pass2 = pass_fix;
                           heap_top()->retry = opt->retry;
 
-                          //strcpybuff(heap_top()->adr,adr);
-                          //strcpybuff(heap_top()->fil,fil);
-                          //strcpybuff(heap_top()->sav,save); 
                           if (!just_test_it) {
                             hts_log_print(opt, LOG_DEBUG,
                                           "OK, NOTE: %s%s -> %s",
@@ -3352,7 +3193,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             if (eadr > html) {
               hts_automate_increment(&saut, (int) (eadr - 1 - html));
             }
-            // adr=eadr-1;  // ** sauter
 
             /* srcset candidate loop: skip the descriptor and comma, then
                re-enter the capture for the next URL. Backward goto, not a loop:
@@ -3382,9 +3222,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                inscript_state_pos = INSCRIPT_START;
                } */
 
-          }                     // if (p) 
+ } // if (p)
 
-        }                       // si '<' ou '>'
+        } // si '<' ou '>'
 
         // plus loin
         html++;                  // automate will be checked next loop
@@ -3454,9 +3294,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             *stre->exit_xh_ = 1;        // exit requested
             XH_uninit;
             return -1;
-            //adr = r->adr + r->size;  // exit
           } else if (opt->state._hts_cancel == 1) {
-            // adr = r->adr + r->size;  // exit
             nofollow = 1;       // moins violent
             opt->state._hts_cancel = 0;
           }
@@ -3514,9 +3352,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
         fp = NULL;
       }
     }
-    // sauver fichier
-    //structcheck(savename());
-    //filesave(opt,r->adr,r->size,savename());
 
   }                             // analyse OK
 
@@ -3555,17 +3390,11 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
   // DEBUT rattrapage des 301,302,307..
   // ------------------------------------------------------------
   if (!error) {
-    ////////{
-    // on a chargé un fichier en plus
-    // if (!error) stat_loaded+=r.size;
 
     // ------------------------------------------------------------
     // Rattrapage des 301,302,307 (moved) et 412,416 - les 304 le sont dans le backing 
     // ------------------------------------------------------------
     if (HTTP_IS_REDIRECT(r->statuscode)) {
-      //if (r->adr!=NULL) {   // adr==null si fichier direct. [catch: davename normalement si cgi]
-      //int i=0;
-      // char* p;
 
       hts_log_print(opt, LOG_WARNING, "%s for %s%s", r->msg, urladr(), urlfil());
 
@@ -3592,7 +3421,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
           // A same-file alias redirect must be followed, not stubbed (#159).
           const hts_boolean same_savefile = hts_redirect_same_savefile(
               opt, urladr(), urlfil(), moved->adr, moved->fil);
-          //if (ident_url_absolute(mov_url,moved->adr,moved->fil)!=-1) {    // ok URL reconnue
           // c'est (en gros) la même URL..
           // si c'est un problème de casse dans le host c'est que le serveur est buggé
           // ("RFC says.." : host name IS case insensitive)
@@ -3632,7 +3460,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
             } /* sinon traité normalement */
           }
 
-          //if ((strfield2(moved->adr,urladr())!=0) && (strfield2(moved->fil,urlfil())!=0)) {  // identique à casse près
           if (get_it == 1) {
             // court-circuiter le reste du traitement
             // et reculer pour mieux sauter
@@ -3643,9 +3470,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
             error = 1;
             hts_invalidate_link(opt, ptr);  // invalidate hashtable entry
             // noter NOUVEAU lien
-            //xxc xxc
-            //  set_prio_to=0+1;  // protection if the moved URL is an html page!!
-            //xxc xxc
             {
               // calculer lien et éventuellement modifier addresse/fichier
               if (url_savename(&savedmoved, NULL,
@@ -3683,8 +3507,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
               }
             }
 
-            //printf("-> %s %s %s\n",liens[lien_tot-1]->adr,liens[lien_tot-1]->fil,liens[lien_tot-1]->sav);
-
             // note métaphysique: il se peut qu'il y ait un index.html et un INDEX.HTML
             // sous DOS ca marche pas très bien... mais comme je suis génial url_savename()
             // est à même de régler ce problème
@@ -3806,7 +3628,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
 
       // cas où l'on peut reessayer
       switch (r->statuscode) {
-        //case -1: can_retry=1; break;
       case STATUSCODE_TIMEOUT:
         if (opt->hostcontrol) { // timeout et retry épuisés
           if ((opt->hostcontrol & HTS_HOSTCONTROL_BAN_TIMEOUT) &&
@@ -3938,7 +3759,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
     // FIN rattrapage des 301,302,307..
     // ------------------------------------------------------------
 
-  }                             // if !error
+  } // if !error
 
   /* Apply changes */
   ENGINE_SAVE_CONTEXT();
@@ -4005,9 +3826,6 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
           }
         }
       }
-      // On désalloue le buffer d'enregistrement des chemins créée, au cas où pendant la pause
-      // l'utilisateur ferait un rm -r après avoir effectué un tar
-      // structcheck_init(1);
       {
         FILE *fp =
           fopen(fconcat
@@ -4388,33 +4206,8 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
                     "link #%d is ready, no more on the stack, skipping: %s%s..",
                     ptr, urladr(), urlfil());
 
-      // prochain lien
+      return 2; // goto jump_if_done;
-      // ptr++;
-
-      return 2;                 // goto jump_if_done;
-
     }
-#if 0
-    /* FIXME - finalized HAS NO MORE THIS MEANING */
-    /* link put in cache by the backing system for memory spare - reclaim */
-    else if (back[b].finalized) {
-      assertf(back[b].r.adr == NULL);
-      /* read file in cache */
-      back[b].r =
-        cache_read_ro(opt, cache, back[b].url_adr, back[b].url_fil,
-                      back[b].url_sav, back[b].location_buffer);
-      /* ensure correct location buffer set */
-      back[b].r.location = back[b].location_buffer;
-      if (back[b].r.statuscode == STATUSCODE_INVALID) {
-        hts_log_print(opt, LOG_ERROR,
-                      "Unexpected error: %s%s not found anymore in cache",
-                      back[b].url_adr, back[b].url_fil);
-      } else {
-        hts_log_print(opt, LOG_DEBUG, "reclaim file %s%s (%d)", back[b].url_adr,
-                      back[b].url_fil, back[b].r.statuscode);
-      }
-    }
-#endif
 
     if (!opt->verbosedisplay) {
       if (!opt->quiet) {
@@ -4438,7 +4231,6 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
       }
       fflush(stdout);
     }
-    //}
 
     // ------------------------------------------------------------
     // Vérificateur d'intégrité
@@ -4519,6 +4311,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
       *forbidden_url == 0 && IS_DELAYED_EXT(afs->save) && !opt->state.stop) {
     int loops;
     int continue_loop;
+    char BIGSTK
+        cookie_before[16384]; /* #15: this URL's Cookie header, pre-request */
 
     hts_log_print(opt, LOG_DEBUG, "Waiting for type to be known: %s%s", afs->af.adr,
                   afs->af.fil);
@@ -4527,6 +4321,12 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
     for(loops = 0, continue_loop = 1;
         IS_DELAYED_EXT(afs->save) && continue_loop && loops < 7; loops++) {
       continue_loop = 0;
+      /* #15: snapshot the Cookie header THIS url would send, so only its own
+         self-redirect Set-Cookie trips the retry, not a concurrent slot's. */
+      cookie_before[0] = '\0';
+      if (opt->accept_cookie && opt->cookie != NULL)
+        http_cookie_header(opt->cookie, jump_identification_const(afs->af.adr),
+                           afs->af.fil, cookie_before, sizeof(cookie_before));
 
       /* Wait for an available slot */
       if (!hts_wait_available_socket(sback, opt, cache, ptr))
@@ -4606,9 +4406,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
         if (back[b].r.statuscode == STATUSCODE_INVALID && back[b].r.adr == NULL) {
           lien_back delayed_back;
 
-          //char BIGSTK delayed_ctype[128];
-          // delayed_ctype[0] = '\0';
-          // strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1);    // copier content-type
           back_copy_static(&back[b], &delayed_back);
 
           /* Delete entry */
@@ -4713,9 +4510,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
         if (b >= 0) {
           lien_back delayed_back;
 
-          //char BIGSTK delayed_ctype[128];
-          //delayed_ctype[0] = '\0';
-          //strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1);    // copier content-type
           back_copy_static(&back[b], &delayed_back);
 
           /* Error */
@@ -4730,6 +4524,17 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           /* Moved! */
           else if (HTTP_IS_REDIRECT(back[b].r.statuscode)) {
             char BIGSTK mov_url[HTS_URLMAXSIZE * 2];
+            char BIGSTK cookie_after[16384];
+            hts_boolean cookies_changed;
+
+            /* #15: cookie-wall signal: this URL's own Cookie header changed
+               across its self-redirect (a Set-Cookie it just set). */
+            cookie_after[0] = '\0';
+            if (opt->accept_cookie && opt->cookie != NULL)
+              http_cookie_header(
+                  opt->cookie, jump_identification_const(afs->af.adr),
+                  afs->af.fil, cookie_after, sizeof(cookie_after));
+            cookies_changed = strcmp(cookie_before, cookie_after) != 0;
 
             mov_url[0] = '\0';
             strcpybuff(mov_url, back[b].r.location);    // copier URL
@@ -4799,11 +4604,24 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
                   url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, 
                                opt, sback, cache, hash, ptr, numero_passe,
                                &delayed_back);
+                } else if (cookies_changed) {
+                  // #15: cookie-wall self-redirect; evict the cached
+                  // fast-header so the re-issue refetches with the cookie.
+                  if (cache->cached_tests != NULL)
+                    coucal_remove(cache->cached_tests,
+                                  concat(OPT_GET_BUFF(opt),
+                                         OPT_GET_BUFF_SIZE(opt), afs->af.adr,
+                                         afs->af.fil));
+                  afs->save[0] = '\0';
+                  url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, opt,
+                               sback, cache, hash, ptr, numero_passe,
+                               &delayed_back);
+                  continue_loop = 1;
                 } else {
                   hts_log_print(opt, LOG_WARNING,
                                 "Unable to test %s%s (loop to same filename)",
                                 afs->af.adr, afs->af.fil);
-                }               // loop to same location
+                } // loop to same location
               }                 // ident_url_relatif()
             }                   // location
           }                     // redirect
@@ -4867,13 +4685,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
       /* 'no error page' selected or file discarded by size rules! */
       if (!opt->errpage || (in_error == STATUSCODE_TOO_BIG)) {
         /* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */
-#if 0
+        /* Do not post-exclude the link here (*forbidden_url): the cache system
-        /* No (3.43) - don't do that. We must not post-exclude an authorized link, because this will prevent the cache
+           would never process it again, and it would be refetched endlessly */
-           system from processing it, leading to refetch it endlessly. Just accept it, and handle the error as
-           usual during parsing.
-         */
-        *forbidden_url = 1;     /* Forbidden! */
-#endif
         if (in_error == STATUSCODE_TOO_BIG) {
           hts_log_print(opt, LOG_INFO,
                         "link not taken because of its size (%d bytes) at %s%s",

src/htsparse.h

@@ -66,7 +66,6 @@ struct htsmoduleStructExtended {
   char ***filters_;
   robots_wizard *robots_;
   hash_struct *hash_;
-  //int *lien_max_;
 
   /* Base & codebase */
   char *base;

src/htsselftest.c

@@ -46,6 +46,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsdefines.h"
 #include "htslib.h"
 #include "htsparse.h"
+#include "htscache.h"
 #include "htscache_selftest.h"
 #include "htsdns_selftest.h"
 #include "htscharset.h"
@@ -579,15 +580,16 @@ static int st_filter(httrackp *opt, int argc, char **argv) {
   int matched;
 
   (void) opt;
-  if (argc < 2) {
+  if (argc < 1) {
     fprintf(stderr, "filter: needs a filter pattern and a string\n");
     return 1;
   }
-  /* exact-size heap copies so a sanitizer traps any over-read of the pattern */
+  /* exact-size heap copies so a sanitizer traps an over-read; a missing
-  str = strdupt(argv[1]);
+     subject means "" (not reachable as a CLI arg) */
+  str = strdupt(argc >= 2 ? argv[1] : "");
   pat = strdupt(argv[0]);
   matched = strjoker(str, pat, NULL, NULL) != NULL;
-  printf("%s does %s %s\n", argv[1], matched ? "match" : "NOT match", argv[0]);
+  printf("%s does %s %s\n", str, matched ? "match" : "NOT match", argv[0]);
   freet(str);
   freet(pat);
   return 0;
@@ -619,6 +621,125 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Mime-type filter verdict via fa_strjoker(type=1): only mime: rules apply. */
+static int st_filtermime(httrackp *opt, int argc, char **argv) {
+  int verdict;
+
+  (void) opt;
+  if (argc < 2) {
+    fprintf(stderr, "filtermime: needs <mime> <filter> [filter...]\n");
+    return 1;
+  }
+  verdict = fa_strjoker(1, &argv[1], argc - 1, argv[0], NULL, NULL, NULL);
+  printf("verdict=%s\n", verdict > 0   ? "allowed"
+                         : verdict < 0 ? "forbidden"
+                                       : "unknown");
+  return 0;
+}
+
+/* SplitMix64: deterministic, platform-independent case generator. */
+static uint64_t st_mix64(uint64_t *state) {
+  uint64_t z = (*state += UINT64_C(0x9E3779B97F4A7C15));
+
+  z = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4B5B9);
+  z = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB);
+  return z ^ (z >> 31);
+}
+
+/* Differential test: memoized strjoker vs the no-memo oracle on seeded random
+   pattern/subject/size cases must agree on result, *size and *size_flag. */
+static int st_filtermemo(httrackp *opt, int argc, char **argv) {
+  static const char *const pieces[] = {
+      "a",       "b",        "A",    "c",      ".",       "/",
+      "?",       "*",        "*[a]", "*[ab]",  "*[a-c]",  "*[A-Z]",
+      "*[<5]",   "*[>5]",    "*(a)", "*(a,b)", "*[file]", "*[path]",
+      "*[name]", "*[param]", "*[]",  "*[\\a]", "*[a-"};
+  static const char subject_chars[] = "abAc./?";
+  uint64_t rng = UINT64_C(0x501);
+  int iters = 20000, matched = 0, unmatched = 0;
+  int it;
+
+  (void) opt;
+  if (argc >= 1)
+    sscanf(argv[0], "%d", &iters);
+  for (it = 0; it < iters; it++) {
+    char pat[64], str[16];
+    size_t pl = 0;
+    const int npieces = (int) (st_mix64(&rng) % 5);
+    const int slen = (int) (st_mix64(&rng) % 13);
+    const int szsel = (int) (st_mix64(&rng) % 4); /* none/unknown/3/20 */
+    LLint sz1, sz2, off1, off2;
+    int i, flag1 = 0, flag2 = 0;
+    char *hpat, *hstr;
+    const char *adr;
+
+    for (i = 0; i < npieces; i++) {
+      const char *p =
+          pieces[st_mix64(&rng) % (sizeof(pieces) / sizeof(pieces[0]))];
+      const size_t len = strlen(p);
+
+      if (len >= sizeof(pat) - pl)
+        break;
+      memcpy(pat + pl, p, len);
+      pl += len;
+    }
+    pat[pl] = '\0';
+    for (i = 0; i < slen; i++)
+      str[i] = subject_chars[st_mix64(&rng) % (sizeof(subject_chars) - 1)];
+    str[slen] = '\0';
+    sz1 = sz2 = (szsel <= 1) ? -1 : (szsel == 2) ? 3 : 20;
+    /* exact-size heap copies per run so a sanitizer traps any over-read */
+    hpat = strdupt(pat);
+    hstr = strdupt(str);
+    adr = strjoker(hstr, hpat, szsel ? &sz1 : NULL, szsel ? &flag1 : NULL);
+    off1 = adr != NULL ? (LLint) (adr - hstr) : -1;
+    freet(hpat);
+    freet(hstr);
+    hpat = strdupt(pat);
+    hstr = strdupt(str);
+    adr =
+        strjoker_nomemo(hstr, hpat, szsel ? &sz2 : NULL, szsel ? &flag2 : NULL);
+    off2 = adr != NULL ? (LLint) (adr - hstr) : -1;
+    freet(hpat);
+    freet(hstr);
+    if (off1 != off2 || sz1 != sz2 || flag1 != flag2) {
+      printf("filtermemo MISMATCH pat=[%s] str=[%s] szsel=%d: "
+             "off %d/%d size %d/%d flag %d/%d\n",
+             pat, str, szsel, (int) off1, (int) off2, (int) sz1, (int) sz2,
+             flag1, flag2);
+      return 1;
+    }
+    if (off1 >= 0)
+      matched++;
+    else
+      unmatched++;
+  }
+  /* both polarities must actually occur or the test proves nothing */
+  assertf(matched > 0 && unmatched > 0);
+  printf("filtermemo: %d cases OK\n", iters);
+  return 0;
+}
+
+/* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
+static int st_filterdual(httrackp *opt, int argc, char **argv) {
+  int depth = -1, verdict;
+
+  (void) opt;
+  if (argc < 3) {
+    fprintf(stderr,
+            "filterdual: needs <string1> <string2> <filter> [filter...]\n");
+    return 1;
+  }
+  verdict = fa_strjoker_dual(0, &argv[2], argc - 2, argv[0], argv[1], NULL,
+                             NULL, &depth);
+  printf("verdict=%s rule=%d\n",
+         verdict > 0   ? "allowed"
+         : verdict < 0 ? "forbidden"
+                       : "unknown",
+         depth);
+  return 0;
+}
+
 static int st_simplify(httrackp *opt, int argc, char **argv) {
   (void) opt;
   if (argc < 1) {
@@ -667,6 +788,30 @@ static int st_charset(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+static int st_metacharset(httrackp *opt, int argc, char **argv) {
+  char *s;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "metacharset: needs an html string\n");
+    return 1;
+  }
+  s = hts_getCharsetFromMeta(argv[0], strlen(argv[0]));
+  printf("%s\n", s != NULL ? s : "(none)");
+  freet(s);
+  return 0;
+}
+
+static int st_isutf8(httrackp *opt, int argc, char **argv) {
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "isutf8: needs a string\n");
+    return 1;
+  }
+  printf("%d\n", hts_isStringUTF8(argv[0], strlen(argv[0])) ? 1 : 0);
+  return 0;
+}
+
 static int st_idna_encode(httrackp *opt, int argc, char **argv) {
   char *s;
 
@@ -1094,6 +1239,33 @@ static int st_resolve(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Split a URL into (adr, fil), or print "error" if rejected. A second arg pads
+   the URL with that many 'a's to reach lengths a CLI arg can't. */
+static int st_identurl(httrackp *opt, int argc, char **argv) {
+  lien_adrfil af;
+  char *url;
+  size_t len, pad = 0;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "identurl: needs a URL\n");
+    return 1;
+  }
+  if (argc >= 2)
+    pad = (size_t) atoi(argv[1]);
+  len = strlen(argv[0]);
+  url = malloct(len + pad + 1);
+  memcpy(url, argv[0], len);
+  memset(url + len, 'a', pad);
+  url[len + pad] = '\0';
+  if (ident_url_absolute(url, &af) >= 0)
+    printf("adr=%s fil=%s\n", af.adr, af.fil);
+  else
+    printf("error\n");
+  freet(url);
+  return 0;
+}
+
 /* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
    no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
    registering an already-crawled link (dedup/collision paths). */
@@ -1115,6 +1287,102 @@ static int st_header(httrackp *opt, int argc, char **argv) {
     treathead(NULL, "www.example.com", "/", &r, line);
   }
   printf("contenttype=%s cdispo=%s\n", r.contenttype, r.cdispo);
+  printf("contentencoding=%s\n", r.contentencoding);
+  return 0;
+}
+
+/* An over-long header value must not overflow treathead's tempo[1100]. */
+static int st_headerlong(httrackp *opt, int argc, char **argv) {
+  htsblk r;
+  char BIGSTK line[HTS_URLMAXSIZE * 2];
+  const char *const name = argc >= 1 ? argv[0] : "Content-Type:";
+  const int pad = 1500; /* > tempo[1100] */
+  size_t n;
+
+  (void) opt;
+  memset(&r, 0, sizeof(r));
+  n = (size_t) snprintf(line, sizeof(line), "%s ", name);
+  memset(line + n, 'a', pad);
+  line[n + pad] = '\0';
+  treathead(NULL, "www.example.com", "/", &r, line);
+  printf("contenttype_len=%d contentencoding_len=%d\n",
+         (int) strlen(r.contenttype), (int) strlen(r.contentencoding));
+  return 0;
+}
+
+/* http_xfread1 must refuse an in-memory buffer whose size would exceed a 32-bit
+   index (hostile Content-Length or endless stream) rather than allocate it.
+   The guard returns before any socket read, so no real connection is needed. */
+static int st_xfread_limit(httrackp *opt, int argc, char **argv) {
+  htsblk r;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+
+  // Content-Length just over 2 GiB.
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = (LLint) INT32_MAX + 1;
+  printf("bylen: refused=%d adr=%s msg=%s\n",
+         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
+         r.msg);
+  if (r.adr != NULL)
+    freet(r.adr);
+
+  // Unknown length, buffer already at the limit: the next read would exceed it.
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = -1;
+  r.size = (LLint) INT32_MAX;
+  printf("bygrow: refused=%d adr=%s msg=%s\n",
+         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
+         r.msg);
+  if (r.adr != NULL)
+    freet(r.adr);
+
+  // Exactly at the 2 GiB index (size + bufl == INT32_MAX): must also be
+  // refused, since the reallocs below add 1 (a `> INT32_MAX` check would let
+  // this through and overflow the int realloc size).
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = -1;
+  r.size = (LLint) INT32_MAX - 8192;
+  http_xfread1(&r, 8192);
+  printf("boundary: msg=%s\n", r.msg);
+
+  // A legitimate small size must NOT be refused by the guard (the read then
+  // fails on the invalid socket, but the size-too-large msg must not be set).
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = 1000;
+  http_xfread1(&r, 8192);
+  printf("accept: msg=%s\n", r.msg);
+  if (r.adr != NULL)
+    freet(r.adr);
+  return 0;
+}
+
+/* Parse a Content-Range header and print the sanitized triple. A hostile value
+   (negative or INT64 extreme) must clamp to 0 without signed-overflow UB. */
+static int st_crange(httrackp *opt, int argc, char **argv) {
+  int i;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "crange: needs at least one raw Content-Range line\n");
+    return 1;
+  }
+  for (i = 0; i < argc; i++) {
+    htsblk r;
+    char BIGSTK line[HTS_URLMAXSIZE * 2];
+
+    memset(&r, 0, sizeof(r));
+    strcpybuff(line, argv[i]);
+    treathead(NULL, "www.example.com", "/", &r, line);
+    printf("crange_start=" LLintP " crange_end=" LLintP " crange=" LLintP "\n",
+           (LLint) r.crange_start, (LLint) r.crange_end, (LLint) r.crange);
+  }
   return 0;
 }
 
@@ -1322,6 +1590,78 @@ static int st_cache(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* A corrupt cache index (.ndx) must not walk the length-prefixed scan past
+   the buffer. Checks the two primitives the loader is built from. */
+static int st_cacheindex(httrackp *opt, int argc, char **argv) {
+  int fail = 0;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+
+  /* A length prefix that overstates the bytes present must bound the advance
+     to the buffer, not trust the declared length. */
+  {
+    static const char src[] = "32768\nCACHE-1.1";
+    const size_t len = sizeof(src) - 1;
+    char *buf = malloct(len + 1);
+    char s[256];
+    int off;
+
+    memcpy(buf, src, len + 1);
+    off = cache_brstr(buf, s, sizeof(s));
+    if (off > (int) len) {
+      printf("cacheindex: over-advance off=%d len=%d\n", off, (int) len);
+      fail = 1;
+    }
+    if (strcmp(s, "CACHE-1.1") != 0) {
+      printf("cacheindex: value=%s\n", s);
+      fail = 1;
+    }
+    freet(buf);
+  }
+
+  /* cache_binput reads a field while in bounds, but refuses one starting at
+     or past end-of-buffer. */
+  {
+    char buf[8] = "ab\ncd";
+    const char *const end = buf + 5;
+    char s[16];
+
+    if (cache_binput(buf, end, s, sizeof(s)) != 3 || strcmp(s, "ab") != 0)
+      fail = 1; /* normal read: "ab" then the '\n', 3 bytes consumed */
+    if (cache_binput(end, end, s, sizeof(s)) != 0 || s[0] != '\0')
+      fail = 1;
+  }
+
+  /* Drive the full loader scan over a truncated index: a declared length that
+     overshoots plus a half-written entry. ASan aborts here on the pre-fix
+     scan; the cursor must never leave the buffer. */
+  {
+    static const char src[] = "9\nCACHE-1.1\n99\nwww.example.com\n/a";
+    const size_t len = sizeof(src) - 1;
+    char *buf = malloct(len + 1);
+    const char *const end = buf + len;
+    char line[256];
+    char *a = buf;
+
+    memcpy(buf, src, len + 1);
+    a += cache_brstr(a, line, sizeof(line));
+    a += cache_brstr(a, line, sizeof(line));
+    while (a != NULL && a < end) {
+      a = strchr(a + 1, '\n');
+      if (a == NULL)
+        break;
+      a++;
+      a += cache_binput(a, end, line, sizeof(line));
+    }
+    freet(buf);
+  }
+
+  printf("cacheindex: %s\n", fail ? "FAIL" : "OK");
+  return fail;
+}
+
 static int st_cache_golden(httrackp *opt, int argc, char **argv) {
   int regen, err;
 
@@ -1359,6 +1699,18 @@ static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+static int st_cache_legacy(httrackp *opt, int argc, char **argv) {
+  int err;
+
+  if (argc < 1) {
+    fprintf(stderr, "cache-legacy: needs a directory\n");
+    return 1;
+  }
+  err = cache_legacy_refused_selftest(opt, argv[0]);
+  printf("cache-legacy: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 static int st_reconcile(httrackp *opt, int argc, char **argv) {
   int err;
 
@@ -1403,13 +1755,33 @@ static int st_cookies(httrackp *opt, int argc, char **argv) {
     return 1;
   }
 
-  http_cookie_header_selftest(&cookie, dom, "/", hdr, sizeof(hdr));
+  http_cookie_header(&cookie, dom, "/", hdr, sizeof(hdr));
   if (strcmp(hdr, expected) != 0)
     err = 1;
   if (strstr(hdr, "$Version") != NULL || strstr(hdr, "$Path") != NULL)
     err = 1;
   if (strstr(hdr, "junk") != NULL) // wrong-domain cookie leaked
     err = 1;
+#ifndef _WIN32
+  /* the jar holds live session cookies: cookie_save must keep it 0600 */
+  {
+    const char *jar = "st-cookies-jar.txt";
+    struct stat st;
+
+    (void) UNLINK(jar);
+    assertf(cookie_save(&cookie, jar) == 0);
+    assertf(stat(jar, &st) == 0);
+    assertf((st.st_mode & 07777) == HTS_PROTECT_FILE);
+    assertf(st.st_size > 0); /* mode-only checks would pass an empty jar */
+    /* a pre-existing world-readable jar must be tightened, not kept */
+    assertf(chmod(jar, 0644) == 0);
+    assertf(cookie_save(&cookie, jar) == 0);
+    assertf(stat(jar, &st) == 0);
+    assertf((st.st_mode & 07777) == HTS_PROTECT_FILE);
+    assertf(st.st_size > 0);
+    (void) UNLINK(jar);
+  }
+#endif
   printf("cookie-header: %s\n", err ? "FAIL" : "OK");
   if (err)
     printf("  got: %s\n", hdr);
@@ -1851,6 +2223,19 @@ static int ae_write_collision(const char *path, const unsigned char *src,
   return ok ? 0 : 1;
 }
 
+/* Write src[0..len) to path as-is; 0 on success. */
+static int ae_write_raw(const char *path, const unsigned char *src,
+                        size_t len) {
+  FILE *const f = FOPEN(path, "wb");
+  int ok;
+
+  if (f == NULL)
+    return 1;
+  ok = fwrite(src, 1, len, f) == len;
+  fclose(f);
+  return ok ? 0 : 1;
+}
+
 /* Compare path's bytes to expect[0..len); 0 if equal. Streams (large files). */
 static int ae_check_decoded(const char *path, const unsigned char *expect,
                             size_t len) {
@@ -1918,6 +2303,46 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
     assertf(ae_write_collision(inpath, body, 64) == 0);
     assertf(hts_zunpack(inpath, outpath) == 64);
     assertf(ae_check_decoded(outpath, body, 64) == 0);
+    /* Identity fallback (#47): a plain body mislabeled as compressed is kept
+       verbatim, small and multi-chunk (> one 8 KiB fread). */
+    assertf(ae_write_raw(inpath, small, slen) == 0);
+    assertf(hts_zunpack(inpath, outpath) == (int) slen);
+    assertf(ae_check_decoded(outpath, small, slen) == 0);
+    {
+      const size_t ilen = 16 * 1024;
+      unsigned char *idbody = malloct(ilen);
+
+      assertf(idbody != NULL);
+      for (i = 0; i < ilen; i++)
+        idbody[i] = small[i % slen];
+      assertf(ae_write_raw(inpath, idbody, ilen) == 0);
+      assertf(hts_zunpack(inpath, outpath) == (int) ilen);
+      assertf(ae_check_decoded(outpath, idbody, ilen) == 0);
+      freet(idbody);
+    }
+    /* Truncated gzip (CRC+ISIZE cut), zlib (ADLER32 cut) and raw deflate
+       must all still fail, not fall back to a verbatim copy. */
+    {
+      static const struct {
+        int wb;
+        size_t cut;
+      } tr[] = {{16 + MAX_WBITS, 8}, {MAX_WBITS, 4}, {-MAX_WBITS, 5}};
+
+      for (i = 0; i < sizeof(tr) / sizeof(tr[0]); i++) {
+        unsigned char z[512];
+        size_t zlen;
+        FILE *f;
+
+        assertf(ae_write_packed(inpath, tr[i].wb, small, slen) == 0);
+        f = FOPEN(inpath, "rb");
+        assertf(f != NULL);
+        zlen = fread(z, 1, sizeof(z), f);
+        fclose(f);
+        assertf(zlen > tr[i].cut && zlen < sizeof(z));
+        assertf(ae_write_raw(inpath, z, zlen - tr[i].cut) == 0);
+        assertf(hts_zunpack(inpath, outpath) < 0);
+      }
+    }
     freet(body);
   }
 #else
@@ -2106,6 +2531,12 @@ static const struct selftest_entry {
     {"filtersize", "<size> <string> <filter>...",
      "size-aware filter verdict (negative size = unknown/scan time)",
      st_filtersize},
+    {"filtermime", "<mime> <filter>...",
+     "mime-type filter verdict (fa_strjoker type=1)", st_filtermime},
+    {"filtermemo", "[iterations]",
+     "memoized vs unmemoized matcher differential", st_filtermemo},
+    {"filterdual", "<string1> <string2> <filter>...",
+     "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
@@ -2116,6 +2547,9 @@ static const struct selftest_entry {
     {"mime", "<filename>", "MIME type for a filename", st_mime},
     {"charset", "<charset> <string>",
      "convert a string to UTF-8 from a charset", st_charset},
+    {"metacharset", "<html>", "extract the <meta> charset from an HTML page",
+     st_metacharset},
+    {"isutf8", "<string>", "is the string valid UTF-8 (1/0)", st_isutf8},
     {"idna-encode", "<host>", "encode a hostname to IDNA/punycode",
      st_idna_encode},
     {"idna-decode", "<host>", "decode an IDNA/punycode hostname",
@@ -2132,19 +2566,31 @@ static const struct selftest_entry {
      st_relative},
     {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
      st_resolve},
+    {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
+    {"headerlong", "[header-name:]",
+     "over-long header value must not overflow the parse scratch",
+     st_headerlong},
+    {"crange", "<raw-content-range-line> ...",
+     "Content-Range parse integer safety", st_crange},
+    {"xfread-limit", "", "in-memory receive buffer size bound",
+     st_xfread_limit},
     {"savename", "<fil> <content-type> [key=value ...]",
      "local save-name for a URL", st_savename},
     {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
      st_sniff},
     {"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
+    {"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
+     st_cacheindex},
     {"cache-golden", "<dir> [regen]", "frozen cache-format read self-test",
      st_cache_golden},
     {"cache-writefail", "<dir>", "cache write-failure handling self-test",
      st_cache_writefail},
     {"reconcile", "<dir>", "cache generation reconcile policy self-test",
      st_reconcile},
+    {"cache-legacy", "<dir>", "pre-3.31 legacy cache refusal self-test",
+     st_cache_legacy},
     {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
      st_cache_corrupt},
     {"dns", "", "DNS resolver/cache self-test", st_dns},

src/htsserver.c

@@ -265,7 +265,6 @@ T_SOC smallserver_init(int *port, char *adr) {
       SOCaddr_initport(server, *port);
       if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
         if (listen(soc, 10) >= 0) {
-          // SOCaddr_inetntoa(adr, 128, server2);
           strcpy(adr, h_loc);
         } else {
 #ifdef _WIN32
@@ -439,7 +438,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
         meth = 10;
       } else {
 #ifdef _DEBUG
-        // assert(FALSE);
 #endif
       }
       if (meth) {
@@ -1360,7 +1358,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
 
             StringCat(headers, error_hdr);
             StringCat(output, error);
-            //assert(file == NULL);
           }
         }
       } else {
@@ -1388,12 +1385,10 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                   != StringLength(output)))
         ) {
 #ifdef _DEBUG
-        //assert(FALSE);
 #endif
       }
     } else {
 #ifdef _DEBUG
-      // assert(FALSE);
 #endif
     }
 
@@ -1540,8 +1535,7 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
             } while(strnotempty(test));
           }
 
-          if (!strnotempty(test)) {     // éviter doublons
+          if (!strnotempty(test)) { // éviter doublons
-            // conv_printf(key,key);
             const size_t len = strlen(intkey);
             char *const buff = (char *) malloc(len + 1);
 
@@ -1632,8 +1626,6 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
                     intkey = "";
                 } else {
                   if (loops > 0) {
-                    //err_msg += intkey;
-                    //err_msg += " ";
                   }
                 }
               }
@@ -1740,13 +1732,7 @@ static void LANG_DELETE(void) {
 }
 
 // sélection de la langue
-static void LANG_INIT(const char *path) {
+static void LANG_INIT(const char *path) { LANG_T(path, 0); }
-  //CWinApp* pApp = AfxGetApp();
-  //if (pApp) {
-  /* pApp->GetProfileInt("Language","IntId",0); */
-  LANG_T(path, 0 /*pApp->GetProfileInt("Language","IntId",0) */ );
-  //}
-}
 
 static int LANG_T(const char *path, int l) {
   if (l >= 0) {

src/htstools.c

@@ -68,16 +68,6 @@ struct find_handle_struct {
   char path[2048];
 };
 #endif
-//#ifndef HTS_DEF_FWSTRUCT_topindex_chain
-//#define HTS_DEF_FWSTRUCT_topindex_chain
-//typedef struct topindex_chain topindex_chain;
-//#endif
-//struct topindex_chain {
-//  int level;                    /* sort level */
-//  char *category;               /* category */
-//  char name[2048];              /* path */
-//  struct topindex_chain *next;  /* next element */
-//};
 
 /* Tools */
 
@@ -293,7 +283,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
   char BIGSTK newcurr_fil[HTS_URLMAXSIZE * 2], newlink[HTS_URLMAXSIZE * 2];
   char *curr;
 
-  //int n=0;
   char *a;
   int slash = 0;
 
@@ -338,7 +327,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
     if (*curr == '/')
       curr++;
     l = link;
-    //c=curr;
     // couper ce qui est commun
     while((streql(*link, *curr)) && (*link != 0)) {
       link++;
@@ -350,8 +338,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
       link--;
       curr--;
     }
-    //if (*link=='/') link++;
-    //if (*curr=='/') curr++;
   }
 
   // calculer la profondeur du répertoire courant et remonter
@@ -362,7 +348,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
   while(*a)
     if (*(a++) == '/')
       strlcatbuff(s, "../", ssize);
-  //if (strlen(s)==0) strcatbuff(s,"/");
 
   if (slash)
     strlcatbuff(s, "/", ssize); // keep it absolute!

src/htsweb.c

@@ -639,8 +639,7 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b
           for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) {        // no lien
             int i = (back_index + _i) % back_max;       // commencer par le "premier" (l'actuel)
 
-            if (back[i].status >= 0) {  // signifie "lien actif"
+            if (back[i].status >= 0) { // signifie "lien actif"
-              // int ok=0;  // OPTI
               ok = 0;
               switch (j) {
               case 0:          // prioritaire

src/htswizard.c

@@ -188,12 +188,10 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
 
   /* Doit-on traiter les non html? */
   if ((opt->getmode & HTS_GETMODE_NONHTML) == 0) { // non on ne doit pas
-    if (!ishtml(opt, fil)) {    // non il ne faut pas
+    if (!ishtml(opt, fil)) {                       // non il ne faut pas
-      //adr[0]='\0';    // ne pas traiter ce lien, pas traiter
       forbidden_url = 1;        // interdire récupération du lien
       hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s", adr,
                     fil);
-
     }
   }
 
@@ -352,7 +350,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
 
   } else {                      // adresse différente, sortir?
 
-    //if (!opt->wizard) {    // mode non wizard
     // doit-on traiter ce lien?.. vérifier droits de sortie
     switch ((opt->travel & HTS_TRAVEL_SCOPE_MASK)) {
     case HTS_TRAVEL_SAME_ADDRESS:
@@ -380,7 +377,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
       if ((i > 0) && (j > 0)) {
         if (!strfield2(adr + i, urladr() + j)) { // !=
           if (!opt->wizard) {                    // mode non wizard
-            // printf("refused: %s\n",adr);
             forbidden_url = 1; // pas même domaine
             hts_log_print(opt, LOG_DEBUG, "foreign domain link canceled: %s%s",
                           adr, fil);
@@ -408,7 +404,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
       if ((i > 0) && (j > 0)) {
         if (!strfield2(adr + i, urladr() + j)) { // !-
           if (!opt->wizard) {                    // mode non wizard
-            // printf("refused: %s\n",adr);
             forbidden_url = 1; // pas même .xx
             hts_log_print(opt, LOG_DEBUG,
                           "foreign location link canceled: %s%s", adr, fil);
@@ -439,7 +434,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
   // récupérer les liens à côtés d'un lien (nearlink) (nvelle pos)
   if (forbidden_url != 0 && opt->nearlink) {
     if (!ishtml(opt, fil)) {    // non html
-      //printf("ok %s%s\n",ad,fil);
       forbidden_url = 0;        // autoriser
       may_set_prio_to = 1 + 1;  // set prio to 1 (parse but skip urls) if near is the winner
       hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s", adr, fil);
@@ -530,28 +524,11 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
 
         // filters, 0=sait pas 1=ok -1=interdit
         {
-          int jokDepth1 = 0, jokDepth2 = 0;
+          int jokDepth = 0;
-          int jok1 = 0, jok2 = 0;
 
-          jok1 =
+          jok = fa_strjoker_dual(/*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, l,
-            fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, NULL, NULL,
+                                 NULL, NULL, &jokDepth);
-                        &jokDepth1);
+          mdepth = _FILTERS[jokDepth];
-          jok2 =
-            fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, l, NULL, NULL,
-                        &jokDepth2);
-          if (jok2 == 0) {      // #2 doesn't know
-            jok = jok1;         // then, use #1
-            mdepth = _FILTERS[jokDepth1];
-          } else if (jok1 == 0) {       // #1 doesn't know
-            jok = jok2;         // then, use #2
-            mdepth = _FILTERS[jokDepth2];
-          } else if (jokDepth1 >= jokDepth2) {  // #1 matching rule is "after" #2, then it is prioritary
-            jok = jok1;
-            mdepth = _FILTERS[jokDepth1];
-          } else {              // #2 matching rule is "after" #1, then it is prioritary
-            jok = jok2;
-            mdepth = _FILTERS[jokDepth2];
-          }
         }
 
         if (jok == 1) {         // autorisé
@@ -887,7 +864,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
         }
       }
     }
-    //adr[0]='\0';  // cancel
   }
   // -------------------- FINAL PHASE --------------------
   // Test if the "Near" test won
@@ -972,34 +948,10 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz
 
       // filters, 0=sait pas 1=ok -1=interdit
       {
-        int jokDepth1 = 0, jokDepth2 = 0;
+        sz = size;
-        int jok1 = 0, jok2 = 0;
+        jok = fa_strjoker_dual(/*url */ 0, *opt->filters.filters,
-        LLint sz1 = size, sz2 = size;
+                               *opt->filters.filptr, lfull, l, &sz, &size_flag,
-        int size_flag1 = 0, size_flag2 = 0;
+                               NULL);
-
-        jok1 =
-          fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
-                      lfull, &sz1, &size_flag1, &jokDepth1);
-        jok2 =
-          fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
-                      l, &sz2, &size_flag2, &jokDepth2);
-        if (jok2 == 0) {        // #2 doesn't know
-          jok = jok1;           // then, use #1
-          sz = sz1;
-          size_flag = size_flag1;
-        } else if (jok1 == 0) { // #1 doesn't know
-          jok = jok2;           // then, use #2
-          sz = sz2;
-          size_flag = size_flag2;
-        } else if (jokDepth1 >= jokDepth2) {    // #1 matching rule is "after" #2, then it is prioritary
-          jok = jok1;
-          sz = sz1;
-          size_flag = size_flag1;
-        } else {                // #2 matching rule is "after" #1, then it is prioritary
-          jok = jok2;
-          sz = sz2;
-          size_flag = size_flag2;
-        }
       }
 
       // log

src/htszlib.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 
 /*
   Unpack file into a new file (gzip, zlib RFC1950 or raw deflate RFC1951).
+  A body provably in no deflate framing is copied verbatim (identity).
   Return value: size of the new file, or -1 if an error occurred
 */
 /* Note: utf-8 */
@@ -68,6 +69,10 @@ int hts_zunpack(char *filename, char *newfile) {
             ((inbuf[0] & 0x0f) == Z_DEFLATED &&
              (((unsigned) inbuf[0] << 8 | inbuf[1]) % 31) == 0)));
       int attempt;
+      /* not_deflate: the raw attempt hit a data error, so the body is in no
+         deflate framing at all. env_error: local I/O or memory failure. */
+      hts_boolean not_deflate = HTS_FALSE;
+      hts_boolean env_error = HTS_FALSE;
 
       /* deflate is ambiguous; on failure retry with the other windowBits */
       for (attempt = 0; attempt < 2 && ret < 0; attempt++) {
@@ -78,15 +83,20 @@ int hts_zunpack(char *filename, char *newfile) {
 
         if (attempt > 0) {
           /* rewind input; reopening fpout "wb" discards the partial output */
-          if (fseek(in, 0, SEEK_SET) != 0)
+          if (fseek(in, 0, SEEK_SET) != 0) {
+            env_error = HTS_TRUE;
             break;
+          }
           navail = fread(inbuf, 1, sizeof(inbuf), in);
         }
         fpout = FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
-        if (fpout == NULL)
+        if (fpout == NULL) {
+          env_error = HTS_TRUE;
           break;
+        }
         memset(&strm, 0, sizeof(strm));
         if (inflateInit2(&strm, windowBits) != Z_OK) {
+          env_error = HTS_TRUE;
           fclose(fpout);
           break;
         }
@@ -108,12 +118,17 @@ int hts_zunpack(char *filename, char *newfile) {
               zerr = inflate(&strm, Z_NO_FLUSH);
               if (zerr == Z_NEED_DICT || zerr == Z_DATA_ERROR ||
                   zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR) {
+                if (zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR)
+                  env_error = HTS_TRUE;
+                else if (windowBits < 0)
+                  not_deflate = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
               produced = sizeof(outbuf) - strm.avail_out;
               if (produced > 0 &&
                   fwrite(outbuf, 1, produced, fpout) != produced) {
+                env_error = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
@@ -129,6 +144,28 @@ int hts_zunpack(char *filename, char *newfile) {
         inflateEnd(&strm);
         fclose(fpout);
       }
+      /* keep a mislabeled identity body verbatim only when provably not
+         deflate; truncation or a local failure must keep failing (#47) */
+      if (ret < 0 && !wrapped && not_deflate && !env_error && !ferror(in)) {
+        FILE *const fpout =
+            FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
+
+        if (fpout != NULL && fseek(in, 0, SEEK_SET) == 0) {
+          int size = 0;
+
+          while ((navail = fread(inbuf, 1, sizeof(inbuf), in)) > 0) {
+            if (fwrite(inbuf, 1, navail, fpout) != navail) {
+              size = -1;
+              break;
+            }
+            size += (int) navail;
+          }
+          if (size >= 0 && !ferror(in))
+            ret = size;
+        }
+        if (fpout != NULL)
+          fclose(fpout);
+      }
       fclose(in);
     }
   }

src/htszlib.h

@@ -36,7 +36,6 @@ Please visit our Website: http://www.httrack.com
 
 /* ZLib */
 #include "zlib.h"
-//#include "zutil.h"
 
 /* MiniZip */
 #include "minizip/zip.h"

src/httrack.c

@@ -54,7 +54,6 @@ static int linput(FILE * fp, char *s, int max);
 #include "htswrap.h"
 
 /* specific definitions */
-//#include "htsbase.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -454,8 +453,7 @@ static int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_b
           for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) {        // no lien
             int i = (back_index + _i) % back_max;       // commencer par le "premier" (l'actuel)
 
-            if (back[i].status >= 0) {  // signifie "lien actif"
+            if (back[i].status >= 0) { // signifie "lien actif"
-              // int ok=0;  // OPTI
               ok = 0;
               switch (j) {
               case 0:          // prioritaire
@@ -773,28 +771,7 @@ static void sig_finish(int code) {      // finir et quitter
   fprintf(stderr, "\nExit requested to engine (signal %d)\n", code);
 }
 
-#ifdef _WIN32
+#ifndef _WIN32
-#if 0
-static void sig_ask(int code) { // demander
-  char s[256];
-
-  signal(code, sig_term);       // quitter si encore
-  printf("\nQuit program/Interrupt/Cancel? (Q/I/C) ");
-  fflush(stdout);
-  scanf("%s", s);
-  if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
-      || (s[0] == 'q') || (s[0] == 'Q'))
-    exit(0);                    // quitter
-  else if ((s[0] == 'i') || (s[0] == 'I')) {
-    if (global_opt != NULL) {
-      // ask for stop
-      global_opt->state.stop = 1;
-    }
-  }
-  signal(code, sig_ask);        // remettre signal
-}
-#endif
-#else
 static void sig_doback(int blind);
 static void sig_back(int code) {        // ignorer et mettre en backing 
   if (global_opt != NULL && !global_opt->background_on_suspend) {
@@ -809,36 +786,6 @@ static void sig_back(int code) {        // ignorer et mettre en backing
   }
 }
 
-#if 0
-static void sig_ask(int code) { // demander
-  char s[256];
-
-  signal(code, sig_term);       // quitter si encore
-  printf
-    ("\nQuit program/Interrupt/Background/bLind background/Cancel? (Q/I/B/L/C) ");
-  fflush(stdout);
-  scanf("%s", s);
-  if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
-      || (s[0] == 'q') || (s[0] == 'Q'))
-    exit(0);                    // quitter
-  else if ((s[0] == 'b') || (s[0] == 'B') || (s[0] == 'a') || (s[0] == 'A'))
-    sig_doback(0);              // arrière plan
-  else if ((s[0] == 'l') || (s[0] == 'L'))
-    sig_doback(1);              // arrière plan
-  else if ((s[0] == 'i') || (s[0] == 'I')) {
-    if (global_opt != NULL) {
-      // ask for stop
-      printf("finishing pending transfers.. please wait\n");
-      global_opt->state.stop = 1;
-    }
-    signal(code, sig_ask);      // remettre signal
-  } else {
-    printf("cancel..\n");
-    signal(code, sig_ask);      // remettre signal
-  }
-}
-#endif
-
 static void sig_brpipe(int code) {      // treat if necessary
   signal(code, sig_brpipe);
 }
@@ -951,23 +898,12 @@ static void sig_leave(int code) {
 
 static void signal_handlers(void) {
 #ifdef _WIN32
-#if 0                           /* BUG366763 */
-  signal(SIGINT, sig_ask);      // ^C
-#else
   signal(SIGINT, sig_leave);    // ^C
-#endif
   signal(SIGTERM, sig_finish);  // kill <process>
 #else
-#if 0                           /* BUG366763 */
-  signal(SIGHUP, sig_back);     // close window
-#endif
   signal(SIGTSTP, sig_back);    // ^Z
   signal(SIGTERM, sig_finish);  // kill <process>
-#if 0                           /* BUG366763 */
-  signal(SIGINT, sig_ask);      // ^C
-#else
   signal(SIGINT, sig_leave);    // ^C
-#endif
   signal(SIGPIPE, sig_brpipe);  // broken pipe (write into non-opened socket)
   signal(SIGCHLD, sig_ignore);  // child change status
 #endif

src/proxy/proxytrack.c

@@ -250,13 +250,6 @@ static int gethost(const char *hostname, SOCaddr * server) {
     struct addrinfo hints;
 
     memset(&hints, 0, sizeof(hints));
-#if 0
-    if (IPV6_resolver == 1)     // V4 only (for bogus V6 entries)
-      hints.ai_family = PF_INET;
-    else if (IPV6_resolver == 2)        // V6 only (for testing V6 only)
-      hints.ai_family = PF_INET6;
-    else
-#endif
     hints.ai_family = PF_UNSPEC;
     hints.ai_socktype = SOCK_STREAM;
     hints.ai_protocol = IPPROTO_TCP;
@@ -340,12 +333,7 @@ int proxytrack_main(char *proxyAddr, int proxyPort, char *icpAddr, int icpPort,
   T_SOC socICP = smallserver_init(proxyAddr, icpPort, SOCK_DGRAM);
 
   if (soc != INVALID_SOCKET && socICP != INVALID_SOCKET) {
-    //char url[HTS_URLMAXSIZE * 2];
-    //char method[32];
-    //char data[32768];
 
-    //url[0] = method[0] = data[0] = '\0';
-    //
     printf("HTTP Proxy installed on %s:%d/\n", proxyAddr, proxyPort);
     printf("ICP Proxy installed on %s:%d/\n", icpAddr, icpPort);
 #ifndef _WIN32
@@ -950,7 +938,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
     char *command;
     char *proto;
     char *surl;
-    //int directHit = 0;
     int headRequest = 0;
     int listRequest = 0;
 
@@ -1092,10 +1079,8 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
 
       /* Post-process request */
       if (link_has_authority(surl)) {
-        if (strncasecmp
+        if (strncasecmp(surl, "http://proxytrack/",
-            (surl, "http://proxytrack/",
+                        sizeof("http://proxytrack/") - 1) == 0) {
-             sizeof("http://proxytrack/") - 1) == 0) {
-          //directHit = 1;        /* Another direct hit hack */
         }
         StringCopy(url, surl);
       } else {
@@ -1116,7 +1101,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
               toHit += 7;
             }
             /* Direct hit */
-            //directHit = 1;
             StringCopy(url, "");
             if (!link_has_authority(toHit))
               StringCat(url, "http://");
@@ -1127,7 +1111,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
             const char *toHit = surl + sizeof("/proxytrack/") - 1;
 
             /* Direct hit */
-            //directHit = 1;
             StringCopy(url, "");
             if (!link_has_authority(toHit))
               StringCat(url, "http://");
@@ -1611,10 +1594,7 @@ static int proxytrack_start_ICP(PT_Indexes indexes, T_SOC soc) {
         unsigned char Opcode = buffer[0];
         unsigned char Version = buffer[1];
         unsigned short Message_Length = READ_NET16(&buffer[2]);
-        unsigned int Request_Number = READ_NET32(&buffer[4]);   /* Session ID */
+        unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */
-        //unsigned int Options = READ_NET32(&buffer[8]);
-        //unsigned int Option_Data = READ_NET32(&buffer[12]);     /* ICP_FLAG_SRC_RTT */
-        //unsigned int Sender_Host_Address = READ_NET32(&buffer[16]);     /* ignored */
         unsigned char *Payload = &buffer[20];
 
         buffer[bufferSize] = '\0';      /* Ensure payload is NULL terminated */

src/proxy/store.c

@@ -1035,7 +1035,6 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
       if (unzOpenCurrentFile(index->zFile) == Z_OK) {
         char headerBuff[8192 + 2];
         int readSizeHeader;
-        //int totalHeader = 0;
         int dataincache = 0;
 
         /* For BIG comments */
@@ -1074,9 +1073,8 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
               ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified);      // last-modified
               ZIP_READFIELD_STRING(line, value, "Etag", r->etag);       // Etag
               ZIP_READFIELD_STRING(line, value, "Location", r->location);       // 'location' pour moved
-              ZIP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo);      // Content-disposition
+              ZIP_READFIELD_STRING(line, value, "Content-Disposition",
-              //ZIP_READFIELD_STRING(line, value, "X-Addr", ..);            // Original address
+                                   r->cdispo); // Content-disposition
-              //ZIP_READFIELD_STRING(line, value, "X-Fil", ..);            // Original URI filename
               ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_);      // Original save filename
               if (line[0] != '\0') {
                 int len = r->headers ? ((int) strlen(r->headers)) : 0;
@@ -1090,8 +1088,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
                 strcat(r->headers, "\r\n");
               }
             }
-          } while(offset < readSizeHeader && !lineEof);
+          } while (offset < readSizeHeader && !lineEof);
-          //totalHeader = offset;
 
           /* Previous entry */
           if (previous_save_[0] != '\0') {
@@ -1202,7 +1199,6 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
                       strcpy(r->msg, "Cache Read Error : Read Data");
                     } else
                       *(r->adr + r->size) = '\0';
-                    //printf(">%s status %d\n",back[p].r->contenttype,back[p].r->statuscode);
                   } else {      // erreur
                     r->statuscode = STATUSCODE_INVALID;
                     strcpy(r->msg, "Cache Memory Error");
@@ -1484,14 +1480,12 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
               a += cache_brstr(a, firstline);
               strcpy(cache->lastmodified, firstline);
             } else {
-              // fprintf(opt->errlog,"Cache: version 1.%d not supported, ignoring current cache"LF,cache->version);
               fclose(cache->dat);
               cache->dat = NULL;
               free(use);
               use = NULL;
             }
-          } else {              // non supporté
+          } else { // non supporté
-            // fspc(opt->errlog,"error"); fprintf(opt->errlog,"Cache: %s not supported, ignoring current cache"LF,firstline);
             fclose(cache->dat);
             cache->dat = NULL;
             free(use);
@@ -1500,7 +1494,6 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
           /* */
         } else {                // Vieille version du cache
           /* */
-          // hts_log_print(opt, LOG_WARNING, "Cache: importing old cache format");
           cache->version = 0;   // cache 1.0
           strcpy(cache->lastmodified, firstline);
         }

tests/01_engine-cacheindex.test

@@ -0,0 +1,10 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Cache-index (.ndx) parse must stay inside the buffer on a corrupt length
+# prefix (ASan aborts here on the pre-fix binary; the OK check gates the
+# non-sanitized build too).
+
+test "$(httrack -O /dev/null -#test=cacheindex)" == 'cacheindex: OK'

tests/01_engine-charset.test

@@ -36,3 +36,64 @@ test "$(httrack -O /dev/null -#test=charset 'no-such-charset-xyz' 'café' 2>/dev
 # malformed UTF-8 (lone continuation byte, truncated lead byte) must not crash
 runs 'utf-8' $'\x80'
 runs 'utf-8' $'\xc3'
+
+# hts_getCharsetFromMeta: <meta> charset extraction, HTML5 and legacy forms.
+# -#test=metacharset <html> prints the charset, or "(none)".
+meta() {
+    test "$(httrack -O /dev/null -#test=metacharset "$1")" == "$2" || exit 1
+}
+
+# HTML5 form, quoting/spacing flavors
+meta '<meta charset="utf-8">' 'utf-8'
+meta "<meta charset='utf-8'>" 'utf-8'
+meta '<meta charset=utf-8>' 'utf-8'
+meta '<meta charset=utf-8 />' 'utf-8'
+meta '<meta charset=utf-8/>' 'utf-8'
+# charset labels are case-insensitive downstream; only require case-insensitive match
+test "$(httrack -O /dev/null -#test=metacharset '<META CHARSET=UTF-8>' | tr '[:upper:]' '[:lower:]')" == 'utf-8' || exit 1
+meta '<html><head><meta   charset = "utf-8"  ></head>' 'utf-8'
+
+# legacy http-equiv form, attributes in any order
+meta '<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">' 'iso-8859-1'
+meta '<meta content="text/html; charset=gb2312" http-equiv="content-type">' 'gb2312'
+meta '<meta http-equiv=Content-Type content="text/html;charset=euc-jp">' 'euc-jp'
+meta '<meta http-equiv="Content-Type" content="text/html; charset = koi8-r ">' 'koi8-r'
+meta '<meta http-equiv="Content-Type" content="text/html; charset=utf-8;">' 'utf-8'
+meta '<meta http-equiv=content-type content=text/html;charset=utf-8 />' 'utf-8'
+meta '<meta http-equiv=content-type content=text/html;charset=utf-8>' 'utf-8'
+
+# first resolvable meta wins; unrelated metas are skipped
+meta '<meta name="viewport" content="width=1"><meta charset="koi8-r">' 'koi8-r'
+meta '<meta charset="utf-8"><meta http-equiv="Content-Type" content="text/html; charset=gb2312">' 'utf-8'
+
+# duplicate attribute: first wins, as in browsers
+meta '<meta http-equiv="Content-Type" content="text/html; charset=big5" http-equiv="refresh">' 'big5'
+
+# negatives: wrong tag/attribute, empty value
+meta '<p>x</p>' '(none)'
+meta '<meta name="desc" content="charset=bogus">' '(none)'
+meta '<meta http-equiv="refresh" content="0; url=x.html">' '(none)'
+meta '<body charset="utf-8">' '(none)'
+meta '<meta charset="">' '(none)'
+
+# hostile truncations: no crash, no match
+meta '<meta charset="utf-8' '(none)'
+meta '<meta charset=' '(none)'
+meta '<meta ' '(none)'
+meta '<meta' '(none)'
+
+# hts_isStringUTF8: valid-UTF-8 probe (guards the #180 link conversion).
+utf8() {
+    test "$(httrack -O /dev/null -#test=isutf8 "$1")" == "$2" || exit 1
+}
+utf8 'plain' '1'
+utf8 'café' '1'
+utf8 '统计' '1'
+utf8 "$(printf 'caf\xe9')" '0'              # latin-1 accent
+utf8 "$(printf '\xc3')" '0'                 # truncated lead byte
+utf8 "$(printf '\xa9')" '0'                 # lone continuation byte
+utf8 "$(printf '\xf0\x9f\x98\x80')" '1'     # 4-byte emoji
+utf8 "$(printf '\xc0\xaf')" '0'             # overlong (latin-1 "À¯" must stay convertible)
+utf8 "$(printf '\xed\xa0\x80')" '0'         # UTF-16 surrogate
+utf8 "$(printf '\xf4\x90\x80\x80')" '0'     # above U+10FFFF
+utf8 "$(printf '\xf8\x88\x80\x80\x80')" '0' # legacy 5-byte form

tests/01_engine-crange.test

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Content-Range parse (treathead via -#test=crange). Hostile values must clamp
+# to 0, and the crange +/- 1 arithmetic must not sign-overflow (UBSan aborts on
+# the pre-fix binary at the INT64_MIN cases).
+
+cr() {
+    local want="$1"
+    shift
+    out="$(httrack -O /dev/null -#test=crange "$@")"
+    test "$out" == "$want" || {
+        echo "FAIL: $* -> '$out' (want '$want')"
+        exit 1
+    }
+}
+
+# Normal range and the '*/N' fallback both parse through unchanged.
+cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes 0-70870/70871'
+cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes */70871'
+
+# INT64_MIN total in the fallback would make crange-1 overflow: clamp to 0.
+cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes */-9223372036854775808'
+
+# Any negative field in the 3-field form clamps the whole triple to 0 (each
+# field independently: a leading negative start/end alone still zeroes all).
+cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -1--1/-1'
+cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -5-10/20'
+
+# INT64_MAX is non-negative: it passes through, the parse must not mangle it.
+cr 'crange_start=0 crange_end=9223372036854775807 crange=9223372036854775807' \
+    'Content-Range: bytes 0-9223372036854775807/9223372036854775807'

tests/01_engine-filter.test

@@ -12,6 +12,11 @@ match() {
 nomatch() {
     test "$(httrack -O /dev/null -#test=filter "$1" "$2")" == "$2 does NOT match $1" || exit 1
 }
+# single-arg call means an empty subject (unreachable as a CLI arg); '*(' used
+# to force max=1 and over-read past it.
+nomatch_empty() {
+    test "$(httrack -O /dev/null -#test=filter "$1")" == " does NOT match $1" || exit 1
+}
 
 # bare star matches everything
 match '*' 'anything/at/all'
@@ -122,3 +127,68 @@ nomatch '*[file]end' 'foo?xend'
 nomatch '*[name]X' 'abc?X'
 match '*[file]' 'foo?x=1' # trailing query: tolerated, as for *.aspx
 match '*.aspx' 'page.aspx?y=2'
+
+# empty subject against a unique-match allow-all pattern (heap over-read regress)
+nomatch_empty '*(('
+nomatch_empty '**(('
+
+# near-miss negatives: word order, exact tails, range boundaries, class case
+nomatch 'foo*bar' 'foobaz'
+nomatch '*foo*bar' 'barfoo' # both words present, wrong order
+nomatch '*.gif' 'a.gifx'    # trailing junk after the extension
+nomatch '*.gif' 'a.gi'
+nomatch '*[b-y]' 'a'
+nomatch '*[b-y]' 'z'
+nomatch '*[a-z].txt' 'B.txt' # classes are case-sensitive...
+match 'FoO*' 'foobar'        # ...literals are not
+
+# a star may match an empty run, but never skip a required literal
+match '*[0-9].gif' '.gif'
+nomatch '*[0-9].gif' 'x.gif'
+
+# a descending range *[c-a] is an empty class, not c..a reversed
+match '*[c-a]x' 'x'
+nomatch '*[c-a]x' 'cx'
+
+# *[param] is all-or-nothing on '?': spans everything after one, else nothing
+match 'a*[param]' 'a?x=1'
+nomatch 'a*[param]' 'ab?x=1'
+
+# the latest matching rule wins, regardless of sign
+fsize 'verdict=forbidden size_flag=0' -1 a.gif '+*.gif' '-*.gif'
+fsize 'verdict=allowed size_flag=0' -1 a.gif '-*.gif' '+*.gif'
+
+# a failed size test sets size_flag but leaves the rule unmatched
+fsize 'verdict=unknown size_flag=1' 5 foo.jpg '-*.jpg*[>10]'
+
+# mime filters (-#test=filtermime = fa_strjoker type=1): only mime: rules
+# apply to a mime type, and mime: rules never apply to a URL
+fmime() {
+    local want="$1"
+    shift
+    test "$(httrack -O /dev/null -#test=filtermime "$@")" == "verdict=$want" || exit 1
+}
+fmime allowed 'image/png' '+mime:image/*'
+fmime unknown 'text/html' '+mime:image/*'
+fmime forbidden 'image/png' '+mime:image/*' '-mime:image/png' # last wins
+fmime allowed 'image/png' '-mime:image/png' '+mime:image/*'
+fmime unknown 'image/png' '+image/*'                               # URL rule: skipped
+fsize 'verdict=unknown size_flag=0' -1 'image/png' '+mime:image/*' # and vice versa
+
+# memoized vs no-memo differential over seeded random patterns and subjects,
+# matching and non-matching (the self-test asserts both polarities occurred)
+test "$(httrack -O /dev/null -#test=filtermemo)" == "filtermemo: 20000 cases OK" || exit 1
+
+# star-heavy non-match must stay polynomial (#501); 400 chars also exercises
+# the heap-allocated memo. timeout (where available) turns a hang into a FAIL.
+with_timeout() {
+    if command -v timeout >/dev/null; then timeout 60 "$@"; else "$@"; fi
+}
+pat=$(printf '*[a]%.0s' $(seq 1 12))b
+subj=$(printf 'a%.0s' $(seq 1 400))
+test "$(with_timeout httrack -O /dev/null -#test=filter "$pat" "$subj")" == "$subj does NOT match $pat" || exit 1
+
+# a later star's class dead-end must re-extend the earlier star, not give up
+# (guards against a single-backtrack-point matcher rewrite)
+match '*[aX]X*[a]Y' 'aXaXaY'
+nomatch '*[aX]X*[a]Y' 'aXbXaY'

tests/01_engine-filterdual.test

@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# fa_strjoker_dual: merged verdict on the two URL forms the engine tests;
+# the match latest in the filter list wins, an unknown verdict defers.
+
+fdual() {
+    local want="$1"
+    shift
+    test "$(httrack -O /dev/null -#test=filterdual "$@")" == "$want" || exit 1
+}
+
+fdual 'verdict=unknown rule=0' zzz yyy '+a*'     # neither form matches
+fdual 'verdict=forbidden rule=0' aaa zzz '-a*'   # only form 1 matches
+fdual 'verdict=forbidden rule=0' zzz aaa '-a*'   # only form 2 matches
+fdual 'verdict=forbidden rule=1' a b '+a*' '-b*' # later rule wins (form 2)
+fdual 'verdict=forbidden rule=1' b a '+a*' '-b*' # later rule wins (form 1)
+fdual 'verdict=allowed rule=0' a aa '+a*'        # tie: form 1 wins

tests/01_engine-header.test

@@ -27,3 +27,16 @@ hdr 'contenttype= cdispo=report.pdf' \
 # Path components in the filename are dropped on the wire (RFC 2616).
 hdr 'contenttype= cdispo=evil.pdf' \
     'Content-Disposition: attachment; filename="../../evil.pdf"'
+
+# An over-long header value must not overflow the parse scratch (ASan aborts
+# here on the pre-fix binary).
+test "$(httrack -O /dev/null -#test=headerlong 'Content-Type:')" \
+    == 'contenttype_len=32 contentencoding_len=0'
+test "$(httrack -O /dev/null -#test=headerlong 'Content-Encoding:')" \
+    == 'contenttype_len=0 contentencoding_len=0'
+
+# An empty Content-Encoding yields no token: leave it empty, don't read the
+# uninitialized scratch (MSan aborts here on the pre-fix binary).
+enc() { httrack -O /dev/null -#test=header "$1" | grep '^contentencoding='; }
+test "$(enc 'Content-Encoding:')" == 'contentencoding='
+test "$(enc 'Content-Encoding: gzip')" == 'contentencoding=gzip'

tests/01_engine-identurl.test

@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# ident_url_absolute splits a raw URL into (adr, fil), the first parser to touch
+# it. -#test=identurl <url> [pad] prints "adr=.. fil=.." or "error"; pad appends
+# N 'a's, reaching lengths a CLI arg can't.
+
+split() {
+    test "$(httrack -O /dev/null -#test=identurl "$1")" == "$2" || exit 1
+}
+
+split 'http://www.example.com/a/b/../c.html' 'adr=www.example.com fil=/a/c.html'
+split 'ftp://ftp.example.com/pub/file.txt' 'adr=ftp://ftp.example.com fil=/pub/file.txt'
+split 'file:///etc/hostname' 'adr=file:// fil=/etc/hostname'
+
+# empty-path file:// used to read one byte past the URL (p[1] with *p=0)
+split 'file://' 'adr=file:// fil=//'
+
+# a root-relative path is copied whole into fil[HTS_URLMAXSIZE*2], so a URL at
+# the buffer size is the tightest overflow; it must be rejected, not abort.
+split_pad() {
+    test "$(httrack -O /dev/null -#test=identurl "$1" "$2")" == "$3" || exit 1
+}
+split_pad '/' 2047 'error' # 1 + 2047 = 2048 = HTS_URLMAXSIZE*2
+split_pad 'http://h/' 3000 'error'

tests/01_engine-idna.test

@@ -8,8 +8,8 @@ set -euo pipefail
 
 enc() { test "$(httrack -O /dev/null -#test=idna-encode "$1")" == "$2" || exit 1; }
 dec() { test "$(httrack -O /dev/null -#test=idna-decode "$1")" == "$2" || exit 1; }
-# crash probe: malformed ACE input must exit cleanly, not abort.
+# a malformed encode input must be rejected (empty output), not encoded or leaked.
-runs() { httrack -O /dev/null -#test=idna-decode "$1" >/dev/null 2>&1 || exit 1; }
+enc_bad() { test -z "$(httrack -O /dev/null -#test=idna-encode "$1" 2>/dev/null)" || exit 1; }
 
 # encode
 enc 'www.café.com' 'www.xn--caf-dma.com'
@@ -33,6 +33,9 @@ enc 'xn--already-encoded.com' 'xn--already-encoded.com'
 # an empty punycode payload decodes back to the bare xn-- label
 dec 'xn--' 'xn--'
 
-# malformed ACE payloads (invalid base-36, garbage) must not crash
+# malformed ACE payloads decode to a stable (garbage) string, not a crash
-runs 'xn--!!!'
+dec 'xn--!!!' '㚯'
-runs 'xn--already-encoded.com'
+dec 'xn--already-encoded.com' 'ǮalǭǮreaǭǫdy.com'
+
+# second label has a truncated UTF-8 sequence: used to leak the segment buffer.
+enc_bad "$(printf 'b\xc3\xbcchev.\xe4\xbe\x8b\xe5\xad\x62\xc3\xbccheple')"