Add libFuzzer harnesses for the pure hostile-input parsers

New fuzz/ tree with a libFuzzer harness per pure (buf,size) parser that ingests
network-sourced bytes: the charset/UTF-8/IDNA codecs, the HTML <meta> charset
scan, HTML entity and URL percent decoders, the wildcard filter matcher, and
the URL splitter. Gated behind --enable-fuzzers (requires clang, forces
--disable-shared so clang's static sanitizer runtimes link); a normal build is
untouched.

A CI job builds the harnesses under ASan+UBSan+LeakSanitizer and replays each
seed corpus. Replay, not open-ended mutation, keeps the job deterministic and
avoids strjoker's catastrophic backtracking (a data-dependent exponential in
the wildcard matcher, tracked separately). Maintainers run the timed mutation
mode locally for discovery; OSS-Fuzz is the next step.

The corpora carry the reproducers for the four parser bugs these harnesses
found, so the job also pins those regressions. It depends on the fixes in the
preceding commit; merge that first.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.clang-format

@@ -16,6 +16,7 @@ BasedOnStyle: LLVM
 SpaceAfterCStyleCast: true   # "(int) x", overwhelmingly dominant (542 vs 7)
 SortIncludes: false          # C include order can be significant; never reorder
 IncludeBlocks: Preserve      # do not merge/reflow include groups
+SeparateDefinitionBlocks: Always  # blank line between definitions (readability)
 
 # Stated explicitly for robustness against base-style drift (these match LLVM):
 IndentWidth: 2

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+# Match black's formatting so the two tools don't fight.
+max-line-length = 88
+# E203/W503 conflict with black's slice and line-break style.
+extend-ignore = E203, W503

.github/workflows/ci.yml

@@ -61,6 +61,50 @@ jobs:
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # Reproduce the Debian buildds: they build in a minimal chroot with no
+  # python3, so the local-server tests must SKIP (exit 77), not fail. GitHub
+  # runners ship python3, so every other job hides this path; here we remove it
+  # before `make check`. This is the guard that would have caught the 3.49.10-1
+  # FTBFS (28_local-pause failed instead of skipping when python3 was absent).
+  buildd-no-python3:
+    name: build (no python3, Debian buildd)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - name: Configure
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Test without python3
+        run: |
+          set -euo pipefail
+          # Hide every python3* so `command -v python3` fails like it does in the
+          # buildd chroot; masking with /bin/false would still resolve.
+          sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
+            -exec mv {} {}.hidden \;
+          ! command -v python3
+          make check
+
+      - name: Print the test log on failure
+        if: failure()
+        run: cat tests/test-suite.log 2>/dev/null || true
+
   # Portability: build and test on macOS (Darwin/clang) on a native runner --
   # no VM. The tree has no __APPLE__ branches, so Darwin exercises the
   # generic-Unix path on a second libc and kernel. brew's openssl@3 is keg-only,
@@ -188,6 +232,91 @@ jobs:
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # MemorySanitizer catches reads of uninitialized memory (#143's stack-garbage
+  # size filter) that ASan/UBSan miss. It flags any byte an uninstrumented lib
+  # wrote, so the job stays in our own code: offline self-tests only, no openssl
+  # (--disable-https), no zlib cache tests, static (the runtime is not in .so's).
+  msan:
+    name: msan (MemorySanitizer, clang)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential clang autoconf automake libtool autoconf-archive \
+            zlib1g-dev
+
+      - name: Configure (MSan, static, no https)
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure CC=clang \
+            CFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
+            LDFLAGS="-fsanitize=memory" \
+            --disable-https --disable-shared --enable-static
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Test (offline self-tests under MSan)
+        env:
+          MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
+        run: |
+          set -euo pipefail
+          # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
+          # skipped here (uninstrumented libz floods MSan with false positives).
+          tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
+          make check TESTS="$tests"
+
+      - name: Print the test log on failure
+        if: failure()
+        run: cat tests/test-suite.log 2>/dev/null || true
+
+  # libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
+  # parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
+  # (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
+  # catastrophic backtracking -- and pins the regression seeds for the bugs the
+  # fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
+  fuzz:
+    name: fuzz (libFuzzer corpus replay, clang)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential clang autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - name: Configure (fuzzers, static)
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure CC=clang \
+            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
+            LDFLAGS="-fsanitize=address,undefined" \
+            --enable-fuzzers --disable-shared
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Replay corpora
+        env:
+          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
+          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
+        run: bash fuzz/run-fuzzers.sh fuzz check
+
   # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
   # the configuration users on minimal systems build, and one libssl is not even
   # installed here so configure cannot silently re-enable it. The matrix above
@@ -227,34 +356,47 @@ jobs:
   # Validate the Debian packaging via the same script maintainers release with.
   # One amd64/gcc run is enough: packaging (control/rules/manifest/lintian/quilt
   # source build) is arch- and compiler-independent, and the build matrix above
-  # already covers compile portability. lintian runs with --fail-on=error.
+  # already covers compile portability. mkdeb.sh runs lintian as an explicit gate
+  # (debuild does not propagate lintian's exit) with --fail-on=error,warning.
   deb:
     name: deb package (lintian)
     runs-on: ubuntu-24.04
+    # Build and gate inside Debian sid, the upload target. A Debian dpkg-deb
+    # produces archive-legal xz members (an Ubuntu host defaults to zstd, which
+    # the archive's lintian rejects), and sid's lintian carries the same
+    # data-driven checks (embedded-lib fingerprints and the like) the buildds and
+    # UDD apply -- so issues surface here instead of after upload.
+    container: debian:sid
     steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
       - name: Install packaging toolchain
         run: |
           set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
+          apt-get update
+          apt-get install -y --no-install-recommends \
+            ca-certificates git \
             build-essential autoconf automake libtool autoconf-archive \
             zlib1g-dev libssl-dev \
             debhelper devscripts lintian fakeroot
 
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
       # --unsigned: CI has no GPG key (also skips the release sig/checksums).
-      # debuild builds every package, then lintian gates on errors.
+      # mkdeb builds every package then runs the lintian gate (--fail-on=error,
+      # warning); debuild runs the packaged test pass.
       #
       # DEB_BUILD_OPTIONS trims work CI does not need (release builds via
       # mkdeb.sh are untouched): noautodbgsym drops the -dbgsym packages whose
       # LTO payloads are slow to compress and that CI never ships; parallel uses
-      # every core. We let debuild run its test pass -- the only one now that
-      # mkdeb no longer runs its own -- so CI exercises the packaged tests.
-      - name: Build Debian packages
+      # every core.
+      - name: Build and lint Debian packages
         run: |
+          set -euo pipefail
+          # The workspace volume is owned by the host runner uid, but the
+          # container runs as root, so mkdeb's git calls (superproject and the
+          # coucal submodule) trip "dubious ownership"; mark them all safe.
+          git config --global --add safe.directory "*"
           export DEB_BUILD_OPTIONS="noautodbgsym parallel=$(nproc)"
           bash tools/mkdeb.sh --unsigned --no-release-artifacts
 
@@ -320,6 +462,21 @@ jobs:
   lint:
     name: lint (shellcheck, shfmt)
     runs-on: ubuntu-24.04
+    # Every tracked shell script; the globs expand at run time. Kept here so the
+    # shellcheck and shfmt steps below cannot drift apart.
+    env:
+      SHELL_SCRIPTS: >-
+        .githooks/pre-commit
+        bootstrap
+        build.sh
+        html/div/search.sh
+        man/makeman.sh
+        src/htsbasiccharsets.sh
+        src/htsentities.sh
+        src/webhttrack
+        tests/*.sh
+        tests/*.test
+        tools/mkdeb.sh
     steps:
       - uses: actions/checkout@v6
 
@@ -332,12 +489,11 @@ jobs:
           sudo apt-get install -y --no-install-recommends shellcheck shfmt
           shfmt --version
 
-      # Lint the scripts we maintain; the legacy scripts are a separate cleanup.
       - name: shellcheck
-        run: shellcheck man/makeman.sh tools/mkdeb.sh .githooks/pre-commit tests/*.test tests/check-network.sh
+        run: shellcheck $SHELL_SCRIPTS
 
       - name: shfmt
-        run: shfmt -d -i 4 man/makeman.sh tools/mkdeb.sh .githooks/pre-commit
+        run: shfmt -d -i 4 $SHELL_SCRIPTS
 
   # Check clang-format on CHANGED LINES ONLY. The engine predates clang-format
   # (it was shaped by an old Visual Studio formatter) and does not round-trip,

.gitignore

@@ -39,3 +39,6 @@ Makefile
 
 # Editor / autotools backup files.
 *~
+
+# Python bytecode (tests/local-server.py).
+__pycache__/

AGENTS.md

@@ -33,8 +33,9 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 - Be terse. Comment the why, in English; translate French comments you touch.
 - Strip AI tells from prose (em-dash overuse, rule-of-three, filler, vague
   attributions). Ref: Wikipedia "Signs of AI writing". Claude Code: `/humanizer`.
-- Behavior change → add a test. Fast path: a hidden `httrack -#N` debug
-  subcommand (`htscoremain.c`) driven by a `tests/NN_*.test`, over a slow crawl.
+- Behavior change → add a test. Fast path: a hidden `httrack -#test=NAME` engine
+  self-test (registry in `htsselftest.c`; `-#test` lists them) driven by a
+  `tests/NN_*.test`, over a slow crawl.
 
 ## Review your change adversarially (strongly suggested)
 Before pushing, and when reviewing others, don't skim for bugs:

CONTRIBUTING.md

@@ -39,6 +39,10 @@ Welcome, and nothing to disclose. Two rules:
 
 The sign-off covers AI-assisted code too.
 
+## Translations
+
+Interface strings live in [`lang/`](lang/). See [lang/README.md](lang/README.md) for the file format and how to add or update a language.
+
 ## Bugs
 
 Open an issue with the version, OS, command used, and expected vs actual result.

Makefile.am

@@ -1,5 +1,5 @@
 
-SUBDIRS = src man m4 libtest templates lang html tests
+SUBDIRS = src man m4 libtest templates lang html tests fuzz
 
 ACLOCAL_AMFLAGS = -I m4
 

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.8], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,9 +29,11 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:0:0: htsblk layout changed (contenttype/charset/contentencoding widened to
-# 128), an incompatible ABI break, so bump current and reset revision/age.
-VERSION_INFO="3:0:0"
+# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
+# installed headers (no struct layout or exported signature changed vs
+# 3.49.10), so it stays soname .so.3; bump revision.
+# (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
+VERSION_INFO="3:3:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 
@@ -62,6 +64,16 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
 # Export version info
 AC_SUBST(VERSION_INFO)
 
+# Versioned plugin name for dlopen() in hts_create_opt(); soname major is
+# libtool's current - age, so this tracks VERSION_INFO bumps automatically.
+HTS_SONAME_MAJOR=$((${VERSION_INFO%%:*} - ${VERSION_INFO##*:}))
+case "$host_os" in
+darwin*) HTS_LIBHTSJAVA_NAME="libhtsjava.$HTS_SONAME_MAJOR.dylib" ;;
+*) HTS_LIBHTSJAVA_NAME="libhtsjava.so.$HTS_SONAME_MAJOR" ;;
+esac
+AC_DEFINE_UNQUOTED([HTS_LIBHTSJAVA_NAME], ["$HTS_LIBHTSJAVA_NAME"],
+  [Versioned libhtsjava runtime name, derived from VERSION_INFO])
+
 ### Default CFLAGS
 DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
 -Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
@@ -214,9 +226,12 @@ AC_SUBST(OPENSSL_LIBS)
 fi
 
 ### Support IPv6
+V6_SUPPORT=no
 AC_CHECK_LIB(c, getaddrinfo, [V6_FLAG="-DINET6"
+V6_SUPPORT=yes
 AC_DEFINE(HTS_INET6, 1, [Check for IPv6])], AC_MSG_WARN([*** IPv6 not found IPv6 compatibility disabled]))
 AC_SUBST(V6_FLAG)
+AC_SUBST(V6_SUPPORT)
 
 ### Check for LFS
 AC_CHECK_LIB(c, fopen64, [LFS_FLAG="-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE"
@@ -295,6 +310,37 @@ AC_ARG_ENABLE([online-unit-tests],
 	])
 AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
 
+## libFuzzer harnesses (fuzz/); requires clang
+AC_MSG_CHECKING(whether to build fuzzers)
+AC_ARG_ENABLE([fuzzers],
+	[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
+	[
+	case "${enableval}" in
+	no|yes)
+		fuzzers=$enableval
+		AC_MSG_RESULT($enableval)
+		;;
+	*)
+		AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
+		;;
+	esac
+	],
+	[
+	fuzzers=no
+	AC_MSG_RESULT(no)
+	])
+if test x"$fuzzers" = x"yes"; then
+	# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
+	AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
+		[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
+		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
+	# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
+	if test x"$enable_shared" != x"no"; then
+		AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
+	fi
+fi
+AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
+
 # Final output
 AC_CONFIG_FILES([
 Makefile
@@ -306,5 +352,6 @@ lang/Makefile
 html/Makefile
 libtest/Makefile
 tests/Makefile
+fuzz/Makefile
 ])
 AC_OUTPUT

debian/changelog

@@ -1,6 +1,67 @@
+httrack (3.49.11-1) unstable; urgency=medium
+
+  * New upstream release: crawl correctness and security fixes (network-facing
+    buffer overflows, file-type detection, redirect handling) and modernized
+    web defaults; full list in history.txt.
+  * Add DEP-12 upstream metadata (#466).
+  * Bump debhelper compat to 14 (#466).
+  * Drop the redundant Priority field and update the NMU lintian override to
+    the current tag names (#466).
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 05 Jul 2026 00:03:18 +0200
+
+httrack (3.49.10-2) unstable; urgency=medium
+
+  * Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
+    absent (the local-server tests need python3, which the buildds lack). Add
+    patches/skip-local-pause-test-without-python3.patch to guard the test on
+    python3 up front, like its siblings, so it skips cleanly.
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 20:18:46 +0200
+
+httrack (3.49.10-1) unstable; urgency=medium
+
+  * New upstream release: new download-pacing and URL-handling options plus a
+    batch of crawl and robustness fixes (full list in history.txt).
+  * Rewrite debian/copyright in machine-readable DEP-5 format, crediting the
+    bundled minizip, md5 and coucal sources (#415).
+  * Lead the webhttrack browser dependency with chromium so httrack is not
+    dragged into the firefox-esr autoremoval cascade (#436).
+  * Override the embedded-library lint for the bundled minizip (#419).
+  * Bump Standards-Version to 4.7.4 (no changes required).
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 14:01:53 +0200
+
+httrack (3.49.9-1) unstable; urgency=medium
+
+  * New upstream release: Content-Type and file-type detection fixes (trust a
+    declared Content-Type over a binary URL extension, honor --assume under the
+    delayed type check, keep a known extension against a bogus or empty
+    Content-Type, and avoid an uninitialised read on an empty Content-Type), and
+    restored C++ source-compatibility of the installed headers so reverse
+    dependencies (httraqt) build again.
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 21 Jun 2026 17:59:38 +0200
+
+httrack (3.49.8-2) unstable; urgency=medium
+
+  * Rename libhttrack2 to libhttrack3 to follow the SONAME, which the 3.49.8
+    ABI bump moved to libhttrack.so.3 (package-name-doesnt-match-sonames). In
+    3.49.8-1 the libhttrack2.files glob still matched .so.2, so the runtime
+    libraries fell through into the httrack package and libhttrack2 shipped no
+    library. The new .files uses a .so.3* wildcard so a future SONAME bump no
+    longer silently misplaces the libraries. New binary package, via NEW.
+  * Drop the stale debian/libhttrack-swf1.files: the swf module is no longer
+    built and no libhttrack-swf1 package exists.
+
+ -- Xavier Roche <xavier@debian.org>  Sat, 20 Jun 2026 14:42:13 +0200
+
 httrack (3.49.8-1) unstable; urgency=medium
 
-  * New upstream release.
+  * New upstream release: HTTPS-proxy CONNECT tunnelling and wider srcset
+    parsing, a batch of crawler and parser fixes (CSS @import, xmlns
+    namespaces, relative paths, RFC 6265 cookies), and security hardening of
+    the parser and of buffer copies throughout the engine.
   * Drop the OpenSSL linking exception from the license: OpenSSL 3.0+ is
     Apache-2.0 and GPL-compatible, so it is no longer needed. httrack is now
     plain GPL-3.0-or-later. Updated debian/copyright accordingly.
@@ -14,7 +75,7 @@ httrack (3.49.8-1) unstable; urgency=medium
     the QA debcheck page. Depend on firefox-esr | chromium | www-browser
     instead.
 
- -- Xavier Roche <xavier@debian.org>  Sun, 07 Jun 2026 14:29:24 +0200
+ -- Xavier Roche <xavier@debian.org>  Sat, 20 Jun 2026 13:02:08 +0200
 
 httrack (3.49.7-2) unstable; urgency=medium
 

debian/control

@@ -1,9 +1,8 @@
 Source: httrack
 Section: web
-Priority: optional
 Maintainer: Xavier Roche <roche@httrack.com>
-Standards-Version: 4.7.0
-Build-Depends: debhelper-compat (= 13), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
+Standards-Version: 4.7.4
+Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
 Rules-Requires-Root: no
 Homepage: http://www.httrack.com
 Vcs-Git: https://github.com/xroche/httrack.git
@@ -30,7 +29,7 @@ Description: Copy websites to your computer (Offline browser)
 Package: webhttrack
 Architecture: any
 Multi-Arch: foreign
-Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, firefox-esr | chromium | www-browser
+Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, chromium | firefox-esr | www-browser
 Replaces: webhttrack-common (<< 3.43.9-2)
 Breaks: webhttrack-common (<< 3.43.9-2)
 Suggests: httrack, httrack-doc
@@ -58,13 +57,13 @@ Description: webhttrack common files
  This package is the common files of webhttrack, website copier and
  mirroring utility
 
-Package: libhttrack2
+Package: libhttrack3
 Architecture: any
 Multi-Arch: same
 Section: libs
-Replaces: libhttrack1
-Conflicts: libhttrack1
 Depends: ${misc:Depends}, ${shlibs:Depends}
+Replaces: libhttrack2, httrack (<< 3.49.8-2~)
+Breaks: libhttrack2, httrack (<< 3.49.8-2~)
 Description: Httrack website copier library
  This package is the library part of httrack, website copier and mirroring
  utility

debian/copyright

@@ -1,21 +1,109 @@
-This package was debianized by Xavier Roche <roche@httrack.com> on
-Fri, 27 Sep 2002 16:42:26 +0200
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
+Upstream-Name: httrack
+Upstream-Contact: Xavier Roche <roche@httrack.com>
+Source: https://www.httrack.com/
 
-The current Debian maintainer is Xavier Roche <xavier@debian.org>
+Files: *
+Copyright: 1998-2026 Xavier Roche and other contributors
+License: GPL-3+
+Comment:
+ The engine includes contributions from Yann Philippot (src/htsjava.c,
+ src/htsjava.h). htsbasenet.h links against the system OpenSSL library
+ (originally by Eric Young); no OpenSSL/SSLeay code is bundled here.
 
-Upstream author: Xavier Roche <roche@httrack.com>
+Files: src/minizip/*
+Copyright: 1998-2010 Gilles Vollant
+           2007-2008 Even Rouault
+           2009-2010 Mathias Svensson
+           1990-2000 Info-ZIP
+License: Zlib
+Comment:
+ The decryption code in src/minizip/crypt.h and src/minizip/unzip.c derives
+ from the Info-ZIP distribution, distributed under the same terms.
 
-Copyright: 1998-2014 Xavier Roche and other contributors
+Files: src/md5.c
+Copyright: 1993 Colin Plumb
+License: public-domain-md5
+ This code implements the MD5 message-digest algorithm, due to Ron Rivest.
+ It was written by Colin Plumb in 1993, no copyright is claimed. This code
+ is in the public domain; do with it what you wish.
 
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
+Files: src/coucal/*
+Copyright: 2013-2014 Xavier Roche
+License: BSD-3-clause
 
-On Debian systems, the complete text of the GNU General Public
-License version 3 can be found in /usr/share/common-licenses/GPL-3 file.
+Files: src/coucal/murmurhash3.h*
+Copyright: Austin Appleby
+License: public-domain-murmurhash3
+ MurmurHash3 was written by Austin Appleby, and is placed in the public
+ domain. The author hereby disclaims copyright to this source code.
 
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
+Files: html/server/div/com.httrack.WebHTTrack.metainfo.xml
+Copyright: 1998-2026 Xavier Roche and other contributors
+License: FSFAP
+ Copying and distribution of this file, with or without modification, are
+ permitted in any medium without royalty provided the copyright notice and
+ this notice are preserved. This file is offered as-is, without any warranty.
+
+Files: debian/*
+Copyright: 2002-2026 Xavier Roche <xavier@debian.org>
+License: GPL-3+
+
+License: GPL-3+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ .
+ On Debian systems, the complete text of the GNU General Public License
+ version 3 can be found in /usr/share/common-licenses/GPL-3.
+
+License: Zlib
+ This software is provided 'as-is', without any express or implied warranty.
+ In no event will the authors be held liable for any damages arising from the
+ use of this software.
+ .
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely, subject to the following restrictions:
+ .
+ 1. The origin of this software must not be misrepresented; you must not claim
+    that you wrote the original software. If you use this software in a product,
+    an acknowledgment in the product documentation would be appreciated but is
+    not required.
+ 2. Altered source versions must be plainly marked as such, and must not be
+    misrepresented as being the original software.
+ 3. This notice may not be removed or altered from any source distribution.
+
+License: BSD-3-clause
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ .
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+ 3. Neither the name of the copyright holder nor the names of its contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.

debian/httrack-doc.lintian-overrides

@@ -4,3 +4,6 @@
 # so the path lives in the display pointer, not the override -- match with '*'.
 httrack-doc: extra-license-file *
 httrack-doc: package-contains-documentation-outside-usr-share-doc *
+# search.sh is a sample CGI shipped alongside the HTML manual, not meant to be
+# run from the package tree; it stays non-executable by design.
+httrack-doc: script-not-executable *

debian/libhttrack-swf1.files

@@ -1,2 +0,0 @@
-usr/lib/*/libhtsswf.so.1.0.0
-usr/lib/*/libhtsswf.so.1

debian/libhttrack2.files

@@ -1,5 +0,0 @@
-usr/lib/*/libhttrack.so.2.0.49
-usr/lib/*/libhttrack.so.2
-usr/lib/*/libhtsjava.so.2.0.49
-usr/lib/*/libhtsjava.so.2
-usr/share/httrack/templates

debian/libhttrack2.lintian-overrides

@@ -1,3 +0,0 @@
-# The shared libraries ship without a versioned symbols control file (ABI is
-# tracked via the SONAME and a strict =version dependency, see debian/rules).
-libhttrack2: no-symbols-control-file usr/lib/*

debian/libhttrack3.files

@@ -0,0 +1,3 @@
+usr/lib/*/libhttrack.so.3*
+usr/lib/*/libhtsjava.so.3*
+usr/share/httrack/templates

debian/libhttrack3.lintian-overrides

@@ -0,0 +1,8 @@
+# The shared libraries ship without a versioned symbols control file (ABI is
+# tracked via the SONAME plus a >= upstream-version dependency, see debian/rules).
+libhttrack3: no-symbols-control-file usr/lib/*
+
+# Bundled, locally patched minizip (src/minizip): it adds a zipFlush() API the
+# system libminizip lacks (htscache.c flushes the cache .zip so an interrupted
+# crawl leaves a valid archive), plus Android/old-zlib portability fixes.
+libhttrack3: embedded-library *libminizip*

debian/proxytrack.lintian-overrides

@@ -0,0 +1,3 @@
+# Statically linked against httrack's bundled, patched minizip (see src/minizip
+# and libhttrack3's override): the zipFlush() API is absent from the system one.
+proxytrack: embedded-library *libminizip*

debian/rules

@@ -135,7 +135,7 @@ binary-arch: build install
 	dh_makeshlibs -a -X/usr/lib/$(DEB_HOST_MULTIARCH)/httrack/libtest --version-info
 	dh_installdeb -a
 	# we depend on the current version (ABI may change)
-	dh_shlibdeps -a -ldebian/libhttrack2/usr/lib/$(DEB_HOST_MULTIARCH)
+	dh_shlibdeps -a -ldebian/libhttrack3/usr/lib/$(DEB_HOST_MULTIARCH)
 	dh_gencontrol -a
 	dh_md5sums -a
 	dh_builddeb -a

debian/source/lintian-overrides

@@ -1,4 +1,6 @@
-httrack source: changelog-should-mention-nmu
+# Maintainer uploads sign the changelog as xavier@debian.org while the control
+# Maintainer is roche@httrack.com; lintian reads the address mismatch as an NMU.
+httrack source: no-nmu-in-changelog
 httrack source: source-nmu-has-incorrect-version-number
 
 # The bundled HTML pages are the genuine upstream documentation taken from

debian/upstream/metadata

@@ -0,0 +1,6 @@
+---
+Repository: https://github.com/xroche/httrack.git
+Repository-Browse: https://github.com/xroche/httrack
+Bug-Database: https://github.com/xroche/httrack/issues
+Bug-Submit: https://github.com/xroche/httrack/issues/new
+Contact: Xavier Roche <roche@httrack.com>

fuzz/Makefile.am

@@ -0,0 +1,38 @@
+# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
+if FUZZERS
+noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
+	fuzz-unescape fuzz-filters fuzz-url
+endif
+
+AM_CPPFLAGS = \
+	@DEFAULT_CFLAGS@ \
+	@THREADS_CFLAGS@ \
+	@V6_FLAG@ \
+	@LFS_FLAG@ \
+	-I$(top_srcdir)/src \
+	-I$(top_srcdir)/src/coucal
+
+# Static-link libhttrack.la: the internal symbols are hidden in the .so.
+AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
+LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
+
+fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
+fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
+fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
+fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
+fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
+fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
+fuzz_url_SOURCES = fuzz-url.c fuzz.h
+
+# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
+EXTRA_DIST = README.md run-fuzzers.sh \
+	corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
+	corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
+	corpus/idna/idna.txt corpus/idna/unicode.txt \
+	corpus/idna/regress-multilabel-leak.txt \
+	corpus/entities/entities.txt \
+	corpus/unescape/percent.txt \
+	corpus/filters/filter.bin corpus/filters/filter-size.bin \
+	corpus/filters/regress-empty-subject-unique.bin \
+	corpus/url/http-url.txt corpus/url/relative-path.txt \
+	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt

fuzz/README.md

@@ -0,0 +1,15 @@
+# Fuzzing httrack
+
+libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
+
+```sh
+./bootstrap
+mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
+CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
+  LDFLAGS="-fsanitize=address,undefined" \
+  bash /path/to/httrack/configure --enable-fuzzers --disable-shared
+make
+bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60   # 60s per target
+```
+
+Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.

fuzz/corpus/charset/latin1.txt

@@ -0,0 +1 @@
+
café naďve ¤

fuzz/corpus/charset/sjis.txt

@@ -0,0 +1 @@
+コンピュータ

fuzz/corpus/entities/entities.txt

@@ -0,0 +1 @@
+&<>A☃é&notarealentity;�

fuzz/corpus/filters/regress-empty-subject-unique.bin

@@ -0,0 +1 @@
+**((

fuzz/corpus/idna/idna.txt

@@ -0,0 +1 @@
+xn--bcher-kva.example.com

fuzz/corpus/idna/regress-multilabel-leak.txt

@@ -0,0 +1 @@
+büchev.例å­bücheple

fuzz/corpus/idna/unicode.txt

@@ -0,0 +1 @@
+bücher.例子.example

fuzz/corpus/meta/meta-charset.html

@@ -0,0 +1 @@
+<html><head><meta charset="utf-8"></head><body>x</body></html>

fuzz/corpus/meta/meta-http-equiv.html

@@ -0,0 +1 @@
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">

fuzz/corpus/unescape/percent.txt

@@ -0,0 +1 @@
+%41%zz%%20%c3%a9+%2e%2e%2f

fuzz/corpus/url/http-url.txt

@@ -0,0 +1 @@
+http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag

fuzz/corpus/url/regress-file-empty-path.txt

@@ -0,0 +1 @@
+file://

fuzz/corpus/url/regress-long-path-abort.txt

@@ -0,0 +1 @@
+ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../

fuzz/corpus/url/relative-path.txt

@@ -0,0 +1 @@
+ftp://ftp.example.com/pub/../file.txt

fuzz/fuzz-charset.c

@@ -0,0 +1,84 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
+   UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+static const char *const charsets[] = {
+    "utf-8",    "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
+    "us-ascii", "shift_jis",  "euc-jp",     "iso-2022-jp", "gb2312",
+    "big5",     "euc-kr",     "koi8-r",     "utf-16",      "unknown-charset",
+};
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  const char *charset;
+  char *s;
+
+  if (size == 0)
+    return 0;
+  charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+
+  {
+    char *utf8 = hts_convertStringToUTF8(s, size, charset);
+    freet(utf8);
+  }
+  {
+    char *enc = hts_convertStringFromUTF8(s, size, charset);
+    freet(enc);
+  }
+  {
+    size_t nChars = 0;
+    hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
+
+    if (ucs != NULL) {
+      char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
+      freet(back);
+      freet(ucs);
+    }
+  }
+  {
+    size_t i = 0;
+
+    while (i < size) {
+      hts_UCS4 uc = 0;
+      const size_t nr = hts_readUTF8(s + i, size - i, &uc);
+      char out[8];
+
+      if (nr == 0)
+        break;
+      hts_writeUTF8(uc, out, sizeof(out));
+      i += nr;
+    }
+  }
+
+  freet(s);
+  return 0;
+}

fuzz/fuzz-entities.c

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
+   destination size, so truncation bounds get exercised too. */
+#include "fuzz.h"
+#include "htsencoding.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t dsizes[] = {1, 2, 8, 64, 4096};
+  size_t dsize;
+  char *src, *dest;
+
+  if (size == 0)
+    return 0;
+  dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
+  data++, size--;
+  src = fuzz_strdup(data, size);
+  dest = malloct(dsize);
+
+  (void) hts_unescapeEntities(src, dest, dsize);
+  (void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
+
+  freet(dest);
+  freet(src);
+  return 0;
+}

fuzz/fuzz-filters.c

@@ -0,0 +1,65 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
+   here). Input splits on the first NUL: pattern, then subject string. */
+#include "fuzz.h"
+#include "htsfilters.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  const char *joker = buf;
+  const uint8_t *sep = memchr(data, '\0', size);
+  /* subject in its own allocation so ASan bounds it apart from the pattern */
+  char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
+                          : fuzz_strdup(data + size, 0);
+
+  (void) strjoker(nom, joker, NULL, NULL);
+  {
+    LLint sz = (LLint) size;
+    int size_flag = 0;
+
+    (void) strjoker(nom, joker, &sz, &size_flag);
+  }
+  (void) strjokerfind(nom, joker);
+  {
+    char *filter = malloct(strlen(joker) + 2);
+    char *filters[1];
+    LLint sz = (LLint) size;
+    int size_flag = 0, depth = 0;
+
+    filter[0] = '-';
+    memcpy(filter + 1, joker, strlen(joker) + 1);
+    filters[0] = filter;
+    (void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
+    freet(filter);
+  }
+
+  freet(nom);
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-idna.c

@@ -0,0 +1,47 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+
+  {
+    char *idna = hts_convertStringUTF8ToIDNA(s, size);
+    freet(idna);
+  }
+  {
+    char *utf8 = hts_convertStringIDNAToUTF8(s, size);
+    freet(utf8);
+  }
+  (void) hts_isStringIDNA(s, size);
+
+  freet(s);
+  return 0;
+}

fuzz/fuzz-meta.c

@@ -0,0 +1,40 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
+   <meta> charset declaration. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *html = fuzz_strdup(data, size);
+  char *charset = hts_getCharsetFromMeta(html, size);
+
+  freet(charset);
+  freet(html);
+  return 0;
+}

fuzz/fuzz-unescape.c

@@ -0,0 +1,53 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
+   output buffer size, so the bounded-copy contract is exercised. */
+#include "fuzz.h"
+#include "httrack-library.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t bsizes[] = {1, 2, 16, 256, 8192};
+  size_t bsize;
+  char *s, *catbuff;
+
+  if (size == 0)
+    return 0;
+  bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+  catbuff = malloct(bsize);
+
+  (void) unescape_http(catbuff, bsize, s);
+  (void) unescape_http_unharm(catbuff, bsize, s, 0);
+  (void) unescape_http_unharm(catbuff, bsize, s, 1);
+  unescape_amp(s);
+
+  freet(catbuff);
+  freet(s);
+  return 0;
+}

fuzz/fuzz-url.c

@@ -0,0 +1,49 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
+   is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
+   in place. */
+#include "fuzz.h"
+#include "htscore.h"
+#include "htslib.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+  lien_adrfil *af = calloct(1, sizeof(*af));
+  /* fil_simplifie rewrites in place and may grow an empty path to "./" */
+  char *path = malloct(size + 3);
+
+  (void) ident_url_absolute(s, af);
+  memcpy(path, s, size + 1);
+  fil_simplifie(path);
+
+  freet(path);
+  freet(af);
+  freet(s);
+  return 0;
+}

fuzz/fuzz.h

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Shared helpers for the libFuzzer harnesses. */
+#ifndef FUZZ_H
+#define FUZZ_H
+
+#define HTS_INTERNAL_BYTECODE
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "htsbase.h"
+
+/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
+static char *fuzz_strdup(const uint8_t *data, size_t size) {
+  char *s = malloct(size + 1);
+
+  memcpy(s, data, size);
+  s[size] = '\0';
+  return s;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
+
+#endif

fuzz/run-fuzzers.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Drive every built harness against its seed corpus.
+#   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
+#   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
+# Replay is crash/leak-only and never mutates, so it can't hit strjoker's
+# catastrophic backtracking; the timed mode can, hence the per-unit -timeout.
+set -euo pipefail
+
+srcdir=$(cd "$(dirname "$0")" && pwd)
+bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
+mode=${2:-20}
+
+status=0
+for f in "$bld"/fuzz-*; do
+    if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
+    case "$f" in *.o | *.c | *.dSYM) continue ;; esac
+    name=$(basename "$f")
+    corpus="$srcdir/corpus/${name#fuzz-}"
+    if [ "$mode" = "check" ]; then
+        echo "=== $name (replay) ==="
+        [ -d "$corpus" ] || continue
+        if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
+            echo "*** $name FAILED on its corpus" >&2
+            status=1
+        fi
+        continue
+    fi
+    work=$(mktemp -d)
+    args=("$work")
+    [ -d "$corpus" ] && args+=("$corpus")
+    echo "=== $name (${mode}s) ==="
+    if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
+        -artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
+        echo "*** $name FAILED; artifacts:" >&2
+        ls -l "$work" >&2
+        status=1
+    else
+        rm -rf "$work"
+    fi
+done
+exit $status

history.txt

@@ -4,13 +4,73 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
+3.49-11
++ New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
++ New: advertise deflate in Accept-Encoding and decode deflate responses (#450)
++ New: follow <source> and <track> media elements as embedded links (#451)
++ New: added modern web MIME types to the type/extension table (#448)
++ Fixed: enforce the -E time limit during a slow transfer instead of only between files (#481)
++ Fixed: sniff the leading bytes of a download so a misdeclared Content-Type no longer renames a correct URL extension
++ Fixed: fast transfers could be saved under their temporary .delayed placeholder name (#5, #107)
++ Fixed: follow a redirect that maps to the same saved file instead of writing a self-pointing stub (#159)
++ Fixed: several network-facing buffer overflows in the FTP, Java and HTML parsers
++ Fixed: the htsjava plugin could not be loaded (hidden entry points, stale library name)
++ Fixed: HTML-escape truncation and a cache-buffer leak in the parser
++ Changed: modernized the default User-Agent to an honest HTTrack identifier (#449)
++ Changed: decode the full WHATWG set of HTML named character references (#443)
++ Changed: refreshed stale HTTP status, proxy-port and TLS-floor constants (#453)
++ Changed: multiple internal hardening, build, test and CI improvements
+
+3.49-10
++ New: --cookies-file to preload a Netscape cookies.txt before crawling (#215)
++ New: --pause to space out file downloads by a random delay (#185)
++ New: --strip-query to drop selected query keys from the dedup naming (#112)
++ Changed: split the -%u URL hacks into independent --keep-www-prefix, --keep-double-slashes and --keep-query-order toggles (#271)
++ Fixed: follow a redirect Location after dropping its #fragment, instead of requesting the fragment and polluting the saved name (#204)
++ Fixed: escaped brackets inside a *[...] filter character class (#148)
++ Fixed: honor the server's Content-Range when resuming a partial download, instead of appending overlapping bytes (#198)
++ Fixed: abort the download as soon as the response type is excluded by -mime:, instead of fetching then discarding the body (#58)
++ Fixed: keep size-based filter rules neutral until the file size is known (#143)
++ Fixed: stop the mirror with a clean fatal error on a cache write failure, instead of crashing (#174, #219)
++ Fixed: stop the 412/416 partial re-get loop on --continue and --update (#206)
++ Fixed: keep an unrecognized URL tail instead of mangling it to .html (#115)
++ Fixed: honor --tolerant (-%B) on a broken Content-Length, and fix an out-of-bounds read it exposed (#32, #41)
++ Fixed: fall back to the next resolved address when a connection fails or stalls, instead of hanging on a dead IPv6 address
++ Fixed: report why a -%L URL list could not be loaded (#49)
++ Changed: multiple internal hardening, build and CI improvements
+
+.49-9
++ Fixed: file-type detection from the Content-Type header: trust a declared type over a binary URL extension, honor --assume under the delayed type check, and keep a known extension against a bogus or empty Content-Type (#267, #29, #56)
++ Fixed: an uninitialized-buffer read when the Content-Type is empty (#411)
++ Fixed: restored C++ source-compatibility of the installed headers so reverse dependencies (httraqt) build again (#413)
++ Changed: multiple internal build, packaging and test-harness improvements
+
 3.49-8
++ New: tunnel HTTPS downloads through the configured HTTP proxy via CONNECT (#85)
++ New: parse every candidate URL in <img> and <source> srcset lists (#326)
 + Changed: dropped the obsolete OpenSSL linking exception (OpenSSL 3.0+ is Apache-2.0 and GPL-compatible); httrack is now plain GPLv3-or-later
-+ Fixed: link libhtsjava and the libtest examples directly against libc
++ Fixed: several out-of-bounds reads in the HTML/CSS parser on hostile input (#94, #396)
++ Fixed: stored XSS via an unescaped URL in the generated page footer (#165)
++ Fixed: hardened buffer copies throughout the engine against overflow
++ Fixed: capture conditional CSS @import URLs (#94)
++ Fixed: don't crawl xmlns namespace declarations as links (#191)
++ Fixed: don't mistake the method argument of XMLHttpRequest.open for a URL (#218)
++ Fixed: percent-encode parentheses when rewriting CSS url() targets (#163)
++ Fixed: collapse ../ in file:// URLs and widen relative-link handling (#137, #162)
++ Fixed: drop the obsolete $Version/$Path attributes from the request Cookie header, per RFC 6265 (#151)
++ Fixed: keep empty quoted arguments when reloading doit.log for --update/--continue (#106)
++ Fixed: raise the User-Agent and custom-header length limits (#152)
++ Fixed: abort on a long log path (lock-file buffer too small) (#183)
++ Fixed: race in lazy mutex initialization (#297)
++ Fixed: sub-second mtime precision when comparing local files on POSIX (#383)
++ Fixed: modernize OpenSSL TLS initialization for the 3.x to 4.x transition (#308)
 + Fixed: in-place changes made by the postprocess callback were not applied (Roman Sęk)
 + Fixed: "preffered" typo in the help text and man page (yosinn1-blip)
 + Fixed: corrections and updates of the Russian translation (German Aizek)
 + Fixed: corrections and updates of the Danish translation (scootergrisen)
++ Fixed: link libhtsjava and the libtest examples directly against libc
++ New: documented the public library API headers and typed the option fields as named enums
++ Fixed: numerous build, packaging, CI and test-coverage improvements (out-of-tree builds, sanitizer/distcheck CI, shell and Python linting, AppStream metainfo)
 
 3.49-7
 + Fixed: keep generated config.h architecture-independent (Debian #1133728)

html/div/search.sh

@@ -1,8 +1,7 @@
-
 #!/bin/sh
 
 # Simple indexing test using HTTrack
-# A "real" script/program would use advanced search, and 
+# A "real" script/program would use advanced search, and
 # use dichotomy to find the word in the index.txt file
 # This script is really basic and NOT optimized, and
 # should not be used for professional purpose :)
@@ -11,50 +10,49 @@ TESTSITE="http://localhost/"
 
 # Create an index if necessary
 if ! test -f "index.txt"; then
-	echo "Building the index .."
-	rm -rf test
-	httrack --display "$TESTSITE"  -%I -O test
-	mv test/index.txt ./
+    echo "Building the index .."
+    rm -rf test
+    httrack --display "$TESTSITE" -%I -O test
+    mv test/index.txt ./
 fi
 
 # Convert crlf to lf
-if test "`head index.txt -n 1 | tr '\r' '#' | grep -c '#'`" = "1"; then
-	echo "Converting index to Unix LF style (not CR/LF) .."
-	mv -f index.txt index.txt.old
-	cat index.txt.old|tr -d '\r' > index.txt
+if test "$(head index.txt -n 1 | tr '\r' '#' | grep -c '#')" = "1"; then
+    echo "Converting index to Unix LF style (not CR/LF) .."
+    mv -f index.txt index.txt.old
+    tr -d '\r' <index.txt.old >index.txt
 fi
 
 keyword=-
 while test -n "$keyword"; do
-	printf "Enter a keyword: "
-	read keyword
+    printf "Enter a keyword: "
+    read -r keyword
 
-	if test -n "$keyword"; then
-		FOUNDK="`grep -niE \"^$keyword\" index.txt`"
+    if test -n "$keyword"; then
+        FOUNDK="$(grep -niE "^$keyword" index.txt)"
 
-		if test -n "$FOUNDK"; then	
-			if ! test `echo "$FOUNDK"|wc -l` = "1"; then
-				# Multiple matches
-				printf "Found multiple keywords: "
-				echo "$FOUNDK"|cut -f2 -d':'|tr '\n' ' '
-				echo ""
-				echo "Use keyword$ to find only one"
-			else
-				# One match
-				N=`echo "$FOUNDK"|cut -f1 -d':'`
-				PM=`tail +$N index.txt|grep -nE "\("|head -n 1`
-				if ! echo "$PM"|grep "ignored">/dev/null; then
-					M=`echo $PM|cut -f1 -d':'`
-					echo "Found in:"
-					cat index.txt | tail "+$N" | head -n "$M" | grep -E "[0-9]* " | cut -f2 -d' '
-				else
-					echo "keyword ignored (too many hits)"
-				fi
-					fi
-		else
-			echo "not found"
-		fi
+        if test -n "$FOUNDK"; then
+            if ! test "$(echo "$FOUNDK" | wc -l)" = "1"; then
+                # Multiple matches
+                printf "Found multiple keywords: "
+                echo "$FOUNDK" | cut -f2 -d':' | tr '\n' ' '
+                echo ""
+                echo "Use keyword$ to find only one"
+            else
+                # One match
+                N=$(echo "$FOUNDK" | cut -f1 -d':')
+                PM=$(tail "+$N" index.txt | grep -nE "\(" | head -n 1)
+                if ! echo "$PM" | grep "ignored" >/dev/null; then
+                    M=$(echo "$PM" | cut -f1 -d':')
+                    echo "Found in:"
+                    tail "+$N" index.txt | head -n "$M" | grep -E "[0-9]* " | cut -f2 -d' '
+                else
+                    echo "keyword ignored (too many hits)"
+                fi
+            fi
+        else
+            echo "not found"
+        fi
 
-	fi
+    fi
 done
-

html/filters.html

@@ -247,7 +247,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
         <td>the \ character</td>
       </tr>
       <tr>
-        <td nowrap><tt>*[\[\]]</tt></td>
+        <td nowrap><tt>*[\[,\]]</tt></td>
         <td>the [ or ] character</td>
       </tr>
       <tr>

lang/English.txt

@@ -295,7 +295,7 @@ Max Depth
 Maximum external depth:
 Maximum external depth:
 Filters (refuse/accept links) :
-Filters (refuse/accept links) :
+Filters (refuse/accept links):
 Paths
 Paths
 Save prefs

lang/README.md

@@ -0,0 +1,37 @@
+# Translating HTTrack
+
+Interface strings live here, one `.txt` file per language. `English.txt` is the reference: every other file maps each English string to its translation.
+
+## File format
+
+Plain text, entries in consecutive pairs of lines:
+
+```
+<English string>
+<translation>
+```
+
+The first line of a pair is the lookup key and must stay identical to the one in `English.txt`; translate only the second line. Missing entries fall back to the English text at runtime, so a partial translation works.
+
+Preserve any `\r\n`, `\t` and `printf` placeholders (`%s`, `%d`, ...) in the translation.
+
+A few `LANGUAGE_*` entries at the top describe the file itself:
+
+| Key | Meaning |
+| --- | --- |
+| `LANGUAGE_NAME` | Name shown in the language picker, in its own language (`Deutsch`, not `German`) |
+| `LANGUAGE_ISO` | ISO 639 code, with region if needed (`de`, `pt_BR`) |
+| `LANGUAGE_CHARSET` | Encoding the file is saved in (`ISO-8859-1`, `windows-1251`, `UTF-8`, ...) |
+| `LANGUAGE_AUTHOR` | Your name and contact |
+| `LANGUAGE_WINDOWSID` | Windows locale name used by WinHTTrack (`German (Standard)`) |
+
+Save the file in exactly its declared `LANGUAGE_CHARSET`; an editor that rewrites it as UTF-8 will corrupt the non-ASCII bytes.
+
+## Adding or updating a language
+
+1. Copy `English.txt` to `<Language>.txt`, or edit the existing file.
+2. Translate each second line; leave the English keys untouched.
+3. Fill in the `LANGUAGE_*` header for a new file.
+4. Open a pull request, or attach the file to a GitHub issue.
+
+When new strings land in `English.txt` they show up untranslated (as English) until a translator fills them in.

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "13 June 2026" "httrack website copier"
+.TH httrack 1 "27 June 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -24,6 +24,7 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-EN, \-\-max\-time[=N]\fR ]
 [ \fB\-AN, \-\-max\-rate[=N]\fR ]
 [ \fB\-%cN, \-\-connection\-per\-second[=N]\fR ]
+[ \fB\-%G, \-\-pause\fR ]
 [ \fB\-GN, \-\-max\-pause[=N]\fR ]
 [ \fB\-cN, \-\-sockets[=N]\fR ]
 [ \fB\-TN, \-\-timeout[=N]\fR ]
@@ -43,11 +44,13 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-x, \-\-replace\-external\fR ]
 [ \fB\-%x, \-\-disable\-passwords\fR ]
 [ \fB\-%q, \-\-include\-query\-string\fR ]
+[ \fB\-%g, \-\-strip\-query\fR ]
 [ \fB\-o, \-\-generate\-errors\fR ]
 [ \fB\-X, \-\-purge\-old[=N]\fR ]
 [ \fB\-%p, \-\-preserve\fR ]
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
+[ \fB\-%K, \-\-cookies\-file\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -153,6 +156,8 @@ maximum mirror time in seconds (60=1 minute, 3600=1 hour) (\-\-max\-time[=N])
 maximum transfer rate in bytes/seconds (1000=1KB/s max) (\-\-max\-rate[=N])
 .IP \-%cN
 maximum number of connections/seconds (*%c10) (\-\-connection\-per\-second[=N])
+.IP \-%G
+random pause of MIN[:MAX] seconds between files (e.g. %G5:10) (\-\-pause <param>)
 .IP \-GN
 pause transfer if N bytes reached, and wait until lock file is deleted (\-\-max\-pause[=N])
 .SS Flow control:
@@ -198,6 +203,8 @@ replace external html links by error pages (\-\-replace\-external)
 do not include any password for external password protected websites (%x0 include) (\-\-disable\-passwords)
 .IP \-%q
 *include query string for local files (useless, for information purpose only) (%q0 don't include) (\-\-include\-query\-string)
+.IP \-%g
+strip query keys for dedup ([host/pattern=]key1,key2,...) (\-\-strip\-query <param>)
 .IP \-o
 *generate output html file in case of error (404..) (o0 don't generate) (\-\-generate\-errors)
 .IP \-X
@@ -209,6 +216,8 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 .SS Spider options:
 .IP \-bN
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
+.IP \-%K
+load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j
@@ -225,6 +234,8 @@ tolerant requests (accept bogus responses on some servers, but not standard!) (\
 update hacks: various hacks to limit re\-transfers when updating (identical size, bogus response..) (\-\-updatehack)
 .IP \-%u
 url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..) (\-\-urlhack)
+.br
+opt out of one url\-hack part: \-\-keep\-www\-prefix (www.foo.com<>foo.com), \-\-keep\-double\-slashes (//), \-\-keep\-query\-order (?b&a)
 .IP \-%A
 assume that a type (cgi,asp..) is always linked with a mime type (\-%A php3,cgi=text/html;dat,bin=application/x\-zip) (\-\-assume <param>)
 .br
@@ -313,12 +324,8 @@ debug HTTP headers in logfile (\-\-debug\-headers)
 .SS Guru options: (do NOT use if possible)
 .IP \-#X
 *use optimized engine (limited memory boundary checks) (\-\-fast\-engine)
-.IP \-#0
-filter test (\-#0 '*.gif' 'www.bar.com/foo.gif') (\-\-debug\-testfilters <param>)
-.IP \-#1
-simplify test (\-#1 ./foo/bar/../foobar)
-.IP \-#2
-type test (\-#2 /foo/bar.php)
+.IP \-#test
+list engine self\-tests (run one with \-#test=NAME [args])
 .IP \-#C
 cache list (\-#C '*.com/spider*.gif' (\-\-debug\-cache <param>)
 .IP \-#R

src/Makefile.am

@@ -56,21 +56,21 @@ whttrackrundir = $(bindir)
 whttrackrun_SCRIPTS = webhttrack
 
 libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
-	htscache_selftest.c \
+	htscache_selftest.c htsdns_selftest.c htsselftest.c \
 	htscatchurl.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
 	htshelp.c htslib.c htscoremain.c \
 	htsname.c htsrobots.c htstools.c htswizard.c \
 	htsalias.c htsthread.c htsindex.c htsbauth.c \
 	htsmd5.c htszlib.c htswrap.c htsconcat.c \
-	htsmodules.c htscharset.c punycode.c htsencoding.c \
+	htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
 	md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
 	hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
-	htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htscatchurl.h  \
+	htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htsdns_selftest.h htsselftest.h htscatchurl.h  \
 	htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h  \
 	htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
 	htshelp.h htsindex.h htslib.h htsmd5.h \
-	htsmodules.h htsname.h htsnet.h \
+	htsmodules.h htsname.h htsnet.h htssniff.h \
 	htsopt.h htsrobots.h htsthread.h  \
 	htstools.h htswizard.h htswrap.h htszlib.h  \
 	htsstrings.h htsarrays.h httrack-library.h \

src/htsalias.c

@@ -60,6 +60,9 @@ Please visit our Website: http://www.httrack.com
   param1 : this option must be alone, and needs one distinct parameter (-P <path>)
   param0 : this option must be alone, but the parameter should be put together (+*.gif)
 */
+/* clang-format off: hand-aligned table; clang-format reflows the whole
+   initializer (2->4 space) on any edit, churning every untouched row. */
+/* clang-format off */
 const char *hts_optalias[][4] = {
   /*   {"","","",""}, */
   {"path", "-O", "param1", "output path"},
@@ -107,6 +110,12 @@ const char *hts_optalias[][4] = {
   {"disable-passwords", "-%x", "single", ""}, {"disable-password", "-%x",
                                                "single", ""},
   {"include-query-string", "-%q", "single", ""},
+  {"strip-query", "-%g", "param1",
+   "strip [host/pattern=]key1,key2,... from URLs"},
+  {"cookies-file", "-%K", "param1",
+   "load extra cookies from a Netscape cookies.txt"},
+  {"pause", "-%G", "param1",
+   "random pause of MIN[:MAX] seconds between files"},
   {"generate-errors", "-o", "single", ""},
   {"do-not-generate-errors", "-o0", "single", ""},
   {"purge-old", "-X", "param", ""},
@@ -123,6 +132,9 @@ const char *hts_optalias[][4] = {
   {"tolerant", "-%B", "single", ""},
   {"updatehack", "-%s", "single", ""}, {"sizehack", "-%s", "single", ""},
   {"urlhack", "-%u", "single", ""},
+  {"keep-www-prefix", "-%j", "single", ""},
+  {"keep-double-slashes", "-%o", "single", ""},
+  {"keep-query-order", "-%y", "single", ""},
   {"user-agent", "-F", "param1", "user-agent identity"},
   {"referer", "-%R", "param1", "default referer URL"},
   {"from", "-%E", "param1", "from email address"},
@@ -241,6 +253,7 @@ const char *hts_optalias[][4] = {
 
   {"", "", "", ""}
 };
+/* clang-format on */
 
 /* 
   Check for alias in command-line 
@@ -270,7 +283,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
       char *position;
       int need_param = 1;
 
-      //int return_param=0;
       int pos;
 
       command[0] = param[0] = addcommand[0] = '\0';
@@ -348,7 +360,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
               strlcatbuff(return_argv[0], "0", return_argv_size);
             else if (strcmp(param, "on") == 0) {
               // on is the default
-              // strcatbuff(return_argv[0],"1");
             } else
               strlcatbuff(return_argv[0], param, return_argv_size);
           }

src/htsarrays.h

@@ -48,112 +48,115 @@ Please visit our Website: http://www.httrack.com
 /* Abort (with the failed byte count) when a growth allocation fails. The
    array macros never return an out-of-memory error; they assert and abort. */
 static void hts_record_assert_memory_failed(const size_t size) {
-  fprintf(stderr, "memory allocation failed (%lu bytes)", \
-          (long int) size); \
-  assertf(! "memory allocation failed"); \
+  fprintf(stderr, "memory allocation failed (%lu bytes)", (long int) size);
+  assertf(!"memory allocation failed");
 }
 
 /** Dynamic array of T elements. **/
-#define TypedArray(T) \
-  struct {            \
-    /** Elements. **/ \
-    union {           \
-      /** Typed. **/  \
-      T* elts;        \
-      /** Opaque. **/ \
-      void* ptr;      \
-    } data;           \
-    /** Count. **/    \
-    size_t size;      \
-    /** Capacity. **/ \
-    size_t capa;      \
+#define TypedArray(T)                                                          \
+  struct {                                                                     \
+    /** Elements. **/                                                          \
+    union {                                                                    \
+      /** Typed. **/                                                           \
+      T *elts;                                                                 \
+      /** Opaque. **/                                                          \
+      void *ptr;                                                               \
+    } data;                                                                    \
+    /** Count. **/                                                             \
+    size_t size;                                                               \
+    /** Capacity. **/                                                          \
+    size_t capa;                                                               \
   }
 
 /** Initializer for an empty array (no backing store, size and capacity 0). **/
-#define EMPTY_TYPED_ARRAY { { NULL }, 0, 0 }
+#define EMPTY_TYPED_ARRAY {{NULL}, 0, 0}
 
 /** Array size, in elements. **/
-#define TypedArraySize(A)   ((A).size)
+#define TypedArraySize(A) ((A).size)
 
 /** Array capacity, in elements. **/
-#define TypedArrayCapa(A)   ((A).capa)
+#define TypedArrayCapa(A) ((A).capa)
 
 /**
- * Remaining free space, in elements. 
+ * Remaining free space, in elements.
  * Macro, first element evaluated multiple times.
  **/
-#define TypedArrayRoom(A)   ( TypedArrayCapa(A) - TypedArraySize(A) )
+#define TypedArrayRoom(A) (TypedArrayCapa(A) - TypedArraySize(A))
 
 /** Array elements, of type T*. **/
-#define TypedArrayElts(A)   ((A).data.elts)
+#define TypedArrayElts(A) ((A).data.elts)
 
 /** Array pointer, of type void*. **/
-#define TypedArrayPtr(A)    ((A).data.ptr)
+#define TypedArrayPtr(A) ((A).data.ptr)
 
 /** Size of T. **/
-#define TypedArrayWidth(A)  (sizeof(*TypedArrayElts(A)))
+#define TypedArrayWidth(A) (sizeof(*TypedArrayElts(A)))
 
 /** Nth element of the array, as an lvalue. No bounds check; N must be
     < TypedArraySize(A). **/
 #define TypedArrayNth(A, N) (TypedArrayElts(A)[N])
 
 /**
- * Tail of the array (outside the array). 
+ * Tail of the array (outside the array).
  * The returned pointer points to the beginning of TypedArrayRoom(A)
  * free elements.
  **/
 #define TypedArrayTail(A) (TypedArrayNth(A, TypedArraySize(A)))
 
 /**
-  * Ensure at least 'ROOM' elements can be put in the remaining space.
-  * After a call to this macro, TypedArrayRoom(A) is guaranteed to be at 
-  * least equal to 'ROOM'.
-  **/
-#define TypedArrayEnsureRoom(A, ROOM) do { \
-  const size_t room_ = (ROOM); \
-  while (TypedArrayRoom(A) < room_) { \
-    TypedArrayCapa(A) = TypedArrayCapa(A) < 16 ? 16 : TypedArrayCapa(A) * 2; \
-  } \
-  TypedArrayPtr(A) = realloc(TypedArrayPtr(A), \
-                             TypedArrayCapa(A)*TypedArrayWidth(A)); \
-  if (TypedArrayPtr(A) == NULL) { \
-    hts_record_assert_memory_failed(TypedArrayCapa(A)*TypedArrayWidth(A)); \
-  } \
-} while(0)
+ * Ensure at least 'ROOM' elements can be put in the remaining space.
+ * After a call to this macro, TypedArrayRoom(A) is guaranteed to be at
+ * least equal to 'ROOM'.
+ **/
+#define TypedArrayEnsureRoom(A, ROOM)                                          \
+  do {                                                                         \
+    const size_t room_ = (ROOM);                                               \
+    while (TypedArrayRoom(A) < room_) {                                        \
+      TypedArrayCapa(A) = TypedArrayCapa(A) < 16 ? 16 : TypedArrayCapa(A) * 2; \
+    }                                                                          \
+    TypedArrayPtr(A) =                                                         \
+        realloc(TypedArrayPtr(A), TypedArrayCapa(A) * TypedArrayWidth(A));     \
+    if (TypedArrayPtr(A) == NULL) {                                            \
+      hts_record_assert_memory_failed(TypedArrayCapa(A) * TypedArrayWidth(A)); \
+    }                                                                          \
+  } while (0)
 
 /** Add an element. Macro, first element evaluated multiple times. **/
-#define TypedArrayAdd(A, E) do { \
-  TypedArrayEnsureRoom(A, 1); \
-  assertf(TypedArraySize(A) < TypedArrayCapa(A)); \
-  TypedArrayTail(A) = (E); \
-  TypedArraySize(A)++; \
-} while(0)
+#define TypedArrayAdd(A, E)                                                    \
+  do {                                                                         \
+    TypedArrayEnsureRoom(A, 1);                                                \
+    assertf(TypedArraySize(A) < TypedArrayCapa(A));                            \
+    TypedArrayTail(A) = (E);                                                   \
+    TypedArraySize(A)++;                                                       \
+  } while (0)
 
 /**
- * Add 'COUNT' elements from 'PTR'. 
+ * Add 'COUNT' elements from 'PTR'.
  * Macro, first element evaluated multiple times.
  **/
-#define TypedArrayAppend(A, PTR, COUNT) do { \
-  const size_t count_ = (COUNT); \
-  /* This 1-case is to benefit from type safety. */ \
-  if (count_ == 1) { \
-    TypedArrayAdd(A, *(PTR)); \
-  } else { \
-    const void *const source_ = (PTR); \
-    TypedArrayEnsureRoom(A, count_); \
-    assertf(count_ <= TypedArrayRoom(A)); \
-    memcpy(&TypedArrayTail(A), source_, count_ * TypedArrayWidth(A)); \
-    TypedArraySize(A) += count_; \
-  } \
-} while(0)
+#define TypedArrayAppend(A, PTR, COUNT)                                        \
+  do {                                                                         \
+    const size_t count_ = (COUNT);                                             \
+    /* This 1-case is to benefit from type safety. */                          \
+    if (count_ == 1) {                                                         \
+      TypedArrayAdd(A, *(PTR));                                                \
+    } else {                                                                   \
+      const void *const source_ = (PTR);                                       \
+      TypedArrayEnsureRoom(A, count_);                                         \
+      assertf(count_ <= TypedArrayRoom(A));                                    \
+      memcpy(&TypedArrayTail(A), source_, count_ *TypedArrayWidth(A));         \
+      TypedArraySize(A) += count_;                                             \
+    }                                                                          \
+  } while (0)
 
 /** Clear an array, freeing memory and clearing size and capacity. **/
-#define TypedArrayFree(A) do { \
-  if (TypedArrayPtr(A) != NULL) { \
-    TypedArrayCapa(A) = TypedArraySize(A) = 0; \
-    free(TypedArrayPtr(A)); \
-    TypedArrayPtr(A) = NULL; \
-  } \
-} while(0)
+#define TypedArrayFree(A)                                                      \
+  do {                                                                         \
+    if (TypedArrayPtr(A) != NULL) {                                            \
+      TypedArrayCapa(A) = TypedArraySize(A) = 0;                               \
+      free(TypedArrayPtr(A));                                                  \
+      TypedArrayPtr(A) = NULL;                                                 \
+    }                                                                          \
+  } while (0)
 
 #endif

src/htsback.c

@@ -43,21 +43,22 @@ Please visit our Website: http://www.httrack.com
 
 #include "htsback.h"
 
-//#ifdef _WIN32
 #include "htsftp.h"
 #if HTS_USEZLIB
 #include "htszlib.h"
 #else
 #error HTS_USEZLIB not defined
 #endif
-//#endif
 
 #ifdef _WIN32
 #ifndef __cplusplus
 // DOS
 #include <process.h>            /* _beginthread, _endthread */
 #endif
+#include <io.h> /* _chsize_s */
+#define HTS_FTRUNCATE(fp, sz) _chsize_s(_fileno(fp), (sz))
 #else
+#define HTS_FTRUNCATE(fp, sz) ftruncate(fileno(fp), (sz))
 #endif
 
 #define VT_CLREOL       "\33[K"
@@ -73,6 +74,8 @@ struct_back *back_new(httrackp *opt, int back_max) {
 
   sback->count = back_max;
   sback->lnk = (lien_back *) calloct((back_max + 1), sizeof(lien_back));
+  sback->connect_fallback = (hts_connect_fallback *) calloct(
+      (back_max + 1), sizeof(hts_connect_fallback));
   sback->ready = coucal_new(0);
   hts_set_hash_handler(sback->ready, opt);
   coucal_set_name(sback->ready, "back_new");
@@ -83,6 +86,7 @@ struct_back *back_new(httrackp *opt, int back_max) {
     sback->lnk[i].r.location = sback->lnk[i].location_buffer;
     sback->lnk[i].status = STATUS_FREE;
     sback->lnk[i].r.soc = INVALID_SOCKET;
+    sback->connect_fallback[i].addr_count = -1; // not yet probed
   }
   return sback;
 }
@@ -93,6 +97,7 @@ void back_free(struct_back ** sback) {
       freet((*sback)->lnk);
       (*sback)->lnk = NULL;
     }
+    freet((*sback)->connect_fallback);
     if ((*sback)->ready != NULL) {
       coucal_delete(&(*sback)->ready);
       (*sback)->ready_size_bytes = 0;
@@ -102,6 +107,72 @@ void back_free(struct_back ** sback) {
   }
 }
 
+/* Per-candidate connect deadline cap (seconds): a connecting slot with another
+   address to try waits at most this long before falling back, instead of the
+   full (default 120s) slot timeout. Caps the dead-IPv6 stall while staying well
+   above a normal handshake. The last candidate still gets the full timeout. */
+#define HTS_CONNECT_FALLBACK_TIMEOUT 10
+
+int back_connect_fallback_due(int addr_index, int addr_count, int elapsed,
+                              int timeout) {
+  int deadline;
+
+  if (addr_index + 1 >= addr_count) // last (or only) candidate: no fallback
+    return 0;
+  if (timeout <= 0) // no timeout management: never force it
+    return 0;
+  deadline = (timeout < HTS_CONNECT_FALLBACK_TIMEOUT)
+                 ? timeout
+                 : HTS_CONNECT_FALLBACK_TIMEOUT;
+  return elapsed >= deadline;
+}
+
+/* Pending-connect result for a non-blocking socket reported ready by select():
+   0 = connected, >0 = the connect errno (refused, unreachable, ...), -1 if the
+   probe itself failed. A failed connect is reported writable too, so this is
+   how success is told from failure without blocking. */
+static int connect_socket_error(T_SOC soc) {
+  int soerr = 0;
+  socklen_t len = (socklen_t) sizeof(soerr);
+
+  if (getsockopt(soc, SOL_SOCKET, SO_ERROR, (char *) &soerr, &len) != 0)
+    return -1;
+  return soerr;
+}
+
+/* Retry a stuck/failed connecting slot against its next resolved address.
+   Closes the current socket and starts a non-blocking connect to the next
+   candidate, leaving the slot in STATUS_CONNECTING. Returns 1 if a new connect
+   was started, 0 if no fallback address remains (caller fails the slot). */
+static int back_connect_next(httrackp *opt, struct_back *sback, int i) {
+  hts_connect_fallback *const cf = &sback->connect_fallback[i];
+  lien_back *const back = sback->lnk;
+  const int next = cf->addr_index + 1;
+  T_SOC soc;
+
+  if (next >= cf->addr_count)
+    return 0;
+
+  if (back[i].r.soc != INVALID_SOCKET) {
+    deletehttp(&back[i].r);
+    back[i].r.soc = INVALID_SOCKET;
+  }
+  soc = newhttp_addr(opt, back[i].url_adr, &back[i].r, -1, 0, next, NULL);
+  if (soc == INVALID_SOCKET)
+    return 0;
+
+  back[i].r.soc = soc;
+  cf->addr_index = next;
+  cf->connect_start = time_local();
+  if (back[i].timeout > 0)
+    back[i].timeout_refresh = cf->connect_start;
+  back[i].status = STATUS_CONNECTING;
+  hts_log_print(opt, LOG_DEBUG,
+                "connect failed, trying next address (%d/%d) for %s", next + 1,
+                cf->addr_count, back[i].url_adr);
+  return 1;
+}
+
 void back_delete_all(httrackp * opt, cache_back * cache, struct_back * sback) {
   if (sback != NULL) {
     int i;
@@ -499,9 +570,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
         && back[p].r.size != back[p].r.totalsize && !opt->tolerant) {
       if (back[p].status == STATUS_READY) {
         hts_log_print(opt, LOG_WARNING,
-                      "file not stored in cache due to bogus state (broken size, expected "
-                      LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
-                      back[p].r.size, back[p].url_adr, back[p].url_fil);
+                      "incomplete transfer (expected " LLintP
+                      " bytes, got " LLintP
+                      "): file not cached, will be retried on the next update"
+                      " (use -%%B to cache anyway): %s%s",
+                      back[p].r.totalsize, back[p].r.size, back[p].url_adr,
+                      back[p].url_fil);
       } else {
         hts_log_print(opt, LOG_INFO,
                       "incomplete file not yet stored in cache (expected "
@@ -521,7 +595,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
 #if HTS_USEZLIB
         if (back[p].r.compressed) {
           if (back[p].r.size > 0) {
-            //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
             // stats
             back[p].compressed_size = back[p].r.size;
             // en mémoire -> passage sur disque
@@ -806,11 +879,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                          back[p].url_fil, NULL);
           } else {
             /* Partial file, but marked as "ok" ? */
-            hts_log_print(opt, LOG_WARNING,
-                          "file not stored in cache due to bogus state (incomplete type with %s (%d), size "
-                          LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode,
-                          (LLint) back[p].r.size, back[p].url_adr,
-                          back[p].url_fil);
+            hts_log_print(
+                opt, LOG_WARNING,
+                "file with unresolved type not cached (%s (%d), size " LLintP
+                "): %s%s",
+                back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size,
+                back[p].url_adr, back[p].url_fil);
           }
         }
 
@@ -850,7 +924,6 @@ int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
     /* clear everything but connection: switch, close, and reswitch */
     back_connxfr(src, &tmp);
     back_delete(opt, cache, sback, p);
-    //deletehttp(src);
     back_connxfr(&tmp, src);
     src->req.flush_garbage = 1; /* ignore CRLF garbage */
     return 1;
@@ -1285,7 +1358,49 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
   return 0;
 }
 
+/* Move a still-writing .delayed placeholder to its final name (#483). */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname) {
+  hts_boolean renamed;
+
+  if (!back->r.is_write || back->tmpfile != NULL ||
+      !IS_DELAYED_EXT(back->url_sav) || strcmp(back->url_sav, newname) == 0)
+    return HTS_TRUE; /* nothing bound to the placeholder name */
+  if (back->r.out != NULL) {
+    fclose(back->r.out);
+    back->r.out = NULL;
+  }
+  renamed = RENAME(back->url_sav, newname) == 0 ? HTS_TRUE : HTS_FALSE;
+  if (renamed && (back->status == STATUS_READY ||
+                  (back->r.out = FOPEN(newname, "ab")) != NULL)) {
+    filenote(&opt->state.strc, newname, NULL);
+    hts_log_print(opt, LOG_DEBUG, "moved placeholder %s to %s", back->url_sav,
+                  newname);
+    return HTS_TRUE;
+  }
+  /* partial lost: drop only what we own (Windows rename won't overwrite) */
+  hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "unable to move %s to %s",
+                back->url_sav, newname);
+  back->r.statuscode = STATUSCODE_INVALID;
+  strcpybuff(back->r.msg, "Write error on disk");
+  back->r.is_write = 0;
+  (void) UNLINK(renamed ? newname : back->url_sav);
+  return HTS_FALSE;
+}
+
 // effacer entrée
+/* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
+void back_delayed_discard(httrackp *opt, lien_back *back) {
+  if (back->r.out != NULL) {
+    fclose(back->r.out);
+    back->r.out = NULL;
+  }
+  back->r.is_write = 0;
+  if (opt != NULL)
+    url_savename_refname_remove(opt, back->url_adr, back->url_fil);
+  (void) UNLINK(back->url_sav);
+}
+
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p) {
   lien_back *const back = sback->lnk;
@@ -1293,6 +1408,12 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
 
   assertf(p >= 0 && p < back_max);
   if (p >= 0 && p < sback->count) {     // on sait jamais..
+    /* mid-write cancel: drop a .delayed placeholder; real-named partials
+       survive for resume (--continue) */
+    if (back[p].r.is_write && IS_DELAYED_EXT(back[p].url_sav) &&
+        (back[p].status != STATUS_READY || back[p].r.statuscode <= 0)) {
+      back_delayed_discard(opt, &back[p]);
+    }
     // Vérificateur d'intégrité
 #if DEBUG_CHECKINT
     _CHECKINT(&back[p], "Appel back_delete")
@@ -1312,7 +1433,6 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                       back[p].url_adr, back[p].url_fil, back[p].url_sav);
       }
       if (cache != NULL) {
-        //hts_log_print(opt, LOG_TRACE, "finalizing from back_delete");
         back_finalize(opt, cache, sback, p);
       }
     }
@@ -1453,7 +1573,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     if (back[p].r.soc != INVALID_SOCKET) {      /* we never know */
       deletehttp(&back[p].r);
     }
-    //memset(&(back[p].r), 0, sizeof(htsblk)); 
     hts_init_htsblk(&back[p].r);
     back[p].r.location = back[p].location_buffer;
 
@@ -1461,7 +1580,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     strcpybuff(back[p].url_adr, adr);
     strcpybuff(back[p].url_fil, fil);
     strcpybuff(back[p].url_sav, save);
-    //back[p].links_index = links_index;
     // copier referer si besoin
     strcpybuff(back[p].referer_adr, "");
     strcpybuff(back[p].referer_fil, "");
@@ -1700,8 +1818,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
                             back[p].url_adr, back[p].url_fil);
             }
             back[p].r.notmodified = 1;  // fichier non modifié
-            back[p].status = STATUS_READY;      // OK prêt
-            //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);        // not modified
+            back[p].status = STATUS_READY; // OK prêt
             back_set_finished(sback, p);
 
             // finalize transfer
@@ -1716,7 +1833,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
           } else {              // erreur
             // effacer r
             hts_init_htsblk(&back[p].r);
-            //memset(&(back[p].r), 0, sizeof(htsblk)); 
             back[p].r.location = back[p].location_buffer;
             // et continuer (chercher le fichier)
           }
@@ -1911,8 +2027,10 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       // ouvrir liaison, envoyer requète
       // ne pas traiter ou recevoir l'en tête immédiatement
       hts_init_htsblk(&back[p].r);
-      //memset(&(back[p].r), 0, sizeof(htsblk)); 
       back[p].r.location = back[p].location_buffer;
+      // fresh connect: address list not yet probed, start at the first
+      sback->connect_fallback[p].addr_index = 0;
+      sback->connect_fallback[p].addr_count = -1;
       // recopier proxy
       if ((back[p].r.req.proxy.active = opt->proxy.active)) {
         if (StringBuff(opt->proxy.bindhost) != NULL)
@@ -1959,7 +2077,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #if HTS_USEOPENSSL
       else if (strfield(back[p].url_adr, "https://")) {     // let's rock
         back[p].r.ssl = 1;
-        // back[p].r.ssl_soc = NULL;
         back[p].r.ssl_con = NULL;
       }
 #endif
@@ -2025,9 +2142,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
         back[p].rateout = -1;   // pas de gestion (default)
       }
 
-      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est
-      // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
-      //if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
+      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où
+      // cela est rattrapable (exemple: 301,302 moved xxx -> refresh sur la
+      // page!)
 
 #if CNXDEBUG
       printf("Xfopen ok, poll..\n");
@@ -2045,7 +2162,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       if (soc == INVALID_SOCKET) {      // erreur socket
         back[p].status = STATUS_READY;  // FINI
         back_set_finished(sback, p);
-        //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
         back[p].r.soc = INVALID_SOCKET;
       } else {
         if (!back[p].r.is_file)
@@ -2066,8 +2182,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
     // note: si il y a erreur (404,etc) status=2 (terminé/échec) mais
     // le lien est considéré comme traité
-    //if (back[p].soc<0)  // erreur
-    //  return -1;
 
     return 0;
   } else {
@@ -2140,9 +2254,6 @@ void back_solve(httrackp * opt, lien_back * back) {
     } else {
       hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
     }
-    //if (hts_dnstest(opt, a, 1) == 2) { // non encore testé!..
-    //  hts_log_print(opt, LOG_DEBUG, "resolving in background: %s", a);
-    //}
   }
 }
 
@@ -2161,12 +2272,13 @@ int host_wait(httrackp * opt, lien_back * back) {
 
 static int slot_can_be_cleaned(const lien_back * back) {
   return (back->status == STATUS_READY) // ready
-    /* Check autoclean */
-    && (!back->testmode)        // not test mode
-    && (strnotempty(back->url_sav))     // filename exists
-    && (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
-    && (back->r.size >= 0)      // size>=0
-    ;
+         /* Check autoclean */
+         && (!back->locked)   // not held by hts_wait_delayed (name pending)
+         && (!back->testmode) // not test mode
+         && (strnotempty(back->url_sav))     // filename exists
+         && (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
+         && (back->r.size >= 0)              // size>=0
+      ;
 }
 
 static int slot_can_be_finalized(httrackp * opt, const lien_back * back) {
@@ -2190,11 +2302,6 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) {
         (void) back_flush_output(opt, cache, sback, i); // flush output buffers
         usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr,
                     back[i].url_fil);
-        //if (back[i].links_index >= 0) {
-        //  assertf(back[i].links_index < opt->hash->max_lien);
-        //  opt->hash->liens[back[i].links_index]->pass2 = -1;
-        //  // *back[i].pass2_ptr=-1;  // Done!
-        //}
         /* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
         {
           int index = hash_read(opt->hash, back[i].url_sav, NULL, HASH_STRUCT_FILENAME );       // lecture type 0 (sav)
@@ -2342,6 +2449,34 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
+  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
+     starves (#481). FTP slots stay, their thread owns the socket. */
+  if (!back_checkmirror(opt)) {
+    int aborted = 0;
+    unsigned int i;
+
+    for (i = 0; i < (unsigned int) back_max; i++) {
+      if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
+        if (back[i].r.soc != INVALID_SOCKET) {
+          deletehttp(&back[i].r);
+        }
+        back[i].r.soc = INVALID_SOCKET;
+        /* drop a .delayed placeholder; real partials survive for resume */
+        if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
+          back_delayed_discard(opt, &back[i]);
+        back[i].r.statuscode = STATUSCODE_TIMEOUT;
+        strcpybuff(back[i].r.msg, "Mirror Time Out");
+        back[i].status = STATUS_READY;
+        back_set_finished(sback, i);
+        aborted++;
+      }
+    }
+    if (aborted > 0)
+      hts_log_print(opt, LOG_WARNING,
+                    "time limit reached, %d transfer(s) aborted", aborted);
+    return;
+  }
+
   // recevoir tant qu'il y a des données (avec un maximum de max_loop boucles)
   do_wait = 0;
   gestion_timeout = 0;
@@ -2350,9 +2485,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
     busy_state = busy_recv = 0;
 
-#if 0
-    check_rate(stat_timestart, opt->maxrate);   // vérifier taux de transfert
-#endif
     // inscrire les sockets actuelles, et rechercher l'ID la plus élevée
     FD_ZERO(&fds);
     FD_ZERO(&fds_c);
@@ -2362,28 +2494,31 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
     nfds = INVALID_SOCKET;
 
     max_c = 1;
-    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-      // for(i=0;i<back_max;i++) {
+    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       // en cas de gestion du connect préemptif
 #if HTS_XCONN
       if (back[i].status == STATUS_CONNECTING) {        // connexion
-        do_wait = 1;
+        // a connecting slot always carries a live socket; guard anyway so a
+        // stray INVALID_SOCKET can never reach FD_SET (mirrors the recv branch)
+        if (back[i].r.soc != INVALID_SOCKET) {
+          do_wait = 1;
 
-        // noter socket write
-        FD_SET(back[i].r.soc, &fds_c);
+          // noter socket write
+          FD_SET(back[i].r.soc, &fds_c);
 
-        // noter socket erreur
-        FD_SET(back[i].r.soc, &fds_e);
+          // noter socket erreur
+          FD_SET(back[i].r.soc, &fds_e);
 
-        // calculer max
-        if (max_c) {
-          max_c = 0;
-          nfds = back[i].r.soc;
-        } else if (back[i].r.soc > nfds) {
-          // ID socket la plus élevée
-          nfds = back[i].r.soc;
+          // calculer max
+          if (max_c) {
+            max_c = 0;
+            nfds = back[i].r.soc;
+          } else if (back[i].r.soc > nfds) {
+            // ID socket la plus élevée
+            nfds = back[i].r.soc;
+          }
         }
 
       } else
@@ -2396,8 +2531,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         // poll pour la lecture sur les sockets
       if ((back[i].status > 0) && (back[i].status < 100)) {     // en réception http
 
-#if BDEBUG==1
-        //printf("....socket in progress: %d\n",back[i].r.soc);
+#if BDEBUG == 1
 #endif
         // non local et non ftp
         if (!back[i].r.is_file) {
@@ -2483,8 +2617,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       busy_recv = 0;
 
     // recevoir les données arrivées
-    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-      // for(i=0;i<back_max;i++) {
+    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       if (back[i].status > 0) {
@@ -2517,8 +2650,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       }
       // ---- FLAG WRITE MIS A UN?: POUR LE CONNECT
       if (back[i].status == STATUS_CONNECTING) {        // attendre connect
+        hts_connect_fallback *const cf = &sback->connect_fallback[i];
         int dispo = 0;
 
+        // probe the resolved address list once per fresh connect (cache hit:
+        // the host was resolved when this connect was opened)
+        if (cf->addr_count < 0 && back[i].r.soc != INVALID_SOCKET &&
+            !back[i].r.is_file) {
+          SOCaddr scratch[HTS_MAXADDRNUM];
+
+          cf->addr_count = hts_dns_resolve_all(opt, back[i].url_adr, scratch,
+                                               HTS_MAXADDRNUM, NULL);
+          cf->connect_start = time_local();
+        }
+
         // vérifier l'existance de timeout-check
         if (!gestion_timeout)
           if (back[i].timeout > 0)
@@ -2526,14 +2671,45 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
         // connecté?
         dispo = FD_ISSET(back[i].r.soc, &fds_c);
-        if (dispo) {            // ok connected!!
+        if (dispo) { // socket ready: connect() finished (ok or failed)
+          // a refused/failed connect is reported writable too; probe SO_ERROR
+          // and, on failure, fall back to the next address (or fail the slot)
+          if (connect_socket_error(back[i].r.soc) != 0) {
+            if (!back_connect_next(opt, sback, i)) {
+              deletehttp(&back[i].r);
+              back[i].r.soc = INVALID_SOCKET;
+              back[i].r.statuscode = STATUSCODE_CONNERROR;
+              strcpybuff(back[i].r.msg, "Connect Error");
+              back[i].status = STATUS_READY;
+              back_set_finished(sback, i);
+            }
+            continue; // reconnected (stay connecting) or failed
+          }
           busy_state = 1;
 
 #if HTS_USEOPENSSL
           /* SSL mode */
           if (back[i].r.ssl) {
+            int tunnel_ok = 1;
+
+            // https via proxy: CONNECT-tunnel before TLS (#85)
+            if (back[i].r.req.proxy.active && back[i].r.ssl_con == NULL) {
+              const int timeout = back[i].timeout > 0 ? back[i].timeout : 30;
+
+              tunnel_ok =
+                  http_proxy_tunnel(opt, &back[i].r, back[i].url_adr, timeout);
+              if (!tunnel_ok) {
+                if (!strnotempty(back[i].r.msg))
+                  strcpybuff(back[i].r.msg, "proxy CONNECT failed");
+                deletehttp(&back[i].r);
+                back[i].r.soc = INVALID_SOCKET;
+                back[i].r.statuscode = STATUSCODE_NON_FATAL;
+                back[i].status = STATUS_READY;
+                back_set_finished(sback, i);
+              }
+            }
             // handshake not yet launched
-            if (!back[i].r.ssl_con) {
+            if (tunnel_ok && !back[i].r.ssl_con) {
               SSL_CTX_set_options(openssl_ctx, SSL_OP_ALL);
               // new session
               back[i].r.ssl_con = SSL_new(openssl_ctx);
@@ -2551,7 +2727,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 back[i].r.statuscode = STATUSCODE_SSL_HANDSHAKE;
             }
             /* Error */
-            if (back[i].r.statuscode == STATUSCODE_SSL_HANDSHAKE) {
+            if (tunnel_ok && back[i].r.statuscode == STATUSCODE_SSL_HANDSHAKE) {
               strcpybuff(back[i].r.msg, "bad SSL/TLS handshake");
               deletehttp(&back[i].r);
               back[i].r.soc = INVALID_SOCKET;
@@ -2581,7 +2757,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back[i].rateout_time = back[i].ka_time_start;
             }
             // envoyer header
-            //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
             HTS_STAT.stat_nrequests++;
             if (!back[i].head_request)
               http_sendhead(opt, opt->cookie, 0, back[i].send_too,
@@ -2648,7 +2823,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 #if HTS_XGETHOST
       else if (back[i].status == STATUS_WAIT_DNS) {     // attendre gethostbyname
 #if DEBUGDNS
-        //printf("status 101 for %s\n",back[i].url_adr);
 #endif
 
         if (!gestion_timeout)
@@ -2768,10 +2942,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
           // range size hack old location
 
 #if HTS_DIRECTDISK
-          // Court-circuit:
-          // Peut-on stocker le fichier directement sur disque?
-          // Ahh que ca serait vachement mieux et que ahh que la mémoire vous dit merci!
-          if (back[i].status) {
+          // Shortcut: store the file directly on disk when possible,
+          // sparing memory
+          if (back[i].status &&
+              !back[i].locked) { // name still pending when locked
             if (back[i].r.is_write == 0) {      // mode mémoire
               if (back[i].r.adr == NULL) {      // rien n'a été écrit
                 if (!back[i].testmode) {        // pas mode test
@@ -2921,8 +3095,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
           // Si réception chunk, tester si on est pas à la fin!
           if (back[i].status == 1) {
             if (back[i].is_chunk) {     // attendre prochain chunk
-              if (back[i].r.size == back[i].r.totalsize) {      // fin chunk!
-                //printf("chunk end at %d\n",back[i].r.size);
+              if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
                 back[i].status = STATUS_CHUNK_CR;       /* fetch ending CRLF */
                 if (back[i].chunk_adr != NULL) {
                   freet(back[i].chunk_adr);
@@ -2985,11 +3158,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back_finalize(opt, cache, sback, i);
             }
 
-            if (back[i].r.totalsize >= 0) {     // tester totalsize
-              //if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) {    // tester totalsize
+            if (back[i].r.totalsize >= 0) { // tester totalsize
               if (back[i].r.totalsize != back[i].r.size) {      // pas la même!
                 if (!opt->tolerant) {
-                  //#if HTS_CL_IS_FATAL
                   deleteaddr(&back[i].r);
                   if (back[i].r.size < back[i].r.totalsize)
                     back[i].r.statuscode = STATUSCODE_CONNERROR;        // recatch
@@ -2998,14 +3169,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           " expected)", (LLint) back[i].r.size,
                           (LLint) back[i].r.totalsize);
                 } else {
-                  //#else
                   // Un warning suffira..
                   hts_log_print(opt, LOG_WARNING,
                                 "Incorrect length (" LLintP "!=" LLintP
-                                " expected) for %s%s", (LLint) back[i].r.size,
+                                " expected) for %s%s",
+                                (LLint) back[i].r.size,
                                 (LLint) back[i].r.totalsize, back[i].url_adr,
                                 back[i].url_fil);
-                  //#endif
                 }
               }
             }
@@ -3132,10 +3302,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                          back[i].r.size);
 #endif
                   /* End */
-                  //if (back[i].status==STATUS_CHUNK_CR) {
-                  back[i].status = STATUS_READY;        // fin  
+                  back[i].status = STATUS_READY; // fin
                   back_set_finished(sback, i);
-                  //}
 
                   // finalize transfer if not temporary
                   if (!IS_DELAYED_EXT(back[i].url_sav)) {
@@ -3602,7 +3770,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                     back[i].url_adr, back[i].url_fil);
 
                       // finalize
-                      //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);        // not modified
                       if (back[i].r.statuscode > 0) {
                         hts_log_print(opt, LOG_TRACE, "finalizing after cache load");
                         back_finalize(opt, cache, sback, i);
@@ -3612,38 +3779,32 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                              back[i].url_adr, back[i].url_fil);
 #endif
 
-                      //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                     } else {    // erreur
                       back[i].status = STATUS_READY;    // terminé
                       back_set_finished(sback, i);
-                      //printf("erreur cache\n");
-
                     }
 
-/********** NO - must complete the body! ********** */
-#if 0
-                  } else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
-                             || (back[i].r.statuscode == 412)
-                             || (back[i].r.statuscode == 416)
-                    ) {         // Ne pas prendre le html, erreurs connues et gérées
-#if HTS_DEBUG_CLOSESOCK
-                    DEBUG_W
-                      ("back_wait(301,302,303,307,412,416..): deletehttp\n");
-#endif
-                    // Couper connexion
-                    /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
-                    back_maydeletehttp(opt, cache, sback, i);
-
-                    back[i].status = STATUS_READY;      // terminé
+                  }
+                  // MIME type excluded by a -mime: filter: abort, don't fetch
+                  // the body (#58)
+                  else if (HTTP_IS_OK(back[i].r.statuscode) &&
+                           !back[i].testmode &&
+                           strnotempty(back[i].r.contenttype) &&
+                           hts_acceptmime(opt, 0, back[i].url_adr,
+                                          back[i].url_fil,
+                                          back[i].r.contenttype) == 1) {
+                    deletehttp(&back[i].r);
+                    back[i].r.soc = INVALID_SOCKET;
+                    back[i].status = STATUS_READY;
                     back_set_finished(sback, i);
-                    // finalize
-                    if (back[i].r.statuscode > 0) {
-                      hts_log_print(opt, LOG_TRACE, "finalizing redirect & 4xx");
-                      back_finalize(opt, cache, sback, i);
-                    }
-#endif
-/********** **************************** ********** */
-                  } else {      // il faut aller le chercher
+                    back[i].r.statuscode = STATUSCODE_EXCLUDED;
+                    strcpybuff(back[i].r.msg, "Excluded by MIME type filter");
+                    hts_log_print(
+                        opt, LOG_NOTICE,
+                        "File excluded by MIME type filter (%s): %s%s",
+                        back[i].r.contenttype, back[i].url_adr,
+                        back[i].url_fil);
+                  } else { // il faut aller le chercher
 
                     // effacer buffer (requète)
                     if (!noFreebuff) {
@@ -3654,35 +3815,70 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     // xxc SI CHUNK VERIFIER QUE CA MARCHE??
                     if (back[i].r.statuscode == 206) {  // on nous envoie un morceau (la fin) coz une partie sur disque!
                       off_t sz = fsize_utf8(back[i].url_sav);
+                      /* RFC 7233: resume at the server's Content-Range start,
+                         not the offset we requested; a server may resume
+                         earlier and appending the overlap duplicates bytes
+                         (#198). */
+                      const LLint resume = back[i].r.crange_start;
+                      const hts_boolean range_ok =
+                          back[i].r.crange > 0 && resume >= 0 &&
+                          resume <= (LLint) sz &&
+                          back[i].r.crange_end + 1 == back[i].r.crange &&
+                          (back[i].r.totalsize < 0 ||
+                           back[i].r.totalsize ==
+                               back[i].r.crange_end - resume + 1);
 
 #if HDEBUG
                       printf("partial content: " LLintP " on disk..\n",
                              (LLint) sz);
 #endif
-                      if (sz >= 0) {
+                      if (sz >= 0 && range_ok) {
                         if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_sav)) {  // pas HTML
                           if (opt->getmode & HTS_GETMODE_NONHTML) {
                             filenote(&opt->state.strc, back[i].url_sav, NULL);  // noter fichier comme connu
                             file_notify(opt, back[i].url_adr, back[i].url_fil,
                                         back[i].url_sav, 0, 1,
                                         back[i].r.notmodified);
-                            back[i].r.out = FOPEN(fconv(catbuff, sizeof(catbuff), back[i].url_sav), "ab");       // append
+                            back[i].r.out =
+                                FOPEN(fconv(catbuff, sizeof(catbuff),
+                                            back[i].url_sav),
+                                      "r+b"); // resume in place
                             if (back[i].r.out && opt->cache != 0) {
-                              back[i].r.is_write = 1;   // écrire
-                              back[i].r.size = sz;      // déja écrit
-                              back[i].r.statuscode = HTTP_OK;   // Forcer 'OK'
+                              back[i].r.is_write = 1;
+                              back[i].r.size = resume; // bytes already on disk
+                              back[i].r.statuscode = HTTP_OK; // force 'OK'
                               if (back[i].r.totalsize >= 0)
-                                back[i].r.totalsize += sz;      // plus en fait
-                              fseek(back[i].r.out, 0, SEEK_END);        // à la fin
-                              /* create a temporary reference file in case of broken mirror */
-                              if (back_serialize_ref(opt, &back[i]) != 0) {
-                                hts_log_print(opt, LOG_WARNING,
-                                              "Could not create temporary reference file for %s%s",
-                                              back[i].url_adr, back[i].url_fil);
-                              }
+                                back[i].r.totalsize += resume; // -> full size
+                              // drop bytes past the resume point; a silent
+                              // failure could leave a stale tail, so on error
+                              // drop the partial and refetch the whole file
+                              if (HTS_FTRUNCATE(back[i].r.out,
+                                                (off_t) resume) != 0) {
+                                fclose(back[i].r.out);
+                                back[i].r.out = NULL;
+                                url_savename_refname_remove(
+                                    opt, back[i].url_adr, back[i].url_fil);
+                                UNLINK(back[i].url_sav);
+                                back[i].status = STATUS_READY;
+                                back_set_finished(sback, i);
+                                strcpybuff(back[i].r.msg,
+                                           "Can not truncate partial file, "
+                                           "restarting");
+                              } else {
+                                fseeko(back[i].r.out, (off_t) resume, SEEK_SET);
+                                /* create a temporary reference file in case of
+                                 * broken mirror */
+                                if (back_serialize_ref(opt, &back[i]) != 0) {
+                                  hts_log_print(opt, LOG_WARNING,
+                                                "Could not create temporary "
+                                                "reference file for %s%s",
+                                                back[i].url_adr,
+                                                back[i].url_fil);
+                                }
 #if HDEBUG
-                              printf("continue interrupted file\n");
+                                printf("continue interrupted file\n");
 #endif
+                              }
                             } else {    // On est dans la m**
                               back[i].status = STATUS_READY;    // terminé (voir plus loin)
                               back_set_finished(sback, i);
@@ -3694,17 +3890,18 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           FILE *fp =
                             FOPEN(fconv(catbuff, sizeof(catbuff), back[i].url_sav), "rb");
                           if (fp) {
-                            LLint alloc_mem = sz + 1;
+                            LLint alloc_mem = resume + 1;
 
                             if (back[i].r.totalsize >= 0)
                               alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
                             if (deleteaddr(&back[i].r)
                                 && (back[i].r.adr =
                                     (char *) malloct((size_t) alloc_mem))) {
-                              back[i].r.size = sz;
+                              back[i].r.size = resume;
                               if (back[i].r.totalsize >= 0)
-                                back[i].r.totalsize += sz;      // plus en fait
-                              if ((fread(back[i].r.adr, 1, sz, fp)) != sz) {
+                                back[i].r.totalsize += resume; // -> full size
+                              if ((fread(back[i].r.adr, 1, (size_t) resume,
+                                         fp)) != (size_t) resume) {
                                 back[i].status = STATUS_READY;  // terminé (voir plus loin)
                                 back_set_finished(sback, i);
                                 strcpybuff(back[i].r.msg,
@@ -3722,14 +3919,30 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                          "No memory for partial file");
                             }
                             fclose(fp);
-                          } else {      // Argh.. 
+                          } else {                              // open failed
                             back[i].status = STATUS_READY;      // terminé (voir plus loin)
                             back_set_finished(sback, i);
                             strcpybuff(back[i].r.msg,
                                        "Can not open partial file");
                           }
                         }
-                      } else {  // Non trouvé??
+                      } else if (sz >=
+                                 0) { // unusable range -> restart whole file
+                        hts_log_print(opt, LOG_WARNING,
+                                      "Unusable partial-content range for %s%s "
+                                      "(have " LLintP " bytes, got " LLintP
+                                      "-" LLintP "/" LLintP "), restarting",
+                                      back[i].url_adr, back[i].url_fil,
+                                      (LLint) sz, back[i].r.crange_start,
+                                      back[i].r.crange_end, back[i].r.crange);
+                        url_savename_refname_remove(opt, back[i].url_adr,
+                                                    back[i].url_fil);
+                        UNLINK(back[i].url_sav);
+                        back[i].status = STATUS_READY;
+                        back_set_finished(sback, i);
+                        strcpybuff(back[i].r.msg,
+                                   "Unusable partial content, restarting");
+                      } else {                          // partial not found
                         back[i].status = STATUS_READY;  // terminé (voir plus loin)
                         back_set_finished(sback, i);
                         strcpybuff(back[i].r.msg, "Can not find partial file");
@@ -3744,7 +3957,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           deletehttp(&back[i].r);
                         }
                         back[i].r.soc = INVALID_SOCKET;
-                        //back[i].r.statuscode=206;  ????????
                         back[i].r.statuscode = STATUSCODE_NON_FATAL;
                         if (strnotempty(back[i].r.msg))
                           strcpybuff(back[i].r.msg,
@@ -3765,10 +3977,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                               && (back[i].r.adr = (char *) malloct(2))) {
                             back[i].r.adr[0] = 0;
                           }
-                          hts_log_print(opt, LOG_TRACE, "finalizing empty");
-                          back_finalize(opt, cache, sback, i);
-                        } else if (!back[i].r.is_chunk) {       // pas de chunk
-                          //if (back[i].r.http11!=2) {    // pas de chunk
+                          /* locked = name pending; the waiter finalizes after
+                             patching url_sav (else: cached as .delayed, #5) */
+                          if (!back[i].locked) {
+                            hts_log_print(opt, LOG_TRACE, "finalizing empty");
+                            back_finalize(opt, cache, sback, i);
+                          }
+                        } else if (!back[i].r.is_chunk) { // pas de chunk
                           back[i].is_chunk = 0;
                           back[i].status = 1;   // start body
                         } else {
@@ -3810,7 +4025,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
                       }
                     }
-
                   }
 
                   /*} */
@@ -3830,15 +4044,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].status < 0) {
           if (!back[i].testmode) {      // pas en test
             UNLINK(back[i].url_sav);    // éliminer fichier (endommagé)
-            //printf("&& %s\n",back[i].url_sav);
           }
         }
 #endif
 
         /* funny log for commandline users */
-        //if (!opt->quiet) {  
-        // petite animation
-        if (opt->verbosedisplay == 1) {
+        if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
           if (back[i].status == STATUS_READY) {
             if (back[i].r.statuscode == HTTP_OK)
               printf("* %s%s (" LLintP " bytes) - OK" VT_CLREOL "\r",
@@ -3850,23 +4061,43 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
             fflush(stdout);
           }
         }
-        //}
 
       }                         // status>0
-    }                           // for
+    } // for
 
     // vérifier timeouts
     if (gestion_timeout) {
       TStamp act;
 
       act = time_local();       // temps en secondes
-      for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-        // for(i=0;i<back_max;i++) {
+      for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
         unsigned int i = (i_mod + mod_random) % (back_max);
 
         if (back[i].status > 0) {       // réception/connexion/..
           if (back[i].timeout > 0) {
-            //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
+            // a stuck connect with a fallback address: retry the next one well
+            // before the full timeout (dead IPv6 on a dual-stack host, ...)
+            if (back[i].status == STATUS_CONNECTING) {
+              const hts_connect_fallback *const cf =
+                  &sback->connect_fallback[i];
+
+              if (back_connect_fallback_due(cf->addr_index, cf->addr_count,
+                                            (int) (act - cf->connect_start),
+                                            back[i].timeout)) {
+                if (back_connect_next(opt, sback, i)) {
+                  continue; // reconnected to the next candidate
+                }
+                // fallback was due but no socket could be opened
+                // (back_connect_next closed the dead one): stop now rather than
+                // spin on an invalid fd
+                back[i].r.soc = INVALID_SOCKET;
+                back[i].r.statuscode = STATUSCODE_CONNERROR;
+                strcpybuff(back[i].r.msg, "Connect Error");
+                back[i].status = STATUS_READY;
+                back_set_finished(sback, i);
+                continue;
+              }
+            }
             if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) {
               hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr,
                 back[i].url_fil);
@@ -3942,6 +4173,11 @@ int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize)
   return 1;
 }
 
+/* Grace left to the smooth stop before in-flight transfers are aborted. */
+static int back_maxtime_grace(const int maxtime) {
+  return maximum(5, minimum(30, maxtime / 10));
+}
+
 int back_checkmirror(httrackp * opt) {
   // Check max size
   if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
@@ -3958,13 +4194,19 @@ int back_checkmirror(httrackp * opt) {
      */
   }
   // Check max time
-  if ((opt->maxtime > 0)
-      && ((time_local() - HTS_STAT.stat_timestart) >= opt->maxtime)) {
-    if (!opt->state.stop) {     /* not yet stopped */
-      hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-                    opt->maxtime);
-      /* cancel mirror smoothly */
-      hts_request_stop(opt, 0);
+  if (opt->maxtime > 0) {
+    const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
+
+    if (elapsed >= opt->maxtime) {
+      if (!opt->state.stop) { /* not yet stopped */
+        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                      opt->maxtime);
+        /* cancel mirror smoothly */
+        hts_request_stop(opt, 0);
+      }
+      /* smooth stop starved past the grace period: stop waiting (#481) */
+      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+        return 0;
     }
   }
   return 1;                     /* Ok, go on */

src/htsback.h

@@ -113,6 +113,12 @@ void back_set_locked(struct_back * sback, const int p);
 void back_set_unlocked(struct_back * sback, const int p);
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p);
+/* Discard back's on-disk .delayed placeholder and its refname. */
+void back_delayed_discard(httrackp *opt, lien_back *back);
+/* Move back's .delayed placeholder (and open stream) to newname;
+   HTS_FALSE = file lost, slot flagged in error. */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname);
 void back_index_unlock(struct_back * sback, const int p);
 int back_clear_entry(lien_back * back);
 int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
@@ -136,6 +142,8 @@ void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
+/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
+   the -E deadline overran its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htsbasenet.h

@@ -41,7 +41,7 @@ Please visit our Website: http://www.httrack.com
 
 #ifdef _WIN32
 
-#if HTS_INET6==0
+#if HTS_INET6 == 0
 #include <winsock2.h>
 #else
 
@@ -49,13 +49,12 @@ Please visit our Website: http://www.httrack.com
 #define WIN32_LEAN_AND_MEAN
 // KB955045 (http://support.microsoft.com/kb/955045)
 // To execute an application using this function on earlier versions of Windows
-// (Windows 2000, Windows NT, and Windows Me/98/95), then it is mandatary to #include Ws2tcpip.h
-// and also Wspiapi.h. When the Wspiapi.h header file is included, the 'getaddrinfo' function is
-// #defined to the 'WspiapiGetAddrInfo' inline function in Wspiapi.h. 
+// (Windows 2000, Windows NT, and Windows Me/98/95), then it is mandatary to
+// #include Ws2tcpip.h and also Wspiapi.h. When the Wspiapi.h header file is
+// included, the 'getaddrinfo' function is #defined to the 'WspiapiGetAddrInfo'
+// inline function in Wspiapi.h.
 #include <ws2tcpip.h>
 #include <Wspiapi.h>
-//#include <winsock2.h>
-//#include <tpipv6.h>
 
 #endif
 
@@ -128,6 +127,8 @@ typedef enum HTTPStatusCode {
   HTTP_UNSUPPORTED_MEDIA_TYPE = 415,
   HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416,
   HTTP_EXPECTATION_FAILED = 417,
+  HTTP_TOO_MANY_REQUESTS = 429,
+  HTTP_UNAVAILABLE_FOR_LEGAL_REASONS = 451,
   HTTP_INTERNAL_SERVER_ERROR = 500,
   HTTP_NOT_IMPLEMENTED = 501,
   HTTP_BAD_GATEWAY = 502,
@@ -145,7 +146,8 @@ typedef enum BackStatusCode {
   STATUSCODE_NON_FATAL = -5,
   STATUSCODE_SSL_HANDSHAKE = -6,
   STATUSCODE_TOO_BIG = -7,
-  STATUSCODE_TEST_OK = -10
+  STATUSCODE_TEST_OK = -10,
+  STATUSCODE_EXCLUDED = -11 /* aborted: MIME excluded by a -mime: filter */
 } BackStatusCode;
 
 /** HTTrack status ('status' member of of 'lien_back') **/

src/htsbasiccharsets.sh

@@ -3,57 +3,59 @@
 
 # Change this to download files
 if false; then
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
-rm -f CP932.TXT CP936.TXT CP949.TXT CP950.TXT
+    echo "mget https://www.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
+    rm -f CP932.TXT CP936.TXT CP949.TXT CP950.TXT
 fi
 
 # Produce code
-printf "/** GENERATED FILE ($0), DO NOT EDIT **/\n\n"
-for i in *.TXT ; do
-  echo "processing $i" >&2
-  grep -vE "^(#|$)" $i | grep -E "^0x" | sed -e 's/[[:space:]]/ /g' | cut -f1,2 -d' ' | \
-  (
-    unset arr
-    while read LINE ; do
-      from=$[$(echo $LINE | cut -f1 -d' ')]
-      if ! test -n "$from"; then
-        echo "error with $i" >&2
-        exit 1
-      elif test $from -ge 256; then
-        echo "out-of-range ($LINE) with $i" >&2
-        exit 1
-      fi
-      to=$(echo $LINE | cut -f2 -d' ') 
-      arr[$from]=$to
-    done
-    name=$(echo $i | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
-    printf "/* Table for $i */\nstatic const hts_UCS4 table_${name}[256] = {\n  "
-    i=0
-    while test "$i" -lt 256; do
-      if test "$i" -gt 0; then
-        printf ", "
-        if test $[${i}%8] -eq 0; then
-          printf "\n  "
-        fi
-      fi
-      value=${arr[$i]:-0}
-      printf "0x%04x" $value
-      i=$[${i}+1]
-    done
-    printf " };\n\n"
-  )
-  echo "processed $i" >&2
+printf '/** GENERATED FILE (%s), DO NOT EDIT **/\n\n' "$0"
+for i in *.TXT; do
+    echo "processing $i" >&2
+    grep -vE "^(#|$)" "$i" | grep -E "^0x" | sed -e 's/[[:space:]]/ /g' | cut -f1,2 -d' ' |
+        (
+            unset arr
+            while read -r LINE; do
+                from=$(($(echo "$LINE" | cut -f1 -d' ')))
+                if ! test -n "$from"; then
+                    echo "error with $i" >&2
+                    exit 1
+                elif test $from -ge 256; then
+                    echo "out-of-range ($LINE) with $i" >&2
+                    exit 1
+                fi
+                to=$(echo "$LINE" | cut -f2 -d' ')
+                arr[from]=$to
+            done
+            # shellcheck disable=SC2018,SC2019 # charset filenames are ASCII; keep C-locale A-Z/a-z
+            name=$(echo "$i" | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
+            printf '/* Table for %s */\nstatic const hts_UCS4 table_%s[256] = {\n  ' "$i" "$name"
+            idx=0
+            while test "$idx" -lt 256; do
+                if test "$idx" -gt 0; then
+                    printf ", "
+                    if test $((idx % 8)) -eq 0; then
+                        printf "\n  "
+                    fi
+                fi
+                value=${arr[$idx]:-0}
+                printf "0x%04x" "$value"
+                idx=$((idx + 1))
+            done
+            printf " };\n\n"
+        )
+    echo "processed $i" >&2
 done
 
 # Indexes
 printf "static const struct {\n  const char *name;\n  const hts_UCS4 *table;\n} table_mappings[] = {\n"
-for i in *.TXT ; do
-  name=$(echo $i | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
-  printf "  { \"$(echo $name | tr -d '_')\", table_${name} },\n"
+for i in *.TXT; do
+    # shellcheck disable=SC2018,SC2019 # charset filenames are ASCII; keep C-locale A-Z/a-z
+    name=$(echo "$i" | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
+    printf '  { "%s", table_%s },\n' "$(echo "$name" | tr -d '_')" "$name"
 done
 printf "  { NULL, NULL }\n};\n"

src/htsbauth.c

@@ -107,7 +107,6 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
 #if DEBUG_COOK
   printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
          cook_name, cook_value, domain, path);
-  //printf(">>>cook: %s<<<\n",cookie->data);
 #endif
   return 0;
 }
@@ -208,8 +207,6 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   char catbuff[CATBUFF_SIZE];
   char buffer[8192];
 
-  //  cookie->data[0]='\0';
-
   // Fusionner d'abord les éventuels cookies IE
 #ifdef _WIN32
   {

src/htsbauth.h

@@ -68,14 +68,15 @@ struct t_cookie {
 #ifdef HTS_INTERNAL_BYTECODE
 
 /* cookies */
-int cookie_add(t_cookie * cookie, const  char *cook_name, const  char *cook_value,
-               const  char *domain, const  char *path);
+int cookie_add(t_cookie *cookie, const char *cook_name, const char *cook_value,
+               const char *domain, const char *path);
 
-int cookie_del(t_cookie * cookie, const char *cook_name, const char *domain, const char *path);
+int cookie_del(t_cookie *cookie, const char *cook_name, const char *domain,
+               const char *path);
 
-int cookie_load(t_cookie * cookie, const char *path, const char *name);
+int cookie_load(t_cookie *cookie, const char *path, const char *name);
 
-int cookie_save(t_cookie * cookie, const char *name);
+int cookie_save(t_cookie *cookie, const char *name);
 
 void cookie_insert(char *s, size_t s_size, const char *ins);
 
@@ -83,7 +84,8 @@ void cookie_delete(char *s, size_t s_size, size_t pos);
 
 const char *cookie_get(char *buffer, const char *cookie_base, int param);
 
-char *cookie_find(char *s, const char *cook_name, const char *domain, const char *path);
+char *cookie_find(char *s, const char *cook_name, const char *domain,
+                  const char *path);
 
 char *cookie_nextfield(char *a);
 
@@ -92,12 +94,13 @@ char *cookie_nextfield(char *a);
 /** Register credentials (auth = base-64 user:pass) for the prefix derived from
     adr (host) and fil (path). No-op returning 0 if cookie is NULL, allocation
     fails, or a matching prefix is already stored; returns 1 on insertion. */
-int bauth_add(t_cookie * cookie, const char *adr, const char *fil, const char *auth);
+int bauth_add(t_cookie *cookie, const char *adr, const char *fil,
+              const char *auth);
 
 /** Return the stored base-64 credentials whose prefix matches adr+fil, or NULL
     if none (or cookie is NULL). Returned pointer aliases the jar's bauth_chain;
     caller must not free it. */
-char *bauth_check(t_cookie * cookie, const char *adr, const char *fil);
+char *bauth_check(t_cookie *cookie, const char *adr, const char *fil);
 
 /** Build the auth lookup key (host + path, query string stripped, truncated at
     the last '/') from adr and fil into prefix; returns prefix. Caller must

src/htscache.c

@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
 #include "htscore.h"
 #include "htsbasenet.h"
 #include "htsmd5.h"
+#include <limits.h>
 #include <time.h>
 
 #include "htszlib.h"
@@ -220,6 +221,40 @@ struct cache_back_zip_entry {
 	} \
 } while(0)
 
+/* Consecutive entry write failures before the cache stream is declared dead. */
+#define CACHE_MAX_WRITE_FAILURES 8
+
+/* Cache write failed: a fatal errno or a failure streak aborts the mirror
+   (exit_xh); an isolated failure only drops the current entry. */
+static void cache_zip_write_failed(httrackp *opt, cache_back *cache,
+                                   const char *what, int zErr,
+                                   hts_boolean entry_open, const char *url_adr,
+                                   const char *url_fil) {
+  const int fatal_errno = zErr == ZIP_ERRNO && check_fatal_io_errno();
+
+  cache->zipWriteFailures++;
+  if (fatal_errno || cache->zipWriteFailures >= CACHE_MAX_WRITE_FAILURES) {
+    if (!cache->zipWriteFailed) {
+      cache->zipWriteFailed = HTS_TRUE;
+      if (fatal_errno) {
+        hts_log_print(opt, LOG_ERROR,
+                      "Mirror aborted: disk full or filesystem problems");
+      } else {
+        hts_log_print(opt, LOG_ERROR,
+                      "Mirror aborted: cache write failed (%s): %s", what,
+                      hts_get_zerror(zErr));
+      }
+    }
+    opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
+  } else {
+    if (entry_open)
+      zipCloseFileInZip((zipFile) cache->zipOutput); /* abandon, best-effort */
+    hts_log_print(opt, LOG_WARNING,
+                  "cache write failed (%s: %s), entry not cached: %s%s", what,
+                  hts_get_zerror(zErr), url_adr, url_fil);
+  }
+}
+
 /* Ajout d'un fichier en cache */
 void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
                const char *url_adr, const char *url_fil, const char *url_save,
@@ -230,12 +265,14 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
   char BIGSTK headers[8192];
   int headersSize = 0;
 
-  //int entryBodySize = 0;
-  //int entryFilenameSize = 0;
   zip_fileinfo fi;
   const char *url_save_suffix = url_save;
   int zErr;
 
+  /* already failed and aborting; don't touch the broken stream again */
+  if (cache->zipWriteFailed)
+    return;
+
   // robots.txt hack
   if (url_save == NULL) {
     dataincache = 0;            // testing links
@@ -263,10 +300,19 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
   if (r->size < 0)              // error
     return;
 
-  // data in cache
-  if (dataincache) {
-    assertf(((int) r->size) == r->size);
-    //entryBodySize = (int) r->size;
+  // data in cache: the body must fit the 32-bit zip write API
+  if (dataincache && (LLint) (int) r->size != r->size) {
+    if (r->is_write && url_save != NULL && strnotempty(url_save)) {
+      hts_log_print(opt, LOG_WARNING,
+                    "file too large for the cache, storing headers only: %s%s",
+                    url_adr, url_fil);
+      dataincache = 0;
+    } else {
+      hts_log_print(opt, LOG_WARNING,
+                    "entry too large for the cache, not cached: %s%s", url_adr,
+                    url_fil);
+      return;
+    }
   }
 
   /* Fields */
@@ -311,8 +357,6 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
   ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil);     // Original URI filename
   ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save_suffix);    // Original save filename
 
-  //entryFilenameSize = (int) ( strlen(url_adr) + strlen(url_fil));
-
   /* Filename */
   if (!link_has_authority(url_adr)) {
     strcpybuff(filename, "http://");
@@ -346,9 +390,9 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
                                    */
                                   headers, (uInt) strlen(headers), NULL, 0, NULL,       /* comment */
                                   Z_DEFLATED, Z_DEFAULT_COMPRESSION)) != Z_OK) {
-    int zip_zipOpenNewFileInZip_failed = 0;
-
-    assertf(zip_zipOpenNewFileInZip_failed);
+    cache_zip_write_failed(opt, cache, "opening a cache entry", zErr, HTS_FALSE,
+                           url_adr, url_fil);
+    return;
   }
 
   /* Write data in cache */
@@ -358,9 +402,9 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
         if ((zErr =
              zipWriteInFileInZip((zipFile) cache->zipOutput, r->adr,
                                  (int) r->size)) != Z_OK) {
-          int zip_zipWriteInFileInZip_failed = 0;
-
-          assertf(zip_zipWriteInFileInZip_failed);
+          cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
+                                 HTS_TRUE, url_adr, url_fil);
+          return;
         }
       }
     } else {
@@ -381,9 +425,10 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
               if ((zErr =
                    zipWriteInFileInZip((zipFile) cache->zipOutput, buff,
                                        (int) nl)) != Z_OK) {
-                int zip_zipWriteInFileInZip_failed = 0;
-
-                assertf(zip_zipWriteInFileInZip_failed);
+                cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
+                                       HTS_TRUE, url_adr, url_fil);
+                fclose(fp);
+                return;
               }
             }
           } while(nl > 0);
@@ -397,17 +442,19 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
 
   /* Close */
   if ((zErr = zipCloseFileInZip((zipFile) cache->zipOutput)) != Z_OK) {
-    int zip_zipCloseFileInZip_failed = 0;
-
-    assertf(zip_zipCloseFileInZip_failed);
+    cache_zip_write_failed(opt, cache, "closing a cache entry", zErr, HTS_FALSE,
+                           url_adr, url_fil);
+    return;
   }
 
   /* Flush */
   if ((zErr = zipFlush((zipFile) cache->zipOutput)) != 0) {
-    int zip_zipFlush_failed = 0;
-
-    assertf(zip_zipFlush_failed);
+    cache_zip_write_failed(opt, cache, "flushing the cache", zErr, HTS_FALSE,
+                           url_adr, url_fil);
+    return;
   }
+
+  cache->zipWriteFailures = 0; /* entry stored: reset the failure streak */
 }
 
 #else
@@ -576,15 +623,18 @@ htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
   return cache_readex(opt, cache, adr, fil, save, location, NULL, 1);
 }
 
-htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
-                                   const char *adr, const char *fil) {
-  htsblk r = cache_read(opt, cache, adr, fil, NULL, NULL);
+htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
+                                   const char *adr, const char *fil,
+                                   char *return_save) {
+  htsblk r = cache_readex(opt, cache, adr, fil, NULL, NULL, return_save, 0);
 
   if (r.statuscode == -1) {
     lien_back *itemback = NULL;
 
     if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
       r = itemback->r;
+      if (return_save != NULL)
+        strlcpybuff(return_save, itemback->url_sav, HTS_URLMAXSIZE * 2);
       /* cleanup */
       back_clear_entry(itemback);       /* delete entry content */
       freet(itemback);          /* delete item */
@@ -671,7 +721,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
         char BIGSTK headerBuff[8192 + 2];
         int readSizeHeader;
 
-        //int totalHeader = 0;
         int dataincache = 0;
 
         /* For BIG comments */
@@ -722,13 +771,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                                    HTS_URLMAXSIZE * 2);
               ZIP_READFIELD_STRING(line, value, "Content-Disposition", r.cdispo,
                                    sizeof(r.cdispo));
-              //ZIP_READFIELD_STRING(line, value, "X-Addr", ..);            // Original address
-              //ZIP_READFIELD_STRING(line, value, "X-Fil", ..);            // Original URI filename
               ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
                                    sizeof(previous_save_));
             }
-          } while(offset < readSizeHeader && !lineEof);
-          //totalHeader = offset;
+          } while (offset < readSizeHeader && !lineEof);
 
           /* Previous entry */
           if (previous_save_[0] != '\0') {
@@ -745,6 +791,15 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
             strlcpybuff(return_save, previous_save, HTS_URLMAXSIZE * 2);
           }
 
+          /* A negative X-Size is corrupt; so is one >= INT_MAX when the data
+             is in the zip (the write path asserts int-sized). Headers-only
+             entries legitimately exceed INT_MAX (>2GB body on disk): keep
+             them, or every update would re-fetch the file. */
+          if (r.size < 0 || (dataincache && r.size >= INT_MAX)) {
+            r.statuscode = STATUSCODE_INVALID;
+            strcpybuff(r.msg, "Cache Read Error : Bad Size");
+          }
+
           /* Complete fields */
           r.totalsize = r.size;
           r.adr = NULL;
@@ -771,7 +826,8 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
             }                   // otherwise, the ZIP file is supposed to be consistent with data.
           }
           /* Read data ? */
-          else {                /* ne pas lire uniquement header */
+          else if (r.statuscode !=
+                   STATUSCODE_INVALID) { /* ne pas lire uniquement header */
             int ok = 0;
 
 #if HTS_DIRECTDISK
@@ -900,7 +956,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg,
                              "Cache Write Error : Unable to Create File");
-                  //printf("%s\n",save);
                 }
               }
 
@@ -935,7 +990,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg,
                                "Previous cache file not found (empty filename)");
                   }
-                } else {        /* Read in memory from disk */
+                } else if (r.size >= INT_MAX) { /* too big to read in memory */
+                  r.statuscode = STATUSCODE_INVALID;
+                  strcpybuff(r.msg, "Cache Read Error : Bad Size");
+                } else { /* Read in memory from disk */
                   FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), previous_save), "rb");
 
                   if (fp != NULL) {
@@ -975,7 +1033,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg, "Cache Read Error : Read Data");
                   } else
                     *(r.adr + r.size) = '\0';
-                  //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                 } else {        // erreur
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg, "Cache Memory Error");
@@ -1154,17 +1211,14 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
         // sécurité
         r.adr = NULL;
         r.out = NULL;
-        ////r.location=NULL;  non, fixée lors des 301 ou 302
         r.fp = NULL;
 
         if ((r.statuscode >= 0) && (r.statuscode <= 999)
             && (r.notmodified >= 0) && (r.notmodified <= 9)) {  // petite vérif intégrité
-          if ((save) && (!header_only)) {       /* ne pas lire uniquement header */
-            //int to_file=0;
+          if ((save) && (!header_only)) { /* ne pas lire uniquement header */
 
             r.adr = NULL;
             r.soc = INVALID_SOCKET;
-            // // r.location=NULL;
 
 #if HTS_DIRECTDISK
             // Court-circuit:
@@ -1173,12 +1227,11 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
               int ok = 0;
 
               r.is_write = 1;   // écrire
-              if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {  // un fichier existe déja
-                //if (fsize_utf8(fconv(save))==r.size) {  // même taille -- NON tant pis (taille mal declaree)
+              if (fexist_utf8(fconv(catbuff, sizeof(catbuff),
+                                    save))) { // un fichier existe déja
                 ok = 1;         // plus rien à faire
                 filenote(&opt->state.strc, save, NULL); // noter comme connu
                 file_notify(opt, adr, fil, save, 0, 0, 0);
-                //}
               }
 
               if ((pos < 0) && (!ok)) { // Pas de donnée en cache et fichier introuvable : erreur!
@@ -1229,7 +1282,6 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg,
                              "Cache Write Error : Unable to Create File");
-                  //printf("%s\n",save);
                 }
               }
 
@@ -1277,14 +1329,13 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg, "Cache Read Error : Read Data");
                   } else
                     *(r.adr + r.size) = '\0';
-                  //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                 } else {        // erreur
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg, "Cache Memory Error");
                 }
               }
             }
-          }                     // si save==null, ne rien charger (juste en tête)
+          } // si save==null, ne rien charger (juste en tête)
         } else {
 #if DEBUGCA
           printf("Cache Read Error : Bad Data");
@@ -1397,6 +1448,86 @@ static int hts_rename(httrackp * opt, const char *a, const char *b) {
   return rename(a, b);
 }
 
+/* Pathname of a file inside the mirror dir (rotating concat buffer). */
+static char *reconcile_path(httrackp *opt, const char *name) {
+  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                 StringBuff(opt->path_log), name);
+}
+
+/* Interrupted-run heuristic: prefer the old generation when the new cache
+   stalled below NEW_TINY while the old one grew past OLD_SOLID (historical
+   arbitrary thresholds). */
+#define CACHE_RECONCILE_NEW_TINY 32768
+#define CACHE_RECONCILE_OLD_SOLID 65536
+
+/* Replace the new-generation file by the old one, when the old one exists. */
+static void reconcile_promote(httrackp *opt, const char *oldname,
+                              const char *newname) {
+  if (fexist(reconcile_path(opt, oldname))) {
+    remove(reconcile_path(opt, newname));
+    rename(reconcile_path(opt, oldname), reconcile_path(opt, newname));
+  }
+}
+
+void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
+  switch (mode) {
+  case CACHE_RECONCILE_PROMOTE:
+    /* Previous run rotated new.* to old.* then died before writing: promote
+       the old generation back, whichever format it uses. */
+    if (!fexist(reconcile_path(opt, "hts-cache/new.zip")))
+      reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if ((!fexist(reconcile_path(opt, "hts-cache/new.dat")) ||
+         !fexist(reconcile_path(opt, "hts-cache/new.ndx"))) &&
+        fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    break;
+  case CACHE_RECONCILE_INTERRUPTED:
+    /* Aborted run: keep the larger generation when the new cache is
+       suspiciously small next to the old one. The new file must exist: fsize()
+       is -1 for a missing file, which would spuriously pass the "< TINY" test
+       and overwrite a solid old generation that PROMOTE/ROLLBACK should keep.
+     */
+    if (!opt->cache || !fexist(reconcile_path(opt, "hts-in_progress.lock")))
+      break;
+    if (fexist(reconcile_path(opt, "hts-cache/new.zip")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.zip")) &&
+        fsize(reconcile_path(opt, "hts-cache/new.zip")) <
+            CACHE_RECONCILE_NEW_TINY &&
+        fsize(reconcile_path(opt, "hts-cache/old.zip")) >
+            CACHE_RECONCILE_OLD_SOLID &&
+        fsize(reconcile_path(opt, "hts-cache/old.zip")) >
+            fsize(reconcile_path(opt, "hts-cache/new.zip")))
+      reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if (fexist(reconcile_path(opt, "hts-cache/new.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx")) &&
+        fsize(reconcile_path(opt, "hts-cache/new.dat")) <
+            CACHE_RECONCILE_NEW_TINY &&
+        fsize(reconcile_path(opt, "hts-cache/old.dat")) >
+            CACHE_RECONCILE_OLD_SOLID &&
+        fsize(reconcile_path(opt, "hts-cache/old.dat")) >
+            fsize(reconcile_path(opt, "hts-cache/new.dat"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    break;
+  case CACHE_RECONCILE_ROLLBACK:
+    /* Nothing transferred: restore the previous generation and sidecars. */
+    reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if (fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    reconcile_promote(opt, "hts-cache/old.lst", "hts-cache/new.lst");
+    reconcile_promote(opt, "hts-cache/old.txt", "hts-cache/new.txt");
+    break;
+  }
+}
+
 // renvoyer uniquement en tête, ou NULL si erreur
 // return NULL upon error, and set -1 to r.statuscode
 htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
@@ -1427,25 +1558,10 @@ void cache_init(cache_back * cache, httrackp * opt) {
       mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"),
             HTS_PROTECT_FOLDER);
 #endif
-      if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.zip")))) {       // il existe déja un cache précédent.. renommer
-        /* Previous cache from the previous cache version */
-#if 0
-        /* No.. reuse with old httrack releases! */
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.dat")))
-          remove(fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.dat"));
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.ndx")))
-          remove(fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.ndx"));
-#endif
+      if ((fexist(fconcat(
+              OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+              StringBuff(opt->path_log),
+              "hts-cache/new.zip")))) { // a previous cache exists.. rename it
         /* Previous cache version */
         if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) {     // il existe déja un cache précédent.. renommer
           rename(fconcat
@@ -1486,7 +1602,13 @@ void cache_init(cache_back * cache, httrackp * opt) {
         } else {
           hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed");
         }
-      } else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) {        // il existe déja un cache précédent.. renommer
+      } else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                 StringBuff(opt->path_log),
+                                 "hts-cache/new.dat"))) &&
+                 (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                 StringBuff(opt->path_log),
+                                 "hts-cache/new.ndx")))) { // a previous cache
+                                                           // exists.. rename it
 #if DEBUGCA
         printf("work with former cache\n");
 #endif
@@ -1515,7 +1637,7 @@ void cache_init(cache_back * cache, httrackp * opt) {
                 "hts-cache/new.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
                                               StringBuff(opt->path_log),
                                               "hts-cache/old.ndx"));
-      } else {                  // un des deux (ou les deux) fichiers cache absents: effacer l'autre éventuel
+      } else { // one or both cache files missing: remove the remaining one
 #if DEBUGCA
         printf("new cache\n");
 #endif
@@ -1884,12 +2006,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                    "hts-cache/new.lst"), "wb");
           strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
           opt->state.strc.lst = cache->lst;
-          //{
-          //filecreate_params tmp;
-          //strcpybuff(tmp.path,StringBuff(opt->path_html));    // chemin
-          //tmp.lst=cache->lst;                 // fichier lst
-          //filenote("",&tmp);        // initialiser filecreate
-          //}
 
           // supprimer old.txt
           if (fexist
@@ -1978,12 +2094,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                    "hts-cache/new.lst"), "wb");
           strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
           opt->state.strc.lst = cache->lst;
-          //{
-          //  filecreate_params tmp;
-          //  strcpybuff(tmp.path,StringBuff(opt->path_html));    // chemin
-          //  tmp.lst=cache->lst;                 // fichier lst
-          //  filenote("",&tmp);        // initialiser filecreate
-          //}
 
           // supprimer old.txt
           if (fexist
@@ -2015,8 +2125,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                     "statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)"
                     LF);
           }
-          // test
-          // cache_writedata(cache->ndx,cache->dat,"//[TEST]//","test1","TEST PIPO",9);
         }                       // cache->ndx!=NULL
       }                         //cache->zipOutput != NULL
 

src/htscache.h

@@ -66,8 +66,11 @@ htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr,
                   const char *fil, const char *save, char *location);
 htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, const char *save, char *location);
-htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
-                                   const char *adr, const char *fil);
+/* Like cache_read, but also yields entries whose transfer broke; return_save
+   (optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name. */
+htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
+                                   const char *adr, const char *fil,
+                                   char *return_save);
 htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr,
                     const char *fil, const char *save, char *location,
                     char *return_save, int readonly);
@@ -75,6 +78,17 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, htsblk * r);
 void cache_init(cache_back * cache, httrackp * opt);
 
+/* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
+typedef enum {
+  CACHE_RECONCILE_PROMOTE,     /* no new cache: promote the old generation */
+  CACHE_RECONCILE_INTERRUPTED, /* aborted run: keep the larger generation */
+  CACHE_RECONCILE_ROLLBACK     /* nothing transferred: restore the old one */
+} hts_cache_reconcile_mode;
+
+/* Reconcile the on-disk cache generations according to mode; a no-op when
+   the involved files are absent. */
+void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
+
 int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
                     const char *str2, char *outbuff, int len);
 int cache_readdata(cache_back * cache, const char *str1, const char *str2,

src/htscache_selftest.c

@@ -47,6 +47,8 @@ Please visit our Website: http://www.httrack.com
 #include "htslib.h"
 #include "htszlib.h"
 
+#include <errno.h>
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -316,6 +318,328 @@ static int disk_fallback_selftest(httrackp *opt) {
   return fail;
 }
 
+typedef struct {
+  size_t budget;  /**< bytes allowed through before writes start failing */
+  int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
+  int writes;     /**< zwrite call count, to detect re-entry into the stream */
+  int fail_once;  /**< recover (unlimited budget) after the first failure */
+} writefail_inject;
+
+/* zwrite that copies until the budget runs out, then fails with inj->fail_errno
+   (the #174/#219 condition). Counts calls so the test can prove a flagged cache
+   never re-enters the stream. */
+static uLong selftest_failing_zwrite(voidpf opaque, voidpf stream,
+                                     const void *buf, uLong size) {
+  writefail_inject *inj = (writefail_inject *) opaque;
+
+  inj->writes++;
+  if (inj->budget >= (size_t) size) {
+    inj->budget -= (size_t) size;
+    return (uLong) fwrite(buf, 1, (size_t) size, (FILE *) stream);
+  }
+  if (inj->fail_once)
+    inj->budget = (size_t) -1; /* the backend recovers after this failure */
+  errno = inj->fail_errno;
+  return 0; /* short write -> the minizip op returns an error */
+}
+
+/* Open a ZIP whose writes fail past inj->budget, so cache_add() hits an error.
+ */
+static zipFile selftest_open_failing_zip(const char *path,
+                                         writefail_inject *inj) {
+  zlib_filefunc_def ff;
+
+  fill_fopen_filefunc(&ff); /* real fopen/read/seek/close; ignores opaque */
+  ff.zwrite_file = selftest_failing_zwrite;
+  ff.opaque = inj;
+  return zipOpen2(path, APPEND_STATUS_CREATE, NULL, &ff);
+}
+
+/* Store one octet-stream body into `cache` (all-in-cache, body in the ZIP). */
+static void writefail_store(httrackp *opt, cache_back *cache, const char *fil,
+                            const char *body, size_t body_len) {
+  htsblk r;
+  char locbuf[4];
+  char *bodycopy = malloct(body_len);
+
+  hts_init_htsblk(&r);
+  r.statuscode = 200;
+  r.size = (LLint) body_len;
+  strcpybuff(r.msg, "OK");
+  strcpybuff(r.contenttype, "application/octet-stream");
+  locbuf[0] = '\0';
+  r.location = locbuf;
+  r.is_write = 0;
+  memcpy(bodycopy, body, body_len);
+  r.adr = bodycopy;
+  cache_add(opt, cache, &r, "example.com", fil, "example.com/blob.bin", 1,
+            NULL);
+  freet(bodycopy);
+}
+
+/* Store an entry claiming a >2GB body; the degrade path never reads data. */
+static void writefail_store_oversized(httrackp *opt, cache_back *cache,
+                                      const char *fil, int is_write) {
+  htsblk r;
+  char locbuf[4];
+
+  hts_init_htsblk(&r);
+  r.statuscode = 200;
+  r.size = (LLint) INT_MAX + 1;
+  strcpybuff(r.msg, "OK");
+  strcpybuff(r.contenttype, "application/octet-stream");
+  locbuf[0] = '\0';
+  r.location = locbuf;
+  r.is_write = (short int) is_write;
+  cache_add(opt, cache, &r, "example.com", fil, "example.com/big.bin", 1, NULL);
+}
+
+/* Read back `entryname`: extra field (cached headers) and body. Returns the
+   body length, or -1 if the entry is absent or unreadable. */
+static int writefail_read_entry(const char *path, const char *entryname,
+                                char *extra, size_t extralen, char *body,
+                                size_t bodylen) {
+  unzFile z = unzOpen(path);
+  int n = -1;
+
+  if (z == NULL)
+    return -1;
+  if (unzLocateFile(z, entryname, 1) == UNZ_OK &&
+      unzOpenCurrentFile(z) == UNZ_OK) {
+    const int elen = unzGetLocalExtrafield(z, extra, (unsigned) (extralen - 1));
+
+    if (elen >= 0) {
+      extra[elen] = '\0';
+      n = unzReadCurrentFile(z, body, (unsigned) bodylen);
+    }
+    unzCloseCurrentFile(z);
+  }
+  unzClose(z);
+  return n;
+}
+
+/* Cache write-failure policy (#174/#219): fatal errno or a failure streak
+   stops the mirror (exit_xh=-1, no crash); isolated/oversized drops the entry.
+ */
+int cache_write_failure_selftest(httrackp *opt, const char *dir) {
+  int fail = 0;
+  char path[HTS_URLMAXSIZE];
+  /* incompressible + big, so deflate flushes (and fails) mid-write, before
+   * close */
+  static const size_t body_len = 256 * 1024;
+  char *body = malloct(body_len);
+  int phase;
+
+  gen_body(body, body_len, 1 /* incompressible */);
+  fconcat(path, sizeof(path), dir, "/wfail.zip");
+
+  /* phase 0: fatal errno (ENOSPC) aborts at once; phase 1: persistent EIO
+     drops entries until the streak caps out, then aborts. */
+  for (phase = 0; phase < 2; phase++) {
+    cache_back cache;
+    writefail_inject inj;
+    int writes_after_fail;
+
+    inj.budget = (phase == 0) ? 4096 : 0;
+    inj.fail_errno = (phase == 0) ? ENOSPC : EIO;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    if (cache.zipOutput == NULL) {
+      fprintf(stderr, "cache-writefail: could not open injected ZIP\n");
+      fail++;
+      continue;
+    }
+
+    opt->state.exit_xh = 0; /* clear; the failing write must set it to -1 */
+    if (phase == 0) {
+      writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    } else {
+      /* the abort must land exactly on the 8th consecutive failure */
+      int i;
+
+      for (i = 0; i < 7; i++) {
+        char fil[32];
+
+        snprintf(fil, sizeof(fil), "/b%d.bin", i);
+        writefail_store(opt, &cache, fil, body, 16);
+      }
+      if (cache.zipWriteFailed) {
+        fprintf(stderr, "cache-writefail: phase 1: aborted before the "
+                        "8th consecutive failure\n");
+        fail++;
+      }
+      writefail_store(opt, &cache, "/b7.bin", body, 16);
+    }
+    if (!cache.zipWriteFailed) {
+      fprintf(stderr, "cache-writefail: phase %d: write error not caught\n",
+              phase);
+      fail++;
+    }
+    if (opt->state.exit_xh != -1) {
+      fprintf(stderr,
+              "cache-writefail: phase %d: mirror not aborted (exit_xh=%d)\n",
+              phase, opt->state.exit_xh);
+      fail++;
+    }
+
+    /* a flagged cache must no-op a sibling write: no further backend write */
+    writes_after_fail = inj.writes;
+    writefail_store(opt, &cache, "/blob2.bin", body, 16);
+    if (inj.writes != writes_after_fail) {
+      fprintf(stderr,
+              "cache-writefail: phase %d: sibling write re-entered the broken "
+              "stream (%d extra backend writes)\n",
+              phase, inj.writes - writes_after_fail);
+      fail++;
+    }
+
+    if (cache.zipOutput != NULL) {
+      zipClose(cache.zipOutput,
+               NULL); /* best-effort; may fail on the backend */
+      cache.zipOutput = NULL;
+    }
+  }
+
+  /* failures with successes in between reset the streak: never aborts */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    int i;
+
+    inj.budget = (size_t) -1;
+    inj.fail_errno = EIO;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    for (i = 0; i < 10; i++) {
+      char fil[32];
+
+      inj.budget = 0; /* this store fails */
+      snprintf(fil, sizeof(fil), "/s%d.bin", i);
+      writefail_store(opt, &cache, fil, body, 16);
+      inj.budget = (size_t) -1; /* this one succeeds and resets the streak */
+      snprintf(fil, sizeof(fil), "/ok%d.bin", i);
+      writefail_store(opt, &cache, fil, body, 16);
+    }
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: scattered: non-consecutive failures aborted "
+              "the mirror (flagged=%d, exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+  }
+
+  /* isolated failure: only that entry drops; a later sibling round-trips */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    char extra[8192];
+    char rbody[64];
+    int n;
+
+    inj.budget = 4096;
+    inj.fail_errno = EIO;
+    inj.writes = 0;
+    inj.fail_once = 1;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: skip: isolated failure aborted the mirror "
+              "(flagged=%d, exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    writefail_store(opt, &cache, "/blob2.bin", body, 16);
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+    n = writefail_read_entry(path, "http://example.com/blob2.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody));
+    if (n != 16 || memcmp(rbody, body, 16) != 0) {
+      fprintf(stderr,
+              "cache-writefail: skip: sibling entry lost after a skipped "
+              "entry (%d)\n",
+              n);
+      fail++;
+    }
+  }
+
+  /* >2GB bodies: in-memory drops the entry, on-disk degrades to headers-only */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    char extra[8192];
+    char rbody[64];
+    int n;
+
+    inj.budget = (size_t) -1; /* no injected failure */
+    inj.fail_errno = 0;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    writefail_store_oversized(opt, &cache, "/bigmem.bin", 0 /* in-memory */);
+    writefail_store_oversized(opt, &cache, "/bigdisk.bin", 1 /* on-disk */);
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: oversize: mirror aborted (flagged=%d, "
+              "exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    if (writefail_read_entry(path, "http://example.com/bigmem.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody)) >= 0) {
+      fprintf(stderr,
+              "cache-writefail: oversize: in-memory entry was stored\n");
+      fail++;
+    }
+    n = writefail_read_entry(path, "http://example.com/bigdisk.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody));
+    if (n != 0 || strstr(extra, "X-In-Cache: 0") == NULL) {
+      fprintf(stderr,
+              "cache-writefail: oversize: on-disk entry not stored "
+              "headers-only (%d)\n",
+              n);
+      fail++;
+    }
+  }
+
+  freet(body);
+  return fail;
+}
+
 int cache_selftests(httrackp *opt, const char *dir) {
   int failures = 0;
   cache_back cache;
@@ -585,3 +909,494 @@ int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
 
   return failures;
 }
+
+/* --- hts_cache_reconcile() policies -------------------------------------- */
+
+/* All reconcile inputs/outputs, wiped between cases. */
+static const char *const reconcile_files[] = {
+    "hts-cache/new.zip", "hts-cache/old.zip",   "hts-cache/new.dat",
+    "hts-cache/old.dat", "hts-cache/new.ndx",   "hts-cache/old.ndx",
+    "hts-cache/new.lst", "hts-cache/old.lst",   "hts-cache/new.txt",
+    "hts-cache/old.txt", "hts-in_progress.lock"};
+
+static char *reconcile_st_path(httrackp *opt, const char *name) {
+  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                 StringBuff(opt->path_log), name);
+}
+
+static void reconcile_wipe(httrackp *opt) {
+  size_t i;
+
+  for (i = 0; i < sizeof(reconcile_files) / sizeof(reconcile_files[0]); i++)
+    remove(reconcile_st_path(opt, reconcile_files[i]));
+}
+
+/* Create a filler file of exactly `size` bytes. */
+static void reconcile_put(httrackp *opt, const char *name, size_t size) {
+  FILE *const fp = fopen(reconcile_st_path(opt, name), "wb");
+  static const char filler[1024] = {'x'};
+
+  assertf(fp != NULL);
+  while (size > 0) {
+    const size_t n = size > sizeof(filler) ? sizeof(filler) : size;
+
+    assertf(fwrite(filler, 1, n, fp) == n);
+    size -= n;
+  }
+  fclose(fp);
+}
+
+/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
+static int reconcile_expect(httrackp *opt, const char *name, off_t size,
+                            const char *what) {
+  const off_t got = fsize(reconcile_st_path(opt, name));
+
+  if (got != size) {
+    fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
+            name, (int) got, (int) size);
+    return 1;
+  }
+  return 0;
+}
+
+int cache_reconcile_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+
+  /* around the interrupted-run thresholds (new < 32768, old > 65536) */
+  static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
+
+  golden_setup(opt, dir);
+#ifdef _WIN32
+  mkdir(reconcile_st_path(opt, "hts-cache"));
+#else
+  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
+#endif
+
+  /* PROMOTE: a zip old generation replaces a missing new one */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "promote-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "promote-zip");
+
+  /* PROMOTE: an existing new.zip is left alone */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "promote-zip-noop");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
+
+  /* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
+     zip cache existed) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
+
+  /* PROMOTE: a half-written legacy new pair is replaced by the old pair */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
+
+  /* INTERRUPTED: no lock file, no action */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-nolock");
+
+  /* INTERRUPTED: an absent new.zip must NOT promote old.zip (fsize(-1) would
+     spuriously pass "< TINY"); leave the solid old generation for ROLLBACK */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", -1, "interrupted-nonew");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "interrupted-nonew");
+
+  /* INTERRUPTED: stalled tiny new.zip loses to a solid old.zip (was dead for
+     zip caches: the arm was gated on a legacy new.dat) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", SOLID, "interrupted-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "interrupted-zip");
+
+  /* INTERRUPTED: old below the confidence threshold, keep new */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-smallold");
+
+  /* INTERRUPTED: new big enough to trust, keep it */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", MID);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
+
+  /* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
+
+  /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
+     its only good generation here) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "rollback-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "rollback-zip");
+
+  /* ROLLBACK: sidecars are restored regardless of format */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.lst", TINY);
+  reconcile_put(opt, "hts-cache/old.lst", MID);
+  reconcile_put(opt, "hts-cache/old.txt", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
+  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
+
+  /* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", MID);
+  reconcile_put(opt, "hts-cache/old.lst", MID);
+  reconcile_put(opt, "hts-cache/old.txt", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
+
+  /* ROLLBACK: nothing to restore, the new generation stays */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", TINY, "rollback-noop");
+
+  reconcile_wipe(opt);
+  return failures;
+}
+
+/* --- read-side corruption injection --------------------------------------- */
+
+/* canary read back intact after each corruption; victim gets the byte surgery
+ */
+#define CORRUPT_ADR "corrupt.example.com"
+static char corrupt_body_a[33 + 1];
+static char corrupt_body_b[44 + 1];
+
+/* Write a fresh two-entry cache: /canary.html then /victim.html. */
+static void corrupt_build(httrackp *opt) {
+  cache_back cache;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
+              strlen(corrupt_body_b));
+  selftest_close(&cache);
+}
+
+/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
+   is later overwritten with a forged oversized X-Size (same byte length). */
+static void corrupt_build_etag(httrackp *opt) {
+  cache_back cache;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
+              "OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
+              corrupt_body_b, strlen(corrupt_body_b));
+  selftest_close(&cache);
+}
+
+/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
+   body on disk): the shape every non-html file is stored with. */
+static void corrupt_build_disk(httrackp *opt) {
+  cache_back cache;
+  htsblk w;
+  char locw[4];
+  char BIGSTK save[HTS_URLMAXSIZE * 2];
+  char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
+  char *path;
+  FILE *fp;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
+          CORRUPT_ADR "/victim.bin");
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  hts_init_htsblk(&w);
+  w.statuscode = 200;
+  w.size = (LLint) sizeof(corrupt_body_b) - 1;
+  strcpybuff(w.msg, "OK");
+  strcpybuff(w.contenttype, "application/octet-stream");
+  strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
+  locw[0] = '\0';
+  w.location = locw;
+  w.is_write = 0;
+  cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
+            0 /* all_in_cache */, NULL);
+  selftest_close(&cache);
+  /* the reader only checks this file exists; it never reads it here */
+  path = fconv(catbuff, sizeof(catbuff), save);
+  (void) structcheck(path);
+  fp = FOPEN(path, "wb");
+  assertf(fp != NULL);
+  fclose(fp);
+}
+
+/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
+static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
+                          const char *rep, size_t nth, size_t total) {
+  LLint fsz = 0;
+  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
+  const size_t n = (size_t) fsz;
+  size_t k, hits = 0, at = 0;
+  FILE *fp;
+
+  assertf(data != NULL);
+  for (k = 0; k + patlen <= n; k++) {
+    if (memcmp(data + k, pat, patlen) == 0) {
+      hits++;
+      if (hits == nth)
+        at = k;
+    }
+  }
+  assertf(hits == total);
+  memcpy(data + at, rep, patlen);
+  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, n, fp) == n);
+  fclose(fp);
+  freet(data);
+}
+
+/* Garbage the first bytes of the victim's deflated data (2nd local header). */
+static void corrupt_victim_body(httrackp *opt) {
+  LLint fsz = 0;
+  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
+  const size_t n = (size_t) fsz;
+  size_t k, hits = 0, off = 0;
+  FILE *fp;
+
+  assertf(data != NULL);
+  for (k = 0; k + 4 <= n; k++) {
+    if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
+      const size_t namelen =
+          (unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
+      const size_t extralen =
+          (unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
+
+      off = k + 30 + namelen + extralen;
+    }
+  }
+  assertf(hits == 2);
+  assertf(off != 0 && off + 4 <= n);
+  memset(data + off, 0xFF, 4);
+  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, n, fp) == n);
+  fclose(fp);
+  freet(data);
+}
+
+/* Read the corrupt /victim.html and, in the SAME read session, the intact
+   /canary.html: the victim must be rejected (wantmsg pins which path) and the
+   canary must still decode byte-exact, proving one bad entry never taints a
+   sibling read. */
+static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
+                                     const char *wantmsg, const char *what) {
+  cache_back cache;
+  htsblk v, c;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  char BIGSTK lc[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = lc[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
+  if (v.statuscode != STATUSCODE_INVALID) {
+    fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
+            selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
+    fail++;
+  }
+  if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
+    fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
+            selftest_tag, what, v.msg, wantmsg);
+    fail++;
+  }
+  c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
+  if (c.statuscode != 200 || c.adr == NULL ||
+      c.size != (LLint) strlen(corrupt_body_a) ||
+      memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
+    fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
+            c.statuscode);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  if (c.adr != NULL)
+    freet(c.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
+static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
+                                 const char *what) {
+  return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
+}
+
+/* Headers-only probe of the disk victim: must parse OK with the size kept. */
+static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
+                                      const char *what) {
+  cache_back cache;
+  htsblk v;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
+  if (v.statuscode != 200 || v.size != wantsize) {
+    fprintf(stderr,
+            "%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
+            selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
+/* One zip corruption case: build, patch, then check victim+canary in-session.
+ */
+static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
+                            size_t nth, size_t total, const char *wantmsg,
+                            const char *what) {
+  corrupt_build(opt);
+  corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
+  return corrupt_expect_victim(opt, wantmsg, what);
+}
+
+int cache_corruption_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+
+  selftest_tag = "cache-corrupt";
+  golden_setup(opt, dir);
+
+  failures +=
+      corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
+                       "Cache Read Error : Read Data", "oversized X-Size");
+  failures +=
+      corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
+                       "Cache Read Error : Bad Size", "negative X-Size");
+  /* both entries carry the line; the victim's is the second */
+  failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
+                               "Previous cache file not found (empty filename)",
+                               "blanked X-In-Cache");
+  /* smashed local file header: the entry is dropped at index load */
+  failures +=
+      corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
+                       "File Cache Entry Not Found", "smashed local header");
+
+  corrupt_build(opt);
+  corrupt_victim_body(opt);
+  failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
+                                    "garbled deflate stream");
+
+  /* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
+     truncates negative in the (int) cast the malloc uses: a wraparound alloc.
+     cache_add asserts size fits an int, so such a value only reaches the reader
+     from a corrupt/foreign cache; inject it by overwriting the victim's long
+     Etag line with a same-length forged X-Size line (the parser keeps the last
+     X-Size it sees), keeping the zip byte-length and offsets intact. */
+  corrupt_build_etag(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
+                                    "X-Size above INT_MAX");
+
+  /* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
+     is how every >2GB non-html file is stored. It must survive a header probe
+     (or every update re-fetches the file); an in-memory read still rejects. */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
+                                         "headers-only X-Size above INT_MAX");
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size above INT_MAX");
+
+  /* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483647AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size at INT_MAX");
+
+  /* the negative check must stay global, headers-only included */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: -2147483648AAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "headers-only negative X-Size");
+
+  return failures;
+}

src/htscache_selftest.h

@@ -52,6 +52,19 @@ int cache_selftests(httrackp *opt, const char *dir);
    committed file, never by the test). Returns the failed-check count. */
 int cache_golden_selftest(httrackp *opt, const char *dir, int regen);
 
+/* Cache write-failure policy (#174/#219): abort on fatal errno or a streak,
+   drop just the entry otherwise. Returns the failed-check count. */
+int cache_write_failure_selftest(httrackp *opt, const char *dir);
+
+/* Exercise the hts_cache_reconcile() generation policies on file fixtures
+   under <dir>. Returns the failed-check count. */
+int cache_reconcile_selftest(httrackp *opt, const char *dir);
+
+/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
+   under <dir> and assert every case degrades to STATUSCODE_INVALID without
+   tainting a sibling entry. */
+int cache_corruption_selftest(httrackp *opt, const char *dir);
+
 #endif
 
 #endif

src/htscatchurl.c

@@ -64,7 +64,7 @@ Please visit our Website: http://www.httrack.com
 // catch_url_init(&port,&return_host);
 HTSEXT_API T_SOC catch_url_init_std(int *port_prox, char *adr_prox) {
   T_SOC soc;
-  int try_to_listen_to[] = { 8080, 3128, 80, 81, 82, 8081, 3129, 31337, 0, -1 };
+  int try_to_listen_to[] = {8080, 3128, 80, 81, 82, 8081, 3129, 0, -1};
   int i = 0;
 
   do {
@@ -175,7 +175,9 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
       //
       socinput(soc, line, 1000);
       if (strnotempty(line)) {
-        if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
+        /* widths bound the caller buffers: method[32], url[HTS_URLMAXSIZE*2],
+           protocol[256] */
+        if (sscanf(line, "%31s %2047s %255s", method, url, protocol) == 3) {
           lien_adrfil af;
 
           // méthode en majuscule
@@ -195,7 +197,6 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
             htsblk blkretour;
 
             hts_init_htsblk(&blkretour);
-            //memset(&blkretour, 0, sizeof(htsblk));    // effacer
             blkretour.location = loc;   // si non nul, contiendra l'adresse véritable en cas de moved xx
             // Lire en têtes restants
             sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
@@ -205,8 +206,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
               strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
               strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
             }
-            // CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
-            //strcatbuff(data,"\r\n");
+            // CR/LF final de l'en tête inutile car déja placé via la ligne vide
+            // juste au dessus
             if (blkretour.totalsize > 0) {
               int len = (int) min(blkretour.totalsize, 32000);
               int pos = (int) strlen(data);

src/htscharset.c

@@ -841,71 +841,6 @@ static unsigned int nlz8(unsigned char x) {
     }                                           \
   } while(0)
 
-/* Sample. */
-#if 0
-
-int main(int argc, char **argv) {
-  int i;
-  int hex = 0;
-  
-#define READ_INT(DEST)                                  \
-  ( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1)      \
-    || (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
-  
-  for(i = 1 ; i < argc ; i++) {
-    unsigned int uc, from, to;
-    
-    if (strcmp(argv[i], "--hex") == 0) {
-      hex = 1;
-    }
-    else if (strcmp(argv[i], "--decimal") == 0) {
-      hex = 0;
-    }
-    else if (strcmp(argv[i], "--decode") == 0) {
-#define RD fgetc_unlocked(stdin)
-#define WR(C) do {                              \
-        if (C != -1) {                          \
-          printf("%04x\n", C);                  \
-        } else if (!feof(stdin)) {              \
-          fprintf(stderr, "read error\n");      \
-          exit(EXIT_FAILURE);                   \
-        }                                       \
-      } while(0)
-      while(!feof(stdin)) {
-        READ_UNICODE(RD, WR);
-      }
-#undef RD
-#undef WR
-    }
-    else if (strcmp(argv[i], "-range") == 0
-             && i + 2 < argc
-             && (++i, 1)
-             && READ_INT(from)
-             && (++i, 1)
-             && READ_INT(to)
-             ) {
-      unsigned int i;
-      for(i = from ; i < to ; i++) {
-#define EM(C) fputc_unlocked(C, stdout)
-        EMIT_UNICODE(i, EM);
-#undef EM
-      }
-    }
-    else if (READ_INT(uc)) {
-#define EM(C) fputc_unlocked(C, stdout)
-      EMIT_UNICODE(uc, EM);
-#undef EM
-    }
-    else {
-      return EXIT_FAILURE;
-    }
-  }
-  
-  return EXIT_SUCCESS;
-}
-
-#endif
-
 /* IDNA helpers. */
 #undef ADD_BYTE
 #undef INCREASE_CAPA
@@ -975,19 +910,19 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
                 /* Reader: can read bytes up to j */
 #define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
 
-                /* Writer: upon error, return FFFD (replacement character) */
-#define WR(C) do { \
-  if ((C) != -1) { \
-    /* copy character */ \
-    assertf(segOutputSize < segSize); \
-    segInt[segOutputSize++] = (C); \
-  } \
-  /* In case of error, abort. */ \
-  else { \
-    FREE_BUFFER(); \
-    return NULL; \
-  } \
-} while(0)
+                /* Writer: a malformed sequence abandons the encode (NULL) */
+#define WR(C)                                                                  \
+  do {                                                                         \
+    if ((C) != -1) {                                                           \
+      /* copy character */                                                     \
+      assertf(segOutputSize < segSize);                                        \
+      segInt[segOutputSize++] = (C);                                           \
+    } else {                                                                   \
+      freet(segInt);                                                           \
+      FREE_BUFFER();                                                           \
+      return NULL;                                                             \
+    }                                                                          \
+  } while (0)
 
                 /* Read/Write Unicode character. */
                 READ_UNICODE(RD, WR);

src/htsconfig.h

@@ -52,12 +52,8 @@ Please visit our Website: http://www.httrack.com
 #define DEFAULT_FTP "index.txt"
 
 // extension par défaut pour fichiers n'en ayant pas
-#define DEFAULT_EXT       ".html"
+#define DEFAULT_EXT ".html"
 #define DEFAULT_EXT_SHORT ".htm"
-//#define DEFAULT_BIN_EXT   ".bin"
-//#define DEFAULT_BIN_EXT_SHORT ".bin"
-//#define DEFAULT_EXT       ".txt"
-//#define DEFAULT_EXT_SHORT ".txt"
 
 // éviter les /nul, /con..
 #define HTS_OVERRIDE_DOS_FOLDERS 1
@@ -80,14 +76,11 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
-// gérer une table de hachage?
-// REMOVED
-// #define HTS_HASH 1
-
 // fast cache (build hash table)
 #define HTS_FAST_CACHE 1
 
-// le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est valide)
+// le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
+// valide)
 #define GT_ENDS_COMMENT 1
 
 // always adds a '/' at the end if a '~' is encountered (/~smith -> /~smith/)
@@ -96,26 +89,10 @@ Please visit our Website: http://www.httrack.com
 // always transform a '//' into a sigle '/'
 #define HTS_STRIP_DOUBLE_SLASH 0
 
-// case-sensitive pour les dossiers et fichiers (0/1)
-// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est pas très utile..
-// ..et pas bcp respecté]
-// REMOVED
-// #define HTS_CASSE 0
-
-// Un fichier ayant une taille différente du content-length doit il être annulé?
-// SEE opt.tolerant and opt.http10
-// #define HTS_CL_IS_FATAL 0
-
 // une erreur supprime le fichier sur disque
 // (non fixé pour cause de retry)
 #define HTS_REMOVE_BAD_FILES 0
 
-// en cas de Range: xx- donnant un Content-length: xx
-// alors skipper le fichier, considéré comme transmis
-// #define HTS_SKIP_FULL_RANGE 1
-
-// nombre max de filtres que l'utilisateur peut fixer
-// #define HTS_FILTERSMAX 10000
 #define HTS_FILTERSINC 1000
 
 // connect non bloquant? (poll sur write)

src/htscore.h

@@ -152,6 +152,15 @@ struct lien_adrfilsave {
   char save[HTS_URLMAXSIZE * 2]; /**< local save path (with directory) */
 };
 
+/** Per-slot connect-fallback bookkeeping (parallel to struct_back.lnk).
+    Tracks which resolved address the slot is currently connecting to so a
+    stuck connect can be retried against the next one. */
+typedef struct hts_connect_fallback {
+  int addr_index;       /**< candidate being connected (0-based) */
+  int addr_count;       /**< resolved addresses; -1 = not yet probed */
+  TStamp connect_start; /**< when the current candidate's connect began */
+} hts_connect_fallback;
+
 /** The download-slot ring: the set of concurrent transfers in flight.
     Allocated/owned by the engine; consumers (status callbacks, the loop)
     read it but do not resize or free it. */
@@ -168,6 +177,7 @@ struct struct_back {
   int count;              /**< number of usable slots (back_max) */
   coucal ready;           /**< index of slots whose transfer completed */
   LLint ready_size_bytes; /**< total bytes buffered in completed slots */
+  hts_connect_fallback *connect_fallback; /**< per-slot, count+1 entries */
 };
 
 typedef struct cache_back_zip_entry cache_back_zip_entry;
@@ -204,6 +214,9 @@ struct cache_back {
   cache_back_zip_entry *zipEntries;
   int zipEntriesOffs;
   int zipEntriesCapa;
+  hts_boolean
+      zipWriteFailed; /**< a cache write failed; stop touching the stream */
+  int zipWriteFailures; /**< consecutive entry write failures; reset on store */
 };
 
 #ifndef HTS_DEF_FWSTRUCT_hash_struct
@@ -222,8 +235,12 @@ struct hash_struct {
   coucal adrfil;
   /* former address+path -> link index (renamed/moved entries) */
   coucal former_adrfil;
-  /* scratch buffers reused across lookups (not reentrant) */
-  int normalized;
+  /* effective urlhack sub-flags: www.==host / // collapse / query-arg sort */
+  hts_boolean norm_host;
+  hts_boolean norm_slash;
+  hts_boolean norm_query;
+  /* query-strip keys (not owned); set from opt->strip_query at hash_init */
+  const char *strip_query;
   char normfil[HTS_URLMAXSIZE * 2];
   char normfil2[HTS_URLMAXSIZE * 2];
   char catbuff[CATBUFF_SIZE];
@@ -346,12 +363,42 @@ void usercommand(httrackp * opt, int exe, const char *cmd, const char *file,
 
 void usercommand_exe(const char *cmd, const char *file);
 
+// Finish the makeindex index.html (footer + refresh meta), run usercommand.
+// Updates *makeindex_done/*makeindex_fp in place; adr/fil are the mode strings.
+void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
+                          FILE **makeindex_fp, int makeindex_links,
+                          const char *makeindex_firstlink,
+                          const char *template_footer, const char *adr,
+                          const char *fil);
+
+// Flush ht_buff[0..ht_len] to save on disk (skip if MD5 unchanged); *fp
+// closed+NULLed on write. Precondition: ht_len>0.
+void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
+                          FILE **fp, const char *ht_buff, size_t ht_len,
+                          const char *adr, const char *fil, const char *save);
+
 int filters_init(char ***ptrfilters, int maxfilter, int filterinc);
 
 int fspc(httrackp * opt, FILE * fp, const char *type);
 
 char *next_token(char *p, int flag);
 
+/* Like fil_normalized(), but first drops query keys in STRIP (comma-separated,
+   "*" = all); STRIP NULL/empty behaves exactly like fil_normalized(). */
+char *fil_normalized_filtered(const char *source, char *dest,
+                              const char *strip);
+
+/* As fil_normalized_filtered(), but DO_SLASH/DO_QUERY gate the // collapse and
+   the query-argument sort independently (the urlhack sub-flags). */
+char *fil_normalized_filtered_ex(const char *source, char *dest,
+                                 const char *strip, int do_slash, int do_query);
+
+/* For URL ADR/FIL, return (in DEST) the comma keylist to strip from the
+   '\n'-separated "[pattern=]keys" RULES (patterns matched on host/path via
+   strjoker, last wins); NULL if none match. Feeds fil_normalized_filtered(). */
+const char *hts_query_strip_keys(const char *rules, const char *adr,
+                                 const char *fil, char *dest, size_t destsize);
+
 /* Read a whole file into a freshly malloc'd, NUL-terminated buffer; the caller
    owns it and must release it with freet(). Return NULL on missing/unreadable
    file (readfile_or substitutes defaultdata instead). The byte content is NOT
@@ -365,13 +412,16 @@ char *readfile_utf8(const char *fil);
 
 char *readfile_or(const char *fil, const char *defaultdata);
 
-#if 0
-void check_rate(TStamp stat_timestart, int maxrate);
-#endif
-
 /* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
    Not thread-safe; call from the single crawl loop. */
 
+/* True if a connecting slot should give up on the current address and try the
+   next one: a fallback address remains (addr_index+1 < addr_count) and the
+   candidate has been connecting for at least its deadline, min(timeout, an
+   internal cap). elapsed/timeout in seconds. Exposed for the -#D self-test. */
+int back_connect_fallback_due(int addr_index, int addr_count, int elapsed,
+                              int timeout);
+
 /* How many new sockets may be opened now, honoring maxsoc and the maxconn rate
    limit (>=0). _strict ignores reserved-slot headroom; the plain form leaves
    room for naming tests and stops at 0 when the stack is nearly full. */
@@ -379,6 +429,19 @@ int back_pluggable_sockets(struct_back * sback, httrackp * opt);
 
 int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);
 
+/* One engine-loop tick: refresh the transfer stats and run the loop callback
+   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
+hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr);
+
+/* Wait until a test socket can be plugged, pumping transfers, stats and the
+   loop callback; gives up past the -E deadline. HTS_FALSE = callback abort. */
+hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
+                                      cache_back *cache, int ptr);
+
+/* Randomized inter-file pause target in [min_ms,max_ms] (#185), derived from a
+   timestamp seed so it is stable within one gap and rerolls per launch. */
+int hts_pause_target_ms(TStamp seed, int min_ms, int max_ms);
+
 /* Schedule more links from the heap into free slots. Returns the number queued,
    or <=0 if none could be added (no free slot / paused / stopped). */
 int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
@@ -400,11 +463,6 @@ int ask_continue(httrackp * opt);
 /* Number of decimal digits in n. */
 int nombre_digit(int n);
 
-// Java
-#if 0
-int hts_add_file(char *file, int file_position);
-#endif
-
 // Polling
 #if HTS_POLL
 int check_flot(T_SOC s);
@@ -427,4 +485,8 @@ void voidf(void);
 /* HTML marker comment marking where the top index is spliced. */
 #define HTS_TOPINDEX "TOP_INDEX_HTTRACK"
 
+/* Worst-case byte expansion HT_ADD_HTMLESCAPED* must reserve per escaper. */
+#define HTS_HTMLESCAPE_MAXEXP 5      /* escape_for_html_print: '&'->"&amp;" */
+#define HTS_HTMLESCAPE_FULL_MAXEXP 6 /* _full: high byte->"&#xHH;" */
+
 #endif

src/htsdefines.h

@@ -69,11 +69,15 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
 typedef struct t_hts_callbackarg t_hts_callbackarg;
 #endif
 
-/* Marks a symbol an external wrapper module exports back to the engine
-   (dllexport on Windows, nothing elsewhere). */
+/* Marks a symbol an external wrapper module exports back to the engine.
+   Must override -fvisibility=hidden on ELF, or dlopen()ed plugins (htsjava)
+   hide their own hts_plug()/hts_unplug() entry points. */
 #ifndef EXTERNAL_FUNCTION
 #ifdef _WIN32
 #define EXTERNAL_FUNCTION __declspec(dllexport)
+#elif ((defined(__GNUC__) && (__GNUC__ >= 4)) ||                               \
+       (defined(HAVE_VISIBILITY) && HAVE_VISIBILITY))
+#define EXTERNAL_FUNCTION __attribute__((visibility("default")))
 #else
 #define EXTERNAL_FUNCTION
 #endif
@@ -82,9 +86,9 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
 /* Entry points of a --wrapper plug-in: hts_plug(opt, argv) is called once to
    install the wrapper (argv is the wrapper's argument string), hts_unplug(opt)
    once to tear it down. Both return non-zero on success. */
-typedef int (*t_hts_plug) (httrackp * opt, const char *argv);
+typedef int (*t_hts_plug)(httrackp *opt, const char *argv);
 
-typedef int (*t_hts_unplug) (httrackp * opt);
+typedef int (*t_hts_unplug)(httrackp *opt);
 
 /* Engine callback prototypes. Each is one hook the engine fires at a defined
    point of a mirror; a wrapper installs the ones it cares about in the
@@ -92,27 +96,27 @@ typedef int (*t_hts_unplug) (httrackp * opt);
    returns are 1 to continue/accept, 0 to abort/refuse unless noted. */
 
 /* Called once when the wrapper is installed; allocate per-run state here. */
-typedef void (*t_hts_htmlcheck_init) (t_hts_callbackarg * carg);
+typedef void (*t_hts_htmlcheck_init)(t_hts_callbackarg *carg);
 
 /* Called once when the wrapper is removed; release per-run state here. */
-typedef void (*t_hts_htmlcheck_uninit) (t_hts_callbackarg * carg);
+typedef void (*t_hts_htmlcheck_uninit)(t_hts_callbackarg *carg);
 
 /* Fired at the start of a mirror, after options are parsed. */
-typedef int (*t_hts_htmlcheck_start) (t_hts_callbackarg * carg, httrackp * opt);
+typedef int (*t_hts_htmlcheck_start)(t_hts_callbackarg *carg, httrackp *opt);
 
 /* Fired at the end of a mirror. */
-typedef int (*t_hts_htmlcheck_end) (t_hts_callbackarg * carg, httrackp * opt);
+typedef int (*t_hts_htmlcheck_end)(t_hts_callbackarg *carg, httrackp *opt);
 
 /* Fired while options are being changed, to validate or adjust them. */
-typedef int (*t_hts_htmlcheck_chopt) (t_hts_callbackarg * carg, httrackp * opt);
+typedef int (*t_hts_htmlcheck_chopt)(t_hts_callbackarg *carg, httrackp *opt);
 
 /* Rewrite hook over an in-memory page: the html and len arguments point at the
    buffer and its length (the callback may reallocate and resize it),
    url_adresse and url_fichier name it. */
-typedef int (*t_hts_htmlcheck_process) (t_hts_callbackarg * carg,
-                                        httrackp * opt, char **html, int *len,
-                                        const char *url_adresse,
-                                        const char *url_fichier);
+typedef int (*t_hts_htmlcheck_process)(t_hts_callbackarg *carg, httrackp *opt,
+                                       char **html, int *len,
+                                       const char *url_adresse,
+                                       const char *url_fichier);
 
 /* Same shape as process, run before HTML parsing. */
 typedef t_hts_htmlcheck_process t_hts_htmlcheck_preprocess;
@@ -121,113 +125,111 @@ typedef t_hts_htmlcheck_process t_hts_htmlcheck_preprocess;
 typedef t_hts_htmlcheck_process t_hts_htmlcheck_postprocess;
 
 /* Inspect a page (read-only html/len) without rewriting it. */
-typedef int (*t_hts_htmlcheck_check_html) (t_hts_callbackarg * carg,
-                                           httrackp * opt, char *html, int len,
-                                           const char *url_adresse,
-                                           const char *url_fichier);
+typedef int (*t_hts_htmlcheck_check_html)(t_hts_callbackarg *carg,
+                                          httrackp *opt, char *html, int len,
+                                          const char *url_adresse,
+                                          const char *url_fichier);
 
 /* Answer an engine query identified by 'question'; returns the answer string
    (owned by the callback, must stay valid until the next call). */
-typedef const char *(*t_hts_htmlcheck_query) (t_hts_callbackarg * carg,
-                                              httrackp * opt,
-                                              const char *question);
+typedef const char *(*t_hts_htmlcheck_query)(t_hts_callbackarg *carg,
+                                             httrackp *opt,
+                                             const char *question);
 
 /* Second query channel, same contract as query. */
-typedef const char *(*t_hts_htmlcheck_query2) (t_hts_callbackarg * carg,
-                                               httrackp * opt,
-                                               const char *question);
+typedef const char *(*t_hts_htmlcheck_query2)(t_hts_callbackarg *carg,
+                                              httrackp *opt,
+                                              const char *question);
 
 /* Third query channel, same contract as query. */
-typedef const char *(*t_hts_htmlcheck_query3) (t_hts_callbackarg * carg,
-                                               httrackp * opt,
-                                               const char *question);
+typedef const char *(*t_hts_htmlcheck_query3)(t_hts_callbackarg *carg,
+                                              httrackp *opt,
+                                              const char *question);
 
 /* Per-tick progress hook: 'back' is the transfer slot array of 'back_max'
    entries, back_index the active one; lien_tot/lien_ntot and stats report
    queue size and running totals, stat_time the elapsed time. */
-typedef int (*t_hts_htmlcheck_loop) (t_hts_callbackarg * carg, httrackp * opt,
-                                     lien_back * back, int back_max,
-                                     int back_index, int lien_tot,
-                                     int lien_ntot, int stat_time,
-                                     hts_stat_struct * stats);
+typedef int (*t_hts_htmlcheck_loop)(t_hts_callbackarg *carg, httrackp *opt,
+                                    lien_back *back, int back_max,
+                                    int back_index, int lien_tot, int lien_ntot,
+                                    int stat_time, hts_stat_struct *stats);
 
 /* Veto a link (adr host, fil path) after its transfer; status is the result.
    Return 0 to drop the link. */
-typedef int (*t_hts_htmlcheck_check_link) (t_hts_callbackarg * carg,
-                                           httrackp * opt, const char *adr,
-                                           const char *fil, int status);
+typedef int (*t_hts_htmlcheck_check_link)(t_hts_callbackarg *carg,
+                                          httrackp *opt, const char *adr,
+                                          const char *fil, int status);
 
 /* Veto a link by its MIME type before download; return 0 to skip it. */
-typedef int (*t_hts_htmlcheck_check_mime) (t_hts_callbackarg * carg,
-                                           httrackp * opt, const char *adr,
-                                           const char *fil, const char *mime,
-                                           int status);
+typedef int (*t_hts_htmlcheck_check_mime)(t_hts_callbackarg *carg,
+                                          httrackp *opt, const char *adr,
+                                          const char *fil, const char *mime,
+                                          int status);
 
 /* Fired when the mirror pauses, waiting on 'lockfile' to be removed. */
-typedef void (*t_hts_htmlcheck_pause) (t_hts_callbackarg * carg, httrackp * opt,
-                                       const char *lockfile);
+typedef void (*t_hts_htmlcheck_pause)(t_hts_callbackarg *carg, httrackp *opt,
+                                      const char *lockfile);
 
 /* Fired after a file is written to disk; 'file' is the local path. */
-typedef void (*t_hts_htmlcheck_filesave) (t_hts_callbackarg * carg,
-                                          httrackp * opt, const char *file);
+typedef void (*t_hts_htmlcheck_filesave)(t_hts_callbackarg *carg, httrackp *opt,
+                                         const char *file);
 
 /* Richer file-saved notification: source host/filename, local path, and flags
    telling whether the file is new, modified, or left unchanged. */
-typedef void (*t_hts_htmlcheck_filesave2) (t_hts_callbackarg * carg,
-                                           httrackp * opt, const char *hostname,
-                                           const char *filename,
-                                           const char *localfile, int is_new,
-                                           int is_modified, int not_updated);
+typedef void (*t_hts_htmlcheck_filesave2)(t_hts_callbackarg *carg,
+                                          httrackp *opt, const char *hostname,
+                                          const char *filename,
+                                          const char *localfile, int is_new,
+                                          int is_modified, int not_updated);
 
 /* Fired for each link parsed out of a page; 'link' may be edited in place. */
-typedef int (*t_hts_htmlcheck_linkdetected) (t_hts_callbackarg * carg,
-                                             httrackp * opt, char *link);
+typedef int (*t_hts_htmlcheck_linkdetected)(t_hts_callbackarg *carg,
+                                            httrackp *opt, char *link);
 
 /* As linkdetected, plus tag_start, the markup the link was found in. */
-typedef int (*t_hts_htmlcheck_linkdetected2) (t_hts_callbackarg * carg,
-                                              httrackp * opt, char *link,
-                                              const char *tag_start);
+typedef int (*t_hts_htmlcheck_linkdetected2)(t_hts_callbackarg *carg,
+                                             httrackp *opt, char *link,
+                                             const char *tag_start);
 
 /* Fired on each transfer-status change of slot 'back'. */
-typedef int (*t_hts_htmlcheck_xfrstatus) (t_hts_callbackarg * carg,
-                                          httrackp * opt, lien_back * back);
+typedef int (*t_hts_htmlcheck_xfrstatus)(t_hts_callbackarg *carg, httrackp *opt,
+                                         lien_back *back);
 
 /* Choose the local save path for a URL; write it into 'save'. adr/fil name the
    target, referer_adr/referer_fil the page that linked it. */
-typedef int (*t_hts_htmlcheck_savename) (t_hts_callbackarg * carg,
-                                         httrackp * opt,
-                                         const char *adr_complete,
-                                         const char *fil_complete,
-                                         const char *referer_adr,
-                                         const char *referer_fil, char *save);
+typedef int (*t_hts_htmlcheck_savename)(t_hts_callbackarg *carg, httrackp *opt,
+                                        const char *adr_complete,
+                                        const char *fil_complete,
+                                        const char *referer_adr,
+                                        const char *referer_fil, char *save);
 
 /* Extended save-name hook, same signature as savename. */
 typedef t_hts_htmlcheck_savename t_hts_htmlcheck_extsavename;
 
 /* Inspect or edit the outgoing request headers in 'buff' before they are sent.
  */
-typedef int (*t_hts_htmlcheck_sendhead) (t_hts_callbackarg * carg,
-                                         httrackp * opt, char *buff,
-                                         const char *adr, const char *fil,
-                                         const char *referer_adr,
-                                         const char *referer_fil,
-                                         htsblk * outgoing);
+typedef int (*t_hts_htmlcheck_sendhead)(t_hts_callbackarg *carg, httrackp *opt,
+                                        char *buff, const char *adr,
+                                        const char *fil,
+                                        const char *referer_adr,
+                                        const char *referer_fil,
+                                        htsblk *outgoing);
 
 /* Inspect the incoming response headers in 'buff' after they are received. */
-typedef int (*t_hts_htmlcheck_receivehead) (t_hts_callbackarg * carg,
-                                            httrackp * opt, char *buff,
-                                            const char *adr, const char *fil,
-                                            const char *referer_adr,
-                                            const char *referer_fil,
-                                            htsblk * incoming);
+typedef int (*t_hts_htmlcheck_receivehead)(t_hts_callbackarg *carg,
+                                           httrackp *opt, char *buff,
+                                           const char *adr, const char *fil,
+                                           const char *referer_adr,
+                                           const char *referer_fil,
+                                           htsblk *incoming);
 
 /* External parser module hooks: detect claims a document type (return 1 to
    take it), parse then extracts its links. 'str' carries the document. */
-typedef int (*t_hts_htmlcheck_detect) (t_hts_callbackarg * carg, httrackp * opt,
-                                       htsmoduleStruct * str);
+typedef int (*t_hts_htmlcheck_detect)(t_hts_callbackarg *carg, httrackp *opt,
+                                      htsmoduleStruct *str);
 
-typedef int (*t_hts_htmlcheck_parse) (t_hts_callbackarg * carg, httrackp * opt,
-                                      htsmoduleStruct * str);
+typedef int (*t_hts_htmlcheck_parse)(t_hts_callbackarg *carg, httrackp *opt,
+                                     htsmoduleStruct *str);
 
 /* Callbacks */
 #ifndef HTS_DEF_FWSTRUCT_t_hts_htmlcheck_callbacks
@@ -237,10 +239,10 @@ typedef struct t_hts_htmlcheck_callbacks t_hts_htmlcheck_callbacks;
 
 /* Declares one named callback slot: its function pointer (typed
    t_hts_htmlcheck_<NAME>) paired with the carg passed to it. */
-#define DEFCALLBACK(NAME)         \
-  struct NAME {                   \
-    t_hts_htmlcheck_ ##NAME fun;  \
-    t_hts_callbackarg *carg;      \
+#define DEFCALLBACK(NAME)                                                      \
+  struct NAME {                                                                \
+    t_hts_htmlcheck_##NAME fun;                                                \
+    t_hts_callbackarg *carg;                                                   \
   } NAME
 
 /* Generic, type-erased callback slot used where the hook type is opaque. */
@@ -324,18 +326,18 @@ extern const t_hts_htmlcheck_callbacks default_callbacks;
 /* Internal helpers for building an HTTP request/response into the engine's
    scratch buffer (opt->state.HTbuff): START resets it, PRINT appends; the
    PANIC variant records a fatal error message. */
-#define HT_PRINT(A) strcatbuff(opt->state.HTbuff,A);
+#define HT_PRINT(A) strcatbuff(opt->state.HTbuff, A);
 
-#define HT_REQUEST_START opt->state.HTbuff[0]='\0';
+#define HT_REQUEST_START opt->state.HTbuff[0] = '\0';
 
 #define HT_REQUEST_END
-#define HTT_REQUEST_START opt->state.HTbuff[0]='\0';
+#define HTT_REQUEST_START opt->state.HTbuff[0] = '\0';
 
 #define HTT_REQUEST_END
-#define HTS_REQUEST_START opt->state.HTbuff[0]='\0';
+#define HTS_REQUEST_START opt->state.HTbuff[0] = '\0';
 
 #define HTS_REQUEST_END
-#define HTS_PANIC_PRINTF(S) strcpybuff(opt->state._hts_errmsg,S);
+#define HTS_PANIC_PRINTF(S) strcpybuff(opt->state._hts_errmsg, S);
 
 #endif
 

src/htsdns_selftest.c

@@ -0,0 +1,359 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: htsdns_selftest.c subroutines:                         */
+/*       in-process self-test for the DNS resolver and cache    */
+/* Author: Xavier Roche                                         */
+/* ------------------------------------------------------------ */
+
+/* Routes the resolver through a scripted getaddrinfo (hts_resolver_backend)
+   instead of the network, so resolution and the DNS cache are testable for a
+   fixed set of scenarios (IPv4/IPv6/dual-stack, errors, family filter,
+   cache reuse) with no live DNS. */
+
+#define HTS_INTERNAL_BYTECODE
+
+#include "htsdns_selftest.h"
+
+#include "htscore.h"
+#include "htslib.h"
+#include "htsnet.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#if HTS_INET6 != 0
+
+/* IPV6_resolver: 0 = v4+v6, 1 = v4 only, 2 = v6 only (htscoremain -@i). */
+extern int IPV6_resolver;
+
+/* One scripted host: either a getaddrinfo error, or an ordered address list. */
+typedef struct mock_addr {
+  int family;             /* AF_INET / AF_INET6 */
+  unsigned char addr[16]; /* 4 (v4) or 16 (v6) meaningful bytes */
+} mock_addr;
+
+typedef struct mock_host {
+  const char *name;
+  int gai_err; /* non-zero: getaddrinfo returns this */
+  int naddr;
+  mock_addr addr[6];
+  int calls; /* times the backend resolved this host */
+} mock_host;
+
+static mock_host mock_hosts[] = {
+    {"v4only.test", 0, 1, {{AF_INET, {1, 2, 3, 4}}}, 0},
+    {"v6only.test", 0, 1, {{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 1}}}, 0},
+    /* dual stack, IPv6 first (RFC 6724 order) then IPv4 */
+    {"dual.test",
+     0,
+     2,
+     {{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 2}}, {AF_INET, {5, 6, 7, 8}}},
+     0},
+    /* dual stack, IPv4 first: distinguishes "keep the first address" from
+       "prefer a family", so the selection contract is actually pinned. */
+    {"dual4.test",
+     0,
+     2,
+     {{AF_INET, {9, 10, 11, 12}},
+      {AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 3}}},
+     0},
+    /* more addresses than HTS_MAXADDRNUM: the list must clamp to the cap. */
+    {"many.test",
+     0,
+     6,
+     {{AF_INET, {10, 0, 0, 1}},
+      {AF_INET, {10, 0, 0, 2}},
+      {AF_INET, {10, 0, 0, 3}},
+      {AF_INET, {10, 0, 0, 4}},
+      {AF_INET, {10, 0, 0, 5}},
+      {AF_INET, {10, 0, 0, 6}}},
+     0},
+    {"nodns.test", EAI_NONAME, 0, {{0}}, 0},
+};
+
+static mock_host *mock_find(const char *name) {
+  for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++) {
+    if (strcmp(mock_hosts[i].name, name) == 0)
+      return &mock_hosts[i];
+  }
+  return NULL;
+}
+
+static void mock_reset_calls(void) {
+  for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++)
+    mock_hosts[i].calls = 0;
+}
+
+/* Build one addrinfo node owning its sockaddr (freed by mock_freeaddrinfo). */
+static struct addrinfo *mock_mkai(const mock_addr *a) {
+  struct addrinfo *ai = calloct(1, sizeof(*ai));
+
+  ai->ai_family = a->family;
+  if (a->family == AF_INET) {
+    struct sockaddr_in *sin = calloct(1, sizeof(*sin));
+
+    sin->sin_family = AF_INET;
+    memcpy(&sin->sin_addr, a->addr, 4);
+    ai->ai_addr = (struct sockaddr *) sin;
+    ai->ai_addrlen = sizeof(*sin);
+  } else {
+    struct sockaddr_in6 *sin6 = calloct(1, sizeof(*sin6));
+
+    sin6->sin6_family = AF_INET6;
+    memcpy(&sin6->sin6_addr, a->addr, 16);
+    ai->ai_addr = (struct sockaddr *) sin6;
+    ai->ai_addrlen = sizeof(*sin6);
+  }
+  return ai;
+}
+
+static int mock_getaddrinfo(const char *node, const char *service,
+                            const struct addrinfo *hints,
+                            struct addrinfo **res) {
+  mock_host *const h = mock_find(node);
+  const int want = (hints != NULL) ? hints->ai_family : PF_UNSPEC;
+  struct addrinfo *head = NULL, *tail = NULL;
+
+  (void) service;
+  *res = NULL;
+  if (h == NULL)
+    return EAI_NONAME;
+  h->calls++; /* a real backend hit; a cached host skips this */
+  if (h->gai_err != 0)
+    return h->gai_err;
+  for (int i = 0; i < h->naddr; i++) {
+    if (want != PF_UNSPEC && want != h->addr[i].family)
+      continue; /* honor the requested family (v4/v6 only) */
+    struct addrinfo *const ai = mock_mkai(&h->addr[i]);
+
+    if (head == NULL)
+      head = ai;
+    else
+      tail->ai_next = ai;
+    tail = ai;
+  }
+  if (head == NULL)
+    return EAI_NONAME; /* filtered to empty, as the libc resolver does */
+  *res = head;
+  return 0;
+}
+
+static void mock_freeaddrinfo(struct addrinfo *res) {
+  while (res != NULL) {
+    struct addrinfo *const next = res->ai_next;
+
+    freet(res->ai_addr);
+    freet(res);
+    res = next;
+  }
+}
+
+static const hts_resolver_backend mock_backend = {mock_getaddrinfo,
+                                                  mock_freeaddrinfo};
+
+static int failures = 0;
+
+#define CHECK(cond)                                                            \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      failures++;                                                              \
+      fprintf(stderr, "dns-selftest: FAIL at %s:%d: %s\n", __FILE__, __LINE__, \
+              #cond);                                                          \
+    }                                                                          \
+  } while (0)
+
+/* Resolve via the uncached entry point; return the address family, or
+   AF_UNSPEC if the host did not resolve. */
+static int resolve_family_nocache(const char *host) {
+  SOCaddr addr;
+  const char *err = NULL;
+
+  if (hts_dns_resolve_nocache2(host, &addr, &err) == NULL)
+    return AF_UNSPEC;
+  return SOCaddr_sinfamily(addr);
+}
+
+int dns_selftests(httrackp *opt) {
+  failures = 0;
+  hts_dns_set_resolver_backend(&mock_backend);
+
+  /* IPv4-only / IPv6-only hosts map to the right family. */
+  IPV6_resolver = 0;
+  CHECK(resolve_family_nocache("v4only.test") == AF_INET);
+  CHECK(resolve_family_nocache("v6only.test") == AF_INET6);
+
+  /* Dual-stack: the single-address API returns the *first* resolved address.
+     Both orderings pin selection by position, not a family preference. The
+     multi-address API (resolve_all, below) exposes the whole list. */
+  CHECK(resolve_family_nocache("dual.test") == AF_INET6); /* v6 listed first */
+  CHECK(resolve_family_nocache("dual4.test") == AF_INET); /* v4 listed first */
+
+  /* Unknown host does not resolve. */
+  CHECK(resolve_family_nocache("nodns.test") == AF_UNSPEC);
+
+  /* Family filter (-@i4 / -@i6) selects v4 / v6 out of the dual-stack host. */
+  IPV6_resolver = 1;
+  CHECK(resolve_family_nocache("dual.test") == AF_INET);
+  IPV6_resolver = 2;
+  CHECK(resolve_family_nocache("dual.test") == AF_INET6);
+  IPV6_resolver = 0;
+
+  /* Cached driver resolves a host once and reuses the *same* address. */
+  mock_reset_calls();
+  {
+    SOCaddr a1, a2;
+    char ip1[64], ip2[64];
+    const char *err = NULL;
+
+    CHECK(hts_dns_resolve2(opt, "v4only.test", &a1, &err) != NULL);
+    CHECK(hts_dns_resolve2(opt, "v4only.test", &a2, &err) != NULL);
+    CHECK(mock_find("v4only.test")->calls == 1);
+    /* the cache returns the right address, not merely a hit for the key */
+    SOCaddr_inetntoa(ip1, sizeof(ip1), a1);
+    SOCaddr_inetntoa(ip2, sizeof(ip2), a2);
+    CHECK(strcmp(ip1, "1.2.3.4") == 0);
+    CHECK(strcmp(ip1, ip2) == 0);
+  }
+
+  /* A negative result is cached too: a second lookup does not re-resolve. */
+  {
+    SOCaddr a1, a2;
+    const char *err = NULL;
+
+    CHECK(hts_dns_resolve2(opt, "nodns.test", &a1, &err) == NULL);
+    CHECK(hts_dns_resolve2(opt, "nodns.test", &a2, &err) == NULL);
+    CHECK(mock_find("nodns.test")->calls == 1); /* resolved once, then cached */
+  }
+
+  /* Multi-address resolution: count and order are the connect-fallback
+     contract. A dead first address is retried against the next, so both must be
+     exact. */
+  mock_reset_calls();
+  {
+    SOCaddr addrs[HTS_MAXADDRNUM];
+    char ip[64];
+    const char *err = NULL;
+
+    /* dual-stack, in resolver order: [0]=v6, [1]=v4 */
+    CHECK(hts_dns_resolve_all(opt, "dual.test", addrs, HTS_MAXADDRNUM, &err) ==
+          2);
+    CHECK(SOCaddr_sinfamily(addrs[0]) == AF_INET6);
+    CHECK(SOCaddr_sinfamily(addrs[1]) == AF_INET);
+    SOCaddr_inetntoa(ip, sizeof(ip), addrs[1]);
+    CHECK(strcmp(ip, "5.6.7.8") == 0);
+    CHECK(mock_find("dual.test")->calls ==
+          1); /* one backend hit for the list */
+
+    /* single-address host: count 1 */
+    CHECK(hts_dns_resolve_all(opt, "v4only.test", addrs, HTS_MAXADDRNUM,
+                              &err) == 1);
+    SOCaddr_inetntoa(ip, sizeof(ip), addrs[0]);
+    CHECK(strcmp(ip, "1.2.3.4") == 0);
+
+    /* does-not-resolve: count 0 (negative), no addresses */
+    CHECK(hts_dns_resolve_all(opt, "nodns.test", addrs, HTS_MAXADDRNUM, &err) ==
+          0);
+
+    /* more than the cap: the kept list is clamped to HTS_MAXADDRNUM, keeping
+       the FIRST addresses in resolver order (not some other window) */
+    CHECK(hts_dns_resolve_all(opt, "many.test", addrs, HTS_MAXADDRNUM, &err) ==
+          HTS_MAXADDRNUM);
+    SOCaddr_inetntoa(ip, sizeof(ip), addrs[0]);
+    CHECK(strcmp(ip, "10.0.0.1") == 0);
+    SOCaddr_inetntoa(ip, sizeof(ip), addrs[HTS_MAXADDRNUM - 1]);
+    CHECK(strcmp(ip, "10.0.0.4") == 0);
+
+    /* family filter still applies through the list path */
+    IPV6_resolver = 1;
+    CHECK(hts_dns_resolve_all(opt, "dual4.test", addrs, HTS_MAXADDRNUM, &err) ==
+          1);
+    CHECK(SOCaddr_sinfamily(addrs[0]) == AF_INET);
+    IPV6_resolver = 0;
+  }
+
+  /* newhttp_addr() must connect to the addr_index-th address, not always the
+     first: this is what back_connect_next relies on to reach the fallback. */
+  {
+    htsblk r;
+    int count = -1;
+    T_SOC s;
+
+    hts_init_htsblk(&r);
+    s = newhttp_addr(opt, "dual.test", &r, 80, 0, 0, &count);
+    CHECK(count == 2);
+    CHECK(SOCaddr_sinfamily(r.address) == AF_INET6); /* index 0 = v6 */
+    if (s != INVALID_SOCKET)
+      deletesoc(s);
+
+    hts_init_htsblk(&r);
+    count = -1;
+    s = newhttp_addr(opt, "dual.test", &r, 80, 0, 1, &count);
+    CHECK(count == 2);
+    CHECK(SOCaddr_sinfamily(r.address) == AF_INET); /* index 1 = v4 */
+    if (s != INVALID_SOCKET)
+      deletesoc(s);
+
+    /* out-of-range index: no address selected (address stays unset) */
+    hts_init_htsblk(&r);
+    s = newhttp_addr(opt, "dual.test", &r, 80, 0, 2, NULL);
+    CHECK(s == INVALID_SOCKET);
+    if (s != INVALID_SOCKET)
+      deletesoc(s);
+  }
+
+  /* Connect-fallback decision (consumer of the multi-address list): when a
+     stuck connect should abandon the current address for the next one. */
+  {
+    /* no fallback for the last/only candidate, whatever the elapsed time */
+    CHECK(back_connect_fallback_due(0, 1, 9999, 120) == 0);
+    CHECK(back_connect_fallback_due(1, 2, 9999, 120) == 0);
+    CHECK(back_connect_fallback_due(3, 4, 9999, 120) == 0);
+    /* fallback available: wait the per-candidate deadline (cap 10s here) */
+    CHECK(back_connect_fallback_due(0, 2, 9, 120) == 0);
+    CHECK(back_connect_fallback_due(0, 2, 10, 120) == 1);
+    CHECK(back_connect_fallback_due(2, 4, 10, 120) == 1);
+    /* a shorter slot timeout shortens the deadline (min(timeout, cap)) */
+    CHECK(back_connect_fallback_due(0, 2, 4, 5) == 0);
+    CHECK(back_connect_fallback_due(0, 2, 5, 5) == 1);
+    /* no timeout management: never force a fallback */
+    CHECK(back_connect_fallback_due(0, 2, 9999, 0) == 0);
+  }
+
+  hts_dns_set_resolver_backend(NULL);
+  return failures;
+}
+
+#else
+
+int dns_selftests(httrackp *opt) {
+  (void) opt;
+  return 0; /* resolver seam only exists in the IPv6 build */
+}
+
+#endif

src/htsdns_selftest.h

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: htsdns_selftest.h                                      */
+/* Author: Xavier Roche                                         */
+/* ------------------------------------------------------------ */
+
+#ifndef HTSDNS_SELFTEST_DEFH
+#define HTSDNS_SELFTEST_DEFH
+
+#ifdef HTS_INTERNAL_BYTECODE
+
+#ifndef HTS_DEF_FWSTRUCT_httrackp
+#define HTS_DEF_FWSTRUCT_httrackp
+typedef struct httrackp httrackp;
+#endif
+
+/* Drive the DNS resolver and cache through a scripted (mock) getaddrinfo,
+   asserting address family, single-address selection, negative caching, the
+   IPv4/IPv6 family filter, and that a cached host is resolved only once.
+   Returns the number of failed checks (0 == success). */
+int dns_selftests(httrackp *opt);
+
+#endif
+
+#endif

src/htsencoding.c

@@ -30,12 +30,14 @@ Please visit our Website: http://www.httrack.com
 /* Author: Xavier Roche                                         */
 /* ------------------------------------------------------------ */
 
+#include <stdint.h>
+
 #include "htscharset.h"
 #include "htsencoding.h"
 #include "htssafe.h"
 
-/* static int decode_entity(const unsigned int hash, const size_t len);
-*/
+/* static int decode_entity(const uint64_t hash, const size_t len);
+ */
 #include "htsentities.h"
 
 /* hexadecimal conversion */
@@ -50,30 +52,31 @@ static int get_hex_value(char c) {
     return -1;
 }
 
-/* Numerical Recipes,
-   see <http://en.wikipedia.org/wiki/Linear_congruential_generator> */
-#define HASH_PRIME ( 1664525 )
-#define HASH_CONST ( 1013904223 )
-#define HASH_ADD(HASH, C) do {                  \
-    (HASH) *= HASH_PRIME;                       \
-    (HASH) += HASH_CONST;                       \
-    (HASH) += (C);                              \
-  } while(0)
+/* 64-bit FNV-1a; must match htsentities.sh, which keys the entity table on it.
+ */
+#define HASH_INIT 0xcbf29ce484222325ULL
+#define HASH_PRIME 0x100000001b3ULL
+#define HASH_ADD(HASH, C)                                                      \
+  do {                                                                         \
+    (HASH) ^= (unsigned char) (C);                                             \
+    (HASH) *= HASH_PRIME;                                                      \
+  } while (0)
 
 int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t max, const char *charset) {
   size_t i, j, ampStart, ampStartDest;
   int uc;
   int hex;
-  unsigned int hash;
+  uint64_t hash;
 
   assertf(max != 0);
-  for(i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0,
-        uc = -1, hex = 0, hash = 0 ; src[i] != '\0' ; i++) {
+  for (i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0, uc = -1, hex = 0,
+      hash = HASH_INIT;
+       src[i] != '\0'; i++) {
     /* start of entity */
     if (src[i] == '&') {
       ampStart = i;
       ampStartDest = j;
-      hash = 0;
+      hash = HASH_INIT;
       uc = -1;
     }
     /* inside a potential entity */
@@ -174,14 +177,11 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
       }
       /* alphanumerical entity */
       else {
-        /* alphanum and not too far ('&thetasym;' is the longest) */
-        if (i <= ampStart + 10 &&
-            (
-             (src[i] >= '0' && src[i] <= '9')
-             || (src[i] >= 'A' && src[i] <= 'Z')
-             || (src[i] >= 'a' && src[i] <= 'z')
-             )
-            ) {
+        /* alphanum, capped at the longest name
+         * '&CounterClockwiseContourIntegral;' (31) */
+        if (i <= ampStart + 31 && ((src[i] >= '0' && src[i] <= '9') ||
+                                   (src[i] >= 'A' && src[i] <= 'Z') ||
+                                   (src[i] >= 'a' && src[i] <= 'z'))) {
           /* compute hash */
           HASH_ADD(hash, (unsigned char) src[i]);
         } else {
@@ -190,9 +190,9 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
         }
       }
     }
-    
-    /* copy */
-    if (j + 1 > max) {
+
+    /* reserve one byte for the trailing NUL written after the loop */
+    if (j + 1 >= max) {
       /* overflow */
       return -1;
     }
@@ -300,6 +300,11 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
 
           /* Was the character read successfully ? */
           if (nRead == utfBufferSize) {
+            /* the 'continue' below skips the NUL-reserve guard: re-check */
+            if (utfBufferJ + utfBufferSize >= max) {
+              return -1;
+            }
+
             /* Rollback write position to sequence start write position */
             j = utfBufferJ;
 
@@ -314,8 +319,8 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
       }
     }
 
-    /* Check for overflow */
-    if (j + 1 > max) {
+    /* reserve one byte for the trailing NUL written after the loop */
+    if (j + 1 >= max) {
       return -1;
     }
 

src/htsentities.sh

@@ -1,75 +1,92 @@
 #!/bin/bash
 #
+# Regenerate htsentities.h from the WHATWG named character references.
 
-src=html40.txt
-url=http://www.w3.org/TR/1998/REC-html40-19980424/html40.txt
+set -euo pipefail
+
+src=entities.json
+url=https://html.spec.whatwg.org/entities.json
 dest=htsentities.h
 
-(
-    cat <<EOF
-/*
-  -- ${dest} --
-  FILE GENERATED BY $0, DO NOT MODIFY
+# 64-bit FNV-1a of $1, printed as a C constant. Must match the hash in
+# htsencoding.c. The offset basis is stored as its wrapped (signed) bit pattern;
+# bash arithmetic is 64-bit two's complement, so the result is bit-exact.
+fnv1a() {
+    local s=$1 i c h=$((0xcbf29ce484222325))
+    for ((i = 0; i < ${#s}; i++)); do
+        printf -v c '%d' "'${s:i:1}"
+        h=$(((h ^ (c & 0xff)) * 0x100000001b3))
+    done
+    printf '0x%016xULL' "$h"
+}
 
-  We compute the LCG hash
-  (see <http://en.wikipedia.org/wiki/Linear_congruential_generator>)
-  for each entity. We should in theory check using strncmp() that we
-  actually have the correct entity, but this is actually statistically
-  not needed.
+if [ ! -f "$src" ]; then
+    curl -fsS "$url" -o "$src"
+fi
 
-  We may want to do better, but we expect the hash function to be uniform, and
-  let the compiler be smart enough to optimize the switch (for example by
-  checking in log2() intervals)
-  
-  This code has been generated using the evil $0 script.
-*/
+# Keep ';'-terminated single-codepoint names; the ~93 multi-codepoint refs can't
+# fit decode_entity's single-codepoint return and are skipped (left verbatim).
+pairs=$(jq -r '
+    to_entries
+    | map(select((.key | endswith(";")) and (.value.codepoints | length == 1)))
+    | sort_by(.key)
+    | .[] | "\(.key | ltrimstr("&") | rtrimstr(";"))\t\(.value.codepoints[0])"' "$src")
 
-static int decode_entity(const unsigned int hash, const size_t len) {
+# Skipped multi-codepoint names, kept to prove none aliases an emitted hash.
+skipped=$(jq -r '
+    to_entries
+    | map(select((.key | endswith(";")) and (.value.codepoints | length > 1)))
+    | .[] | .key | ltrimstr("&") | rtrimstr(";")' "$src")
+
+cases=""
+emit_hashes=""
+while IFS=$'\t' read -r name cp; do
+    hash=$(fnv1a "$name")
+    cases+="    /* $name */"$'\n'
+    cases+="  case $hash:"$'\n'
+    cases+="    if (len == ${#name}) {"$'\n'
+    cases+="      return $cp;"$'\n'
+    cases+="    }"$'\n'
+    cases+="    break;"$'\n'
+    emit_hashes+="$hash"$'\n'
+done <<<"$pairs"
+
+skip_hashes=""
+while IFS= read -r name; do
+    [ -n "$name" ] && skip_hashes+="$(fnv1a "$name")"$'\n'
+done <<<"$skipped"
+
+# The switch keys on the hash alone, so the dispatch is correct only while every
+# emitted name hashes uniquely; prove it here, no runtime name compare needed.
+dups=$(printf '%s' "$emit_hashes" | sort | uniq -d || true)
+if [ -n "$dups" ]; then
+    echo "FATAL: two entity names share a hash (duplicate switch case); change the hash:" >&2
+    echo "$dups" >&2
+    exit 1
+fi
+# A skipped name colliding with an emitted hash would mis-decode instead of
+# staying verbatim; forbid that too.
+aliased=$(comm -12 <(printf '%s' "$emit_hashes" | sort -u) <(printf '%s' "$skip_hashes" | sort -u) || true)
+if [ -n "$aliased" ]; then
+    echo "FATAL: a skipped multi-codepoint name aliases an emitted hash:" >&2
+    echo "$aliased" >&2
+    exit 1
+fi
+
+cat >"$dest" <<EOF
+/* GENERATED by $0 from the WHATWG named character references
+   (${url}). DO NOT EDIT.
+   Dispatch keys on a 64-bit FNV-1a hash of the entity name; the generator
+   aborts on any hash collision, so no runtime name compare is needed. */
+
+#include <stdint.h>
+
+static int decode_entity(const uint64_t hash, const size_t len) {
   switch(hash) {
-EOF
-    (
-        if test -f ${src}; then
-            cat ${src}
-        else
-            GET "${url}"
-        fi
-    ) \
-        | grep -E '^<!ENTITY [a-zA-Z0-9_]' \
-        | sed \
-        -e 's/<!ENTITY //' -e "s/[[:space:]][[:space:]]*/ /g" \
-        -e 's/-->$//' \
-        -e 's/\([^ ]*\) CDATA "&#\([^\"]*\);" -- \(.*\)/\1 \2 \3/'\
-| ( \
-        read A
-        while test -n "$A"; do
-            ent="${A%% *}"
-            code=$(echo "$A"|cut -f2 -d' ')
-            # compute hash
-            hash=0
-            i=0
-            a=1664525
-            c=1013904223
-            m="$[1 << 32]"
-            while test "$i" -lt ${#ent}; do
-                d="$(echo -n "${ent:${i}:1}"|hexdump -v -e '/1 "%d"')"
-                hash="$[((${hash}*${a})%(${m})+${d}+${c})%(${m})]"
-                i=$[${i}+1]
-            done
-            echo -e "    /* $A */"
-            echo -e "  case ${hash}u:"
-            echo -e "    if (len == ${#ent} /* && strncmp(ent, \"${ent}\") == 0 */) {"
-            echo -e "      return ${code};"
-            echo -e "    }"
-            echo -e "    break;"
-
-            # next
-            read A
-        done
-    )
-    cat <<EOF
-  }
+${cases}  }
   /* unknown */
   return -1;
 }
 EOF
-) > ${dest}
+
+echo "wrote $dest ($(grep -c '^  case ' "$dest") entities)" >&2

src/htsfilters.c

@@ -76,7 +76,8 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
     }
     if (size)
       sz = *size;
-    if (strjoker(nom, filters[i] + filteroffs, &sz, size_flag)) {       // reconnu
+    /* size unknown (scan time): no size pointer => size tests stay neutral */
+    if (strjoker(nom, filters[i] + filteroffs, size ? &sz : NULL, size_flag)) {
       if (size)
         if (sz != *size)
           sizelimit = sz;
@@ -101,9 +102,8 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
 // cet algo est 'un peu' récursif mais ne consomme pas trop de tm
 // * = toute lettre
 // --?-- : spécifique à HTTrack et aux ?
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
-                          int *size_flag) {
-  //int err=0;
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
+                                LLint *size, int *size_flag) {
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -192,7 +192,12 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
           int len = (int) strlen(joker);
 
           while((joker[i] != RIGHT) && (joker[i]) && (i < len)) {
-            if ((joker[i] == '<') || (joker[i] == '>')) {       // *[<10]
+            // '\' escapes the next char as a literal member, e.g. *[\[\]]
+            if (joker[i] == '\\' && joker[i + 1] != '\0') {
+              i++;
+              pass[(int) (unsigned char) joker[i]] = 1;
+              i++;
+            } else if ((joker[i] == '<') || (joker[i] == '>')) { // *[<10]
               int lsize = 0;
               int lverdict;
 
@@ -220,7 +225,9 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
                 while(isdigit((unsigned char) joker[i]))
                   i++;
               }
-            } else if (joker[i + 1] == '-') {   // 2 car, ex: *[A-Z]
+            } else if (joker[i + 1] == '-' && joker[i + 2] != '\0') {
+              // range *[A-Z]; the '\0' guard rejects a truncated *[a- (else
+              // i+=3 overshoots the NUL)
               if ((int) (unsigned char) joker[i + 2] >
                   (int) (unsigned char) joker[i]) {
                 int j;
@@ -228,14 +235,9 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
                 for(j = (int) (unsigned char) joker[i];
                     j <= (int) (unsigned char) joker[i + 2]; j++)
                   pass[j] = 1;
-
               }
-              // else err=1;
               i += 3;
-            } else {            // 1 car, ex: *[ ]
-              if (joker[i + 2] == '\\' && joker[i + 3] != 0) {  // escaped char, such as *[\[] or *[\]]
-                i++;
-              }
+            } else { // 1 car, ex: *[ ]
               pass[(int) (unsigned char) joker[i]] = 1;
               i++;
             }
@@ -257,7 +259,6 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       for(i = 0; i < 256; i++)
         pass[i] = 1;            // tout autoriser
       jmp = 1;
-      ////if (joker[2]==LEFT) jmp=3;        // permet de recher *<crochet ouvrant>
     }
 
     {
@@ -285,7 +286,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       if (!unique)
         max = (int) strlen(chaine);
       else                      /* *(a) only match a (not aaaaa) */
-        max = 1;
+        max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
           if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {

src/htsftp.c

@@ -54,15 +54,9 @@ Please visit our Website: http://www.httrack.com
 #endif
 
 // ftp mode passif
-// #if HTS_INET6==0
 #define FTP_PASV 1
-// #else
-// no passive mode for v6
-// #define FTP_PASV 0
-// #endif
 
 #define FTP_DEBUG 0
-//#define FORK_DEBUG 0
 
 #if USE_BEGINTHREAD
 
@@ -128,6 +122,33 @@ void launch_ftp(FTPDownloadStruct * params) {
   return 0; \
   }
 
+/* Bounded split of a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
+void ftp_split_userpass(const char *src, const char *end, char *user,
+                        size_t user_size, char *pass, size_t pass_size) {
+  size_t n = 0;
+
+  assertf(user_size > 0 && pass_size > 0); /* the size-1 math underflows on 0 */
+
+  while (src[n] != '\0' && src[n] != ':') {
+    if (n < user_size - 1)
+      user[n] = src[n];
+    n++;
+  }
+  user[n < user_size ? n : user_size - 1] = '\0';
+  pass[0] = '\0';
+  if (src[n] == ':') { // password follows the colon
+    const size_t base = n + 1;
+    size_t k = 0;
+
+    while (&src[base + k + 1] < end && src[base + k] != '\0') {
+      if (k < pass_size - 1)
+        pass[k] = src[base + k];
+      k++;
+    }
+    pass[k < pass_size ? k : pass_size - 1] = '\0';
+  }
+}
+
 // la véritable fonction une fois lancées les routines thread/fork
 int run_launch_ftp(FTPDownloadStruct * pStruct) {
   lien_back *back = pStruct->pBack;
@@ -173,43 +194,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   while(*real_adr == '/')
     real_adr++;                 // sauter /
   if ((adr = jump_identification(real_adr)) != real_adr) {      // user
-    int i = -1;
-
-    pass[0] = '\0';
-    do {
-      i++;
-      user[i] = real_adr[i];
-    } while((real_adr[i] != ':') && (real_adr[i]));
-    user[i] = '\0';
-    if (real_adr[i] == ':') {   // pass
-      int j = -1;
-
-      i++;                      // oui on saute aussi le :
-      do {
-        j++;
-        pass[j] = real_adr[i + j];
-      } while(((&real_adr[i + j + 1]) < adr) && (real_adr[i + j]));
-      pass[j] = '\0';
-    }
+    ftp_split_userpass(real_adr, adr, user, sizeof(user), pass, sizeof(pass));
   }
   // Calculer RETR <nom>
   {
     char *a;
 
-#if 0
-    a = back->url_fil + strlen(back->url_fil) - 1;
-    while((a > back->url_fil) && (*a != '/'))
-      a--;
-    if (*a != '/') {
-      a = NULL;
-    }
-#else
     a = back->url_fil;
-#endif
     if (a != NULL && *a != '\0') {
-#if 0
-      a++;                      // sauter /
-#endif
       ftp_filename = a;
       if (strnotempty(a)) {
         char catbuff[CATBUFF_SIZE];
@@ -233,7 +225,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
       }
     } else {
       strcpybuff(back->r.msg, "Unexpected PORT error");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
     }
   }
@@ -267,7 +258,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
       snprintf(back->r.msg, sizeof(back->r.msg),
                "Unable to get server's address: %s", error);
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_NON_FATAL;
       _HALT_FTP return 0;
     }
@@ -280,20 +270,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
     if (soc_ctl == INVALID_SOCKET) {
       strcpybuff(back->r.msg, "Unable to create a socket");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
     }
 
     SOCaddr_initport(server, port);
-    // server.sin_port = htons((unsigned short int) port);
 
     // connexion (bloquante, on est en thread)
     strcpybuff(back->info, "connect");
 
     if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
       strcpybuff(back->r.msg, "Unable to connect to the server");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
 #ifdef _WIN32
@@ -335,64 +322,22 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               // ok
             } else {
               strcpybuff(back->r.msg, "TYPE I error");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
-#if 0
-            // --CWD--
-            char *a;
-
-            a = back->url_fil + strlen(back->url_fil) - 1;
-            while((a > back->url_fil) && (*a != '/'))
-              a--;
-            if (*a == '/') {    // ok repéré
-              char BIGSTK target[1024];
-
-              target[0] = '\0';
-              strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
-              if (strnotempty(target) == 0)
-                strcatbuff(target, "/");
-              strcpybuff(back->info, "cwd");
-              snprintf(line, sizeof(line), "CWD %s", target);
-              send_line(soc_ctl, line);
-              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-              _CHECK_HALT_FTP;
-              if (line[0] == '2') {
-                send_line(soc_ctl, "TYPE I");
-                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-                _CHECK_HALT_FTP;
-                if (line[0] == '2') {
-                  // ok..
-                } else {
-                  strcpybuff(back->r.msg, "TYPE I error");
-                  // back->status=STATUS_FTP_READY;    // fini
-                  back->r.statuscode = STATUSCODE_INVALID;
-                }
-              } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
-                // back->status=STATUS_FTP_READY;    // fini
-                back->r.statuscode = STATUSCODE_INVALID;
-              }                 // sinon on est prêts
-            } else {
-              strcpybuff(back->r.msg, "Unexpected ftp error");
-              // back->status=STATUS_FTP_READY;    // fini
-              back->r.statuscode = STATUSCODE_INVALID;
-            }
-#endif
 
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
-          // back->status=STATUS_FTP_READY;    // fini
+          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
+                   linejmp(line));
           back->r.statuscode = STATUSCODE_INVALID;
         }
       } else {
-        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
-        // back->status=STATUS_FTP_READY;    // fini
+        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
+                 linejmp(line));
         back->r.statuscode = STATUSCODE_INVALID;
       }
 
@@ -459,8 +404,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
             // -- fin analyse de l'adresse IP et du port --
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
@@ -491,13 +436,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 }
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
-              // back->status=STATUS_FTP_READY;    // fini
+              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
+                       linejmp(line));
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         }
@@ -603,9 +548,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   deletesoc(soc_dat);
                   soc_dat = INVALID_SOCKET;
                   //
-                  snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
-                          linejmp(line));
-                  // back->status=STATUS_FTP_READY;    // fini
+                  snprintf(back->r.msg, sizeof(back->r.msg),
+                           "RETR command error: %s", linejmp(line));
                   back->r.statuscode = STATUSCODE_INVALID;
                 }               // sinon on est prêts
               } else {
@@ -616,23 +560,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 soc_dat = INVALID_SOCKET;
                 //
                 strcpybuff(back->r.msg, "Unable to connect");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }                 // sinon on est prêts
             } else {
               strcpybuff(back->r.msg, "Unable to create a socket");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }                   // sinon on est prêts
           } else {
             snprintf(back->r.msg, sizeof(back->r.msg),
                      "Unable to resolve IP %s: %s", adr_ip, error);
-            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
-          // back->status=STATUS_FTP_READY;    // fini
+          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
+                   linejmp(line));
           back->r.statuscode = STATUSCODE_INVALID;
         }                       // sinon on est prêts
 #else
@@ -653,17 +594,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               //T_SOC soc_dat;
               if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
                 strcpybuff(back->r.msg, "Unable to accept connection");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
-              // back->status=STATUS_FTP_READY;    // fini
+              snprintf(back->r.msg, sizeof(back->r.msg),
+                       "RETR command error: %s", linejmp(line));
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -673,7 +613,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 #endif
         } else {
           strcpybuff(back->r.msg, "Unable to listen to a port");
-          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
 #endif
@@ -697,21 +636,18 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             int len = 1;
             int read_len = 1024;
 
-            //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
-
             while((len > 0) && (!stop_ftp(back))) {
               // attendre les données
               len = 1;          // pas d'erreur pour le moment
               switch (wait_socket_receive(soc_dat, timeout)) {
               case -1:
                 strcpybuff(back->r.msg, "FTP read error");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
               case 0:
-                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
-                // back->status=STATUS_FTP_READY;    // fini
+                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
+                         timeout);
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
@@ -733,17 +669,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                          }
                        */
                       strcpybuff(back->r.msg, "Write error");
-                      // back->status=STATUS_FTP_READY;    // fini
                       back->r.statuscode = STATUSCODE_INVALID;
                       len = 0;  // error
                     }
                   } else {
                     strcpybuff(back->r.msg, "Unexpected write error");
-                    // back->status=STATUS_FTP_READY;    // fini
                     back->r.statuscode = STATUSCODE_INVALID;
                   }
-                } else {        // Erreur ou terminé
-                  // back->status=STATUS_FTP_READY;    // fini
+                } else { // Erreur ou terminé
                   back->r.statuscode = 0;
                   if (back->r.totalsize > 0
                       && back->r.size != back->r.totalsize) {
@@ -752,7 +685,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   }
                 }
                 read_len = 1024;
-                //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
               }
             }
             if (back->r.fp) {
@@ -761,7 +693,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
           } else {
             strcpybuff(back->r.msg, "Unable to write file");
-            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -777,16 +708,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               get_ftp_line(soc_ctl, line, sizeof(line), timeout);
               if (line[0] == '2') {     // OK
                 strcpybuff(back->r.msg, "OK");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = HTTP_OK;
               } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
-                // back->status=STATUS_FTP_READY;    // fini
+                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
+                         linejmp(line));
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
               strcpybuff(back->r.msg, "FTP read error");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           }
@@ -812,7 +741,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     back->r.statuscode = HTTP_OK;
     strcpybuff(back->r.msg, "OK");
   }
-  // back->status=STATUS_FTP_READY;    // fini
   return 0;
 }
 
@@ -979,13 +907,11 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
       break;
     }
 
-    //HTS_TOTAL_RECV_CHECK(dummy);     // Diminuer au besoin si trop de données reçues
     switch (recv(soc, &b, 1, 0)) {
-      //case 0: break;    // pas encore --> erreur (on attend)!
     case 1:
       HTS_STAT.HTS_TOTAL_RECV += 1;     // compter flux entrant
-      if ((b != 10) && (b != 13))
-        data[i++] = b;
+      if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
+        data[i++] = b; // truncate hostile over-long reply lines
       break;
     default:
       if (ptrline)
@@ -1108,7 +1034,6 @@ int wait_socket_receive(T_SOC soc, int timeout) {
 int stop_ftp(lien_back * back) {
   if (back->stop_ftp) {
     strcpybuff(back->r.msg, "Cancelled by User");
-    // back->status=STATUS_FTP_READY;    // fini
     back->r.statuscode = STATUSCODE_INVALID;
     return 1;
   }

src/htsftp.h

@@ -70,6 +70,11 @@ int back_launch_ftp(FTPDownloadStruct * params);
 int run_launch_ftp(FTPDownloadStruct * params);
 int send_line(T_SOC soc, const char *data);
 int get_ftp_line(T_SOC soc, char *line, size_t line_size, int timeout);
+/* Split a "user[:pass]@" prefix (end = jump_identification result) into
+   bounded, NUL-terminated user/pass buffers, truncating to fit.
+   Both sizes must be nonzero. */
+void ftp_split_userpass(const char *src, const char *end, char *user,
+                        size_t user_size, char *pass, size_t pass_size);
 T_SOC get_datasocket(char *to_send, size_t to_send_size);
 int stop_ftp(lien_back * back);
 char *linejmp(char *line);

src/htsglobal.h

@@ -43,10 +43,10 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION      "3.49-8"
-#define HTTRACK_VERSIONID    "3.49.8"
-#define HTTRACK_AFF_VERSION  "3.x"
-#define HTTRACK_LIB_VERSION  "2.0"
+#define HTTRACK_VERSION "3.49-11"
+#define HTTRACK_VERSIONID "3.49.11"
+#define HTTRACK_AFF_VERSION "3.x"
+#define HTTRACK_LIB_VERSION "2.0"
 
 #ifndef HTS_NOINCLUDES
 #include <stdio.h>
@@ -71,11 +71,11 @@ Please visit our Website: http://www.httrack.com
    varargs starting at arg. */
 #ifndef HTS_UNUSED
 #ifdef __GNUC__
-#define HTS_UNUSED __attribute__ ((unused))
+#define HTS_UNUSED __attribute__((unused))
 
-#define HTS_STATIC static __attribute__ ((unused))
+#define HTS_STATIC static __attribute__((unused))
 
-#define HTS_PRINTF_FUN(fmt, arg) __attribute__ ((format (printf, fmt, arg)))
+#define HTS_PRINTF_FUN(fmt, arg) __attribute__((format(printf, fmt, arg)))
 #else
 #define HTS_UNUSED
 #define HTS_STATIC static
@@ -113,7 +113,7 @@ Please visit our Website: http://www.httrack.com
 
 #ifndef HTS_LONGLONG
 #ifdef SIZEOF_LONG_LONG
-#if SIZEOF_LONG_LONG==8
+#if SIZEOF_LONG_LONG == 8
 #define HTS_LONGLONG 1
 #endif
 #endif
@@ -204,12 +204,12 @@ Please visit our Website: http://www.httrack.com
 #endif
 
 #define HTS_HTTRACKRC ".httrackrc"
-#define HTS_HTTRACKCNF HTS_ETCPATH"/httrack.conf"
+#define HTS_HTTRACKCNF HTS_ETCPATH "/httrack.conf"
 
 #ifdef DATADIR
-#define HTS_HTTRACKDIR DATADIR"/httrack/"
+#define HTS_HTTRACKDIR DATADIR "/httrack/"
 #else
-#define HTS_HTTRACKDIR HTS_PREFIX"/share/httrack/"
+#define HTS_HTTRACKDIR HTS_PREFIX "/share/httrack/"
 #endif
 
 #endif
@@ -226,12 +226,21 @@ Please visit our Website: http://www.httrack.com
 
 /* Copyright (C) 1998 Xavier Roche and other contributors */
 #define HTTRACK_AFF_AUTHORS "[XR&CO'2014]"
-#define HTS_DEFAULT_FOOTER "<!-- Mirrored from %s%s by HTTrack Website Copier/" HTTRACK_AFF_VERSION " " HTTRACK_AFF_AUTHORS ", %s -->"
+#define HTS_DEFAULT_FOOTER                                                     \
+  "<!-- Mirrored from %s%s by HTTrack Website Copier/" HTTRACK_AFF_VERSION     \
+  " " HTTRACK_AFF_AUTHORS ", %s -->"
+/* Honest crawler User-Agent; no fake OS/browser to go stale. */
+#define HTS_DEFAULT_USER_AGENT                                                 \
+  "Mozilla/5.0 (compatible; HTTrack/" HTTRACK_AFF_VERSION                      \
+  "; +https://www.httrack.com/)"
 #define HTTRACK_WEB "http://www.httrack.com"
-#define HTS_UPDATE_WEBSITE "http://www.httrack.com/update.php3?Product=HTTrack&Version=" HTTRACK_VERSIONID "&VersionStr=" HTTRACK_VERSION "&Platform=%d&Language=%s"
+#define HTS_UPDATE_WEBSITE                                                     \
+  "http://www.httrack.com/"                                                    \
+  "update.php3?Product=HTTrack&Version=" HTTRACK_VERSIONID                     \
+  "&VersionStr=" HTTRACK_VERSION "&Platform=%d&Language=%s"
 
 #define H_CRLF "\x0d\x0a"
-#define CRLF   "\x0d\x0a"
+#define CRLF "\x0d\x0a"
 #ifdef _WIN32
 #define LF "\x0d\x0a"
 #else
@@ -242,18 +251,29 @@ Please visit our Website: http://www.httrack.com
 #define HTS_NOPARAM "(none)"
 #define HTS_NOPARAM2 "\"(none)\""
 
-/* Boolean flag for option fields and API yes/no returns. An enum (not C bool)
-   so it stays int-sized: option fields keep the httrackp layout/ABI, and a
-   return type stays compatible with the int it replaces. */
+/* Boolean flag for option fields and API yes/no returns. Int-backed, not an
+   enum: an enum makes C++ reject `field = 1` / `f(0)` on the exported fields
+   and params. Int-sized, so the httrackp layout and the ABI are unchanged. */
 #ifndef HTS_DEF_DEFSTRUCT_hts_boolean
 #define HTS_DEF_DEFSTRUCT_hts_boolean
-typedef enum hts_boolean { HTS_FALSE = 0, HTS_TRUE = 1 } hts_boolean;
+
+typedef int hts_boolean;
+#define HTS_FALSE 0
+#define HTS_TRUE 1
+#endif
+
+#ifndef HTS_DEF_DEFSTRUCT_hts_tristate
+#define HTS_DEF_DEFSTRUCT_hts_tristate
+/* Tri-state hts_boolean: HTS_DEFAULT (-1) = "unspecified" (copy_htsopt leaves
+   the target untouched); HTS_FALSE/HTS_TRUE = off/on. */
+typedef int hts_tristate;
+#define HTS_DEFAULT (-1)
 #endif
 
 /* Larger/smaller of two values. Macros: arguments are evaluated twice. */
-#define maximum(A,B) ( (A) > (B) ? (A) : (B) )
+#define maximum(A, B) ((A) > (B) ? (A) : (B))
 
-#define minimum(A,B) ( (A) < (B) ? (A) : (B) )
+#define minimum(A, B) ((A) < (B) ? (A) : (B))
 
 /* True when A is a non-NULL, non-empty string. */
 #define strnotempty(A) (((A) != NULL && (A)[0] != '\0'))
@@ -278,10 +298,10 @@ typedef enum hts_boolean { HTS_FALSE = 0, HTS_TRUE = 1 } hts_boolean;
 #endif
 #else
 /* See <http://gcc.gnu.org/wiki/Visibility> */
-#if ( ( defined(__GNUC__) && ( __GNUC__ >= 4 ) ) \
-      || ( defined(HAVE_VISIBILITY) && HAVE_VISIBILITY ) )
+#if ((defined(__GNUC__) && (__GNUC__ >= 4)) ||                                 \
+     (defined(HAVE_VISIBILITY) && HAVE_VISIBILITY))
 
-#define HTSEXT_API __attribute__ ((visibility ("default")))
+#define HTSEXT_API __attribute__((visibility("default")))
 #else
 #define HTSEXT_API
 #endif
@@ -335,8 +355,8 @@ typedef __int64 LLint;
 typedef __int64 TStamp;
 
 #define LLintP "%I64d"
-#elif (defined(_LP64) || defined(__x86_64__) \
-       || defined(__powerpc64__) || defined(__64BIT__))
+#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
+       defined(__64BIT__))
 
 typedef long int LLint;
 
@@ -392,6 +412,10 @@ typedef int T_SOC;
 /* Buffer size for a printed network address (IPv4 or IPv6, NUL included). */
 #define HTS_MAXADDRLEN 64
 
+/* Max resolved addresses kept per host for connect fallback (dead IPv6 etc.).
+ */
+#define HTS_MAXADDRNUM 4
+
 #ifdef _WIN32
 #else
 #define __cdecl
@@ -400,16 +424,17 @@ typedef int T_SOC;
 /* Permission bits for created folders and files (mkdir and chmod).
    PROTECT_FOLDER is owner-only. With HTS_ACCESS set (the default) the ACCESS_
    modes also grant group/other read; otherwise they stay owner-only. */
-#define HTS_PROTECT_FOLDER (S_IRUSR|S_IWUSR|S_IXUSR)
+#define HTS_PROTECT_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
 
 #if HTS_ACCESS
-#define HTS_ACCESS_FILE (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)
+#define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)
 
-#define HTS_ACCESS_FOLDER (S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH)
+#define HTS_ACCESS_FOLDER                                                      \
+  (S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
 #else
-#define HTS_ACCESS_FILE (S_IRUSR|S_IWUSR)
+#define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR)
 
-#define HTS_ACCESS_FOLDER (S_IRUSR|S_IWUSR|S_IXUSR)
+#define HTS_ACCESS_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
 #endif
 
 /* Sanity-check that the required preprocessor switches are defined */
@@ -427,7 +452,11 @@ typedef int T_SOC;
 #endif
 
 /* fflush sur stdout */
-#define io_flush { fflush(stdout); fflush(stdin); }
+#define io_flush                                                               \
+  {                                                                            \
+    fflush(stdout);                                                            \
+    fflush(stdin);                                                             \
+  }
 
 /* HTSLib */
 
@@ -447,7 +476,7 @@ typedef int T_SOC;
 
 #ifdef _DEBUG
 // trace mallocs
-//#define HTS_TRACE_MALLOC
+// #define HTS_TRACE_MALLOC
 #ifdef HTS_TRACE_MALLOC
 typedef unsigned long int t_htsboundary;
 
@@ -524,7 +553,13 @@ static const t_htsboundary htsboundary = 0xDEADBEEF;
 #if _HTS_WIDE
 extern FILE *DEBUG_fp;
 
-#define DEBUG_W(A)  { if (DEBUG_fp==NULL) DEBUG_fp=fopen("bug.out","wb"); fprintf(DEBUG_fp,":>"A); fflush(DEBUG_fp); }
+#define DEBUG_W(A)                                                             \
+  {                                                                            \
+    if (DEBUG_fp == NULL)                                                      \
+      DEBUG_fp = fopen("bug.out", "wb");                                       \
+    fprintf(DEBUG_fp, ":>" A);                                                 \
+    fflush(DEBUG_fp);                                                          \
+  }
 #undef _
 #define _ ,
 #endif

src/htshash.c

@@ -106,10 +106,10 @@ static coucal_hashkeys key_adrfil_hashes_generic(void *arg,
   const lien_url*const lien = (const lien_url*) value;
   const char *const adr = !former ? lien->adr : lien->former_adr;
   const char *const fil = !former ? lien->fil : lien->former_fil;
-  const char *const adr_norm = adr != NULL ? 
-    ( hash->normalized  ? jump_normalized_const(adr)
-                        : jump_identification_const(adr) )
-    : NULL;
+  const char *const adr_norm =
+      adr != NULL ? (hash->norm_host ? jump_normalized_const(adr)
+                                     : jump_identification_const(adr))
+                  : NULL;
 
   // copy address
   assertf(adr_norm != NULL);
@@ -117,10 +117,18 @@ static coucal_hashkeys key_adrfil_hashes_generic(void *arg,
 
   // copy link
   assertf(fil != NULL);
-  if (hash->normalized) {
-    fil_normalized(fil, &hash->normfil[strlen(hash->normfil)]);
-  } else {
-    strcpy(&hash->normfil[strlen(hash->normfil)], fil);
+  {
+    /* resolve the per-URL strip keys; strip applies even when urlhack is off */
+    char BIGSTK keybuf[HTS_URLMAXSIZE];
+    const char *const keys = hts_query_strip_keys(hash->strip_query, adr, fil,
+                                                  keybuf, sizeof(keybuf));
+
+    if (hash->norm_slash || hash->norm_query || keys != NULL) {
+      fil_normalized_filtered_ex(fil, &hash->normfil[strlen(hash->normfil)],
+                                 keys, hash->norm_slash, hash->norm_query);
+    } else {
+      strcpy(&hash->normfil[strlen(hash->normfil)], fil);
+    }
   }
 
   // hash
@@ -132,8 +140,7 @@ static int key_adrfil_equals_generic(void *arg,
                                      coucal_key_const a_,
                                      coucal_key_const b_, 
                                      const int former) {
-  hash_struct *const hash = (hash_struct*) arg;
-  const int normalized = hash->normalized;
+  hash_struct *const hash = (hash_struct *) arg;
   const lien_url*const a = (const lien_url*) a_;
   const lien_url*const b = (const lien_url*) b_;
   const char *const a_adr = !former ? a->adr : a->former_adr;
@@ -150,10 +157,10 @@ static int key_adrfil_equals_generic(void *arg,
   assertf(b_fil != NULL);
 
   // skip scheme and authentication to the domain (possibly without www.)
-  ja = normalized
-    ? jump_normalized_const(a_adr) : jump_identification_const(a_adr);
-  jb = normalized
-    ? jump_normalized_const(b_adr) : jump_identification_const(b_adr);
+  ja = hash->norm_host ? jump_normalized_const(a_adr)
+                       : jump_identification_const(a_adr);
+  jb = hash->norm_host ? jump_normalized_const(b_adr)
+                       : jump_identification_const(b_adr);
   assertf(ja != NULL);
   assertf(jb != NULL);
   if (strcasecmp(ja, jb) != 0) {
@@ -161,12 +168,23 @@ static int key_adrfil_equals_generic(void *arg,
   }
 
   // now compare pathes
-  if (normalized) {
-    fil_normalized(a_fil, hash->normfil);
-    fil_normalized(b_fil, hash->normfil2);
-    return strcmp(hash->normfil, hash->normfil2) == 0;
-  } else {
-    return strcmp(a_fil, b_fil) == 0;
+  {
+    char BIGSTK ka[HTS_URLMAXSIZE], kb[HTS_URLMAXSIZE];
+    const char *const keysa =
+        hts_query_strip_keys(hash->strip_query, a_adr, a_fil, ka, sizeof(ka));
+    const char *const keysb =
+        hts_query_strip_keys(hash->strip_query, b_adr, b_fil, kb, sizeof(kb));
+
+    if (hash->norm_slash || hash->norm_query || keysa != NULL ||
+        keysb != NULL) {
+      fil_normalized_filtered_ex(a_fil, hash->normfil, keysa, hash->norm_slash,
+                                 hash->norm_query);
+      fil_normalized_filtered_ex(b_fil, hash->normfil2, keysb, hash->norm_slash,
+                                 hash->norm_query);
+      return strcmp(hash->normfil, hash->normfil2) == 0;
+    } else {
+      return strcmp(a_fil, b_fil) == 0;
+    }
   }
 }
 
@@ -222,11 +240,17 @@ static int key_former_adrfil_equals(void *arg,
   return key_adrfil_equals_generic(arg, a, b, 1);
 }
 
-void hash_init(httrackp *opt, hash_struct * hash, int normalized) {
+void hash_init(httrackp *opt, hash_struct *hash, hts_boolean normalized) {
   hash->sav = coucal_new(0);
   hash->adrfil = coucal_new(0);
   hash->former_adrfil = coucal_new(0);
-  hash->normalized = normalized;
+  /* urlhack is the umbrella; per-feature negatives opt out of each part */
+  hash->norm_host = normalized && !opt->no_www_dedup;
+  hash->norm_slash = normalized && !opt->no_slash_dedup;
+  hash->norm_query = normalized && !opt->no_query_dedup;
+  /* snapshot the query-strip list (not owned; valid for the hash lifetime) */
+  hash->strip_query =
+      StringNotEmpty(opt->strip_query) ? StringBuff(opt->strip_query) : NULL;
 
   hts_set_hash_handler(hash->sav, opt);
   hts_set_hash_handler(hash->adrfil, opt);
@@ -282,6 +306,26 @@ void hash_free(hash_struct *hash) {
   }
 }
 
+/* Test helper: do the two URLs dedupe to the same key under opt's urlhack
+   flags? Exercises the live hash compare (norm_host/slash/query resolution). */
+hts_boolean hash_url_equals(httrackp *opt, const char *adra, const char *fila,
+                            const char *adrb, const char *filb) {
+  hash_struct hash;
+  lien_url la, lb;
+  hts_boolean eq;
+
+  memset(&la, 0, sizeof(la));
+  memset(&lb, 0, sizeof(lb));
+  la.adr = key_duphandler(NULL, adra);
+  la.fil = key_duphandler(NULL, fila);
+  lb.adr = key_duphandler(NULL, adrb);
+  lb.fil = key_duphandler(NULL, filb);
+  hash_init(opt, &hash, opt->urlhack);
+  eq = key_adrfil_equals(&hash, &la, &lb);
+  hash_free(&hash);
+  return eq;
+}
+
 // retour: position ou -1 si non trouvé
 int hash_read(const hash_struct * hash, const char *nom1, const char *nom2,
               hash_struct_type type) {

src/htshash.h

@@ -51,8 +51,12 @@ typedef enum hash_struct_type {
 } hash_struct_type;
 
 // tables de hachage
-void hash_init(httrackp *opt, hash_struct *hash, int normalized);
+void hash_init(httrackp *opt, hash_struct *hash, hts_boolean normalized);
 void hash_free(hash_struct *hash);
+/* Test helper: HTS_TRUE if the two URLs dedupe together under opt's urlhack
+   flags. */
+hts_boolean hash_url_equals(httrackp *opt, const char *adra, const char *fila,
+                            const char *adrb, const char *filb);
 int hash_read(const hash_struct * hash, const char *nom1, const char *nom2,
               hash_struct_type type);
 void hash_write(hash_struct * hash, size_t lpos);

src/htshelp.c

@@ -152,16 +152,6 @@ void help_wizard(httrackp * opt) {
 #define str (buffers->str)
 #define argv (buffers->argv)
 
-  //char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
-  //char *mainpath = (char *) malloct(256);
-  //char *projname = (char *) malloct(256);
-  //char *stropt = (char *) malloct(2048);        // options
-  //char *stropt2 = (char *) malloct(2048);       // options longues
-  //char *strwild = (char *) malloct(2048);       // wildcards
-  //char *cmd = (char *) malloct(4096);
-  //char *str = (char *) malloct(256);
-  //char **argv = (char **) malloct(256 * sizeof(char *));
-
   //
   char *a;
 
@@ -227,8 +217,6 @@ void help_wizard(httrackp * opt) {
   strcatbuff(stropt2, str);
   strcatbuff(stropt2, projname);
   strcatbuff(stropt2, "\" ");
-  // Créer si ce n'est fait un index.html 1er niveau
-  make_empty_index(str);
   //
   printf("\n");
   printf("Enter URLs (separated by commas or blank spaces) :");
@@ -349,8 +337,6 @@ void help_wizard(httrackp * opt) {
       }
       hts_main(argc, argv);
     }
-    //} else {
-    //  help("httrack",1);
   }
 
   /* Free buffers */
@@ -455,23 +441,6 @@ void help_catchurl(const char *dest_path) {
     printf("Unable to create a temporary proxy (no remaining port)\n");
 }
 
-// Créer un index.html vide
-void make_empty_index(const char *str) {
-#if 0
-  if (!fexist(fconcat(str, "index.html"))) {
-    FILE *fp = fopen(fconcat(str, "index.html"), "wb");
-
-    if (fp) {
-      fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
-      fprintf(fp,
-              "<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
-              CRLF);
-      fclose(fp);
-    }
-  }
-#endif
-}
-
 // mini-aide  (h: help)
 //           y
 void help(const char *app, int more) {
@@ -521,6 +490,7 @@ void help(const char *app, int more) {
   infomsg("  EN maximum mirror time in seconds (60=1 minute, 3600=1 hour)");
   infomsg("  AN maximum transfer rate in bytes/seconds (1000=1KB/s max)");
   infomsg(" %cN maximum number of connections/seconds (*%c10)");
+  infomsg(" %G  random pause of MIN[:MAX] seconds between files (e.g. %G5:10)");
   infomsg
     ("  GN pause transfer if N bytes reached, and wait until lock file is deleted");
   infomsg("");
@@ -563,6 +533,7 @@ void help(const char *app, int more) {
     (" %x  do not include any password for external password protected websites (%x0 include)");
   infomsg
     (" %q *include query string for local files (useless, for information purpose only) (%q0 don't include)");
+  infomsg(" %g  strip query keys for dedup ([host/pattern=]key1,key2,...)");
   infomsg
     ("  o *generate output html file in case of error (404..) (o0 don't generate)");
   infomsg("  X *purge old files after update (X0 keep delete)");
@@ -571,6 +542,7 @@ void help(const char *app, int more) {
   infomsg("");
   infomsg("Spider options:");
   infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
+  infomsg(" %K  load extra cookies from a Netscape cookies.txt");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
   infomsg
@@ -587,6 +559,9 @@ void help(const char *app, int more) {
     (" %s  update hacks: various hacks to limit re-transfers when updating (identical size, bogus response..)");
   infomsg
     (" %u  url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..)");
+  infomsg("     opt out of one url-hack part: --keep-www-prefix "
+          "(www.foo.com<>foo.com), --keep-double-slashes (//), "
+          "--keep-query-order (?b&a)");
   infomsg
     (" %A  assume that a type (cgi,asp..) is always linked with a mime type (-%A php3,cgi=text/html;dat,bin=application/x-zip)");
   infomsg("     shortcut: '--assume standard' is equivalent to -%A "
@@ -646,9 +621,7 @@ void help(const char *app, int more) {
   infomsg("");
   infomsg("Guru options: (do NOT use if possible)");
   infomsg(" #X *use optimized engine (limited memory boundary checks)");
-  infomsg(" #0  filter test (-#0 '*.gif' 'www.bar.com/foo.gif')");
-  infomsg(" #1  simplify test (-#1 ./foo/bar/../foobar)");
-  infomsg(" #2  type test (-#2 /foo/bar.php)");
+  infomsg(" #test  list engine self-tests (run one with -#test=NAME [args])");
   infomsg(" #C  cache list (-#C '*.com/spider*.gif'");
   infomsg(" #R  cache repair (damaged cache)");
   infomsg(" #d  debug parser");
@@ -804,7 +777,4 @@ void help(const char *app, int more) {
   infomsg("[compiled: " HTS_PLATFORM_NAME "]");
 #endif
   infomsg(NULL);
-
-//  infomsg("  R  *relative links (e.g ../link)\n");
-//  infomsg("  A   absolute links (e.g /www.adr/link)\n");
 }

src/htshelp.h

@@ -45,7 +45,6 @@ typedef struct httrackp httrackp;
 
 void infomsg(const char *msg);
 void help(const char *app, int more);
-void make_empty_index(const char *str);
 void help_wizard(httrackp * opt);
 int help_query(const char *list, int def);
 void help_catchurl(const char *dest_path);

src/htsindex.c

@@ -142,8 +142,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
   char keyword[KEYW_LEN + 32];
   int i = 0;
 
-  //
-  //int WordIndexSize = 1024;
   coucal WordIndexHash = NULL;
   FILE *tmpfp = NULL;
 
@@ -178,8 +176,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
              || (strfield2(mime, "text/css"))
     ) {
     inscript = 1;
-    //} else if (strfield2(mime, "text/vnd.wap.wml")) {   // humm won't work in many cases
-    //  inscript=0;
   } else
     return 0;
 
@@ -275,8 +271,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
 
   // Process indexing for this page
   {
-    //FILE* fp=NULL;
-    //fp=fopen(concat(indexpath,"index.txt"),"ab");
     if (fp_tmpproject) {
       while(!feof(tmpfp)) {
         char line[KEYW_LEN + 32];
@@ -286,7 +280,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
           intptr_t e = 0;
 
           if (coucal_read(WordIndexHash, line, &e)) {
-            //if (e) {
             char BIGSTK savelst[HTS_URLMAXSIZE * 2];
 
             e++;                /* 0 means "once" */
@@ -300,11 +293,9 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
             fprintf(fp_tmpproject, "%s %d %s\n", line,
                     (int) (KEYW_SORT_MAXCOUNT - e), savelst);
             hts_primindex_size++;
-            //}
           }
         }
       }
-      //fclose(fp);
     }
   }
 
@@ -330,7 +321,6 @@ void index_finish(const char *indexpath, int mode) {
   off_t size = fpsize(fp_tmpproject);
 
   if (size > 0) {
-    //FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
     if (fp_tmpproject) {
       tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
       if (tab) {
@@ -398,8 +388,6 @@ void index_finish(const char *indexpath, int mode) {
                       if (total_hit) {
                         if (mode == 1)  // TEXT
                           fprintf(fp, "\t=%d\r\n", total_hit);
-                        //else                // HTML
-                        //  fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
                         if ((((total_hit * 1000) / hts_primindex_words) >=
                              KEYW_USELESS1K)
                             || (((total_line * 1000) / index) >=
@@ -416,8 +404,6 @@ void index_finish(const char *indexpath, int mode) {
                           if (mode == 1)        // TEXT
                             fprintf(fp, "\t(%d)\r\n",
                                     ((total_hit * 1000) / hts_primindex_words));
-                          //else                // HTML
-                          //  fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
                         }
                       }
                       if (mode == 1)    // TEXT

src/htsjava.c

@@ -63,6 +63,9 @@ Please visit our Website: http://www.httrack.com
 /* This file */
 #include "htsjava.h"
 
+/* calloct/freet wrappers */
+#include "htssafe.h"
+
 static int reverse_endian(void) {
   int endian = 1;
 
@@ -172,15 +175,14 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
 #if JAVADEBUG
       printf("fopen\n");
 #endif
-      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
-        //fprintf(stderr, "Cannot open input file.\n");
+      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) ==
+          NULL) {
         sprintf(str->err_msg, "Unable to open file %s", file);
         return 0;               // une erreur..
       }
 #if JAVADEBUG
       printf("fread\n");
 #endif
-      //if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) {   // pas complet..
       if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
         fclose(fpout);
         sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
@@ -204,7 +206,16 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
         return 0;
       }
 
-      tab = (RESP_STRUCT *) calloc(header.count, sizeof(RESP_STRUCT));
+      /* A constant-pool entry is >= 1 byte on disk; reject a count exceeding
+         the file size (hostile .class ~68 MB alloc DoS). */
+      if (!hts_count_fits(header.count, (LLint) fsize(file))) {
+        fclose(fpout);
+        sprintf(str->err_msg,
+                "Invalid constant pool count %u (file len " LLintP ")",
+                (unsigned) header.count, (LLint) fsize(file));
+        return 0;
+      }
+      tab = (RESP_STRUCT *) calloct(header.count, sizeof(RESP_STRUCT));
       if (!tab) {
         sprintf(str->err_msg, "Unable to alloc %d bytes",
                 (int) sizeof(RESP_STRUCT));
@@ -230,7 +241,7 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
           } else {              // ++ une erreur est survenue!
             if (strnotempty(str->err_msg) == 0)
               strcpy(str->err_msg, "Internal readtable error");
-            free(tab);
+            freet(tab);
             if (fpout) {
               fclose(fpout);
               fpout = NULL;
@@ -245,12 +256,10 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
       printf("addfiles\n");
 #endif
       {
-        //unsigned int acess;
         unsigned int Class;
         unsigned int SClass;
         int i;
 
-        //acess = readshort(fpout);
         Class = readshort(fpout);
         SClass = readshort(fpout);
 
@@ -288,7 +297,7 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
 #if JAVADEBUG
       printf("end\n");
 #endif
-      free(tab);
+      freet(tab);
       if (fpout) {
         fclose(fpout);
         fpout = NULL;
@@ -395,11 +404,9 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
 
       p = &buffer[0];
 
-      //fflush(fp); 
       trans.file_position = ftell(fp);
       length = readshort(fp);
       if (length < HTS_URLMAXSIZE) {
-        // while ((length > 0) && (length<500)) {
         while(length > 0) {
           *p++ = fgetc(fp);
 
@@ -407,17 +414,13 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
         }
         *p = '\0';
 
-        //#if JDEBUG
-        //      if(tris(buffer)==1) printf("%s\n ",buffer);
-        //      if(tris(buffer)==2)  printf("%s\n ",printname(buffer));
-        //#endif
         if (tris(str->opt, buffer) == 1)
           str->addLink(str, buffer);    /* trans.file_position */
         else if (tris(str->opt, buffer) == 2)
           str->addLink(str, printname(rname, buffer));
 
         strcpy(trans.name, buffer);
-      } else {                  // gros pb
+      } else { // gros pb
         while((length > 0) && (!feof(fp))) {
           fgetc(fp);
           length--;
@@ -433,7 +436,6 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
     }
     break;
   default:
-    // printf("Type inconnue\n");
     // on arrête tout 
     sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
     *error = 1;
@@ -496,7 +498,6 @@ static char *printname(char rname[1024], char name[1024]) {
     return rname;  // ""
   }
   p += 2;
-  //rname=(char*)calloct(strlen(name)+8,sizeof(char));
   p1 = rname;
   for(j = 0; name[j] != '\0'; j++, p++) {
     if (*p == '/')

src/htsjava.h

@@ -33,15 +33,19 @@ Please visit our Website: http://www.httrack.com
 #ifndef HTSJAVA_DEFH
 #define HTSJAVA_DEFH
 
+#include <stdint.h>
+
 #ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER
 #define HTS_DEF_FWSTRUCT_JAVA_HEADER
 typedef struct JAVA_HEADER JAVA_HEADER;
 #endif
+/* 10-byte on-disk .class header image, fread() directly: fields need exact
+   widths (LP64's 8-byte 'unsigned long' magic never matched 0xCAFEBABE). */
 struct JAVA_HEADER {
-  unsigned long int magic;
-  unsigned short int minor;
-  unsigned short int major;
-  unsigned short int count;
+  uint32_t magic;
+  uint16_t minor;
+  uint16_t major;
+  uint16_t count;
 };
 
 #ifndef HTS_DEF_FWSTRUCT_RESP_STRUCT

src/htslib.h

@@ -147,11 +147,13 @@ struct OLD_htsblk {
 #define HTS_DEF_FWSTRUCT_t_dnscache
 typedef struct t_dnscache t_dnscache;
 #endif
+// One DNS cache record, stored as a coucal value keyed by hostname.
 struct t_dnscache {
-  struct t_dnscache *next;
-  const char *iadr;
-  size_t host_length;                   // length ; (4 or 16) ; 0 for error
-  char host_addr[HTS_MAXADDRLEN];
+  // resolved addresses, in resolver (RFC 6724) order; host_count==0 means the
+  // name does not resolve (negative cache). host_count<=HTS_MAXADDRNUM.
+  int host_count;
+  size_t host_length[HTS_MAXADDRNUM]; // sockaddr length of each (16 or 28)
+  char host_addr[HTS_MAXADDRNUM][HTS_MAXADDRLEN];
 };
 
 /* Library internal definictions */
@@ -169,7 +171,6 @@ HTSEXT_API const char* hts_version(void);
 // fonctions unix/winsock
 int hts_read(htsblk * r, char *buff, int size);
 
-//int HTS_TOTAL_RECV_CHECK(int var);
 LLint check_downloadable_bytes(int rate);
 
 HTSEXT_API int hts_uninit_module(void);
@@ -182,10 +183,21 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
                   const char *adr, const char *fil,
                   const char *referer_adr, const char *referer_fil,
                   htsblk * retour);
+/* Build the request "Cookie:" header line for stored cookies matching
+   domain/path into dst (NUL-terminated). Exposed for the -#Q self-test;
+   wraps the same logic http_sendhead() uses. Returns cookies emitted. */
+int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
+                                const char *path, char *dst, size_t dst_size);
 
-//int newhttp(char* iadr,char* err=NULL);
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
               int waitconnect);
+/* Like newhttp(), but connect to the addr_index-th resolved address of the host
+   (0-based) instead of always the first; *addr_count, if non-NULL, is set to
+   the total resolved addresses. newhttp() == newhttp_addr(...,0,NULL). Used by
+   the slot scheduler to try the next address when a connect fails (dead IPv6
+   etc.). */
+T_SOC newhttp_addr(httrackp *opt, const char *iadr, htsblk *retour, int port,
+                   int waitconnect, int addr_index, int *addr_count);
 HTS_INLINE void deletehttp(htsblk * r);
 HTS_INLINE int deleteaddr(htsblk * r);
 HTS_INLINE void deletesoc(T_SOC soc);
@@ -193,15 +205,31 @@ HTS_INLINE void deletesoc_r(htsblk * r);
 htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc);
 int check_readinput(htsblk * r);
 int check_readinput_t(T_SOC soc, int timeout);
+int check_writeinput_t(T_SOC soc, int timeout);
+
+/* Open an HTTP CONNECT tunnel through the active proxy for an https request:
+   `retour->soc` must already be TCP-connected to the proxy, and `adr` is the
+   origin authority (url_adr, e.g. "https://host:port"). Sends the CONNECT
+   request (with Proxy-Authorization when the proxy carries credentials) and
+   reads the proxy's status line, so the caller's TLS handshake then runs
+   end-to-end with the origin. Blocks up to `timeout` seconds. Returns 1 on a
+   2xx tunnel, 0 on failure (retour->msg/statuscode set). */
+int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
+                      int timeout);
 void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * retour,
                char *rcvd);
 void treatfirstline(htsblk * retour, const char *rcvd);
 
 // sous-fonctions
 LLint http_xfread1(htsblk * r, int bufl);
-HTS_INLINE SOCaddr* hts_dns_resolve2(httrackp * opt, const char *iadr,
-                                     SOCaddr *const addr, 
-                                     const char **error);
+/* Cached resolver: fill out[0..count-1] with up to max addresses for iadr (in
+   resolver order), returning the count (0 = does not resolve, negative-cached).
+   Resolves once per host; later calls read the DNS cache. Must hold no lock
+   (brackets opt->state.lock itself). */
+int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
+                        const char **error);
+HTS_INLINE SOCaddr *hts_dns_resolve2(httrackp *opt, const char *iadr,
+                                     SOCaddr *const addr, const char **error);
 HTS_INLINE SOCaddr* hts_dns_resolve(httrackp * opt, const char *iadr,
                                     SOCaddr *const addr);
 HTSEXT_API SOCaddr* hts_dns_resolve_nocache2(const char *const hostname, 
@@ -214,8 +242,9 @@ HTSEXT_API int check_hostname_dns(const char *const hostname);
 int ftp_available(void);
 
 #if HTS_DNSCACHE
-void hts_cache_free(t_dnscache *const cache);
-t_dnscache *hts_cache(httrackp * opt);
+/* Return opt's DNS cache hashtable (hostname -> t_dnscache record), creating it
+   on first use. Records are owned by the table and freed on coucal_delete. */
+coucal hts_cache(httrackp *opt);
 #endif
 
 // outils divers
@@ -254,6 +283,9 @@ int ishttperror(int err);
 int get_userhttptype(httrackp * opt, char *s, const char *fil);
 int give_mimext(char *s, size_t ssize, const char *st);
 
+/* Advertised Accept-Encoding value (no header name/CRLF); see htslib.c. */
+const char *hts_acceptencoding(hts_boolean compressible);
+
 int may_bogus_multiple(httrackp * opt, const char *mime, const char *filename);
 int may_unknown2(httrackp * opt, const char *mime, const char *filename);
 
@@ -465,10 +497,22 @@ HTS_STATIC int strcmpnocase(const char *a, const char *b) {
 
 // is this MIME an hypertext MIME (text/html), html/js-style or other script/text type?
 #define HTS_HYPERTEXT_DEFAULT_MIME "text/html"
+/* Sentinel stored when the server declared no Content-Type. It is html-ish
+   for every type test (so a typeless response still parses/stores as today),
+   but the naming code (wire_patches_ext) treats it as "no declared type" and
+   keeps the URL extension. It rides the cache, so updates name consistently. */
+#define HTS_UNKNOWN_MIME "unknown/unknown"
+/* Map the no-declared-type sentinel back to a real type for any header or
+   record we EMIT or PERSIST, so "unknown/unknown" never reaches a consumer
+   (a served Content-Type, a ProxyTrack .arc record, ...). */
+#define hts_effective_mime(m)                                                  \
+  (strfield2((m), HTS_UNKNOWN_MIME) ? HTS_HYPERTEXT_DEFAULT_MIME : (m))
 
-#define is_html_mime_type(a) \
-  ( (strfield2((a),"text/html")!=0)\
-  || (strfield2((a),"application/xhtml+xml")!=0) \
+#define is_html_mime_type(a)                                                   \
+  ((strfield2((a), "text/html") != 0) ||                                       \
+   (strfield2((a), "application/xhtml+xml") != 0) ||                           \
+   (strfield2((a), HTS_UNKNOWN_MIME) !=                                        \
+    0) /* no declared type: treat as html */                                   \
   )
 #define is_hypertext_mime__(a) \
   ( \

src/htsmodules.c

@@ -55,12 +55,6 @@ extern int fspc(httrackp * opt, FILE * fp, const char *type);
 
 /* >>> Put all modules variables here */
 
-#if 0
-t_gzopen gzopen = NULL;
-t_gzread gzread = NULL;
-t_gzclose gzclose = NULL;
-#endif
-
 int V6_is_available = HTS_INET6;
 
 static char WHAT_is_available[64] = "";
@@ -112,18 +106,6 @@ int hts_parse_externals(htsmoduleStruct * str) {
   return -1;
 }
 
-//static void addCallback(htscallbacks* chain, void* moduleHandle, htscallbacksfncptr exitFnc) {
-//  while(chain->next != NULL) {
-//    chain = chain->next;
-//  }
-//  chain->next = calloct(1, sizeof(htscallbacks));
-//  assertf(chain->next != NULL);
-//  chain = chain->next;
-//  memset(chain, 0, sizeof(*chain));
-//  chain->exitFnc = exitFnc;
-//  chain->moduleHandle = moduleHandle;
-//}
-
 void clearCallbacks(htscallbacks * chain_);
 void clearCallbacks(htscallbacks * chain_) {
   htscallbacks *chain;

src/htsmodules.h

@@ -69,41 +69,41 @@ typedef struct hash_struct hash_struct;
 #define HTS_DEF_FWSTRUCT_htsmoduleStruct
 typedef struct htsmoduleStruct htsmoduleStruct;
 #endif
-typedef int (*t_htsAddLink) (htsmoduleStruct * str, char *link);
+typedef int (*t_htsAddLink)(htsmoduleStruct *str, char *link);
 
 /** Per-object context passed to a parser module for one downloaded file.
     Field access classes are noted; engine owns all pointers unless stated. */
 struct htsmoduleStruct {
   /* Read-only elements */
-  const char *filename;         /* filename (C:\My Web Sites\...) */
-  int size;                     /* size of filename (should be > 0) */
-  const char *mime;             /* MIME type of the object */
-  const char *url_host;         /* incoming hostname (www.foo.com) */
-  const char *url_file;         /* incoming filename (/bar/bar.gny) */
+  const char *filename; /* filename (C:\My Web Sites\...) */
+  int size;             /* size of filename (should be > 0) */
+  const char *mime;     /* MIME type of the object */
+  const char *url_host; /* incoming hostname (www.foo.com) */
+  const char *url_file; /* incoming filename (/bar/bar.gny) */
 
   /* Write-only */
-  const char *wrapper_name;     /* name of wrapper (static string) */
-  char *err_msg;                /* if an error occurred, the error message (max. 1KB) */
+  const char *wrapper_name; /* name of wrapper (static string) */
+  char *err_msg; /* if an error occurred, the error message (max. 1KB) */
 
   /* Read/Write */
-  int relativeToHtmlLink;       /* set this to 1 if all urls you pass to addLink
-                                   are in fact relative to the html file where your
-                                   module was originally */
+  int relativeToHtmlLink; /* set this to 1 if all urls you pass to addLink
+                             are in fact relative to the html file where your
+                             module was originally */
 
   /* Callbacks */
-  t_htsAddLink addLink;         /* call this function when links are 
-                                   being detected. it if not your responsability to decide
-                                   if the engine will keep them, or not. */
+  t_htsAddLink addLink; /* call this function when links are
+                           being detected. it if not your responsability to
+                           decide if the engine will keep them, or not. */
 
   /* Optional */
-  char *localLink;              /* if non null, the engine will write there the local
-                                   relative filename of the link added by addLink(), or
-                                   the absolute path if the link was refused by the wizard */
-  int localLinkSize;            /* size of the optionnal buffer */
+  char *localLink;   /* if non null, the engine will write there the local
+                        relative filename of the link added by addLink(), or
+                        the absolute path if the link was refused by the wizard */
+  int localLinkSize; /* size of the optionnal buffer */
 
   /* User-defined */
-  void *userdef;                /* can be used by callback routines
-                                 */
+  void *userdef; /* can be used by callback routines
+                  */
 
   /* The parser httrackp structure (may be used) */
   httrackp *opt;
@@ -117,7 +117,6 @@ struct htsmoduleStruct {
   int *ptr_;
   const char *page_charset_;
   /* Internal use - please don't touch */
-
 };
 
 #ifdef __cplusplus
@@ -126,11 +125,11 @@ extern "C" {
 
 /** Module lifecycle hooks. Init/PlugInit return 1 on success, 0 on failure;
     Exit returns its own status (ignored by the engine). */
-typedef int (*t_htsWrapperInit) (char *fn, char *args);
+typedef int (*t_htsWrapperInit)(char *fn, char *args);
 
-typedef int (*t_htsWrapperExit) (void);
+typedef int (*t_htsWrapperExit)(void);
 
-typedef int (*t_htsWrapperPlugInit) (char *args);
+typedef int (*t_htsWrapperPlugInit)(char *args);
 
 /* Library internal definictions */
 #ifdef HTS_INTERNAL_BYTECODE
@@ -138,7 +137,7 @@ typedef int (*t_htsWrapperPlugInit) (char *args);
 /** Capabilities string ("-noV6", "-nossl", ...) followed by "+name" for each
     loaded module. Returned pointer aliases opt->state.HTbuff; do not free, and
     it is overwritten by the next call. */
-HTSEXT_API const char *hts_get_version_info(httrackp * opt);
+HTSEXT_API const char *hts_get_version_info(httrackp *opt);
 
 /** Static capabilities string set by htspe_init(); valid for the process
     lifetime, do not free. */
@@ -154,7 +153,7 @@ extern void htspe_uninit(void);
 /** Run the external-parser callbacks for the object described by str.
     Returns the parse callback result (>=0) on a handled object, or -1 if no
     module claimed it or its wrapper_name is blacklisted. */
-extern int hts_parse_externals(htsmoduleStruct * str);
+extern int hts_parse_externals(htsmoduleStruct *str);
 
 /** Nonzero if IPv6 support was compiled in (== HTS_INET6). */
 extern int V6_is_available;

src/htsname.c

@@ -41,6 +41,10 @@ Please visit our Website: http://www.httrack.com
 #include "htstools.h"
 #include "htscharset.h"
 #include "htsencoding.h"
+#include "htssniff.h"
+#if HTS_USEZLIB
+#include "htszlib.h"
+#endif
 #include <ctype.h>
 
 #define ADD_STANDARD_PATH \
@@ -70,31 +74,6 @@ static const char *hts_tbdev[] = {
   ""
 };
 
-#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET() do { \
-  int prev = opt->state._hts_in_html_parsing; \
-  while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
-    opt->state. _hts_in_html_parsing = 6; \
-    /* Wait .. */ \
-    back_wait(sback,opt,cache,0); \
-    /* Transfer rate */ \
-    engine_stats(); \
-    /* Refresh various stats */ \
-    HTS_STAT.stat_nsocket=back_nsoc(sback); \
-    HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
-    HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
-    HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
-    HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
-    HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
-    /* Check */ \
-    { \
-      if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,-1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
-        return -1; \
-      } \
-    } \
-  } \
-  opt->state._hts_in_html_parsing = prev; \
-} while(0)
-
 /* Strip all // */
 static void cleanDoubleSlash(char *s) {
   int i, j;
@@ -138,8 +117,191 @@ static void cleanEndingSpaceOrDot(char *s) {
   }
 }
 
-// forme le nom du fichier à sauver (save) à partir de fil et adr
-// système intelligent, qui renomme en cas de besoin (exemple: deux INDEX.HTML et index.html)
+/* Wire Content-Type vs URL extension: a patchable wire type wins over an
+   unspecific ext, the HTS_UNKNOWN_MIME sentinel keeps a specific non-HTML ext
+   (#267 guard), a declared disagreement is CONTESTED (sniffed below). */
+typedef enum wire_verdict {
+  WIRE_KEEPS_EXT,
+  WIRE_WINS,
+  WIRE_CONTESTED
+} wire_verdict;
+
+static wire_verdict wire_ext_verdict(httrackp *opt, const char *wiremime,
+                                     const char *file, char *urlmime,
+                                     size_t urlmime_size) {
+  if (may_unknown2(opt, wiremime, file))
+    return WIRE_KEEPS_EXT; /* type kept verbatim (keep-list / bogus-multiple) */
+  urlmime[0] = '\0';
+  /* type implied by the URL extension, only when confidently known (flag 0) */
+  if (!get_httptype_sized(opt, urlmime, urlmime_size, file, 0))
+    return WIRE_WINS; /* URL ext implies no known type */
+  if (strfield2(wiremime, urlmime))
+    return WIRE_KEEPS_EXT; /* agreement (no .htm->.html churn) */
+  if (!is_hypertext_mime(opt, urlmime, file) &&
+      strfield2(wiremime, HTS_UNKNOWN_MIME))
+    return WIRE_KEEPS_EXT; /* no declared type */
+  return WIRE_CONTESTED;
+}
+
+/* Optional evidence for a contested wire-vs-ext verdict. */
+typedef struct sniff_src {
+  struct_back *sback;       /* live backing (looked up by adr/fil) */
+  const lien_back *headers; /* snapshot: r.adr, else the url_sav file */
+  const char *adr, *fil;
+  const char *prev_save; /* previous run's save name (cache X-Save) */
+} sniff_src;
+
+#if HTS_USEZLIB
+/* Inflate the head of a gzip/zlib stream; 0 when undecodable. */
+static size_t sniff_inflate_head(const void *in, size_t in_len, void *out,
+                                 size_t out_len) {
+  z_stream zs;
+  size_t n = 0;
+  int err;
+
+  memset(&zs, 0, sizeof(zs));
+  if (inflateInit2(&zs, 47) != Z_OK) /* 47: gzip or zlib, autodetected */
+    return 0;
+  zs.next_in = (const Bytef *) in;
+  zs.avail_in = (uInt) in_len;
+  zs.next_out = (Bytef *) out;
+  zs.avail_out = (uInt) out_len;
+  err = inflate(&zs, Z_SYNC_FLUSH);
+  if (err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR)
+    n = out_len - zs.avail_out;
+  inflateEnd(&zs);
+  return n;
+}
+#endif
+
+static size_t sniff_read_head(const char *path, void *buf, size_t len) {
+  char catbuff[CATBUFF_SIZE];
+  FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "rb");
+  size_t n = 0;
+
+  if (fp != NULL) {
+    n = fread(buf, 1, len, fp);
+    fclose(fp);
+  }
+  return n;
+}
+
+/* Body head of one slot: memory, else its flushed on-disk file (url_sav, or
+   tmpfile for a compressed stream); inflated so the sniff sees the final body.
+ */
+static size_t sniff_slot_head(const lien_back *slot, void *buf, size_t len) {
+  const htsblk *const r = &slot->r;
+  size_t n = 0;
+
+  if (r->adr != NULL && r->size > 0) {
+    n = (size_t) r->size < len ? (size_t) r->size : len;
+    memcpy(buf, r->adr, n);
+  } else {
+    if (r->out != NULL)
+      fflush(r->out);
+    if (slot->url_sav[0] != '\0')
+      n = sniff_read_head(slot->url_sav, buf, len);
+    if (n == 0 && slot->tmpfile != NULL && slot->tmpfile[0] != '\0')
+      n = sniff_read_head(slot->tmpfile, buf, len);
+  }
+  if (n > 0 && r->compressed) {
+#if HTS_USEZLIB
+    unsigned char raw[HTS_SNIFF_LEN];
+
+    if (n > sizeof(raw))
+      n = sizeof(raw);
+    memcpy(raw, buf, n);
+    n = sniff_inflate_head(raw, n, buf, len);
+#else
+    n = 0;
+#endif
+  }
+  return n;
+}
+
+/* Up to len leading body bytes; 0 when unavailable, and always in
+   non-delayed mode (its HEAD-probe first run couldn't sniff either). */
+static size_t sniff_body_head(httrackp *opt, const sniff_src *src, void *buf,
+                              size_t len) {
+  size_t n = 0;
+
+  if (src == NULL || opt->savename_delayed == HTS_SAVENAME_DELAYED_NONE)
+    return 0;
+  /* live backing slot: a snapshot (back_copy_static) loses r.adr/r.out */
+  if (src->sback != NULL && src->adr != NULL && src->fil != NULL) {
+    const int b = back_index(opt, src->sback, src->adr, src->fil, NULL);
+
+    if (b >= 0)
+      n = sniff_slot_head(&src->sback->lnk[b], buf, len);
+  }
+  if (n == 0 && src->headers != NULL)
+    n = sniff_slot_head(src->headers, buf, len);
+  return n;
+}
+
+/* Contested verdicts: magic proving the URL ext keeps it, else wire wins. */
+static int wire_patches_ext(httrackp *opt, const sniff_src *src,
+                            const char *wiremime, const char *file) {
+  char urlmime[256];
+
+  switch (wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime))) {
+  case WIRE_KEEPS_EXT:
+    return 0;
+  case WIRE_WINS:
+    return 1;
+  case WIRE_CONTESTED:
+    break;
+  }
+  if (src != NULL) {
+    if (hts_sniff_mime_known(urlmime)) {
+      unsigned char head[HTS_SNIFF_LEN];
+      const size_t n = sniff_body_head(opt, src, head, sizeof(head));
+
+      if (n > 0)
+        return hts_sniff_mime_consistent(head, n, urlmime) ? 0 : 1;
+    }
+    /* no bytes: reproduce the previous run's verdict (cached X-Save name) */
+    if (src->prev_save != NULL && src->prev_save[0] != '\0') {
+      char prevmime[256];
+
+      prevmime[0] = '\0';
+      if (get_httptype_sized(opt, prevmime, sizeof(prevmime), src->prev_save,
+                             0) &&
+          strfield2(prevmime, urlmime))
+        return 0;
+    }
+  }
+  return 1;
+}
+
+int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime,
+                         const char *file) {
+  char urlmime[256];
+
+  return wiremime != NULL && strnotempty(wiremime) &&
+         wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime)) ==
+             WIRE_CONTESTED &&
+         hts_sniff_mime_known(urlmime);
+}
+
+/* Wire-metadata name change: a Content-Disposition filename wins (returns 2),
+   else the declared type's ext when wire_patches_ext() allows (returns 1),
+   else 0. ext receives the new extension or replacement filename. */
+static int resolve_extension(httrackp *opt, const sniff_src *src,
+                             const char *cdispo, const char *contenttype,
+                             const char *fil, char *ext, size_t ext_size) {
+  if (strnotempty(cdispo)) {
+    strlcpybuff(ext, cdispo, ext_size);
+    return 2;
+  }
+  if (wire_patches_ext(opt, src, contenttype, fil) &&
+      give_mimext(ext, ext_size, contenttype))
+    return 1;
+  return 0;
+}
+
+// Build the local save name (save) from adr/fil; renames on collision
+// (e.g. INDEX.HTML vs index.html).
 int url_savename(lien_adrfilsave *const afs,
                  lien_adrfil *const former,
                  const char *referer_adr, const char *referer_fil, 
@@ -169,6 +331,13 @@ int url_savename(lien_adrfilsave *const afs,
   // copy of fil, used for lookups (see urlhack)
   const char *normadr = adr;
   const char *normfil = fil_complete;
+  /* query keys to strip for this URL (NULL = none); decoupled from urlhack */
+  char BIGSTK stripkeys[HTS_URLMAXSIZE];
+  const char *const strip =
+      StringNotEmpty(opt->strip_query)
+          ? hts_query_strip_keys(StringBuff(opt->strip_query), adr,
+                                 fil_complete, stripkeys, sizeof(stripkeys))
+          : NULL;
   const char *const print_adr = jump_protocol_const(adr);
   const char *start_pos = NULL, *nom_pos = NULL, *dot_pos = NULL;     // Position nom et point
 
@@ -184,10 +353,11 @@ int url_savename(lien_adrfilsave *const afs,
 
   /* 8-3 ? */
   switch (opt->savename_83) {
-  case 1:                      // 8-3
+  case HTS_SAVENAME_83_DOS: // 8-3
     max_char = 8;
     break;
-  case 2:                      // Level 2 File names may be up to 31 characters.
+  case HTS_SAVENAME_83_ISO9660: // Level 2 File names may be up to 31
+                                // characters.
     max_char = 31;
     break;
   default:
@@ -200,9 +370,13 @@ int url_savename(lien_adrfilsave *const afs,
   // www-42.foo.com -> foo.com
   // foo.com/bar//foobar -> foo.com/bar/foobar
   if (opt->urlhack) {
-    // copy of adr (without protocol), used for lookups (see urlhack)
-    normadr = adr_normalized_sized(adr, normadr_, sizeof(normadr_));
-    normfil = fil_normalized(fil_complete, normfil_);
+    // dedup-lookup key; honor the per-feature negatives like htshash.c so
+    // distinct URLs keep distinct savenames (else keep normadr = adr)
+    if (!opt->no_www_dedup)
+      normadr = adr_normalized_sized(adr, normadr_, sizeof(normadr_));
+    normfil =
+        fil_normalized_filtered_ex(fil_complete, normfil_, strip,
+                                   !opt->no_slash_dedup, !opt->no_query_dedup);
   } else {
     if (link_has_authority(adr_complete)) {     // https or other protocols : in "http/" subfolder
       char *pos = strchr(adr_complete, ':');
@@ -215,6 +389,11 @@ int url_savename(lien_adrfilsave *const afs,
         normadr = normadr_;
       }
     }
+    // strip still applies with urlhack off (host left untouched); no // or
+    // query-sort here, to match the hash key (norm_slash/norm_query are 0 when
+    // urlhack is off) so a URL is looked up under the key it was stored with
+    if (strip != NULL)
+      normfil = fil_normalized_filtered_ex(fil_complete, normfil_, strip, 0, 0);
   }
 
   // à afficher sans ftp://
@@ -324,7 +503,10 @@ int url_savename(lien_adrfilsave *const afs,
   }
 
   /* replace shtml to html.. */
-  if (opt->savename_delayed == 2)
+  /* HARD delays every type, except one the user pinned with --assume: honor it
+     immediately (ishtml() consults the user type), no delayed name (#56) */
+  if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD &&
+      !is_userknowntype(opt, fil))
     is_html = -1;               /* ALWAYS delay type */
   else
     is_html = ishtml(opt, fil);
@@ -356,48 +538,36 @@ int url_savename(lien_adrfilsave *const afs,
 
   // si option check_type activée
   if (is_html < 0 && opt->check_type && !ext_chg) {
-    int ishtest = 0;
-
     if (protocol != PROTOCOL_FILE
         && protocol != PROTOCOL_FTP
       ) {
       // tester type avec requète HEAD si on ne connait pas le type du fichier
       if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/')))   // slash doit être html?
-        if (opt->savename_delayed == 2 || (ishtest = ishtml(opt, fil)) < 0) {   // on ne sait pas si c'est un html ou un fichier..
+        if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
+            ishtml(opt, fil) < 0) { // unsure whether it's html or a file
           // lire dans le cache
-          htsblk r = cache_read_including_broken(opt, cache, adr, fil); // test uniquement
+          char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
+          htsblk r;
 
-          if (r.statuscode != -1) {     // pas d'erreur de lecture cache
-            char s[32];
+          previous_save[0] = '\0';
+          r = cache_read_including_broken(opt, cache, adr, fil,
+                                          previous_save); // test uniquement
 
-            s[0] = '\0';
+          if (r.statuscode != -1) { // cache entry read OK
             hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s",
                           adr_complete, fil_complete);
             if (!HTTP_IS_REDIRECT(r.statuscode)) {
-              if (strnotempty(r.cdispo)) {        /* filename given */
-                ext_chg = 2;      /* change filename */
-                strcpybuff(ext, r.cdispo);
-              } else if (!may_unknown2(opt, r.contenttype, fil)) {        // on peut patcher à priori?
-                if (give_mimext(s, sizeof(s),
-                                r.contenttype)) { // recognized extension
-                  ext_chg = 1;
-                  strcpybuff(ext, s);
-                }
-              }
+              const sniff_src src = {sback, NULL, adr, fil, previous_save};
+
+              ext_chg = resolve_extension(opt, &src, r.cdispo, r.contenttype,
+                                          fil, ext, sizeof(ext));
             }
-#ifdef DEFAULT_BIN_EXT
-            // no extension and potentially bogus
-            else if (ishtest == -2) {
-              ext_chg = 1;
-              strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-            }
-#endif
-            //
-          } else if (opt->savename_delayed != 2 && is_userknowntype(opt, fil)) {        /* PATCH BY BRIAN SCHRÖDER. 
-                                                                                           Lookup mimetype not only by extension, 
-                                                                                           but also by filename */
-            /* Note: "foo.cgi => text/html" means that foo.cgi shall have the text/html MIME file type,
-               that is, ".html" */
+          } else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD &&
+                     is_userknowntype(opt, fil)) { /* PATCH BY BRIAN SCHRÖDER.
+                              Lookup mimetype not only by extension,
+                              but also by filename */
+            /* Note: "foo.cgi => text/html" means that foo.cgi shall have the
+               text/html MIME file type, that is, ".html" */
             char BIGSTK mime[1024];
 
             mime[0] = ext[0] = '\0';
@@ -408,24 +578,20 @@ int url_savename(lien_adrfilsave *const afs,
               }
             }
           }
-          // note: if savename_delayed is enabled, the naming will be temporary (and slightly invalid!)
-          // note: if we are about to stop (opt->state.stop), back_add() will fail later
-          else if (opt->savename_delayed != 0 && !opt->state.stop) {
-            // Check if the file is ready in backing. We basically take the same logic as later.
-            // FIXME: we should cleanup and factorize this unholy mess
+          // note: if savename_delayed is enabled, the naming will be temporary
+          // (and slightly invalid!)
+          //
+          // note: if we are about to stop (opt->state.stop), back_add() will
+          // fail later
+          else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_NONE &&
+                   !opt->state.stop) {
+            // Check if the file is ready in backing.
             if (headers != NULL && headers->status >= 0 && !is_redirect) {
-              if (strnotempty(headers->r.cdispo)) {        /* filename given */
-                ext_chg = 2;      /* change filename */
-                strcpybuff(ext, headers->r.cdispo);
-              } else if (!may_unknown2(opt, headers->r.contenttype, headers->url_fil)) {    // on peut patcher à priori? (pas interdit ou pas de type)
-                char s[16];
-                if (give_mimext(
-                        s, sizeof(s),
-                        headers->r.contenttype)) { // recognized extension
-                  ext_chg = 1;
-                  strcpybuff(ext, s);
-                }
-              }
+              const sniff_src src = {sback, headers, adr, fil, NULL};
+
+              ext_chg = resolve_extension(opt, &src, headers->r.cdispo,
+                                          headers->r.contenttype,
+                                          headers->url_fil, ext, sizeof(ext));
             }
             else if (mime_type != NULL) {
               ext[0] = '\0';
@@ -443,13 +609,6 @@ int url_savename(lien_adrfilsave *const afs,
                   if (!may_unknown2(opt, mime_type, fil)) {
                     ext_chg = 1;
                   }
-#ifdef DEFAULT_BIN_EXT
-                  // no extension and potentially bogus
-                  else if (ishtml(opt, fil) == -2) {
-                    ext_chg = 1;
-                    strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-                  }
-#endif
                 } else {
                   ext_chg = 0;
                 }
@@ -468,11 +627,10 @@ int url_savename(lien_adrfilsave *const afs,
             int has_been_moved = 0;
             lien_adrfil current;
 
-            /* Ensure we don't use too many sockets by using a "testing" one
-               If we have only 1 simultaneous connection authorized, wait for pending download
-               Wait for an available slot 
+            /* Wait for an available test slot, honoring the connection limits
              */
-            URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
+            if (!hts_wait_available_socket(sback, opt, cache, ptr))
+              return -1;
 
             /* Rock'in */
             current.adr[0] = current.fil[0] = '\0';
@@ -502,24 +660,11 @@ int url_savename(lien_adrfilsave *const afs,
                   if (ptr >= 0) {
                     back_fillmax(sback, opt, cache, ptr, numero_passe);
                   }
-                  // on est obligé d'appeler le shell pour le refresh..
-                  // Transfer rate
-                  engine_stats();
-
-                  // Refresh various stats
-                  HTS_STAT.stat_nsocket = back_nsoc(sback);
-                  HTS_STAT.stat_errors = fspc(opt, NULL, "error");
-                  HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
-                  HTS_STAT.stat_infos = fspc(opt, NULL, "info");
-                  HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
-                  HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
-
-                  if (!RUN_CALLBACK7
-                      (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
-                       (int) (time_local() - HTS_STAT.stat_timestart),
-                       &HTS_STAT)) {
+                  if (!hts_loop_tick(sback, opt, b, ptr)) {
                     return -1;
-                  } else if (opt->state._hts_cancel || !back_checkmirror(opt)) {        // cancel 2 ou 1 (cancel parsing)
+                  } else if (opt->state._hts_cancel ||
+                             !back_checkmirror(
+                                 opt)) { // cancel level 2 or 1 (cancel parsing)
                     back_delete(opt, cache, sback, b);  // cancel test
                     stop_looping = 1;
                   }
@@ -584,8 +729,9 @@ int url_savename(lien_adrfilsave *const afs,
                                                 "Loop with HEAD request (during prefetch) at %s%s",
                                                 current.adr, current.fil);
                                 }
-                                // Ajouter
-                                URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
+                                if (!hts_wait_available_socket(sback, opt,
+                                                               cache, ptr))
+                                  return -1;
                                 if (back_add(sback, opt, cache, moved.adr, moved.fil, methode, referer_adr, referer_fil, 1) != -1) {        // OK
                                   hts_log_print(opt, LOG_DEBUG,
                                                 "(during prefetch) %s (%d) to link %s at %s%s",
@@ -633,34 +779,16 @@ int url_savename(lien_adrfilsave *const afs,
                 if (!has_been_moved) {
                   if (back[b].r.statuscode != -10) {    // erreur
                     if (strnotempty(back[b].r.contenttype) == 0)
-                      strcpybuff(back[b].r.contenttype, "text/html");   // message d'erreur en html
+                      strcpybuff(back[b].r.contenttype,
+                                 HTS_UNKNOWN_MIME); // no declared type
                     // Finalement on, renvoie un erreur, pour ne toucher à rien dans le code
                     // libérer emplacement backing
                   }
 
-                  {             // pas d'erreur, changer type?
-                    char s[16];
-
-                    s[0] = '\0';
-                    if (strnotempty(back[b].r.cdispo)) {        /* filename given */
-                      ext_chg = 2;      /* change filename */
-                      strcpybuff(ext, back[b].r.cdispo);
-                    } else if (!may_unknown2(opt, back[b].r.contenttype, back[b].url_fil)) {    // on peut patcher à priori? (pas interdit ou pas de type)
-                      if (give_mimext(
-                              s, sizeof(s),
-                              back[b].r.contenttype)) { // recognized extension
-                        ext_chg = 1;
-                        strcpybuff(ext, s);
-                      }
-                    }
-#ifdef DEFAULT_BIN_EXT
-                    // no extension and potentially bogus
-                    else if (ishtest == -2) {
-                      ext_chg = 1;
-                      strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-                    }
-#endif
-                  }
+                  // no error: change the type?
+                  ext_chg = resolve_extension(
+                      opt, NULL, back[b].r.cdispo, back[b].r.contenttype,
+                      back[b].url_fil, ext, sizeof(ext));
                 }
                 // FIN Si non déplacé, forcer type?
 
@@ -698,7 +826,7 @@ int url_savename(lien_adrfilsave *const afs,
             }
             // restaurer
             opt->state._hts_in_html_parsing = hihp;
-          }                     // caché?
+          } // caché?
         }
     }
   }
@@ -717,9 +845,9 @@ int url_savename(lien_adrfilsave *const afs,
         strcatbuff(fil, DEFAULT_HTML);  // nommer page par défaut (à priori ici html depuis un proxy http)
     }
   }
-  // Changer extension?
-  // par exemple, php3 sera sauvé en html, cgi en html ou gif, xbm etc.. selon les cas
-  if (ext_chg && !opt->no_type_change) {                // changer ext
+  // Change the extension? e.g. php3 saved as html, cgi as html or gif/xbm
+  // depending on the resolved type.
+  if (ext_chg && !opt->no_type_change) {
     char *a = fil + strlen(fil) - 1;
 
     if ((opt->debug > 1) && (opt->log != NULL)) {
@@ -731,11 +859,19 @@ int url_savename(lien_adrfilsave *const afs,
                       adr_complete, fil_complete, ext);
     }
     if (ext_chg == 1) {
+      // Cut the old extension only when it is empty (a bare trailing dot), the
+      // new one, or a recognized one; an unknown trailing ".token" (e.g.
+      // /article-1.884291, #115) is part of the name, not an extension.
+      const char *const old_ext = get_ext(catbuff, sizeof(catbuff), fil);
+      const int known_ext = !*old_ext || strfield2(old_ext, ext) ||
+                            is_knowntype(opt, fil) || is_dyntype(old_ext) ||
+                            ishtml_ext(old_ext) != -1;
+
       while((a > fil) && (*a != '.') && (*a != '/'))
         a--;
-      if (*a == '.')
-        *a = '\0';              // couper
-      strcatbuff(fil, ".");     // recopier point
+      if (*a == '.' && known_ext)
+        *a = '\0';          // cut
+      strcatbuff(fil, "."); // re-add the dot
     } else {
       while((a > fil) && (*a != '/'))
         a--;
@@ -743,7 +879,7 @@ int url_savename(lien_adrfilsave *const afs,
         a++;
       *a = '\0';
     }
-    strcatbuff(fil, ext);       // copier ext/nom
+    strcatbuff(fil, ext); // append ext/name
   }
   // Rechercher premier / et dernier .
   {
@@ -1190,13 +1326,13 @@ int url_savename(lien_adrfilsave *const afs,
   // Not used anymore unless non-delayed types.
   // de même en cas de manque d'extension on en place une de manière forcée..
   // cela évite les /chez/toto et les /chez/toto/index.html incompatibles
-  if (opt->savename_type != -1 && opt->savename_delayed != 2) {
+  if (opt->savename_type != -1 &&
+      opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD) {
     char *a = afs->save + strlen(afs->save) - 1;
 
     while((a > afs->save) && (*a != '.') && (*a != '/'))
       a--;
-    if (*a != '.') {            // agh pas de point
-      //strcatbuff(save,".none");                 // a éviter
+    if (*a != '.') {                         // agh pas de point
       strcatbuff(afs->save, ".html");        // préférable!
       hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",
                     adr_complete, fil_complete, afs->save);
@@ -1236,31 +1372,21 @@ int url_savename(lien_adrfilsave *const afs,
     size_t i;
     for(i = 0 ; afs->save[i] != '\0' ; i++) {
       unsigned char c = (unsigned char) afs->save[i];
-      if (c < 32      // control
-        || c == 127   // unwise
-        || c == '~'   // unix unwise
-        || c == '\\'  // windows separator
-        || c == ':'   // windows forbidden
-        || c == '*'   // windows forbidden
-        || c == '?'   // windows forbidden
-        || c == '\"'  // windows forbidden
-        || c == '<'   // windows forbidden
-        || c == '>'   // windows forbidden
-        || c == '|'   // windows forbidden
-        //|| c == '@' // ?
-        ||
-          (
-            opt->savename_83 == 2 // CDROM
-            &&
-            (
-              c == '-'
-              || c == '='
-              || c == '+'
-            )
-          )
-        )
-      {
-         afs->save[i] = '_';
+      if (c < 32       // control
+          || c == 127  // unwise
+          || c == '~'  // unix unwise
+          || c == '\\' // windows separator
+          || c == ':'  // windows forbidden
+          || c == '*'  // windows forbidden
+          || c == '?'  // windows forbidden
+          || c == '\"' // windows forbidden
+          || c == '<'  // windows forbidden
+          || c == '>'  // windows forbidden
+          || c == '|'  // windows forbidden
+          //|| c == '@' // ?
+          || (opt->savename_83 == HTS_SAVENAME_83_ISO9660 // CDROM
+              && (c == '-' || c == '=' || c == '+'))) {
+        afs->save[i] = '_';
       }
     }
   }
@@ -1344,17 +1470,6 @@ int url_savename(lien_adrfilsave *const afs,
   fil_simplifie(afs->save);
 
   /* convert name to UTF-8 ? Note: already done while parsing. */
-  //if (charset != NULL && charset[0] != '\0') {
-  //  char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
-
-  //  if (s != NULL) {
-  //    hts_log_print(opt, LOG_DEBUG,
-  //                  "engine: save-name: charset conversion from '%s' to '%s' using charset '%s'",
-  //                  save, s, charset);
-  //    strcpybuff(save, s);
-  //    free(s);
-  //  }
-  //}
 
   /* callback */
   RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr,
@@ -1502,10 +1617,6 @@ int url_savename(lien_adrfilsave *const afs,
         int sameAdr = (strfield2(heap(i)->adr, normadr) != 0);
         int sameFil;
 
-        // NO - URL hack is only for stripping // and www.
-        //if (opt->urlhack != 0)
-        //  sameFil = ( strfield2(heap(i)->fil, normfil) != 0);
-        //else
         sameFil = (strcmp(heap(i)->fil, normfil) == 0);
         if (sameAdr && sameFil) {       // ok c'est le même lien, adresse déja définie
           /* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */
@@ -1521,7 +1632,8 @@ int url_savename(lien_adrfilsave *const afs,
           char *a = afs->save + strlen(afs->save) - 1;
           char *b;
           int n = 2;
-          char collisionSeparator = ((opt->savename_83 != 2) ? '-' : '_');
+          char collisionSeparator =
+              ((opt->savename_83 != HTS_SAVENAME_83_ISO9660) ? '-' : '_');
 
           tempo[0] = '\0';
 
@@ -1570,17 +1682,13 @@ int url_savename(lien_adrfilsave *const afs,
 
           strcpybuff(afs->save, tempo);
 
-          //printf("switched: %s\n",save);
-
         }                       // if
       }
 #if DEBUG_SAVENAME
       printf("\nEnd search, %s\n", fil_complete);
 #endif
-    } while(!nom_ok);
-
+    } while (!nom_ok);
   }
-  //printf("'%s' %s %s\n",save,adr,fil);
 
   return 0;
 }
@@ -1686,10 +1794,10 @@ char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
     StringBuff(opt->path_log), digest_filename);
 }
 
-/* remove refname if any */
-void url_savename_refname_remove(httrackp * opt, const char *adr,
-                                 const char *fil) {
+/* remove refname if any; HTS_TRUE if it was removed */
+hts_boolean url_savename_refname_remove(httrackp *opt, const char *adr,
+                                        const char *fil) {
   char *filename = url_savename_refname_fullpath(opt, adr, fil);
 
-  (void) UNLINK(filename);
+  return UNLINK(filename) == 0 ? HTS_TRUE : HTS_FALSE;
 }

src/htsname.h

@@ -100,12 +100,15 @@ void standard_name(char *b, size_t bsize, const char *dot_pos,
                    const char *nom_pos, const char *fil_complete,
                    int short_ver);
 void url_savename_addstr(char *d, const char *s);
+/* Contested wire-vs-ext verdict that a body sniff could settle (htssniff.h). */
+int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime, const char *file);
 char *url_md5(char *digest_buffer, const char *fil_complete);
 void url_savename_refname(const char *adr, const char *fil, char *filename);
 char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
                                     const char *fil);
-void url_savename_refname_remove(httrackp * opt, const char *adr,
-                                 const char *fil);
+/* Remove the temp-ref for (adr,fil); HTS_TRUE if it was removed. */
+hts_boolean url_savename_refname_remove(httrackp *opt, const char *adr,
+                                        const char *fil);
 #endif
 
 #endif

src/htsnet.h

@@ -112,10 +112,10 @@ struct SOCaddr {
 
 /** Pointer to the port field (network byte order) for the active family.
     Asserts on NULL or an unset/unknown family. */
-static HTS_INLINE HTS_UNUSED in_port_t* SOCaddr_sinport_(SOCaddr *const addr,
-                                                         const char *file, const int line) {
+static HTS_INLINE HTS_UNUSED in_port_t *
+SOCaddr_sinport_(SOCaddr *const addr, const char *file, const int line) {
   assertf_(addr != NULL, file, line);
-  switch(addr->m_addr.sa.sa_family) {
+  switch (addr->m_addr.sa.sa_family) {
   case AF_INET:
     return &addr->m_addr.in.sin_port;
     break;
@@ -125,7 +125,7 @@ static HTS_INLINE HTS_UNUSED in_port_t* SOCaddr_sinport_(SOCaddr *const addr,
     break;
 #endif
   default:
-    assertf_(! "invalid structure", file, line);
+    assertf_(!"invalid structure", file, line);
     return 0;
     break;
   }
@@ -133,10 +133,11 @@ static HTS_INLINE HTS_UNUSED in_port_t* SOCaddr_sinport_(SOCaddr *const addr,
 
 /** Length of the active sockaddr (sockaddr_in or sockaddr_in6), or 0 if the
     family is unset/unknown. The 0 case doubles as the "not valid" test. */
-static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_size_(const SOCaddr*const addr,
-                                                     const char *file, const int line) {
+static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_size_(const SOCaddr *const addr,
+                                                     const char *file,
+                                                     const int line) {
   assertf_(addr != NULL, file, line);
-  switch(addr->m_addr.sa.sa_family) {
+  switch (addr->m_addr.sa.sa_family) {
   case AF_INET:
     return sizeof(addr->m_addr.in);
     break;
@@ -152,8 +153,8 @@ static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_size_(const SOCaddr*const addr,
 }
 
 /** Reset to the unset state (family AF_UNSPEC), making the address invalid. */
-static HTS_INLINE HTS_UNUSED void SOCaddr_clear_(SOCaddr*const addr,
-                                                 const char *file, const int line) {
+static HTS_INLINE HTS_UNUSED void
+SOCaddr_clear_(SOCaddr *const addr, const char *file, const int line) {
   assertf_(addr != NULL, file, line);
   addr->m_addr.sa.sa_family = AF_UNSPEC;
 }
@@ -191,14 +192,16 @@ static HTS_INLINE HTS_UNUSED void SOCaddr_clear_(SOCaddr*const addr,
 
 /** Set the port (host-order argument, stored network-order) on the active
  * family. */
-#define SOCaddr_initport(server, port) do { \
-  SOCaddr_sinport(server) = htons((in_port_t) (port)); \
-} while(0)
+#define SOCaddr_initport(server, port)                                         \
+  do {                                                                         \
+    SOCaddr_sinport(server) = htons((in_port_t) (port));                       \
+  } while (0)
 
 /** Initialize as an all-zero IPv4 wildcard (INADDR_ANY) address; returns its
     sockaddr length. */
-static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_initany_(SOCaddr*const addr,
-                                                        const char *file, const int line) {
+static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_initany_(SOCaddr *const addr,
+                                                        const char *file,
+                                                        const int line) {
   assertf_(addr != NULL, file, line);
   memset(&addr->m_addr.in, 0, sizeof(addr->m_addr.in));
   addr->m_addr.in.sin_family = AF_INET;
@@ -206,17 +209,20 @@ static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_initany_(SOCaddr*const addr,
 }
 
 /** Initialize server as an IPv4 wildcard (INADDR_ANY) address. */
-#define SOCaddr_initany(server) do { \
-  SOCaddr_initany_(&(server), __FILE__, __LINE__); \
-} while(0)
+#define SOCaddr_initany(server)                                                \
+  do {                                                                         \
+    SOCaddr_initany_(&(server), __FILE__, __LINE__);                           \
+  } while (0)
 
 /** Populate server from data. data_size selects the source form: a full
     sockaddr_in / sockaddr_in6, or a raw 4-byte (IPv4) / 16-byte (IPv6) address
     with port zeroed. Any other size leaves an AF_INET shell. Returns the
     resulting sockaddr length. */
-static HTS_UNUSED socklen_t SOCaddr_copyaddr_(SOCaddr*const server, 
-                                              const void *data, const size_t data_size,
-                                              const char *file, const int line) {
+static HTS_UNUSED socklen_t SOCaddr_copyaddr_(SOCaddr *const server,
+                                              const void *data,
+                                              const size_t data_size,
+                                              const char *file,
+                                              const int line) {
   assertf_(server != NULL, file, line);
   assertf_(data != NULL, file, line);
 
@@ -248,32 +254,35 @@ static HTS_UNUSED socklen_t SOCaddr_copyaddr_(SOCaddr*const server,
 
 /** Copy hpaddr (length hpsize) into server, writing the result length into the
     lvalue server_len (int). See SOCaddr_copyaddr_ for accepted forms. */
-#define SOCaddr_copyaddr(server, server_len, hpaddr, hpsize) do { \
-  server_len = (int) SOCaddr_copyaddr_(&(server), hpaddr, hpsize, __FILE__, __LINE__); \
-} while(0)
+#define SOCaddr_copyaddr(server, server_len, hpaddr, hpsize)                   \
+  do {                                                                         \
+    server_len = (int) SOCaddr_copyaddr_(&(server), hpaddr, hpsize, __FILE__,  \
+                                         __LINE__);                            \
+  } while (0)
 
 /** Like SOCaddr_copyaddr but discards the result length. */
-#define SOCaddr_copyaddr2(server, hpaddr, hpsize) do { \
-  (void) SOCaddr_copyaddr_(&(server), hpaddr, hpsize, __FILE__, __LINE__); \
-} while(0)
+#define SOCaddr_copyaddr2(server, hpaddr, hpsize)                              \
+  do {                                                                         \
+    (void) SOCaddr_copyaddr_(&(server), hpaddr, hpsize, __FILE__, __LINE__);   \
+  } while (0)
 
 /** Copy one SOCaddr (src) into another (dest), preserving family and port. */
-#define SOCaddr_copy_SOCaddr(dest, src) do { \
-  SOCaddr_copyaddr_(&(dest), &(src).m_addr.sa, SOCaddr_size(src), __FILE__, __LINE__); \
-} while(0)
+#define SOCaddr_copy_SOCaddr(dest, src)                                        \
+  do {                                                                         \
+    SOCaddr_copyaddr_(&(dest), &(src).m_addr.sa, SOCaddr_size(src), __FILE__,  \
+                      __LINE__);                                               \
+  } while (0)
 
 /** Write the numeric (dotted/colon) host of ss into namebuf (capacity
     namebuflen), scope id stripped. On failure namebuf becomes "". */
-static HTS_UNUSED void SOCaddr_inetntoa_(char *namebuf, size_t namebuflen, 
-                                         SOCaddr *const ss,
-                                         const char *file, const int line) {
+static HTS_UNUSED void SOCaddr_inetntoa_(char *namebuf, size_t namebuflen,
+                                         SOCaddr *const ss, const char *file,
+                                         const int line) {
   assertf_(namebuf != NULL, file, line);
   assertf_(ss != NULL, file, line);
 
-  if (getnameinfo(&ss->m_addr.sa, sizeof(ss->m_addr),
-                  namebuf, namebuflen, 
-                  NULL, 0, 
-                  NI_NUMERICHOST) == 0) {
+  if (getnameinfo(&ss->m_addr.sa, sizeof(ss->m_addr), namebuf, namebuflen, NULL,
+                  0, NI_NUMERICHOST) == 0) {
     /* remove scope id(s) */
     char *const pos = strchr(namebuf, '%');
     if (pos != NULL) {
@@ -285,15 +294,32 @@ static HTS_UNUSED void SOCaddr_inetntoa_(char *namebuf, size_t namebuflen,
 }
 
 /** Numeric host of ss into namebuf (capacity namebuflen); "" on failure. */
-#define SOCaddr_inetntoa(namebuf, namebuflen, ss) \
+#define SOCaddr_inetntoa(namebuf, namebuflen, ss)                              \
   SOCaddr_inetntoa_(namebuf, namebuflen, &(ss), __FILE__, __LINE__)
 
 /** Single-char family tag: '1' for IPv4, '2' otherwise (used in the cache). */
-#define SOCaddr_getproto(ss) ( SOCaddr_size(ss) == sizeof(struct sockaddr_in) ? '1' : '2')
+#define SOCaddr_getproto(ss)                                                   \
+  (SOCaddr_size(ss) == sizeof(struct sockaddr_in) ? '1' : '2')
 
 /** Length type for socket APIs (getsockname, accept, ...). */
 typedef socklen_t SOClen;
 
+#if HTS_INET6 != 0
+/** Resolver backend: getaddrinfo/freeaddrinfo as a swappable pair, so the
+    self-test can script DNS answers (families, multiplicity, errors)
+    in-process. The free function must match its getaddrinfo (a fake allocates
+    its own chain), hence the pair. */
+typedef struct hts_resolver_backend {
+  int (*getaddrinfo)(const char *node, const char *service,
+                     const struct addrinfo *hints, struct addrinfo **res);
+  void (*freeaddrinfo)(struct addrinfo *res);
+} hts_resolver_backend;
+
+/** Install a resolver backend for the process; NULL restores the libc default.
+    Test-only seam, not thread-safe; callers must serialize against resolves. */
+void hts_dns_set_resolver_backend(const hts_resolver_backend *backend);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

src/htsopt.h

@@ -72,6 +72,7 @@ typedef struct String String;
 #endif
 #ifndef HTS_DEF_STRUCT_String
 #define HTS_DEF_STRUCT_String
+
 struct String {
   char *buffer_;
   size_t length_;
@@ -80,7 +81,7 @@ struct String {
 #endif
 
 /* Defines */
-#define CATBUFF_SIZE (STRING_SIZE*2*2)
+#define CATBUFF_SIZE (STRING_SIZE * 2 * 2)
 
 #define STRING_SIZE 2048
 
@@ -104,11 +105,10 @@ typedef struct htsfilters htsfilters;
 struct htsfilters {
   char ***filters; /**< pointer to the +/-pattern filter array */
   int *filptr;     /**< pointer to the current filter count */
-  // int*    filter_max;
 };
 
 /* User callbacks chain */
-typedef int (*htscallbacksfncptr) (void);
+typedef int (*htscallbacksfncptr)(void);
 
 typedef struct htscallbacks htscallbacks;
 
@@ -179,6 +179,7 @@ typedef struct lien_url lien_url;
 
 #ifndef HTS_DEF_DEFSTRUCT_hts_log_type
 #define HTS_DEF_DEFSTRUCT_hts_log_type
+
 typedef enum hts_log_type {
   LOG_PANIC,
   LOG_ERROR,
@@ -239,7 +240,7 @@ struct htsoptstate {
   char *userhttptype;
   int verif_backblue_done; /**< backblue.gif/fade.gif already emitted */
   int verif_external_status;
-  t_dnscache *dns_cache;  /**< DNS resolution cache */
+  coucal dns_cache; /**< DNS resolution cache: hostname -> t_dnscache record */
   int dns_cache_nthreads; /**< number of in-flight DNS resolver threads */
   /* HTML parsing state */
   char _hts_errmsg[HTS_CDLMAXSIZE + 256]; /**< last engine error message */
@@ -278,16 +279,17 @@ struct htslibhandles {
 
 /* Javascript parser flags */
 typedef enum htsparsejava_flags {
-  HTSPARSE_NONE = 0,            // don't parse
-  HTSPARSE_DEFAULT = 1,         // parse default (all)
-  HTSPARSE_NO_CLASS = 2,        // don't parse .java
-  HTSPARSE_NO_JAVASCRIPT = 4,   // don't parse .js
-  HTSPARSE_NO_AGGRESSIVE = 8    // don't aggressively parse .js or .java
+  HTSPARSE_NONE = 0,          // don't parse
+  HTSPARSE_DEFAULT = 1,       // parse default (all)
+  HTSPARSE_NO_CLASS = 2,      // don't parse .java
+  HTSPARSE_NO_JAVASCRIPT = 4, // don't parse .js
+  HTSPARSE_NO_AGGRESSIVE = 8  // don't aggressively parse .js or .java
 } htsparsejava_flags;
 
 /* Link-rewriting style for saved pages (opt->urlmode). */
 #ifndef HTS_DEF_DEFSTRUCT_hts_urlmode
 #define HTS_DEF_DEFSTRUCT_hts_urlmode
+
 typedef enum hts_urlmode {
   HTS_URLMODE_ABSOLUTE = 0, /**< absolute URL (http://host/path) everywhere */
   HTS_URLMODE_ABSOLUTE_FILE = 1, /**< legacy file: form, unused */
@@ -301,6 +303,7 @@ typedef enum hts_urlmode {
 /* Cache policy for updates and retries (opt->cache). */
 #ifndef HTS_DEF_DEFSTRUCT_hts_cachemode
 #define HTS_DEF_DEFSTRUCT_hts_cachemode
+
 typedef enum hts_cachemode {
   HTS_CACHE_NONE = 0,       /**< no cache */
   HTS_CACHE_PRIORITY = 1,   /**< cache takes priority over the network */
@@ -311,6 +314,7 @@ typedef enum hts_cachemode {
 /* Interactive wizard level (opt->wizard). */
 #ifndef HTS_DEF_DEFSTRUCT_hts_wizard
 #define HTS_DEF_DEFSTRUCT_hts_wizard
+
 typedef enum hts_wizard {
   HTS_WIZARD_NONE = 0, /**< no wizard */
   HTS_WIZARD_ASK = 1,  /**< wizard asks questions */
@@ -321,6 +325,7 @@ typedef enum hts_wizard {
 /* robots.txt / meta-robots obedience level (opt->robots). */
 #ifndef HTS_DEF_DEFSTRUCT_hts_robots
 #define HTS_DEF_DEFSTRUCT_hts_robots
+
 typedef enum hts_robots {
   HTS_ROBOTS_NEVER = 0,        /**< ignore robots rules */
   HTS_ROBOTS_SOMETIMES = 1,    /**< partial obedience (default) */
@@ -342,17 +347,17 @@ typedef enum hts_seeker {
   HTS_SEEKER_UP = 1 << 1    /**< may ascend to parent directories */
 } hts_seeker;
 
-/* Link-following scope, stored in the low byte of opt->travel. */
+/* opt->travel: link-following scope in the low byte, flags OR'd in above it. */
 typedef enum hts_travel_scope {
   HTS_TRAVEL_SAME_ADDRESS = 0, /**< stay on the same address (host) */
   HTS_TRAVEL_SAME_DOMAIN = 1,  /**< stay on the same principal domain */
   HTS_TRAVEL_SAME_TLD = 2,     /**< stay on the same TLD (e.g. .com) */
-  HTS_TRAVEL_EVERYWHERE = 7    /**< follow links anywhere on the web */
+  HTS_TRAVEL_EVERYWHERE = 7,   /**< follow links anywhere on the web */
+  HTS_TRAVEL_TEST_ALL = 1 << 8 /**< also test forbidden URLs (-t) */
 } hts_travel_scope;
 
-/* Flags OR'd into opt->travel above the scope value. */
-#define HTS_TRAVEL_SCOPE_MASK 0xff   /**< mask selecting the scope value */
-#define HTS_TRAVEL_TEST_ALL (1 << 8) /**< also test forbidden URLs (-t) */
+/* Mask selecting the scope value out of opt->travel. */
+#define HTS_TRAVEL_SCOPE_MASK 0xff
 
 /* Text progress display detail (opt->verbosedisplay). */
 typedef enum hts_verbosedisplay {
@@ -368,6 +373,13 @@ typedef enum hts_savename_delayed {
   HTS_SAVENAME_DELAYED_HARD = 2  /**< always delay the type check (default) */
 } hts_savename_delayed;
 
+/* Saved-name length layout (opt->savename_83). */
+typedef enum hts_savename_83 {
+  HTS_SAVENAME_83_LONG = 0,   /**< long file names (default) */
+  HTS_SAVENAME_83_DOS = 1,    /**< DOS 8.3 names (ISO9660 level 1) */
+  HTS_SAVENAME_83_ISO9660 = 2 /**< ISO9660 level 2 names (up to 31 chars) */
+} hts_savename_83;
+
 /* Host-banning triggers (opt->hostcontrol bitmask). */
 typedef enum hts_hostcontrol {
   HTS_HOSTCONTROL_BAN_TIMEOUT = 1 << 0, /**< ban a timing-out host */
@@ -399,38 +411,38 @@ struct httrackp {
   /* */
   hts_wizard wizard; /**< interactive wizard level (none/ask/auto) */
   hts_boolean flush; /**< fflush() log files after each write */
-  int travel;   /**< link-following scope (same domain, etc.) */
-  int seeker;   /**< allowed direction: go up and/or down the tree */
-  int depth;    /**< maximum recursion depth (-rN) */
-  int extdepth; /**< maximum recursion depth outside the start domain */
+  int travel;        /**< link-following scope (same domain, etc.) */
+  int seeker;        /**< allowed direction: go up and/or down the tree */
+  int depth;         /**< maximum recursion depth (-rN) */
+  int extdepth;      /**< maximum recursion depth outside the start domain */
   hts_urlmode
       urlmode; /**< saved-link rewriting style (relative, absolute, etc.) */
-  hts_boolean no_type_change;   // do not change file type according to MIME
-  hts_log_type debug;           /**< debug logging level */
-  int getmode;           /**< what to fetch (HTML, images, ...) bitmask */
-  FILE *log;             /**< informational log stream; NULL mutes it */
-  FILE *errlog;          /**< error log stream; NULL mutes it */
-  LLint maxsite;         /**< max total bytes for the whole mirror */
-  LLint maxfile_nonhtml; /**< max bytes per non-HTML file */
-  LLint maxfile_html;    /**< max bytes per HTML file */
-  int maxsoc;            /**< max simultaneous sockets (-cN) */
-  LLint fragment;        /**< split site after this many bytes */
-  hts_boolean
+  hts_boolean no_type_change; // do not change file type according to MIME
+  hts_log_type debug;         /**< debug logging level */
+  int getmode;                /**< what to fetch (HTML, images, ...) bitmask */
+  FILE *log;                  /**< informational log stream; NULL mutes it */
+  FILE *errlog;               /**< error log stream; NULL mutes it */
+  LLint maxsite;              /**< max total bytes for the whole mirror */
+  LLint maxfile_nonhtml;      /**< max bytes per non-HTML file */
+  LLint maxfile_html;         /**< max bytes per HTML file */
+  int maxsoc;                 /**< max simultaneous sockets (-cN) */
+  LLint fragment;             /**< split site after this many bytes */
+  hts_tristate
       nearlink; /**< also fetch images/data adjacent to a page but off-site */
   hts_boolean makeindex;  /**< build a top-level index.html */
   hts_boolean kindex;     /**< build a keyword index */
-  hts_boolean delete_old; /**< delete locally obsolete files after update */
-  int timeout;    /**< connection timeout in seconds */
-  int rateout;    /**< minimum transfer rate (bytes/s) before abort */
-  int maxtime;    /**< max total mirror duration in seconds */
-  int maxrate;    /**< max transfer rate cap (bytes/s) */
-  float maxconn;  /**< max connections per second */
-  int waittime;   /**< scheduled start time (wall-clock seconds) */
-  hts_cachemode cache; /**< cache generation mode */
-  // int aff_progress;     // progress bar
+  hts_tristate delete_old; /**< delete locally obsolete files after update */
+  int timeout;            /**< connection timeout in seconds */
+  int rateout;            /**< minimum transfer rate (bytes/s) before abort */
+  int maxtime;            /**< max total mirror duration in seconds */
+  int maxrate;            /**< max transfer rate cap (bytes/s) */
+  float maxconn;          /**< max connections per second */
+  int waittime;           /**< scheduled start time (wall-clock seconds) */
+  hts_cachemode cache;    /**< cache generation mode */
   hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
   t_proxy proxy;     /**< proxy configuration */
-  int savename_83;   /**< force 8.3 (DOS) file names */
+  hts_savename_83
+      savename_83;   /**< saved-name length layout (long/DOS/ISO9660) */
   int savename_type; /**< saved-name layout (original tree, flat, ...) */
   String
       savename_userdef; /**< user-defined name template (e.g. %h%p/%n%q.%t) */
@@ -438,12 +450,12 @@ struct httrackp {
   hts_boolean
       delayed_cached;   // delayed type check can be cached to speedup updates
   hts_boolean mimehtml; /**< produce a single MIME/MHTML archive */
-  hts_boolean user_agent_send;  /**< send a User-Agent header */
-  String user_agent;            /**< User-Agent value (e.g. httrack/1.0) */
-  String referer;               /**< Referer value to send */
-  String from;                  /**< From value to send */
-  String path_log;              /**< directory for cache and logs */
-  String path_html;             /**< output directory for the mirror */
+  hts_boolean user_agent_send; /**< send a User-Agent header */
+  String user_agent;           /**< User-Agent value (e.g. httrack/1.0) */
+  String referer;              /**< Referer value to send */
+  String from;                 /**< From value to send */
+  String path_log;             /**< directory for cache and logs */
+  String path_html;            /**< output directory for the mirror */
   String path_html_utf8; /**< output directory for the mirror, UTF-8 form */
   String path_bin;       /**< directory for HTML templates */
   int retry;             /**< extra retries on a failed transfer */
@@ -451,49 +463,48 @@ struct httrackp {
   hts_boolean maketrack; /**< maintain an operations-statistics log */
   int parsejava;         /**< Java/JS parsing mode; see htsparsejava_flags */
   int hostcontrol; /**< ban slow/timing-out hosts; see hts_hostcontrol bits */
-  hts_boolean errpage;   /**< generate an error page on 404 and similar */
+  hts_tristate errpage; /**< generate an error page on 404 and similar */
   hts_boolean
       check_type; /**< probe unknown-type links (cgi/asp/dir) and follow moves
                    */
-  hts_boolean all_in_cache; /**< keep all retrieved data in the cache */
-  hts_robots robots; /**< robots.txt handling level */
-  hts_boolean external;          /**< render external links as error pages */
+  hts_boolean all_in_cache;      /**< keep all retrieved data in the cache */
+  hts_robots robots;             /**< robots.txt handling level */
+  hts_tristate external;         /**< render external links as error pages */
   hts_boolean passprivacy;       /**< strip passwords from external links */
   hts_boolean includequery;      /**< include the query string in saved names */
   hts_boolean mirror_first_page; /**< only mirror the links of the first page */
-  String sys_com;               /**< system command to run */
-  hts_boolean sys_com_exec;     /**< actually execute sys_com */
-  hts_boolean accept_cookie;    /**< accept and send cookies */
-  t_cookie *cookie;             /**< cookie store */
-  hts_boolean http10;           /**< force HTTP/1.0 */
-  hts_boolean nokeepalive;      /**< disable keep-alive */
-  hts_boolean nocompression;    /**< disable content compression */
-  hts_boolean sizehack;         /**< treat same-size response as "updated" */
-  hts_boolean urlhack;          // force "url normalization" to avoid loops
-  hts_boolean tolerant;         /**< accept an incorrect Content-Length */
-  hts_boolean
+  String sys_com;                /**< system command to run */
+  hts_boolean sys_com_exec;      /**< actually execute sys_com */
+  hts_boolean accept_cookie;     /**< accept and send cookies */
+  t_cookie *cookie;              /**< cookie store */
+  hts_boolean http10;            /**< force HTTP/1.0 */
+  hts_boolean nokeepalive;       /**< disable keep-alive */
+  hts_boolean nocompression;     /**< disable content compression */
+  hts_boolean sizehack;          /**< treat same-size response as "updated" */
+  hts_boolean urlhack;           // force "url normalization" to avoid loops
+  hts_boolean tolerant;          /**< accept an incorrect Content-Length */
+  hts_tristate
       parseall; /**< parse aggressively, including unknown tags with links */
   hts_boolean parsedebug; /**< parser debug mode */
   hts_boolean norecatch;  /**< do not re-fetch files the user deleted locally */
   hts_verbosedisplay verbosedisplay; /**< animated text progress display */
-  String footer;      /**< footer/info line injected into pages */
-  int maxcache;       /**< in-memory cache backing limit (bytes) */
-  // int maxcache_anticipate; // maximum links to anticipate (upper bound)
-  hts_boolean ftp_proxy;        /**< use the HTTP proxy for FTP too */
-  String filelist;              /**< file listing URLs to include */
-  String urllist;               /**< file listing filters to include */
-  htsfilters filters;           /**< filter pointers (+/-pattern rules) */
-  hash_struct *hash;            // hash structure
-  lien_url **liens;             // links
-  int lien_tot;                 // top index of "links" heap (always out-of-range)
-  lien_buffers *liensbuf;       // links buffers
-  robots_wizard *robotsptr;     // robots ptr
-  String lang_iso;              /**< Accept-Language value (en, fr, ...) */
-  String accept;                // Accept:
-  String headers;               // Additional headers
-  String mimedefs;              // ext1=mimetype1\next2=mimetype2..
-  String mod_blacklist;         /**< blacklisted modules */
-  hts_boolean convert_utf8;     // filenames UTF-8 conversion (3.46)
+  String footer; /**< footer/info line injected into pages */
+  int maxcache;  /**< in-memory cache backing limit (bytes) */
+  hts_boolean ftp_proxy;    /**< use the HTTP proxy for FTP too */
+  String filelist;          /**< file listing URLs to include */
+  String urllist;           /**< file listing filters to include */
+  htsfilters filters;       /**< filter pointers (+/-pattern rules) */
+  hash_struct *hash;        // hash structure
+  lien_url **liens;         // links
+  int lien_tot;             // top index of "links" heap (always out-of-range)
+  lien_buffers *liensbuf;   // links buffers
+  robots_wizard *robotsptr; // robots ptr
+  String lang_iso;          /**< Accept-Language value (en, fr, ...) */
+  String accept;            // Accept:
+  String headers;           // Additional headers
+  String mimedefs;          // ext1=mimetype1\next2=mimetype2..
+  String mod_blacklist;     /**< blacklisted modules */
+  hts_boolean convert_utf8; // filenames UTF-8 conversion (3.46)
   //
   int maxlink;   /**< max number of links */
   int maxfilter; /**< max number of filters */
@@ -515,6 +526,16 @@ struct httrackp {
   htslibhandles libHandles; /**< loaded external module handles */
   //
   htsoptstate state; /**< embedded live engine state */
+  String strip_query; /**< query keys to drop when deduping URLs (-strip-query);
+                           appended at the tail to keep field offsets stable */
+  hts_boolean
+      no_www_dedup; /**< with urlhack, keep www.host distinct from host */
+  hts_boolean no_slash_dedup; /**< with urlhack, keep redundant // in paths */
+  hts_boolean no_query_dedup; /**< with urlhack, keep query-argument order */
+  String cookies_file;        /**< extra Netscape cookies.txt to preload
+                                 (--cookies-file) */
+  int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
+  int pause_max_ms; /**< inter-file pause upper bound, ms */
 };
 
 /* Running statistics for a mirror. */
@@ -525,8 +546,6 @@ typedef struct hts_stat_struct hts_stat_struct;
 struct hts_stat_struct {
   LLint HTS_TOTAL_RECV; /**< total bytes received from the network */
   LLint stat_bytes;     /**< total bytes written to disk */
-  // int HTS_TOTAL_RECV_STATE;  // status: 0 ok 1: slow down a little 2: slow
-  // down 3: a lot
   TStamp stat_timestart; /**< mirror start time */
   //
   LLint total_packed;    /**< compressed bytes received (on the wire) */
@@ -579,17 +598,17 @@ typedef struct htsrequest htsrequest;
 struct htsrequest {
   short int user_agent_send; /**< send a User-Agent header */
   short int http11; /**< sign the request as HTTP/1.1 rather than HTTP/1.0 */
-  short int nokeepalive;        /**< disable keep-alive */
-  short int range_used;         /**< a Range header is in use */
-  short int nocompression;      /**< disable compression */
-  short int flush_garbage;      // recycled
-  const char *user_agent;       /**< User-Agent value */
-  const char *referer;          /**< Referer value */
-  const char *from;             /**< From value */
-  const char *lang_iso;         /**< Accept-Language value */
-  const char *accept;           /**< Accept value */
-  const char *headers;          /**< extra request headers */
-  htsrequest_proxy proxy;       /**< proxy for this request */
+  short int nokeepalive;   /**< disable keep-alive */
+  short int range_used;    /**< a Range header is in use */
+  short int nocompression; /**< disable compression */
+  short int flush_garbage; // recycled
+  const char *user_agent;  /**< User-Agent value */
+  const char *referer;     /**< Referer value */
+  const char *from;        /**< From value */
+  const char *lang_iso;    /**< Accept-Language value */
+  const char *accept;      /**< Accept value */
+  const char *headers;     /**< extra request headers */
+  htsrequest_proxy proxy;  /**< proxy for this request */
 };
 
 /* Result of a connection / header fetch. */
@@ -621,8 +640,8 @@ struct htsblk {
   short int is_file; /**< 1 if a file descriptor rather than a socket */
   T_SOC soc;         /**< socket id */
   SOCaddr address;   /**< peer IP address */
-  int address_size;             // IP address structure length (unused internally)
-  FILE *fp;                     /**< file handle for file:// */
+  int address_size;  // IP address structure length (unused internally)
+  FILE *fp;          /**< file handle for file:// */
 #if HTS_USEOPENSSL
   short int ssl; /**< nonzero if this is an SSL connection (https) */
   // BIO* ssl_soc;          // SSL structure
@@ -658,7 +677,6 @@ struct lien_url {
   int depth;        /**< remaining allowed depth; >0 strong, 0 weak */
   int pass2;        /**< second-pass marker; -1 means handled in background */
   char link_import; /**< imported after a move; skip the usual up/down rules */
-  // int moved;          // pointer to moved
   int retry;    /**< remaining retries */
   int testmode; /**< test only: send just a HEAD */
 };
@@ -703,9 +721,6 @@ struct lien_back {
   LLint chunk_size; /**< size of the chunk being loaded */
   LLint chunk_blocksize; /**< data size declared by the chunk */
   LLint compressed_size; /**< compressed size (stats only) */
-  //
-  //int links_index;        // to access liens[links_index]
-  //
   char info[256]; /**< status text, e.g. for FTP */
   int stop_ftp;   /**< stop flag for FTP */
   int finalized;  /**< finalized (memory optimization) */