Add a CodeQL C analysis workflow

Scan on push to master, PRs, and a weekly cron, with the security-extended
query suite; findings land in the code-scanning dashboard. Manual build mode
so CodeQL traces the real autotools build.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.clang-format

@@ -16,6 +16,7 @@ BasedOnStyle: LLVM
 SpaceAfterCStyleCast: true   # "(int) x", overwhelmingly dominant (542 vs 7)
 SortIncludes: false          # C include order can be significant; never reorder
 IncludeBlocks: Preserve      # do not merge/reflow include groups
+SeparateDefinitionBlocks: Always  # blank line between definitions (readability)
 
 # Stated explicitly for robustness against base-style drift (these match LLVM):
 IndentWidth: 2

.flake8

@@ -0,0 +1,5 @@
+[flake8]
+# Match black's formatting so the two tools don't fight.
+max-line-length = 88
+# E203/W503 conflict with black's slice and line-break style.
+extend-ignore = E203, W503

.github/workflows/ci.yml

@@ -61,6 +61,50 @@ jobs:
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # Reproduce the Debian buildds: they build in a minimal chroot with no
+  # python3, so the local-server tests must SKIP (exit 77), not fail. GitHub
+  # runners ship python3, so every other job hides this path; here we remove it
+  # before `make check`. This is the guard that would have caught the 3.49.10-1
+  # FTBFS (28_local-pause failed instead of skipping when python3 was absent).
+  buildd-no-python3:
+    name: build (no python3, Debian buildd)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - name: Configure
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Test without python3
+        run: |
+          set -euo pipefail
+          # Hide every python3* so `command -v python3` fails like it does in the
+          # buildd chroot; masking with /bin/false would still resolve.
+          sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
+            -exec mv {} {}.hidden \;
+          ! command -v python3
+          make check
+
+      - name: Print the test log on failure
+        if: failure()
+        run: cat tests/test-suite.log 2>/dev/null || true
+
   # Portability: build and test on macOS (Darwin/clang) on a native runner --
   # no VM. The tree has no __APPLE__ branches, so Darwin exercises the
   # generic-Unix path on a second libc and kernel. brew's openssl@3 is keg-only,
@@ -188,6 +232,91 @@ jobs:
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # MemorySanitizer catches reads of uninitialized memory (#143's stack-garbage
+  # size filter) that ASan/UBSan miss. It flags any byte an uninstrumented lib
+  # wrote, so the job stays in our own code: offline self-tests only, no openssl
+  # (--disable-https), no zlib cache tests, static (the runtime is not in .so's).
+  msan:
+    name: msan (MemorySanitizer, clang)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential clang autoconf automake libtool autoconf-archive \
+            zlib1g-dev
+
+      - name: Configure (MSan, static, no https)
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure CC=clang \
+            CFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
+            LDFLAGS="-fsanitize=memory" \
+            --disable-https --disable-shared --enable-static
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Test (offline self-tests under MSan)
+        env:
+          MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
+        run: |
+          set -euo pipefail
+          # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
+          # skipped here (uninstrumented libz floods MSan with false positives).
+          tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
+          make check TESTS="$tests"
+
+      - name: Print the test log on failure
+        if: failure()
+        run: cat tests/test-suite.log 2>/dev/null || true
+
+  # libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
+  # parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
+  # (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
+  # catastrophic backtracking -- and pins the regression seeds for the bugs the
+  # fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
+  fuzz:
+    name: fuzz (libFuzzer corpus replay, clang)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential clang autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - name: Configure (fuzzers, static)
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure CC=clang \
+            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
+            LDFLAGS="-fsanitize=address,undefined" \
+            --enable-fuzzers --disable-shared
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Replay corpora
+        env:
+          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
+          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
+        run: bash fuzz/run-fuzzers.sh fuzz check
+
   # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
   # the configuration users on minimal systems build, and one libssl is not even
   # installed here so configure cannot silently re-enable it. The matrix above
@@ -227,34 +356,47 @@ jobs:
   # Validate the Debian packaging via the same script maintainers release with.
   # One amd64/gcc run is enough: packaging (control/rules/manifest/lintian/quilt
   # source build) is arch- and compiler-independent, and the build matrix above
-  # already covers compile portability. lintian runs with --fail-on=error.
+  # already covers compile portability. mkdeb.sh runs lintian as an explicit gate
+  # (debuild does not propagate lintian's exit) with --fail-on=error,warning.
   deb:
     name: deb package (lintian)
     runs-on: ubuntu-24.04
+    # Build and gate inside Debian sid, the upload target. A Debian dpkg-deb
+    # produces archive-legal xz members (an Ubuntu host defaults to zstd, which
+    # the archive's lintian rejects), and sid's lintian carries the same
+    # data-driven checks (embedded-lib fingerprints and the like) the buildds and
+    # UDD apply -- so issues surface here instead of after upload.
+    container: debian:sid
     steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
       - name: Install packaging toolchain
         run: |
           set -euo pipefail
-          sudo apt-get update
+          apt-get update
-          sudo apt-get install -y --no-install-recommends \
+          apt-get install -y --no-install-recommends \
+            ca-certificates git \
             build-essential autoconf automake libtool autoconf-archive \
             zlib1g-dev libssl-dev \
             debhelper devscripts lintian fakeroot
 
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
       # --unsigned: CI has no GPG key (also skips the release sig/checksums).
-      # debuild builds every package, then lintian gates on errors.
+      # mkdeb builds every package then runs the lintian gate (--fail-on=error,
+      # warning); debuild runs the packaged test pass.
       #
       # DEB_BUILD_OPTIONS trims work CI does not need (release builds via
       # mkdeb.sh are untouched): noautodbgsym drops the -dbgsym packages whose
       # LTO payloads are slow to compress and that CI never ships; parallel uses
-      # every core. We let debuild run its test pass -- the only one now that
+      # every core.
-      # mkdeb no longer runs its own -- so CI exercises the packaged tests.
+      - name: Build and lint Debian packages
-      - name: Build Debian packages
         run: |
+          set -euo pipefail
+          # The workspace volume is owned by the host runner uid, but the
+          # container runs as root, so mkdeb's git calls (superproject and the
+          # coucal submodule) trip "dubious ownership"; mark them all safe.
+          git config --global --add safe.directory "*"
           export DEB_BUILD_OPTIONS="noautodbgsym parallel=$(nproc)"
           bash tools/mkdeb.sh --unsigned --no-release-artifacts
 
@@ -320,6 +462,21 @@ jobs:
   lint:
     name: lint (shellcheck, shfmt)
     runs-on: ubuntu-24.04
+    # Every tracked shell script; the globs expand at run time. Kept here so the
+    # shellcheck and shfmt steps below cannot drift apart.
+    env:
+      SHELL_SCRIPTS: >-
+        .githooks/pre-commit
+        bootstrap
+        build.sh
+        html/div/search.sh
+        man/makeman.sh
+        src/htsbasiccharsets.sh
+        src/htsentities.sh
+        src/webhttrack
+        tests/*.sh
+        tests/*.test
+        tools/mkdeb.sh
     steps:
       - uses: actions/checkout@v6
 
@@ -332,12 +489,11 @@ jobs:
           sudo apt-get install -y --no-install-recommends shellcheck shfmt
           shfmt --version
 
-      # Lint the scripts we maintain; the legacy scripts are a separate cleanup.
       - name: shellcheck
-        run: shellcheck man/makeman.sh tools/mkdeb.sh .githooks/pre-commit tests/*.test tests/check-network.sh
+        run: shellcheck $SHELL_SCRIPTS
 
       - name: shfmt
-        run: shfmt -d -i 4 man/makeman.sh tools/mkdeb.sh .githooks/pre-commit
+        run: shfmt -d -i 4 $SHELL_SCRIPTS
 
   # Check clang-format on CHANGED LINES ONLY. The engine predates clang-format
   # (it was shaped by an old Visual Studio formatter) and does not round-trip,

.github/workflows/codeql.yml

@@ -0,0 +1,57 @@
+# CodeQL static analysis (C). The security-extended suite covers the classes
+# this codebase actually fights: overflows, tainted-size allocs, format bugs.
+name: CodeQL
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+  schedule:
+    # Weekly re-scan of master so new/updated queries land without a push.
+    - cron: "17 4 * * 1"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: codeql-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: analyze (c-cpp)
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      # Upload findings to the repo's code-scanning dashboard.
+      security-events: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - uses: github/codeql-action/init@v4
+        with:
+          languages: c-cpp
+          build-mode: manual
+          queries: security-extended
+
+      # Manual build: CodeQL traces the compiler, so build exactly what ships.
+      - name: Build
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure
+          make -j"$(nproc)"
+
+      - uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:c-cpp"

.gitignore

@@ -39,3 +39,6 @@ Makefile
 
 # Editor / autotools backup files.
 *~
+
+# Python bytecode (tests/local-server.py).
+__pycache__/

AGENTS.md

@@ -33,8 +33,9 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 - Be terse. Comment the why, in English; translate French comments you touch.
 - Strip AI tells from prose (em-dash overuse, rule-of-three, filler, vague
   attributions). Ref: Wikipedia "Signs of AI writing". Claude Code: `/humanizer`.
-- Behavior change → add a test. Fast path: a hidden `httrack -#N` debug
+- Behavior change → add a test. Fast path: a hidden `httrack -#test=NAME` engine
-  subcommand (`htscoremain.c`) driven by a `tests/NN_*.test`, over a slow crawl.
+  self-test (registry in `htsselftest.c`; `-#test` lists them) driven by a
+  `tests/NN_*.test`, over a slow crawl.
 
 ## Review your change adversarially (strongly suggested)
 Before pushing, and when reviewing others, don't skim for bugs:

CONTRIBUTING.md

@@ -39,6 +39,10 @@ Welcome, and nothing to disclose. Two rules:
 
 The sign-off covers AI-assisted code too.
 
+## Translations
+
+Interface strings live in [`lang/`](lang/). See [lang/README.md](lang/README.md) for the file format and how to add or update a language.
+
 ## Bugs
 
 Open an issue with the version, OS, command used, and expected vs actual result.

Makefile.am

@@ -1,5 +1,5 @@
 
-SUBDIRS = src man m4 libtest templates lang html tests
+SUBDIRS = src man m4 libtest templates lang html tests fuzz
 
 ACLOCAL_AMFLAGS = -I m4
 

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.8], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,9 +29,11 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:0:0: htsblk layout changed (contenttype/charset/contentencoding widened to
+# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
-# 128), an incompatible ABI break, so bump current and reset revision/age.
+# installed headers (no struct layout or exported signature changed vs
-VERSION_INFO="3:0:0"
+# 3.49.10), so it stays soname .so.3; bump revision.
+# (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
+VERSION_INFO="3:3:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 
@@ -62,6 +64,16 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
 # Export version info
 AC_SUBST(VERSION_INFO)
 
+# Versioned plugin name for dlopen() in hts_create_opt(); soname major is
+# libtool's current - age, so this tracks VERSION_INFO bumps automatically.
+HTS_SONAME_MAJOR=$((${VERSION_INFO%%:*} - ${VERSION_INFO##*:}))
+case "$host_os" in
+darwin*) HTS_LIBHTSJAVA_NAME="libhtsjava.$HTS_SONAME_MAJOR.dylib" ;;
+*) HTS_LIBHTSJAVA_NAME="libhtsjava.so.$HTS_SONAME_MAJOR" ;;
+esac
+AC_DEFINE_UNQUOTED([HTS_LIBHTSJAVA_NAME], ["$HTS_LIBHTSJAVA_NAME"],
+  [Versioned libhtsjava runtime name, derived from VERSION_INFO])
+
 ### Default CFLAGS
 DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
 -Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
@@ -214,9 +226,12 @@ AC_SUBST(OPENSSL_LIBS)
 fi
 
 ### Support IPv6
+V6_SUPPORT=no
 AC_CHECK_LIB(c, getaddrinfo, [V6_FLAG="-DINET6"
+V6_SUPPORT=yes
 AC_DEFINE(HTS_INET6, 1, [Check for IPv6])], AC_MSG_WARN([*** IPv6 not found IPv6 compatibility disabled]))
 AC_SUBST(V6_FLAG)
+AC_SUBST(V6_SUPPORT)
 
 ### Check for LFS
 AC_CHECK_LIB(c, fopen64, [LFS_FLAG="-D_FILE_OFFSET_BITS=64 -D_LARGEFILE_SOURCE -D_LARGEFILE64_SOURCE"
@@ -295,6 +310,37 @@ AC_ARG_ENABLE([online-unit-tests],
 	])
 AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
 
+## libFuzzer harnesses (fuzz/); requires clang
+AC_MSG_CHECKING(whether to build fuzzers)
+AC_ARG_ENABLE([fuzzers],
+	[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
+	[
+	case "${enableval}" in
+	no|yes)
+		fuzzers=$enableval
+		AC_MSG_RESULT($enableval)
+		;;
+	*)
+		AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
+		;;
+	esac
+	],
+	[
+	fuzzers=no
+	AC_MSG_RESULT(no)
+	])
+if test x"$fuzzers" = x"yes"; then
+	# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
+	AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
+		[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
+		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
+	# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
+	if test x"$enable_shared" != x"no"; then
+		AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
+	fi
+fi
+AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
+
 # Final output
 AC_CONFIG_FILES([
 Makefile
@@ -306,5 +352,6 @@ lang/Makefile
 html/Makefile
 libtest/Makefile
 tests/Makefile
+fuzz/Makefile
 ])
 AC_OUTPUT

debian/changelog

@@ -1,6 +1,67 @@
+httrack (3.49.11-1) unstable; urgency=medium
+
+  * New upstream release: crawl correctness and security fixes (network-facing
+    buffer overflows, file-type detection, redirect handling) and modernized
+    web defaults; full list in history.txt.
+  * Add DEP-12 upstream metadata (#466).
+  * Bump debhelper compat to 14 (#466).
+  * Drop the redundant Priority field and update the NMU lintian override to
+    the current tag names (#466).
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 05 Jul 2026 00:03:18 +0200
+
+httrack (3.49.10-2) unstable; urgency=medium
+
+  * Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
+    absent (the local-server tests need python3, which the buildds lack). Add
+    patches/skip-local-pause-test-without-python3.patch to guard the test on
+    python3 up front, like its siblings, so it skips cleanly.
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 20:18:46 +0200
+
+httrack (3.49.10-1) unstable; urgency=medium
+
+  * New upstream release: new download-pacing and URL-handling options plus a
+    batch of crawl and robustness fixes (full list in history.txt).
+  * Rewrite debian/copyright in machine-readable DEP-5 format, crediting the
+    bundled minizip, md5 and coucal sources (#415).
+  * Lead the webhttrack browser dependency with chromium so httrack is not
+    dragged into the firefox-esr autoremoval cascade (#436).
+  * Override the embedded-library lint for the bundled minizip (#419).
+  * Bump Standards-Version to 4.7.4 (no changes required).
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 14:01:53 +0200
+
+httrack (3.49.9-1) unstable; urgency=medium
+
+  * New upstream release: Content-Type and file-type detection fixes (trust a
+    declared Content-Type over a binary URL extension, honor --assume under the
+    delayed type check, keep a known extension against a bogus or empty
+    Content-Type, and avoid an uninitialised read on an empty Content-Type), and
+    restored C++ source-compatibility of the installed headers so reverse
+    dependencies (httraqt) build again.
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 21 Jun 2026 17:59:38 +0200
+
+httrack (3.49.8-2) unstable; urgency=medium
+
+  * Rename libhttrack2 to libhttrack3 to follow the SONAME, which the 3.49.8
+    ABI bump moved to libhttrack.so.3 (package-name-doesnt-match-sonames). In
+    3.49.8-1 the libhttrack2.files glob still matched .so.2, so the runtime
+    libraries fell through into the httrack package and libhttrack2 shipped no
+    library. The new .files uses a .so.3* wildcard so a future SONAME bump no
+    longer silently misplaces the libraries. New binary package, via NEW.
+  * Drop the stale debian/libhttrack-swf1.files: the swf module is no longer
+    built and no libhttrack-swf1 package exists.
+
+ -- Xavier Roche <xavier@debian.org>  Sat, 20 Jun 2026 14:42:13 +0200
+
 httrack (3.49.8-1) unstable; urgency=medium
 
-  * New upstream release.
+  * New upstream release: HTTPS-proxy CONNECT tunnelling and wider srcset
+    parsing, a batch of crawler and parser fixes (CSS @import, xmlns
+    namespaces, relative paths, RFC 6265 cookies), and security hardening of
+    the parser and of buffer copies throughout the engine.
   * Drop the OpenSSL linking exception from the license: OpenSSL 3.0+ is
     Apache-2.0 and GPL-compatible, so it is no longer needed. httrack is now
     plain GPL-3.0-or-later. Updated debian/copyright accordingly.
@@ -14,7 +75,7 @@ httrack (3.49.8-1) unstable; urgency=medium
     the QA debcheck page. Depend on firefox-esr | chromium | www-browser
     instead.
 
- -- Xavier Roche <xavier@debian.org>  Sun, 07 Jun 2026 14:29:24 +0200
+ -- Xavier Roche <xavier@debian.org>  Sat, 20 Jun 2026 13:02:08 +0200
 
 httrack (3.49.7-2) unstable; urgency=medium
 

debian/control

@@ -1,9 +1,8 @@
 Source: httrack
 Section: web
-Priority: optional
 Maintainer: Xavier Roche <roche@httrack.com>
-Standards-Version: 4.7.0
+Standards-Version: 4.7.4
-Build-Depends: debhelper-compat (= 13), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
+Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
 Rules-Requires-Root: no
 Homepage: http://www.httrack.com
 Vcs-Git: https://github.com/xroche/httrack.git
@@ -30,7 +29,7 @@ Description: Copy websites to your computer (Offline browser)
 Package: webhttrack
 Architecture: any
 Multi-Arch: foreign
-Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, firefox-esr | chromium | www-browser
+Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, chromium | firefox-esr | www-browser
 Replaces: webhttrack-common (<< 3.43.9-2)
 Breaks: webhttrack-common (<< 3.43.9-2)
 Suggests: httrack, httrack-doc
@@ -58,13 +57,13 @@ Description: webhttrack common files
  This package is the common files of webhttrack, website copier and
  mirroring utility
 
-Package: libhttrack2
+Package: libhttrack3
 Architecture: any
 Multi-Arch: same
 Section: libs
-Replaces: libhttrack1
-Conflicts: libhttrack1
 Depends: ${misc:Depends}, ${shlibs:Depends}
+Replaces: libhttrack2, httrack (<< 3.49.8-2~)
+Breaks: libhttrack2, httrack (<< 3.49.8-2~)
 Description: Httrack website copier library
  This package is the library part of httrack, website copier and mirroring
  utility

debian/copyright

@@ -1,21 +1,109 @@
-This package was debianized by Xavier Roche <roche@httrack.com> on
+Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/
-Fri, 27 Sep 2002 16:42:26 +0200
+Upstream-Name: httrack
+Upstream-Contact: Xavier Roche <roche@httrack.com>
+Source: https://www.httrack.com/
 
-The current Debian maintainer is Xavier Roche <xavier@debian.org>
+Files: *
+Copyright: 1998-2026 Xavier Roche and other contributors
+License: GPL-3+
+Comment:
+ The engine includes contributions from Yann Philippot (src/htsjava.c,
+ src/htsjava.h). htsbasenet.h links against the system OpenSSL library
+ (originally by Eric Young); no OpenSSL/SSLeay code is bundled here.
 
-Upstream author: Xavier Roche <roche@httrack.com>
+Files: src/minizip/*
+Copyright: 1998-2010 Gilles Vollant
+           2007-2008 Even Rouault
+           2009-2010 Mathias Svensson
+           1990-2000 Info-ZIP
+License: Zlib
+Comment:
+ The decryption code in src/minizip/crypt.h and src/minizip/unzip.c derives
+ from the Info-ZIP distribution, distributed under the same terms.
 
-Copyright: 1998-2014 Xavier Roche and other contributors
+Files: src/md5.c
+Copyright: 1993 Colin Plumb
+License: public-domain-md5
+ This code implements the MD5 message-digest algorithm, due to Ron Rivest.
+ It was written by Colin Plumb in 1993, no copyright is claimed. This code
+ is in the public domain; do with it what you wish.
 
-This program is free software: you can redistribute it and/or modify
+Files: src/coucal/*
-it under the terms of the GNU General Public License as published by
+Copyright: 2013-2014 Xavier Roche
-the Free Software Foundation, either version 3 of the License, or
+License: BSD-3-clause
-(at your option) any later version.
 
-On Debian systems, the complete text of the GNU General Public
+Files: src/coucal/murmurhash3.h*
-License version 3 can be found in /usr/share/common-licenses/GPL-3 file.
+Copyright: Austin Appleby
+License: public-domain-murmurhash3
+ MurmurHash3 was written by Austin Appleby, and is placed in the public
+ domain. The author hereby disclaims copyright to this source code.
 
-This program is distributed in the hope that it will be useful,
+Files: html/server/div/com.httrack.WebHTTrack.metainfo.xml
-but WITHOUT ANY WARRANTY; without even the implied warranty of
+Copyright: 1998-2026 Xavier Roche and other contributors
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+License: FSFAP
-GNU General Public License for more details.
+ Copying and distribution of this file, with or without modification, are
+ permitted in any medium without royalty provided the copyright notice and
+ this notice are preserved. This file is offered as-is, without any warranty.
+
+Files: debian/*
+Copyright: 2002-2026 Xavier Roche <xavier@debian.org>
+License: GPL-3+
+
+License: GPL-3+
+ This program is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License as published by
+ the Free Software Foundation, either version 3 of the License, or
+ (at your option) any later version.
+ .
+ This program is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ GNU General Public License for more details.
+ .
+ You should have received a copy of the GNU General Public License
+ along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ .
+ On Debian systems, the complete text of the GNU General Public License
+ version 3 can be found in /usr/share/common-licenses/GPL-3.
+
+License: Zlib
+ This software is provided 'as-is', without any express or implied warranty.
+ In no event will the authors be held liable for any damages arising from the
+ use of this software.
+ .
+ Permission is granted to anyone to use this software for any purpose,
+ including commercial applications, and to alter it and redistribute it
+ freely, subject to the following restrictions:
+ .
+ 1. The origin of this software must not be misrepresented; you must not claim
+    that you wrote the original software. If you use this software in a product,
+    an acknowledgment in the product documentation would be appreciated but is
+    not required.
+ 2. Altered source versions must be plainly marked as such, and must not be
+    misrepresented as being the original software.
+ 3. This notice may not be removed or altered from any source distribution.
+
+License: BSD-3-clause
+ Redistribution and use in source and binary forms, with or without
+ modification, are permitted provided that the following conditions are met:
+ .
+ 1. Redistributions of source code must retain the above copyright notice,
+    this list of conditions and the following disclaimer.
+ 2. Redistributions in binary form must reproduce the above copyright notice,
+    this list of conditions and the following disclaimer in the documentation
+    and/or other materials provided with the distribution.
+ 3. Neither the name of the copyright holder nor the names of its contributors
+    may be used to endorse or promote products derived from this software
+    without specific prior written permission.
+ .
+ THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+ AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE
+ LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ POSSIBILITY OF SUCH DAMAGE.

debian/httrack-doc.lintian-overrides

@@ -4,3 +4,6 @@
 # so the path lives in the display pointer, not the override -- match with '*'.
 httrack-doc: extra-license-file *
 httrack-doc: package-contains-documentation-outside-usr-share-doc *
+# search.sh is a sample CGI shipped alongside the HTML manual, not meant to be
+# run from the package tree; it stays non-executable by design.
+httrack-doc: script-not-executable *

debian/libhttrack-swf1.files

@@ -1,2 +0,0 @@
-usr/lib/*/libhtsswf.so.1.0.0
-usr/lib/*/libhtsswf.so.1

debian/libhttrack2.files

@@ -1,5 +0,0 @@
-usr/lib/*/libhttrack.so.2.0.49
-usr/lib/*/libhttrack.so.2
-usr/lib/*/libhtsjava.so.2.0.49
-usr/lib/*/libhtsjava.so.2
-usr/share/httrack/templates

debian/libhttrack2.lintian-overrides

@@ -1,3 +0,0 @@
-# The shared libraries ship without a versioned symbols control file (ABI is
-# tracked via the SONAME and a strict =version dependency, see debian/rules).
-libhttrack2: no-symbols-control-file usr/lib/*

debian/libhttrack3.files

@@ -0,0 +1,3 @@
+usr/lib/*/libhttrack.so.3*
+usr/lib/*/libhtsjava.so.3*
+usr/share/httrack/templates

debian/libhttrack3.lintian-overrides

@@ -0,0 +1,8 @@
+# The shared libraries ship without a versioned symbols control file (ABI is
+# tracked via the SONAME plus a >= upstream-version dependency, see debian/rules).
+libhttrack3: no-symbols-control-file usr/lib/*
+
+# Bundled, locally patched minizip (src/minizip): it adds a zipFlush() API the
+# system libminizip lacks (htscache.c flushes the cache .zip so an interrupted
+# crawl leaves a valid archive), plus Android/old-zlib portability fixes.
+libhttrack3: embedded-library *libminizip*

debian/proxytrack.lintian-overrides

@@ -0,0 +1,3 @@
+# Statically linked against httrack's bundled, patched minizip (see src/minizip
+# and libhttrack3's override): the zipFlush() API is absent from the system one.
+proxytrack: embedded-library *libminizip*

debian/rules

@@ -135,7 +135,7 @@ binary-arch: build install
 	dh_makeshlibs -a -X/usr/lib/$(DEB_HOST_MULTIARCH)/httrack/libtest --version-info
 	dh_installdeb -a
 	# we depend on the current version (ABI may change)
-	dh_shlibdeps -a -ldebian/libhttrack2/usr/lib/$(DEB_HOST_MULTIARCH)
+	dh_shlibdeps -a -ldebian/libhttrack3/usr/lib/$(DEB_HOST_MULTIARCH)
 	dh_gencontrol -a
 	dh_md5sums -a
 	dh_builddeb -a

debian/source/lintian-overrides

@@ -1,4 +1,6 @@
-httrack source: changelog-should-mention-nmu
+# Maintainer uploads sign the changelog as xavier@debian.org while the control
+# Maintainer is roche@httrack.com; lintian reads the address mismatch as an NMU.
+httrack source: no-nmu-in-changelog
 httrack source: source-nmu-has-incorrect-version-number
 
 # The bundled HTML pages are the genuine upstream documentation taken from

debian/upstream/metadata

@@ -0,0 +1,6 @@
+---
+Repository: https://github.com/xroche/httrack.git
+Repository-Browse: https://github.com/xroche/httrack
+Bug-Database: https://github.com/xroche/httrack/issues
+Bug-Submit: https://github.com/xroche/httrack/issues/new
+Contact: Xavier Roche <roche@httrack.com>

fuzz/Makefile.am

@@ -0,0 +1,38 @@
+# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
+if FUZZERS
+noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
+	fuzz-unescape fuzz-filters fuzz-url
+endif
+
+AM_CPPFLAGS = \
+	@DEFAULT_CFLAGS@ \
+	@THREADS_CFLAGS@ \
+	@V6_FLAG@ \
+	@LFS_FLAG@ \
+	-I$(top_srcdir)/src \
+	-I$(top_srcdir)/src/coucal
+
+# Static-link libhttrack.la: the internal symbols are hidden in the .so.
+AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
+LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
+
+fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
+fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
+fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
+fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
+fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
+fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
+fuzz_url_SOURCES = fuzz-url.c fuzz.h
+
+# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
+EXTRA_DIST = README.md run-fuzzers.sh \
+	corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
+	corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
+	corpus/idna/idna.txt corpus/idna/unicode.txt \
+	corpus/idna/regress-multilabel-leak.txt \
+	corpus/entities/entities.txt \
+	corpus/unescape/percent.txt \
+	corpus/filters/filter.bin corpus/filters/filter-size.bin \
+	corpus/filters/regress-empty-subject-unique.bin \
+	corpus/url/http-url.txt corpus/url/relative-path.txt \
+	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt

fuzz/README.md

@@ -0,0 +1,15 @@
+# Fuzzing httrack
+
+libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
+
+```sh
+./bootstrap
+mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
+CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
+  LDFLAGS="-fsanitize=address,undefined" \
+  bash /path/to/httrack/configure --enable-fuzzers --disable-shared
+make
+bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60   # 60s per target
+```
+
+Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.

fuzz/corpus/charset/latin1.txt

@@ -0,0 +1 @@
+
café naďve ¤

fuzz/corpus/charset/sjis.txt

@@ -0,0 +1 @@
+コンピュータ

fuzz/corpus/entities/entities.txt

@@ -0,0 +1 @@
+&<>A☃é&notarealentity;�

fuzz/corpus/filters/regress-empty-subject-unique.bin

@@ -0,0 +1 @@
+**((

fuzz/corpus/idna/idna.txt

@@ -0,0 +1 @@
+xn--bcher-kva.example.com

fuzz/corpus/idna/regress-multilabel-leak.txt

@@ -0,0 +1 @@
+büchev.例å­bücheple

fuzz/corpus/idna/unicode.txt

@@ -0,0 +1 @@
+bücher.例子.example

fuzz/corpus/meta/meta-charset.html

@@ -0,0 +1 @@
+<html><head><meta charset="utf-8"></head><body>x</body></html>

fuzz/corpus/meta/meta-http-equiv.html

@@ -0,0 +1 @@
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">

fuzz/corpus/unescape/percent.txt

@@ -0,0 +1 @@
+%41%zz%%20%c3%a9+%2e%2e%2f

fuzz/corpus/url/http-url.txt

@@ -0,0 +1 @@
+http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag

fuzz/corpus/url/regress-file-empty-path.txt

@@ -0,0 +1 @@
+file://

fuzz/corpus/url/regress-long-path-abort.txt

@@ -0,0 +1 @@
+ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../

fuzz/corpus/url/relative-path.txt

@@ -0,0 +1 @@
+ftp://ftp.example.com/pub/../file.txt

fuzz/fuzz-charset.c

@@ -0,0 +1,84 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
+   UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+static const char *const charsets[] = {
+    "utf-8",    "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
+    "us-ascii", "shift_jis",  "euc-jp",     "iso-2022-jp", "gb2312",
+    "big5",     "euc-kr",     "koi8-r",     "utf-16",      "unknown-charset",
+};
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  const char *charset;
+  char *s;
+
+  if (size == 0)
+    return 0;
+  charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+
+  {
+    char *utf8 = hts_convertStringToUTF8(s, size, charset);
+    freet(utf8);
+  }
+  {
+    char *enc = hts_convertStringFromUTF8(s, size, charset);
+    freet(enc);
+  }
+  {
+    size_t nChars = 0;
+    hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
+
+    if (ucs != NULL) {
+      char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
+      freet(back);
+      freet(ucs);
+    }
+  }
+  {
+    size_t i = 0;
+
+    while (i < size) {
+      hts_UCS4 uc = 0;
+      const size_t nr = hts_readUTF8(s + i, size - i, &uc);
+      char out[8];
+
+      if (nr == 0)
+        break;
+      hts_writeUTF8(uc, out, sizeof(out));
+      i += nr;
+    }
+  }
+
+  freet(s);
+  return 0;
+}

fuzz/fuzz-entities.c

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
+   destination size, so truncation bounds get exercised too. */
+#include "fuzz.h"
+#include "htsencoding.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t dsizes[] = {1, 2, 8, 64, 4096};
+  size_t dsize;
+  char *src, *dest;
+
+  if (size == 0)
+    return 0;
+  dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
+  data++, size--;
+  src = fuzz_strdup(data, size);
+  dest = malloct(dsize);
+
+  (void) hts_unescapeEntities(src, dest, dsize);
+  (void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
+
+  freet(dest);
+  freet(src);
+  return 0;
+}

fuzz/fuzz-filters.c

@@ -0,0 +1,65 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
+   here). Input splits on the first NUL: pattern, then subject string. */
+#include "fuzz.h"
+#include "htsfilters.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  const char *joker = buf;
+  const uint8_t *sep = memchr(data, '\0', size);
+  /* subject in its own allocation so ASan bounds it apart from the pattern */
+  char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
+                          : fuzz_strdup(data + size, 0);
+
+  (void) strjoker(nom, joker, NULL, NULL);
+  {
+    LLint sz = (LLint) size;
+    int size_flag = 0;
+
+    (void) strjoker(nom, joker, &sz, &size_flag);
+  }
+  (void) strjokerfind(nom, joker);
+  {
+    char *filter = malloct(strlen(joker) + 2);
+    char *filters[1];
+    LLint sz = (LLint) size;
+    int size_flag = 0, depth = 0;
+
+    filter[0] = '-';
+    memcpy(filter + 1, joker, strlen(joker) + 1);
+    filters[0] = filter;
+    (void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
+    freet(filter);
+  }
+
+  freet(nom);
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-idna.c

@@ -0,0 +1,47 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+
+  {
+    char *idna = hts_convertStringUTF8ToIDNA(s, size);
+    freet(idna);
+  }
+  {
+    char *utf8 = hts_convertStringIDNAToUTF8(s, size);
+    freet(utf8);
+  }
+  (void) hts_isStringIDNA(s, size);
+
+  freet(s);
+  return 0;
+}

fuzz/fuzz-meta.c

@@ -0,0 +1,40 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
+   <meta> charset declaration. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *html = fuzz_strdup(data, size);
+  char *charset = hts_getCharsetFromMeta(html, size);
+
+  freet(charset);
+  freet(html);
+  return 0;
+}

fuzz/fuzz-unescape.c

@@ -0,0 +1,53 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
+   output buffer size, so the bounded-copy contract is exercised. */
+#include "fuzz.h"
+#include "httrack-library.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t bsizes[] = {1, 2, 16, 256, 8192};
+  size_t bsize;
+  char *s, *catbuff;
+
+  if (size == 0)
+    return 0;
+  bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+  catbuff = malloct(bsize);
+
+  (void) unescape_http(catbuff, bsize, s);
+  (void) unescape_http_unharm(catbuff, bsize, s, 0);
+  (void) unescape_http_unharm(catbuff, bsize, s, 1);
+  unescape_amp(s);
+
+  freet(catbuff);
+  freet(s);
+  return 0;
+}

fuzz/fuzz-url.c

@@ -0,0 +1,49 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
+   is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
+   in place. */
+#include "fuzz.h"
+#include "htscore.h"
+#include "htslib.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+  lien_adrfil *af = calloct(1, sizeof(*af));
+  /* fil_simplifie rewrites in place and may grow an empty path to "./" */
+  char *path = malloct(size + 3);
+
+  (void) ident_url_absolute(s, af);
+  memcpy(path, s, size + 1);
+  fil_simplifie(path);
+
+  freet(path);
+  freet(af);
+  freet(s);
+  return 0;
+}

fuzz/fuzz.h

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Shared helpers for the libFuzzer harnesses. */
+#ifndef FUZZ_H
+#define FUZZ_H
+
+#define HTS_INTERNAL_BYTECODE
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "htsbase.h"
+
+/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
+static char *fuzz_strdup(const uint8_t *data, size_t size) {
+  char *s = malloct(size + 1);
+
+  memcpy(s, data, size);
+  s[size] = '\0';
+  return s;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
+
+#endif

fuzz/run-fuzzers.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Drive every built harness against its seed corpus.
+#   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
+#   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
+# Replay is crash/leak-only and never mutates, so it can't hit strjoker's
+# catastrophic backtracking; the timed mode can, hence the per-unit -timeout.
+set -euo pipefail
+
+srcdir=$(cd "$(dirname "$0")" && pwd)
+bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
+mode=${2:-20}
+
+status=0
+for f in "$bld"/fuzz-*; do
+    if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
+    case "$f" in *.o | *.c | *.dSYM) continue ;; esac
+    name=$(basename "$f")
+    corpus="$srcdir/corpus/${name#fuzz-}"
+    if [ "$mode" = "check" ]; then
+        echo "=== $name (replay) ==="
+        [ -d "$corpus" ] || continue
+        if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
+            echo "*** $name FAILED on its corpus" >&2
+            status=1
+        fi
+        continue
+    fi
+    work=$(mktemp -d)
+    args=("$work")
+    [ -d "$corpus" ] && args+=("$corpus")
+    echo "=== $name (${mode}s) ==="
+    if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
+        -artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
+        echo "*** $name FAILED; artifacts:" >&2
+        ls -l "$work" >&2
+        status=1
+    else
+        rm -rf "$work"
+    fi
+done
+exit $status

history.txt

@@ -4,13 +4,73 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
+3.49-11
++ New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
++ New: advertise deflate in Accept-Encoding and decode deflate responses (#450)
++ New: follow <source> and <track> media elements as embedded links (#451)
++ New: added modern web MIME types to the type/extension table (#448)
++ Fixed: enforce the -E time limit during a slow transfer instead of only between files (#481)
++ Fixed: sniff the leading bytes of a download so a misdeclared Content-Type no longer renames a correct URL extension
++ Fixed: fast transfers could be saved under their temporary .delayed placeholder name (#5, #107)
++ Fixed: follow a redirect that maps to the same saved file instead of writing a self-pointing stub (#159)
++ Fixed: several network-facing buffer overflows in the FTP, Java and HTML parsers
++ Fixed: the htsjava plugin could not be loaded (hidden entry points, stale library name)
++ Fixed: HTML-escape truncation and a cache-buffer leak in the parser
++ Changed: modernized the default User-Agent to an honest HTTrack identifier (#449)
++ Changed: decode the full WHATWG set of HTML named character references (#443)
++ Changed: refreshed stale HTTP status, proxy-port and TLS-floor constants (#453)
++ Changed: multiple internal hardening, build, test and CI improvements
+
+3.49-10
++ New: --cookies-file to preload a Netscape cookies.txt before crawling (#215)
++ New: --pause to space out file downloads by a random delay (#185)
++ New: --strip-query to drop selected query keys from the dedup naming (#112)
++ Changed: split the -%u URL hacks into independent --keep-www-prefix, --keep-double-slashes and --keep-query-order toggles (#271)
++ Fixed: follow a redirect Location after dropping its #fragment, instead of requesting the fragment and polluting the saved name (#204)
++ Fixed: escaped brackets inside a *[...] filter character class (#148)
++ Fixed: honor the server's Content-Range when resuming a partial download, instead of appending overlapping bytes (#198)
++ Fixed: abort the download as soon as the response type is excluded by -mime:, instead of fetching then discarding the body (#58)
++ Fixed: keep size-based filter rules neutral until the file size is known (#143)
++ Fixed: stop the mirror with a clean fatal error on a cache write failure, instead of crashing (#174, #219)
++ Fixed: stop the 412/416 partial re-get loop on --continue and --update (#206)
++ Fixed: keep an unrecognized URL tail instead of mangling it to .html (#115)
++ Fixed: honor --tolerant (-%B) on a broken Content-Length, and fix an out-of-bounds read it exposed (#32, #41)
++ Fixed: fall back to the next resolved address when a connection fails or stalls, instead of hanging on a dead IPv6 address
++ Fixed: report why a -%L URL list could not be loaded (#49)
++ Changed: multiple internal hardening, build and CI improvements
+
+.49-9
++ Fixed: file-type detection from the Content-Type header: trust a declared type over a binary URL extension, honor --assume under the delayed type check, and keep a known extension against a bogus or empty Content-Type (#267, #29, #56)
++ Fixed: an uninitialized-buffer read when the Content-Type is empty (#411)
++ Fixed: restored C++ source-compatibility of the installed headers so reverse dependencies (httraqt) build again (#413)
++ Changed: multiple internal build, packaging and test-harness improvements
+
 3.49-8
++ New: tunnel HTTPS downloads through the configured HTTP proxy via CONNECT (#85)
++ New: parse every candidate URL in <img> and <source> srcset lists (#326)
 + Changed: dropped the obsolete OpenSSL linking exception (OpenSSL 3.0+ is Apache-2.0 and GPL-compatible); httrack is now plain GPLv3-or-later
-+ Fixed: link libhtsjava and the libtest examples directly against libc
++ Fixed: several out-of-bounds reads in the HTML/CSS parser on hostile input (#94, #396)
++ Fixed: stored XSS via an unescaped URL in the generated page footer (#165)
++ Fixed: hardened buffer copies throughout the engine against overflow
++ Fixed: capture conditional CSS @import URLs (#94)
++ Fixed: don't crawl xmlns namespace declarations as links (#191)
++ Fixed: don't mistake the method argument of XMLHttpRequest.open for a URL (#218)
++ Fixed: percent-encode parentheses when rewriting CSS url() targets (#163)
++ Fixed: collapse ../ in file:// URLs and widen relative-link handling (#137, #162)
++ Fixed: drop the obsolete $Version/$Path attributes from the request Cookie header, per RFC 6265 (#151)
++ Fixed: keep empty quoted arguments when reloading doit.log for --update/--continue (#106)
++ Fixed: raise the User-Agent and custom-header length limits (#152)
++ Fixed: abort on a long log path (lock-file buffer too small) (#183)
++ Fixed: race in lazy mutex initialization (#297)
++ Fixed: sub-second mtime precision when comparing local files on POSIX (#383)
++ Fixed: modernize OpenSSL TLS initialization for the 3.x to 4.x transition (#308)
 + Fixed: in-place changes made by the postprocess callback were not applied (Roman Sęk)
 + Fixed: "preffered" typo in the help text and man page (yosinn1-blip)
 + Fixed: corrections and updates of the Russian translation (German Aizek)
 + Fixed: corrections and updates of the Danish translation (scootergrisen)
++ Fixed: link libhtsjava and the libtest examples directly against libc
++ New: documented the public library API headers and typed the option fields as named enums
++ Fixed: numerous build, packaging, CI and test-coverage improvements (out-of-tree builds, sanitizer/distcheck CI, shell and Python linting, AppStream metainfo)
 
 3.49-7
 + Fixed: keep generated config.h architecture-independent (Debian #1133728)

html/div/search.sh

@@ -1,8 +1,7 @@
-
 #!/bin/sh
 
 # Simple indexing test using HTTrack
-# A "real" script/program would use advanced search, and 
+# A "real" script/program would use advanced search, and
 # use dichotomy to find the word in the index.txt file
 # This script is really basic and NOT optimized, and
 # should not be used for professional purpose :)
@@ -11,50 +10,49 @@ TESTSITE="http://localhost/"
 
 # Create an index if necessary
 if ! test -f "index.txt"; then
-	echo "Building the index .."
+    echo "Building the index .."
-	rm -rf test
+    rm -rf test
-	httrack --display "$TESTSITE"  -%I -O test
+    httrack --display "$TESTSITE" -%I -O test
-	mv test/index.txt ./
+    mv test/index.txt ./
 fi
 
 # Convert crlf to lf
-if test "`head index.txt -n 1 | tr '\r' '#' | grep -c '#'`" = "1"; then
+if test "$(head index.txt -n 1 | tr '\r' '#' | grep -c '#')" = "1"; then
-	echo "Converting index to Unix LF style (not CR/LF) .."
+    echo "Converting index to Unix LF style (not CR/LF) .."
-	mv -f index.txt index.txt.old
+    mv -f index.txt index.txt.old
-	cat index.txt.old|tr -d '\r' > index.txt
+    tr -d '\r' <index.txt.old >index.txt
 fi
 
 keyword=-
 while test -n "$keyword"; do
-	printf "Enter a keyword: "
+    printf "Enter a keyword: "
-	read keyword
+    read -r keyword
 
-	if test -n "$keyword"; then
+    if test -n "$keyword"; then
-		FOUNDK="`grep -niE \"^$keyword\" index.txt`"
+        FOUNDK="$(grep -niE "^$keyword" index.txt)"
 
-		if test -n "$FOUNDK"; then	
+        if test -n "$FOUNDK"; then
-			if ! test `echo "$FOUNDK"|wc -l` = "1"; then
+            if ! test "$(echo "$FOUNDK" | wc -l)" = "1"; then
-				# Multiple matches
+                # Multiple matches
-				printf "Found multiple keywords: "
+                printf "Found multiple keywords: "
-				echo "$FOUNDK"|cut -f2 -d':'|tr '\n' ' '
+                echo "$FOUNDK" | cut -f2 -d':' | tr '\n' ' '
-				echo ""
+                echo ""
-				echo "Use keyword$ to find only one"
+                echo "Use keyword$ to find only one"
-			else
+            else
-				# One match
+                # One match
-				N=`echo "$FOUNDK"|cut -f1 -d':'`
+                N=$(echo "$FOUNDK" | cut -f1 -d':')
-				PM=`tail +$N index.txt|grep -nE "\("|head -n 1`
+                PM=$(tail "+$N" index.txt | grep -nE "\(" | head -n 1)
-				if ! echo "$PM"|grep "ignored">/dev/null; then
+                if ! echo "$PM" | grep "ignored" >/dev/null; then
-					M=`echo $PM|cut -f1 -d':'`
+                    M=$(echo "$PM" | cut -f1 -d':')
-					echo "Found in:"
+                    echo "Found in:"
-					cat index.txt | tail "+$N" | head -n "$M" | grep -E "[0-9]* " | cut -f2 -d' '
+                    tail "+$N" index.txt | head -n "$M" | grep -E "[0-9]* " | cut -f2 -d' '
-				else
+                else
-					echo "keyword ignored (too many hits)"
+                    echo "keyword ignored (too many hits)"
-				fi
+                fi
-					fi
+            fi
-		else
+        else
-			echo "not found"
+            echo "not found"
-		fi
+        fi
 
-	fi
+    fi
 done
-

html/filters.html

@@ -247,7 +247,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
         <td>the \ character</td>
       </tr>
       <tr>
-        <td nowrap><tt>*[\[\]]</tt></td>
+        <td nowrap><tt>*[\[,\]]</tt></td>
         <td>the [ or ] character</td>
       </tr>
       <tr>

lang/English.txt

@@ -295,7 +295,7 @@ Max Depth
 Maximum external depth:
 Maximum external depth:
 Filters (refuse/accept links) :
-Filters (refuse/accept links) :
+Filters (refuse/accept links):
 Paths
 Paths
 Save prefs

lang/README.md

@@ -0,0 +1,37 @@
+# Translating HTTrack
+
+Interface strings live here, one `.txt` file per language. `English.txt` is the reference: every other file maps each English string to its translation.
+
+## File format
+
+Plain text, entries in consecutive pairs of lines:
+
+```
+<English string>
+<translation>
+```
+
+The first line of a pair is the lookup key and must stay identical to the one in `English.txt`; translate only the second line. Missing entries fall back to the English text at runtime, so a partial translation works.
+
+Preserve any `\r\n`, `\t` and `printf` placeholders (`%s`, `%d`, ...) in the translation.
+
+A few `LANGUAGE_*` entries at the top describe the file itself:
+
+| Key | Meaning |
+| --- | --- |
+| `LANGUAGE_NAME` | Name shown in the language picker, in its own language (`Deutsch`, not `German`) |
+| `LANGUAGE_ISO` | ISO 639 code, with region if needed (`de`, `pt_BR`) |
+| `LANGUAGE_CHARSET` | Encoding the file is saved in (`ISO-8859-1`, `windows-1251`, `UTF-8`, ...) |
+| `LANGUAGE_AUTHOR` | Your name and contact |
+| `LANGUAGE_WINDOWSID` | Windows locale name used by WinHTTrack (`German (Standard)`) |
+
+Save the file in exactly its declared `LANGUAGE_CHARSET`; an editor that rewrites it as UTF-8 will corrupt the non-ASCII bytes.
+
+## Adding or updating a language
+
+1. Copy `English.txt` to `<Language>.txt`, or edit the existing file.
+2. Translate each second line; leave the English keys untouched.
+3. Fill in the `LANGUAGE_*` header for a new file.
+4. Open a pull request, or attach the file to a GitHub issue.
+
+When new strings land in `English.txt` they show up untranslated (as English) until a translator fills them in.

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "13 June 2026" "httrack website copier"
+.TH httrack 1 "27 June 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -24,6 +24,7 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-EN, \-\-max\-time[=N]\fR ]
 [ \fB\-AN, \-\-max\-rate[=N]\fR ]
 [ \fB\-%cN, \-\-connection\-per\-second[=N]\fR ]
+[ \fB\-%G, \-\-pause\fR ]
 [ \fB\-GN, \-\-max\-pause[=N]\fR ]
 [ \fB\-cN, \-\-sockets[=N]\fR ]
 [ \fB\-TN, \-\-timeout[=N]\fR ]
@@ -43,11 +44,13 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-x, \-\-replace\-external\fR ]
 [ \fB\-%x, \-\-disable\-passwords\fR ]
 [ \fB\-%q, \-\-include\-query\-string\fR ]
+[ \fB\-%g, \-\-strip\-query\fR ]
 [ \fB\-o, \-\-generate\-errors\fR ]
 [ \fB\-X, \-\-purge\-old[=N]\fR ]
 [ \fB\-%p, \-\-preserve\fR ]
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
+[ \fB\-%K, \-\-cookies\-file\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -153,6 +156,8 @@ maximum mirror time in seconds (60=1 minute, 3600=1 hour) (\-\-max\-time[=N])
 maximum transfer rate in bytes/seconds (1000=1KB/s max) (\-\-max\-rate[=N])
 .IP \-%cN
 maximum number of connections/seconds (*%c10) (\-\-connection\-per\-second[=N])
+.IP \-%G
+random pause of MIN[:MAX] seconds between files (e.g. %G5:10) (\-\-pause <param>)
 .IP \-GN
 pause transfer if N bytes reached, and wait until lock file is deleted (\-\-max\-pause[=N])
 .SS Flow control:
@@ -198,6 +203,8 @@ replace external html links by error pages (\-\-replace\-external)
 do not include any password for external password protected websites (%x0 include) (\-\-disable\-passwords)
 .IP \-%q
 *include query string for local files (useless, for information purpose only) (%q0 don't include) (\-\-include\-query\-string)
+.IP \-%g
+strip query keys for dedup ([host/pattern=]key1,key2,...) (\-\-strip\-query <param>)
 .IP \-o
 *generate output html file in case of error (404..) (o0 don't generate) (\-\-generate\-errors)
 .IP \-X
@@ -209,6 +216,8 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 .SS Spider options:
 .IP \-bN
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
+.IP \-%K
+load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j
@@ -225,6 +234,8 @@ tolerant requests (accept bogus responses on some servers, but not standard!) (\
 update hacks: various hacks to limit re\-transfers when updating (identical size, bogus response..) (\-\-updatehack)
 .IP \-%u
 url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..) (\-\-urlhack)
+.br
+opt out of one url\-hack part: \-\-keep\-www\-prefix (www.foo.com<>foo.com), \-\-keep\-double\-slashes (//), \-\-keep\-query\-order (?b&a)
 .IP \-%A
 assume that a type (cgi,asp..) is always linked with a mime type (\-%A php3,cgi=text/html;dat,bin=application/x\-zip) (\-\-assume <param>)
 .br
@@ -313,12 +324,8 @@ debug HTTP headers in logfile (\-\-debug\-headers)
 .SS Guru options: (do NOT use if possible)
 .IP \-#X
 *use optimized engine (limited memory boundary checks) (\-\-fast\-engine)
-.IP \-#0
+.IP \-#test
-filter test (\-#0 '*.gif' 'www.bar.com/foo.gif') (\-\-debug\-testfilters <param>)
+list engine self\-tests (run one with \-#test=NAME [args])
-.IP \-#1
-simplify test (\-#1 ./foo/bar/../foobar)
-.IP \-#2
-type test (\-#2 /foo/bar.php)
 .IP \-#C
 cache list (\-#C '*.com/spider*.gif' (\-\-debug\-cache <param>)
 .IP \-#R

src/Makefile.am

@@ -56,21 +56,21 @@ whttrackrundir = $(bindir)
 whttrackrun_SCRIPTS = webhttrack
 
 libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
-	htscache_selftest.c \
+	htscache_selftest.c htsdns_selftest.c htsselftest.c \
 	htscatchurl.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
 	htshelp.c htslib.c htscoremain.c \
 	htsname.c htsrobots.c htstools.c htswizard.c \
 	htsalias.c htsthread.c htsindex.c htsbauth.c \
 	htsmd5.c htszlib.c htswrap.c htsconcat.c \
-	htsmodules.c htscharset.c punycode.c htsencoding.c \
+	htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
 	md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
 	hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
-	htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htscatchurl.h  \
+	htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htsdns_selftest.h htsselftest.h htscatchurl.h  \
 	htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h  \
 	htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
 	htshelp.h htsindex.h htslib.h htsmd5.h \
-	htsmodules.h htsname.h htsnet.h \
+	htsmodules.h htsname.h htsnet.h htssniff.h \
 	htsopt.h htsrobots.h htsthread.h  \
 	htstools.h htswizard.h htswrap.h htszlib.h  \
 	htsstrings.h htsarrays.h httrack-library.h \

src/htsalias.c

@@ -60,6 +60,9 @@ Please visit our Website: http://www.httrack.com
   param1 : this option must be alone, and needs one distinct parameter (-P <path>)
   param0 : this option must be alone, but the parameter should be put together (+*.gif)
 */
+/* clang-format off: hand-aligned table; clang-format reflows the whole
+   initializer (2->4 space) on any edit, churning every untouched row. */
+/* clang-format off */
 const char *hts_optalias[][4] = {
   /*   {"","","",""}, */
   {"path", "-O", "param1", "output path"},
@@ -107,6 +110,12 @@ const char *hts_optalias[][4] = {
   {"disable-passwords", "-%x", "single", ""}, {"disable-password", "-%x",
                                                "single", ""},
   {"include-query-string", "-%q", "single", ""},
+  {"strip-query", "-%g", "param1",
+   "strip [host/pattern=]key1,key2,... from URLs"},
+  {"cookies-file", "-%K", "param1",
+   "load extra cookies from a Netscape cookies.txt"},
+  {"pause", "-%G", "param1",
+   "random pause of MIN[:MAX] seconds between files"},
   {"generate-errors", "-o", "single", ""},
   {"do-not-generate-errors", "-o0", "single", ""},
   {"purge-old", "-X", "param", ""},
@@ -123,6 +132,9 @@ const char *hts_optalias[][4] = {
   {"tolerant", "-%B", "single", ""},
   {"updatehack", "-%s", "single", ""}, {"sizehack", "-%s", "single", ""},
   {"urlhack", "-%u", "single", ""},
+  {"keep-www-prefix", "-%j", "single", ""},
+  {"keep-double-slashes", "-%o", "single", ""},
+  {"keep-query-order", "-%y", "single", ""},
   {"user-agent", "-F", "param1", "user-agent identity"},
   {"referer", "-%R", "param1", "default referer URL"},
   {"from", "-%E", "param1", "from email address"},
@@ -241,6 +253,7 @@ const char *hts_optalias[][4] = {
 
   {"", "", "", ""}
 };
+/* clang-format on */
 
 /* 
   Check for alias in command-line 
@@ -270,7 +283,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
       char *position;
       int need_param = 1;
 
-      //int return_param=0;
       int pos;
 
       command[0] = param[0] = addcommand[0] = '\0';
@@ -348,7 +360,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
               strlcatbuff(return_argv[0], "0", return_argv_size);
             else if (strcmp(param, "on") == 0) {
               // on is the default
-              // strcatbuff(return_argv[0],"1");
             } else
               strlcatbuff(return_argv[0], param, return_argv_size);
           }

src/htsarrays.h

@@ -48,112 +48,115 @@ Please visit our Website: http://www.httrack.com
 /* Abort (with the failed byte count) when a growth allocation fails. The
    array macros never return an out-of-memory error; they assert and abort. */
 static void hts_record_assert_memory_failed(const size_t size) {
-  fprintf(stderr, "memory allocation failed (%lu bytes)", \
+  fprintf(stderr, "memory allocation failed (%lu bytes)", (long int) size);
-          (long int) size); \
+  assertf(!"memory allocation failed");
-  assertf(! "memory allocation failed"); \
 }
 
 /** Dynamic array of T elements. **/
-#define TypedArray(T) \
+#define TypedArray(T)                                                          \
-  struct {            \
+  struct {                                                                     \
-    /** Elements. **/ \
+    /** Elements. **/                                                          \
-    union {           \
+    union {                                                                    \
-      /** Typed. **/  \
+      /** Typed. **/                                                           \
-      T* elts;        \
+      T *elts;                                                                 \
-      /** Opaque. **/ \
+      /** Opaque. **/                                                          \
-      void* ptr;      \
+      void *ptr;                                                               \
-    } data;           \
+    } data;                                                                    \
-    /** Count. **/    \
+    /** Count. **/                                                             \
-    size_t size;      \
+    size_t size;                                                               \
-    /** Capacity. **/ \
+    /** Capacity. **/                                                          \
-    size_t capa;      \
+    size_t capa;                                                               \
   }
 
 /** Initializer for an empty array (no backing store, size and capacity 0). **/
-#define EMPTY_TYPED_ARRAY { { NULL }, 0, 0 }
+#define EMPTY_TYPED_ARRAY {{NULL}, 0, 0}
 
 /** Array size, in elements. **/
-#define TypedArraySize(A)   ((A).size)
+#define TypedArraySize(A) ((A).size)
 
 /** Array capacity, in elements. **/
-#define TypedArrayCapa(A)   ((A).capa)
+#define TypedArrayCapa(A) ((A).capa)
 
 /**
- * Remaining free space, in elements. 
+ * Remaining free space, in elements.
  * Macro, first element evaluated multiple times.
  **/
-#define TypedArrayRoom(A)   ( TypedArrayCapa(A) - TypedArraySize(A) )
+#define TypedArrayRoom(A) (TypedArrayCapa(A) - TypedArraySize(A))
 
 /** Array elements, of type T*. **/
-#define TypedArrayElts(A)   ((A).data.elts)
+#define TypedArrayElts(A) ((A).data.elts)
 
 /** Array pointer, of type void*. **/
-#define TypedArrayPtr(A)    ((A).data.ptr)
+#define TypedArrayPtr(A) ((A).data.ptr)
 
 /** Size of T. **/
-#define TypedArrayWidth(A)  (sizeof(*TypedArrayElts(A)))
+#define TypedArrayWidth(A) (sizeof(*TypedArrayElts(A)))
 
 /** Nth element of the array, as an lvalue. No bounds check; N must be
     < TypedArraySize(A). **/
 #define TypedArrayNth(A, N) (TypedArrayElts(A)[N])
 
 /**
- * Tail of the array (outside the array). 
+ * Tail of the array (outside the array).
  * The returned pointer points to the beginning of TypedArrayRoom(A)
  * free elements.
  **/
 #define TypedArrayTail(A) (TypedArrayNth(A, TypedArraySize(A)))
 
 /**
-  * Ensure at least 'ROOM' elements can be put in the remaining space.
+ * Ensure at least 'ROOM' elements can be put in the remaining space.
-  * After a call to this macro, TypedArrayRoom(A) is guaranteed to be at 
+ * After a call to this macro, TypedArrayRoom(A) is guaranteed to be at
-  * least equal to 'ROOM'.
+ * least equal to 'ROOM'.
-  **/
+ **/
-#define TypedArrayEnsureRoom(A, ROOM) do { \
+#define TypedArrayEnsureRoom(A, ROOM)                                          \
-  const size_t room_ = (ROOM); \
+  do {                                                                         \
-  while (TypedArrayRoom(A) < room_) { \
+    const size_t room_ = (ROOM);                                               \
-    TypedArrayCapa(A) = TypedArrayCapa(A) < 16 ? 16 : TypedArrayCapa(A) * 2; \
+    while (TypedArrayRoom(A) < room_) {                                        \
-  } \
+      TypedArrayCapa(A) = TypedArrayCapa(A) < 16 ? 16 : TypedArrayCapa(A) * 2; \
-  TypedArrayPtr(A) = realloc(TypedArrayPtr(A), \
+    }                                                                          \
-                             TypedArrayCapa(A)*TypedArrayWidth(A)); \
+    TypedArrayPtr(A) =                                                         \
-  if (TypedArrayPtr(A) == NULL) { \
+        realloc(TypedArrayPtr(A), TypedArrayCapa(A) * TypedArrayWidth(A));     \
-    hts_record_assert_memory_failed(TypedArrayCapa(A)*TypedArrayWidth(A)); \
+    if (TypedArrayPtr(A) == NULL) {                                            \
-  } \
+      hts_record_assert_memory_failed(TypedArrayCapa(A) * TypedArrayWidth(A)); \
-} while(0)
+    }                                                                          \
+  } while (0)
 
 /** Add an element. Macro, first element evaluated multiple times. **/
-#define TypedArrayAdd(A, E) do { \
+#define TypedArrayAdd(A, E)                                                    \
-  TypedArrayEnsureRoom(A, 1); \
+  do {                                                                         \
-  assertf(TypedArraySize(A) < TypedArrayCapa(A)); \
+    TypedArrayEnsureRoom(A, 1);                                                \
-  TypedArrayTail(A) = (E); \
+    assertf(TypedArraySize(A) < TypedArrayCapa(A));                            \
-  TypedArraySize(A)++; \
+    TypedArrayTail(A) = (E);                                                   \
-} while(0)
+    TypedArraySize(A)++;                                                       \
+  } while (0)
 
 /**
- * Add 'COUNT' elements from 'PTR'. 
+ * Add 'COUNT' elements from 'PTR'.
  * Macro, first element evaluated multiple times.
  **/
-#define TypedArrayAppend(A, PTR, COUNT) do { \
+#define TypedArrayAppend(A, PTR, COUNT)                                        \
-  const size_t count_ = (COUNT); \
+  do {                                                                         \
-  /* This 1-case is to benefit from type safety. */ \
+    const size_t count_ = (COUNT);                                             \
-  if (count_ == 1) { \
+    /* This 1-case is to benefit from type safety. */                          \
-    TypedArrayAdd(A, *(PTR)); \
+    if (count_ == 1) {                                                         \
-  } else { \
+      TypedArrayAdd(A, *(PTR));                                                \
-    const void *const source_ = (PTR); \
+    } else {                                                                   \
-    TypedArrayEnsureRoom(A, count_); \
+      const void *const source_ = (PTR);                                       \
-    assertf(count_ <= TypedArrayRoom(A)); \
+      TypedArrayEnsureRoom(A, count_);                                         \
-    memcpy(&TypedArrayTail(A), source_, count_ * TypedArrayWidth(A)); \
+      assertf(count_ <= TypedArrayRoom(A));                                    \
-    TypedArraySize(A) += count_; \
+      memcpy(&TypedArrayTail(A), source_, count_ *TypedArrayWidth(A));         \
-  } \
+      TypedArraySize(A) += count_;                                             \
-} while(0)
+    }                                                                          \
+  } while (0)
 
 /** Clear an array, freeing memory and clearing size and capacity. **/
-#define TypedArrayFree(A) do { \
+#define TypedArrayFree(A)                                                      \
-  if (TypedArrayPtr(A) != NULL) { \
+  do {                                                                         \
-    TypedArrayCapa(A) = TypedArraySize(A) = 0; \
+    if (TypedArrayPtr(A) != NULL) {                                            \
-    free(TypedArrayPtr(A)); \
+      TypedArrayCapa(A) = TypedArraySize(A) = 0;                               \
-    TypedArrayPtr(A) = NULL; \
+      free(TypedArrayPtr(A));                                                  \
-  } \
+      TypedArrayPtr(A) = NULL;                                                 \
-} while(0)
+    }                                                                          \
+  } while (0)
 
 #endif

src/htsback.c

@@ -43,21 +43,22 @@ Please visit our Website: http://www.httrack.com
 
 #include "htsback.h"
 
-//#ifdef _WIN32
 #include "htsftp.h"
 #if HTS_USEZLIB
 #include "htszlib.h"
 #else
 #error HTS_USEZLIB not defined
 #endif
-//#endif
 
 #ifdef _WIN32
 #ifndef __cplusplus
 // DOS
 #include <process.h>            /* _beginthread, _endthread */
 #endif
+#include <io.h> /* _chsize_s */
+#define HTS_FTRUNCATE(fp, sz) _chsize_s(_fileno(fp), (sz))
 #else
+#define HTS_FTRUNCATE(fp, sz) ftruncate(fileno(fp), (sz))
 #endif
 
 #define VT_CLREOL       "\33[K"
@@ -73,6 +74,8 @@ struct_back *back_new(httrackp *opt, int back_max) {
 
   sback->count = back_max;
   sback->lnk = (lien_back *) calloct((back_max + 1), sizeof(lien_back));
+  sback->connect_fallback = (hts_connect_fallback *) calloct(
+      (back_max + 1), sizeof(hts_connect_fallback));
   sback->ready = coucal_new(0);
   hts_set_hash_handler(sback->ready, opt);
   coucal_set_name(sback->ready, "back_new");
@@ -83,6 +86,7 @@ struct_back *back_new(httrackp *opt, int back_max) {
     sback->lnk[i].r.location = sback->lnk[i].location_buffer;
     sback->lnk[i].status = STATUS_FREE;
     sback->lnk[i].r.soc = INVALID_SOCKET;
+    sback->connect_fallback[i].addr_count = -1; // not yet probed
   }
   return sback;
 }
@@ -93,6 +97,7 @@ void back_free(struct_back ** sback) {
       freet((*sback)->lnk);
       (*sback)->lnk = NULL;
     }
+    freet((*sback)->connect_fallback);
     if ((*sback)->ready != NULL) {
       coucal_delete(&(*sback)->ready);
       (*sback)->ready_size_bytes = 0;
@@ -102,6 +107,72 @@ void back_free(struct_back ** sback) {
   }
 }
 
+/* Per-candidate connect deadline cap (seconds): a connecting slot with another
+   address to try waits at most this long before falling back, instead of the
+   full (default 120s) slot timeout. Caps the dead-IPv6 stall while staying well
+   above a normal handshake. The last candidate still gets the full timeout. */
+#define HTS_CONNECT_FALLBACK_TIMEOUT 10
+
+int back_connect_fallback_due(int addr_index, int addr_count, int elapsed,
+                              int timeout) {
+  int deadline;
+
+  if (addr_index + 1 >= addr_count) // last (or only) candidate: no fallback
+    return 0;
+  if (timeout <= 0) // no timeout management: never force it
+    return 0;
+  deadline = (timeout < HTS_CONNECT_FALLBACK_TIMEOUT)
+                 ? timeout
+                 : HTS_CONNECT_FALLBACK_TIMEOUT;
+  return elapsed >= deadline;
+}
+
+/* Pending-connect result for a non-blocking socket reported ready by select():
+   0 = connected, >0 = the connect errno (refused, unreachable, ...), -1 if the
+   probe itself failed. A failed connect is reported writable too, so this is
+   how success is told from failure without blocking. */
+static int connect_socket_error(T_SOC soc) {
+  int soerr = 0;
+  socklen_t len = (socklen_t) sizeof(soerr);
+
+  if (getsockopt(soc, SOL_SOCKET, SO_ERROR, (char *) &soerr, &len) != 0)
+    return -1;
+  return soerr;
+}
+
+/* Retry a stuck/failed connecting slot against its next resolved address.
+   Closes the current socket and starts a non-blocking connect to the next
+   candidate, leaving the slot in STATUS_CONNECTING. Returns 1 if a new connect
+   was started, 0 if no fallback address remains (caller fails the slot). */
+static int back_connect_next(httrackp *opt, struct_back *sback, int i) {
+  hts_connect_fallback *const cf = &sback->connect_fallback[i];
+  lien_back *const back = sback->lnk;
+  const int next = cf->addr_index + 1;
+  T_SOC soc;
+
+  if (next >= cf->addr_count)
+    return 0;
+
+  if (back[i].r.soc != INVALID_SOCKET) {
+    deletehttp(&back[i].r);
+    back[i].r.soc = INVALID_SOCKET;
+  }
+  soc = newhttp_addr(opt, back[i].url_adr, &back[i].r, -1, 0, next, NULL);
+  if (soc == INVALID_SOCKET)
+    return 0;
+
+  back[i].r.soc = soc;
+  cf->addr_index = next;
+  cf->connect_start = time_local();
+  if (back[i].timeout > 0)
+    back[i].timeout_refresh = cf->connect_start;
+  back[i].status = STATUS_CONNECTING;
+  hts_log_print(opt, LOG_DEBUG,
+                "connect failed, trying next address (%d/%d) for %s", next + 1,
+                cf->addr_count, back[i].url_adr);
+  return 1;
+}
+
 void back_delete_all(httrackp * opt, cache_back * cache, struct_back * sback) {
   if (sback != NULL) {
     int i;
@@ -499,9 +570,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
         && back[p].r.size != back[p].r.totalsize && !opt->tolerant) {
       if (back[p].status == STATUS_READY) {
         hts_log_print(opt, LOG_WARNING,
-                      "file not stored in cache due to bogus state (broken size, expected "
+                      "incomplete transfer (expected " LLintP
-                      LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
+                      " bytes, got " LLintP
-                      back[p].r.size, back[p].url_adr, back[p].url_fil);
+                      "): file not cached, will be retried on the next update"
+                      " (use -%%B to cache anyway): %s%s",
+                      back[p].r.totalsize, back[p].r.size, back[p].url_adr,
+                      back[p].url_fil);
       } else {
         hts_log_print(opt, LOG_INFO,
                       "incomplete file not yet stored in cache (expected "
@@ -521,7 +595,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
 #if HTS_USEZLIB
         if (back[p].r.compressed) {
           if (back[p].r.size > 0) {
-            //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
             // stats
             back[p].compressed_size = back[p].r.size;
             // en mémoire -> passage sur disque
@@ -806,11 +879,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                          back[p].url_fil, NULL);
           } else {
             /* Partial file, but marked as "ok" ? */
-            hts_log_print(opt, LOG_WARNING,
+            hts_log_print(
-                          "file not stored in cache due to bogus state (incomplete type with %s (%d), size "
+                opt, LOG_WARNING,
-                          LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode,
+                "file with unresolved type not cached (%s (%d), size " LLintP
-                          (LLint) back[p].r.size, back[p].url_adr,
+                "): %s%s",
-                          back[p].url_fil);
+                back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size,
+                back[p].url_adr, back[p].url_fil);
           }
         }
 
@@ -850,7 +924,6 @@ int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
     /* clear everything but connection: switch, close, and reswitch */
     back_connxfr(src, &tmp);
     back_delete(opt, cache, sback, p);
-    //deletehttp(src);
     back_connxfr(&tmp, src);
     src->req.flush_garbage = 1; /* ignore CRLF garbage */
     return 1;
@@ -1285,7 +1358,49 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
   return 0;
 }
 
+/* Move a still-writing .delayed placeholder to its final name (#483). */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname) {
+  hts_boolean renamed;
+
+  if (!back->r.is_write || back->tmpfile != NULL ||
+      !IS_DELAYED_EXT(back->url_sav) || strcmp(back->url_sav, newname) == 0)
+    return HTS_TRUE; /* nothing bound to the placeholder name */
+  if (back->r.out != NULL) {
+    fclose(back->r.out);
+    back->r.out = NULL;
+  }
+  renamed = RENAME(back->url_sav, newname) == 0 ? HTS_TRUE : HTS_FALSE;
+  if (renamed && (back->status == STATUS_READY ||
+                  (back->r.out = FOPEN(newname, "ab")) != NULL)) {
+    filenote(&opt->state.strc, newname, NULL);
+    hts_log_print(opt, LOG_DEBUG, "moved placeholder %s to %s", back->url_sav,
+                  newname);
+    return HTS_TRUE;
+  }
+  /* partial lost: drop only what we own (Windows rename won't overwrite) */
+  hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "unable to move %s to %s",
+                back->url_sav, newname);
+  back->r.statuscode = STATUSCODE_INVALID;
+  strcpybuff(back->r.msg, "Write error on disk");
+  back->r.is_write = 0;
+  (void) UNLINK(renamed ? newname : back->url_sav);
+  return HTS_FALSE;
+}
+
 // effacer entrée
+/* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
+void back_delayed_discard(httrackp *opt, lien_back *back) {
+  if (back->r.out != NULL) {
+    fclose(back->r.out);
+    back->r.out = NULL;
+  }
+  back->r.is_write = 0;
+  if (opt != NULL)
+    url_savename_refname_remove(opt, back->url_adr, back->url_fil);
+  (void) UNLINK(back->url_sav);
+}
+
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p) {
   lien_back *const back = sback->lnk;
@@ -1293,6 +1408,12 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
 
   assertf(p >= 0 && p < back_max);
   if (p >= 0 && p < sback->count) {     // on sait jamais..
+    /* mid-write cancel: drop a .delayed placeholder; real-named partials
+       survive for resume (--continue) */
+    if (back[p].r.is_write && IS_DELAYED_EXT(back[p].url_sav) &&
+        (back[p].status != STATUS_READY || back[p].r.statuscode <= 0)) {
+      back_delayed_discard(opt, &back[p]);
+    }
     // Vérificateur d'intégrité
 #if DEBUG_CHECKINT
     _CHECKINT(&back[p], "Appel back_delete")
@@ -1312,7 +1433,6 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                       back[p].url_adr, back[p].url_fil, back[p].url_sav);
       }
       if (cache != NULL) {
-        //hts_log_print(opt, LOG_TRACE, "finalizing from back_delete");
         back_finalize(opt, cache, sback, p);
       }
     }
@@ -1453,7 +1573,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     if (back[p].r.soc != INVALID_SOCKET) {      /* we never know */
       deletehttp(&back[p].r);
     }
-    //memset(&(back[p].r), 0, sizeof(htsblk)); 
     hts_init_htsblk(&back[p].r);
     back[p].r.location = back[p].location_buffer;
 
@@ -1461,7 +1580,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     strcpybuff(back[p].url_adr, adr);
     strcpybuff(back[p].url_fil, fil);
     strcpybuff(back[p].url_sav, save);
-    //back[p].links_index = links_index;
     // copier referer si besoin
     strcpybuff(back[p].referer_adr, "");
     strcpybuff(back[p].referer_fil, "");
@@ -1700,8 +1818,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
                             back[p].url_adr, back[p].url_fil);
             }
             back[p].r.notmodified = 1;  // fichier non modifié
-            back[p].status = STATUS_READY;      // OK prêt
+            back[p].status = STATUS_READY; // OK prêt
-            //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);        // not modified
             back_set_finished(sback, p);
 
             // finalize transfer
@@ -1716,7 +1833,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
           } else {              // erreur
             // effacer r
             hts_init_htsblk(&back[p].r);
-            //memset(&(back[p].r), 0, sizeof(htsblk)); 
             back[p].r.location = back[p].location_buffer;
             // et continuer (chercher le fichier)
           }
@@ -1911,8 +2027,10 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       // ouvrir liaison, envoyer requète
       // ne pas traiter ou recevoir l'en tête immédiatement
       hts_init_htsblk(&back[p].r);
-      //memset(&(back[p].r), 0, sizeof(htsblk)); 
       back[p].r.location = back[p].location_buffer;
+      // fresh connect: address list not yet probed, start at the first
+      sback->connect_fallback[p].addr_index = 0;
+      sback->connect_fallback[p].addr_count = -1;
       // recopier proxy
       if ((back[p].r.req.proxy.active = opt->proxy.active)) {
         if (StringBuff(opt->proxy.bindhost) != NULL)
@@ -1959,7 +2077,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #if HTS_USEOPENSSL
       else if (strfield(back[p].url_adr, "https://")) {     // let's rock
         back[p].r.ssl = 1;
-        // back[p].r.ssl_soc = NULL;
         back[p].r.ssl_con = NULL;
       }
 #endif
@@ -2025,9 +2142,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
         back[p].rateout = -1;   // pas de gestion (default)
       }
 
-      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est
+      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où
-      // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
+      // cela est rattrapable (exemple: 301,302 moved xxx -> refresh sur la
-      //if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
+      // page!)
 
 #if CNXDEBUG
       printf("Xfopen ok, poll..\n");
@@ -2045,7 +2162,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       if (soc == INVALID_SOCKET) {      // erreur socket
         back[p].status = STATUS_READY;  // FINI
         back_set_finished(sback, p);
-        //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
         back[p].r.soc = INVALID_SOCKET;
       } else {
         if (!back[p].r.is_file)
@@ -2066,8 +2182,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
     // note: si il y a erreur (404,etc) status=2 (terminé/échec) mais
     // le lien est considéré comme traité
-    //if (back[p].soc<0)  // erreur
-    //  return -1;
 
     return 0;
   } else {
@@ -2140,9 +2254,6 @@ void back_solve(httrackp * opt, lien_back * back) {
     } else {
       hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
     }
-    //if (hts_dnstest(opt, a, 1) == 2) { // non encore testé!..
-    //  hts_log_print(opt, LOG_DEBUG, "resolving in background: %s", a);
-    //}
   }
 }
 
@@ -2161,12 +2272,13 @@ int host_wait(httrackp * opt, lien_back * back) {
 
 static int slot_can_be_cleaned(const lien_back * back) {
   return (back->status == STATUS_READY) // ready
-    /* Check autoclean */
+         /* Check autoclean */
-    && (!back->testmode)        // not test mode
+         && (!back->locked)   // not held by hts_wait_delayed (name pending)
-    && (strnotempty(back->url_sav))     // filename exists
+         && (!back->testmode) // not test mode
-    && (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
+         && (strnotempty(back->url_sav))     // filename exists
-    && (back->r.size >= 0)      // size>=0
+         && (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
-    ;
+         && (back->r.size >= 0)              // size>=0
+      ;
 }
 
 static int slot_can_be_finalized(httrackp * opt, const lien_back * back) {
@@ -2190,11 +2302,6 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) {
         (void) back_flush_output(opt, cache, sback, i); // flush output buffers
         usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr,
                     back[i].url_fil);
-        //if (back[i].links_index >= 0) {
-        //  assertf(back[i].links_index < opt->hash->max_lien);
-        //  opt->hash->liens[back[i].links_index]->pass2 = -1;
-        //  // *back[i].pass2_ptr=-1;  // Done!
-        //}
         /* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
         {
           int index = hash_read(opt->hash, back[i].url_sav, NULL, HASH_STRUCT_FILENAME );       // lecture type 0 (sav)
@@ -2342,6 +2449,34 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
+  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
+     starves (#481). FTP slots stay, their thread owns the socket. */
+  if (!back_checkmirror(opt)) {
+    int aborted = 0;
+    unsigned int i;
+
+    for (i = 0; i < (unsigned int) back_max; i++) {
+      if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
+        if (back[i].r.soc != INVALID_SOCKET) {
+          deletehttp(&back[i].r);
+        }
+        back[i].r.soc = INVALID_SOCKET;
+        /* drop a .delayed placeholder; real partials survive for resume */
+        if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
+          back_delayed_discard(opt, &back[i]);
+        back[i].r.statuscode = STATUSCODE_TIMEOUT;
+        strcpybuff(back[i].r.msg, "Mirror Time Out");
+        back[i].status = STATUS_READY;
+        back_set_finished(sback, i);
+        aborted++;
+      }
+    }
+    if (aborted > 0)
+      hts_log_print(opt, LOG_WARNING,
+                    "time limit reached, %d transfer(s) aborted", aborted);
+    return;
+  }
+
   // recevoir tant qu'il y a des données (avec un maximum de max_loop boucles)
   do_wait = 0;
   gestion_timeout = 0;
@@ -2350,9 +2485,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
     busy_state = busy_recv = 0;
 
-#if 0
-    check_rate(stat_timestart, opt->maxrate);   // vérifier taux de transfert
-#endif
     // inscrire les sockets actuelles, et rechercher l'ID la plus élevée
     FD_ZERO(&fds);
     FD_ZERO(&fds_c);
@@ -2362,28 +2494,31 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
     nfds = INVALID_SOCKET;
 
     max_c = 1;
-    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-      // for(i=0;i<back_max;i++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       // en cas de gestion du connect préemptif
 #if HTS_XCONN
       if (back[i].status == STATUS_CONNECTING) {        // connexion
-        do_wait = 1;
+        // a connecting slot always carries a live socket; guard anyway so a
+        // stray INVALID_SOCKET can never reach FD_SET (mirrors the recv branch)
+        if (back[i].r.soc != INVALID_SOCKET) {
+          do_wait = 1;
 
-        // noter socket write
+          // noter socket write
-        FD_SET(back[i].r.soc, &fds_c);
+          FD_SET(back[i].r.soc, &fds_c);
 
-        // noter socket erreur
+          // noter socket erreur
-        FD_SET(back[i].r.soc, &fds_e);
+          FD_SET(back[i].r.soc, &fds_e);
 
-        // calculer max
+          // calculer max
-        if (max_c) {
+          if (max_c) {
-          max_c = 0;
+            max_c = 0;
-          nfds = back[i].r.soc;
+            nfds = back[i].r.soc;
-        } else if (back[i].r.soc > nfds) {
+          } else if (back[i].r.soc > nfds) {
-          // ID socket la plus élevée
+            // ID socket la plus élevée
-          nfds = back[i].r.soc;
+            nfds = back[i].r.soc;
+          }
         }
 
       } else
@@ -2396,8 +2531,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         // poll pour la lecture sur les sockets
       if ((back[i].status > 0) && (back[i].status < 100)) {     // en réception http
 
-#if BDEBUG==1
+#if BDEBUG == 1
-        //printf("....socket in progress: %d\n",back[i].r.soc);
 #endif
         // non local et non ftp
         if (!back[i].r.is_file) {
@@ -2483,8 +2617,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       busy_recv = 0;
 
     // recevoir les données arrivées
-    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-      // for(i=0;i<back_max;i++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       if (back[i].status > 0) {
@@ -2517,8 +2650,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       }
       // ---- FLAG WRITE MIS A UN?: POUR LE CONNECT
       if (back[i].status == STATUS_CONNECTING) {        // attendre connect
+        hts_connect_fallback *const cf = &sback->connect_fallback[i];
         int dispo = 0;
 
+        // probe the resolved address list once per fresh connect (cache hit:
+        // the host was resolved when this connect was opened)
+        if (cf->addr_count < 0 && back[i].r.soc != INVALID_SOCKET &&
+            !back[i].r.is_file) {
+          SOCaddr scratch[HTS_MAXADDRNUM];
+
+          cf->addr_count = hts_dns_resolve_all(opt, back[i].url_adr, scratch,
+                                               HTS_MAXADDRNUM, NULL);
+          cf->connect_start = time_local();
+        }
+
         // vérifier l'existance de timeout-check
         if (!gestion_timeout)
           if (back[i].timeout > 0)
@@ -2526,14 +2671,45 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
         // connecté?
         dispo = FD_ISSET(back[i].r.soc, &fds_c);
-        if (dispo) {            // ok connected!!
+        if (dispo) { // socket ready: connect() finished (ok or failed)
+          // a refused/failed connect is reported writable too; probe SO_ERROR
+          // and, on failure, fall back to the next address (or fail the slot)
+          if (connect_socket_error(back[i].r.soc) != 0) {
+            if (!back_connect_next(opt, sback, i)) {
+              deletehttp(&back[i].r);
+              back[i].r.soc = INVALID_SOCKET;
+              back[i].r.statuscode = STATUSCODE_CONNERROR;
+              strcpybuff(back[i].r.msg, "Connect Error");
+              back[i].status = STATUS_READY;
+              back_set_finished(sback, i);
+            }
+            continue; // reconnected (stay connecting) or failed
+          }
           busy_state = 1;
 
 #if HTS_USEOPENSSL
           /* SSL mode */
           if (back[i].r.ssl) {
+            int tunnel_ok = 1;
+
+            // https via proxy: CONNECT-tunnel before TLS (#85)
+            if (back[i].r.req.proxy.active && back[i].r.ssl_con == NULL) {
+              const int timeout = back[i].timeout > 0 ? back[i].timeout : 30;
+
+              tunnel_ok =
+                  http_proxy_tunnel(opt, &back[i].r, back[i].url_adr, timeout);
+              if (!tunnel_ok) {
+                if (!strnotempty(back[i].r.msg))
+                  strcpybuff(back[i].r.msg, "proxy CONNECT failed");
+                deletehttp(&back[i].r);
+                back[i].r.soc = INVALID_SOCKET;
+                back[i].r.statuscode = STATUSCODE_NON_FATAL;
+                back[i].status = STATUS_READY;
+                back_set_finished(sback, i);
+              }
+            }
             // handshake not yet launched
-            if (!back[i].r.ssl_con) {
+            if (tunnel_ok && !back[i].r.ssl_con) {
               SSL_CTX_set_options(openssl_ctx, SSL_OP_ALL);
               // new session
               back[i].r.ssl_con = SSL_new(openssl_ctx);
@@ -2551,7 +2727,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 back[i].r.statuscode = STATUSCODE_SSL_HANDSHAKE;
             }
             /* Error */
-            if (back[i].r.statuscode == STATUSCODE_SSL_HANDSHAKE) {
+            if (tunnel_ok && back[i].r.statuscode == STATUSCODE_SSL_HANDSHAKE) {
               strcpybuff(back[i].r.msg, "bad SSL/TLS handshake");
               deletehttp(&back[i].r);
               back[i].r.soc = INVALID_SOCKET;
@@ -2581,7 +2757,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back[i].rateout_time = back[i].ka_time_start;
             }
             // envoyer header
-            //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
             HTS_STAT.stat_nrequests++;
             if (!back[i].head_request)
               http_sendhead(opt, opt->cookie, 0, back[i].send_too,
@@ -2648,7 +2823,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 #if HTS_XGETHOST
       else if (back[i].status == STATUS_WAIT_DNS) {     // attendre gethostbyname
 #if DEBUGDNS
-        //printf("status 101 for %s\n",back[i].url_adr);
 #endif
 
         if (!gestion_timeout)
@@ -2768,10 +2942,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
           // range size hack old location
 
 #if HTS_DIRECTDISK
-          // Court-circuit:
+          // Shortcut: store the file directly on disk when possible,
-          // Peut-on stocker le fichier directement sur disque?
+          // sparing memory
-          // Ahh que ca serait vachement mieux et que ahh que la mémoire vous dit merci!
+          if (back[i].status &&
-          if (back[i].status) {
+              !back[i].locked) { // name still pending when locked
             if (back[i].r.is_write == 0) {      // mode mémoire
               if (back[i].r.adr == NULL) {      // rien n'a été écrit
                 if (!back[i].testmode) {        // pas mode test
@@ -2779,7 +2953,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     if (strcmp(back[i].url_fil, "/robots.txt")) {
                       if (back[i].r.statuscode == HTTP_OK) {    // 'OK'
                         if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) {  // pas HTML
-                          if (opt->getmode & 2) {       // on peut ecrire des non html
+                          if (opt->getmode & HTS_GETMODE_NONHTML) {
                             int fcheck = 0;
                             int last_errno = 0;
 
@@ -2852,7 +3026,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                 }
                               }
                             }
-                          } else {      // on coupe tout!
+                          } else { // on coupe tout!
                             hts_log_print(opt, LOG_DEBUG,
                                           "File cancelled (non HTML): %s%s",
                                           back[i].url_adr, back[i].url_fil);
@@ -2921,8 +3095,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
           // Si réception chunk, tester si on est pas à la fin!
           if (back[i].status == 1) {
             if (back[i].is_chunk) {     // attendre prochain chunk
-              if (back[i].r.size == back[i].r.totalsize) {      // fin chunk!
+              if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
-                //printf("chunk end at %d\n",back[i].r.size);
                 back[i].status = STATUS_CHUNK_CR;       /* fetch ending CRLF */
                 if (back[i].chunk_adr != NULL) {
                   freet(back[i].chunk_adr);
@@ -2985,11 +3158,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back_finalize(opt, cache, sback, i);
             }
 
-            if (back[i].r.totalsize >= 0) {     // tester totalsize
+            if (back[i].r.totalsize >= 0) { // tester totalsize
-              //if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) {    // tester totalsize
               if (back[i].r.totalsize != back[i].r.size) {      // pas la même!
                 if (!opt->tolerant) {
-                  //#if HTS_CL_IS_FATAL
                   deleteaddr(&back[i].r);
                   if (back[i].r.size < back[i].r.totalsize)
                     back[i].r.statuscode = STATUSCODE_CONNERROR;        // recatch
@@ -2998,14 +3169,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           " expected)", (LLint) back[i].r.size,
                           (LLint) back[i].r.totalsize);
                 } else {
-                  //#else
                   // Un warning suffira..
                   hts_log_print(opt, LOG_WARNING,
                                 "Incorrect length (" LLintP "!=" LLintP
-                                " expected) for %s%s", (LLint) back[i].r.size,
+                                " expected) for %s%s",
+                                (LLint) back[i].r.size,
                                 (LLint) back[i].r.totalsize, back[i].url_adr,
                                 back[i].url_fil);
-                  //#endif
                 }
               }
             }
@@ -3132,10 +3302,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                          back[i].r.size);
 #endif
                   /* End */
-                  //if (back[i].status==STATUS_CHUNK_CR) {
+                  back[i].status = STATUS_READY; // fin
-                  back[i].status = STATUS_READY;        // fin  
                   back_set_finished(sback, i);
-                  //}
 
                   // finalize transfer if not temporary
                   if (!IS_DELAYED_EXT(back[i].url_sav)) {
@@ -3602,7 +3770,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                     back[i].url_adr, back[i].url_fil);
 
                       // finalize
-                      //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);        // not modified
                       if (back[i].r.statuscode > 0) {
                         hts_log_print(opt, LOG_TRACE, "finalizing after cache load");
                         back_finalize(opt, cache, sback, i);
@@ -3612,38 +3779,32 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                              back[i].url_adr, back[i].url_fil);
 #endif
 
-                      //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                     } else {    // erreur
                       back[i].status = STATUS_READY;    // terminé
                       back_set_finished(sback, i);
-                      //printf("erreur cache\n");
-
                     }
 
-/********** NO - must complete the body! ********** */
+                  }
-#if 0
+                  // MIME type excluded by a -mime: filter: abort, don't fetch
-                  } else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
+                  // the body (#58)
-                             || (back[i].r.statuscode == 412)
+                  else if (HTTP_IS_OK(back[i].r.statuscode) &&
-                             || (back[i].r.statuscode == 416)
+                           !back[i].testmode &&
-                    ) {         // Ne pas prendre le html, erreurs connues et gérées
+                           strnotempty(back[i].r.contenttype) &&
-#if HTS_DEBUG_CLOSESOCK
+                           hts_acceptmime(opt, 0, back[i].url_adr,
-                    DEBUG_W
+                                          back[i].url_fil,
-                      ("back_wait(301,302,303,307,412,416..): deletehttp\n");
+                                          back[i].r.contenttype) == 1) {
-#endif
+                    deletehttp(&back[i].r);
-                    // Couper connexion
+                    back[i].r.soc = INVALID_SOCKET;
-                    /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
+                    back[i].status = STATUS_READY;
-                    back_maydeletehttp(opt, cache, sback, i);
-
-                    back[i].status = STATUS_READY;      // terminé
                     back_set_finished(sback, i);
-                    // finalize
+                    back[i].r.statuscode = STATUSCODE_EXCLUDED;
-                    if (back[i].r.statuscode > 0) {
+                    strcpybuff(back[i].r.msg, "Excluded by MIME type filter");
-                      hts_log_print(opt, LOG_TRACE, "finalizing redirect & 4xx");
+                    hts_log_print(
-                      back_finalize(opt, cache, sback, i);
+                        opt, LOG_NOTICE,
-                    }
+                        "File excluded by MIME type filter (%s): %s%s",
-#endif
+                        back[i].r.contenttype, back[i].url_adr,
-/********** **************************** ********** */
+                        back[i].url_fil);
-                  } else {      // il faut aller le chercher
+                  } else { // il faut aller le chercher
 
                     // effacer buffer (requète)
                     if (!noFreebuff) {
@@ -3654,35 +3815,70 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     // xxc SI CHUNK VERIFIER QUE CA MARCHE??
                     if (back[i].r.statuscode == 206) {  // on nous envoie un morceau (la fin) coz une partie sur disque!
                       off_t sz = fsize_utf8(back[i].url_sav);
+                      /* RFC 7233: resume at the server's Content-Range start,
+                         not the offset we requested; a server may resume
+                         earlier and appending the overlap duplicates bytes
+                         (#198). */
+                      const LLint resume = back[i].r.crange_start;
+                      const hts_boolean range_ok =
+                          back[i].r.crange > 0 && resume >= 0 &&
+                          resume <= (LLint) sz &&
+                          back[i].r.crange_end + 1 == back[i].r.crange &&
+                          (back[i].r.totalsize < 0 ||
+                           back[i].r.totalsize ==
+                               back[i].r.crange_end - resume + 1);
 
 #if HDEBUG
                       printf("partial content: " LLintP " on disk..\n",
                              (LLint) sz);
 #endif
-                      if (sz >= 0) {
+                      if (sz >= 0 && range_ok) {
                         if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_sav)) {  // pas HTML
-                          if (opt->getmode & 2) {       // on peut ecrire des non html  **sinon ben euhh sera intercepté plus loin, donc rap sur ce qui va sortir**
+                          if (opt->getmode & HTS_GETMODE_NONHTML) {
                             filenote(&opt->state.strc, back[i].url_sav, NULL);  // noter fichier comme connu
                             file_notify(opt, back[i].url_adr, back[i].url_fil,
                                         back[i].url_sav, 0, 1,
                                         back[i].r.notmodified);
-                            back[i].r.out = FOPEN(fconv(catbuff, sizeof(catbuff), back[i].url_sav), "ab");       // append
+                            back[i].r.out =
+                                FOPEN(fconv(catbuff, sizeof(catbuff),
+                                            back[i].url_sav),
+                                      "r+b"); // resume in place
                             if (back[i].r.out && opt->cache != 0) {
-                              back[i].r.is_write = 1;   // écrire
+                              back[i].r.is_write = 1;
-                              back[i].r.size = sz;      // déja écrit
+                              back[i].r.size = resume; // bytes already on disk
-                              back[i].r.statuscode = HTTP_OK;   // Forcer 'OK'
+                              back[i].r.statuscode = HTTP_OK; // force 'OK'
                               if (back[i].r.totalsize >= 0)
-                                back[i].r.totalsize += sz;      // plus en fait
+                                back[i].r.totalsize += resume; // -> full size
-                              fseek(back[i].r.out, 0, SEEK_END);        // à la fin
+                              // drop bytes past the resume point; a silent
-                              /* create a temporary reference file in case of broken mirror */
+                              // failure could leave a stale tail, so on error
-                              if (back_serialize_ref(opt, &back[i]) != 0) {
+                              // drop the partial and refetch the whole file
-                                hts_log_print(opt, LOG_WARNING,
+                              if (HTS_FTRUNCATE(back[i].r.out,
-                                              "Could not create temporary reference file for %s%s",
+                                                (off_t) resume) != 0) {
-                                              back[i].url_adr, back[i].url_fil);
+                                fclose(back[i].r.out);
-                              }
+                                back[i].r.out = NULL;
+                                url_savename_refname_remove(
+                                    opt, back[i].url_adr, back[i].url_fil);
+                                UNLINK(back[i].url_sav);
+                                back[i].status = STATUS_READY;
+                                back_set_finished(sback, i);
+                                strcpybuff(back[i].r.msg,
+                                           "Can not truncate partial file, "
+                                           "restarting");
+                              } else {
+                                fseeko(back[i].r.out, (off_t) resume, SEEK_SET);
+                                /* create a temporary reference file in case of
+                                 * broken mirror */
+                                if (back_serialize_ref(opt, &back[i]) != 0) {
+                                  hts_log_print(opt, LOG_WARNING,
+                                                "Could not create temporary "
+                                                "reference file for %s%s",
+                                                back[i].url_adr,
+                                                back[i].url_fil);
+                                }
 #if HDEBUG
-                              printf("continue interrupted file\n");
+                                printf("continue interrupted file\n");
 #endif
+                              }
                             } else {    // On est dans la m**
                               back[i].status = STATUS_READY;    // terminé (voir plus loin)
                               back_set_finished(sback, i);
@@ -3694,17 +3890,18 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           FILE *fp =
                             FOPEN(fconv(catbuff, sizeof(catbuff), back[i].url_sav), "rb");
                           if (fp) {
-                            LLint alloc_mem = sz + 1;
+                            LLint alloc_mem = resume + 1;
 
                             if (back[i].r.totalsize >= 0)
                               alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
                             if (deleteaddr(&back[i].r)
                                 && (back[i].r.adr =
                                     (char *) malloct((size_t) alloc_mem))) {
-                              back[i].r.size = sz;
+                              back[i].r.size = resume;
                               if (back[i].r.totalsize >= 0)
-                                back[i].r.totalsize += sz;      // plus en fait
+                                back[i].r.totalsize += resume; // -> full size
-                              if ((fread(back[i].r.adr, 1, sz, fp)) != sz) {
+                              if ((fread(back[i].r.adr, 1, (size_t) resume,
+                                         fp)) != (size_t) resume) {
                                 back[i].status = STATUS_READY;  // terminé (voir plus loin)
                                 back_set_finished(sback, i);
                                 strcpybuff(back[i].r.msg,
@@ -3722,14 +3919,30 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                          "No memory for partial file");
                             }
                             fclose(fp);
-                          } else {      // Argh.. 
+                          } else {                              // open failed
                             back[i].status = STATUS_READY;      // terminé (voir plus loin)
                             back_set_finished(sback, i);
                             strcpybuff(back[i].r.msg,
                                        "Can not open partial file");
                           }
                         }
-                      } else {  // Non trouvé??
+                      } else if (sz >=
+                                 0) { // unusable range -> restart whole file
+                        hts_log_print(opt, LOG_WARNING,
+                                      "Unusable partial-content range for %s%s "
+                                      "(have " LLintP " bytes, got " LLintP
+                                      "-" LLintP "/" LLintP "), restarting",
+                                      back[i].url_adr, back[i].url_fil,
+                                      (LLint) sz, back[i].r.crange_start,
+                                      back[i].r.crange_end, back[i].r.crange);
+                        url_savename_refname_remove(opt, back[i].url_adr,
+                                                    back[i].url_fil);
+                        UNLINK(back[i].url_sav);
+                        back[i].status = STATUS_READY;
+                        back_set_finished(sback, i);
+                        strcpybuff(back[i].r.msg,
+                                   "Unusable partial content, restarting");
+                      } else {                          // partial not found
                         back[i].status = STATUS_READY;  // terminé (voir plus loin)
                         back_set_finished(sback, i);
                         strcpybuff(back[i].r.msg, "Can not find partial file");
@@ -3744,7 +3957,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           deletehttp(&back[i].r);
                         }
                         back[i].r.soc = INVALID_SOCKET;
-                        //back[i].r.statuscode=206;  ????????
                         back[i].r.statuscode = STATUSCODE_NON_FATAL;
                         if (strnotempty(back[i].r.msg))
                           strcpybuff(back[i].r.msg,
@@ -3765,10 +3977,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                               && (back[i].r.adr = (char *) malloct(2))) {
                             back[i].r.adr[0] = 0;
                           }
-                          hts_log_print(opt, LOG_TRACE, "finalizing empty");
+                          /* locked = name pending; the waiter finalizes after
-                          back_finalize(opt, cache, sback, i);
+                             patching url_sav (else: cached as .delayed, #5) */
-                        } else if (!back[i].r.is_chunk) {       // pas de chunk
+                          if (!back[i].locked) {
-                          //if (back[i].r.http11!=2) {    // pas de chunk
+                            hts_log_print(opt, LOG_TRACE, "finalizing empty");
+                            back_finalize(opt, cache, sback, i);
+                          }
+                        } else if (!back[i].r.is_chunk) { // pas de chunk
                           back[i].is_chunk = 0;
                           back[i].status = 1;   // start body
                         } else {
@@ -3810,7 +4025,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
                       }
                     }
-
                   }
 
                   /*} */
@@ -3830,15 +4044,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].status < 0) {
           if (!back[i].testmode) {      // pas en test
             UNLINK(back[i].url_sav);    // éliminer fichier (endommagé)
-            //printf("&& %s\n",back[i].url_sav);
           }
         }
 #endif
 
         /* funny log for commandline users */
-        //if (!opt->quiet) {  
+        if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
-        // petite animation
-        if (opt->verbosedisplay == 1) {
           if (back[i].status == STATUS_READY) {
             if (back[i].r.statuscode == HTTP_OK)
               printf("* %s%s (" LLintP " bytes) - OK" VT_CLREOL "\r",
@@ -3850,23 +4061,43 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
             fflush(stdout);
           }
         }
-        //}
 
       }                         // status>0
-    }                           // for
+    } // for
 
     // vérifier timeouts
     if (gestion_timeout) {
       TStamp act;
 
       act = time_local();       // temps en secondes
-      for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+      for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-        // for(i=0;i<back_max;i++) {
         unsigned int i = (i_mod + mod_random) % (back_max);
 
         if (back[i].status > 0) {       // réception/connexion/..
           if (back[i].timeout > 0) {
-            //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
+            // a stuck connect with a fallback address: retry the next one well
+            // before the full timeout (dead IPv6 on a dual-stack host, ...)
+            if (back[i].status == STATUS_CONNECTING) {
+              const hts_connect_fallback *const cf =
+                  &sback->connect_fallback[i];
+
+              if (back_connect_fallback_due(cf->addr_index, cf->addr_count,
+                                            (int) (act - cf->connect_start),
+                                            back[i].timeout)) {
+                if (back_connect_next(opt, sback, i)) {
+                  continue; // reconnected to the next candidate
+                }
+                // fallback was due but no socket could be opened
+                // (back_connect_next closed the dead one): stop now rather than
+                // spin on an invalid fd
+                back[i].r.soc = INVALID_SOCKET;
+                back[i].r.statuscode = STATUSCODE_CONNERROR;
+                strcpybuff(back[i].r.msg, "Connect Error");
+                back[i].status = STATUS_READY;
+                back_set_finished(sback, i);
+                continue;
+              }
+            }
             if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) {
               hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr,
                 back[i].url_fil);
@@ -3942,6 +4173,11 @@ int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize)
   return 1;
 }
 
+/* Grace left to the smooth stop before in-flight transfers are aborted. */
+static int back_maxtime_grace(const int maxtime) {
+  return maximum(5, minimum(30, maxtime / 10));
+}
+
 int back_checkmirror(httrackp * opt) {
   // Check max size
   if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
@@ -3958,13 +4194,19 @@ int back_checkmirror(httrackp * opt) {
      */
   }
   // Check max time
-  if ((opt->maxtime > 0)
+  if (opt->maxtime > 0) {
-      && ((time_local() - HTS_STAT.stat_timestart) >= opt->maxtime)) {
+    const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
-    if (!opt->state.stop) {     /* not yet stopped */
+
-      hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+    if (elapsed >= opt->maxtime) {
-                    opt->maxtime);
+      if (!opt->state.stop) { /* not yet stopped */
-      /* cancel mirror smoothly */
+        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-      hts_request_stop(opt, 0);
+                      opt->maxtime);
+        /* cancel mirror smoothly */
+        hts_request_stop(opt, 0);
+      }
+      /* smooth stop starved past the grace period: stop waiting (#481) */
+      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+        return 0;
     }
   }
   return 1;                     /* Ok, go on */

src/htsback.h

@@ -113,6 +113,12 @@ void back_set_locked(struct_back * sback, const int p);
 void back_set_unlocked(struct_back * sback, const int p);
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p);
+/* Discard back's on-disk .delayed placeholder and its refname. */
+void back_delayed_discard(httrackp *opt, lien_back *back);
+/* Move back's .delayed placeholder (and open stream) to newname;
+   HTS_FALSE = file lost, slot flagged in error. */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname);
 void back_index_unlock(struct_back * sback, const int p);
 int back_clear_entry(lien_back * back);
 int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
@@ -136,6 +142,8 @@ void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
+/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
+   the -E deadline overran its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htsbasenet.h

@@ -41,7 +41,7 @@ Please visit our Website: http://www.httrack.com
 
 #ifdef _WIN32
 
-#if HTS_INET6==0
+#if HTS_INET6 == 0
 #include <winsock2.h>
 #else
 
@@ -49,13 +49,12 @@ Please visit our Website: http://www.httrack.com
 #define WIN32_LEAN_AND_MEAN
 // KB955045 (http://support.microsoft.com/kb/955045)
 // To execute an application using this function on earlier versions of Windows
-// (Windows 2000, Windows NT, and Windows Me/98/95), then it is mandatary to #include Ws2tcpip.h
+// (Windows 2000, Windows NT, and Windows Me/98/95), then it is mandatary to
-// and also Wspiapi.h. When the Wspiapi.h header file is included, the 'getaddrinfo' function is
+// #include Ws2tcpip.h and also Wspiapi.h. When the Wspiapi.h header file is
-// #defined to the 'WspiapiGetAddrInfo' inline function in Wspiapi.h. 
+// included, the 'getaddrinfo' function is #defined to the 'WspiapiGetAddrInfo'
+// inline function in Wspiapi.h.
 #include <ws2tcpip.h>
 #include <Wspiapi.h>
-//#include <winsock2.h>
-//#include <tpipv6.h>
 
 #endif
 
@@ -128,6 +127,8 @@ typedef enum HTTPStatusCode {
   HTTP_UNSUPPORTED_MEDIA_TYPE = 415,
   HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416,
   HTTP_EXPECTATION_FAILED = 417,
+  HTTP_TOO_MANY_REQUESTS = 429,
+  HTTP_UNAVAILABLE_FOR_LEGAL_REASONS = 451,
   HTTP_INTERNAL_SERVER_ERROR = 500,
   HTTP_NOT_IMPLEMENTED = 501,
   HTTP_BAD_GATEWAY = 502,
@@ -145,7 +146,8 @@ typedef enum BackStatusCode {
   STATUSCODE_NON_FATAL = -5,
   STATUSCODE_SSL_HANDSHAKE = -6,
   STATUSCODE_TOO_BIG = -7,
-  STATUSCODE_TEST_OK = -10
+  STATUSCODE_TEST_OK = -10,
+  STATUSCODE_EXCLUDED = -11 /* aborted: MIME excluded by a -mime: filter */
 } BackStatusCode;
 
 /** HTTrack status ('status' member of of 'lien_back') **/

src/htsbasiccharsets.sh

@@ -3,57 +3,59 @@
 
 # Change this to download files
 if false; then
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
-echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
+    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
-rm -f CP932.TXT CP936.TXT CP949.TXT CP950.TXT
+    rm -f CP932.TXT CP936.TXT CP949.TXT CP950.TXT
 fi
 
 # Produce code
-printf "/** GENERATED FILE ($0), DO NOT EDIT **/\n\n"
+printf '/** GENERATED FILE (%s), DO NOT EDIT **/\n\n' "$0"
-for i in *.TXT ; do
+for i in *.TXT; do
-  echo "processing $i" >&2
+    echo "processing $i" >&2
-  grep -vE "^(#|$)" $i | grep -E "^0x" | sed -e 's/[[:space:]]/ /g' | cut -f1,2 -d' ' | \
+    grep -vE "^(#|$)" "$i" | grep -E "^0x" | sed -e 's/[[:space:]]/ /g' | cut -f1,2 -d' ' |
-  (
+        (
-    unset arr
+            unset arr
-    while read LINE ; do
+            while read -r LINE; do
-      from=$[$(echo $LINE | cut -f1 -d' ')]
+                from=$(($(echo "$LINE" | cut -f1 -d' ')))
-      if ! test -n "$from"; then
+                if ! test -n "$from"; then
-        echo "error with $i" >&2
+                    echo "error with $i" >&2
-        exit 1
+                    exit 1
-      elif test $from -ge 256; then
+                elif test $from -ge 256; then
-        echo "out-of-range ($LINE) with $i" >&2
+                    echo "out-of-range ($LINE) with $i" >&2
-        exit 1
+                    exit 1
-      fi
+                fi
-      to=$(echo $LINE | cut -f2 -d' ') 
+                to=$(echo "$LINE" | cut -f2 -d' ')
-      arr[$from]=$to
+                arr[from]=$to
-    done
+            done
-    name=$(echo $i | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
+            # shellcheck disable=SC2018,SC2019 # charset filenames are ASCII; keep C-locale A-Z/a-z
-    printf "/* Table for $i */\nstatic const hts_UCS4 table_${name}[256] = {\n  "
+            name=$(echo "$i" | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
-    i=0
+            printf '/* Table for %s */\nstatic const hts_UCS4 table_%s[256] = {\n  ' "$i" "$name"
-    while test "$i" -lt 256; do
+            idx=0
-      if test "$i" -gt 0; then
+            while test "$idx" -lt 256; do
-        printf ", "
+                if test "$idx" -gt 0; then
-        if test $[${i}%8] -eq 0; then
+                    printf ", "
-          printf "\n  "
+                    if test $((idx % 8)) -eq 0; then
-        fi
+                        printf "\n  "
-      fi
+                    fi
-      value=${arr[$i]:-0}
+                fi
-      printf "0x%04x" $value
+                value=${arr[$idx]:-0}
-      i=$[${i}+1]
+                printf "0x%04x" "$value"
-    done
+                idx=$((idx + 1))
-    printf " };\n\n"
+            done
-  )
+            printf " };\n\n"
-  echo "processed $i" >&2
+        )
+    echo "processed $i" >&2
 done
 
 # Indexes
 printf "static const struct {\n  const char *name;\n  const hts_UCS4 *table;\n} table_mappings[] = {\n"
-for i in *.TXT ; do
+for i in *.TXT; do
-  name=$(echo $i | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
+    # shellcheck disable=SC2018,SC2019 # charset filenames are ASCII; keep C-locale A-Z/a-z
-  printf "  { \"$(echo $name | tr -d '_')\", table_${name} },\n"
+    name=$(echo "$i" | tr 'A-Z' 'a-z' | tr '-' '_' | sed -e 's/\.txt//' -e 's/8859/iso_8859/')
+    printf '  { "%s", table_%s },\n' "$(echo "$name" | tr -d '_')" "$name"
 done
 printf "  { NULL, NULL }\n};\n"

src/htsbauth.c

@@ -107,7 +107,6 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
 #if DEBUG_COOK
   printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
          cook_name, cook_value, domain, path);
-  //printf(">>>cook: %s<<<\n",cookie->data);
 #endif
   return 0;
 }
@@ -208,8 +207,6 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   char catbuff[CATBUFF_SIZE];
   char buffer[8192];
 
-  //  cookie->data[0]='\0';
-
   // Fusionner d'abord les éventuels cookies IE
 #ifdef _WIN32
   {

src/htsbauth.h

@@ -68,14 +68,15 @@ struct t_cookie {
 #ifdef HTS_INTERNAL_BYTECODE
 
 /* cookies */
-int cookie_add(t_cookie * cookie, const  char *cook_name, const  char *cook_value,
+int cookie_add(t_cookie *cookie, const char *cook_name, const char *cook_value,
-               const  char *domain, const  char *path);
+               const char *domain, const char *path);
 
-int cookie_del(t_cookie * cookie, const char *cook_name, const char *domain, const char *path);
+int cookie_del(t_cookie *cookie, const char *cook_name, const char *domain,
+               const char *path);
 
-int cookie_load(t_cookie * cookie, const char *path, const char *name);
+int cookie_load(t_cookie *cookie, const char *path, const char *name);
 
-int cookie_save(t_cookie * cookie, const char *name);
+int cookie_save(t_cookie *cookie, const char *name);
 
 void cookie_insert(char *s, size_t s_size, const char *ins);
 
@@ -83,7 +84,8 @@ void cookie_delete(char *s, size_t s_size, size_t pos);
 
 const char *cookie_get(char *buffer, const char *cookie_base, int param);
 
-char *cookie_find(char *s, const char *cook_name, const char *domain, const char *path);
+char *cookie_find(char *s, const char *cook_name, const char *domain,
+                  const char *path);
 
 char *cookie_nextfield(char *a);
 
@@ -92,12 +94,13 @@ char *cookie_nextfield(char *a);
 /** Register credentials (auth = base-64 user:pass) for the prefix derived from
     adr (host) and fil (path). No-op returning 0 if cookie is NULL, allocation
     fails, or a matching prefix is already stored; returns 1 on insertion. */
-int bauth_add(t_cookie * cookie, const char *adr, const char *fil, const char *auth);
+int bauth_add(t_cookie *cookie, const char *adr, const char *fil,
+              const char *auth);
 
 /** Return the stored base-64 credentials whose prefix matches adr+fil, or NULL
     if none (or cookie is NULL). Returned pointer aliases the jar's bauth_chain;
     caller must not free it. */
-char *bauth_check(t_cookie * cookie, const char *adr, const char *fil);
+char *bauth_check(t_cookie *cookie, const char *adr, const char *fil);
 
 /** Build the auth lookup key (host + path, query string stripped, truncated at
     the last '/') from adr and fil into prefix; returns prefix. Caller must

src/htscache.c

@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
 #include "htscore.h"
 #include "htsbasenet.h"
 #include "htsmd5.h"
+#include <limits.h>
 #include <time.h>
 
 #include "htszlib.h"
@@ -220,6 +221,40 @@ struct cache_back_zip_entry {
 	} \
 } while(0)
 
+/* Consecutive entry write failures before the cache stream is declared dead. */
+#define CACHE_MAX_WRITE_FAILURES 8
+
+/* Cache write failed: a fatal errno or a failure streak aborts the mirror
+   (exit_xh); an isolated failure only drops the current entry. */
+static void cache_zip_write_failed(httrackp *opt, cache_back *cache,
+                                   const char *what, int zErr,
+                                   hts_boolean entry_open, const char *url_adr,
+                                   const char *url_fil) {
+  const int fatal_errno = zErr == ZIP_ERRNO && check_fatal_io_errno();
+
+  cache->zipWriteFailures++;
+  if (fatal_errno || cache->zipWriteFailures >= CACHE_MAX_WRITE_FAILURES) {
+    if (!cache->zipWriteFailed) {
+      cache->zipWriteFailed = HTS_TRUE;
+      if (fatal_errno) {
+        hts_log_print(opt, LOG_ERROR,
+                      "Mirror aborted: disk full or filesystem problems");
+      } else {
+        hts_log_print(opt, LOG_ERROR,
+                      "Mirror aborted: cache write failed (%s): %s", what,
+                      hts_get_zerror(zErr));
+      }
+    }
+    opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
+  } else {
+    if (entry_open)
+      zipCloseFileInZip((zipFile) cache->zipOutput); /* abandon, best-effort */
+    hts_log_print(opt, LOG_WARNING,
+                  "cache write failed (%s: %s), entry not cached: %s%s", what,
+                  hts_get_zerror(zErr), url_adr, url_fil);
+  }
+}
+
 /* Ajout d'un fichier en cache */
 void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
                const char *url_adr, const char *url_fil, const char *url_save,
@@ -230,12 +265,14 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
   char BIGSTK headers[8192];
   int headersSize = 0;
 
-  //int entryBodySize = 0;
-  //int entryFilenameSize = 0;
   zip_fileinfo fi;
   const char *url_save_suffix = url_save;
   int zErr;
 
+  /* already failed and aborting; don't touch the broken stream again */
+  if (cache->zipWriteFailed)
+    return;
+
   // robots.txt hack
   if (url_save == NULL) {
     dataincache = 0;            // testing links
@@ -263,10 +300,19 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
   if (r->size < 0)              // error
     return;
 
-  // data in cache
+  // data in cache: the body must fit the 32-bit zip write API
-  if (dataincache) {
+  if (dataincache && (LLint) (int) r->size != r->size) {
-    assertf(((int) r->size) == r->size);
+    if (r->is_write && url_save != NULL && strnotempty(url_save)) {
-    //entryBodySize = (int) r->size;
+      hts_log_print(opt, LOG_WARNING,
+                    "file too large for the cache, storing headers only: %s%s",
+                    url_adr, url_fil);
+      dataincache = 0;
+    } else {
+      hts_log_print(opt, LOG_WARNING,
+                    "entry too large for the cache, not cached: %s%s", url_adr,
+                    url_fil);
+      return;
+    }
   }
 
   /* Fields */
@@ -311,8 +357,6 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
   ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil);     // Original URI filename
   ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save_suffix);    // Original save filename
 
-  //entryFilenameSize = (int) ( strlen(url_adr) + strlen(url_fil));
-
   /* Filename */
   if (!link_has_authority(url_adr)) {
     strcpybuff(filename, "http://");
@@ -346,9 +390,9 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
                                    */
                                   headers, (uInt) strlen(headers), NULL, 0, NULL,       /* comment */
                                   Z_DEFLATED, Z_DEFAULT_COMPRESSION)) != Z_OK) {
-    int zip_zipOpenNewFileInZip_failed = 0;
+    cache_zip_write_failed(opt, cache, "opening a cache entry", zErr, HTS_FALSE,
-
+                           url_adr, url_fil);
-    assertf(zip_zipOpenNewFileInZip_failed);
+    return;
   }
 
   /* Write data in cache */
@@ -358,9 +402,9 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
         if ((zErr =
              zipWriteInFileInZip((zipFile) cache->zipOutput, r->adr,
                                  (int) r->size)) != Z_OK) {
-          int zip_zipWriteInFileInZip_failed = 0;
+          cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
-
+                                 HTS_TRUE, url_adr, url_fil);
-          assertf(zip_zipWriteInFileInZip_failed);
+          return;
         }
       }
     } else {
@@ -381,9 +425,10 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
               if ((zErr =
                    zipWriteInFileInZip((zipFile) cache->zipOutput, buff,
                                        (int) nl)) != Z_OK) {
-                int zip_zipWriteInFileInZip_failed = 0;
+                cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
-
+                                       HTS_TRUE, url_adr, url_fil);
-                assertf(zip_zipWriteInFileInZip_failed);
+                fclose(fp);
+                return;
               }
             }
           } while(nl > 0);
@@ -397,17 +442,19 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
 
   /* Close */
   if ((zErr = zipCloseFileInZip((zipFile) cache->zipOutput)) != Z_OK) {
-    int zip_zipCloseFileInZip_failed = 0;
+    cache_zip_write_failed(opt, cache, "closing a cache entry", zErr, HTS_FALSE,
-
+                           url_adr, url_fil);
-    assertf(zip_zipCloseFileInZip_failed);
+    return;
   }
 
   /* Flush */
   if ((zErr = zipFlush((zipFile) cache->zipOutput)) != 0) {
-    int zip_zipFlush_failed = 0;
+    cache_zip_write_failed(opt, cache, "flushing the cache", zErr, HTS_FALSE,
-
+                           url_adr, url_fil);
-    assertf(zip_zipFlush_failed);
+    return;
   }
+
+  cache->zipWriteFailures = 0; /* entry stored: reset the failure streak */
 }
 
 #else
@@ -576,15 +623,18 @@ htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
   return cache_readex(opt, cache, adr, fil, save, location, NULL, 1);
 }
 
-htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
+htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
-                                   const char *adr, const char *fil) {
+                                   const char *adr, const char *fil,
-  htsblk r = cache_read(opt, cache, adr, fil, NULL, NULL);
+                                   char *return_save) {
+  htsblk r = cache_readex(opt, cache, adr, fil, NULL, NULL, return_save, 0);
 
   if (r.statuscode == -1) {
     lien_back *itemback = NULL;
 
     if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
       r = itemback->r;
+      if (return_save != NULL)
+        strlcpybuff(return_save, itemback->url_sav, HTS_URLMAXSIZE * 2);
       /* cleanup */
       back_clear_entry(itemback);       /* delete entry content */
       freet(itemback);          /* delete item */
@@ -671,7 +721,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
         char BIGSTK headerBuff[8192 + 2];
         int readSizeHeader;
 
-        //int totalHeader = 0;
         int dataincache = 0;
 
         /* For BIG comments */
@@ -722,13 +771,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                                    HTS_URLMAXSIZE * 2);
               ZIP_READFIELD_STRING(line, value, "Content-Disposition", r.cdispo,
                                    sizeof(r.cdispo));
-              //ZIP_READFIELD_STRING(line, value, "X-Addr", ..);            // Original address
-              //ZIP_READFIELD_STRING(line, value, "X-Fil", ..);            // Original URI filename
               ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
                                    sizeof(previous_save_));
             }
-          } while(offset < readSizeHeader && !lineEof);
+          } while (offset < readSizeHeader && !lineEof);
-          //totalHeader = offset;
 
           /* Previous entry */
           if (previous_save_[0] != '\0') {
@@ -745,6 +791,15 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
             strlcpybuff(return_save, previous_save, HTS_URLMAXSIZE * 2);
           }
 
+          /* A negative X-Size is corrupt; so is one >= INT_MAX when the data
+             is in the zip (the write path asserts int-sized). Headers-only
+             entries legitimately exceed INT_MAX (>2GB body on disk): keep
+             them, or every update would re-fetch the file. */
+          if (r.size < 0 || (dataincache && r.size >= INT_MAX)) {
+            r.statuscode = STATUSCODE_INVALID;
+            strcpybuff(r.msg, "Cache Read Error : Bad Size");
+          }
+
           /* Complete fields */
           r.totalsize = r.size;
           r.adr = NULL;
@@ -771,7 +826,8 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
             }                   // otherwise, the ZIP file is supposed to be consistent with data.
           }
           /* Read data ? */
-          else {                /* ne pas lire uniquement header */
+          else if (r.statuscode !=
+                   STATUSCODE_INVALID) { /* ne pas lire uniquement header */
             int ok = 0;
 
 #if HTS_DIRECTDISK
@@ -900,7 +956,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg,
                              "Cache Write Error : Unable to Create File");
-                  //printf("%s\n",save);
                 }
               }
 
@@ -935,7 +990,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg,
                                "Previous cache file not found (empty filename)");
                   }
-                } else {        /* Read in memory from disk */
+                } else if (r.size >= INT_MAX) { /* too big to read in memory */
+                  r.statuscode = STATUSCODE_INVALID;
+                  strcpybuff(r.msg, "Cache Read Error : Bad Size");
+                } else { /* Read in memory from disk */
                   FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), previous_save), "rb");
 
                   if (fp != NULL) {
@@ -975,7 +1033,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg, "Cache Read Error : Read Data");
                   } else
                     *(r.adr + r.size) = '\0';
-                  //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                 } else {        // erreur
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg, "Cache Memory Error");
@@ -1154,17 +1211,14 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
         // sécurité
         r.adr = NULL;
         r.out = NULL;
-        ////r.location=NULL;  non, fixée lors des 301 ou 302
         r.fp = NULL;
 
         if ((r.statuscode >= 0) && (r.statuscode <= 999)
             && (r.notmodified >= 0) && (r.notmodified <= 9)) {  // petite vérif intégrité
-          if ((save) && (!header_only)) {       /* ne pas lire uniquement header */
+          if ((save) && (!header_only)) { /* ne pas lire uniquement header */
-            //int to_file=0;
 
             r.adr = NULL;
             r.soc = INVALID_SOCKET;
-            // // r.location=NULL;
 
 #if HTS_DIRECTDISK
             // Court-circuit:
@@ -1173,12 +1227,11 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
               int ok = 0;
 
               r.is_write = 1;   // écrire
-              if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {  // un fichier existe déja
+              if (fexist_utf8(fconv(catbuff, sizeof(catbuff),
-                //if (fsize_utf8(fconv(save))==r.size) {  // même taille -- NON tant pis (taille mal declaree)
+                                    save))) { // un fichier existe déja
                 ok = 1;         // plus rien à faire
                 filenote(&opt->state.strc, save, NULL); // noter comme connu
                 file_notify(opt, adr, fil, save, 0, 0, 0);
-                //}
               }
 
               if ((pos < 0) && (!ok)) { // Pas de donnée en cache et fichier introuvable : erreur!
@@ -1229,7 +1282,6 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg,
                              "Cache Write Error : Unable to Create File");
-                  //printf("%s\n",save);
                 }
               }
 
@@ -1277,14 +1329,13 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg, "Cache Read Error : Read Data");
                   } else
                     *(r.adr + r.size) = '\0';
-                  //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                 } else {        // erreur
                   r.statuscode = STATUSCODE_INVALID;
                   strcpybuff(r.msg, "Cache Memory Error");
                 }
               }
             }
-          }                     // si save==null, ne rien charger (juste en tête)
+          } // si save==null, ne rien charger (juste en tête)
         } else {
 #if DEBUGCA
           printf("Cache Read Error : Bad Data");
@@ -1397,6 +1448,86 @@ static int hts_rename(httrackp * opt, const char *a, const char *b) {
   return rename(a, b);
 }
 
+/* Pathname of a file inside the mirror dir (rotating concat buffer). */
+static char *reconcile_path(httrackp *opt, const char *name) {
+  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                 StringBuff(opt->path_log), name);
+}
+
+/* Interrupted-run heuristic: prefer the old generation when the new cache
+   stalled below NEW_TINY while the old one grew past OLD_SOLID (historical
+   arbitrary thresholds). */
+#define CACHE_RECONCILE_NEW_TINY 32768
+#define CACHE_RECONCILE_OLD_SOLID 65536
+
+/* Replace the new-generation file by the old one, when the old one exists. */
+static void reconcile_promote(httrackp *opt, const char *oldname,
+                              const char *newname) {
+  if (fexist(reconcile_path(opt, oldname))) {
+    remove(reconcile_path(opt, newname));
+    rename(reconcile_path(opt, oldname), reconcile_path(opt, newname));
+  }
+}
+
+void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
+  switch (mode) {
+  case CACHE_RECONCILE_PROMOTE:
+    /* Previous run rotated new.* to old.* then died before writing: promote
+       the old generation back, whichever format it uses. */
+    if (!fexist(reconcile_path(opt, "hts-cache/new.zip")))
+      reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if ((!fexist(reconcile_path(opt, "hts-cache/new.dat")) ||
+         !fexist(reconcile_path(opt, "hts-cache/new.ndx"))) &&
+        fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    break;
+  case CACHE_RECONCILE_INTERRUPTED:
+    /* Aborted run: keep the larger generation when the new cache is
+       suspiciously small next to the old one. The new file must exist: fsize()
+       is -1 for a missing file, which would spuriously pass the "< TINY" test
+       and overwrite a solid old generation that PROMOTE/ROLLBACK should keep.
+     */
+    if (!opt->cache || !fexist(reconcile_path(opt, "hts-in_progress.lock")))
+      break;
+    if (fexist(reconcile_path(opt, "hts-cache/new.zip")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.zip")) &&
+        fsize(reconcile_path(opt, "hts-cache/new.zip")) <
+            CACHE_RECONCILE_NEW_TINY &&
+        fsize(reconcile_path(opt, "hts-cache/old.zip")) >
+            CACHE_RECONCILE_OLD_SOLID &&
+        fsize(reconcile_path(opt, "hts-cache/old.zip")) >
+            fsize(reconcile_path(opt, "hts-cache/new.zip")))
+      reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if (fexist(reconcile_path(opt, "hts-cache/new.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx")) &&
+        fsize(reconcile_path(opt, "hts-cache/new.dat")) <
+            CACHE_RECONCILE_NEW_TINY &&
+        fsize(reconcile_path(opt, "hts-cache/old.dat")) >
+            CACHE_RECONCILE_OLD_SOLID &&
+        fsize(reconcile_path(opt, "hts-cache/old.dat")) >
+            fsize(reconcile_path(opt, "hts-cache/new.dat"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    break;
+  case CACHE_RECONCILE_ROLLBACK:
+    /* Nothing transferred: restore the previous generation and sidecars. */
+    reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if (fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    reconcile_promote(opt, "hts-cache/old.lst", "hts-cache/new.lst");
+    reconcile_promote(opt, "hts-cache/old.txt", "hts-cache/new.txt");
+    break;
+  }
+}
+
 // renvoyer uniquement en tête, ou NULL si erreur
 // return NULL upon error, and set -1 to r.statuscode
 htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
@@ -1427,25 +1558,10 @@ void cache_init(cache_back * cache, httrackp * opt) {
       mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"),
             HTS_PROTECT_FOLDER);
 #endif
-      if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.zip")))) {       // il existe déja un cache précédent.. renommer
+      if ((fexist(fconcat(
-        /* Previous cache from the previous cache version */
+              OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-#if 0
+              StringBuff(opt->path_log),
-        /* No.. reuse with old httrack releases! */
+              "hts-cache/new.zip")))) { // a previous cache exists.. rename it
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.dat")))
-          remove(fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.dat"));
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.ndx")))
-          remove(fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.ndx"));
-#endif
         /* Previous cache version */
         if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) {     // il existe déja un cache précédent.. renommer
           rename(fconcat
@@ -1486,7 +1602,13 @@ void cache_init(cache_back * cache, httrackp * opt) {
         } else {
           hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed");
         }
-      } else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) {        // il existe déja un cache précédent.. renommer
+      } else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                 StringBuff(opt->path_log),
+                                 "hts-cache/new.dat"))) &&
+                 (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                 StringBuff(opt->path_log),
+                                 "hts-cache/new.ndx")))) { // a previous cache
+                                                           // exists.. rename it
 #if DEBUGCA
         printf("work with former cache\n");
 #endif
@@ -1515,7 +1637,7 @@ void cache_init(cache_back * cache, httrackp * opt) {
                 "hts-cache/new.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
                                               StringBuff(opt->path_log),
                                               "hts-cache/old.ndx"));
-      } else {                  // un des deux (ou les deux) fichiers cache absents: effacer l'autre éventuel
+      } else { // one or both cache files missing: remove the remaining one
 #if DEBUGCA
         printf("new cache\n");
 #endif
@@ -1884,12 +2006,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                    "hts-cache/new.lst"), "wb");
           strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
           opt->state.strc.lst = cache->lst;
-          //{
-          //filecreate_params tmp;
-          //strcpybuff(tmp.path,StringBuff(opt->path_html));    // chemin
-          //tmp.lst=cache->lst;                 // fichier lst
-          //filenote("",&tmp);        // initialiser filecreate
-          //}
 
           // supprimer old.txt
           if (fexist
@@ -1978,12 +2094,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                    "hts-cache/new.lst"), "wb");
           strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
           opt->state.strc.lst = cache->lst;
-          //{
-          //  filecreate_params tmp;
-          //  strcpybuff(tmp.path,StringBuff(opt->path_html));    // chemin
-          //  tmp.lst=cache->lst;                 // fichier lst
-          //  filenote("",&tmp);        // initialiser filecreate
-          //}
 
           // supprimer old.txt
           if (fexist
@@ -2015,8 +2125,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                     "statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)"
                     LF);
           }
-          // test
-          // cache_writedata(cache->ndx,cache->dat,"//[TEST]//","test1","TEST PIPO",9);
         }                       // cache->ndx!=NULL
       }                         //cache->zipOutput != NULL
 

src/htscache.h

@@ -66,8 +66,11 @@ htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr,
                   const char *fil, const char *save, char *location);
 htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, const char *save, char *location);
-htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
+/* Like cache_read, but also yields entries whose transfer broke; return_save
-                                   const char *adr, const char *fil);
+   (optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name. */
+htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
+                                   const char *adr, const char *fil,
+                                   char *return_save);
 htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr,
                     const char *fil, const char *save, char *location,
                     char *return_save, int readonly);
@@ -75,6 +78,17 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, htsblk * r);
 void cache_init(cache_back * cache, httrackp * opt);
 
+/* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
+typedef enum {
+  CACHE_RECONCILE_PROMOTE,     /* no new cache: promote the old generation */
+  CACHE_RECONCILE_INTERRUPTED, /* aborted run: keep the larger generation */
+  CACHE_RECONCILE_ROLLBACK     /* nothing transferred: restore the old one */
+} hts_cache_reconcile_mode;
+
+/* Reconcile the on-disk cache generations according to mode; a no-op when
+   the involved files are absent. */
+void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
+
 int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
                     const char *str2, char *outbuff, int len);
 int cache_readdata(cache_back * cache, const char *str1, const char *str2,

src/htscache_selftest.c

@@ -47,6 +47,8 @@ Please visit our Website: http://www.httrack.com
 #include "htslib.h"
 #include "htszlib.h"
 
+#include <errno.h>
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -316,6 +318,328 @@ static int disk_fallback_selftest(httrackp *opt) {
   return fail;
 }
 
+typedef struct {
+  size_t budget;  /**< bytes allowed through before writes start failing */
+  int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
+  int writes;     /**< zwrite call count, to detect re-entry into the stream */
+  int fail_once;  /**< recover (unlimited budget) after the first failure */
+} writefail_inject;
+
+/* zwrite that copies until the budget runs out, then fails with inj->fail_errno
+   (the #174/#219 condition). Counts calls so the test can prove a flagged cache
+   never re-enters the stream. */
+static uLong selftest_failing_zwrite(voidpf opaque, voidpf stream,
+                                     const void *buf, uLong size) {
+  writefail_inject *inj = (writefail_inject *) opaque;
+
+  inj->writes++;
+  if (inj->budget >= (size_t) size) {
+    inj->budget -= (size_t) size;
+    return (uLong) fwrite(buf, 1, (size_t) size, (FILE *) stream);
+  }
+  if (inj->fail_once)
+    inj->budget = (size_t) -1; /* the backend recovers after this failure */
+  errno = inj->fail_errno;
+  return 0; /* short write -> the minizip op returns an error */
+}
+
+/* Open a ZIP whose writes fail past inj->budget, so cache_add() hits an error.
+ */
+static zipFile selftest_open_failing_zip(const char *path,
+                                         writefail_inject *inj) {
+  zlib_filefunc_def ff;
+
+  fill_fopen_filefunc(&ff); /* real fopen/read/seek/close; ignores opaque */
+  ff.zwrite_file = selftest_failing_zwrite;
+  ff.opaque = inj;
+  return zipOpen2(path, APPEND_STATUS_CREATE, NULL, &ff);
+}
+
+/* Store one octet-stream body into `cache` (all-in-cache, body in the ZIP). */
+static void writefail_store(httrackp *opt, cache_back *cache, const char *fil,
+                            const char *body, size_t body_len) {
+  htsblk r;
+  char locbuf[4];
+  char *bodycopy = malloct(body_len);
+
+  hts_init_htsblk(&r);
+  r.statuscode = 200;
+  r.size = (LLint) body_len;
+  strcpybuff(r.msg, "OK");
+  strcpybuff(r.contenttype, "application/octet-stream");
+  locbuf[0] = '\0';
+  r.location = locbuf;
+  r.is_write = 0;
+  memcpy(bodycopy, body, body_len);
+  r.adr = bodycopy;
+  cache_add(opt, cache, &r, "example.com", fil, "example.com/blob.bin", 1,
+            NULL);
+  freet(bodycopy);
+}
+
+/* Store an entry claiming a >2GB body; the degrade path never reads data. */
+static void writefail_store_oversized(httrackp *opt, cache_back *cache,
+                                      const char *fil, int is_write) {
+  htsblk r;
+  char locbuf[4];
+
+  hts_init_htsblk(&r);
+  r.statuscode = 200;
+  r.size = (LLint) INT_MAX + 1;
+  strcpybuff(r.msg, "OK");
+  strcpybuff(r.contenttype, "application/octet-stream");
+  locbuf[0] = '\0';
+  r.location = locbuf;
+  r.is_write = (short int) is_write;
+  cache_add(opt, cache, &r, "example.com", fil, "example.com/big.bin", 1, NULL);
+}
+
+/* Read back `entryname`: extra field (cached headers) and body. Returns the
+   body length, or -1 if the entry is absent or unreadable. */
+static int writefail_read_entry(const char *path, const char *entryname,
+                                char *extra, size_t extralen, char *body,
+                                size_t bodylen) {
+  unzFile z = unzOpen(path);
+  int n = -1;
+
+  if (z == NULL)
+    return -1;
+  if (unzLocateFile(z, entryname, 1) == UNZ_OK &&
+      unzOpenCurrentFile(z) == UNZ_OK) {
+    const int elen = unzGetLocalExtrafield(z, extra, (unsigned) (extralen - 1));
+
+    if (elen >= 0) {
+      extra[elen] = '\0';
+      n = unzReadCurrentFile(z, body, (unsigned) bodylen);
+    }
+    unzCloseCurrentFile(z);
+  }
+  unzClose(z);
+  return n;
+}
+
+/* Cache write-failure policy (#174/#219): fatal errno or a failure streak
+   stops the mirror (exit_xh=-1, no crash); isolated/oversized drops the entry.
+ */
+int cache_write_failure_selftest(httrackp *opt, const char *dir) {
+  int fail = 0;
+  char path[HTS_URLMAXSIZE];
+  /* incompressible + big, so deflate flushes (and fails) mid-write, before
+   * close */
+  static const size_t body_len = 256 * 1024;
+  char *body = malloct(body_len);
+  int phase;
+
+  gen_body(body, body_len, 1 /* incompressible */);
+  fconcat(path, sizeof(path), dir, "/wfail.zip");
+
+  /* phase 0: fatal errno (ENOSPC) aborts at once; phase 1: persistent EIO
+     drops entries until the streak caps out, then aborts. */
+  for (phase = 0; phase < 2; phase++) {
+    cache_back cache;
+    writefail_inject inj;
+    int writes_after_fail;
+
+    inj.budget = (phase == 0) ? 4096 : 0;
+    inj.fail_errno = (phase == 0) ? ENOSPC : EIO;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    if (cache.zipOutput == NULL) {
+      fprintf(stderr, "cache-writefail: could not open injected ZIP\n");
+      fail++;
+      continue;
+    }
+
+    opt->state.exit_xh = 0; /* clear; the failing write must set it to -1 */
+    if (phase == 0) {
+      writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    } else {
+      /* the abort must land exactly on the 8th consecutive failure */
+      int i;
+
+      for (i = 0; i < 7; i++) {
+        char fil[32];
+
+        snprintf(fil, sizeof(fil), "/b%d.bin", i);
+        writefail_store(opt, &cache, fil, body, 16);
+      }
+      if (cache.zipWriteFailed) {
+        fprintf(stderr, "cache-writefail: phase 1: aborted before the "
+                        "8th consecutive failure\n");
+        fail++;
+      }
+      writefail_store(opt, &cache, "/b7.bin", body, 16);
+    }
+    if (!cache.zipWriteFailed) {
+      fprintf(stderr, "cache-writefail: phase %d: write error not caught\n",
+              phase);
+      fail++;
+    }
+    if (opt->state.exit_xh != -1) {
+      fprintf(stderr,
+              "cache-writefail: phase %d: mirror not aborted (exit_xh=%d)\n",
+              phase, opt->state.exit_xh);
+      fail++;
+    }
+
+    /* a flagged cache must no-op a sibling write: no further backend write */
+    writes_after_fail = inj.writes;
+    writefail_store(opt, &cache, "/blob2.bin", body, 16);
+    if (inj.writes != writes_after_fail) {
+      fprintf(stderr,
+              "cache-writefail: phase %d: sibling write re-entered the broken "
+              "stream (%d extra backend writes)\n",
+              phase, inj.writes - writes_after_fail);
+      fail++;
+    }
+
+    if (cache.zipOutput != NULL) {
+      zipClose(cache.zipOutput,
+               NULL); /* best-effort; may fail on the backend */
+      cache.zipOutput = NULL;
+    }
+  }
+
+  /* failures with successes in between reset the streak: never aborts */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    int i;
+
+    inj.budget = (size_t) -1;
+    inj.fail_errno = EIO;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    for (i = 0; i < 10; i++) {
+      char fil[32];
+
+      inj.budget = 0; /* this store fails */
+      snprintf(fil, sizeof(fil), "/s%d.bin", i);
+      writefail_store(opt, &cache, fil, body, 16);
+      inj.budget = (size_t) -1; /* this one succeeds and resets the streak */
+      snprintf(fil, sizeof(fil), "/ok%d.bin", i);
+      writefail_store(opt, &cache, fil, body, 16);
+    }
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: scattered: non-consecutive failures aborted "
+              "the mirror (flagged=%d, exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+  }
+
+  /* isolated failure: only that entry drops; a later sibling round-trips */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    char extra[8192];
+    char rbody[64];
+    int n;
+
+    inj.budget = 4096;
+    inj.fail_errno = EIO;
+    inj.writes = 0;
+    inj.fail_once = 1;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: skip: isolated failure aborted the mirror "
+              "(flagged=%d, exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    writefail_store(opt, &cache, "/blob2.bin", body, 16);
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+    n = writefail_read_entry(path, "http://example.com/blob2.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody));
+    if (n != 16 || memcmp(rbody, body, 16) != 0) {
+      fprintf(stderr,
+              "cache-writefail: skip: sibling entry lost after a skipped "
+              "entry (%d)\n",
+              n);
+      fail++;
+    }
+  }
+
+  /* >2GB bodies: in-memory drops the entry, on-disk degrades to headers-only */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    char extra[8192];
+    char rbody[64];
+    int n;
+
+    inj.budget = (size_t) -1; /* no injected failure */
+    inj.fail_errno = 0;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    writefail_store_oversized(opt, &cache, "/bigmem.bin", 0 /* in-memory */);
+    writefail_store_oversized(opt, &cache, "/bigdisk.bin", 1 /* on-disk */);
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: oversize: mirror aborted (flagged=%d, "
+              "exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    if (writefail_read_entry(path, "http://example.com/bigmem.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody)) >= 0) {
+      fprintf(stderr,
+              "cache-writefail: oversize: in-memory entry was stored\n");
+      fail++;
+    }
+    n = writefail_read_entry(path, "http://example.com/bigdisk.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody));
+    if (n != 0 || strstr(extra, "X-In-Cache: 0") == NULL) {
+      fprintf(stderr,
+              "cache-writefail: oversize: on-disk entry not stored "
+              "headers-only (%d)\n",
+              n);
+      fail++;
+    }
+  }
+
+  freet(body);
+  return fail;
+}
+
 int cache_selftests(httrackp *opt, const char *dir) {
   int failures = 0;
   cache_back cache;
@@ -370,7 +694,7 @@ int cache_selftests(httrackp *opt, const char *dir) {
     StringCopy(opt->path_html, base);
     StringCopy(opt->path_html_utf8, base);
   }
-  opt->cache = 1;
+  opt->cache = HTS_CACHE_PRIORITY;
 
   /* pass 1: create everything in a single write session */
   selftest_open_for_write(&cache, opt);
@@ -547,7 +871,7 @@ static void golden_setup(httrackp *opt, const char *dir) {
   StringCopy(opt->path_log, base);
   StringCopy(opt->path_html, base);
   StringCopy(opt->path_html_utf8, base);
-  opt->cache = 1;
+  opt->cache = HTS_CACHE_PRIORITY;
 }
 
 int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
@@ -585,3 +909,494 @@ int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
 
   return failures;
 }
+
+/* --- hts_cache_reconcile() policies -------------------------------------- */
+
+/* All reconcile inputs/outputs, wiped between cases. */
+static const char *const reconcile_files[] = {
+    "hts-cache/new.zip", "hts-cache/old.zip",   "hts-cache/new.dat",
+    "hts-cache/old.dat", "hts-cache/new.ndx",   "hts-cache/old.ndx",
+    "hts-cache/new.lst", "hts-cache/old.lst",   "hts-cache/new.txt",
+    "hts-cache/old.txt", "hts-in_progress.lock"};
+
+static char *reconcile_st_path(httrackp *opt, const char *name) {
+  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                 StringBuff(opt->path_log), name);
+}
+
+static void reconcile_wipe(httrackp *opt) {
+  size_t i;
+
+  for (i = 0; i < sizeof(reconcile_files) / sizeof(reconcile_files[0]); i++)
+    remove(reconcile_st_path(opt, reconcile_files[i]));
+}
+
+/* Create a filler file of exactly `size` bytes. */
+static void reconcile_put(httrackp *opt, const char *name, size_t size) {
+  FILE *const fp = fopen(reconcile_st_path(opt, name), "wb");
+  static const char filler[1024] = {'x'};
+
+  assertf(fp != NULL);
+  while (size > 0) {
+    const size_t n = size > sizeof(filler) ? sizeof(filler) : size;
+
+    assertf(fwrite(filler, 1, n, fp) == n);
+    size -= n;
+  }
+  fclose(fp);
+}
+
+/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
+static int reconcile_expect(httrackp *opt, const char *name, off_t size,
+                            const char *what) {
+  const off_t got = fsize(reconcile_st_path(opt, name));
+
+  if (got != size) {
+    fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
+            name, (int) got, (int) size);
+    return 1;
+  }
+  return 0;
+}
+
+int cache_reconcile_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+
+  /* around the interrupted-run thresholds (new < 32768, old > 65536) */
+  static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
+
+  golden_setup(opt, dir);
+#ifdef _WIN32
+  mkdir(reconcile_st_path(opt, "hts-cache"));
+#else
+  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
+#endif
+
+  /* PROMOTE: a zip old generation replaces a missing new one */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "promote-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "promote-zip");
+
+  /* PROMOTE: an existing new.zip is left alone */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "promote-zip-noop");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
+
+  /* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
+     zip cache existed) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
+
+  /* PROMOTE: a half-written legacy new pair is replaced by the old pair */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
+
+  /* INTERRUPTED: no lock file, no action */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-nolock");
+
+  /* INTERRUPTED: an absent new.zip must NOT promote old.zip (fsize(-1) would
+     spuriously pass "< TINY"); leave the solid old generation for ROLLBACK */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", -1, "interrupted-nonew");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "interrupted-nonew");
+
+  /* INTERRUPTED: stalled tiny new.zip loses to a solid old.zip (was dead for
+     zip caches: the arm was gated on a legacy new.dat) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", SOLID, "interrupted-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "interrupted-zip");
+
+  /* INTERRUPTED: old below the confidence threshold, keep new */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-smallold");
+
+  /* INTERRUPTED: new big enough to trust, keep it */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", MID);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
+
+  /* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
+
+  /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
+     its only good generation here) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "rollback-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "rollback-zip");
+
+  /* ROLLBACK: sidecars are restored regardless of format */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.lst", TINY);
+  reconcile_put(opt, "hts-cache/old.lst", MID);
+  reconcile_put(opt, "hts-cache/old.txt", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
+  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
+
+  /* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", MID);
+  reconcile_put(opt, "hts-cache/old.lst", MID);
+  reconcile_put(opt, "hts-cache/old.txt", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
+
+  /* ROLLBACK: nothing to restore, the new generation stays */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", TINY, "rollback-noop");
+
+  reconcile_wipe(opt);
+  return failures;
+}
+
+/* --- read-side corruption injection --------------------------------------- */
+
+/* canary read back intact after each corruption; victim gets the byte surgery
+ */
+#define CORRUPT_ADR "corrupt.example.com"
+static char corrupt_body_a[33 + 1];
+static char corrupt_body_b[44 + 1];
+
+/* Write a fresh two-entry cache: /canary.html then /victim.html. */
+static void corrupt_build(httrackp *opt) {
+  cache_back cache;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
+              strlen(corrupt_body_b));
+  selftest_close(&cache);
+}
+
+/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
+   is later overwritten with a forged oversized X-Size (same byte length). */
+static void corrupt_build_etag(httrackp *opt) {
+  cache_back cache;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
+              "OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
+              corrupt_body_b, strlen(corrupt_body_b));
+  selftest_close(&cache);
+}
+
+/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
+   body on disk): the shape every non-html file is stored with. */
+static void corrupt_build_disk(httrackp *opt) {
+  cache_back cache;
+  htsblk w;
+  char locw[4];
+  char BIGSTK save[HTS_URLMAXSIZE * 2];
+  char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
+  char *path;
+  FILE *fp;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
+          CORRUPT_ADR "/victim.bin");
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  hts_init_htsblk(&w);
+  w.statuscode = 200;
+  w.size = (LLint) sizeof(corrupt_body_b) - 1;
+  strcpybuff(w.msg, "OK");
+  strcpybuff(w.contenttype, "application/octet-stream");
+  strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
+  locw[0] = '\0';
+  w.location = locw;
+  w.is_write = 0;
+  cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
+            0 /* all_in_cache */, NULL);
+  selftest_close(&cache);
+  /* the reader only checks this file exists; it never reads it here */
+  path = fconv(catbuff, sizeof(catbuff), save);
+  (void) structcheck(path);
+  fp = FOPEN(path, "wb");
+  assertf(fp != NULL);
+  fclose(fp);
+}
+
+/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
+static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
+                          const char *rep, size_t nth, size_t total) {
+  LLint fsz = 0;
+  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
+  const size_t n = (size_t) fsz;
+  size_t k, hits = 0, at = 0;
+  FILE *fp;
+
+  assertf(data != NULL);
+  for (k = 0; k + patlen <= n; k++) {
+    if (memcmp(data + k, pat, patlen) == 0) {
+      hits++;
+      if (hits == nth)
+        at = k;
+    }
+  }
+  assertf(hits == total);
+  memcpy(data + at, rep, patlen);
+  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, n, fp) == n);
+  fclose(fp);
+  freet(data);
+}
+
+/* Garbage the first bytes of the victim's deflated data (2nd local header). */
+static void corrupt_victim_body(httrackp *opt) {
+  LLint fsz = 0;
+  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
+  const size_t n = (size_t) fsz;
+  size_t k, hits = 0, off = 0;
+  FILE *fp;
+
+  assertf(data != NULL);
+  for (k = 0; k + 4 <= n; k++) {
+    if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
+      const size_t namelen =
+          (unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
+      const size_t extralen =
+          (unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
+
+      off = k + 30 + namelen + extralen;
+    }
+  }
+  assertf(hits == 2);
+  assertf(off != 0 && off + 4 <= n);
+  memset(data + off, 0xFF, 4);
+  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, n, fp) == n);
+  fclose(fp);
+  freet(data);
+}
+
+/* Read the corrupt /victim.html and, in the SAME read session, the intact
+   /canary.html: the victim must be rejected (wantmsg pins which path) and the
+   canary must still decode byte-exact, proving one bad entry never taints a
+   sibling read. */
+static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
+                                     const char *wantmsg, const char *what) {
+  cache_back cache;
+  htsblk v, c;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  char BIGSTK lc[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = lc[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
+  if (v.statuscode != STATUSCODE_INVALID) {
+    fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
+            selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
+    fail++;
+  }
+  if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
+    fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
+            selftest_tag, what, v.msg, wantmsg);
+    fail++;
+  }
+  c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
+  if (c.statuscode != 200 || c.adr == NULL ||
+      c.size != (LLint) strlen(corrupt_body_a) ||
+      memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
+    fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
+            c.statuscode);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  if (c.adr != NULL)
+    freet(c.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
+static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
+                                 const char *what) {
+  return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
+}
+
+/* Headers-only probe of the disk victim: must parse OK with the size kept. */
+static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
+                                      const char *what) {
+  cache_back cache;
+  htsblk v;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
+  if (v.statuscode != 200 || v.size != wantsize) {
+    fprintf(stderr,
+            "%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
+            selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
+/* One zip corruption case: build, patch, then check victim+canary in-session.
+ */
+static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
+                            size_t nth, size_t total, const char *wantmsg,
+                            const char *what) {
+  corrupt_build(opt);
+  corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
+  return corrupt_expect_victim(opt, wantmsg, what);
+}
+
+int cache_corruption_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+
+  selftest_tag = "cache-corrupt";
+  golden_setup(opt, dir);
+
+  failures +=
+      corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
+                       "Cache Read Error : Read Data", "oversized X-Size");
+  failures +=
+      corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
+                       "Cache Read Error : Bad Size", "negative X-Size");
+  /* both entries carry the line; the victim's is the second */
+  failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
+                               "Previous cache file not found (empty filename)",
+                               "blanked X-In-Cache");
+  /* smashed local file header: the entry is dropped at index load */
+  failures +=
+      corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
+                       "File Cache Entry Not Found", "smashed local header");
+
+  corrupt_build(opt);
+  corrupt_victim_body(opt);
+  failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
+                                    "garbled deflate stream");
+
+  /* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
+     truncates negative in the (int) cast the malloc uses: a wraparound alloc.
+     cache_add asserts size fits an int, so such a value only reaches the reader
+     from a corrupt/foreign cache; inject it by overwriting the victim's long
+     Etag line with a same-length forged X-Size line (the parser keeps the last
+     X-Size it sees), keeping the zip byte-length and offsets intact. */
+  corrupt_build_etag(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
+                                    "X-Size above INT_MAX");
+
+  /* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
+     is how every >2GB non-html file is stored. It must survive a header probe
+     (or every update re-fetches the file); an in-memory read still rejects. */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
+                                         "headers-only X-Size above INT_MAX");
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size above INT_MAX");
+
+  /* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483647AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size at INT_MAX");
+
+  /* the negative check must stay global, headers-only included */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: -2147483648AAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "headers-only negative X-Size");
+
+  return failures;
+}

src/htscache_selftest.h

@@ -52,6 +52,19 @@ int cache_selftests(httrackp *opt, const char *dir);
    committed file, never by the test). Returns the failed-check count. */
 int cache_golden_selftest(httrackp *opt, const char *dir, int regen);
 
+/* Cache write-failure policy (#174/#219): abort on fatal errno or a streak,
+   drop just the entry otherwise. Returns the failed-check count. */
+int cache_write_failure_selftest(httrackp *opt, const char *dir);
+
+/* Exercise the hts_cache_reconcile() generation policies on file fixtures
+   under <dir>. Returns the failed-check count. */
+int cache_reconcile_selftest(httrackp *opt, const char *dir);
+
+/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
+   under <dir> and assert every case degrades to STATUSCODE_INVALID without
+   tainting a sibling entry. */
+int cache_corruption_selftest(httrackp *opt, const char *dir);
+
 #endif
 
 #endif

src/htscatchurl.c

@@ -64,7 +64,7 @@ Please visit our Website: http://www.httrack.com
 // catch_url_init(&port,&return_host);
 HTSEXT_API T_SOC catch_url_init_std(int *port_prox, char *adr_prox) {
   T_SOC soc;
-  int try_to_listen_to[] = { 8080, 3128, 80, 81, 82, 8081, 3129, 31337, 0, -1 };
+  int try_to_listen_to[] = {8080, 3128, 80, 81, 82, 8081, 3129, 0, -1};
   int i = 0;
 
   do {
@@ -135,7 +135,8 @@ HTSEXT_API T_SOC catch_url_init(int *port, /* 128 bytes */ char *adr) {
 // returns 0 if error
 // url: buffer where URL must be stored - or ip:port in case of failure
 // data: 32Kb
-HTSEXT_API int catch_url(T_SOC soc, char *url, char *method, char *data) {
+HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
+                                 char *data) {
   int retour = 0;
 
   // connexion (accept)
@@ -174,7 +175,9 @@ HTSEXT_API int catch_url(T_SOC soc, char *url, char *method, char *data) {
       //
       socinput(soc, line, 1000);
       if (strnotempty(line)) {
-        if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
+        /* widths bound the caller buffers: method[32], url[HTS_URLMAXSIZE*2],
+           protocol[256] */
+        if (sscanf(line, "%31s %2047s %255s", method, url, protocol) == 3) {
           lien_adrfil af;
 
           // méthode en majuscule
@@ -194,7 +197,6 @@ HTSEXT_API int catch_url(T_SOC soc, char *url, char *method, char *data) {
             htsblk blkretour;
 
             hts_init_htsblk(&blkretour);
-            //memset(&blkretour, 0, sizeof(htsblk));    // effacer
             blkretour.location = loc;   // si non nul, contiendra l'adresse véritable en cas de moved xx
             // Lire en têtes restants
             sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
@@ -204,8 +206,8 @@ HTSEXT_API int catch_url(T_SOC soc, char *url, char *method, char *data) {
               strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
               strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
             }
-            // CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
+            // CR/LF final de l'en tête inutile car déja placé via la ligne vide
-            //strcatbuff(data,"\r\n");
+            // juste au dessus
             if (blkretour.totalsize > 0) {
               int len = (int) min(blkretour.totalsize, 32000);
               int pos = (int) strlen(data);

src/htscharset.c

@@ -841,71 +841,6 @@ static unsigned int nlz8(unsigned char x) {
     }                                           \
   } while(0)
 
-/* Sample. */
-#if 0
-
-int main(int argc, char **argv) {
-  int i;
-  int hex = 0;
-  
-#define READ_INT(DEST)                                  \
-  ( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1)      \
-    || (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
-  
-  for(i = 1 ; i < argc ; i++) {
-    unsigned int uc, from, to;
-    
-    if (strcmp(argv[i], "--hex") == 0) {
-      hex = 1;
-    }
-    else if (strcmp(argv[i], "--decimal") == 0) {
-      hex = 0;
-    }
-    else if (strcmp(argv[i], "--decode") == 0) {
-#define RD fgetc_unlocked(stdin)
-#define WR(C) do {                              \
-        if (C != -1) {                          \
-          printf("%04x\n", C);                  \
-        } else if (!feof(stdin)) {              \
-          fprintf(stderr, "read error\n");      \
-          exit(EXIT_FAILURE);                   \
-        }                                       \
-      } while(0)
-      while(!feof(stdin)) {
-        READ_UNICODE(RD, WR);
-      }
-#undef RD
-#undef WR
-    }
-    else if (strcmp(argv[i], "-range") == 0
-             && i + 2 < argc
-             && (++i, 1)
-             && READ_INT(from)
-             && (++i, 1)
-             && READ_INT(to)
-             ) {
-      unsigned int i;
-      for(i = from ; i < to ; i++) {
-#define EM(C) fputc_unlocked(C, stdout)
-        EMIT_UNICODE(i, EM);
-#undef EM
-      }
-    }
-    else if (READ_INT(uc)) {
-#define EM(C) fputc_unlocked(C, stdout)
-      EMIT_UNICODE(uc, EM);
-#undef EM
-    }
-    else {
-      return EXIT_FAILURE;
-    }
-  }
-  
-  return EXIT_SUCCESS;
-}
-
-#endif
-
 /* IDNA helpers. */
 #undef ADD_BYTE
 #undef INCREASE_CAPA
@@ -975,19 +910,19 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
                 /* Reader: can read bytes up to j */
 #define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
 
-                /* Writer: upon error, return FFFD (replacement character) */
+                /* Writer: a malformed sequence abandons the encode (NULL) */
-#define WR(C) do { \
+#define WR(C)                                                                  \
-  if ((C) != -1) { \
+  do {                                                                         \
-    /* copy character */ \
+    if ((C) != -1) {                                                           \
-    assertf(segOutputSize < segSize); \
+      /* copy character */                                                     \
-    segInt[segOutputSize++] = (C); \
+      assertf(segOutputSize < segSize);                                        \
-  } \
+      segInt[segOutputSize++] = (C);                                           \
-  /* In case of error, abort. */ \
+    } else {                                                                   \
-  else { \
+      freet(segInt);                                                           \
-    FREE_BUFFER(); \
+      FREE_BUFFER();                                                           \
-    return NULL; \
+      return NULL;                                                             \
-  } \
+    }                                                                          \
-} while(0)
+  } while (0)
 
                 /* Read/Write Unicode character. */
                 READ_UNICODE(RD, WR);
@@ -1209,7 +1144,7 @@ int hts_isStringUTF8(const char *s, size_t size) {
     /* Reader: can read bytes up to j */
 #define RD ( i < size ? data[i++] : -1 )
 
-    /* Writer: upon error, return FFFD (replacement character) */
+    /* Writer: a malformed sequence means the string is not UTF-8 (return 0) */
 #define WR(C) if ((C) == -1) { return 0; }
 
     /* Read Unicode character. */

src/htsconfig.h

@@ -52,12 +52,8 @@ Please visit our Website: http://www.httrack.com
 #define DEFAULT_FTP "index.txt"
 
 // extension par défaut pour fichiers n'en ayant pas
-#define DEFAULT_EXT       ".html"
+#define DEFAULT_EXT ".html"
 #define DEFAULT_EXT_SHORT ".htm"
-//#define DEFAULT_BIN_EXT   ".bin"
-//#define DEFAULT_BIN_EXT_SHORT ".bin"
-//#define DEFAULT_EXT       ".txt"
-//#define DEFAULT_EXT_SHORT ".txt"
 
 // éviter les /nul, /con..
 #define HTS_OVERRIDE_DOS_FOLDERS 1
@@ -80,14 +76,11 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
-// gérer une table de hachage?
-// REMOVED
-// #define HTS_HASH 1
-
 // fast cache (build hash table)
 #define HTS_FAST_CACHE 1
 
-// le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est valide)
+// le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
+// valide)
 #define GT_ENDS_COMMENT 1
 
 // always adds a '/' at the end if a '~' is encountered (/~smith -> /~smith/)
@@ -96,26 +89,10 @@ Please visit our Website: http://www.httrack.com
 // always transform a '//' into a sigle '/'
 #define HTS_STRIP_DOUBLE_SLASH 0
 
-// case-sensitive pour les dossiers et fichiers (0/1)
-// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est pas très utile..
-// ..et pas bcp respecté]
-// REMOVED
-// #define HTS_CASSE 0
-
-// Un fichier ayant une taille différente du content-length doit il être annulé?
-// SEE opt.tolerant and opt.http10
-// #define HTS_CL_IS_FATAL 0
-
 // une erreur supprime le fichier sur disque
 // (non fixé pour cause de retry)
 #define HTS_REMOVE_BAD_FILES 0
 
-// en cas de Range: xx- donnant un Content-length: xx
-// alors skipper le fichier, considéré comme transmis
-// #define HTS_SKIP_FULL_RANGE 1
-
-// nombre max de filtres que l'utilisateur peut fixer
-// #define HTS_FILTERSMAX 10000
 #define HTS_FILTERSINC 1000
 
 // connect non bloquant? (poll sur write)

src/htscore.h

@@ -152,6 +152,15 @@ struct lien_adrfilsave {
   char save[HTS_URLMAXSIZE * 2]; /**< local save path (with directory) */
 };
 
+/** Per-slot connect-fallback bookkeeping (parallel to struct_back.lnk).
+    Tracks which resolved address the slot is currently connecting to so a
+    stuck connect can be retried against the next one. */
+typedef struct hts_connect_fallback {
+  int addr_index;       /**< candidate being connected (0-based) */
+  int addr_count;       /**< resolved addresses; -1 = not yet probed */
+  TStamp connect_start; /**< when the current candidate's connect began */
+} hts_connect_fallback;
+
 /** The download-slot ring: the set of concurrent transfers in flight.
     Allocated/owned by the engine; consumers (status callbacks, the loop)
     read it but do not resize or free it. */
@@ -168,6 +177,7 @@ struct struct_back {
   int count;              /**< number of usable slots (back_max) */
   coucal ready;           /**< index of slots whose transfer completed */
   LLint ready_size_bytes; /**< total bytes buffered in completed slots */
+  hts_connect_fallback *connect_fallback; /**< per-slot, count+1 entries */
 };
 
 typedef struct cache_back_zip_entry cache_back_zip_entry;
@@ -204,6 +214,9 @@ struct cache_back {
   cache_back_zip_entry *zipEntries;
   int zipEntriesOffs;
   int zipEntriesCapa;
+  hts_boolean
+      zipWriteFailed; /**< a cache write failed; stop touching the stream */
+  int zipWriteFailures; /**< consecutive entry write failures; reset on store */
 };
 
 #ifndef HTS_DEF_FWSTRUCT_hash_struct
@@ -222,8 +235,12 @@ struct hash_struct {
   coucal adrfil;
   /* former address+path -> link index (renamed/moved entries) */
   coucal former_adrfil;
-  /* scratch buffers reused across lookups (not reentrant) */
+  /* effective urlhack sub-flags: www.==host / // collapse / query-arg sort */
-  int normalized;
+  hts_boolean norm_host;
+  hts_boolean norm_slash;
+  hts_boolean norm_query;
+  /* query-strip keys (not owned); set from opt->strip_query at hash_init */
+  const char *strip_query;
   char normfil[HTS_URLMAXSIZE * 2];
   char normfil2[HTS_URLMAXSIZE * 2];
   char catbuff[CATBUFF_SIZE];
@@ -346,12 +363,42 @@ void usercommand(httrackp * opt, int exe, const char *cmd, const char *file,
 
 void usercommand_exe(const char *cmd, const char *file);
 
+// Finish the makeindex index.html (footer + refresh meta), run usercommand.
+// Updates *makeindex_done/*makeindex_fp in place; adr/fil are the mode strings.
+void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
+                          FILE **makeindex_fp, int makeindex_links,
+                          const char *makeindex_firstlink,
+                          const char *template_footer, const char *adr,
+                          const char *fil);
+
+// Flush ht_buff[0..ht_len] to save on disk (skip if MD5 unchanged); *fp
+// closed+NULLed on write. Precondition: ht_len>0.
+void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
+                          FILE **fp, const char *ht_buff, size_t ht_len,
+                          const char *adr, const char *fil, const char *save);
+
 int filters_init(char ***ptrfilters, int maxfilter, int filterinc);
 
 int fspc(httrackp * opt, FILE * fp, const char *type);
 
 char *next_token(char *p, int flag);
 
+/* Like fil_normalized(), but first drops query keys in STRIP (comma-separated,
+   "*" = all); STRIP NULL/empty behaves exactly like fil_normalized(). */
+char *fil_normalized_filtered(const char *source, char *dest,
+                              const char *strip);
+
+/* As fil_normalized_filtered(), but DO_SLASH/DO_QUERY gate the // collapse and
+   the query-argument sort independently (the urlhack sub-flags). */
+char *fil_normalized_filtered_ex(const char *source, char *dest,
+                                 const char *strip, int do_slash, int do_query);
+
+/* For URL ADR/FIL, return (in DEST) the comma keylist to strip from the
+   '\n'-separated "[pattern=]keys" RULES (patterns matched on host/path via
+   strjoker, last wins); NULL if none match. Feeds fil_normalized_filtered(). */
+const char *hts_query_strip_keys(const char *rules, const char *adr,
+                                 const char *fil, char *dest, size_t destsize);
+
 /* Read a whole file into a freshly malloc'd, NUL-terminated buffer; the caller
    owns it and must release it with freet(). Return NULL on missing/unreadable
    file (readfile_or substitutes defaultdata instead). The byte content is NOT
@@ -365,17 +412,16 @@ char *readfile_utf8(const char *fil);
 
 char *readfile_or(const char *fil, const char *defaultdata);
 
-#if 0
-void check_rate(TStamp stat_timestart, int maxrate);
-#endif
-
-// links
-int liens_record(char *adr, char *fil, char *save, char *former_adr,
-                 char *former_fil, char *codebase);
-
 /* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
    Not thread-safe; call from the single crawl loop. */
 
+/* True if a connecting slot should give up on the current address and try the
+   next one: a fallback address remains (addr_index+1 < addr_count) and the
+   candidate has been connecting for at least its deadline, min(timeout, an
+   internal cap). elapsed/timeout in seconds. Exposed for the -#D self-test. */
+int back_connect_fallback_due(int addr_index, int addr_count, int elapsed,
+                              int timeout);
+
 /* How many new sockets may be opened now, honoring maxsoc and the maxconn rate
    limit (>=0). _strict ignores reserved-slot headroom; the plain form leaves
    room for naming tests and stops at 0 when the stack is nearly full. */
@@ -383,6 +429,19 @@ int back_pluggable_sockets(struct_back * sback, httrackp * opt);
 
 int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);
 
+/* One engine-loop tick: refresh the transfer stats and run the loop callback
+   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
+hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr);
+
+/* Wait until a test socket can be plugged, pumping transfers, stats and the
+   loop callback; gives up past the -E deadline. HTS_FALSE = callback abort. */
+hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
+                                      cache_back *cache, int ptr);
+
+/* Randomized inter-file pause target in [min_ms,max_ms] (#185), derived from a
+   timestamp seed so it is stable within one gap and rerolls per launch. */
+int hts_pause_target_ms(TStamp seed, int min_ms, int max_ms);
+
 /* Schedule more links from the heap into free slots. Returns the number queued,
    or <=0 if none could be added (no free slot / paused / stopped). */
 int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
@@ -404,11 +463,6 @@ int ask_continue(httrackp * opt);
 /* Number of decimal digits in n. */
 int nombre_digit(int n);
 
-// Java
-#if 0
-int hts_add_file(char *file, int file_position);
-#endif
-
 // Polling
 #if HTS_POLL
 int check_flot(T_SOC s);
@@ -431,4 +485,8 @@ void voidf(void);
 /* HTML marker comment marking where the top index is spliced. */
 #define HTS_TOPINDEX "TOP_INDEX_HTTRACK"
 
+/* Worst-case byte expansion HT_ADD_HTMLESCAPED* must reserve per escaper. */
+#define HTS_HTMLESCAPE_MAXEXP 5      /* escape_for_html_print: '&'->"&amp;" */
+#define HTS_HTMLESCAPE_FULL_MAXEXP 6 /* _full: high byte->"&#xHH;" */
+
 #endif

src/htsdefines.h

@@ -69,11 +69,15 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
 typedef struct t_hts_callbackarg t_hts_callbackarg;
 #endif
 
-/* Marks a symbol an external wrapper module exports back to the engine
+/* Marks a symbol an external wrapper module exports back to the engine.
-   (dllexport on Windows, nothing elsewhere). */
+   Must override -fvisibility=hidden on ELF, or dlopen()ed plugins (htsjava)
+   hide their own hts_plug()/hts_unplug() entry points. */
 #ifndef EXTERNAL_FUNCTION
 #ifdef _WIN32
 #define EXTERNAL_FUNCTION __declspec(dllexport)
+#elif ((defined(__GNUC__) && (__GNUC__ >= 4)) ||                               \
+       (defined(HAVE_VISIBILITY) && HAVE_VISIBILITY))
+#define EXTERNAL_FUNCTION __attribute__((visibility("default")))
 #else
 #define EXTERNAL_FUNCTION
 #endif
@@ -82,9 +86,9 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
 /* Entry points of a --wrapper plug-in: hts_plug(opt, argv) is called once to
    install the wrapper (argv is the wrapper's argument string), hts_unplug(opt)
    once to tear it down. Both return non-zero on success. */
-typedef int (*t_hts_plug) (httrackp * opt, const char *argv);
+typedef int (*t_hts_plug)(httrackp *opt, const char *argv);
 
-typedef int (*t_hts_unplug) (httrackp * opt);
+typedef int (*t_hts_unplug)(httrackp *opt);
 
 /* Engine callback prototypes. Each is one hook the engine fires at a defined
    point of a mirror; a wrapper installs the ones it cares about in the
@@ -92,27 +96,27 @@ typedef int (*t_hts_unplug) (httrackp * opt);
    returns are 1 to continue/accept, 0 to abort/refuse unless noted. */
 
 /* Called once when the wrapper is installed; allocate per-run state here. */
-typedef void (*t_hts_htmlcheck_init) (t_hts_callbackarg * carg);
+typedef void (*t_hts_htmlcheck_init)(t_hts_callbackarg *carg);
 
 /* Called once when the wrapper is removed; release per-run state here. */
-typedef void (*t_hts_htmlcheck_uninit) (t_hts_callbackarg * carg);
+typedef void (*t_hts_htmlcheck_uninit)(t_hts_callbackarg *carg);
 
 /* Fired at the start of a mirror, after options are parsed. */
-typedef int (*t_hts_htmlcheck_start) (t_hts_callbackarg * carg, httrackp * opt);
+typedef int (*t_hts_htmlcheck_start)(t_hts_callbackarg *carg, httrackp *opt);
 
 /* Fired at the end of a mirror. */
-typedef int (*t_hts_htmlcheck_end) (t_hts_callbackarg * carg, httrackp * opt);
+typedef int (*t_hts_htmlcheck_end)(t_hts_callbackarg *carg, httrackp *opt);
 
 /* Fired while options are being changed, to validate or adjust them. */
-typedef int (*t_hts_htmlcheck_chopt) (t_hts_callbackarg * carg, httrackp * opt);
+typedef int (*t_hts_htmlcheck_chopt)(t_hts_callbackarg *carg, httrackp *opt);
 
 /* Rewrite hook over an in-memory page: the html and len arguments point at the
    buffer and its length (the callback may reallocate and resize it),
    url_adresse and url_fichier name it. */
-typedef int (*t_hts_htmlcheck_process) (t_hts_callbackarg * carg,
+typedef int (*t_hts_htmlcheck_process)(t_hts_callbackarg *carg, httrackp *opt,
-                                        httrackp * opt, char **html, int *len,
+                                       char **html, int *len,
-                                        const char *url_adresse,
+                                       const char *url_adresse,
-                                        const char *url_fichier);
+                                       const char *url_fichier);
 
 /* Same shape as process, run before HTML parsing. */
 typedef t_hts_htmlcheck_process t_hts_htmlcheck_preprocess;
@@ -121,113 +125,111 @@ typedef t_hts_htmlcheck_process t_hts_htmlcheck_preprocess;
 typedef t_hts_htmlcheck_process t_hts_htmlcheck_postprocess;
 
 /* Inspect a page (read-only html/len) without rewriting it. */
-typedef int (*t_hts_htmlcheck_check_html) (t_hts_callbackarg * carg,
+typedef int (*t_hts_htmlcheck_check_html)(t_hts_callbackarg *carg,
-                                           httrackp * opt, char *html, int len,
+                                          httrackp *opt, char *html, int len,
-                                           const char *url_adresse,
+                                          const char *url_adresse,
-                                           const char *url_fichier);
+                                          const char *url_fichier);
 
 /* Answer an engine query identified by 'question'; returns the answer string
    (owned by the callback, must stay valid until the next call). */
-typedef const char *(*t_hts_htmlcheck_query) (t_hts_callbackarg * carg,
+typedef const char *(*t_hts_htmlcheck_query)(t_hts_callbackarg *carg,
-                                              httrackp * opt,
+                                             httrackp *opt,
-                                              const char *question);
+                                             const char *question);
 
 /* Second query channel, same contract as query. */
-typedef const char *(*t_hts_htmlcheck_query2) (t_hts_callbackarg * carg,
+typedef const char *(*t_hts_htmlcheck_query2)(t_hts_callbackarg *carg,
-                                               httrackp * opt,
+                                              httrackp *opt,
-                                               const char *question);
+                                              const char *question);
 
 /* Third query channel, same contract as query. */
-typedef const char *(*t_hts_htmlcheck_query3) (t_hts_callbackarg * carg,
+typedef const char *(*t_hts_htmlcheck_query3)(t_hts_callbackarg *carg,
-                                               httrackp * opt,
+                                              httrackp *opt,
-                                               const char *question);
+                                              const char *question);
 
 /* Per-tick progress hook: 'back' is the transfer slot array of 'back_max'
    entries, back_index the active one; lien_tot/lien_ntot and stats report
    queue size and running totals, stat_time the elapsed time. */
-typedef int (*t_hts_htmlcheck_loop) (t_hts_callbackarg * carg, httrackp * opt,
+typedef int (*t_hts_htmlcheck_loop)(t_hts_callbackarg *carg, httrackp *opt,
-                                     lien_back * back, int back_max,
+                                    lien_back *back, int back_max,
-                                     int back_index, int lien_tot,
+                                    int back_index, int lien_tot, int lien_ntot,
-                                     int lien_ntot, int stat_time,
+                                    int stat_time, hts_stat_struct *stats);
-                                     hts_stat_struct * stats);
 
 /* Veto a link (adr host, fil path) after its transfer; status is the result.
    Return 0 to drop the link. */
-typedef int (*t_hts_htmlcheck_check_link) (t_hts_callbackarg * carg,
+typedef int (*t_hts_htmlcheck_check_link)(t_hts_callbackarg *carg,
-                                           httrackp * opt, const char *adr,
+                                          httrackp *opt, const char *adr,
-                                           const char *fil, int status);
+                                          const char *fil, int status);
 
 /* Veto a link by its MIME type before download; return 0 to skip it. */
-typedef int (*t_hts_htmlcheck_check_mime) (t_hts_callbackarg * carg,
+typedef int (*t_hts_htmlcheck_check_mime)(t_hts_callbackarg *carg,
-                                           httrackp * opt, const char *adr,
+                                          httrackp *opt, const char *adr,
-                                           const char *fil, const char *mime,
+                                          const char *fil, const char *mime,
-                                           int status);
+                                          int status);
 
 /* Fired when the mirror pauses, waiting on 'lockfile' to be removed. */
-typedef void (*t_hts_htmlcheck_pause) (t_hts_callbackarg * carg, httrackp * opt,
+typedef void (*t_hts_htmlcheck_pause)(t_hts_callbackarg *carg, httrackp *opt,
-                                       const char *lockfile);
+                                      const char *lockfile);
 
 /* Fired after a file is written to disk; 'file' is the local path. */
-typedef void (*t_hts_htmlcheck_filesave) (t_hts_callbackarg * carg,
+typedef void (*t_hts_htmlcheck_filesave)(t_hts_callbackarg *carg, httrackp *opt,
-                                          httrackp * opt, const char *file);
+                                         const char *file);
 
 /* Richer file-saved notification: source host/filename, local path, and flags
    telling whether the file is new, modified, or left unchanged. */
-typedef void (*t_hts_htmlcheck_filesave2) (t_hts_callbackarg * carg,
+typedef void (*t_hts_htmlcheck_filesave2)(t_hts_callbackarg *carg,
-                                           httrackp * opt, const char *hostname,
+                                          httrackp *opt, const char *hostname,
-                                           const char *filename,
+                                          const char *filename,
-                                           const char *localfile, int is_new,
+                                          const char *localfile, int is_new,
-                                           int is_modified, int not_updated);
+                                          int is_modified, int not_updated);
 
 /* Fired for each link parsed out of a page; 'link' may be edited in place. */
-typedef int (*t_hts_htmlcheck_linkdetected) (t_hts_callbackarg * carg,
+typedef int (*t_hts_htmlcheck_linkdetected)(t_hts_callbackarg *carg,
-                                             httrackp * opt, char *link);
+                                            httrackp *opt, char *link);
 
 /* As linkdetected, plus tag_start, the markup the link was found in. */
-typedef int (*t_hts_htmlcheck_linkdetected2) (t_hts_callbackarg * carg,
+typedef int (*t_hts_htmlcheck_linkdetected2)(t_hts_callbackarg *carg,
-                                              httrackp * opt, char *link,
+                                             httrackp *opt, char *link,
-                                              const char *tag_start);
+                                             const char *tag_start);
 
 /* Fired on each transfer-status change of slot 'back'. */
-typedef int (*t_hts_htmlcheck_xfrstatus) (t_hts_callbackarg * carg,
+typedef int (*t_hts_htmlcheck_xfrstatus)(t_hts_callbackarg *carg, httrackp *opt,
-                                          httrackp * opt, lien_back * back);
+                                         lien_back *back);
 
 /* Choose the local save path for a URL; write it into 'save'. adr/fil name the
    target, referer_adr/referer_fil the page that linked it. */
-typedef int (*t_hts_htmlcheck_savename) (t_hts_callbackarg * carg,
+typedef int (*t_hts_htmlcheck_savename)(t_hts_callbackarg *carg, httrackp *opt,
-                                         httrackp * opt,
+                                        const char *adr_complete,
-                                         const char *adr_complete,
+                                        const char *fil_complete,
-                                         const char *fil_complete,
+                                        const char *referer_adr,
-                                         const char *referer_adr,
+                                        const char *referer_fil, char *save);
-                                         const char *referer_fil, char *save);
 
 /* Extended save-name hook, same signature as savename. */
 typedef t_hts_htmlcheck_savename t_hts_htmlcheck_extsavename;
 
 /* Inspect or edit the outgoing request headers in 'buff' before they are sent.
  */
-typedef int (*t_hts_htmlcheck_sendhead) (t_hts_callbackarg * carg,
+typedef int (*t_hts_htmlcheck_sendhead)(t_hts_callbackarg *carg, httrackp *opt,
-                                         httrackp * opt, char *buff,
+                                        char *buff, const char *adr,
-                                         const char *adr, const char *fil,
+                                        const char *fil,
-                                         const char *referer_adr,
+                                        const char *referer_adr,
-                                         const char *referer_fil,
+                                        const char *referer_fil,
-                                         htsblk * outgoing);
+                                        htsblk *outgoing);
 
 /* Inspect the incoming response headers in 'buff' after they are received. */
-typedef int (*t_hts_htmlcheck_receivehead) (t_hts_callbackarg * carg,
+typedef int (*t_hts_htmlcheck_receivehead)(t_hts_callbackarg *carg,
-                                            httrackp * opt, char *buff,
+                                           httrackp *opt, char *buff,
-                                            const char *adr, const char *fil,
+                                           const char *adr, const char *fil,
-                                            const char *referer_adr,
+                                           const char *referer_adr,
-                                            const char *referer_fil,
+                                           const char *referer_fil,
-                                            htsblk * incoming);
+                                           htsblk *incoming);
 
 /* External parser module hooks: detect claims a document type (return 1 to
    take it), parse then extracts its links. 'str' carries the document. */
-typedef int (*t_hts_htmlcheck_detect) (t_hts_callbackarg * carg, httrackp * opt,
+typedef int (*t_hts_htmlcheck_detect)(t_hts_callbackarg *carg, httrackp *opt,
-                                       htsmoduleStruct * str);
+                                      htsmoduleStruct *str);
 
-typedef int (*t_hts_htmlcheck_parse) (t_hts_callbackarg * carg, httrackp * opt,
+typedef int (*t_hts_htmlcheck_parse)(t_hts_callbackarg *carg, httrackp *opt,
-                                      htsmoduleStruct * str);
+                                     htsmoduleStruct *str);
 
 /* Callbacks */
 #ifndef HTS_DEF_FWSTRUCT_t_hts_htmlcheck_callbacks
@@ -237,10 +239,10 @@ typedef struct t_hts_htmlcheck_callbacks t_hts_htmlcheck_callbacks;
 
 /* Declares one named callback slot: its function pointer (typed
    t_hts_htmlcheck_<NAME>) paired with the carg passed to it. */
-#define DEFCALLBACK(NAME)         \
+#define DEFCALLBACK(NAME)                                                      \
-  struct NAME {                   \
+  struct NAME {                                                                \
-    t_hts_htmlcheck_ ##NAME fun;  \
+    t_hts_htmlcheck_##NAME fun;                                                \
-    t_hts_callbackarg *carg;      \
+    t_hts_callbackarg *carg;                                                   \
   } NAME
 
 /* Generic, type-erased callback slot used where the hook type is opaque. */
@@ -324,18 +326,18 @@ extern const t_hts_htmlcheck_callbacks default_callbacks;
 /* Internal helpers for building an HTTP request/response into the engine's
    scratch buffer (opt->state.HTbuff): START resets it, PRINT appends; the
    PANIC variant records a fatal error message. */
-#define HT_PRINT(A) strcatbuff(opt->state.HTbuff,A);
+#define HT_PRINT(A) strcatbuff(opt->state.HTbuff, A);
 
-#define HT_REQUEST_START opt->state.HTbuff[0]='\0';
+#define HT_REQUEST_START opt->state.HTbuff[0] = '\0';
 
 #define HT_REQUEST_END
-#define HTT_REQUEST_START opt->state.HTbuff[0]='\0';
+#define HTT_REQUEST_START opt->state.HTbuff[0] = '\0';
 
 #define HTT_REQUEST_END
-#define HTS_REQUEST_START opt->state.HTbuff[0]='\0';
+#define HTS_REQUEST_START opt->state.HTbuff[0] = '\0';
 
 #define HTS_REQUEST_END
-#define HTS_PANIC_PRINTF(S) strcpybuff(opt->state._hts_errmsg,S);
+#define HTS_PANIC_PRINTF(S) strcpybuff(opt->state._hts_errmsg, S);
 
 #endif
 

src/htsdns_selftest.c

@@ -0,0 +1,359 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: htsdns_selftest.c subroutines:                         */
+/*       in-process self-test for the DNS resolver and cache    */
+/* Author: Xavier Roche                                         */
+/* ------------------------------------------------------------ */
+
+/* Routes the resolver through a scripted getaddrinfo (hts_resolver_backend)
+   instead of the network, so resolution and the DNS cache are testable for a
+   fixed set of scenarios (IPv4/IPv6/dual-stack, errors, family filter,
+   cache reuse) with no live DNS. */
+
+#define HTS_INTERNAL_BYTECODE
+
+#include "htsdns_selftest.h"
+
+#include "htscore.h"
+#include "htslib.h"
+#include "htsnet.h"
+
+#include <stdio.h>
+#include <string.h>
+
+#if HTS_INET6 != 0
+
+/* IPV6_resolver: 0 = v4+v6, 1 = v4 only, 2 = v6 only (htscoremain -@i). */
+extern int IPV6_resolver;
+
+/* One scripted host: either a getaddrinfo error, or an ordered address list. */
+typedef struct mock_addr {
+  int family;             /* AF_INET / AF_INET6 */
+  unsigned char addr[16]; /* 4 (v4) or 16 (v6) meaningful bytes */
+} mock_addr;
+
+typedef struct mock_host {
+  const char *name;
+  int gai_err; /* non-zero: getaddrinfo returns this */
+  int naddr;
+  mock_addr addr[6];
+  int calls; /* times the backend resolved this host */
+} mock_host;
+
+static mock_host mock_hosts[] = {
+    {"v4only.test", 0, 1, {{AF_INET, {1, 2, 3, 4}}}, 0},
+    {"v6only.test", 0, 1, {{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 1}}}, 0},
+    /* dual stack, IPv6 first (RFC 6724 order) then IPv4 */
+    {"dual.test",
+     0,
+     2,
+     {{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 2}}, {AF_INET, {5, 6, 7, 8}}},
+     0},
+    /* dual stack, IPv4 first: distinguishes "keep the first address" from
+       "prefer a family", so the selection contract is actually pinned. */
+    {"dual4.test",
+     0,
+     2,
+     {{AF_INET, {9, 10, 11, 12}},
+      {AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 3}}},
+     0},
+    /* more addresses than HTS_MAXADDRNUM: the list must clamp to the cap. */
+    {"many.test",
+     0,
+     6,
+     {{AF_INET, {10, 0, 0, 1}},
+      {AF_INET, {10, 0, 0, 2}},
+      {AF_INET, {10, 0, 0, 3}},
+      {AF_INET, {10, 0, 0, 4}},
+      {AF_INET, {10, 0, 0, 5}},
+      {AF_INET, {10, 0, 0, 6}}},
+     0},
+    {"nodns.test", EAI_NONAME, 0, {{0}}, 0},
+};
+
+static mock_host *mock_find(const char *name) {
+  for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++) {
+    if (strcmp(mock_hosts[i].name, name) == 0)
+      return &mock_hosts[i];
+  }
+  return NULL;
+}
+
+static void mock_reset_calls(void) {
+  for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++)
+    mock_hosts[i].calls = 0;
+}
+
+/* Build one addrinfo node owning its sockaddr (freed by mock_freeaddrinfo). */
+static struct addrinfo *mock_mkai(const mock_addr *a) {
+  struct addrinfo *ai = calloct(1, sizeof(*ai));
+
+  ai->ai_family = a->family;
+  if (a->family == AF_INET) {
+    struct sockaddr_in *sin = calloct(1, sizeof(*sin));
+
+    sin->sin_family = AF_INET;
+    memcpy(&sin->sin_addr, a->addr, 4);
+    ai->ai_addr = (struct sockaddr *) sin;
+    ai->ai_addrlen = sizeof(*sin);
+  } else {
+    struct sockaddr_in6 *sin6 = calloct(1, sizeof(*sin6));
+
+    sin6->sin6_family = AF_INET6;
+    memcpy(&sin6->sin6_addr, a->addr, 16);
+    ai->ai_addr = (struct sockaddr *) sin6;
+    ai->ai_addrlen = sizeof(*sin6);
+  }
+  return ai;
+}
+
+static int mock_getaddrinfo(const char *node, const char *service,
+                            const struct addrinfo *hints,
+                            struct addrinfo **res) {
+  mock_host *const h = mock_find(node);
+  const int want = (hints != NULL) ? hints->ai_family : PF_UNSPEC;
+  struct addrinfo *head = NULL, *tail = NULL;
+
+  (void) service;
+  *res = NULL;
+  if (h == NULL)
+    return EAI_NONAME;
+  h->calls++; /* a real backend hit; a cached host skips this */
+  if (h->gai_err != 0)
+    return h->gai_err;
+  for (int i = 0; i < h->naddr; i++) {
+    if (want != PF_UNSPEC && want != h->addr[i].family)
+      continue; /* honor the requested family (v4/v6 only) */
+    struct addrinfo *const ai = mock_mkai(&h->addr[i]);
+
+    if (head == NULL)
+      head = ai;
+    else
+      tail->ai_next = ai;
+    tail = ai;
+  }
+  if (head == NULL)
+    return EAI_NONAME; /* filtered to empty, as the libc resolver does */
+  *res = head;
+  return 0;
+}
+
+static void mock_freeaddrinfo(struct addrinfo *res) {
+  while (res != NULL) {
+    struct addrinfo *const next = res->ai_next;
+
+    freet(res->ai_addr);
+    freet(res);
+    res = next;
+  }
+}
+
+static const hts_resolver_backend mock_backend = {mock_getaddrinfo,
+                                                  mock_freeaddrinfo};
+
+static int failures = 0;
+
+#define CHECK(cond)                                                            \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      failures++;                                                              \
+      fprintf(stderr, "dns-selftest: FAIL at %s:%d: %s\n", __FILE__, __LINE__, \
+              #cond);                                                          \
+    }                                                                          \
+  } while (0)
+
+/* Resolve via the uncached entry point; return the address family, or
+   AF_UNSPEC if the host did not resolve. */
+static int resolve_family_nocache(const char *host) {
+  SOCaddr addr;
+  const char *err = NULL;
+
+  if (hts_dns_resolve_nocache2(host, &addr, &err) == NULL)
+    return AF_UNSPEC;
+  return SOCaddr_sinfamily(addr);
+}
+
+int dns_selftests(httrackp *opt) {
+  failures = 0;
+  hts_dns_set_resolver_backend(&mock_backend);
+
+  /* IPv4-only / IPv6-only hosts map to the right family. */
+  IPV6_resolver = 0;
+  CHECK(resolve_family_nocache("v4only.test") == AF_INET);
+  CHECK(resolve_family_nocache("v6only.test") == AF_INET6);
+
+  /* Dual-stack: the single-address API returns the *first* resolved address.
+     Both orderings pin selection by position, not a family preference. The
+     multi-address API (resolve_all, below) exposes the whole list. */
+  CHECK(resolve_family_nocache("dual.test") == AF_INET6); /* v6 listed first */
+  CHECK(resolve_family_nocache("dual4.test") == AF_INET); /* v4 listed first */
+
+  /* Unknown host does not resolve. */
+  CHECK(resolve_family_nocache("nodns.test") == AF_UNSPEC);
+
+  /* Family filter (-@i4 / -@i6) selects v4 / v6 out of the dual-stack host. */
+  IPV6_resolver = 1;
+  CHECK(resolve_family_nocache("dual.test") == AF_INET);
+  IPV6_resolver = 2;
+  CHECK(resolve_family_nocache("dual.test") == AF_INET6);
+  IPV6_resolver = 0;
+
+  /* Cached driver resolves a host once and reuses the *same* address. */
+  mock_reset_calls();
+  {
+    SOCaddr a1, a2;
+    char ip1[64], ip2[64];
+    const char *err = NULL;
+
+    CHECK(hts_dns_resolve2(opt, "v4only.test", &a1, &err) != NULL);
+    CHECK(hts_dns_resolve2(opt, "v4only.test", &a2, &err) != NULL);
+    CHECK(mock_find("v4only.test")->calls == 1);
+    /* the cache returns the right address, not merely a hit for the key */
+    SOCaddr_inetntoa(ip1, sizeof(ip1), a1);
+    SOCaddr_inetntoa(ip2, sizeof(ip2), a2);
+    CHECK(strcmp(ip1, "1.2.3.4") == 0);
+    CHECK(strcmp(ip1, ip2) == 0);
+  }
+
+  /* A negative result is cached too: a second lookup does not re-resolve. */
+  {
+    SOCaddr a1, a2;
+    const char *err = NULL;
+
+    CHECK(hts_dns_resolve2(opt, "nodns.test", &a1, &err) == NULL);
+    CHECK(hts_dns_resolve2(opt, "nodns.test", &a2, &err) == NULL);
+    CHECK(mock_find("nodns.test")->calls == 1); /* resolved once, then cached */
+  }
+
+  /* Multi-address resolution: count and order are the connect-fallback
+     contract. A dead first address is retried against the next, so both must be
+     exact. */
+  mock_reset_calls();
+  {
+    SOCaddr addrs[HTS_MAXADDRNUM];
+    char ip[64];
+    const char *err = NULL;
+
+    /* dual-stack, in resolver order: [0]=v6, [1]=v4 */
+    CHECK(hts_dns_resolve_all(opt, "dual.test", addrs, HTS_MAXADDRNUM, &err) ==
+          2);
+    CHECK(SOCaddr_sinfamily(addrs[0]) == AF_INET6);
+    CHECK(SOCaddr_sinfamily(addrs[1]) == AF_INET);
+    SOCaddr_inetntoa(ip, sizeof(ip), addrs[1]);
+    CHECK(strcmp(ip, "5.6.7.8") == 0);
+    CHECK(mock_find("dual.test")->calls ==
+          1); /* one backend hit for the list */
+
+    /* single-address host: count 1 */
+    CHECK(hts_dns_resolve_all(opt, "v4only.test", addrs, HTS_MAXADDRNUM,
+                              &err) == 1);
+    SOCaddr_inetntoa(ip, sizeof(ip), addrs[0]);
+    CHECK(strcmp(ip, "1.2.3.4") == 0);
+
+    /* does-not-resolve: count 0 (negative), no addresses */
+    CHECK(hts_dns_resolve_all(opt, "nodns.test", addrs, HTS_MAXADDRNUM, &err) ==
+          0);
+
+    /* more than the cap: the kept list is clamped to HTS_MAXADDRNUM, keeping
+       the FIRST addresses in resolver order (not some other window) */
+    CHECK(hts_dns_resolve_all(opt, "many.test", addrs, HTS_MAXADDRNUM, &err) ==
+          HTS_MAXADDRNUM);
+    SOCaddr_inetntoa(ip, sizeof(ip), addrs[0]);
+    CHECK(strcmp(ip, "10.0.0.1") == 0);
+    SOCaddr_inetntoa(ip, sizeof(ip), addrs[HTS_MAXADDRNUM - 1]);
+    CHECK(strcmp(ip, "10.0.0.4") == 0);
+
+    /* family filter still applies through the list path */
+    IPV6_resolver = 1;
+    CHECK(hts_dns_resolve_all(opt, "dual4.test", addrs, HTS_MAXADDRNUM, &err) ==
+          1);
+    CHECK(SOCaddr_sinfamily(addrs[0]) == AF_INET);
+    IPV6_resolver = 0;
+  }
+
+  /* newhttp_addr() must connect to the addr_index-th address, not always the
+     first: this is what back_connect_next relies on to reach the fallback. */
+  {
+    htsblk r;
+    int count = -1;
+    T_SOC s;
+
+    hts_init_htsblk(&r);
+    s = newhttp_addr(opt, "dual.test", &r, 80, 0, 0, &count);
+    CHECK(count == 2);
+    CHECK(SOCaddr_sinfamily(r.address) == AF_INET6); /* index 0 = v6 */
+    if (s != INVALID_SOCKET)
+      deletesoc(s);
+
+    hts_init_htsblk(&r);
+    count = -1;
+    s = newhttp_addr(opt, "dual.test", &r, 80, 0, 1, &count);
+    CHECK(count == 2);
+    CHECK(SOCaddr_sinfamily(r.address) == AF_INET); /* index 1 = v4 */
+    if (s != INVALID_SOCKET)
+      deletesoc(s);
+
+    /* out-of-range index: no address selected (address stays unset) */
+    hts_init_htsblk(&r);
+    s = newhttp_addr(opt, "dual.test", &r, 80, 0, 2, NULL);
+    CHECK(s == INVALID_SOCKET);
+    if (s != INVALID_SOCKET)
+      deletesoc(s);
+  }
+
+  /* Connect-fallback decision (consumer of the multi-address list): when a
+     stuck connect should abandon the current address for the next one. */
+  {
+    /* no fallback for the last/only candidate, whatever the elapsed time */
+    CHECK(back_connect_fallback_due(0, 1, 9999, 120) == 0);
+    CHECK(back_connect_fallback_due(1, 2, 9999, 120) == 0);
+    CHECK(back_connect_fallback_due(3, 4, 9999, 120) == 0);
+    /* fallback available: wait the per-candidate deadline (cap 10s here) */
+    CHECK(back_connect_fallback_due(0, 2, 9, 120) == 0);
+    CHECK(back_connect_fallback_due(0, 2, 10, 120) == 1);
+    CHECK(back_connect_fallback_due(2, 4, 10, 120) == 1);
+    /* a shorter slot timeout shortens the deadline (min(timeout, cap)) */
+    CHECK(back_connect_fallback_due(0, 2, 4, 5) == 0);
+    CHECK(back_connect_fallback_due(0, 2, 5, 5) == 1);
+    /* no timeout management: never force a fallback */
+    CHECK(back_connect_fallback_due(0, 2, 9999, 0) == 0);
+  }
+
+  hts_dns_set_resolver_backend(NULL);
+  return failures;
+}
+
+#else
+
+int dns_selftests(httrackp *opt) {
+  (void) opt;
+  return 0; /* resolver seam only exists in the IPv6 build */
+}
+
+#endif

src/htsdns_selftest.h

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: htsdns_selftest.h                                      */
+/* Author: Xavier Roche                                         */
+/* ------------------------------------------------------------ */
+
+#ifndef HTSDNS_SELFTEST_DEFH
+#define HTSDNS_SELFTEST_DEFH
+
+#ifdef HTS_INTERNAL_BYTECODE
+
+#ifndef HTS_DEF_FWSTRUCT_httrackp
+#define HTS_DEF_FWSTRUCT_httrackp
+typedef struct httrackp httrackp;
+#endif
+
+/* Drive the DNS resolver and cache through a scripted (mock) getaddrinfo,
+   asserting address family, single-address selection, negative caching, the
+   IPv4/IPv6 family filter, and that a cached host is resolved only once.
+   Returns the number of failed checks (0 == success). */
+int dns_selftests(httrackp *opt);
+
+#endif
+
+#endif

src/htsencoding.c

@@ -30,12 +30,14 @@ Please visit our Website: http://www.httrack.com
 /* Author: Xavier Roche                                         */
 /* ------------------------------------------------------------ */
 
+#include <stdint.h>
+
 #include "htscharset.h"
 #include "htsencoding.h"
 #include "htssafe.h"
 
-/* static int decode_entity(const unsigned int hash, const size_t len);
+/* static int decode_entity(const uint64_t hash, const size_t len);
-*/
+ */
 #include "htsentities.h"
 
 /* hexadecimal conversion */
@@ -50,30 +52,31 @@ static int get_hex_value(char c) {
     return -1;
 }
 
-/* Numerical Recipes,
+/* 64-bit FNV-1a; must match htsentities.sh, which keys the entity table on it.
-   see <http://en.wikipedia.org/wiki/Linear_congruential_generator> */
+ */
-#define HASH_PRIME ( 1664525 )
+#define HASH_INIT 0xcbf29ce484222325ULL
-#define HASH_CONST ( 1013904223 )
+#define HASH_PRIME 0x100000001b3ULL
-#define HASH_ADD(HASH, C) do {                  \
+#define HASH_ADD(HASH, C)                                                      \
-    (HASH) *= HASH_PRIME;                       \
+  do {                                                                         \
-    (HASH) += HASH_CONST;                       \
+    (HASH) ^= (unsigned char) (C);                                             \
-    (HASH) += (C);                              \
+    (HASH) *= HASH_PRIME;                                                      \
-  } while(0)
+  } while (0)
 
 int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t max, const char *charset) {
   size_t i, j, ampStart, ampStartDest;
   int uc;
   int hex;
-  unsigned int hash;
+  uint64_t hash;
 
   assertf(max != 0);
-  for(i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0,
+  for (i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0, uc = -1, hex = 0,
-        uc = -1, hex = 0, hash = 0 ; src[i] != '\0' ; i++) {
+      hash = HASH_INIT;
+       src[i] != '\0'; i++) {
     /* start of entity */
     if (src[i] == '&') {
       ampStart = i;
       ampStartDest = j;
-      hash = 0;
+      hash = HASH_INIT;
       uc = -1;
     }
     /* inside a potential entity */
@@ -174,14 +177,11 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
       }
       /* alphanumerical entity */
       else {
-        /* alphanum and not too far ('&thetasym;' is the longest) */
+        /* alphanum, capped at the longest name
-        if (i <= ampStart + 10 &&
+         * '&CounterClockwiseContourIntegral;' (31) */
-            (
+        if (i <= ampStart + 31 && ((src[i] >= '0' && src[i] <= '9') ||
-             (src[i] >= '0' && src[i] <= '9')
+                                   (src[i] >= 'A' && src[i] <= 'Z') ||
-             || (src[i] >= 'A' && src[i] <= 'Z')
+                                   (src[i] >= 'a' && src[i] <= 'z'))) {
-             || (src[i] >= 'a' && src[i] <= 'z')
-             )
-            ) {
           /* compute hash */
           HASH_ADD(hash, (unsigned char) src[i]);
         } else {
@@ -190,9 +190,9 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
         }
       }
     }
-    
+
-    /* copy */
+    /* reserve one byte for the trailing NUL written after the loop */
-    if (j + 1 > max) {
+    if (j + 1 >= max) {
       /* overflow */
       return -1;
     }
@@ -300,6 +300,11 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
 
           /* Was the character read successfully ? */
           if (nRead == utfBufferSize) {
+            /* the 'continue' below skips the NUL-reserve guard: re-check */
+            if (utfBufferJ + utfBufferSize >= max) {
+              return -1;
+            }
+
             /* Rollback write position to sequence start write position */
             j = utfBufferJ;
 
@@ -314,8 +319,8 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
       }
     }
 
-    /* Check for overflow */
+    /* reserve one byte for the trailing NUL written after the loop */
-    if (j + 1 > max) {
+    if (j + 1 >= max) {
       return -1;
     }
 

src/htsentities.sh

@@ -1,75 +1,92 @@
 #!/bin/bash
 #
+# Regenerate htsentities.h from the WHATWG named character references.
 
-src=html40.txt
+set -euo pipefail
-url=http://www.w3.org/TR/1998/REC-html40-19980424/html40.txt
+
+src=entities.json
+url=https://html.spec.whatwg.org/entities.json
 dest=htsentities.h
 
-(
+# 64-bit FNV-1a of $1, printed as a C constant. Must match the hash in
-    cat <<EOF
+# htsencoding.c. The offset basis is stored as its wrapped (signed) bit pattern;
-/*
+# bash arithmetic is 64-bit two's complement, so the result is bit-exact.
-  -- ${dest} --
+fnv1a() {
-  FILE GENERATED BY $0, DO NOT MODIFY
+    local s=$1 i c h=$((0xcbf29ce484222325))
+    for ((i = 0; i < ${#s}; i++)); do
+        printf -v c '%d' "'${s:i:1}"
+        h=$(((h ^ (c & 0xff)) * 0x100000001b3))
+    done
+    printf '0x%016xULL' "$h"
+}
 
-  We compute the LCG hash
+if [ ! -f "$src" ]; then
-  (see <http://en.wikipedia.org/wiki/Linear_congruential_generator>)
+    curl -fsS "$url" -o "$src"
-  for each entity. We should in theory check using strncmp() that we
+fi
-  actually have the correct entity, but this is actually statistically
-  not needed.
 
-  We may want to do better, but we expect the hash function to be uniform, and
+# Keep ';'-terminated single-codepoint names; the ~93 multi-codepoint refs can't
-  let the compiler be smart enough to optimize the switch (for example by
+# fit decode_entity's single-codepoint return and are skipped (left verbatim).
-  checking in log2() intervals)
+pairs=$(jq -r '
-  
+    to_entries
-  This code has been generated using the evil $0 script.
+    | map(select((.key | endswith(";")) and (.value.codepoints | length == 1)))
-*/
+    | sort_by(.key)
+    | .[] | "\(.key | ltrimstr("&") | rtrimstr(";"))\t\(.value.codepoints[0])"' "$src")
 
-static int decode_entity(const unsigned int hash, const size_t len) {
+# Skipped multi-codepoint names, kept to prove none aliases an emitted hash.
+skipped=$(jq -r '
+    to_entries
+    | map(select((.key | endswith(";")) and (.value.codepoints | length > 1)))
+    | .[] | .key | ltrimstr("&") | rtrimstr(";")' "$src")
+
+cases=""
+emit_hashes=""
+while IFS=$'\t' read -r name cp; do
+    hash=$(fnv1a "$name")
+    cases+="    /* $name */"$'\n'
+    cases+="  case $hash:"$'\n'
+    cases+="    if (len == ${#name}) {"$'\n'
+    cases+="      return $cp;"$'\n'
+    cases+="    }"$'\n'
+    cases+="    break;"$'\n'
+    emit_hashes+="$hash"$'\n'
+done <<<"$pairs"
+
+skip_hashes=""
+while IFS= read -r name; do
+    [ -n "$name" ] && skip_hashes+="$(fnv1a "$name")"$'\n'
+done <<<"$skipped"
+
+# The switch keys on the hash alone, so the dispatch is correct only while every
+# emitted name hashes uniquely; prove it here, no runtime name compare needed.
+dups=$(printf '%s' "$emit_hashes" | sort | uniq -d || true)
+if [ -n "$dups" ]; then
+    echo "FATAL: two entity names share a hash (duplicate switch case); change the hash:" >&2
+    echo "$dups" >&2
+    exit 1
+fi
+# A skipped name colliding with an emitted hash would mis-decode instead of
+# staying verbatim; forbid that too.
+aliased=$(comm -12 <(printf '%s' "$emit_hashes" | sort -u) <(printf '%s' "$skip_hashes" | sort -u) || true)
+if [ -n "$aliased" ]; then
+    echo "FATAL: a skipped multi-codepoint name aliases an emitted hash:" >&2
+    echo "$aliased" >&2
+    exit 1
+fi
+
+cat >"$dest" <<EOF
+/* GENERATED by $0 from the WHATWG named character references
+   (${url}). DO NOT EDIT.
+   Dispatch keys on a 64-bit FNV-1a hash of the entity name; the generator
+   aborts on any hash collision, so no runtime name compare is needed. */
+
+#include <stdint.h>
+
+static int decode_entity(const uint64_t hash, const size_t len) {
   switch(hash) {
-EOF
+${cases}  }
-    (
-        if test -f ${src}; then
-            cat ${src}
-        else
-            GET "${url}"
-        fi
-    ) \
-        | grep -E '^<!ENTITY [a-zA-Z0-9_]' \
-        | sed \
-        -e 's/<!ENTITY //' -e "s/[[:space:]][[:space:]]*/ /g" \
-        -e 's/-->$//' \
-        -e 's/\([^ ]*\) CDATA "&#\([^\"]*\);" -- \(.*\)/\1 \2 \3/'\
-| ( \
-        read A
-        while test -n "$A"; do
-            ent="${A%% *}"
-            code=$(echo "$A"|cut -f2 -d' ')
-            # compute hash
-            hash=0
-            i=0
-            a=1664525
-            c=1013904223
-            m="$[1 << 32]"
-            while test "$i" -lt ${#ent}; do
-                d="$(echo -n "${ent:${i}:1}"|hexdump -v -e '/1 "%d"')"
-                hash="$[((${hash}*${a})%(${m})+${d}+${c})%(${m})]"
-                i=$[${i}+1]
-            done
-            echo -e "    /* $A */"
-            echo -e "  case ${hash}u:"
-            echo -e "    if (len == ${#ent} /* && strncmp(ent, \"${ent}\") == 0 */) {"
-            echo -e "      return ${code};"
-            echo -e "    }"
-            echo -e "    break;"
-
-            # next
-            read A
-        done
-    )
-    cat <<EOF
-  }
   /* unknown */
   return -1;
 }
 EOF
-) > ${dest}
+
+echo "wrote $dest ($(grep -c '^  case ' "$dest") entities)" >&2

src/htsfilters.c

@@ -76,7 +76,8 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
     }
     if (size)
       sz = *size;
-    if (strjoker(nom, filters[i] + filteroffs, &sz, size_flag)) {       // reconnu
+    /* size unknown (scan time): no size pointer => size tests stay neutral */
+    if (strjoker(nom, filters[i] + filteroffs, size ? &sz : NULL, size_flag)) {
       if (size)
         if (sz != *size)
           sizelimit = sz;
@@ -101,9 +102,8 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
 // cet algo est 'un peu' récursif mais ne consomme pas trop de tm
 // * = toute lettre
 // --?-- : spécifique à HTTrack et aux ?
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
-                          int *size_flag) {
+                                LLint *size, int *size_flag) {
-  //int err=0;
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -192,7 +192,12 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
           int len = (int) strlen(joker);
 
           while((joker[i] != RIGHT) && (joker[i]) && (i < len)) {
-            if ((joker[i] == '<') || (joker[i] == '>')) {       // *[<10]
+            // '\' escapes the next char as a literal member, e.g. *[\[\]]
+            if (joker[i] == '\\' && joker[i + 1] != '\0') {
+              i++;
+              pass[(int) (unsigned char) joker[i]] = 1;
+              i++;
+            } else if ((joker[i] == '<') || (joker[i] == '>')) { // *[<10]
               int lsize = 0;
               int lverdict;
 
@@ -220,7 +225,9 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
                 while(isdigit((unsigned char) joker[i]))
                   i++;
               }
-            } else if (joker[i + 1] == '-') {   // 2 car, ex: *[A-Z]
+            } else if (joker[i + 1] == '-' && joker[i + 2] != '\0') {
+              // range *[A-Z]; the '\0' guard rejects a truncated *[a- (else
+              // i+=3 overshoots the NUL)
               if ((int) (unsigned char) joker[i + 2] >
                   (int) (unsigned char) joker[i]) {
                 int j;
@@ -228,14 +235,9 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
                 for(j = (int) (unsigned char) joker[i];
                     j <= (int) (unsigned char) joker[i + 2]; j++)
                   pass[j] = 1;
-
               }
-              // else err=1;
               i += 3;
-            } else {            // 1 car, ex: *[ ]
+            } else { // 1 car, ex: *[ ]
-              if (joker[i + 2] == '\\' && joker[i + 3] != 0) {  // escaped char, such as *[\[] or *[\]]
-                i++;
-              }
               pass[(int) (unsigned char) joker[i]] = 1;
               i++;
             }
@@ -257,7 +259,6 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       for(i = 0; i < 256; i++)
         pass[i] = 1;            // tout autoriser
       jmp = 1;
-      ////if (joker[2]==LEFT) jmp=3;        // permet de recher *<crochet ouvrant>
     }
 
     {
@@ -285,7 +286,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       if (!unique)
         max = (int) strlen(chaine);
       else                      /* *(a) only match a (not aaaaa) */
-        max = 1;
+        max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
           if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {

src/htsftp.c

@@ -54,15 +54,9 @@ Please visit our Website: http://www.httrack.com
 #endif
 
 // ftp mode passif
-// #if HTS_INET6==0
 #define FTP_PASV 1
-// #else
-// no passive mode for v6
-// #define FTP_PASV 0
-// #endif
 
 #define FTP_DEBUG 0
-//#define FORK_DEBUG 0
 
 #if USE_BEGINTHREAD
 
@@ -128,6 +122,33 @@ void launch_ftp(FTPDownloadStruct * params) {
   return 0; \
   }
 
+/* Bounded split of a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
+void ftp_split_userpass(const char *src, const char *end, char *user,
+                        size_t user_size, char *pass, size_t pass_size) {
+  size_t n = 0;
+
+  assertf(user_size > 0 && pass_size > 0); /* the size-1 math underflows on 0 */
+
+  while (src[n] != '\0' && src[n] != ':') {
+    if (n < user_size - 1)
+      user[n] = src[n];
+    n++;
+  }
+  user[n < user_size ? n : user_size - 1] = '\0';
+  pass[0] = '\0';
+  if (src[n] == ':') { // password follows the colon
+    const size_t base = n + 1;
+    size_t k = 0;
+
+    while (&src[base + k + 1] < end && src[base + k] != '\0') {
+      if (k < pass_size - 1)
+        pass[k] = src[base + k];
+      k++;
+    }
+    pass[k < pass_size ? k : pass_size - 1] = '\0';
+  }
+}
+
 // la véritable fonction une fois lancées les routines thread/fork
 int run_launch_ftp(FTPDownloadStruct * pStruct) {
   lien_back *back = pStruct->pBack;
@@ -173,43 +194,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   while(*real_adr == '/')
     real_adr++;                 // sauter /
   if ((adr = jump_identification(real_adr)) != real_adr) {      // user
-    int i = -1;
+    ftp_split_userpass(real_adr, adr, user, sizeof(user), pass, sizeof(pass));
-
-    pass[0] = '\0';
-    do {
-      i++;
-      user[i] = real_adr[i];
-    } while((real_adr[i] != ':') && (real_adr[i]));
-    user[i] = '\0';
-    if (real_adr[i] == ':') {   // pass
-      int j = -1;
-
-      i++;                      // oui on saute aussi le :
-      do {
-        j++;
-        pass[j] = real_adr[i + j];
-      } while(((&real_adr[i + j + 1]) < adr) && (real_adr[i + j]));
-      pass[j] = '\0';
-    }
   }
   // Calculer RETR <nom>
   {
     char *a;
 
-#if 0
-    a = back->url_fil + strlen(back->url_fil) - 1;
-    while((a > back->url_fil) && (*a != '/'))
-      a--;
-    if (*a != '/') {
-      a = NULL;
-    }
-#else
     a = back->url_fil;
-#endif
     if (a != NULL && *a != '\0') {
-#if 0
-      a++;                      // sauter /
-#endif
       ftp_filename = a;
       if (strnotempty(a)) {
         char catbuff[CATBUFF_SIZE];
@@ -233,7 +225,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
       }
     } else {
       strcpybuff(back->r.msg, "Unexpected PORT error");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
     }
   }
@@ -267,7 +258,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
       snprintf(back->r.msg, sizeof(back->r.msg),
                "Unable to get server's address: %s", error);
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_NON_FATAL;
       _HALT_FTP return 0;
     }
@@ -280,20 +270,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
     if (soc_ctl == INVALID_SOCKET) {
       strcpybuff(back->r.msg, "Unable to create a socket");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
     }
 
     SOCaddr_initport(server, port);
-    // server.sin_port = htons((unsigned short int) port);
 
     // connexion (bloquante, on est en thread)
     strcpybuff(back->info, "connect");
 
     if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
       strcpybuff(back->r.msg, "Unable to connect to the server");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
 #ifdef _WIN32
@@ -335,64 +322,22 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               // ok
             } else {
               strcpybuff(back->r.msg, "TYPE I error");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
-#if 0
-            // --CWD--
-            char *a;
-
-            a = back->url_fil + strlen(back->url_fil) - 1;
-            while((a > back->url_fil) && (*a != '/'))
-              a--;
-            if (*a == '/') {    // ok repéré
-              char BIGSTK target[1024];
-
-              target[0] = '\0';
-              strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
-              if (strnotempty(target) == 0)
-                strcatbuff(target, "/");
-              strcpybuff(back->info, "cwd");
-              snprintf(line, sizeof(line), "CWD %s", target);
-              send_line(soc_ctl, line);
-              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-              _CHECK_HALT_FTP;
-              if (line[0] == '2') {
-                send_line(soc_ctl, "TYPE I");
-                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-                _CHECK_HALT_FTP;
-                if (line[0] == '2') {
-                  // ok..
-                } else {
-                  strcpybuff(back->r.msg, "TYPE I error");
-                  // back->status=STATUS_FTP_READY;    // fini
-                  back->r.statuscode = STATUSCODE_INVALID;
-                }
-              } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
-                // back->status=STATUS_FTP_READY;    // fini
-                back->r.statuscode = STATUSCODE_INVALID;
-              }                 // sinon on est prêts
-            } else {
-              strcpybuff(back->r.msg, "Unexpected ftp error");
-              // back->status=STATUS_FTP_READY;    // fini
-              back->r.statuscode = STATUSCODE_INVALID;
-            }
-#endif
 
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
+          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
-          // back->status=STATUS_FTP_READY;    // fini
+                   linejmp(line));
           back->r.statuscode = STATUSCODE_INVALID;
         }
       } else {
-        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
+        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
-        // back->status=STATUS_FTP_READY;    // fini
+                 linejmp(line));
         back->r.statuscode = STATUSCODE_INVALID;
       }
 
@@ -459,8 +404,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
             // -- fin analyse de l'adresse IP et du port --
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
@@ -491,13 +436,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 }
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
+              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
-              // back->status=STATUS_FTP_READY;    // fini
+                       linejmp(line));
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         }
@@ -603,9 +548,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   deletesoc(soc_dat);
                   soc_dat = INVALID_SOCKET;
                   //
-                  snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
+                  snprintf(back->r.msg, sizeof(back->r.msg),
-                          linejmp(line));
+                           "RETR command error: %s", linejmp(line));
-                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = STATUSCODE_INVALID;
                 }               // sinon on est prêts
               } else {
@@ -616,23 +560,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 soc_dat = INVALID_SOCKET;
                 //
                 strcpybuff(back->r.msg, "Unable to connect");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }                 // sinon on est prêts
             } else {
               strcpybuff(back->r.msg, "Unable to create a socket");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }                   // sinon on est prêts
           } else {
             snprintf(back->r.msg, sizeof(back->r.msg),
                      "Unable to resolve IP %s: %s", adr_ip, error);
-            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
+          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
-          // back->status=STATUS_FTP_READY;    // fini
+                   linejmp(line));
           back->r.statuscode = STATUSCODE_INVALID;
         }                       // sinon on est prêts
 #else
@@ -653,17 +594,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               //T_SOC soc_dat;
               if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
                 strcpybuff(back->r.msg, "Unable to accept connection");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
+              snprintf(back->r.msg, sizeof(back->r.msg),
-              // back->status=STATUS_FTP_READY;    // fini
+                       "RETR command error: %s", linejmp(line));
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
-            // back->status=STATUS_FTP_READY;    // fini
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -673,7 +613,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 #endif
         } else {
           strcpybuff(back->r.msg, "Unable to listen to a port");
-          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
 #endif
@@ -697,21 +636,18 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             int len = 1;
             int read_len = 1024;
 
-            //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
-
             while((len > 0) && (!stop_ftp(back))) {
               // attendre les données
               len = 1;          // pas d'erreur pour le moment
               switch (wait_socket_receive(soc_dat, timeout)) {
               case -1:
                 strcpybuff(back->r.msg, "FTP read error");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
               case 0:
-                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
+                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
-                // back->status=STATUS_FTP_READY;    // fini
+                         timeout);
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
@@ -733,17 +669,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                          }
                        */
                       strcpybuff(back->r.msg, "Write error");
-                      // back->status=STATUS_FTP_READY;    // fini
                       back->r.statuscode = STATUSCODE_INVALID;
                       len = 0;  // error
                     }
                   } else {
                     strcpybuff(back->r.msg, "Unexpected write error");
-                    // back->status=STATUS_FTP_READY;    // fini
                     back->r.statuscode = STATUSCODE_INVALID;
                   }
-                } else {        // Erreur ou terminé
+                } else { // Erreur ou terminé
-                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = 0;
                   if (back->r.totalsize > 0
                       && back->r.size != back->r.totalsize) {
@@ -752,7 +685,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   }
                 }
                 read_len = 1024;
-                //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
               }
             }
             if (back->r.fp) {
@@ -761,7 +693,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
           } else {
             strcpybuff(back->r.msg, "Unable to write file");
-            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -777,16 +708,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               get_ftp_line(soc_ctl, line, sizeof(line), timeout);
               if (line[0] == '2') {     // OK
                 strcpybuff(back->r.msg, "OK");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = HTTP_OK;
               } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
+                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
-                // back->status=STATUS_FTP_READY;    // fini
+                         linejmp(line));
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
               strcpybuff(back->r.msg, "FTP read error");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           }
@@ -812,7 +741,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     back->r.statuscode = HTTP_OK;
     strcpybuff(back->r.msg, "OK");
   }
-  // back->status=STATUS_FTP_READY;    // fini
   return 0;
 }
 
@@ -979,13 +907,11 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
       break;
     }
 
-    //HTS_TOTAL_RECV_CHECK(dummy);     // Diminuer au besoin si trop de données reçues
     switch (recv(soc, &b, 1, 0)) {
-      //case 0: break;    // pas encore --> erreur (on attend)!
     case 1:
       HTS_STAT.HTS_TOTAL_RECV += 1;     // compter flux entrant
-      if ((b != 10) && (b != 13))
+      if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
-        data[i++] = b;
+        data[i++] = b; // truncate hostile over-long reply lines
       break;
     default:
       if (ptrline)
@@ -1108,7 +1034,6 @@ int wait_socket_receive(T_SOC soc, int timeout) {
 int stop_ftp(lien_back * back) {
   if (back->stop_ftp) {
     strcpybuff(back->r.msg, "Cancelled by User");
-    // back->status=STATUS_FTP_READY;    // fini
     back->r.statuscode = STATUSCODE_INVALID;
     return 1;
   }

src/htsftp.h

@@ -70,6 +70,11 @@ int back_launch_ftp(FTPDownloadStruct * params);
 int run_launch_ftp(FTPDownloadStruct * params);
 int send_line(T_SOC soc, const char *data);
 int get_ftp_line(T_SOC soc, char *line, size_t line_size, int timeout);
+/* Split a "user[:pass]@" prefix (end = jump_identification result) into
+   bounded, NUL-terminated user/pass buffers, truncating to fit.
+   Both sizes must be nonzero. */
+void ftp_split_userpass(const char *src, const char *end, char *user,
+                        size_t user_size, char *pass, size_t pass_size);
 T_SOC get_datasocket(char *to_send, size_t to_send_size);
 int stop_ftp(lien_back * back);
 char *linejmp(char *line);

src/htsglobal.h

@@ -43,10 +43,10 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION      "3.49-8"
+#define HTTRACK_VERSION "3.49-11"
-#define HTTRACK_VERSIONID    "3.49.8"
+#define HTTRACK_VERSIONID "3.49.11"
-#define HTTRACK_AFF_VERSION  "3.x"
+#define HTTRACK_AFF_VERSION "3.x"
-#define HTTRACK_LIB_VERSION  "2.0"
+#define HTTRACK_LIB_VERSION "2.0"
 
 #ifndef HTS_NOINCLUDES
 #include <stdio.h>
@@ -71,11 +71,11 @@ Please visit our Website: http://www.httrack.com
    varargs starting at arg. */
 #ifndef HTS_UNUSED
 #ifdef __GNUC__
-#define HTS_UNUSED __attribute__ ((unused))
+#define HTS_UNUSED __attribute__((unused))
 
-#define HTS_STATIC static __attribute__ ((unused))
+#define HTS_STATIC static __attribute__((unused))
 
-#define HTS_PRINTF_FUN(fmt, arg) __attribute__ ((format (printf, fmt, arg)))
+#define HTS_PRINTF_FUN(fmt, arg) __attribute__((format(printf, fmt, arg)))
 #else
 #define HTS_UNUSED
 #define HTS_STATIC static
@@ -113,7 +113,7 @@ Please visit our Website: http://www.httrack.com
 
 #ifndef HTS_LONGLONG
 #ifdef SIZEOF_LONG_LONG
-#if SIZEOF_LONG_LONG==8
+#if SIZEOF_LONG_LONG == 8
 #define HTS_LONGLONG 1
 #endif
 #endif
@@ -204,12 +204,12 @@ Please visit our Website: http://www.httrack.com
 #endif
 
 #define HTS_HTTRACKRC ".httrackrc"
-#define HTS_HTTRACKCNF HTS_ETCPATH"/httrack.conf"
+#define HTS_HTTRACKCNF HTS_ETCPATH "/httrack.conf"
 
 #ifdef DATADIR
-#define HTS_HTTRACKDIR DATADIR"/httrack/"
+#define HTS_HTTRACKDIR DATADIR "/httrack/"
 #else
-#define HTS_HTTRACKDIR HTS_PREFIX"/share/httrack/"
+#define HTS_HTTRACKDIR HTS_PREFIX "/share/httrack/"
 #endif
 
 #endif
@@ -226,12 +226,21 @@ Please visit our Website: http://www.httrack.com
 
 /* Copyright (C) 1998 Xavier Roche and other contributors */
 #define HTTRACK_AFF_AUTHORS "[XR&CO'2014]"
-#define HTS_DEFAULT_FOOTER "<!-- Mirrored from %s%s by HTTrack Website Copier/" HTTRACK_AFF_VERSION " " HTTRACK_AFF_AUTHORS ", %s -->"
+#define HTS_DEFAULT_FOOTER                                                     \
+  "<!-- Mirrored from %s%s by HTTrack Website Copier/" HTTRACK_AFF_VERSION     \
+  " " HTTRACK_AFF_AUTHORS ", %s -->"
+/* Honest crawler User-Agent; no fake OS/browser to go stale. */
+#define HTS_DEFAULT_USER_AGENT                                                 \
+  "Mozilla/5.0 (compatible; HTTrack/" HTTRACK_AFF_VERSION                      \
+  "; +https://www.httrack.com/)"
 #define HTTRACK_WEB "http://www.httrack.com"
-#define HTS_UPDATE_WEBSITE "http://www.httrack.com/update.php3?Product=HTTrack&Version=" HTTRACK_VERSIONID "&VersionStr=" HTTRACK_VERSION "&Platform=%d&Language=%s"
+#define HTS_UPDATE_WEBSITE                                                     \
+  "http://www.httrack.com/"                                                    \
+  "update.php3?Product=HTTrack&Version=" HTTRACK_VERSIONID                     \
+  "&VersionStr=" HTTRACK_VERSION "&Platform=%d&Language=%s"
 
 #define H_CRLF "\x0d\x0a"
-#define CRLF   "\x0d\x0a"
+#define CRLF "\x0d\x0a"
 #ifdef _WIN32
 #define LF "\x0d\x0a"
 #else
@@ -242,10 +251,29 @@ Please visit our Website: http://www.httrack.com
 #define HTS_NOPARAM "(none)"
 #define HTS_NOPARAM2 "\"(none)\""
 
-/* Larger/smaller of two values. Macros: arguments are evaluated twice. */
+/* Boolean flag for option fields and API yes/no returns. Int-backed, not an
-#define maximum(A,B) ( (A) > (B) ? (A) : (B) )
+   enum: an enum makes C++ reject `field = 1` / `f(0)` on the exported fields
+   and params. Int-sized, so the httrackp layout and the ABI are unchanged. */
+#ifndef HTS_DEF_DEFSTRUCT_hts_boolean
+#define HTS_DEF_DEFSTRUCT_hts_boolean
 
-#define minimum(A,B) ( (A) < (B) ? (A) : (B) )
+typedef int hts_boolean;
+#define HTS_FALSE 0
+#define HTS_TRUE 1
+#endif
+
+#ifndef HTS_DEF_DEFSTRUCT_hts_tristate
+#define HTS_DEF_DEFSTRUCT_hts_tristate
+/* Tri-state hts_boolean: HTS_DEFAULT (-1) = "unspecified" (copy_htsopt leaves
+   the target untouched); HTS_FALSE/HTS_TRUE = off/on. */
+typedef int hts_tristate;
+#define HTS_DEFAULT (-1)
+#endif
+
+/* Larger/smaller of two values. Macros: arguments are evaluated twice. */
+#define maximum(A, B) ((A) > (B) ? (A) : (B))
+
+#define minimum(A, B) ((A) < (B) ? (A) : (B))
 
 /* True when A is a non-NULL, non-empty string. */
 #define strnotempty(A) (((A) != NULL && (A)[0] != '\0'))
@@ -270,10 +298,10 @@ Please visit our Website: http://www.httrack.com
 #endif
 #else
 /* See <http://gcc.gnu.org/wiki/Visibility> */
-#if ( ( defined(__GNUC__) && ( __GNUC__ >= 4 ) ) \
+#if ((defined(__GNUC__) && (__GNUC__ >= 4)) ||                                 \
-      || ( defined(HAVE_VISIBILITY) && HAVE_VISIBILITY ) )
+     (defined(HAVE_VISIBILITY) && HAVE_VISIBILITY))
 
-#define HTSEXT_API __attribute__ ((visibility ("default")))
+#define HTSEXT_API __attribute__((visibility("default")))
 #else
 #define HTSEXT_API
 #endif
@@ -327,8 +355,8 @@ typedef __int64 LLint;
 typedef __int64 TStamp;
 
 #define LLintP "%I64d"
-#elif (defined(_LP64) || defined(__x86_64__) \
+#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
-       || defined(__powerpc64__) || defined(__64BIT__))
+       defined(__64BIT__))
 
 typedef long int LLint;
 
@@ -384,6 +412,10 @@ typedef int T_SOC;
 /* Buffer size for a printed network address (IPv4 or IPv6, NUL included). */
 #define HTS_MAXADDRLEN 64
 
+/* Max resolved addresses kept per host for connect fallback (dead IPv6 etc.).
+ */
+#define HTS_MAXADDRNUM 4
+
 #ifdef _WIN32
 #else
 #define __cdecl
@@ -392,16 +424,17 @@ typedef int T_SOC;
 /* Permission bits for created folders and files (mkdir and chmod).
    PROTECT_FOLDER is owner-only. With HTS_ACCESS set (the default) the ACCESS_
    modes also grant group/other read; otherwise they stay owner-only. */
-#define HTS_PROTECT_FOLDER (S_IRUSR|S_IWUSR|S_IXUSR)
+#define HTS_PROTECT_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
 
 #if HTS_ACCESS
-#define HTS_ACCESS_FILE (S_IRUSR|S_IWUSR|S_IRGRP|S_IROTH)
+#define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)
 
-#define HTS_ACCESS_FOLDER (S_IRUSR|S_IWUSR|S_IXUSR|S_IRGRP|S_IXGRP|S_IROTH|S_IXOTH)
+#define HTS_ACCESS_FOLDER                                                      \
+  (S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP | S_IROTH | S_IXOTH)
 #else
-#define HTS_ACCESS_FILE (S_IRUSR|S_IWUSR)
+#define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR)
 
-#define HTS_ACCESS_FOLDER (S_IRUSR|S_IWUSR|S_IXUSR)
+#define HTS_ACCESS_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
 #endif
 
 /* Sanity-check that the required preprocessor switches are defined */
@@ -419,7 +452,11 @@ typedef int T_SOC;
 #endif
 
 /* fflush sur stdout */
-#define io_flush { fflush(stdout); fflush(stdin); }
+#define io_flush                                                               \
+  {                                                                            \
+    fflush(stdout);                                                            \
+    fflush(stdin);                                                             \
+  }
 
 /* HTSLib */
 
@@ -439,7 +476,7 @@ typedef int T_SOC;
 
 #ifdef _DEBUG
 // trace mallocs
-//#define HTS_TRACE_MALLOC
+// #define HTS_TRACE_MALLOC
 #ifdef HTS_TRACE_MALLOC
 typedef unsigned long int t_htsboundary;
 
@@ -516,7 +553,13 @@ static const t_htsboundary htsboundary = 0xDEADBEEF;
 #if _HTS_WIDE
 extern FILE *DEBUG_fp;
 
-#define DEBUG_W(A)  { if (DEBUG_fp==NULL) DEBUG_fp=fopen("bug.out","wb"); fprintf(DEBUG_fp,":>"A); fflush(DEBUG_fp); }
+#define DEBUG_W(A)                                                             \
+  {                                                                            \
+    if (DEBUG_fp == NULL)                                                      \
+      DEBUG_fp = fopen("bug.out", "wb");                                       \
+    fprintf(DEBUG_fp, ":>" A);                                                 \
+    fflush(DEBUG_fp);                                                          \
+  }
 #undef _
 #define _ ,
 #endif

src/htshash.c

@@ -106,10 +106,10 @@ static coucal_hashkeys key_adrfil_hashes_generic(void *arg,
   const lien_url*const lien = (const lien_url*) value;
   const char *const adr = !former ? lien->adr : lien->former_adr;
   const char *const fil = !former ? lien->fil : lien->former_fil;
-  const char *const adr_norm = adr != NULL ? 
+  const char *const adr_norm =
-    ( hash->normalized  ? jump_normalized_const(adr)
+      adr != NULL ? (hash->norm_host ? jump_normalized_const(adr)
-                        : jump_identification_const(adr) )
+                                     : jump_identification_const(adr))
-    : NULL;
+                  : NULL;
 
   // copy address
   assertf(adr_norm != NULL);
@@ -117,10 +117,18 @@ static coucal_hashkeys key_adrfil_hashes_generic(void *arg,
 
   // copy link
   assertf(fil != NULL);
-  if (hash->normalized) {
+  {
-    fil_normalized(fil, &hash->normfil[strlen(hash->normfil)]);
+    /* resolve the per-URL strip keys; strip applies even when urlhack is off */
-  } else {
+    char BIGSTK keybuf[HTS_URLMAXSIZE];
-    strcpy(&hash->normfil[strlen(hash->normfil)], fil);
+    const char *const keys = hts_query_strip_keys(hash->strip_query, adr, fil,
+                                                  keybuf, sizeof(keybuf));
+
+    if (hash->norm_slash || hash->norm_query || keys != NULL) {
+      fil_normalized_filtered_ex(fil, &hash->normfil[strlen(hash->normfil)],
+                                 keys, hash->norm_slash, hash->norm_query);
+    } else {
+      strcpy(&hash->normfil[strlen(hash->normfil)], fil);
+    }
   }
 
   // hash
@@ -132,8 +140,7 @@ static int key_adrfil_equals_generic(void *arg,
                                      coucal_key_const a_,
                                      coucal_key_const b_, 
                                      const int former) {
-  hash_struct *const hash = (hash_struct*) arg;
+  hash_struct *const hash = (hash_struct *) arg;
-  const int normalized = hash->normalized;
   const lien_url*const a = (const lien_url*) a_;
   const lien_url*const b = (const lien_url*) b_;
   const char *const a_adr = !former ? a->adr : a->former_adr;
@@ -150,10 +157,10 @@ static int key_adrfil_equals_generic(void *arg,
   assertf(b_fil != NULL);
 
   // skip scheme and authentication to the domain (possibly without www.)
-  ja = normalized
+  ja = hash->norm_host ? jump_normalized_const(a_adr)
-    ? jump_normalized_const(a_adr) : jump_identification_const(a_adr);
+                       : jump_identification_const(a_adr);
-  jb = normalized
+  jb = hash->norm_host ? jump_normalized_const(b_adr)
-    ? jump_normalized_const(b_adr) : jump_identification_const(b_adr);
+                       : jump_identification_const(b_adr);
   assertf(ja != NULL);
   assertf(jb != NULL);
   if (strcasecmp(ja, jb) != 0) {
@@ -161,12 +168,23 @@ static int key_adrfil_equals_generic(void *arg,
   }
 
   // now compare pathes
-  if (normalized) {
+  {
-    fil_normalized(a_fil, hash->normfil);
+    char BIGSTK ka[HTS_URLMAXSIZE], kb[HTS_URLMAXSIZE];
-    fil_normalized(b_fil, hash->normfil2);
+    const char *const keysa =
-    return strcmp(hash->normfil, hash->normfil2) == 0;
+        hts_query_strip_keys(hash->strip_query, a_adr, a_fil, ka, sizeof(ka));
-  } else {
+    const char *const keysb =
-    return strcmp(a_fil, b_fil) == 0;
+        hts_query_strip_keys(hash->strip_query, b_adr, b_fil, kb, sizeof(kb));
+
+    if (hash->norm_slash || hash->norm_query || keysa != NULL ||
+        keysb != NULL) {
+      fil_normalized_filtered_ex(a_fil, hash->normfil, keysa, hash->norm_slash,
+                                 hash->norm_query);
+      fil_normalized_filtered_ex(b_fil, hash->normfil2, keysb, hash->norm_slash,
+                                 hash->norm_query);
+      return strcmp(hash->normfil, hash->normfil2) == 0;
+    } else {
+      return strcmp(a_fil, b_fil) == 0;
+    }
   }
 }
 
@@ -222,11 +240,17 @@ static int key_former_adrfil_equals(void *arg,
   return key_adrfil_equals_generic(arg, a, b, 1);
 }
 
-void hash_init(httrackp *opt, hash_struct * hash, int normalized) {
+void hash_init(httrackp *opt, hash_struct *hash, hts_boolean normalized) {
   hash->sav = coucal_new(0);
   hash->adrfil = coucal_new(0);
   hash->former_adrfil = coucal_new(0);
-  hash->normalized = normalized;
+  /* urlhack is the umbrella; per-feature negatives opt out of each part */
+  hash->norm_host = normalized && !opt->no_www_dedup;
+  hash->norm_slash = normalized && !opt->no_slash_dedup;
+  hash->norm_query = normalized && !opt->no_query_dedup;
+  /* snapshot the query-strip list (not owned; valid for the hash lifetime) */
+  hash->strip_query =
+      StringNotEmpty(opt->strip_query) ? StringBuff(opt->strip_query) : NULL;
 
   hts_set_hash_handler(hash->sav, opt);
   hts_set_hash_handler(hash->adrfil, opt);
@@ -282,6 +306,26 @@ void hash_free(hash_struct *hash) {
   }
 }
 
+/* Test helper: do the two URLs dedupe to the same key under opt's urlhack
+   flags? Exercises the live hash compare (norm_host/slash/query resolution). */
+hts_boolean hash_url_equals(httrackp *opt, const char *adra, const char *fila,
+                            const char *adrb, const char *filb) {
+  hash_struct hash;
+  lien_url la, lb;
+  hts_boolean eq;
+
+  memset(&la, 0, sizeof(la));
+  memset(&lb, 0, sizeof(lb));
+  la.adr = key_duphandler(NULL, adra);
+  la.fil = key_duphandler(NULL, fila);
+  lb.adr = key_duphandler(NULL, adrb);
+  lb.fil = key_duphandler(NULL, filb);
+  hash_init(opt, &hash, opt->urlhack);
+  eq = key_adrfil_equals(&hash, &la, &lb);
+  hash_free(&hash);
+  return eq;
+}
+
 // retour: position ou -1 si non trouvé
 int hash_read(const hash_struct * hash, const char *nom1, const char *nom2,
               hash_struct_type type) {

src/htshash.h

@@ -51,8 +51,12 @@ typedef enum hash_struct_type {
 } hash_struct_type;
 
 // tables de hachage
-void hash_init(httrackp *opt, hash_struct *hash, int normalized);
+void hash_init(httrackp *opt, hash_struct *hash, hts_boolean normalized);
 void hash_free(hash_struct *hash);
+/* Test helper: HTS_TRUE if the two URLs dedupe together under opt's urlhack
+   flags. */
+hts_boolean hash_url_equals(httrackp *opt, const char *adra, const char *fila,
+                            const char *adrb, const char *filb);
 int hash_read(const hash_struct * hash, const char *nom1, const char *nom2,
               hash_struct_type type);
 void hash_write(hash_struct * hash, size_t lpos);

src/htshelp.c

@@ -152,16 +152,6 @@ void help_wizard(httrackp * opt) {
 #define str (buffers->str)
 #define argv (buffers->argv)
 
-  //char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
-  //char *mainpath = (char *) malloct(256);
-  //char *projname = (char *) malloct(256);
-  //char *stropt = (char *) malloct(2048);        // options
-  //char *stropt2 = (char *) malloct(2048);       // options longues
-  //char *strwild = (char *) malloct(2048);       // wildcards
-  //char *cmd = (char *) malloct(4096);
-  //char *str = (char *) malloct(256);
-  //char **argv = (char **) malloct(256 * sizeof(char *));
-
   //
   char *a;
 
@@ -227,8 +217,6 @@ void help_wizard(httrackp * opt) {
   strcatbuff(stropt2, str);
   strcatbuff(stropt2, projname);
   strcatbuff(stropt2, "\" ");
-  // Créer si ce n'est fait un index.html 1er niveau
-  make_empty_index(str);
   //
   printf("\n");
   printf("Enter URLs (separated by commas or blank spaces) :");
@@ -349,8 +337,6 @@ void help_wizard(httrackp * opt) {
       }
       hts_main(argc, argv);
     }
-    //} else {
-    //  help("httrack",1);
   }
 
   /* Free buffers */
@@ -455,23 +441,6 @@ void help_catchurl(const char *dest_path) {
     printf("Unable to create a temporary proxy (no remaining port)\n");
 }
 
-// Créer un index.html vide
-void make_empty_index(const char *str) {
-#if 0
-  if (!fexist(fconcat(str, "index.html"))) {
-    FILE *fp = fopen(fconcat(str, "index.html"), "wb");
-
-    if (fp) {
-      fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
-      fprintf(fp,
-              "<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
-              CRLF);
-      fclose(fp);
-    }
-  }
-#endif
-}
-
 // mini-aide  (h: help)
 //           y
 void help(const char *app, int more) {
@@ -521,6 +490,7 @@ void help(const char *app, int more) {
   infomsg("  EN maximum mirror time in seconds (60=1 minute, 3600=1 hour)");
   infomsg("  AN maximum transfer rate in bytes/seconds (1000=1KB/s max)");
   infomsg(" %cN maximum number of connections/seconds (*%c10)");
+  infomsg(" %G  random pause of MIN[:MAX] seconds between files (e.g. %G5:10)");
   infomsg
     ("  GN pause transfer if N bytes reached, and wait until lock file is deleted");
   infomsg("");
@@ -563,6 +533,7 @@ void help(const char *app, int more) {
     (" %x  do not include any password for external password protected websites (%x0 include)");
   infomsg
     (" %q *include query string for local files (useless, for information purpose only) (%q0 don't include)");
+  infomsg(" %g  strip query keys for dedup ([host/pattern=]key1,key2,...)");
   infomsg
     ("  o *generate output html file in case of error (404..) (o0 don't generate)");
   infomsg("  X *purge old files after update (X0 keep delete)");
@@ -571,6 +542,7 @@ void help(const char *app, int more) {
   infomsg("");
   infomsg("Spider options:");
   infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
+  infomsg(" %K  load extra cookies from a Netscape cookies.txt");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
   infomsg
@@ -587,6 +559,9 @@ void help(const char *app, int more) {
     (" %s  update hacks: various hacks to limit re-transfers when updating (identical size, bogus response..)");
   infomsg
     (" %u  url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..)");
+  infomsg("     opt out of one url-hack part: --keep-www-prefix "
+          "(www.foo.com<>foo.com), --keep-double-slashes (//), "
+          "--keep-query-order (?b&a)");
   infomsg
     (" %A  assume that a type (cgi,asp..) is always linked with a mime type (-%A php3,cgi=text/html;dat,bin=application/x-zip)");
   infomsg("     shortcut: '--assume standard' is equivalent to -%A "
@@ -646,9 +621,7 @@ void help(const char *app, int more) {
   infomsg("");
   infomsg("Guru options: (do NOT use if possible)");
   infomsg(" #X *use optimized engine (limited memory boundary checks)");
-  infomsg(" #0  filter test (-#0 '*.gif' 'www.bar.com/foo.gif')");
+  infomsg(" #test  list engine self-tests (run one with -#test=NAME [args])");
-  infomsg(" #1  simplify test (-#1 ./foo/bar/../foobar)");
-  infomsg(" #2  type test (-#2 /foo/bar.php)");
   infomsg(" #C  cache list (-#C '*.com/spider*.gif'");
   infomsg(" #R  cache repair (damaged cache)");
   infomsg(" #d  debug parser");
@@ -804,7 +777,4 @@ void help(const char *app, int more) {
   infomsg("[compiled: " HTS_PLATFORM_NAME "]");
 #endif
   infomsg(NULL);
-
-//  infomsg("  R  *relative links (e.g ../link)\n");
-//  infomsg("  A   absolute links (e.g /www.adr/link)\n");
 }

src/htshelp.h

@@ -45,7 +45,6 @@ typedef struct httrackp httrackp;
 
 void infomsg(const char *msg);
 void help(const char *app, int more);
-void make_empty_index(const char *str);
 void help_wizard(httrackp * opt);
 int help_query(const char *list, int def);
 void help_catchurl(const char *dest_path);

src/htsindex.c

@@ -142,8 +142,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
   char keyword[KEYW_LEN + 32];
   int i = 0;
 
-  //
-  //int WordIndexSize = 1024;
   coucal WordIndexHash = NULL;
   FILE *tmpfp = NULL;
 
@@ -178,8 +176,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
              || (strfield2(mime, "text/css"))
     ) {
     inscript = 1;
-    //} else if (strfield2(mime, "text/vnd.wap.wml")) {   // humm won't work in many cases
-    //  inscript=0;
   } else
     return 0;
 
@@ -275,8 +271,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
 
   // Process indexing for this page
   {
-    //FILE* fp=NULL;
-    //fp=fopen(concat(indexpath,"index.txt"),"ab");
     if (fp_tmpproject) {
       while(!feof(tmpfp)) {
         char line[KEYW_LEN + 32];
@@ -286,7 +280,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
           intptr_t e = 0;
 
           if (coucal_read(WordIndexHash, line, &e)) {
-            //if (e) {
             char BIGSTK savelst[HTS_URLMAXSIZE * 2];
 
             e++;                /* 0 means "once" */
@@ -300,11 +293,9 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
             fprintf(fp_tmpproject, "%s %d %s\n", line,
                     (int) (KEYW_SORT_MAXCOUNT - e), savelst);
             hts_primindex_size++;
-            //}
           }
         }
       }
-      //fclose(fp);
     }
   }
 
@@ -330,7 +321,6 @@ void index_finish(const char *indexpath, int mode) {
   off_t size = fpsize(fp_tmpproject);
 
   if (size > 0) {
-    //FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
     if (fp_tmpproject) {
       tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
       if (tab) {
@@ -398,8 +388,6 @@ void index_finish(const char *indexpath, int mode) {
                       if (total_hit) {
                         if (mode == 1)  // TEXT
                           fprintf(fp, "\t=%d\r\n", total_hit);
-                        //else                // HTML
-                        //  fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
                         if ((((total_hit * 1000) / hts_primindex_words) >=
                              KEYW_USELESS1K)
                             || (((total_line * 1000) / index) >=
@@ -416,8 +404,6 @@ void index_finish(const char *indexpath, int mode) {
                           if (mode == 1)        // TEXT
                             fprintf(fp, "\t(%d)\r\n",
                                     ((total_hit * 1000) / hts_primindex_words));
-                          //else                // HTML
-                          //  fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
                         }
                       }
                       if (mode == 1)    // TEXT

src/htsjava.c

@@ -63,6 +63,9 @@ Please visit our Website: http://www.httrack.com
 /* This file */
 #include "htsjava.h"
 
+/* calloct/freet wrappers */
+#include "htssafe.h"
+
 static int reverse_endian(void) {
   int endian = 1;
 
@@ -172,15 +175,14 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
 #if JAVADEBUG
       printf("fopen\n");
 #endif
-      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
+      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) ==
-        //fprintf(stderr, "Cannot open input file.\n");
+          NULL) {
         sprintf(str->err_msg, "Unable to open file %s", file);
         return 0;               // une erreur..
       }
 #if JAVADEBUG
       printf("fread\n");
 #endif
-      //if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) {   // pas complet..
       if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
         fclose(fpout);
         sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
@@ -204,7 +206,16 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
         return 0;
       }
 
-      tab = (RESP_STRUCT *) calloc(header.count, sizeof(RESP_STRUCT));
+      /* A constant-pool entry is >= 1 byte on disk; reject a count exceeding
+         the file size (hostile .class ~68 MB alloc DoS). */
+      if (!hts_count_fits(header.count, (LLint) fsize(file))) {
+        fclose(fpout);
+        sprintf(str->err_msg,
+                "Invalid constant pool count %u (file len " LLintP ")",
+                (unsigned) header.count, (LLint) fsize(file));
+        return 0;
+      }
+      tab = (RESP_STRUCT *) calloct(header.count, sizeof(RESP_STRUCT));
       if (!tab) {
         sprintf(str->err_msg, "Unable to alloc %d bytes",
                 (int) sizeof(RESP_STRUCT));
@@ -230,7 +241,7 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
           } else {              // ++ une erreur est survenue!
             if (strnotempty(str->err_msg) == 0)
               strcpy(str->err_msg, "Internal readtable error");
-            free(tab);
+            freet(tab);
             if (fpout) {
               fclose(fpout);
               fpout = NULL;
@@ -245,12 +256,10 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
       printf("addfiles\n");
 #endif
       {
-        //unsigned int acess;
         unsigned int Class;
         unsigned int SClass;
         int i;
 
-        //acess = readshort(fpout);
         Class = readshort(fpout);
         SClass = readshort(fpout);
 
@@ -288,7 +297,7 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
 #if JAVADEBUG
       printf("end\n");
 #endif
-      free(tab);
+      freet(tab);
       if (fpout) {
         fclose(fpout);
         fpout = NULL;
@@ -395,11 +404,9 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
 
       p = &buffer[0];
 
-      //fflush(fp); 
       trans.file_position = ftell(fp);
       length = readshort(fp);
       if (length < HTS_URLMAXSIZE) {
-        // while ((length > 0) && (length<500)) {
         while(length > 0) {
           *p++ = fgetc(fp);
 
@@ -407,17 +414,13 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
         }
         *p = '\0';
 
-        //#if JDEBUG
-        //      if(tris(buffer)==1) printf("%s\n ",buffer);
-        //      if(tris(buffer)==2)  printf("%s\n ",printname(buffer));
-        //#endif
         if (tris(str->opt, buffer) == 1)
           str->addLink(str, buffer);    /* trans.file_position */
         else if (tris(str->opt, buffer) == 2)
           str->addLink(str, printname(rname, buffer));
 
         strcpy(trans.name, buffer);
-      } else {                  // gros pb
+      } else { // gros pb
         while((length > 0) && (!feof(fp))) {
           fgetc(fp);
           length--;
@@ -433,7 +436,6 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
     }
     break;
   default:
-    // printf("Type inconnue\n");
     // on arrête tout 
     sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
     *error = 1;
@@ -496,7 +498,6 @@ static char *printname(char rname[1024], char name[1024]) {
     return rname;  // ""
   }
   p += 2;
-  //rname=(char*)calloct(strlen(name)+8,sizeof(char));
   p1 = rname;
   for(j = 0; name[j] != '\0'; j++, p++) {
     if (*p == '/')

src/htsjava.h

@@ -33,15 +33,19 @@ Please visit our Website: http://www.httrack.com
 #ifndef HTSJAVA_DEFH
 #define HTSJAVA_DEFH
 
+#include <stdint.h>
+
 #ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER
 #define HTS_DEF_FWSTRUCT_JAVA_HEADER
 typedef struct JAVA_HEADER JAVA_HEADER;
 #endif
+/* 10-byte on-disk .class header image, fread() directly: fields need exact
+   widths (LP64's 8-byte 'unsigned long' magic never matched 0xCAFEBABE). */
 struct JAVA_HEADER {
-  unsigned long int magic;
+  uint32_t magic;
-  unsigned short int minor;
+  uint16_t minor;
-  unsigned short int major;
+  uint16_t major;
-  unsigned short int count;
+  uint16_t count;
 };
 
 #ifndef HTS_DEF_FWSTRUCT_RESP_STRUCT

src/htslib.h

@@ -147,11 +147,13 @@ struct OLD_htsblk {
 #define HTS_DEF_FWSTRUCT_t_dnscache
 typedef struct t_dnscache t_dnscache;
 #endif
+// One DNS cache record, stored as a coucal value keyed by hostname.
 struct t_dnscache {
-  struct t_dnscache *next;
+  // resolved addresses, in resolver (RFC 6724) order; host_count==0 means the
-  const char *iadr;
+  // name does not resolve (negative cache). host_count<=HTS_MAXADDRNUM.
-  size_t host_length;                   // length ; (4 or 16) ; 0 for error
+  int host_count;
-  char host_addr[HTS_MAXADDRLEN];
+  size_t host_length[HTS_MAXADDRNUM]; // sockaddr length of each (16 or 28)
+  char host_addr[HTS_MAXADDRNUM][HTS_MAXADDRLEN];
 };
 
 /* Library internal definictions */
@@ -169,7 +171,6 @@ HTSEXT_API const char* hts_version(void);
 // fonctions unix/winsock
 int hts_read(htsblk * r, char *buff, int size);
 
-//int HTS_TOTAL_RECV_CHECK(int var);
 LLint check_downloadable_bytes(int rate);
 
 HTSEXT_API int hts_uninit_module(void);
@@ -182,10 +183,21 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
                   const char *adr, const char *fil,
                   const char *referer_adr, const char *referer_fil,
                   htsblk * retour);
+/* Build the request "Cookie:" header line for stored cookies matching
+   domain/path into dst (NUL-terminated). Exposed for the -#Q self-test;
+   wraps the same logic http_sendhead() uses. Returns cookies emitted. */
+int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
+                                const char *path, char *dst, size_t dst_size);
 
-//int newhttp(char* iadr,char* err=NULL);
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
               int waitconnect);
+/* Like newhttp(), but connect to the addr_index-th resolved address of the host
+   (0-based) instead of always the first; *addr_count, if non-NULL, is set to
+   the total resolved addresses. newhttp() == newhttp_addr(...,0,NULL). Used by
+   the slot scheduler to try the next address when a connect fails (dead IPv6
+   etc.). */
+T_SOC newhttp_addr(httrackp *opt, const char *iadr, htsblk *retour, int port,
+                   int waitconnect, int addr_index, int *addr_count);
 HTS_INLINE void deletehttp(htsblk * r);
 HTS_INLINE int deleteaddr(htsblk * r);
 HTS_INLINE void deletesoc(T_SOC soc);
@@ -193,15 +205,31 @@ HTS_INLINE void deletesoc_r(htsblk * r);
 htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc);
 int check_readinput(htsblk * r);
 int check_readinput_t(T_SOC soc, int timeout);
+int check_writeinput_t(T_SOC soc, int timeout);
+
+/* Open an HTTP CONNECT tunnel through the active proxy for an https request:
+   `retour->soc` must already be TCP-connected to the proxy, and `adr` is the
+   origin authority (url_adr, e.g. "https://host:port"). Sends the CONNECT
+   request (with Proxy-Authorization when the proxy carries credentials) and
+   reads the proxy's status line, so the caller's TLS handshake then runs
+   end-to-end with the origin. Blocks up to `timeout` seconds. Returns 1 on a
+   2xx tunnel, 0 on failure (retour->msg/statuscode set). */
+int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
+                      int timeout);
 void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * retour,
                char *rcvd);
 void treatfirstline(htsblk * retour, const char *rcvd);
 
 // sous-fonctions
 LLint http_xfread1(htsblk * r, int bufl);
-HTS_INLINE SOCaddr* hts_dns_resolve2(httrackp * opt, const char *iadr,
+/* Cached resolver: fill out[0..count-1] with up to max addresses for iadr (in
-                                     SOCaddr *const addr, 
+   resolver order), returning the count (0 = does not resolve, negative-cached).
-                                     const char **error);
+   Resolves once per host; later calls read the DNS cache. Must hold no lock
+   (brackets opt->state.lock itself). */
+int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
+                        const char **error);
+HTS_INLINE SOCaddr *hts_dns_resolve2(httrackp *opt, const char *iadr,
+                                     SOCaddr *const addr, const char **error);
 HTS_INLINE SOCaddr* hts_dns_resolve(httrackp * opt, const char *iadr,
                                     SOCaddr *const addr);
 HTSEXT_API SOCaddr* hts_dns_resolve_nocache2(const char *const hostname, 
@@ -214,8 +242,9 @@ HTSEXT_API int check_hostname_dns(const char *const hostname);
 int ftp_available(void);
 
 #if HTS_DNSCACHE
-void hts_cache_free(t_dnscache *const cache);
+/* Return opt's DNS cache hashtable (hostname -> t_dnscache record), creating it
-t_dnscache *hts_cache(httrackp * opt);
+   on first use. Records are owned by the table and freed on coucal_delete. */
+coucal hts_cache(httrackp *opt);
 #endif
 
 // outils divers
@@ -254,6 +283,9 @@ int ishttperror(int err);
 int get_userhttptype(httrackp * opt, char *s, const char *fil);
 int give_mimext(char *s, size_t ssize, const char *st);
 
+/* Advertised Accept-Encoding value (no header name/CRLF); see htslib.c. */
+const char *hts_acceptencoding(hts_boolean compressible);
+
 int may_bogus_multiple(httrackp * opt, const char *mime, const char *filename);
 int may_unknown2(httrackp * opt, const char *mime, const char *filename);
 
@@ -465,10 +497,22 @@ HTS_STATIC int strcmpnocase(const char *a, const char *b) {
 
 // is this MIME an hypertext MIME (text/html), html/js-style or other script/text type?
 #define HTS_HYPERTEXT_DEFAULT_MIME "text/html"
+/* Sentinel stored when the server declared no Content-Type. It is html-ish
+   for every type test (so a typeless response still parses/stores as today),
+   but the naming code (wire_patches_ext) treats it as "no declared type" and
+   keeps the URL extension. It rides the cache, so updates name consistently. */
+#define HTS_UNKNOWN_MIME "unknown/unknown"
+/* Map the no-declared-type sentinel back to a real type for any header or
+   record we EMIT or PERSIST, so "unknown/unknown" never reaches a consumer
+   (a served Content-Type, a ProxyTrack .arc record, ...). */
+#define hts_effective_mime(m)                                                  \
+  (strfield2((m), HTS_UNKNOWN_MIME) ? HTS_HYPERTEXT_DEFAULT_MIME : (m))
 
-#define is_html_mime_type(a) \
+#define is_html_mime_type(a)                                                   \
-  ( (strfield2((a),"text/html")!=0)\
+  ((strfield2((a), "text/html") != 0) ||                                       \
-  || (strfield2((a),"application/xhtml+xml")!=0) \
+   (strfield2((a), "application/xhtml+xml") != 0) ||                           \
+   (strfield2((a), HTS_UNKNOWN_MIME) !=                                        \
+    0) /* no declared type: treat as html */                                   \
   )
 #define is_hypertext_mime__(a) \
   ( \

src/htsmodules.c

@@ -55,12 +55,6 @@ extern int fspc(httrackp * opt, FILE * fp, const char *type);
 
 /* >>> Put all modules variables here */
 
-#if 0
-t_gzopen gzopen = NULL;
-t_gzread gzread = NULL;
-t_gzclose gzclose = NULL;
-#endif
-
 int V6_is_available = HTS_INET6;
 
 static char WHAT_is_available[64] = "";
@@ -112,18 +106,6 @@ int hts_parse_externals(htsmoduleStruct * str) {
   return -1;
 }
 
-//static void addCallback(htscallbacks* chain, void* moduleHandle, htscallbacksfncptr exitFnc) {
-//  while(chain->next != NULL) {
-//    chain = chain->next;
-//  }
-//  chain->next = calloct(1, sizeof(htscallbacks));
-//  assertf(chain->next != NULL);
-//  chain = chain->next;
-//  memset(chain, 0, sizeof(*chain));
-//  chain->exitFnc = exitFnc;
-//  chain->moduleHandle = moduleHandle;
-//}
-
 void clearCallbacks(htscallbacks * chain_);
 void clearCallbacks(htscallbacks * chain_) {
   htscallbacks *chain;

src/htsmodules.h

@@ -69,41 +69,41 @@ typedef struct hash_struct hash_struct;
 #define HTS_DEF_FWSTRUCT_htsmoduleStruct
 typedef struct htsmoduleStruct htsmoduleStruct;
 #endif
-typedef int (*t_htsAddLink) (htsmoduleStruct * str, char *link);
+typedef int (*t_htsAddLink)(htsmoduleStruct *str, char *link);
 
 /** Per-object context passed to a parser module for one downloaded file.
     Field access classes are noted; engine owns all pointers unless stated. */
 struct htsmoduleStruct {
   /* Read-only elements */
-  const char *filename;         /* filename (C:\My Web Sites\...) */
+  const char *filename; /* filename (C:\My Web Sites\...) */
-  int size;                     /* size of filename (should be > 0) */
+  int size;             /* size of filename (should be > 0) */
-  const char *mime;             /* MIME type of the object */
+  const char *mime;     /* MIME type of the object */
-  const char *url_host;         /* incoming hostname (www.foo.com) */
+  const char *url_host; /* incoming hostname (www.foo.com) */
-  const char *url_file;         /* incoming filename (/bar/bar.gny) */
+  const char *url_file; /* incoming filename (/bar/bar.gny) */
 
   /* Write-only */
-  const char *wrapper_name;     /* name of wrapper (static string) */
+  const char *wrapper_name; /* name of wrapper (static string) */
-  char *err_msg;                /* if an error occurred, the error message (max. 1KB) */
+  char *err_msg; /* if an error occurred, the error message (max. 1KB) */
 
   /* Read/Write */
-  int relativeToHtmlLink;       /* set this to 1 if all urls you pass to addLink
+  int relativeToHtmlLink; /* set this to 1 if all urls you pass to addLink
-                                   are in fact relative to the html file where your
+                             are in fact relative to the html file where your
-                                   module was originally */
+                             module was originally */
 
   /* Callbacks */
-  t_htsAddLink addLink;         /* call this function when links are 
+  t_htsAddLink addLink; /* call this function when links are
-                                   being detected. it if not your responsability to decide
+                           being detected. it if not your responsability to
-                                   if the engine will keep them, or not. */
+                           decide if the engine will keep them, or not. */
 
   /* Optional */
-  char *localLink;              /* if non null, the engine will write there the local
+  char *localLink;   /* if non null, the engine will write there the local
-                                   relative filename of the link added by addLink(), or
+                        relative filename of the link added by addLink(), or
-                                   the absolute path if the link was refused by the wizard */
+                        the absolute path if the link was refused by the wizard */
-  int localLinkSize;            /* size of the optionnal buffer */
+  int localLinkSize; /* size of the optionnal buffer */
 
   /* User-defined */
-  void *userdef;                /* can be used by callback routines
+  void *userdef; /* can be used by callback routines
-                                 */
+                  */
 
   /* The parser httrackp structure (may be used) */
   httrackp *opt;
@@ -117,7 +117,6 @@ struct htsmoduleStruct {
   int *ptr_;
   const char *page_charset_;
   /* Internal use - please don't touch */
-
 };
 
 #ifdef __cplusplus
@@ -126,11 +125,11 @@ extern "C" {
 
 /** Module lifecycle hooks. Init/PlugInit return 1 on success, 0 on failure;
     Exit returns its own status (ignored by the engine). */
-typedef int (*t_htsWrapperInit) (char *fn, char *args);
+typedef int (*t_htsWrapperInit)(char *fn, char *args);
 
-typedef int (*t_htsWrapperExit) (void);
+typedef int (*t_htsWrapperExit)(void);
 
-typedef int (*t_htsWrapperPlugInit) (char *args);
+typedef int (*t_htsWrapperPlugInit)(char *args);
 
 /* Library internal definictions */
 #ifdef HTS_INTERNAL_BYTECODE
@@ -138,7 +137,7 @@ typedef int (*t_htsWrapperPlugInit) (char *args);
 /** Capabilities string ("-noV6", "-nossl", ...) followed by "+name" for each
     loaded module. Returned pointer aliases opt->state.HTbuff; do not free, and
     it is overwritten by the next call. */
-HTSEXT_API const char *hts_get_version_info(httrackp * opt);
+HTSEXT_API const char *hts_get_version_info(httrackp *opt);
 
 /** Static capabilities string set by htspe_init(); valid for the process
     lifetime, do not free. */
@@ -154,7 +153,7 @@ extern void htspe_uninit(void);
 /** Run the external-parser callbacks for the object described by str.
     Returns the parse callback result (>=0) on a handled object, or -1 if no
     module claimed it or its wrapper_name is blacklisted. */
-extern int hts_parse_externals(htsmoduleStruct * str);
+extern int hts_parse_externals(htsmoduleStruct *str);
 
 /** Nonzero if IPv6 support was compiled in (== HTS_INET6). */
 extern int V6_is_available;

src/htsname.c

@@ -41,6 +41,10 @@ Please visit our Website: http://www.httrack.com
 #include "htstools.h"
 #include "htscharset.h"
 #include "htsencoding.h"
+#include "htssniff.h"
+#if HTS_USEZLIB
+#include "htszlib.h"
+#endif
 #include <ctype.h>
 
 #define ADD_STANDARD_PATH \
@@ -70,31 +74,6 @@ static const char *hts_tbdev[] = {
   ""
 };
 
-#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET() do { \
-  int prev = opt->state._hts_in_html_parsing; \
-  while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
-    opt->state. _hts_in_html_parsing = 6; \
-    /* Wait .. */ \
-    back_wait(sback,opt,cache,0); \
-    /* Transfer rate */ \
-    engine_stats(); \
-    /* Refresh various stats */ \
-    HTS_STAT.stat_nsocket=back_nsoc(sback); \
-    HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
-    HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
-    HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
-    HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
-    HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
-    /* Check */ \
-    { \
-      if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,-1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
-        return -1; \
-      } \
-    } \
-  } \
-  opt->state._hts_in_html_parsing = prev; \
-} while(0)
-
 /* Strip all // */
 static void cleanDoubleSlash(char *s) {
   int i, j;
@@ -138,8 +117,191 @@ static void cleanEndingSpaceOrDot(char *s) {
   }
 }
 
-// forme le nom du fichier à sauver (save) à partir de fil et adr
+/* Wire Content-Type vs URL extension: a patchable wire type wins over an
-// système intelligent, qui renomme en cas de besoin (exemple: deux INDEX.HTML et index.html)
+   unspecific ext, the HTS_UNKNOWN_MIME sentinel keeps a specific non-HTML ext
+   (#267 guard), a declared disagreement is CONTESTED (sniffed below). */
+typedef enum wire_verdict {
+  WIRE_KEEPS_EXT,
+  WIRE_WINS,
+  WIRE_CONTESTED
+} wire_verdict;
+
+static wire_verdict wire_ext_verdict(httrackp *opt, const char *wiremime,
+                                     const char *file, char *urlmime,
+                                     size_t urlmime_size) {
+  if (may_unknown2(opt, wiremime, file))
+    return WIRE_KEEPS_EXT; /* type kept verbatim (keep-list / bogus-multiple) */
+  urlmime[0] = '\0';
+  /* type implied by the URL extension, only when confidently known (flag 0) */
+  if (!get_httptype_sized(opt, urlmime, urlmime_size, file, 0))
+    return WIRE_WINS; /* URL ext implies no known type */
+  if (strfield2(wiremime, urlmime))
+    return WIRE_KEEPS_EXT; /* agreement (no .htm->.html churn) */
+  if (!is_hypertext_mime(opt, urlmime, file) &&
+      strfield2(wiremime, HTS_UNKNOWN_MIME))
+    return WIRE_KEEPS_EXT; /* no declared type */
+  return WIRE_CONTESTED;
+}
+
+/* Optional evidence for a contested wire-vs-ext verdict. */
+typedef struct sniff_src {
+  struct_back *sback;       /* live backing (looked up by adr/fil) */
+  const lien_back *headers; /* snapshot: r.adr, else the url_sav file */
+  const char *adr, *fil;
+  const char *prev_save; /* previous run's save name (cache X-Save) */
+} sniff_src;
+
+#if HTS_USEZLIB
+/* Inflate the head of a gzip/zlib stream; 0 when undecodable. */
+static size_t sniff_inflate_head(const void *in, size_t in_len, void *out,
+                                 size_t out_len) {
+  z_stream zs;
+  size_t n = 0;
+  int err;
+
+  memset(&zs, 0, sizeof(zs));
+  if (inflateInit2(&zs, 47) != Z_OK) /* 47: gzip or zlib, autodetected */
+    return 0;
+  zs.next_in = (const Bytef *) in;
+  zs.avail_in = (uInt) in_len;
+  zs.next_out = (Bytef *) out;
+  zs.avail_out = (uInt) out_len;
+  err = inflate(&zs, Z_SYNC_FLUSH);
+  if (err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR)
+    n = out_len - zs.avail_out;
+  inflateEnd(&zs);
+  return n;
+}
+#endif
+
+static size_t sniff_read_head(const char *path, void *buf, size_t len) {
+  char catbuff[CATBUFF_SIZE];
+  FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "rb");
+  size_t n = 0;
+
+  if (fp != NULL) {
+    n = fread(buf, 1, len, fp);
+    fclose(fp);
+  }
+  return n;
+}
+
+/* Body head of one slot: memory, else its flushed on-disk file (url_sav, or
+   tmpfile for a compressed stream); inflated so the sniff sees the final body.
+ */
+static size_t sniff_slot_head(const lien_back *slot, void *buf, size_t len) {
+  const htsblk *const r = &slot->r;
+  size_t n = 0;
+
+  if (r->adr != NULL && r->size > 0) {
+    n = (size_t) r->size < len ? (size_t) r->size : len;
+    memcpy(buf, r->adr, n);
+  } else {
+    if (r->out != NULL)
+      fflush(r->out);
+    if (slot->url_sav[0] != '\0')
+      n = sniff_read_head(slot->url_sav, buf, len);
+    if (n == 0 && slot->tmpfile != NULL && slot->tmpfile[0] != '\0')
+      n = sniff_read_head(slot->tmpfile, buf, len);
+  }
+  if (n > 0 && r->compressed) {
+#if HTS_USEZLIB
+    unsigned char raw[HTS_SNIFF_LEN];
+
+    if (n > sizeof(raw))
+      n = sizeof(raw);
+    memcpy(raw, buf, n);
+    n = sniff_inflate_head(raw, n, buf, len);
+#else
+    n = 0;
+#endif
+  }
+  return n;
+}
+
+/* Up to len leading body bytes; 0 when unavailable, and always in
+   non-delayed mode (its HEAD-probe first run couldn't sniff either). */
+static size_t sniff_body_head(httrackp *opt, const sniff_src *src, void *buf,
+                              size_t len) {
+  size_t n = 0;
+
+  if (src == NULL || opt->savename_delayed == HTS_SAVENAME_DELAYED_NONE)
+    return 0;
+  /* live backing slot: a snapshot (back_copy_static) loses r.adr/r.out */
+  if (src->sback != NULL && src->adr != NULL && src->fil != NULL) {
+    const int b = back_index(opt, src->sback, src->adr, src->fil, NULL);
+
+    if (b >= 0)
+      n = sniff_slot_head(&src->sback->lnk[b], buf, len);
+  }
+  if (n == 0 && src->headers != NULL)
+    n = sniff_slot_head(src->headers, buf, len);
+  return n;
+}
+
+/* Contested verdicts: magic proving the URL ext keeps it, else wire wins. */
+static int wire_patches_ext(httrackp *opt, const sniff_src *src,
+                            const char *wiremime, const char *file) {
+  char urlmime[256];
+
+  switch (wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime))) {
+  case WIRE_KEEPS_EXT:
+    return 0;
+  case WIRE_WINS:
+    return 1;
+  case WIRE_CONTESTED:
+    break;
+  }
+  if (src != NULL) {
+    if (hts_sniff_mime_known(urlmime)) {
+      unsigned char head[HTS_SNIFF_LEN];
+      const size_t n = sniff_body_head(opt, src, head, sizeof(head));
+
+      if (n > 0)
+        return hts_sniff_mime_consistent(head, n, urlmime) ? 0 : 1;
+    }
+    /* no bytes: reproduce the previous run's verdict (cached X-Save name) */
+    if (src->prev_save != NULL && src->prev_save[0] != '\0') {
+      char prevmime[256];
+
+      prevmime[0] = '\0';
+      if (get_httptype_sized(opt, prevmime, sizeof(prevmime), src->prev_save,
+                             0) &&
+          strfield2(prevmime, urlmime))
+        return 0;
+    }
+  }
+  return 1;
+}
+
+int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime,
+                         const char *file) {
+  char urlmime[256];
+
+  return wiremime != NULL && strnotempty(wiremime) &&
+         wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime)) ==
+             WIRE_CONTESTED &&
+         hts_sniff_mime_known(urlmime);
+}
+
+/* Wire-metadata name change: a Content-Disposition filename wins (returns 2),
+   else the declared type's ext when wire_patches_ext() allows (returns 1),
+   else 0. ext receives the new extension or replacement filename. */
+static int resolve_extension(httrackp *opt, const sniff_src *src,
+                             const char *cdispo, const char *contenttype,
+                             const char *fil, char *ext, size_t ext_size) {
+  if (strnotempty(cdispo)) {
+    strlcpybuff(ext, cdispo, ext_size);
+    return 2;
+  }
+  if (wire_patches_ext(opt, src, contenttype, fil) &&
+      give_mimext(ext, ext_size, contenttype))
+    return 1;
+  return 0;
+}
+
+// Build the local save name (save) from adr/fil; renames on collision
+// (e.g. INDEX.HTML vs index.html).
 int url_savename(lien_adrfilsave *const afs,
                  lien_adrfil *const former,
                  const char *referer_adr, const char *referer_fil, 
@@ -169,6 +331,13 @@ int url_savename(lien_adrfilsave *const afs,
   // copy of fil, used for lookups (see urlhack)
   const char *normadr = adr;
   const char *normfil = fil_complete;
+  /* query keys to strip for this URL (NULL = none); decoupled from urlhack */
+  char BIGSTK stripkeys[HTS_URLMAXSIZE];
+  const char *const strip =
+      StringNotEmpty(opt->strip_query)
+          ? hts_query_strip_keys(StringBuff(opt->strip_query), adr,
+                                 fil_complete, stripkeys, sizeof(stripkeys))
+          : NULL;
   const char *const print_adr = jump_protocol_const(adr);
   const char *start_pos = NULL, *nom_pos = NULL, *dot_pos = NULL;     // Position nom et point
 
@@ -184,10 +353,11 @@ int url_savename(lien_adrfilsave *const afs,
 
   /* 8-3 ? */
   switch (opt->savename_83) {
-  case 1:                      // 8-3
+  case HTS_SAVENAME_83_DOS: // 8-3
     max_char = 8;
     break;
-  case 2:                      // Level 2 File names may be up to 31 characters.
+  case HTS_SAVENAME_83_ISO9660: // Level 2 File names may be up to 31
+                                // characters.
     max_char = 31;
     break;
   default:
@@ -200,9 +370,13 @@ int url_savename(lien_adrfilsave *const afs,
   // www-42.foo.com -> foo.com
   // foo.com/bar//foobar -> foo.com/bar/foobar
   if (opt->urlhack) {
-    // copy of adr (without protocol), used for lookups (see urlhack)
+    // dedup-lookup key; honor the per-feature negatives like htshash.c so
-    normadr = adr_normalized_sized(adr, normadr_, sizeof(normadr_));
+    // distinct URLs keep distinct savenames (else keep normadr = adr)
-    normfil = fil_normalized(fil_complete, normfil_);
+    if (!opt->no_www_dedup)
+      normadr = adr_normalized_sized(adr, normadr_, sizeof(normadr_));
+    normfil =
+        fil_normalized_filtered_ex(fil_complete, normfil_, strip,
+                                   !opt->no_slash_dedup, !opt->no_query_dedup);
   } else {
     if (link_has_authority(adr_complete)) {     // https or other protocols : in "http/" subfolder
       char *pos = strchr(adr_complete, ':');
@@ -215,6 +389,11 @@ int url_savename(lien_adrfilsave *const afs,
         normadr = normadr_;
       }
     }
+    // strip still applies with urlhack off (host left untouched); no // or
+    // query-sort here, to match the hash key (norm_slash/norm_query are 0 when
+    // urlhack is off) so a URL is looked up under the key it was stored with
+    if (strip != NULL)
+      normfil = fil_normalized_filtered_ex(fil_complete, normfil_, strip, 0, 0);
   }
 
   // à afficher sans ftp://
@@ -324,7 +503,10 @@ int url_savename(lien_adrfilsave *const afs,
   }
 
   /* replace shtml to html.. */
-  if (opt->savename_delayed == 2)
+  /* HARD delays every type, except one the user pinned with --assume: honor it
+     immediately (ishtml() consults the user type), no delayed name (#56) */
+  if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD &&
+      !is_userknowntype(opt, fil))
     is_html = -1;               /* ALWAYS delay type */
   else
     is_html = ishtml(opt, fil);
@@ -356,48 +538,36 @@ int url_savename(lien_adrfilsave *const afs,
 
   // si option check_type activée
   if (is_html < 0 && opt->check_type && !ext_chg) {
-    int ishtest = 0;
-
     if (protocol != PROTOCOL_FILE
         && protocol != PROTOCOL_FTP
       ) {
       // tester type avec requète HEAD si on ne connait pas le type du fichier
       if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/')))   // slash doit être html?
-        if (opt->savename_delayed == 2 || (ishtest = ishtml(opt, fil)) < 0) {   // on ne sait pas si c'est un html ou un fichier..
+        if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
+            ishtml(opt, fil) < 0) { // unsure whether it's html or a file
           // lire dans le cache
-          htsblk r = cache_read_including_broken(opt, cache, adr, fil); // test uniquement
+          char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
+          htsblk r;
 
-          if (r.statuscode != -1) {     // pas d'erreur de lecture cache
+          previous_save[0] = '\0';
-            char s[32];
+          r = cache_read_including_broken(opt, cache, adr, fil,
+                                          previous_save); // test uniquement
 
-            s[0] = '\0';
+          if (r.statuscode != -1) { // cache entry read OK
             hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s",
                           adr_complete, fil_complete);
             if (!HTTP_IS_REDIRECT(r.statuscode)) {
-              if (strnotempty(r.cdispo)) {        /* filename given */
+              const sniff_src src = {sback, NULL, adr, fil, previous_save};
-                ext_chg = 2;      /* change filename */
+
-                strcpybuff(ext, r.cdispo);
+              ext_chg = resolve_extension(opt, &src, r.cdispo, r.contenttype,
-              } else if (!may_unknown2(opt, r.contenttype, fil)) {        // on peut patcher à priori?
+                                          fil, ext, sizeof(ext));
-                if (give_mimext(s, sizeof(s),
-                                r.contenttype)) { // recognized extension
-                  ext_chg = 1;
-                  strcpybuff(ext, s);
-                }
-              }
             }
-#ifdef DEFAULT_BIN_EXT
+          } else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD &&
-            // no extension and potentially bogus
+                     is_userknowntype(opt, fil)) { /* PATCH BY BRIAN SCHRÖDER.
-            else if (ishtest == -2) {
+                              Lookup mimetype not only by extension,
-              ext_chg = 1;
+                              but also by filename */
-              strcpybuff(ext, DEFAULT_BIN_EXT + 1);
+            /* Note: "foo.cgi => text/html" means that foo.cgi shall have the
-            }
+               text/html MIME file type, that is, ".html" */
-#endif
-            //
-          } else if (opt->savename_delayed != 2 && is_userknowntype(opt, fil)) {        /* PATCH BY BRIAN SCHRÖDER. 
-                                                                                           Lookup mimetype not only by extension, 
-                                                                                           but also by filename */
-            /* Note: "foo.cgi => text/html" means that foo.cgi shall have the text/html MIME file type,
-               that is, ".html" */
             char BIGSTK mime[1024];
 
             mime[0] = ext[0] = '\0';
@@ -408,24 +578,20 @@ int url_savename(lien_adrfilsave *const afs,
               }
             }
           }
-          // note: if savename_delayed is enabled, the naming will be temporary (and slightly invalid!)
+          // note: if savename_delayed is enabled, the naming will be temporary
-          // note: if we are about to stop (opt->state.stop), back_add() will fail later
+          // (and slightly invalid!)
-          else if (opt->savename_delayed != 0 && !opt->state.stop) {
+          //
-            // Check if the file is ready in backing. We basically take the same logic as later.
+          // note: if we are about to stop (opt->state.stop), back_add() will
-            // FIXME: we should cleanup and factorize this unholy mess
+          // fail later
+          else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_NONE &&
+                   !opt->state.stop) {
+            // Check if the file is ready in backing.
             if (headers != NULL && headers->status >= 0 && !is_redirect) {
-              if (strnotempty(headers->r.cdispo)) {        /* filename given */
+              const sniff_src src = {sback, headers, adr, fil, NULL};
-                ext_chg = 2;      /* change filename */
+
-                strcpybuff(ext, headers->r.cdispo);
+              ext_chg = resolve_extension(opt, &src, headers->r.cdispo,
-              } else if (!may_unknown2(opt, headers->r.contenttype, headers->url_fil)) {    // on peut patcher à priori? (pas interdit ou pas de type)
+                                          headers->r.contenttype,
-                char s[16];
+                                          headers->url_fil, ext, sizeof(ext));
-                if (give_mimext(
-                        s, sizeof(s),
-                        headers->r.contenttype)) { // recognized extension
-                  ext_chg = 1;
-                  strcpybuff(ext, s);
-                }
-              }
             }
             else if (mime_type != NULL) {
               ext[0] = '\0';
@@ -443,13 +609,6 @@ int url_savename(lien_adrfilsave *const afs,
                   if (!may_unknown2(opt, mime_type, fil)) {
                     ext_chg = 1;
                   }
-#ifdef DEFAULT_BIN_EXT
-                  // no extension and potentially bogus
-                  else if (ishtml(opt, fil) == -2) {
-                    ext_chg = 1;
-                    strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-                  }
-#endif
                 } else {
                   ext_chg = 0;
                 }
@@ -468,11 +627,10 @@ int url_savename(lien_adrfilsave *const afs,
             int has_been_moved = 0;
             lien_adrfil current;
 
-            /* Ensure we don't use too many sockets by using a "testing" one
+            /* Wait for an available test slot, honoring the connection limits
-               If we have only 1 simultaneous connection authorized, wait for pending download
-               Wait for an available slot 
              */
-            URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
+            if (!hts_wait_available_socket(sback, opt, cache, ptr))
+              return -1;
 
             /* Rock'in */
             current.adr[0] = current.fil[0] = '\0';
@@ -502,24 +660,11 @@ int url_savename(lien_adrfilsave *const afs,
                   if (ptr >= 0) {
                     back_fillmax(sback, opt, cache, ptr, numero_passe);
                   }
-                  // on est obligé d'appeler le shell pour le refresh..
+                  if (!hts_loop_tick(sback, opt, b, ptr)) {
-                  // Transfer rate
-                  engine_stats();
-
-                  // Refresh various stats
-                  HTS_STAT.stat_nsocket = back_nsoc(sback);
-                  HTS_STAT.stat_errors = fspc(opt, NULL, "error");
-                  HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
-                  HTS_STAT.stat_infos = fspc(opt, NULL, "info");
-                  HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
-                  HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
-
-                  if (!RUN_CALLBACK7
-                      (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
-                       (int) (time_local() - HTS_STAT.stat_timestart),
-                       &HTS_STAT)) {
                     return -1;
-                  } else if (opt->state._hts_cancel || !back_checkmirror(opt)) {        // cancel 2 ou 1 (cancel parsing)
+                  } else if (opt->state._hts_cancel ||
+                             !back_checkmirror(
+                                 opt)) { // cancel level 2 or 1 (cancel parsing)
                     back_delete(opt, cache, sback, b);  // cancel test
                     stop_looping = 1;
                   }
@@ -584,8 +729,9 @@ int url_savename(lien_adrfilsave *const afs,
                                                 "Loop with HEAD request (during prefetch) at %s%s",
                                                 current.adr, current.fil);
                                 }
-                                // Ajouter
+                                if (!hts_wait_available_socket(sback, opt,
-                                URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
+                                                               cache, ptr))
+                                  return -1;
                                 if (back_add(sback, opt, cache, moved.adr, moved.fil, methode, referer_adr, referer_fil, 1) != -1) {        // OK
                                   hts_log_print(opt, LOG_DEBUG,
                                                 "(during prefetch) %s (%d) to link %s at %s%s",
@@ -633,34 +779,16 @@ int url_savename(lien_adrfilsave *const afs,
                 if (!has_been_moved) {
                   if (back[b].r.statuscode != -10) {    // erreur
                     if (strnotempty(back[b].r.contenttype) == 0)
-                      strcpybuff(back[b].r.contenttype, "text/html");   // message d'erreur en html
+                      strcpybuff(back[b].r.contenttype,
+                                 HTS_UNKNOWN_MIME); // no declared type
                     // Finalement on, renvoie un erreur, pour ne toucher à rien dans le code
                     // libérer emplacement backing
                   }
 
-                  {             // pas d'erreur, changer type?
+                  // no error: change the type?
-                    char s[16];
+                  ext_chg = resolve_extension(
-
+                      opt, NULL, back[b].r.cdispo, back[b].r.contenttype,
-                    s[0] = '\0';
+                      back[b].url_fil, ext, sizeof(ext));
-                    if (strnotempty(back[b].r.cdispo)) {        /* filename given */
-                      ext_chg = 2;      /* change filename */
-                      strcpybuff(ext, back[b].r.cdispo);
-                    } else if (!may_unknown2(opt, back[b].r.contenttype, back[b].url_fil)) {    // on peut patcher à priori? (pas interdit ou pas de type)
-                      if (give_mimext(
-                              s, sizeof(s),
-                              back[b].r.contenttype)) { // recognized extension
-                        ext_chg = 1;
-                        strcpybuff(ext, s);
-                      }
-                    }
-#ifdef DEFAULT_BIN_EXT
-                    // no extension and potentially bogus
-                    else if (ishtest == -2) {
-                      ext_chg = 1;
-                      strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-                    }
-#endif
-                  }
                 }
                 // FIN Si non déplacé, forcer type?
 
@@ -698,7 +826,7 @@ int url_savename(lien_adrfilsave *const afs,
             }
             // restaurer
             opt->state._hts_in_html_parsing = hihp;
-          }                     // caché?
+          } // caché?
         }
     }
   }
@@ -717,9 +845,9 @@ int url_savename(lien_adrfilsave *const afs,
         strcatbuff(fil, DEFAULT_HTML);  // nommer page par défaut (à priori ici html depuis un proxy http)
     }
   }
-  // Changer extension?
+  // Change the extension? e.g. php3 saved as html, cgi as html or gif/xbm
-  // par exemple, php3 sera sauvé en html, cgi en html ou gif, xbm etc.. selon les cas
+  // depending on the resolved type.
-  if (ext_chg && !opt->no_type_change) {                // changer ext
+  if (ext_chg && !opt->no_type_change) {
     char *a = fil + strlen(fil) - 1;
 
     if ((opt->debug > 1) && (opt->log != NULL)) {
@@ -731,11 +859,19 @@ int url_savename(lien_adrfilsave *const afs,
                       adr_complete, fil_complete, ext);
     }
     if (ext_chg == 1) {
+      // Cut the old extension only when it is empty (a bare trailing dot), the
+      // new one, or a recognized one; an unknown trailing ".token" (e.g.
+      // /article-1.884291, #115) is part of the name, not an extension.
+      const char *const old_ext = get_ext(catbuff, sizeof(catbuff), fil);
+      const int known_ext = !*old_ext || strfield2(old_ext, ext) ||
+                            is_knowntype(opt, fil) || is_dyntype(old_ext) ||
+                            ishtml_ext(old_ext) != -1;
+
       while((a > fil) && (*a != '.') && (*a != '/'))
         a--;
-      if (*a == '.')
+      if (*a == '.' && known_ext)
-        *a = '\0';              // couper
+        *a = '\0';          // cut
-      strcatbuff(fil, ".");     // recopier point
+      strcatbuff(fil, "."); // re-add the dot
     } else {
       while((a > fil) && (*a != '/'))
         a--;
@@ -743,7 +879,7 @@ int url_savename(lien_adrfilsave *const afs,
         a++;
       *a = '\0';
     }
-    strcatbuff(fil, ext);       // copier ext/nom
+    strcatbuff(fil, ext); // append ext/name
   }
   // Rechercher premier / et dernier .
   {
@@ -1190,13 +1326,13 @@ int url_savename(lien_adrfilsave *const afs,
   // Not used anymore unless non-delayed types.
   // de même en cas de manque d'extension on en place une de manière forcée..
   // cela évite les /chez/toto et les /chez/toto/index.html incompatibles
-  if (opt->savename_type != -1 && opt->savename_delayed != 2) {
+  if (opt->savename_type != -1 &&
+      opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD) {
     char *a = afs->save + strlen(afs->save) - 1;
 
     while((a > afs->save) && (*a != '.') && (*a != '/'))
       a--;
-    if (*a != '.') {            // agh pas de point
+    if (*a != '.') {                         // agh pas de point
-      //strcatbuff(save,".none");                 // a éviter
       strcatbuff(afs->save, ".html");        // préférable!
       hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",
                     adr_complete, fil_complete, afs->save);
@@ -1236,31 +1372,21 @@ int url_savename(lien_adrfilsave *const afs,
     size_t i;
     for(i = 0 ; afs->save[i] != '\0' ; i++) {
       unsigned char c = (unsigned char) afs->save[i];
-      if (c < 32      // control
+      if (c < 32       // control
-        || c == 127   // unwise
+          || c == 127  // unwise
-        || c == '~'   // unix unwise
+          || c == '~'  // unix unwise
-        || c == '\\'  // windows separator
+          || c == '\\' // windows separator
-        || c == ':'   // windows forbidden
+          || c == ':'  // windows forbidden
-        || c == '*'   // windows forbidden
+          || c == '*'  // windows forbidden
-        || c == '?'   // windows forbidden
+          || c == '?'  // windows forbidden
-        || c == '\"'  // windows forbidden
+          || c == '\"' // windows forbidden
-        || c == '<'   // windows forbidden
+          || c == '<'  // windows forbidden
-        || c == '>'   // windows forbidden
+          || c == '>'  // windows forbidden
-        || c == '|'   // windows forbidden
+          || c == '|'  // windows forbidden
-        //|| c == '@' // ?
+          //|| c == '@' // ?
-        ||
+          || (opt->savename_83 == HTS_SAVENAME_83_ISO9660 // CDROM
-          (
+              && (c == '-' || c == '=' || c == '+'))) {
-            opt->savename_83 == 2 // CDROM
+        afs->save[i] = '_';
-            &&
-            (
-              c == '-'
-              || c == '='
-              || c == '+'
-            )
-          )
-        )
-      {
-         afs->save[i] = '_';
       }
     }
   }
@@ -1344,17 +1470,6 @@ int url_savename(lien_adrfilsave *const afs,
   fil_simplifie(afs->save);
 
   /* convert name to UTF-8 ? Note: already done while parsing. */
-  //if (charset != NULL && charset[0] != '\0') {
-  //  char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
-
-  //  if (s != NULL) {
-  //    hts_log_print(opt, LOG_DEBUG,
-  //                  "engine: save-name: charset conversion from '%s' to '%s' using charset '%s'",
-  //                  save, s, charset);
-  //    strcpybuff(save, s);
-  //    free(s);
-  //  }
-  //}
 
   /* callback */
   RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr,
@@ -1502,10 +1617,6 @@ int url_savename(lien_adrfilsave *const afs,
         int sameAdr = (strfield2(heap(i)->adr, normadr) != 0);
         int sameFil;
 
-        // NO - URL hack is only for stripping // and www.
-        //if (opt->urlhack != 0)
-        //  sameFil = ( strfield2(heap(i)->fil, normfil) != 0);
-        //else
         sameFil = (strcmp(heap(i)->fil, normfil) == 0);
         if (sameAdr && sameFil) {       // ok c'est le même lien, adresse déja définie
           /* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */
@@ -1521,7 +1632,8 @@ int url_savename(lien_adrfilsave *const afs,
           char *a = afs->save + strlen(afs->save) - 1;
           char *b;
           int n = 2;
-          char collisionSeparator = ((opt->savename_83 != 2) ? '-' : '_');
+          char collisionSeparator =
+              ((opt->savename_83 != HTS_SAVENAME_83_ISO9660) ? '-' : '_');
 
           tempo[0] = '\0';
 
@@ -1570,17 +1682,13 @@ int url_savename(lien_adrfilsave *const afs,
 
           strcpybuff(afs->save, tempo);
 
-          //printf("switched: %s\n",save);
-
         }                       // if
       }
 #if DEBUG_SAVENAME
       printf("\nEnd search, %s\n", fil_complete);
 #endif
-    } while(!nom_ok);
+    } while (!nom_ok);
-
   }
-  //printf("'%s' %s %s\n",save,adr,fil);
 
   return 0;
 }
@@ -1686,10 +1794,10 @@ char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
     StringBuff(opt->path_log), digest_filename);
 }
 
-/* remove refname if any */
+/* remove refname if any; HTS_TRUE if it was removed */
-void url_savename_refname_remove(httrackp * opt, const char *adr,
+hts_boolean url_savename_refname_remove(httrackp *opt, const char *adr,
-                                 const char *fil) {
+                                        const char *fil) {
   char *filename = url_savename_refname_fullpath(opt, adr, fil);
 
-  (void) UNLINK(filename);
+  return UNLINK(filename) == 0 ? HTS_TRUE : HTS_FALSE;
 }

src/htsname.h

@@ -100,12 +100,15 @@ void standard_name(char *b, size_t bsize, const char *dot_pos,
                    const char *nom_pos, const char *fil_complete,
                    int short_ver);
 void url_savename_addstr(char *d, const char *s);
+/* Contested wire-vs-ext verdict that a body sniff could settle (htssniff.h). */
+int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime, const char *file);
 char *url_md5(char *digest_buffer, const char *fil_complete);
 void url_savename_refname(const char *adr, const char *fil, char *filename);
 char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
                                     const char *fil);
-void url_savename_refname_remove(httrackp * opt, const char *adr,
+/* Remove the temp-ref for (adr,fil); HTS_TRUE if it was removed. */
-                                 const char *fil);
+hts_boolean url_savename_refname_remove(httrackp *opt, const char *adr,
+                                        const char *fil);
 #endif
 
 #endif

src/htsnet.h

@@ -112,10 +112,10 @@ struct SOCaddr {
 
 /** Pointer to the port field (network byte order) for the active family.
     Asserts on NULL or an unset/unknown family. */
-static HTS_INLINE HTS_UNUSED in_port_t* SOCaddr_sinport_(SOCaddr *const addr,
+static HTS_INLINE HTS_UNUSED in_port_t *
-                                                         const char *file, const int line) {
+SOCaddr_sinport_(SOCaddr *const addr, const char *file, const int line) {
   assertf_(addr != NULL, file, line);
-  switch(addr->m_addr.sa.sa_family) {
+  switch (addr->m_addr.sa.sa_family) {
   case AF_INET:
     return &addr->m_addr.in.sin_port;
     break;
@@ -125,7 +125,7 @@ static HTS_INLINE HTS_UNUSED in_port_t* SOCaddr_sinport_(SOCaddr *const addr,
     break;
 #endif
   default:
-    assertf_(! "invalid structure", file, line);
+    assertf_(!"invalid structure", file, line);
     return 0;
     break;
   }
@@ -133,10 +133,11 @@ static HTS_INLINE HTS_UNUSED in_port_t* SOCaddr_sinport_(SOCaddr *const addr,
 
 /** Length of the active sockaddr (sockaddr_in or sockaddr_in6), or 0 if the
     family is unset/unknown. The 0 case doubles as the "not valid" test. */
-static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_size_(const SOCaddr*const addr,
+static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_size_(const SOCaddr *const addr,
-                                                     const char *file, const int line) {
+                                                     const char *file,
+                                                     const int line) {
   assertf_(addr != NULL, file, line);
-  switch(addr->m_addr.sa.sa_family) {
+  switch (addr->m_addr.sa.sa_family) {
   case AF_INET:
     return sizeof(addr->m_addr.in);
     break;
@@ -152,8 +153,8 @@ static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_size_(const SOCaddr*const addr,
 }
 
 /** Reset to the unset state (family AF_UNSPEC), making the address invalid. */
-static HTS_INLINE HTS_UNUSED void SOCaddr_clear_(SOCaddr*const addr,
+static HTS_INLINE HTS_UNUSED void
-                                                 const char *file, const int line) {
+SOCaddr_clear_(SOCaddr *const addr, const char *file, const int line) {
   assertf_(addr != NULL, file, line);
   addr->m_addr.sa.sa_family = AF_UNSPEC;
 }
@@ -191,14 +192,16 @@ static HTS_INLINE HTS_UNUSED void SOCaddr_clear_(SOCaddr*const addr,
 
 /** Set the port (host-order argument, stored network-order) on the active
  * family. */
-#define SOCaddr_initport(server, port) do { \
+#define SOCaddr_initport(server, port)                                         \
-  SOCaddr_sinport(server) = htons((in_port_t) (port)); \
+  do {                                                                         \
-} while(0)
+    SOCaddr_sinport(server) = htons((in_port_t) (port));                       \
+  } while (0)
 
 /** Initialize as an all-zero IPv4 wildcard (INADDR_ANY) address; returns its
     sockaddr length. */
-static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_initany_(SOCaddr*const addr,
+static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_initany_(SOCaddr *const addr,
-                                                        const char *file, const int line) {
+                                                        const char *file,
+                                                        const int line) {
   assertf_(addr != NULL, file, line);
   memset(&addr->m_addr.in, 0, sizeof(addr->m_addr.in));
   addr->m_addr.in.sin_family = AF_INET;
@@ -206,17 +209,20 @@ static HTS_INLINE HTS_UNUSED socklen_t SOCaddr_initany_(SOCaddr*const addr,
 }
 
 /** Initialize server as an IPv4 wildcard (INADDR_ANY) address. */
-#define SOCaddr_initany(server) do { \
+#define SOCaddr_initany(server)                                                \
-  SOCaddr_initany_(&(server), __FILE__, __LINE__); \
+  do {                                                                         \
-} while(0)
+    SOCaddr_initany_(&(server), __FILE__, __LINE__);                           \
+  } while (0)
 
 /** Populate server from data. data_size selects the source form: a full
     sockaddr_in / sockaddr_in6, or a raw 4-byte (IPv4) / 16-byte (IPv6) address
     with port zeroed. Any other size leaves an AF_INET shell. Returns the
     resulting sockaddr length. */
-static HTS_UNUSED socklen_t SOCaddr_copyaddr_(SOCaddr*const server, 
+static HTS_UNUSED socklen_t SOCaddr_copyaddr_(SOCaddr *const server,
-                                              const void *data, const size_t data_size,
+                                              const void *data,
-                                              const char *file, const int line) {
+                                              const size_t data_size,
+                                              const char *file,
+                                              const int line) {
   assertf_(server != NULL, file, line);
   assertf_(data != NULL, file, line);
 
@@ -248,32 +254,35 @@ static HTS_UNUSED socklen_t SOCaddr_copyaddr_(SOCaddr*const server,
 
 /** Copy hpaddr (length hpsize) into server, writing the result length into the
     lvalue server_len (int). See SOCaddr_copyaddr_ for accepted forms. */
-#define SOCaddr_copyaddr(server, server_len, hpaddr, hpsize) do { \
+#define SOCaddr_copyaddr(server, server_len, hpaddr, hpsize)                   \
-  server_len = (int) SOCaddr_copyaddr_(&(server), hpaddr, hpsize, __FILE__, __LINE__); \
+  do {                                                                         \
-} while(0)
+    server_len = (int) SOCaddr_copyaddr_(&(server), hpaddr, hpsize, __FILE__,  \
+                                         __LINE__);                            \
+  } while (0)
 
 /** Like SOCaddr_copyaddr but discards the result length. */
-#define SOCaddr_copyaddr2(server, hpaddr, hpsize) do { \
+#define SOCaddr_copyaddr2(server, hpaddr, hpsize)                              \
-  (void) SOCaddr_copyaddr_(&(server), hpaddr, hpsize, __FILE__, __LINE__); \
+  do {                                                                         \
-} while(0)
+    (void) SOCaddr_copyaddr_(&(server), hpaddr, hpsize, __FILE__, __LINE__);   \
+  } while (0)
 
 /** Copy one SOCaddr (src) into another (dest), preserving family and port. */
-#define SOCaddr_copy_SOCaddr(dest, src) do { \
+#define SOCaddr_copy_SOCaddr(dest, src)                                        \
-  SOCaddr_copyaddr_(&(dest), &(src).m_addr.sa, SOCaddr_size(src), __FILE__, __LINE__); \
+  do {                                                                         \
-} while(0)
+    SOCaddr_copyaddr_(&(dest), &(src).m_addr.sa, SOCaddr_size(src), __FILE__,  \
+                      __LINE__);                                               \
+  } while (0)
 
 /** Write the numeric (dotted/colon) host of ss into namebuf (capacity
     namebuflen), scope id stripped. On failure namebuf becomes "". */
-static HTS_UNUSED void SOCaddr_inetntoa_(char *namebuf, size_t namebuflen, 
+static HTS_UNUSED void SOCaddr_inetntoa_(char *namebuf, size_t namebuflen,
-                                         SOCaddr *const ss,
+                                         SOCaddr *const ss, const char *file,
-                                         const char *file, const int line) {
+                                         const int line) {
   assertf_(namebuf != NULL, file, line);
   assertf_(ss != NULL, file, line);
 
-  if (getnameinfo(&ss->m_addr.sa, sizeof(ss->m_addr),
+  if (getnameinfo(&ss->m_addr.sa, sizeof(ss->m_addr), namebuf, namebuflen, NULL,
-                  namebuf, namebuflen, 
+                  0, NI_NUMERICHOST) == 0) {
-                  NULL, 0, 
-                  NI_NUMERICHOST) == 0) {
     /* remove scope id(s) */
     char *const pos = strchr(namebuf, '%');
     if (pos != NULL) {
@@ -285,15 +294,32 @@ static HTS_UNUSED void SOCaddr_inetntoa_(char *namebuf, size_t namebuflen,
 }
 
 /** Numeric host of ss into namebuf (capacity namebuflen); "" on failure. */
-#define SOCaddr_inetntoa(namebuf, namebuflen, ss) \
+#define SOCaddr_inetntoa(namebuf, namebuflen, ss)                              \
   SOCaddr_inetntoa_(namebuf, namebuflen, &(ss), __FILE__, __LINE__)
 
 /** Single-char family tag: '1' for IPv4, '2' otherwise (used in the cache). */
-#define SOCaddr_getproto(ss) ( SOCaddr_size(ss) == sizeof(struct sockaddr_in) ? '1' : '2')
+#define SOCaddr_getproto(ss)                                                   \
+  (SOCaddr_size(ss) == sizeof(struct sockaddr_in) ? '1' : '2')
 
 /** Length type for socket APIs (getsockname, accept, ...). */
 typedef socklen_t SOClen;
 
+#if HTS_INET6 != 0
+/** Resolver backend: getaddrinfo/freeaddrinfo as a swappable pair, so the
+    self-test can script DNS answers (families, multiplicity, errors)
+    in-process. The free function must match its getaddrinfo (a fake allocates
+    its own chain), hence the pair. */
+typedef struct hts_resolver_backend {
+  int (*getaddrinfo)(const char *node, const char *service,
+                     const struct addrinfo *hints, struct addrinfo **res);
+  void (*freeaddrinfo)(struct addrinfo *res);
+} hts_resolver_backend;
+
+/** Install a resolver backend for the process; NULL restores the libc default.
+    Test-only seam, not thread-safe; callers must serialize against resolves. */
+void hts_dns_set_resolver_backend(const hts_resolver_backend *backend);
+#endif
+
 #ifdef __cplusplus
 }
 #endif

src/htsopt.h

@@ -72,6 +72,7 @@ typedef struct String String;
 #endif
 #ifndef HTS_DEF_STRUCT_String
 #define HTS_DEF_STRUCT_String
+
 struct String {
   char *buffer_;
   size_t length_;
@@ -80,7 +81,7 @@ struct String {
 #endif
 
 /* Defines */
-#define CATBUFF_SIZE (STRING_SIZE*2*2)
+#define CATBUFF_SIZE (STRING_SIZE * 2 * 2)
 
 #define STRING_SIZE 2048
 
@@ -104,11 +105,10 @@ typedef struct htsfilters htsfilters;
 struct htsfilters {
   char ***filters; /**< pointer to the +/-pattern filter array */
   int *filptr;     /**< pointer to the current filter count */
-  // int*    filter_max;
 };
 
 /* User callbacks chain */
-typedef int (*htscallbacksfncptr) (void);
+typedef int (*htscallbacksfncptr)(void);
 
 typedef struct htscallbacks htscallbacks;
 
@@ -179,6 +179,7 @@ typedef struct lien_url lien_url;
 
 #ifndef HTS_DEF_DEFSTRUCT_hts_log_type
 #define HTS_DEF_DEFSTRUCT_hts_log_type
+
 typedef enum hts_log_type {
   LOG_PANIC,
   LOG_ERROR,
@@ -239,7 +240,7 @@ struct htsoptstate {
   char *userhttptype;
   int verif_backblue_done; /**< backblue.gif/fade.gif already emitted */
   int verif_external_status;
-  t_dnscache *dns_cache;  /**< DNS resolution cache */
+  coucal dns_cache; /**< DNS resolution cache: hostname -> t_dnscache record */
   int dns_cache_nthreads; /**< number of in-flight DNS resolver threads */
   /* HTML parsing state */
   char _hts_errmsg[HTS_CDLMAXSIZE + 256]; /**< last engine error message */
@@ -278,13 +279,113 @@ struct htslibhandles {
 
 /* Javascript parser flags */
 typedef enum htsparsejava_flags {
-  HTSPARSE_NONE = 0,            // don't parse
+  HTSPARSE_NONE = 0,          // don't parse
-  HTSPARSE_DEFAULT = 1,         // parse default (all)
+  HTSPARSE_DEFAULT = 1,       // parse default (all)
-  HTSPARSE_NO_CLASS = 2,        // don't parse .java
+  HTSPARSE_NO_CLASS = 2,      // don't parse .java
-  HTSPARSE_NO_JAVASCRIPT = 4,   // don't parse .js
+  HTSPARSE_NO_JAVASCRIPT = 4, // don't parse .js
-  HTSPARSE_NO_AGGRESSIVE = 8    // don't aggressively parse .js or .java
+  HTSPARSE_NO_AGGRESSIVE = 8  // don't aggressively parse .js or .java
 } htsparsejava_flags;
 
+/* Link-rewriting style for saved pages (opt->urlmode). */
+#ifndef HTS_DEF_DEFSTRUCT_hts_urlmode
+#define HTS_DEF_DEFSTRUCT_hts_urlmode
+
+typedef enum hts_urlmode {
+  HTS_URLMODE_ABSOLUTE = 0, /**< absolute URL (http://host/path) everywhere */
+  HTS_URLMODE_ABSOLUTE_FILE = 1, /**< legacy file: form, unused */
+  HTS_URLMODE_RELATIVE = 2,      /**< relative link (default) */
+  HTS_URLMODE_ABSOLUTE_URI = 3,  /**< absolute URI from root (/path) */
+  HTS_URLMODE_KEEP_ORIGINAL = 4, /**< keep the original link, do not rewrite */
+  HTS_URLMODE_TRANSPARENT_PROXY = 5 /**< transparent-proxy URL */
+} hts_urlmode;
+#endif
+
+/* Cache policy for updates and retries (opt->cache). */
+#ifndef HTS_DEF_DEFSTRUCT_hts_cachemode
+#define HTS_DEF_DEFSTRUCT_hts_cachemode
+
+typedef enum hts_cachemode {
+  HTS_CACHE_NONE = 0,       /**< no cache */
+  HTS_CACHE_PRIORITY = 1,   /**< cache takes priority over the network */
+  HTS_CACHE_TEST_UPDATE = 2 /**< check for update before reuse (default) */
+} hts_cachemode;
+#endif
+
+/* Interactive wizard level (opt->wizard). */
+#ifndef HTS_DEF_DEFSTRUCT_hts_wizard
+#define HTS_DEF_DEFSTRUCT_hts_wizard
+
+typedef enum hts_wizard {
+  HTS_WIZARD_NONE = 0, /**< no wizard */
+  HTS_WIZARD_ASK = 1,  /**< wizard asks questions */
+  HTS_WIZARD_AUTO = 2  /**< wizard runs without asking */
+} hts_wizard;
+#endif
+
+/* robots.txt / meta-robots obedience level (opt->robots). */
+#ifndef HTS_DEF_DEFSTRUCT_hts_robots
+#define HTS_DEF_DEFSTRUCT_hts_robots
+
+typedef enum hts_robots {
+  HTS_ROBOTS_NEVER = 0,        /**< ignore robots rules */
+  HTS_ROBOTS_SOMETIMES = 1,    /**< partial obedience (default) */
+  HTS_ROBOTS_ALWAYS = 2,       /**< obey robots rules */
+  HTS_ROBOTS_ALWAYS_STRICT = 3 /**< obey even strict rules */
+} hts_robots;
+#endif
+
+/* What to fetch (opt->getmode bitmask). */
+typedef enum hts_getmode {
+  HTS_GETMODE_HTML = 1 << 0,      /**< save HTML files */
+  HTS_GETMODE_NONHTML = 1 << 1,   /**< save non-HTML files */
+  HTS_GETMODE_HTML_FIRST = 1 << 2 /**< fetch HTML first, then the other files */
+} hts_getmode;
+
+/* Allowed directions in the directory tree (opt->seeker bitmask). */
+typedef enum hts_seeker {
+  HTS_SEEKER_DOWN = 1 << 0, /**< may descend into subdirectories */
+  HTS_SEEKER_UP = 1 << 1    /**< may ascend to parent directories */
+} hts_seeker;
+
+/* opt->travel: link-following scope in the low byte, flags OR'd in above it. */
+typedef enum hts_travel_scope {
+  HTS_TRAVEL_SAME_ADDRESS = 0, /**< stay on the same address (host) */
+  HTS_TRAVEL_SAME_DOMAIN = 1,  /**< stay on the same principal domain */
+  HTS_TRAVEL_SAME_TLD = 2,     /**< stay on the same TLD (e.g. .com) */
+  HTS_TRAVEL_EVERYWHERE = 7,   /**< follow links anywhere on the web */
+  HTS_TRAVEL_TEST_ALL = 1 << 8 /**< also test forbidden URLs (-t) */
+} hts_travel_scope;
+
+/* Mask selecting the scope value out of opt->travel. */
+#define HTS_TRAVEL_SCOPE_MASK 0xff
+
+/* Text progress display detail (opt->verbosedisplay). */
+typedef enum hts_verbosedisplay {
+  HTS_VERBOSE_NONE = 0,   /**< no animated progress display (default) */
+  HTS_VERBOSE_SIMPLE = 1, /**< minimal single-line progress */
+  HTS_VERBOSE_FULL = 2    /**< full animated progress */
+} hts_verbosedisplay;
+
+/* Delayed file-type resolution policy (opt->savename_delayed). */
+typedef enum hts_savename_delayed {
+  HTS_SAVENAME_DELAYED_NONE = 0, /**< resolve the type immediately */
+  HTS_SAVENAME_DELAYED_SOFT = 1, /**< delay the type check when unknown */
+  HTS_SAVENAME_DELAYED_HARD = 2  /**< always delay the type check (default) */
+} hts_savename_delayed;
+
+/* Saved-name length layout (opt->savename_83). */
+typedef enum hts_savename_83 {
+  HTS_SAVENAME_83_LONG = 0,   /**< long file names (default) */
+  HTS_SAVENAME_83_DOS = 1,    /**< DOS 8.3 names (ISO9660 level 1) */
+  HTS_SAVENAME_83_ISO9660 = 2 /**< ISO9660 level 2 names (up to 31 chars) */
+} hts_savename_83;
+
+/* Host-banning triggers (opt->hostcontrol bitmask). */
+typedef enum hts_hostcontrol {
+  HTS_HOSTCONTROL_BAN_TIMEOUT = 1 << 0, /**< ban a timing-out host */
+  HTS_HOSTCONTROL_BAN_SLOW = 1 << 1     /**< ban a too-slow host */
+} hts_hostcontrol;
+
 #ifndef HTS_DEF_FWSTRUCT_lien_buffers
 #define HTS_DEF_FWSTRUCT_lien_buffers
 typedef struct lien_buffers lien_buffers;
@@ -308,111 +409,115 @@ typedef struct httrackp httrackp;
 struct httrackp {
   size_t size_httrackp; /**< size of this structure (version/ABI guard) */
   /* */
-  int wizard;   /**< interactive wizard level (none/full/light) */
+  hts_wizard wizard; /**< interactive wizard level (none/ask/auto) */
-  int flush;    /**< fflush() log files after each write */
+  hts_boolean flush; /**< fflush() log files after each write */
-  int travel;   /**< link-following scope (same domain, etc.) */
+  int travel;        /**< link-following scope (same domain, etc.) */
-  int seeker;   /**< allowed direction: go up and/or down the tree */
+  int seeker;        /**< allowed direction: go up and/or down the tree */
-  int depth;    /**< maximum recursion depth (-rN) */
+  int depth;         /**< maximum recursion depth (-rN) */
-  int extdepth; /**< maximum recursion depth outside the start domain */
+  int extdepth;      /**< maximum recursion depth outside the start domain */
-  int urlmode;  /**< saved-link rewriting style (relative, absolute, etc.) */
+  hts_urlmode
-  int no_type_change;           // do not change file type according to MIME
+      urlmode; /**< saved-link rewriting style (relative, absolute, etc.) */
-  int debug;                    /**< debug logging level */
+  hts_boolean no_type_change; // do not change file type according to MIME
-  int getmode;           /**< what to fetch (HTML, images, ...) bitmask */
+  hts_log_type debug;         /**< debug logging level */
-  FILE *log;             /**< informational log stream; NULL mutes it */
+  int getmode;                /**< what to fetch (HTML, images, ...) bitmask */
-  FILE *errlog;          /**< error log stream; NULL mutes it */
+  FILE *log;                  /**< informational log stream; NULL mutes it */
-  LLint maxsite;         /**< max total bytes for the whole mirror */
+  FILE *errlog;               /**< error log stream; NULL mutes it */
-  LLint maxfile_nonhtml; /**< max bytes per non-HTML file */
+  LLint maxsite;              /**< max total bytes for the whole mirror */
-  LLint maxfile_html;    /**< max bytes per HTML file */
+  LLint maxfile_nonhtml;      /**< max bytes per non-HTML file */
-  int maxsoc;            /**< max simultaneous sockets (-cN) */
+  LLint maxfile_html;         /**< max bytes per HTML file */
-  LLint fragment;        /**< split site after this many bytes */
+  int maxsoc;                 /**< max simultaneous sockets (-cN) */
-  int nearlink;   /**< also fetch images/data adjacent to a page but off-site */
+  LLint fragment;             /**< split site after this many bytes */
-  int makeindex;  /**< build a top-level index.html */
+  hts_tristate
-  int kindex;     /**< build a keyword index */
+      nearlink; /**< also fetch images/data adjacent to a page but off-site */
-  int delete_old; /**< delete locally obsolete files after update */
+  hts_boolean makeindex;  /**< build a top-level index.html */
-  int timeout;    /**< connection timeout in seconds */
+  hts_boolean kindex;     /**< build a keyword index */
-  int rateout;    /**< minimum transfer rate (bytes/s) before abort */
+  hts_tristate delete_old; /**< delete locally obsolete files after update */
-  int maxtime;    /**< max total mirror duration in seconds */
+  int timeout;            /**< connection timeout in seconds */
-  int maxrate;    /**< max transfer rate cap (bytes/s) */
+  int rateout;            /**< minimum transfer rate (bytes/s) before abort */
-  float maxconn;  /**< max connections per second */
+  int maxtime;            /**< max total mirror duration in seconds */
-  int waittime;   /**< scheduled start time (wall-clock seconds) */
+  int maxrate;            /**< max transfer rate cap (bytes/s) */
-  int cache;      /**< cache generation mode */
+  float maxconn;          /**< max connections per second */
-  // int aff_progress;     // progress bar
+  int waittime;           /**< scheduled start time (wall-clock seconds) */
-  int shell;         /**< driven by a shell over stdin/stdout pipes */
+  hts_cachemode cache;    /**< cache generation mode */
+  hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
   t_proxy proxy;     /**< proxy configuration */
-  int savename_83;   /**< force 8.3 (DOS) file names */
+  hts_savename_83
+      savename_83;   /**< saved-name length layout (long/DOS/ISO9660) */
   int savename_type; /**< saved-name layout (original tree, flat, ...) */
   String
       savename_userdef; /**< user-defined name template (e.g. %h%p/%n%q.%t) */
-  int savename_delayed;         // delayed type check
+  hts_savename_delayed savename_delayed; /**< delayed type-check policy */
-  int delayed_cached;           // delayed type check can be cached to speedup updates
+  hts_boolean
-  int mimehtml;                 /**< produce a single MIME/MHTML archive */
+      delayed_cached;   // delayed type check can be cached to speedup updates
-  int user_agent_send;          /**< send a User-Agent header */
+  hts_boolean mimehtml; /**< produce a single MIME/MHTML archive */
-  String user_agent;            /**< User-Agent value (e.g. httrack/1.0) */
+  hts_boolean user_agent_send; /**< send a User-Agent header */
-  String referer;               /**< Referer value to send */
+  String user_agent;           /**< User-Agent value (e.g. httrack/1.0) */
-  String from;                  /**< From value to send */
+  String referer;              /**< Referer value to send */
-  String path_log;              /**< directory for cache and logs */
+  String from;                 /**< From value to send */
-  String path_html;             /**< output directory for the mirror */
+  String path_log;             /**< directory for cache and logs */
+  String path_html;            /**< output directory for the mirror */
   String path_html_utf8; /**< output directory for the mirror, UTF-8 form */
   String path_bin;       /**< directory for HTML templates */
   int retry;             /**< extra retries on a failed transfer */
-  int makestat;          /**< maintain a transfer-statistics log */
+  hts_boolean makestat;  /**< maintain a transfer-statistics log */
-  int maketrack;         /**< maintain an operations-statistics log */
+  hts_boolean maketrack; /**< maintain an operations-statistics log */
   int parsejava;         /**< Java/JS parsing mode; see htsparsejava_flags */
-  int hostcontrol;       /**< drop hosts that are too slow, etc. */
+  int hostcontrol; /**< ban slow/timing-out hosts; see hts_hostcontrol bits */
-  int errpage;           /**< generate an error page on 404 and similar */
+  hts_tristate errpage; /**< generate an error page on 404 and similar */
-  int check_type;   /**< probe unknown-type links (cgi/asp/dir) and follow moves
+  hts_boolean
-                     */
+      check_type; /**< probe unknown-type links (cgi/asp/dir) and follow moves
-  int all_in_cache; /**< keep all retrieved data in the cache */
+                   */
-  int robots;       /**< robots.txt handling level */
+  hts_boolean all_in_cache;      /**< keep all retrieved data in the cache */
-  int external;     /**< render external links as error pages */
+  hts_robots robots;             /**< robots.txt handling level */
-  int passprivacy;  /**< strip passwords from external links */
+  hts_tristate external;         /**< render external links as error pages */
-  int includequery; /**< include the query string in saved names */
+  hts_boolean passprivacy;       /**< strip passwords from external links */
-  int mirror_first_page;        /**< only mirror the links of the first page */
+  hts_boolean includequery;      /**< include the query string in saved names */
-  String sys_com;               /**< system command to run */
+  hts_boolean mirror_first_page; /**< only mirror the links of the first page */
-  int sys_com_exec;             /**< actually execute sys_com */
+  String sys_com;                /**< system command to run */
-  int accept_cookie;            /**< accept and send cookies */
+  hts_boolean sys_com_exec;      /**< actually execute sys_com */
-  t_cookie *cookie;             /**< cookie store */
+  hts_boolean accept_cookie;     /**< accept and send cookies */
-  int http10;                   /**< force HTTP/1.0 */
+  t_cookie *cookie;              /**< cookie store */
-  int nokeepalive;              /**< disable keep-alive */
+  hts_boolean http10;            /**< force HTTP/1.0 */
-  int nocompression;            /**< disable content compression */
+  hts_boolean nokeepalive;       /**< disable keep-alive */
-  int sizehack;                 /**< treat same-size response as "updated" */
+  hts_boolean nocompression;     /**< disable content compression */
-  int urlhack;                  // force "url normalization" to avoid loops
+  hts_boolean sizehack;          /**< treat same-size response as "updated" */
-  int tolerant;                 /**< accept an incorrect Content-Length */
+  hts_boolean urlhack;           // force "url normalization" to avoid loops
-  int parseall;   /**< parse aggressively, including unknown tags with links */
+  hts_boolean tolerant;          /**< accept an incorrect Content-Length */
-  int parsedebug; /**< parser debug mode */
+  hts_tristate
-  int norecatch;  /**< do not re-fetch files the user deleted locally */
+      parseall; /**< parse aggressively, including unknown tags with links */
-  int verbosedisplay; /**< animated text progress display */
+  hts_boolean parsedebug; /**< parser debug mode */
-  String footer;      /**< footer/info line injected into pages */
+  hts_boolean norecatch;  /**< do not re-fetch files the user deleted locally */
-  int maxcache;       /**< in-memory cache backing limit (bytes) */
+  hts_verbosedisplay verbosedisplay; /**< animated text progress display */
-  // int maxcache_anticipate; // maximum links to anticipate (upper bound)
+  String footer; /**< footer/info line injected into pages */
-  int ftp_proxy;                /**< use the HTTP proxy for FTP too */
+  int maxcache;  /**< in-memory cache backing limit (bytes) */
-  String filelist;              /**< file listing URLs to include */
+  hts_boolean ftp_proxy;    /**< use the HTTP proxy for FTP too */
-  String urllist;               /**< file listing filters to include */
+  String filelist;          /**< file listing URLs to include */
-  htsfilters filters;           /**< filter pointers (+/-pattern rules) */
+  String urllist;           /**< file listing filters to include */
-  hash_struct *hash;            // hash structure
+  htsfilters filters;       /**< filter pointers (+/-pattern rules) */
-  lien_url **liens;             // links
+  hash_struct *hash;        // hash structure
-  int lien_tot;                 // top index of "links" heap (always out-of-range)
+  lien_url **liens;         // links
-  lien_buffers *liensbuf;       // links buffers
+  int lien_tot;             // top index of "links" heap (always out-of-range)
-  robots_wizard *robotsptr;     // robots ptr
+  lien_buffers *liensbuf;   // links buffers
-  String lang_iso;              /**< Accept-Language value (en, fr, ...) */
+  robots_wizard *robotsptr; // robots ptr
-  String accept;                // Accept:
+  String lang_iso;          /**< Accept-Language value (en, fr, ...) */
-  String headers;               // Additional headers
+  String accept;            // Accept:
-  String mimedefs;              // ext1=mimetype1\next2=mimetype2..
+  String headers;           // Additional headers
-  String mod_blacklist;         /**< blacklisted modules */
+  String mimedefs;          // ext1=mimetype1\next2=mimetype2..
-  int convert_utf8;             // filenames UTF-8 conversion (3.46)
+  String mod_blacklist;     /**< blacklisted modules */
+  hts_boolean convert_utf8; // filenames UTF-8 conversion (3.46)
   //
   int maxlink;   /**< max number of links */
   int maxfilter; /**< max number of filters */
   //
   const char *exec; /**< path of the running executable */
   //
-  int quiet;                    /**< suppress non-wizard questions */
+  hts_boolean quiet;                 /**< suppress non-wizard questions */
-  int keyboard;                 /**< poll stdin for keyboard input */
+  hts_boolean keyboard;              /**< poll stdin for keyboard input */
-  int bypass_limits;            // bypass built-in limits
+  hts_boolean bypass_limits;         // bypass built-in limits
-  int background_on_suspend;    // background process on suspend signal
+  hts_boolean background_on_suspend; // background process on suspend signal
   //
-  int is_update;    /**< this run is an update (show "File updated...") */
+  hts_boolean is_update; /**< this run is an update (show "File updated...") */
-  int dir_topindex; /**< rebuild the top index afterwards */
+  hts_boolean dir_topindex; /**< rebuild the top index afterwards */
   //
   // callbacks
   t_hts_htmlcheck_callbacks
@@ -421,6 +526,16 @@ struct httrackp {
   htslibhandles libHandles; /**< loaded external module handles */
   //
   htsoptstate state; /**< embedded live engine state */
+  String strip_query; /**< query keys to drop when deduping URLs (-strip-query);
+                           appended at the tail to keep field offsets stable */
+  hts_boolean
+      no_www_dedup; /**< with urlhack, keep www.host distinct from host */
+  hts_boolean no_slash_dedup; /**< with urlhack, keep redundant // in paths */
+  hts_boolean no_query_dedup; /**< with urlhack, keep query-argument order */
+  String cookies_file;        /**< extra Netscape cookies.txt to preload
+                                 (--cookies-file) */
+  int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
+  int pause_max_ms; /**< inter-file pause upper bound, ms */
 };
 
 /* Running statistics for a mirror. */
@@ -431,8 +546,6 @@ typedef struct hts_stat_struct hts_stat_struct;
 struct hts_stat_struct {
   LLint HTS_TOTAL_RECV; /**< total bytes received from the network */
   LLint stat_bytes;     /**< total bytes written to disk */
-  // int HTS_TOTAL_RECV_STATE;  // status: 0 ok 1: slow down a little 2: slow
-  // down 3: a lot
   TStamp stat_timestart; /**< mirror start time */
   //
   LLint total_packed;    /**< compressed bytes received (on the wire) */
@@ -485,17 +598,17 @@ typedef struct htsrequest htsrequest;
 struct htsrequest {
   short int user_agent_send; /**< send a User-Agent header */
   short int http11; /**< sign the request as HTTP/1.1 rather than HTTP/1.0 */
-  short int nokeepalive;        /**< disable keep-alive */
+  short int nokeepalive;   /**< disable keep-alive */
-  short int range_used;         /**< a Range header is in use */
+  short int range_used;    /**< a Range header is in use */
-  short int nocompression;      /**< disable compression */
+  short int nocompression; /**< disable compression */
-  short int flush_garbage;      // recycled
+  short int flush_garbage; // recycled
-  const char *user_agent;       /**< User-Agent value */
+  const char *user_agent;  /**< User-Agent value */
-  const char *referer;          /**< Referer value */
+  const char *referer;     /**< Referer value */
-  const char *from;             /**< From value */
+  const char *from;        /**< From value */
-  const char *lang_iso;         /**< Accept-Language value */
+  const char *lang_iso;    /**< Accept-Language value */
-  const char *accept;           /**< Accept value */
+  const char *accept;      /**< Accept value */
-  const char *headers;          /**< extra request headers */
+  const char *headers;     /**< extra request headers */
-  htsrequest_proxy proxy;       /**< proxy for this request */
+  htsrequest_proxy proxy;  /**< proxy for this request */
 };
 
 /* Result of a connection / header fetch. */
@@ -527,8 +640,8 @@ struct htsblk {
   short int is_file; /**< 1 if a file descriptor rather than a socket */
   T_SOC soc;         /**< socket id */
   SOCaddr address;   /**< peer IP address */
-  int address_size;             // IP address structure length (unused internally)
+  int address_size;  // IP address structure length (unused internally)
-  FILE *fp;                     /**< file handle for file:// */
+  FILE *fp;          /**< file handle for file:// */
 #if HTS_USEOPENSSL
   short int ssl; /**< nonzero if this is an SSL connection (https) */
   // BIO* ssl_soc;          // SSL structure
@@ -564,7 +677,6 @@ struct lien_url {
   int depth;        /**< remaining allowed depth; >0 strong, 0 weak */
   int pass2;        /**< second-pass marker; -1 means handled in background */
   char link_import; /**< imported after a move; skip the usual up/down rules */
-  // int moved;          // pointer to moved
   int retry;    /**< remaining retries */
   int testmode; /**< test only: send just a HEAD */
 };
@@ -609,9 +721,6 @@ struct lien_back {
   LLint chunk_size; /**< size of the chunk being loaded */
   LLint chunk_blocksize; /**< data size declared by the chunk */
   LLint compressed_size; /**< compressed size (stats only) */
-  //
-  //int links_index;        // to access liens[links_index]
-  //
   char info[256]; /**< status text, e.g. for FTP */
   int stop_ftp;   /**< stop flag for FTP */
   int finalized;  /**< finalized (memory optimization) */