Finish scrubbing the java-class docs across the guides and man source

The -j option still exists (it drives JavaScript parsing), so every "parse
Java Classes" summary in the HTML guides becomes "parse scripts", and the
obsolete .class-parsing descriptions and java-applet troubleshooting entries
are reworded to drop the removed capability. The "Some java classes may not
find..." engine-limit line is dropped at its source (README, from which
makeman.sh regenerates the man LIMITS section), and httrack.1 regenerated.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -55,7 +55,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -99,7 +101,8 @@ jobs:
           sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
             -exec mv {} {}.hidden \;
           ! command -v python3
-          make check
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -132,13 +135,52 @@ jobs:
       - name: Build
         run: make -j"$(sysctl -n hw.ncpu)"
 
+      - name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
+        # 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
+        # instantly like Linux; alias them onto lo0 so they don't stall to timeout.
+        run: |
+          set -euo pipefail
+          sudo ifconfig lo0 alias 127.0.0.2 up
+          sudo ifconfig lo0 alias 127.0.0.3 up
+
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # Runtime smoke of the WebHTTrack launcher on macOS: it carries a Darwin-only
+  # browser path (open -W) and nothing else exercises htsserver. Install into a
+  # temp prefix, then check webhttrack brings up htsserver and serves the UI.
+  webhttrack-macos:
+    name: webhttrack smoke (macOS arm64)
+    runs-on: macos-14
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          brew install autoconf automake libtool autoconf-archive
+
+      - name: Build and install into a temp prefix
+        run: |
+          set -euo pipefail
+          ssl="$(brew --prefix openssl@3)"
+          autoreconf -fi
+          ./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib" \
+            --prefix="$RUNNER_TEMP/inst"
+          make -j"$(sysctl -n hw.ncpu)"
+          make install
+
+      - name: Smoke-test webhttrack
+        run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
+
   # Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
   # -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
   # and bounds math can truncate or wrap in ways 64-bit never reveals (the axis
@@ -171,7 +213,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -226,7 +270,9 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -272,12 +318,52 @@ jobs:
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
           # skipped here (uninstrumented libz floods MSan with false positives).
           tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
-          make check TESTS="$tests"
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs" TESTS="$tests"
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
+  # parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
+  # (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
+  # catastrophic backtracking -- and pins the regression seeds for the bugs the
+  # fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
+  fuzz:
+    name: fuzz (libFuzzer corpus replay, clang)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential clang autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - name: Configure (fuzzers, static)
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure CC=clang \
+            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
+            LDFLAGS="-fsanitize=address,undefined" \
+            --enable-fuzzers --disable-shared
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Replay corpora
+        env:
+          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
+          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
+        run: bash fuzz/run-fuzzers.sh fuzz check
+
   # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
   # the configuration users on minimal systems build, and one libssl is not even
   # installed here so configure cannot silently re-enable it. The matrix above
@@ -308,7 +394,9 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: make check
+        run: |
+          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()

.github/workflows/codeql.yml

@@ -0,0 +1,63 @@
+# CodeQL static analysis (C). The security-extended suite covers the classes
+# this codebase actually fights: overflows, tainted-size allocs, format bugs.
+name: CodeQL
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+  schedule:
+    # Weekly re-scan of master so new/updated queries land without a push.
+    - cron: "17 4 * * 1"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: codeql-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: analyze (c-cpp)
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      # Upload findings to the repo's code-scanning dashboard.
+      security-events: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - uses: github/codeql-action/init@v4
+        with:
+          languages: c-cpp
+          build-mode: manual
+          queries: security-extended
+          # fopen's umask-controlled 0666 is intended for mirror/cache/log
+          # output; the one credential file (cookies.txt) is kept 0600 on Unix.
+          config: |
+            query-filters:
+              - exclude:
+                  id: cpp/world-writable-file-creation
+
+      # Manual build: CodeQL traces the compiler, so build exactly what ships.
+      - name: Build
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure
+          make -j"$(nproc)"
+
+      - uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:c-cpp"

.gitignore

@@ -34,8 +34,12 @@ Makefile
 *.so.*
 *.a
 
-# make dist output.
+# make dist output; dist/ holds httrack-gh-release staging artifacts.
 /httrack-*.tar.gz
+/dist/
 
 # Editor / autotools backup files.
 *~
+
+# Python bytecode (tests/local-server.py).
+__pycache__/

AGENTS.md

@@ -6,8 +6,19 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 ## Build & test
 - Fresh clone first: `git submodule update --init src/coucal`
 - `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
-  automake, libtool), then `bash configure && make && make check`. Or run
-  `sh build.sh` to do bootstrap + configure + make in one shot.
+  automake, libtool), then `bash configure && make -j"$(nproc)" && make check
+  -j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
+  automake's parallel harness and each crawl test binds its own ephemeral-port
+  server, so `-j` never contends and a multi-minute serial run drops to
+  seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
+  automatically; only a test slower than the current longest raises the floor.
+  On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
+  spend much of their wall time asleep (server trickles, httrack self-pacing),
+  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
+  platform, macOS included: the test server raises its listen backlog
+  (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
+  `-c16` bigcrawl the way Python's default backlog of 5 did.
+  Or run `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

Makefile.am

@@ -1,5 +1,5 @@
 
-SUBDIRS = src man m4 libtest templates lang html tests
+SUBDIRS = src man m4 libtest templates lang html tests fuzz
 
 ACLOCAL_AMFLAGS = -I m4
 

README

@@ -40,7 +40,6 @@ These are the principals limits of HTTrack for that moment. Note that we did not
 that would have solved them.
 
 - Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
-- Some java classes may not find some files on them (class included)
 - Cgi-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like -*cgi-bin*
 
 

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.10], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,10 +29,11 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:2:0: 3.49.10 only appends tail fields to the options struct (no existing
-# symbol or offset changed vs 3.49.9), so it stays soname .so.3; bump revision.
+# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
+# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
+# so it stays soname .so.3; bump revision.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:2:0"
+VERSION_INFO="3:4:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 
@@ -91,7 +92,8 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
 AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
 AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
 AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
-AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
+AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
+	[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
 AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
 AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
 AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -99,9 +101,16 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
 AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
 AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
 
+# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
+# sanitizer builds, whose interceptors want the unfortified calls.
+case "$CFLAGS" in
+*-fsanitize=*) ;;
+*) AX_ADD_FORTIFY_SOURCE ;;
+esac
+
 # Force libc back into DT_NEEDED for libraries that reach it only through
-# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
-# linker; Apple ld rejects these flags and links libSystem unconditionally.
+# libhttrack (the libtest callbacks), but only with a GNU-style linker; Apple
+# ld rejects these flags and links libSystem unconditionally.
 AX_CHECK_LINK_FLAG([-Wl,--push-state,--no-as-needed,-lc,--pop-state],
 	[LIBC_FORCE_LINK="-Wl,--push-state,--no-as-needed,-lc,--pop-state"])
 AC_SUBST([LIBC_FORCE_LINK])
@@ -299,6 +308,37 @@ AC_ARG_ENABLE([online-unit-tests],
 	])
 AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
 
+## libFuzzer harnesses (fuzz/); requires clang
+AC_MSG_CHECKING(whether to build fuzzers)
+AC_ARG_ENABLE([fuzzers],
+	[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
+	[
+	case "${enableval}" in
+	no|yes)
+		fuzzers=$enableval
+		AC_MSG_RESULT($enableval)
+		;;
+	*)
+		AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
+		;;
+	esac
+	],
+	[
+	fuzzers=no
+	AC_MSG_RESULT(no)
+	])
+if test x"$fuzzers" = x"yes"; then
+	# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
+	AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
+		[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
+		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
+	# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
+	if test x"$enable_shared" != x"no"; then
+		AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
+	fi
+fi
+AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
+
 # Final output
 AC_CONFIG_FILES([
 Makefile
@@ -310,5 +350,6 @@ lang/Makefile
 html/Makefile
 libtest/Makefile
 tests/Makefile
+fuzz/Makefile
 ])
 AC_OUTPUT

debian/changelog

@@ -1,3 +1,34 @@
+httrack (3.49.12-1) unstable; urgency=medium
+
+  * New upstream release: security and crawl-correctness fixes (remote stack
+    overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
+    world-readable cookies.txt, filter-pattern denial of service) plus a new
+    --why filter diagnostic; full list in history.txt.
+  * Build with _FORTIFY_SOURCE and -fstack-protector-strong.
+
+ -- Xavier Roche <xavier@debian.org>  Fri, 10 Jul 2026 21:11:09 +0200
+
+httrack (3.49.11-1) unstable; urgency=medium
+
+  * New upstream release: crawl correctness and security fixes (network-facing
+    buffer overflows, file-type detection, redirect handling) and modernized
+    web defaults; full list in history.txt.
+  * Add DEP-12 upstream metadata (#466).
+  * Bump debhelper compat to 14 (#466).
+  * Drop the redundant Priority field and update the NMU lintian override to
+    the current tag names (#466).
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 05 Jul 2026 00:03:18 +0200
+
+httrack (3.49.10-2) unstable; urgency=medium
+
+  * Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
+    absent (the local-server tests need python3, which the buildds lack). Add
+    patches/skip-local-pause-test-without-python3.patch to guard the test on
+    python3 up front, like its siblings, so it skips cleanly.
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 20:18:46 +0200
+
 httrack (3.49.10-1) unstable; urgency=medium
 
   * New upstream release: new download-pacing and URL-handling options plus a

debian/copyright

@@ -7,9 +7,8 @@ Files: *
 Copyright: 1998-2026 Xavier Roche and other contributors
 License: GPL-3+
 Comment:
- The engine includes contributions from Yann Philippot (src/htsjava.c,
- src/htsjava.h). htsbasenet.h links against the system OpenSSL library
- (originally by Eric Young); no OpenSSL/SSLeay code is bundled here.
+ htsbasenet.h links against the system OpenSSL library (originally by Eric
+ Young); no OpenSSL/SSLeay code is bundled here.
 
 Files: src/minizip/*
 Copyright: 1998-2010 Gilles Vollant

debian/libhttrack-dev.files

@@ -1,5 +1,4 @@
 usr/include/httrack
 usr/lib/*/libhttrack.{so}
-usr/lib/*/libhtsjava.{so}
 usr/share/httrack/libtest
 usr/lib/*/httrack/libtest

debian/libhttrack3.files

@@ -1,3 +1,2 @@
 usr/lib/*/libhttrack.so.3*
-usr/lib/*/libhtsjava.so.3*
 usr/share/httrack/templates

fuzz/Makefile.am

@@ -0,0 +1,45 @@
+# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
+if FUZZERS
+noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
+	fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx
+endif
+
+AM_CPPFLAGS = \
+	@DEFAULT_CFLAGS@ \
+	@THREADS_CFLAGS@ \
+	@V6_FLAG@ \
+	@LFS_FLAG@ \
+	-I$(top_srcdir)/src \
+	-I$(top_srcdir)/src/coucal
+
+# Static-link libhttrack.la: the internal symbols are hidden in the .so.
+AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
+LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
+
+fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
+fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
+fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
+fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
+fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
+fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
+fuzz_url_SOURCES = fuzz-url.c fuzz.h
+fuzz_header_SOURCES = fuzz-header.c fuzz.h
+fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
+
+# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
+EXTRA_DIST = README.md run-fuzzers.sh \
+	corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
+	corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
+	corpus/idna/idna.txt corpus/idna/unicode.txt \
+	corpus/idna/regress-multilabel-leak.txt \
+	corpus/entities/entities.txt \
+	corpus/unescape/percent.txt \
+	corpus/filters/filter.bin corpus/filters/filter-size.bin \
+	corpus/filters/regress-empty-subject-unique.bin \
+	corpus/filters/redos-star-classes.bin \
+	corpus/url/http-url.txt corpus/url/relative-path.txt \
+	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
+	corpus/header/full-response.txt corpus/header/redirect.txt \
+	corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
+	corpus/cachendx/regress-overadvance.bin \
+	corpus/cachendx/regress-truncated-entry.bin

fuzz/README.md

@@ -0,0 +1,15 @@
+# Fuzzing httrack
+
+libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
+
+```sh
+./bootstrap
+mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
+CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
+  LDFLAGS="-fsanitize=address,undefined" \
+  bash /path/to/httrack/configure --enable-fuzzers --disable-shared
+make
+bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60   # 60s per target
+```
+
+Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.

fuzz/corpus/cachendx/new-format.txt

@@ -0,0 +1,7 @@
+8
+CACHE-1.1
+28
+Mon, 01 Jan 2024 00:00:00 GMT
+www.example.com
+/index.html
+123

fuzz/corpus/cachendx/old-format.txt

@@ -0,0 +1,5 @@
+3
+1.0
+www.example.com
+/page
+5

fuzz/corpus/cachendx/regress-overadvance.bin

@@ -0,0 +1,2 @@
+32768
+CACHE-1.1

fuzz/corpus/cachendx/regress-truncated-entry.bin

@@ -0,0 +1,5 @@
+8
+CACHE-1.1
+1
+x
+www.example.com

fuzz/corpus/charset/latin1.txt

@@ -0,0 +1 @@
+
café naďve ¤

fuzz/corpus/charset/sjis.txt

@@ -0,0 +1 @@
+コンピュータ

fuzz/corpus/entities/entities.txt

@@ -0,0 +1 @@
+&<>A☃é&notarealentity;�

fuzz/corpus/filters/regress-empty-subject-unique.bin

@@ -0,0 +1 @@
+**((

fuzz/corpus/header/full-response.txt

@@ -0,0 +1,10 @@
+HTTP/1.1 200 OK
+Content-Type: text/html; charset=utf-8
+Content-Length: 1234
+Content-Encoding: gzip
+Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
+Etag: "abc"
+Location: http://example.com/x
+Set-Cookie: ID=42; path=/; domain=.example.com
+Content-Range: bytes 0-99/100
+Transfer-Encoding: chunked

fuzz/corpus/header/redirect.txt

@@ -0,0 +1,3 @@
+HTTP/1.0 301 Moved
+Location: /elsewhere
+Content-Disposition: attachment; filename="a.pdf"

fuzz/corpus/idna/idna.txt

@@ -0,0 +1 @@
+xn--bcher-kva.example.com

fuzz/corpus/idna/regress-multilabel-leak.txt

@@ -0,0 +1 @@
+büchev.例å­bücheple

fuzz/corpus/idna/unicode.txt

@@ -0,0 +1 @@
+bücher.例子.example

fuzz/corpus/meta/meta-charset.html

@@ -0,0 +1 @@
+<html><head><meta charset="utf-8"></head><body>x</body></html>

fuzz/corpus/meta/meta-content-first.html

@@ -0,0 +1 @@
+<meta content="text/html; charset=gb2312" http-equiv="content-type">

fuzz/corpus/meta/meta-http-equiv.html

@@ -0,0 +1 @@
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">

fuzz/corpus/meta/meta-truncated.html

@@ -0,0 +1 @@
+<meta charset='utf-8

fuzz/corpus/meta/meta-unquoted.html

@@ -0,0 +1 @@
+<meta   charset = utf-8 />

fuzz/corpus/unescape/percent.txt

@@ -0,0 +1 @@
+%41%zz%%20%c3%a9+%2e%2e%2f

fuzz/corpus/url/http-url.txt

@@ -0,0 +1 @@
+http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag

fuzz/corpus/url/regress-file-empty-path.txt

@@ -0,0 +1 @@
+file://

fuzz/corpus/url/regress-long-path-abort.txt

@@ -0,0 +1 @@
+ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../

fuzz/corpus/url/relative-path.txt

@@ -0,0 +1 @@
+ftp://ftp.example.com/pub/../file.txt

fuzz/fuzz-cachendx.c

@@ -0,0 +1,65 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
+   walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
+   Mirrors the -#C cache-listing scan (htscoremain.c). */
+#include "fuzz.h"
+#include "htscache.h"
+#include "htslib.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  const char *const end = buf + size;
+  char firstline[256];
+  char *a = buf;
+
+  /* header: two length-prefixed fields (version, last-modified) */
+  a += cache_brstr(a, firstline, sizeof(firstline));
+  a += cache_brstr(a, firstline, sizeof(firstline));
+
+  /* body: newline-delimited host/file/position triples; the length-prefixed
+     scan must stay inside the buffer */
+  while (a != NULL && a < end) {
+    char BIGSTK line[HTS_URLMAXSIZE * 2];
+    char linepos[256];
+    int pos;
+
+    a = strchr(a + 1, '\n');
+    if (a == NULL)
+      break;
+    a++;
+    a += cache_binput(a, end, line, HTS_URLMAXSIZE);
+    a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
+    a += cache_binput(a, end, linepos, 200);
+    sscanf(linepos, "%d", &pos);
+    (void) pos;
+  }
+
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-charset.c

@@ -0,0 +1,84 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
+   UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+static const char *const charsets[] = {
+    "utf-8",    "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
+    "us-ascii", "shift_jis",  "euc-jp",     "iso-2022-jp", "gb2312",
+    "big5",     "euc-kr",     "koi8-r",     "utf-16",      "unknown-charset",
+};
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  const char *charset;
+  char *s;
+
+  if (size == 0)
+    return 0;
+  charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+
+  {
+    char *utf8 = hts_convertStringToUTF8(s, size, charset);
+    freet(utf8);
+  }
+  {
+    char *enc = hts_convertStringFromUTF8(s, size, charset);
+    freet(enc);
+  }
+  {
+    size_t nChars = 0;
+    hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
+
+    if (ucs != NULL) {
+      char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
+      freet(back);
+      freet(ucs);
+    }
+  }
+  {
+    size_t i = 0;
+
+    while (i < size) {
+      hts_UCS4 uc = 0;
+      const size_t nr = hts_readUTF8(s + i, size - i, &uc);
+      char out[8];
+
+      if (nr == 0)
+        break;
+      hts_writeUTF8(uc, out, sizeof(out));
+      i += nr;
+    }
+  }
+
+  freet(s);
+  return 0;
+}

fuzz/fuzz-entities.c

@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
+   destination size, so truncation bounds get exercised too. */
+#include "fuzz.h"
+#include "htsencoding.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t dsizes[] = {1, 2, 8, 64, 4096};
+  size_t dsize;
+  char *src, *dest;
+
+  if (size == 0)
+    return 0;
+  dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
+  data++, size--;
+  src = fuzz_strdup(data, size);
+  dest = malloct(dsize);
+
+  (void) hts_unescapeEntities(src, dest, dsize);
+  (void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
+
+  freet(dest);
+  freet(src);
+  return 0;
+}

fuzz/fuzz-filters.c

@@ -0,0 +1,65 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
+   here). Input splits on the first NUL: pattern, then subject string. */
+#include "fuzz.h"
+#include "htsfilters.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  const char *joker = buf;
+  const uint8_t *sep = memchr(data, '\0', size);
+  /* subject in its own allocation so ASan bounds it apart from the pattern */
+  char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
+                          : fuzz_strdup(data + size, 0);
+
+  (void) strjoker(nom, joker, NULL, NULL);
+  {
+    LLint sz = (LLint) size;
+    int size_flag = 0;
+
+    (void) strjoker(nom, joker, &sz, &size_flag);
+  }
+  (void) strjokerfind(nom, joker);
+  {
+    char *filter = malloct(strlen(joker) + 2);
+    char *filters[1];
+    LLint sz = (LLint) size;
+    int size_flag = 0, depth = 0;
+
+    filter[0] = '-';
+    memcpy(filter + 1, joker, strlen(joker) + 1);
+    filters[0] = filter;
+    (void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
+    freet(filter);
+  }
+
+  freet(nom);
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-header.c

@@ -0,0 +1,74 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
+   status line, treathead on each following header. Both consume raw bytes
+   off the wire and copy fields into fixed htsblk buffers; treathead also
+   mutates its line in place and drives cookie parsing. */
+#include "fuzz.h"
+#include "htslib.h"
+#include "htsbauth.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  htsblk r;
+  t_cookie *cookie = calloct(1, sizeof(*cookie));
+  char *line = malloct(size + 1);
+  char *p = buf;
+  int first = 1;
+
+  memset(&r, 0, sizeof(r));
+  cookie->max_len = (int) sizeof(cookie->data);
+  r.location = malloct(HTS_URLMAXSIZE * 2);
+  r.location[0] = '\0';
+
+  /* feed one header line at a time, as the receive loop does */
+  while (p != NULL && *p != '\0') {
+    char *nl = strchr(p, '\n');
+    size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
+    size_t i, len = 0;
+
+    /* binput drops every '\r' on the wire; mirror it */
+    for (i = 0; i < n; i++)
+      if (p[i] != '\r')
+        line[len++] = p[i];
+    line[len] = '\0';
+    if (first) {
+      treatfirstline(&r, line);
+      first = 0;
+    } else {
+      treathead(cookie, "www.example.com", "/", &r, line);
+    }
+    p = (nl != NULL) ? nl + 1 : NULL;
+  }
+
+  freet(r.location);
+  freet(line);
+  freet(cookie);
+  freet(buf);
+  return 0;
+}

fuzz/fuzz-idna.c

@@ -0,0 +1,47 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+
+  {
+    char *idna = hts_convertStringUTF8ToIDNA(s, size);
+    freet(idna);
+  }
+  {
+    char *utf8 = hts_convertStringIDNAToUTF8(s, size);
+    freet(utf8);
+  }
+  (void) hts_isStringIDNA(s, size);
+
+  freet(s);
+  return 0;
+}

fuzz/fuzz-meta.c

@@ -0,0 +1,40 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
+   <meta> charset declaration. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *html = fuzz_strdup(data, size);
+  char *charset = hts_getCharsetFromMeta(html, size);
+
+  freet(charset);
+  freet(html);
+  return 0;
+}

fuzz/fuzz-unescape.c

@@ -0,0 +1,53 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
+   output buffer size, so the bounded-copy contract is exercised. */
+#include "fuzz.h"
+#include "httrack-library.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t bsizes[] = {1, 2, 16, 256, 8192};
+  size_t bsize;
+  char *s, *catbuff;
+
+  if (size == 0)
+    return 0;
+  bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+  catbuff = malloct(bsize);
+
+  (void) unescape_http(catbuff, bsize, s);
+  (void) unescape_http_unharm(catbuff, bsize, s, 0);
+  (void) unescape_http_unharm(catbuff, bsize, s, 1);
+  unescape_amp(s);
+
+  freet(catbuff);
+  freet(s);
+  return 0;
+}

fuzz/fuzz-url.c

@@ -0,0 +1,49 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
+   is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
+   in place. */
+#include "fuzz.h"
+#include "htscore.h"
+#include "htslib.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+  lien_adrfil *af = calloct(1, sizeof(*af));
+  /* fil_simplifie rewrites in place and may grow an empty path to "./" */
+  char *path = malloct(size + 3);
+
+  (void) ident_url_absolute(s, af);
+  memcpy(path, s, size + 1);
+  fil_simplifie(path);
+
+  freet(path);
+  freet(af);
+  freet(s);
+  return 0;
+}

fuzz/fuzz.h

@@ -25,42 +25,27 @@ would dishonor our work and waste the many hours we have spent on it.
 Please visit our Website: http://www.httrack.com
 */
 
-/* ------------------------------------------------------------ */
-/* File: Java classes parser .h                                 */
-/* Author: Yann Philippot                                       */
-/* ------------------------------------------------------------ */
+/* Shared helpers for the libFuzzer harnesses. */
+#ifndef FUZZ_H
+#define FUZZ_H
 
-#ifndef HTSJAVA_DEFH
-#define HTSJAVA_DEFH
+#define HTS_INTERNAL_BYTECODE
 
-#ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER
-#define HTS_DEF_FWSTRUCT_JAVA_HEADER
-typedef struct JAVA_HEADER JAVA_HEADER;
-#endif
-struct JAVA_HEADER {
-  unsigned long int magic;
-  unsigned short int minor;
-  unsigned short int major;
-  unsigned short int count;
-};
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
 
-#ifndef HTS_DEF_FWSTRUCT_RESP_STRUCT
-#define HTS_DEF_FWSTRUCT_RESP_STRUCT
-typedef struct RESP_STRUCT RESP_STRUCT;
-#endif
-struct RESP_STRUCT {
-  int file_position;
-  //
-  unsigned int index1;
-  unsigned int type;
-  char name[1024];
-};
+#include "htsbase.h"
 
-/* Library internal definictions */
-#ifdef HTS_INTERNAL_BYTECODE
+/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
+static char *fuzz_strdup(const uint8_t *data, size_t size) {
+  char *s = malloct(size + 1);
 
-EXTERNAL_FUNCTION int hts_plug_java(httrackp * opt, const char *argv);
-
-#endif
+  memcpy(s, data, size);
+  s[size] = '\0';
+  return s;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
 
 #endif

fuzz/run-fuzzers.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+# Drive every built harness against its seed corpus.
+#   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
+#   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
+# Replay is crash/leak-only and never mutates; the per-unit -timeout guards
+# both modes against pathological slowdowns (e.g. pre-#501 strjoker).
+set -euo pipefail
+
+srcdir=$(cd "$(dirname "$0")" && pwd)
+bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
+mode=${2:-20}
+
+status=0
+for f in "$bld"/fuzz-*; do
+    if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
+    case "$f" in *.o | *.c | *.dSYM) continue ;; esac
+    name=$(basename "$f")
+    corpus="$srcdir/corpus/${name#fuzz-}"
+    if [ "$mode" = "check" ]; then
+        echo "=== $name (replay) ==="
+        [ -d "$corpus" ] || continue
+        if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
+            echo "*** $name FAILED on its corpus" >&2
+            status=1
+        fi
+        continue
+    fi
+    work=$(mktemp -d)
+    args=("$work")
+    [ -d "$corpus" ] && args+=("$corpus")
+    echo "=== $name (${mode}s) ==="
+    if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
+        -artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
+        echo "*** $name FAILED; artifacts:" >&2
+        ls -l "$work" >&2
+        status=1
+    else
+        rm -rf "$work"
+    fi
+done
+exit $status

history.txt

@@ -4,6 +4,40 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
+3.49-12
++ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
++ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
++ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
++ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
++ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
++ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
++ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
++ Fixed: cookies.txt was created world-readable (#511)
++ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
++ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
++ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
++ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
++ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
++ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
++ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
+
+3.49-11
++ New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
++ New: advertise deflate in Accept-Encoding and decode deflate responses (#450)
++ New: follow <source> and <track> media elements as embedded links (#451)
++ New: added modern web MIME types to the type/extension table (#448)
++ Fixed: enforce the -E time limit during a slow transfer instead of only between files (#481)
++ Fixed: sniff the leading bytes of a download so a misdeclared Content-Type no longer renames a correct URL extension
++ Fixed: fast transfers could be saved under their temporary .delayed placeholder name (#5, #107)
++ Fixed: follow a redirect that maps to the same saved file instead of writing a self-pointing stub (#159)
++ Fixed: several network-facing buffer overflows in the FTP, Java and HTML parsers
++ Fixed: the htsjava plugin could not be loaded (hidden entry points, stale library name)
++ Fixed: HTML-escape truncation and a cache-buffer leak in the parser
++ Changed: modernized the default User-Agent to an honest HTTrack identifier (#449)
++ Changed: decode the full WHATWG set of HTML named character references (#443)
++ Changed: refreshed stale HTTP status, proxy-port and TLS-floor constants (#453)
++ Changed: multiple internal hardening, build, test and CI improvements
+
 3.49-10
 + New: --cookies-file to preload a Netscape cookies.txt before crawling (#215)
 + New: --pause to space out file downloads by a random delay (#185)

html/faq.html

@@ -609,10 +609,9 @@ most mirrors do work. We still are working to improve the mirror quality of HTTr
 <br>
 
 </a><a NAME="Q2b">Q: <strong>Some Java applets do not work properly!</strong><br>
-A: <em>Java applets may not work in some cases, for example if HTTrack failed to detect all included classes
-or files called within the class file. Sometimes, Java applets need to be online, because remote files are
-directly caught. Finally, the site structure can be incompatible with the class (always try to keep the original site structure
-when you want to get Java classes)<br>
+A: <em>Java applets may not work in some cases, because HTTrack does not parse compiled class files to find
+the resources they load. Java applets often need to be online anyway, since remote files are fetched directly
+at runtime.<br>
 If there is no way to make some classes work properly, you can exclude them with the filters. 
 They will be available, but only online.
 </em>

html/fcguide.html

@@ -243,7 +243,7 @@ Build options:
 Spider options:
   bN accept cookies in cookies.txt (0=do not accept,* 1=accept) (--cookies[=N])
   u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (--check-type[=N])
-  j *parse Java Classes (j0 don't parse) (--parse-java[=N])
+  j *parse scripts (j0 don't parse) (--parse-java[=N])
   sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) (--robots[=N])
  %h  force HTTP/1.0 requests (reduce update features, only for old servers or proxies) (--http-10)
  %B  tolerant requests (accept bogus responses on some servers, but not standard!) (--tolerant)
@@ -417,7 +417,7 @@ site. Specifically, the defauls are:
   X *purge old files after update (X0 keep delete)
   bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
   u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
-  j *parse Java Classes (j0 don't parse)
+  j *parse scripts (j0 don't parse)
   sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
   C  create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)
   f *log file mode
@@ -497,9 +497,9 @@ about your browser retaining them.
 <p align=justify> This causes different document types to be analyzed
 differently.
 
-<pre><b><i>  j *parse Java Classes (j0 don't parse) </i></b></pre>
+<pre><b><i>  j *parse scripts (j0 don't parse) </i></b></pre>
 
-<p align=justify> This causes Java class files to be parsed looking for
+<p align=justify> This causes scripts to be parsed looking for
 URLs.
 
 <pre><b><i>  sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) </i></b></pre>
@@ -1178,7 +1178,7 @@ accepted and sent back in order to allow access.
 Spider options:
  bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
  u   check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
- j   *parse Java Classes (j0 don't parse)
+ j   *parse scripts (j0 don't parse)
  sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
  %h  force HTTP/1.0 requests (reduce update features, only for old servers or proxies)
  %B  tolerant requests (accept bogus responses on some servers, but not standard!)
@@ -2451,12 +2451,10 @@ work.  We still are working to improve the mirror quality of HTTrack.
 
 <p align=justify><b>Q: Some Java applets do not work properly! </b>
 
-<p align=justify> A: Java applets may not work in some cases, for
-example if HTTrack failed to detect all included classes or files called
-within the class file.  Sometimes, Java applets need to be online,
-because remote files are directly caught.  Finally, the site structure
-can be incompatible with the class (always try to keep the original site
-structure when you want to get Java classes)
+<p align=justify> A: Java applets may not work in some cases, because
+HTTrack does not parse compiled class files to find the resources they
+load.  Java applets often need to be online anyway, since remote files
+are fetched directly at runtime.
 
 <p align=justify> If there is no way to make some classes work properly,
 you can exclude them with the filters.  They will be available, but only

html/httrack.man.html

@@ -764,9 +764,9 @@ check, * u1 check but /, u2 check always)
 <td width="78%">
 
 
-<p>*parse Java Classes (j0 don t parse, bitmask: |1 parse
-default, |2 don t parse .class |4 don t parse .js |8 don t
-be aggressive) (--parse-java[=N])</p></td></tr>
+<p>*parse scripts (j0 don t parse, bitmask: |1 parse
+default, |4 don t parse .js |8 don t be aggressive)
+(--parse-java[=N])</p></td></tr>
 <tr valign="top" align="left">
 <td width="11%"></td>
 <td width="6%">
@@ -889,8 +889,8 @@ only) (--protocol[=N])</p></td></tr>
 <td width="78%">
 
 
-<p>disable a specific external mime module (-%w htsswf -%w
-htsjava) (--disable-module &lt;param&gt;)</p></td></tr>
+<p>disable a specific external mime module (-%w
+httrack-plugin) (--disable-module &lt;param&gt;)</p></td></tr>
 </table>
 
 <p style="margin-left:11%; margin-top: 1em"><b>Browser
@@ -2175,10 +2175,6 @@ Several scripts generating complex filenames may not find
 them (ex:
 img.src=&rsquo;image&rsquo;+a+Mobj.dst+&rsquo;.gif&rsquo;)</small></p>
 
-<p style="margin-left:11%; margin-top: 1em"><small>- Some
-java classes may not find some files on them (class
-included)</small></p>
-
 <p style="margin-left:11%; margin-top: 1em"><small>-
 Cgi-bin links may not work properly in some cases
 (parameters needed). To avoid them: use filters like

html/options.html

@@ -288,8 +288,8 @@ This option define whether the engine has to abandon a host if a timeout/"too sl
 &P extended parsing, attempt to parse all links (even in unknown tags or Javascript)
 This option activates the extended parsing, that attempt to find links in unknown Html code/javascript
 
-j *parse Java Classes (j0 don't parse)
-This option define whether the engine has to parse java files or not to catch included files
+j *parse scripts (j0 don't parse)
+This option defines whether the engine has to parse scripts or not to catch included files
 
 I *make an index (I0 don't make)
 This option define whether the engine has to generate an index.html on the top directory

html/plug.html

@@ -103,220 +103,219 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
 
 <br>
 
-You can write external functions to be plugged in the httrack library very easily. 
-We'll see there some examples.
+You can write external functions to be plugged in the httrack library very easily. 
+We'll see there some examples.
 
 <br><br>
-
-The <tt>httrack</tt> commandline tool allows (since the 3.30 release) to plug external functions to various callbacks defined in httrack.
-The 3.41 release introduces a cleaned up verion of callbacks, with two major changes:
-<ul>
-<li>Cleaned up function prototypes, with two arguments always passed (the caller carg structure, and the httrackp* object), convenient to pass an user-defined pointer (see <tt>CALLBACKARG_USERDEF(carg)</tt>)</li>
-<li>The httrackp* option structure can be directly accessed to plug callbacks (no need to give the callback name and function name in the commandline!)</li>
-<li>The callback plug is made through the CHAIN_FUNCTION() helper, allowing to chain multiple callbacks of the same type (the callbacks MUST preserve the chain by calling ancestors)</li>
-</ul>
-
-<br>
-References: 
-<ul>
-<li>the <tt>httrack-library.h</tt> prototype file
-<br />
+
+The <tt>httrack</tt> commandline tool allows (since the 3.30 release) to plug external functions to various callbacks defined in httrack.
+The 3.41 release introduces a cleaned up verion of callbacks, with two major changes:
+<ul>
+<li>Cleaned up function prototypes, with two arguments always passed (the caller carg structure, and the httrackp* object), convenient to pass an user-defined pointer (see <tt>CALLBACKARG_USERDEF(carg)</tt>)</li>
+<li>The httrackp* option structure can be directly accessed to plug callbacks (no need to give the callback name and function name in the commandline!)</li>
+<li>The callback plug is made through the CHAIN_FUNCTION() helper, allowing to chain multiple callbacks of the same type (the callbacks MUST preserve the chain by calling ancestors)</li>
+</ul>
+
+<br>
+References: 
+<ul>
+<li>the <tt>httrack-library.h</tt> prototype file
+<br />
 Note: the <i>Initialization</i>, <i>Main functions</i>, <i>Options handling</i> and <i>Wrapper functions</i> sections are generally the only ones to be considered.
 </li>
-<li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
-<li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
-<li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
-<li>the <tt>htsjava.c</tt> source file (the java class plugin ; overrides 'detect' and 'parse')</li>
-<li>the example given at the end of this document</li>
-</ul>
-
-<br />
-Below the list of functions to be defined in the module (plugin).<br />
-<br />
-
-<table width="100%">
-<tr><td><b><i>module function name</i></b></td><td><b>function description</b></td><td><b>function signature</b></td></tr>
-<tr><td background="img/fade.gif"><i>hts_plug</i></td><td background="img/fade.gif">
-The module entry point. The opt structure can be used to plug callbacks, using the CHAIN_FUNCTION() macro helper. The argv optional argument is the one passed in the commandline as --wrapper parameter.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)<br />
-
-<br />
-Wrappers can be plugged inside hts_plug() using:<br />
-<tt>
-CHAIN_FUNCTION(opt, &lt;callback name&gt;, &lt;our callback function name&gt;, &lt;our callback function optional custom pointer argument&gt;);
-</tt>
-<br />
-
-<br />
-Example:
-<br />
-<tt>
-CHAIN_FUNCTION(opt, check_html, process, userdef);
-</tt>
-<br />
-
-</td><td background="img/fade.gif"><tt>extern int hts_plug(httrackp *opt, const char* argv);</tt></td></tr>
-
-<!-- -->
-
-<tr><td background="img/fade.gif"><i>hts_unplug</i></td><td background="img/fade.gif">
-The module exit point. To free allocated resources without using global variables, use the uninit callback (see below)</td><td background="img/fade.gif"><tt>extern int hts_unplug(httrackp *opt);</tt></td></tr>
-
-</table>
-
-
-<br />
-Note that all callbacks (except init and uninit) take as first two argument:
-<ul>
-<li>the t_hts_callbackarg structure<br />
-this structure holds the callback chain (parent callbacks defined before the current callback) pointers, and the user-defined pointer ; see <tt>CALLBACKARG_USERDEF(carg)</tt>)
-</li>
-<li>the httrackp structure<br />
-this structure, holding all current httrack options and mirror state, can be read or mofidied
-</li>
-</ul>
-
-<br />
-Below the list of callbacks, and associated external wrappers.
-
-<table width="100%">
-<tr><td><b><i>callback name</i></b></td><td><b>callback description</b></td><td><b>callback function signature</b></td></tr>
-
-<tr><td background="img/fade.gif"><i>init</i></td><td background="img/fade.gif">Note: the use the "start" callback is advised. Called during initialization.<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>uninit</i></td><td background="img/fade.gif">Note: the use os the "end" callback is advised.<br />Called during un-initialization<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>start</i></td><td background="img/fade.gif">Called when the mirror starts. The <tt>opt</tt> structure passed lists all options defined for this mirror. You may modify the <tt>opt</tt> structure to fit your needs.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>end</i></td><td background="img/fade.gif">Called when the mirror ends<br>return value: 1 upon success, 0 upon error (the mirror will then be considered aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>chopt</i></td><td background="img/fade.gif">Called when options are to be changed. The <tt>opt</tt> structure passed lists all options, updated to take account of recent changes<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>preprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is to be parsed (original, not yet modified document). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>postprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is parsed and transformed (links rewritten). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_html</i></td><td background="img/fade.gif">Called when a document (which may not be an html document) is to be parsed. The <tt>html</tt> address points to the document data, of lenth <tt>len</tt>. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the parsing can be processed, 0 if the file must be skipped without being parsed</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question. The <tt>question</tt> string contains the question for the (human) user<br>return value: the string answer ("" for default reply)</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query2</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query3</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>loop</i></td><td background="img/fade.gif">Called periodically (informational, to display statistics)<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back, int back_max, int back_index, int lien_tot, int lien_ntot, int stat_time, hts_stat_struct* stats);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_link</i></td><td background="img/fade.gif">Called when a link has to be tested. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, int status);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_mime</i></td><td background="img/fade.gif">Called when a link download has begun, and needs to be tested against its MIME type. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested, and the <tt>mime</tt> string contains the link type being processed. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, const char* mime, int status);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>pause</i></td><td background="img/fade.gif">Called when the engine must pause. When the <tt>lockfile</tt> passed is deleted, the function can return<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* lockfile);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>filesave</i></td><td background="img/fade.gif">Called when a file is to be saved on disk<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>filesave2</i></td><td background="img/fade.gif">Called when a file is to be saved or checked on disk<br>The hostname, filename and local filename are given. Two additional flags tells if the local file is new (is_new), if the local file is to be modified (is_modified), and if the file was not updated remotely (not_updated).<br>(!is_new && !is_modified): the file is up-to-date, and will not be modified<br>(is_new && is_modified): a new file will be written (or an updated file is being written)<br>(!is_new && is_modified): a file is being updated (append)<br>(is_new && !is_modified): an empty file will be written ("do not recatch locally erased files")<br>not_updated: the file was not re-downloaded because it was up-to-date (no data transferred again)<br><br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* hostname, const char* filename, const char* localfile, int is_new, int is_modified, int not_updated);</tt></td></tr>
-
-<tr><td background="img/fade.gif"><i>linkdetected</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>linkdetected2</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link, const const char* tag_start);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>xfrstatus</i></td><td background="img/fade.gif">Called when a file has been processed (downloaded, updated, or error)<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>savename</i></td><td background="img/fade.gif">Called when a local filename has to be processed. The <tt>adr_complete</tt> and <tt>fil_complete</tt> are the address and URI of the file being saved ; the <tt>referer_adr</tt> and <tt>referer_fil</tt> are the address and URI of the referer link. The <tt>save</tt> string contains the local filename being used. You may modifiy the <tt>save</tt> string to fit your needs, up to 1024 bytes (note: filename collisions, if any, will be handled by the engine by renaming the file into file-2.ext, file-3.ext ..).<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>sendhead</i></td><td background="img/fade.gif">Called when HTTP headers are to be sent to the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>outgoing</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>receivehead</i></td><td background="img/fade.gif">Called when HTTP headers are recevived from the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>incoming</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);</tt></td></tr>
-
-<tr><td background="img/fade.gif"><i>detect</i></td><td background="img/fade.gif">Called when an unknown document is to be parsed. The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the type is known and can be parsed, 0 if the document type is unknown</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>parse</i></td><td background="img/fade.gif">The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the document was successfully parsed, 0 if an error occurred</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
-
-
-</table>
+<li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
+<li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
+<li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
+<li>the example given at the end of this document</li>
+</ul>
+
+<br />
+Below the list of functions to be defined in the module (plugin).<br />
+<br />
+
+<table width="100%">
+<tr><td><b><i>module function name</i></b></td><td><b>function description</b></td><td><b>function signature</b></td></tr>
+<tr><td background="img/fade.gif"><i>hts_plug</i></td><td background="img/fade.gif">
+The module entry point. The opt structure can be used to plug callbacks, using the CHAIN_FUNCTION() macro helper. The argv optional argument is the one passed in the commandline as --wrapper parameter.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)<br />
+
+<br />
+Wrappers can be plugged inside hts_plug() using:<br />
+<tt>
+CHAIN_FUNCTION(opt, &lt;callback name&gt;, &lt;our callback function name&gt;, &lt;our callback function optional custom pointer argument&gt;);
+</tt>
+<br />
+
+<br />
+Example:
+<br />
+<tt>
+CHAIN_FUNCTION(opt, check_html, process, userdef);
+</tt>
+<br />
+
+</td><td background="img/fade.gif"><tt>extern int hts_plug(httrackp *opt, const char* argv);</tt></td></tr>
+
+<!-- -->
+
+<tr><td background="img/fade.gif"><i>hts_unplug</i></td><td background="img/fade.gif">
+The module exit point. To free allocated resources without using global variables, use the uninit callback (see below)</td><td background="img/fade.gif"><tt>extern int hts_unplug(httrackp *opt);</tt></td></tr>
+
+</table>
+
+
+<br />
+Note that all callbacks (except init and uninit) take as first two argument:
+<ul>
+<li>the t_hts_callbackarg structure<br />
+this structure holds the callback chain (parent callbacks defined before the current callback) pointers, and the user-defined pointer ; see <tt>CALLBACKARG_USERDEF(carg)</tt>)
+</li>
+<li>the httrackp structure<br />
+this structure, holding all current httrack options and mirror state, can be read or mofidied
+</li>
+</ul>
+
+<br />
+Below the list of callbacks, and associated external wrappers.
+
+<table width="100%">
+<tr><td><b><i>callback name</i></b></td><td><b>callback description</b></td><td><b>callback function signature</b></td></tr>
+
+<tr><td background="img/fade.gif"><i>init</i></td><td background="img/fade.gif">Note: the use the "start" callback is advised. Called during initialization.<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>uninit</i></td><td background="img/fade.gif">Note: the use os the "end" callback is advised.<br />Called during un-initialization<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>start</i></td><td background="img/fade.gif">Called when the mirror starts. The <tt>opt</tt> structure passed lists all options defined for this mirror. You may modify the <tt>opt</tt> structure to fit your needs.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>end</i></td><td background="img/fade.gif">Called when the mirror ends<br>return value: 1 upon success, 0 upon error (the mirror will then be considered aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>chopt</i></td><td background="img/fade.gif">Called when options are to be changed. The <tt>opt</tt> structure passed lists all options, updated to take account of recent changes<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>preprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is to be parsed (original, not yet modified document). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>postprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is parsed and transformed (links rewritten). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_html</i></td><td background="img/fade.gif">Called when a document (which may not be an html document) is to be parsed. The <tt>html</tt> address points to the document data, of lenth <tt>len</tt>. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the parsing can be processed, 0 if the file must be skipped without being parsed</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question. The <tt>question</tt> string contains the question for the (human) user<br>return value: the string answer ("" for default reply)</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query2</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query3</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>loop</i></td><td background="img/fade.gif">Called periodically (informational, to display statistics)<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back, int back_max, int back_index, int lien_tot, int lien_ntot, int stat_time, hts_stat_struct* stats);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_link</i></td><td background="img/fade.gif">Called when a link has to be tested. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, int status);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_mime</i></td><td background="img/fade.gif">Called when a link download has begun, and needs to be tested against its MIME type. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested, and the <tt>mime</tt> string contains the link type being processed. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, const char* mime, int status);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>pause</i></td><td background="img/fade.gif">Called when the engine must pause. When the <tt>lockfile</tt> passed is deleted, the function can return<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* lockfile);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>filesave</i></td><td background="img/fade.gif">Called when a file is to be saved on disk<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>filesave2</i></td><td background="img/fade.gif">Called when a file is to be saved or checked on disk<br>The hostname, filename and local filename are given. Two additional flags tells if the local file is new (is_new), if the local file is to be modified (is_modified), and if the file was not updated remotely (not_updated).<br>(!is_new && !is_modified): the file is up-to-date, and will not be modified<br>(is_new && is_modified): a new file will be written (or an updated file is being written)<br>(!is_new && is_modified): a file is being updated (append)<br>(is_new && !is_modified): an empty file will be written ("do not recatch locally erased files")<br>not_updated: the file was not re-downloaded because it was up-to-date (no data transferred again)<br><br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* hostname, const char* filename, const char* localfile, int is_new, int is_modified, int not_updated);</tt></td></tr>
+
+<tr><td background="img/fade.gif"><i>linkdetected</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>linkdetected2</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link, const const char* tag_start);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>xfrstatus</i></td><td background="img/fade.gif">Called when a file has been processed (downloaded, updated, or error)<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>savename</i></td><td background="img/fade.gif">Called when a local filename has to be processed. The <tt>adr_complete</tt> and <tt>fil_complete</tt> are the address and URI of the file being saved ; the <tt>referer_adr</tt> and <tt>referer_fil</tt> are the address and URI of the referer link. The <tt>save</tt> string contains the local filename being used. You may modifiy the <tt>save</tt> string to fit your needs, up to 1024 bytes (note: filename collisions, if any, will be handled by the engine by renaming the file into file-2.ext, file-3.ext ..).<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>sendhead</i></td><td background="img/fade.gif">Called when HTTP headers are to be sent to the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>outgoing</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>receivehead</i></td><td background="img/fade.gif">Called when HTTP headers are recevived from the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>incoming</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);</tt></td></tr>
+
+<tr><td background="img/fade.gif"><i>detect</i></td><td background="img/fade.gif">Called when an unknown document is to be parsed. The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the type is known and can be parsed, 0 if the document type is unknown</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>parse</i></td><td background="img/fade.gif">The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the document was successfully parsed, 0 if an error occurred</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
+
+
+</table>
 
 <br><br>
-Note: the optional libhttrack-plugin module (libhttrack-plugin.dll or libhttrack-plugin.so), if found in the library environment, is loaded automatically, and its <tt>hts_plug()</tt> function being called.<br />
-
+Note: the optional libhttrack-plugin module (libhttrack-plugin.dll or libhttrack-plugin.so), if found in the library environment, is loaded automatically, and its <tt>hts_plug()</tt> function being called.<br />
+
+
+<br />
+An example is generally more efficient than anything else, so let's write our first module, aimed to stupidely print all parsed html files:
+<table width="100%" border="2">
+<tr><td>
+<pre>
+/* system includes */
+#include &lt;stdio.h&gt;
+#include &lt;stdlib.h&gt;
+#include &lt;string.h&gt;
+
+/* standard httrack module includes */
+#include "httrack-library.h"
+#include "htsopt.h"
+#include "htsdefines.h"
+
+/* local function called as "check_html" callback */
+static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt, 
+                        /*other parameters are callback-specific*/
+                        char* html, int len, const char* url_address, const char* url_file) {
+  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
+
+  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
+  if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
+    if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
+                                                html, len, url_address, url_file)) {
+        return 0;  /* abort */
+      }
+  }
+
+  printf("file %s%s content: %s\n", url_address, url_file, html);
+  return 1;  /* success */
+}
+
+/* local function called as "end" callback */
+static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
+  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
+
+  /* processing */
+  fprintf(stderr, "That's all, folks!\n");
+
+  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
+  if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
+    /* status is ok on our side, return other callabck's status */
+    return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
+  }
+
+  return 1;  /* success */
+}
+
+/*
+module entry point
+the function name and prototype MUST match this prototype
+*/
+EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
+  /* optional argument passed in the commandline we won't be using here */
+  const char *arg = strchr(argv, ',');
+  if (arg != NULL)
+    arg++;
+
+  /* plug callback functions */
+  CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
+  CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
+
+  return 1;  /* success */
+}
+
+/*
+module exit point
+the function name and prototype MUST match this prototype
+*/
+EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
+  fprintf(stder, "Module unplugged");
+
+  return 1;  /* success */
+}
+</pre>
+</td></tr></table>
+
+<br />
+Compile this file ; for example:
+<br />
+<tt>
+gcc -O -g3 -shared -o mylibrary.so myexample.c
+</tt>
+<br />
+and plug the module using the commandline ; for example:
+<br />
+<tt>
+httrack --wrapper mylibrary http://www.example.com
+</tt>
+<br />
+or, if some parameters are desired:
+<br />
+<tt>
+httrack --wrapper mylibrary,myparameter-string http://www.example.com
+</tt>
+<br />
+(the "myparameter-string" string will be available in the 'arg' parameter passed to the hts_plug entry point)
+<br />
+
 
-<br />
-An example is generally more efficient than anything else, so let's write our first module, aimed to stupidely print all parsed html files:
-<table width="100%" border="2">
-<tr><td>
-<pre>
-/* system includes */
-#include &lt;stdio.h&gt;
-#include &lt;stdlib.h&gt;
-#include &lt;string.h&gt;
-
-/* standard httrack module includes */
-#include "httrack-library.h"
-#include "htsopt.h"
-#include "htsdefines.h"
-
-/* local function called as "check_html" callback */
-static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt, 
-                        /*other parameters are callback-specific*/
-                        char* html, int len, const char* url_address, const char* url_file) {
-  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
-
-  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
-  if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
-    if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
-                                                html, len, url_address, url_file)) {
-        return 0;  /* abort */
-      }
-  }
-
-  printf("file %s%s content: %s\n", url_address, url_file, html);
-  return 1;  /* success */
-}
-
-/* local function called as "end" callback */
-static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
-  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
-
-  /* processing */
-  fprintf(stderr, "That's all, folks!\n");
-
-  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
-  if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
-    /* status is ok on our side, return other callabck's status */
-    return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
-  }
-
-  return 1;  /* success */
-}
-
-/*
-module entry point
-the function name and prototype MUST match this prototype
-*/
-EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
-  /* optional argument passed in the commandline we won't be using here */
-  const char *arg = strchr(argv, ',');
-  if (arg != NULL)
-    arg++;
-
-  /* plug callback functions */
-  CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
-  CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
-
-  return 1;  /* success */
-}
-
-/*
-module exit point
-the function name and prototype MUST match this prototype
-*/
-EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
-  fprintf(stder, "Module unplugged");
-
-  return 1;  /* success */
-}
-</pre>
-</td></tr></table>
-
-<br />
-Compile this file ; for example:
-<br />
-<tt>
-gcc -O -g3 -shared -o mylibrary.so myexample.c
-</tt>
-<br />
-and plug the module using the commandline ; for example:
-<br />
-<tt>
-httrack --wrapper mylibrary http://www.example.com
-</tt>
-<br />
-or, if some parameters are desired:
-<br />
-<tt>
-httrack --wrapper mylibrary,myparameter-string http://www.example.com
-</tt>
-<br />
-(the "myparameter-string" string will be available in the 'arg' parameter passed to the hts_plug entry point)
-<br />
-
-
 <br><br>
 
 <!-- ==================== Start epilogue ==================== -->

html/step9_opt6.html

@@ -118,8 +118,8 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
   <br>Avoid "never", because the local mirror could be bogus
   </small><br><br>
 <!-- -->
-  <li>Parse java files</li>
-  <br><small>Must the engine parse .java files (java classes) to seek included filenames?
+  <li>Parse scripts</li>
+  <br><small>Must the engine parse scripts to seek included filenames?
   <br>It is checked by default
   </small><br><br>
 <!-- -->

m4/ax_add_fortify_source.m4

@@ -0,0 +1,119 @@
+# ===========================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_ADD_FORTIFY_SOURCE
+#
+# DESCRIPTION
+#
+#   Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
+#   redefinition warnings, other cpp warnings or linker. Some distributions
+#   (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
+#   their compilers, leading to unnecessary warnings in the form of
+#
+#     <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
+#     <built-in>: note: this is the location of the previous definition
+#
+#   which is a problem if -Werror is enabled. This macro checks whether
+#   _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
+#   to CPPFLAGS.
+#
+#   Newer mingw-w64 msys2 package comes with a bug in
+#   headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
+#   and would need -lssp or -fstack-protector.  See
+#   https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
+#   link it.
+#
+# LICENSE
+#
+#   Copyright (c) 2017 David Seifert <soap@gentoo.org>
+#   Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#serial 10
+
+AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
+    ac_save_cflags=$CFLAGS
+    ac_cwerror_flag=yes
+    AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
+    ax_add_fortify_3_failed=
+    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
+    AC_LINK_IFELSE([
+        AC_LANG_PROGRAM([],
+            [[
+                #ifndef _FORTIFY_SOURCE
+                    return 0;
+                #else
+                    _FORTIFY_SOURCE_already_defined;
+                #endif
+            ]]
+        )],
+        AC_LINK_IFELSE([
+            AC_LANG_SOURCE([[
+                #define _FORTIFY_SOURCE 3
+                #include <string.h>
+                int main(void) {
+                    char *s = " ";
+                    strcpy(s, "x");
+                    return strlen(s)-1;
+                }
+              ]]
+            )],
+            [
+              AC_MSG_RESULT([yes])
+              CFLAGS=$ac_save_cflags
+              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
+            ], [
+              AC_MSG_RESULT([no])
+              ax_add_fortify_3_failed=1
+            ],
+        ),
+        [
+          AC_MSG_RESULT([no])
+          ax_add_fortify_3_failed=1
+        ])
+    if test -n "$ax_add_fortify_3_failed"
+    then
+    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
+    AC_LINK_IFELSE([
+        AC_LANG_PROGRAM([],
+            [[
+                #ifndef _FORTIFY_SOURCE
+                    return 0;
+                #else
+                    _FORTIFY_SOURCE_already_defined;
+                #endif
+            ]]
+        )],
+        AC_LINK_IFELSE([
+            AC_LANG_SOURCE([[
+                #define _FORTIFY_SOURCE 2
+                #include <string.h>
+                int main(void) {
+                    char *s = " ";
+                    strcpy(s, "x");
+                    return strlen(s)-1;
+                }
+              ]]
+            )],
+            [
+              AC_MSG_RESULT([yes])
+              CFLAGS=$ac_save_cflags
+              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
+            ], [
+              AC_MSG_RESULT([no])
+              CFLAGS=$ac_save_cflags
+            ],
+        ),
+        [
+          AC_MSG_RESULT([no])
+          CFLAGS=$ac_save_cflags
+        ])
+    fi
+])

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "27 June 2026" "httrack website copier"
+.TH httrack 1 "12 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -51,6 +51,7 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
 [ \fB\-%K, \-\-cookies\-file\fR ]
+[ \fB\-%Y, \-\-why\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -218,10 +219,12 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
 .IP \-%K
 load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
+.IP \-%Y
+explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j
-*parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
+*parse scripts (j0 don't parse, bitmask: |1 parse default, |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
 .IP \-sN
 follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules)) (\-\-robots[=N])
 .IP \-%h
@@ -245,7 +248,7 @@ can also be used to force a specific file type: \-\-assume foo.cgi=text/html
 .IP \-@iN
 internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only) (\-\-protocol[=N])
 .IP \-%w
-disable a specific external mime module (\-%w htsswf \-%w htsjava) (\-\-disable\-module <param>)
+disable a specific external mime module (\-%w httrack\-plugin) (\-\-disable\-module <param>)
 .SS Browser ID:
 .IP \-F
 user\-agent field sent in HTTP headers (\-F "user\-agent name") (\-\-user\-agent <param>)
@@ -511,8 +514,6 @@ that would have solved them.
 .SM
 \- Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
 .SM
-\- Some java classes may not find some files on them (class included)
-.SM
 \- Cgi\-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like \-*cgi\-bin*
 .SH BUGS
 Please reports bugs to

src/Makefile.am

@@ -44,7 +44,7 @@ httrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 proxytrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 
-lib_LTLIBRARIES = libhttrack.la libhtsjava.la
+lib_LTLIBRARIES = libhttrack.la
 
 htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h
 proxytrack_SOURCES = proxy/main.c \
@@ -62,7 +62,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htsname.c htsrobots.c htstools.c htswizard.c \
 	htsalias.c htsthread.c htsindex.c htsbauth.c \
 	htsmd5.c htszlib.c htswrap.c htsconcat.c \
-	htsmodules.c htscharset.c punycode.c htsencoding.c \
+	htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
 	md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
 	hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
@@ -70,7 +70,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h  \
 	htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
 	htshelp.h htsindex.h htslib.h htsmd5.h \
-	htsmodules.h htsname.h htsnet.h \
+	htsmodules.h htsname.h htsnet.h htssniff.h \
 	htsopt.h htsrobots.h htsthread.h  \
 	htstools.h htswizard.h htswrap.h htszlib.h  \
 	htsstrings.h htsarrays.h httrack-library.h \
@@ -82,14 +82,6 @@ libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(OPENSSL_LIBS) $(DL_LIBS) $
 libhttrack_la_CFLAGS = $(AM_CFLAGS) -DLIBHTTRACK_EXPORTS -DZLIB_CONST
 libhttrack_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO)
 
-libhtsjava_la_SOURCES = htsjava.c htsjava.h
-libhtsjava_la_LIBADD = $(THREADS_LIBS) $(DL_LIBS) libhttrack.la
-# This thin JNI wrapper reaches libc only through libhttrack, so the direct
-# libc edge is dropped from DT_NEEDED (library-not-linked-against-libc). Force
-# libc back as a dependency; configure gates the flag since only a GNU-style
-# linker accepts it (LIBC_FORCE_LINK is empty on e.g. macOS).
-libhtsjava_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO) $(LIBC_FORCE_LINK)
-
 EXTRA_DIST = httrack.h webhttrack \
 		coucal/murmurhash3.h.diff \
 		coucal/murmurhash3.h.orig \
@@ -119,7 +111,6 @@ EXTRA_DIST = httrack.h webhttrack \
 		coucal/README.md \
 		coucal/sample.c \
 		coucal/tests.c \
-		htsjava.vcproj \
 		httrack.dsp httrack.dsw httrack.vcproj \
 		libhttrack.dsp libhttrack.dsw libhttrack.vcproj \
 		webhttrack.dsp webhttrack.dsw webhttrack.vcproj

src/htsalias.c

@@ -114,6 +114,8 @@ const char *hts_optalias[][4] = {
    "strip [host/pattern=]key1,key2,... from URLs"},
   {"cookies-file", "-%K", "param1",
    "load extra cookies from a Netscape cookies.txt"},
+  {"why", "-%Y", "param1",
+   "explain which filter rule accepts or rejects a URL, then exit"},
   {"pause", "-%G", "param1",
    "random pause of MIN[:MAX] seconds between files"},
   {"generate-errors", "-o", "single", ""},
@@ -283,7 +285,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
       char *position;
       int need_param = 1;
 
-      //int return_param=0;
       int pos;
 
       command[0] = param[0] = addcommand[0] = '\0';
@@ -361,7 +362,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
               strlcatbuff(return_argv[0], "0", return_argv_size);
             else if (strcmp(param, "on") == 0) {
               // on is the default
-              // strcatbuff(return_argv[0],"1");
             } else
               strlcatbuff(return_argv[0], param, return_argv_size);
           }

src/htsback.h

@@ -113,6 +113,12 @@ void back_set_locked(struct_back * sback, const int p);
 void back_set_unlocked(struct_back * sback, const int p);
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p);
+/* Discard back's on-disk .delayed placeholder and its refname. */
+void back_delayed_discard(httrackp *opt, lien_back *back);
+/* Move back's .delayed placeholder (and open stream) to newname;
+   HTS_FALSE = file lost, slot flagged in error. */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname);
 void back_index_unlock(struct_back * sback, const int p);
 int back_clear_entry(lien_back * back);
 int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
@@ -132,10 +138,11 @@ LLint back_transferred(LLint add, struct_back * sback);
 
 // hostback
 #if HTS_XGETHOST
-void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
+/* Enforce -M/-E quotas: smooth-stops when reached; returns 0 once the -M cap
+   or -E deadline overruns its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htsbasenet.h

@@ -55,8 +55,6 @@ Please visit our Website: http://www.httrack.com
 // inline function in Wspiapi.h.
 #include <ws2tcpip.h>
 #include <Wspiapi.h>
-// #include <winsock2.h>
-// #include <tpipv6.h>
 
 #endif
 

src/htsbauth.c

@@ -107,7 +107,6 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
 #if DEBUG_COOK
   printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
          cook_name, cook_value, domain, path);
-  //printf(">>>cook: %s<<<\n",cookie->data);
 #endif
   return 0;
 }
@@ -208,8 +207,6 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   char catbuff[CATBUFF_SIZE];
   char buffer[8192];
 
-  //  cookie->data[0]='\0';
-
   // Fusionner d'abord les éventuels cookies IE
 #ifdef _WIN32
   {
@@ -310,14 +307,25 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   return -1;
 }
 
-// écrire cookies.txt
-// !=0 : erreur
+/* Write cookies.txt; returns 0 on success. The jar holds live session
+   cookies, so keep it owner-only on Unix (Windows inherits folder ACLs). */
 int cookie_save(t_cookie * cookie, const char *name) {
   char catbuff[CATBUFF_SIZE];
 
   if (strnotempty(cookie->data)) {
     char BIGSTK line[8192];
+#ifdef _WIN32
     FILE *fp = fopen(fconv(catbuff, sizeof(catbuff), name), "wb");
+#else
+    const int fd = open(fconv(catbuff, sizeof(catbuff), name),
+                        O_WRONLY | O_CREAT | O_TRUNC, HTS_PROTECT_FILE);
+    FILE *fp = fd != -1 ? fdopen(fd, "wb") : NULL;
+
+    if (fd != -1 && fp == NULL)
+      close(fd);    /* fdopen failed: don't leak the descriptor */
+    if (fp != NULL) /* O_CREAT's mode skips pre-existing jars: tighten those */
+      (void) fchmod(fd, HTS_PROTECT_FILE);
+#endif
 
     if (fp) {
       char *a = cookie->data;

src/htscache.h

@@ -66,8 +66,11 @@ htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr,
                   const char *fil, const char *save, char *location);
 htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, const char *save, char *location);
-htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
-                                   const char *adr, const char *fil);
+/* Like cache_read, but also yields entries whose transfer broke; return_save
+   (optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name. */
+htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
+                                   const char *adr, const char *fil,
+                                   char *return_save);
 htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr,
                     const char *fil, const char *save, char *location,
                     char *return_save, int readonly);
@@ -75,14 +78,22 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, htsblk * r);
 void cache_init(cache_back * cache, httrackp * opt);
 
-int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
-                    const char *str2, char *outbuff, int len);
-int cache_readdata(cache_back * cache, const char *str1, const char *str2,
-                   char **inbuff, int *len);
+/* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
+typedef enum {
+  CACHE_RECONCILE_PROMOTE,     /* no new cache: promote the old generation */
+  CACHE_RECONCILE_INTERRUPTED, /* aborted run: keep the larger generation */
+  CACHE_RECONCILE_ROLLBACK     /* nothing transferred: restore the old one */
+} hts_cache_reconcile_mode;
+
+/* Reconcile the on-disk cache generations according to mode; a no-op when
+   the involved files are absent. */
+void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
 
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);
+/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
+int cache_binput(char *adr, const char *end, char *s, int max);
 int cache_brint(char *adr, int *i);
 void cache_rint(FILE * fp, int *i);
 void cache_rLLint(FILE * fp, LLint * i);

src/htscache_selftest.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 #include "htszlib.h"
 
 #include <errno.h>
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -77,14 +78,6 @@ static void selftest_open_for_read(cache_back *cache, httrackp *opt) {
 }
 
 static void selftest_close(cache_back *cache) {
-  if (cache->dat != NULL) {
-    fclose(cache->dat);
-    cache->dat = NULL;
-  }
-  if (cache->ndx != NULL) {
-    fclose(cache->ndx);
-    cache->ndx = NULL;
-  }
   if (cache->zipOutput != NULL) {
     zipClose(cache->zipOutput,
              "Created by HTTrack Website Copier (cache self-test)");
@@ -321,6 +314,7 @@ typedef struct {
   size_t budget;  /**< bytes allowed through before writes start failing */
   int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
   int writes;     /**< zwrite call count, to detect re-entry into the stream */
+  int fail_once;  /**< recover (unlimited budget) after the first failure */
 } writefail_inject;
 
 /* zwrite that copies until the budget runs out, then fails with inj->fail_errno
@@ -335,6 +329,8 @@ static uLong selftest_failing_zwrite(voidpf opaque, voidpf stream,
     inj->budget -= (size_t) size;
     return (uLong) fwrite(buf, 1, (size_t) size, (FILE *) stream);
   }
+  if (inj->fail_once)
+    inj->budget = (size_t) -1; /* the backend recovers after this failure */
   errno = inj->fail_errno;
   return 0; /* short write -> the minizip op returns an error */
 }
@@ -373,9 +369,50 @@ static void writefail_store(httrackp *opt, cache_back *cache, const char *fil,
   freet(bodycopy);
 }
 
-/* #174/#219: a failing cache write used to crash via assertf(); it must instead
-   stop the mirror (exit_xh = -1) without crashing. Assert that, plus the cache
-   is flagged and a sibling write doesn't re-enter the broken stream. */
+/* Store an entry claiming a >2GB body; the degrade path never reads data. */
+static void writefail_store_oversized(httrackp *opt, cache_back *cache,
+                                      const char *fil, int is_write) {
+  htsblk r;
+  char locbuf[4];
+
+  hts_init_htsblk(&r);
+  r.statuscode = 200;
+  r.size = (LLint) INT_MAX + 1;
+  strcpybuff(r.msg, "OK");
+  strcpybuff(r.contenttype, "application/octet-stream");
+  locbuf[0] = '\0';
+  r.location = locbuf;
+  r.is_write = (short int) is_write;
+  cache_add(opt, cache, &r, "example.com", fil, "example.com/big.bin", 1, NULL);
+}
+
+/* Read back `entryname`: extra field (cached headers) and body. Returns the
+   body length, or -1 if the entry is absent or unreadable. */
+static int writefail_read_entry(const char *path, const char *entryname,
+                                char *extra, size_t extralen, char *body,
+                                size_t bodylen) {
+  unzFile z = unzOpen(path);
+  int n = -1;
+
+  if (z == NULL)
+    return -1;
+  if (unzLocateFile(z, entryname, 1) == UNZ_OK &&
+      unzOpenCurrentFile(z) == UNZ_OK) {
+    const int elen = unzGetLocalExtrafield(z, extra, (unsigned) (extralen - 1));
+
+    if (elen >= 0) {
+      extra[elen] = '\0';
+      n = unzReadCurrentFile(z, body, (unsigned) bodylen);
+    }
+    unzCloseCurrentFile(z);
+  }
+  unzClose(z);
+  return n;
+}
+
+/* Cache write-failure policy (#174/#219): fatal errno or a failure streak
+   stops the mirror (exit_xh=-1, no crash); isolated/oversized drops the entry.
+ */
 int cache_write_failure_selftest(httrackp *opt, const char *dir) {
   int fail = 0;
   char path[HTS_URLMAXSIZE];
@@ -388,9 +425,8 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
   gen_body(body, body_len, 1 /* incompressible */);
   fconcat(path, sizeof(path), dir, "/wfail.zip");
 
-  /* phase 0: fail on the body write, fatal errno (ENOSPC, the disk-full
-     branch). phase 1: fail on the open, non-fatal errno (EIO, dropped-share
-     branch). Both must abort the mirror. */
+  /* phase 0: fatal errno (ENOSPC) aborts at once; phase 1: persistent EIO
+     drops entries until the streak caps out, then aborts. */
   for (phase = 0; phase < 2; phase++) {
     cache_back cache;
     writefail_inject inj;
@@ -399,6 +435,7 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     inj.budget = (phase == 0) ? 4096 : 0;
     inj.fail_errno = (phase == 0) ? ENOSPC : EIO;
     inj.writes = 0;
+    inj.fail_once = 0;
     memset(&cache, 0, sizeof(cache));
     cache.type = 1;
     cache.log = stderr;
@@ -412,7 +449,25 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     }
 
     opt->state.exit_xh = 0; /* clear; the failing write must set it to -1 */
-    writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    if (phase == 0) {
+      writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    } else {
+      /* the abort must land exactly on the 8th consecutive failure */
+      int i;
+
+      for (i = 0; i < 7; i++) {
+        char fil[32];
+
+        snprintf(fil, sizeof(fil), "/b%d.bin", i);
+        writefail_store(opt, &cache, fil, body, 16);
+      }
+      if (cache.zipWriteFailed) {
+        fprintf(stderr, "cache-writefail: phase 1: aborted before the "
+                        "8th consecutive failure\n");
+        fail++;
+      }
+      writefail_store(opt, &cache, "/b7.bin", body, 16);
+    }
     if (!cache.zipWriteFailed) {
       fprintf(stderr, "cache-writefail: phase %d: write error not caught\n",
               phase);
@@ -443,6 +498,136 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     }
   }
 
+  /* failures with successes in between reset the streak: never aborts */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    int i;
+
+    inj.budget = (size_t) -1;
+    inj.fail_errno = EIO;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    for (i = 0; i < 10; i++) {
+      char fil[32];
+
+      inj.budget = 0; /* this store fails */
+      snprintf(fil, sizeof(fil), "/s%d.bin", i);
+      writefail_store(opt, &cache, fil, body, 16);
+      inj.budget = (size_t) -1; /* this one succeeds and resets the streak */
+      snprintf(fil, sizeof(fil), "/ok%d.bin", i);
+      writefail_store(opt, &cache, fil, body, 16);
+    }
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: scattered: non-consecutive failures aborted "
+              "the mirror (flagged=%d, exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+  }
+
+  /* isolated failure: only that entry drops; a later sibling round-trips */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    char extra[8192];
+    char rbody[64];
+    int n;
+
+    inj.budget = 4096;
+    inj.fail_errno = EIO;
+    inj.writes = 0;
+    inj.fail_once = 1;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: skip: isolated failure aborted the mirror "
+              "(flagged=%d, exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    writefail_store(opt, &cache, "/blob2.bin", body, 16);
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+    n = writefail_read_entry(path, "http://example.com/blob2.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody));
+    if (n != 16 || memcmp(rbody, body, 16) != 0) {
+      fprintf(stderr,
+              "cache-writefail: skip: sibling entry lost after a skipped "
+              "entry (%d)\n",
+              n);
+      fail++;
+    }
+  }
+
+  /* >2GB bodies: in-memory drops the entry, on-disk degrades to headers-only */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    char extra[8192];
+    char rbody[64];
+    int n;
+
+    inj.budget = (size_t) -1; /* no injected failure */
+    inj.fail_errno = 0;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    writefail_store_oversized(opt, &cache, "/bigmem.bin", 0 /* in-memory */);
+    writefail_store_oversized(opt, &cache, "/bigdisk.bin", 1 /* on-disk */);
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: oversize: mirror aborted (flagged=%d, "
+              "exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    if (writefail_read_entry(path, "http://example.com/bigmem.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody)) >= 0) {
+      fprintf(stderr,
+              "cache-writefail: oversize: in-memory entry was stored\n");
+      fail++;
+    }
+    n = writefail_read_entry(path, "http://example.com/bigdisk.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody));
+    if (n != 0 || strstr(extra, "X-In-Cache: 0") == NULL) {
+      fprintf(stderr,
+              "cache-writefail: oversize: on-disk entry not stored "
+              "headers-only (%d)\n",
+              n);
+      fail++;
+    }
+  }
+
   freet(body);
   return fail;
 }
@@ -716,3 +901,583 @@ int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
 
   return failures;
 }
+
+/* --- hts_cache_reconcile() policies -------------------------------------- */
+
+/* All reconcile inputs/outputs, wiped between cases. */
+static const char *const reconcile_files[] = {
+    "hts-cache/new.zip", "hts-cache/old.zip",   "hts-cache/new.dat",
+    "hts-cache/old.dat", "hts-cache/new.ndx",   "hts-cache/old.ndx",
+    "hts-cache/new.lst", "hts-cache/old.lst",   "hts-cache/new.txt",
+    "hts-cache/old.txt", "hts-in_progress.lock"};
+
+static char *reconcile_st_path(httrackp *opt, const char *name) {
+  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                 StringBuff(opt->path_log), name);
+}
+
+static void reconcile_wipe(httrackp *opt) {
+  size_t i;
+
+  for (i = 0; i < sizeof(reconcile_files) / sizeof(reconcile_files[0]); i++)
+    remove(reconcile_st_path(opt, reconcile_files[i]));
+}
+
+/* Create a filler file of exactly `size` bytes. */
+static void reconcile_put(httrackp *opt, const char *name, size_t size) {
+  FILE *const fp = fopen(reconcile_st_path(opt, name), "wb");
+  static const char filler[1024] = {'x'};
+
+  assertf(fp != NULL);
+  while (size > 0) {
+    const size_t n = size > sizeof(filler) ? sizeof(filler) : size;
+
+    assertf(fwrite(filler, 1, n, fp) == n);
+    size -= n;
+  }
+  fclose(fp);
+}
+
+/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
+static int reconcile_expect(httrackp *opt, const char *name, off_t size,
+                            const char *what) {
+  const off_t got = fsize(reconcile_st_path(opt, name));
+
+  if (got != size) {
+    fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
+            name, (int) got, (int) size);
+    return 1;
+  }
+  return 0;
+}
+
+int cache_reconcile_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+
+  /* around the interrupted-run thresholds (new < 32768, old > 65536) */
+  static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
+
+  golden_setup(opt, dir);
+#ifdef _WIN32
+  mkdir(reconcile_st_path(opt, "hts-cache"));
+#else
+  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
+#endif
+
+  /* PROMOTE: a zip old generation replaces a missing new one */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "promote-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "promote-zip");
+
+  /* PROMOTE: an existing new.zip is left alone */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "promote-zip-noop");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
+
+  /* PROMOTE: the removed pre-3.31 legacy pair is left alone */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures += reconcile_expect(opt, "hts-cache/new.dat", -1, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.ndx", TINY, "promote-dat");
+
+  /* INTERRUPTED: no lock file, no action */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-nolock");
+
+  /* INTERRUPTED: an absent new.zip must NOT promote old.zip (fsize(-1) would
+     spuriously pass "< TINY"); leave the solid old generation for ROLLBACK */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", -1, "interrupted-nonew");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "interrupted-nonew");
+
+  /* INTERRUPTED: stalled tiny new.zip loses to a solid old.zip (was dead for
+     zip caches: the arm was gated on a legacy new.dat) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", SOLID, "interrupted-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "interrupted-zip");
+
+  /* INTERRUPTED: old below the confidence threshold, keep new */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-smallold");
+
+  /* INTERRUPTED: new big enough to trust, keep it */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", MID);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
+
+  /* INTERRUPTED: the removed pre-3.31 legacy pair is left alone */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", TINY, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.dat", SOLID, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.ndx", MID, "interrupted-dat");
+
+  /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
+     its only good generation here) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "rollback-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "rollback-zip");
+
+  /* ROLLBACK: sidecars are restored regardless of format */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.lst", TINY);
+  reconcile_put(opt, "hts-cache/old.lst", MID);
+  reconcile_put(opt, "hts-cache/old.txt", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
+  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
+
+  /* ROLLBACK: sidecars restored, the removed pre-3.31 pair left alone */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", MID);
+  reconcile_put(opt, "hts-cache/old.lst", MID);
+  reconcile_put(opt, "hts-cache/old.txt", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.dat", TINY, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.ndx", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
+
+  /* ROLLBACK: nothing to restore, the new generation stays */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", TINY, "rollback-noop");
+
+  reconcile_wipe(opt);
+  return failures;
+}
+
+/* The pre-3.31 .dat/.ndx import was removed: cache_init must refuse the
+   legacy pair, leave the files in place, and not flag an update run. */
+int cache_legacy_refused_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+  int variant;
+
+  selftest_tag = "cache-legacy";
+  golden_setup(opt, dir);
+#ifdef _WIN32
+  mkdir(reconcile_st_path(opt, "hts-cache"));
+#else
+  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
+#endif
+
+  /* variant 0: old.dat/old.ndx (--update layout); 1: new.dat/new.ndx (ro) */
+  for (variant = 0; variant < 2; variant++) {
+    cache_back cache;
+    const char *const dat =
+        variant == 0 ? "hts-cache/old.dat" : "hts-cache/new.dat";
+    const char *const ndx =
+        variant == 0 ? "hts-cache/old.ndx" : "hts-cache/new.ndx";
+
+    reconcile_wipe(opt);
+    reconcile_put(opt, dat, 4096);
+    reconcile_put(opt, ndx, 4096);
+    opt->is_update = 0;
+    selftest_open(&cache, opt, /*ro*/ variant == 1);
+    if (cache_readable(&cache)) {
+      printf("cache-legacy: variant %d imported a removed format\n", variant);
+      failures++;
+    }
+    if (coucal_nitems(cache.hashtable) != 0) { /* no phantom index entries */
+      printf("cache-legacy: variant %d imported index entries\n", variant);
+      failures++;
+    }
+    if (opt->is_update) {
+      printf("cache-legacy: variant %d flagged an update\n", variant);
+      failures++;
+    }
+    if (fsize(reconcile_st_path(opt, dat)) != 4096 ||
+        fsize(reconcile_st_path(opt, ndx)) != 4096) {
+      printf("cache-legacy: variant %d touched the legacy files\n", variant);
+      failures++;
+    }
+    if (cache.lst != NULL) {
+      fclose(cache.lst);
+      cache.lst = NULL;
+    }
+    if (cache.txt != NULL) {
+      fclose(cache.txt);
+      cache.txt = NULL;
+    }
+    selftest_close(&cache);
+  }
+
+  /* a healthy zip must win over stray legacy files, with no refusal */
+  {
+    cache_back cache;
+    const char *const body = "<html>zip wins</html>";
+
+    reconcile_wipe(opt);
+    selftest_open_for_write(&cache, opt);
+    store_entry(opt, &cache, "example.com", "/", "example.com/index.html", 200,
+                "OK", "text/html", "utf-8", "", "", "", "", body, strlen(body));
+    selftest_close(&cache);
+    reconcile_put(opt, "hts-cache/old.ndx", 4096);
+    reconcile_put(opt, "hts-cache/old.dat", 4096);
+    selftest_open_for_read(&cache, opt);
+    if (!cache_readable(&cache)) {
+      printf("cache-legacy: stray legacy files shadowed the zip cache\n");
+      failures++;
+    }
+    failures +=
+        check_entry(opt, &cache, "example.com", "/", 200, "OK", "text/html",
+                    "utf-8", "", "", "", "", body, strlen(body));
+    if (fsize(reconcile_st_path(opt, "hts-cache/old.ndx")) != 4096 ||
+        fsize(reconcile_st_path(opt, "hts-cache/old.dat")) != 4096) {
+      printf("cache-legacy: zip-wins pass touched the legacy files\n");
+      failures++;
+    }
+    if (cache.lst != NULL) {
+      fclose(cache.lst);
+      cache.lst = NULL;
+    }
+    if (cache.txt != NULL) {
+      fclose(cache.txt);
+      cache.txt = NULL;
+    }
+    selftest_close(&cache);
+  }
+
+  reconcile_wipe(opt);
+  return failures;
+}
+
+/* --- read-side corruption injection --------------------------------------- */
+
+/* canary read back intact after each corruption; victim gets the byte surgery
+ */
+#define CORRUPT_ADR "corrupt.example.com"
+static char corrupt_body_a[33 + 1];
+static char corrupt_body_b[44 + 1];
+
+/* Write a fresh two-entry cache: /canary.html then /victim.html. */
+static void corrupt_build(httrackp *opt) {
+  cache_back cache;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
+              strlen(corrupt_body_b));
+  selftest_close(&cache);
+}
+
+/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
+   is later overwritten with a forged oversized X-Size (same byte length). */
+static void corrupt_build_etag(httrackp *opt) {
+  cache_back cache;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
+              "OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
+              corrupt_body_b, strlen(corrupt_body_b));
+  selftest_close(&cache);
+}
+
+/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
+   body on disk): the shape every non-html file is stored with. */
+static void corrupt_build_disk(httrackp *opt) {
+  cache_back cache;
+  htsblk w;
+  char locw[4];
+  char BIGSTK save[HTS_URLMAXSIZE * 2];
+  char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
+  char *path;
+  FILE *fp;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
+          CORRUPT_ADR "/victim.bin");
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  hts_init_htsblk(&w);
+  w.statuscode = 200;
+  w.size = (LLint) sizeof(corrupt_body_b) - 1;
+  strcpybuff(w.msg, "OK");
+  strcpybuff(w.contenttype, "application/octet-stream");
+  strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
+  locw[0] = '\0';
+  w.location = locw;
+  w.is_write = 0;
+  cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
+            0 /* all_in_cache */, NULL);
+  selftest_close(&cache);
+  /* the reader only checks this file exists; it never reads it here */
+  path = fconv(catbuff, sizeof(catbuff), save);
+  (void) structcheck(path);
+  fp = FOPEN(path, "wb");
+  assertf(fp != NULL);
+  fclose(fp);
+}
+
+/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
+static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
+                          const char *rep, size_t nth, size_t total) {
+  LLint fsz = 0;
+  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
+  const size_t n = (size_t) fsz;
+  size_t k, hits = 0, at = 0;
+  FILE *fp;
+
+  assertf(data != NULL);
+  for (k = 0; k + patlen <= n; k++) {
+    if (memcmp(data + k, pat, patlen) == 0) {
+      hits++;
+      if (hits == nth)
+        at = k;
+    }
+  }
+  assertf(hits == total);
+  memcpy(data + at, rep, patlen);
+  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, n, fp) == n);
+  fclose(fp);
+  freet(data);
+}
+
+/* Garbage the first bytes of the victim's deflated data (2nd local header). */
+static void corrupt_victim_body(httrackp *opt) {
+  LLint fsz = 0;
+  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
+  const size_t n = (size_t) fsz;
+  size_t k, hits = 0, off = 0;
+  FILE *fp;
+
+  assertf(data != NULL);
+  for (k = 0; k + 4 <= n; k++) {
+    if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
+      const size_t namelen =
+          (unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
+      const size_t extralen =
+          (unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
+
+      off = k + 30 + namelen + extralen;
+    }
+  }
+  assertf(hits == 2);
+  assertf(off != 0 && off + 4 <= n);
+  memset(data + off, 0xFF, 4);
+  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, n, fp) == n);
+  fclose(fp);
+  freet(data);
+}
+
+/* Read the corrupt /victim.html and, in the SAME read session, the intact
+   /canary.html: the victim must be rejected (wantmsg pins which path) and the
+   canary must still decode byte-exact, proving one bad entry never taints a
+   sibling read. */
+static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
+                                     const char *wantmsg, const char *what) {
+  cache_back cache;
+  htsblk v, c;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  char BIGSTK lc[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = lc[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
+  if (v.statuscode != STATUSCODE_INVALID) {
+    fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
+            selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
+    fail++;
+  }
+  if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
+    fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
+            selftest_tag, what, v.msg, wantmsg);
+    fail++;
+  }
+  c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
+  if (c.statuscode != 200 || c.adr == NULL ||
+      c.size != (LLint) strlen(corrupt_body_a) ||
+      memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
+    fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
+            c.statuscode);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  if (c.adr != NULL)
+    freet(c.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
+static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
+                                 const char *what) {
+  return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
+}
+
+/* Headers-only probe of the disk victim: must parse OK with the size kept. */
+static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
+                                      const char *what) {
+  cache_back cache;
+  htsblk v;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
+  if (v.statuscode != 200 || v.size != wantsize) {
+    fprintf(stderr,
+            "%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
+            selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
+/* One zip corruption case: build, patch, then check victim+canary in-session.
+ */
+static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
+                            size_t nth, size_t total, const char *wantmsg,
+                            const char *what) {
+  corrupt_build(opt);
+  corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
+  return corrupt_expect_victim(opt, wantmsg, what);
+}
+
+int cache_corruption_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+
+  selftest_tag = "cache-corrupt";
+  golden_setup(opt, dir);
+
+  failures +=
+      corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
+                       "Cache Read Error : Read Data", "oversized X-Size");
+  failures +=
+      corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
+                       "Cache Read Error : Bad Size", "negative X-Size");
+  /* both entries carry the line; the victim's is the second */
+  failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
+                               "Previous cache file not found (empty filename)",
+                               "blanked X-In-Cache");
+  /* smashed local file header: the entry is dropped at index load */
+  failures +=
+      corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
+                       "File Cache Entry Not Found", "smashed local header");
+
+  corrupt_build(opt);
+  corrupt_victim_body(opt);
+  failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
+                                    "garbled deflate stream");
+
+  /* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
+     truncates negative in the (int) cast the malloc uses: a wraparound alloc.
+     cache_add asserts size fits an int, so such a value only reaches the reader
+     from a corrupt/foreign cache; inject it by overwriting the victim's long
+     Etag line with a same-length forged X-Size line (the parser keeps the last
+     X-Size it sees), keeping the zip byte-length and offsets intact. */
+  corrupt_build_etag(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
+                                    "X-Size above INT_MAX");
+
+  /* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
+     is how every >2GB non-html file is stored. It must survive a header probe
+     (or every update re-fetches the file); an in-memory read still rejects. */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
+                                         "headers-only X-Size above INT_MAX");
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size above INT_MAX");
+
+  /* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483647AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size at INT_MAX");
+
+  /* the negative check must stay global, headers-only included */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: -2147483648AAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "headers-only negative X-Size");
+
+  return failures;
+}

src/htscache_selftest.h

@@ -52,10 +52,23 @@ int cache_selftests(httrackp *opt, const char *dir);
    committed file, never by the test). Returns the failed-check count. */
 int cache_golden_selftest(httrackp *opt, const char *dir, int regen);
 
-/* #174/#219: assert a failing cache write aborts the mirror cleanly instead of
-   crashing. Returns the failed-check count. */
+/* Cache write-failure policy (#174/#219): abort on fatal errno or a streak,
+   drop just the entry otherwise. Returns the failed-check count. */
 int cache_write_failure_selftest(httrackp *opt, const char *dir);
 
+/* Exercise the hts_cache_reconcile() generation policies on file fixtures
+   under <dir>. Returns the failed-check count. */
+int cache_reconcile_selftest(httrackp *opt, const char *dir);
+
+/* Verify cache_init refuses a pre-3.31 .dat/.ndx cache without touching it.
+   Returns the number of failed checks (0 = pass). */
+int cache_legacy_refused_selftest(httrackp *opt, const char *dir);
+
+/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
+   under <dir> and assert every case degrades to STATUSCODE_INVALID without
+   tainting a sibling entry. */
+int cache_corruption_selftest(httrackp *opt, const char *dir);
+
 #endif
 
 #endif

src/htscatchurl.c

@@ -175,7 +175,9 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
       //
       socinput(soc, line, 1000);
       if (strnotempty(line)) {
-        if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
+        /* widths bound the caller buffers: method[32], url[HTS_URLMAXSIZE*2],
+           protocol[256] */
+        if (sscanf(line, "%31s %2047s %255s", method, url, protocol) == 3) {
           lien_adrfil af;
 
           // méthode en majuscule
@@ -195,7 +197,6 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
             htsblk blkretour;
 
             hts_init_htsblk(&blkretour);
-            //memset(&blkretour, 0, sizeof(htsblk));    // effacer
             blkretour.location = loc;   // si non nul, contiendra l'adresse véritable en cas de moved xx
             // Lire en têtes restants
             sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
@@ -205,8 +206,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
               strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
               strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
             }
-            // CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
-            //strcatbuff(data,"\r\n");
+            // CR/LF final de l'en tête inutile car déja placé via la ligne vide
+            // juste au dessus
             if (blkretour.totalsize > 0) {
               int len = (int) min(blkretour.totalsize, 32000);
               int pos = (int) strlen(data);

src/htscharset.c

@@ -84,7 +84,7 @@ static int hts_equalsAlphanum(const char *a, const char *b) {
 
 /* Copy the memory region [s .. s + size - 1 ] as a \0-terminated string. */
 static char *hts_stringMemCopy(const char *s, size_t size) {
-  char *dest = malloc(size + 1);
+  char *dest = malloct(size + 1);
 
   if (dest != NULL) {
     memcpy(dest, s, size);
@@ -549,42 +549,6 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
 
 #endif
 
-HTS_STATIC char *hts_getCharsetFromContentType(const char *mime) {
-  /* text/html; charset=utf-8 */
-  const char *const charset = "charset";
-  char *pos = strstr(mime, charset);
-
-  if (pos != NULL) {
-    /* Skip spaces */
-    int eq = 0;
-
-    for(pos += strlen(charset);
-        *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\''; pos++) {
-      if (*pos == '=') {
-        eq = 1;
-      }
-    }
-    if (eq == 1) {
-      int len;
-
-      for(len = 0;
-          pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'';
-          pos++) ;
-      if (len != 0) {
-        char *const s = malloc(len + 1);
-        int i;
-
-        for(i = 0; i < len; i++) {
-          s[i] = pos[i];
-        }
-        s[len] = '\0';
-        return s;
-      }
-    }
-  }
-  return NULL;
-}
-
 #ifdef _WIN32
 #define strcasecmp(a,b) stricmp(a,b)
 #define strncasecmp(a,b,n) strnicmp(a,b,n)
@@ -644,58 +608,106 @@ int hts_isCharsetUTF8(const char *charset) {
          || strcasecmp(charset, "utf8") == 0 );
 }
 
+/* Extract X from a "text/html; charset=X" content= attribute value. */
+static char *charset_from_content(const char *s, size_t len) {
+  size_t i;
+
+  for (i = 0; i + 7 < len; i++) {
+    if ((i == 0 || is_space(s[i - 1]) || s[i - 1] == ';') &&
+        strncasecmp(&s[i], "charset", 7) == 0 && is_space_or_equal(s[i + 7])) {
+      size_t j, val;
+
+      for (j = i + 7; j < len && is_space_or_equal_or_quote(s[j]); j++)
+        ;
+      for (val = j; j < len && s[j] != '"' && s[j] != '\'' && s[j] != ';' &&
+                    !is_space(s[j]);
+           j++)
+        ;
+      if (j != val) {
+        return hts_stringMemCopy(&s[val], j - val);
+      }
+    }
+  }
+  return NULL;
+}
+
 char *hts_getCharsetFromMeta(const char *html, size_t size) {
-  int i;
+  size_t i;
 
-  /* <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8" > */
-  for(i = 0; i < size; i++) {
-    if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0
-        && is_space(html[i + 5])) {
-      /* Skip spaces */
-      for(i += 5; is_space(html[i]); i++) ;
-      if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0
-          && is_space_or_equal(html[i + 10])) {
-        for(i += 10; is_space_or_equal_or_quote(html[i]); i++) ;
-        if (strncasecmp(&html[i], "CONTENT-TYPE", 12) == 0) {
-          for(i += 12; is_space_or_equal_or_quote(html[i]); i++) ;
-          if (strncasecmp(&html[i], "CONTENT", 7) == 0
-              && is_space_or_equal(html[i + 7])) {
-            for(i += 7; is_space_or_equal_or_quote(html[i]); i++) ;
-            /* Skip content-type */
-            for(;
-                i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'';
-                i++) ;
-            /* Expect charset attribute here */
-            if (html[i] == ';') {
-              for(i++; is_space(html[i]); i++) ;
-              /* Look for charset */
-              if (strncasecmp(&html[i], "charset", 7) == 0
-                  && is_space_or_equal(html[i + 7])) {
-                int len;
+  /* HTML5 <meta charset=X> or legacy
+     <meta http-equiv="Content-Type" content="text/html; charset=X">,
+     attributes in any order; first resolvable tag wins. */
+  for (i = 0; i + 6 < size; i++) {
+    size_t j;
+    const char *equiv = NULL, *content = NULL;
+    size_t equiv_len = 0, content_len = 0;
 
-                for(i += 7; is_space_or_equal(html[i]) || html[i] == '\'';
-                    i++) ;
-                /* Charset */
-                for(len = 0;
-                    i + len < size && html[i + len] != '"'
-                    && html[i + len] != '\'' && html[i + len] != ' '; len++) ;
-                /* No error ? */
-                if (len != 0 && i < size) {
-                  char *const s = malloc(len + 1);
-                  int j;
+    if (html[i] != '<' || strncasecmp(&html[i + 1], "meta", 4) != 0 ||
+        !is_space(html[i + 5])) {
+      continue;
+    }
+    /* Attribute scan, strictly bounded by size. */
+    for (j = i + 6; j < size && html[j] != '>';) {
+      size_t name, name_len, val = 0, val_len = 0;
 
-                  for(j = 0; j < len; j++) {
-                    s[j] = html[i + j];
-                  }
-                  s[len] = '\0';
-                  return s;
-                }
-              }
-            }
+      if (is_space(html[j]) || html[j] == '/') {
+        j++;
+        continue;
+      }
+      for (name = j; j < size && html[j] != '=' && html[j] != '>' &&
+                     html[j] != '/' && !is_space(html[j]);
+           j++)
+        ;
+      name_len = j - name;
+      for (; j < size && is_space(html[j]); j++)
+        ;
+      if (j < size && html[j] == '=') {
+        for (j++; j < size && is_space(html[j]); j++)
+          ;
+        if (j < size && (html[j] == '"' || html[j] == '\'')) {
+          const char quote = html[j++];
+
+          for (val = j; j < size && html[j] != quote; j++)
+            ;
+          if (j >= size) /* unterminated quote: drop the tag */
+            break;
+          val_len = j++ - val;
+        } else {
+          /* unquoted: ends at whitespace or '>' only (HTML5 prescan); '/'
+             belongs to the value except as the tail of a self-close */
+          for (val = j; j < size && !is_space(html[j]) && html[j] != '>'; j++)
+            ;
+          val_len = j - val;
+          if (val_len != 0 && j < size && html[j] == '>' &&
+              html[val + val_len - 1] == '/') {
+            val_len--;
           }
         }
       }
+      /* first occurrence of each attribute wins, as in browsers */
+      if (val_len != 0) {
+        if (name_len == 7 && strncasecmp(&html[name], "charset", 7) == 0) {
+          return hts_stringMemCopy(&html[val], val_len);
+        } else if (equiv == NULL && name_len == 10 &&
+                   strncasecmp(&html[name], "http-equiv", 10) == 0) {
+          equiv = &html[val];
+          equiv_len = val_len;
+        } else if (content == NULL && name_len == 7 &&
+                   strncasecmp(&html[name], "content", 7) == 0) {
+          content = &html[val];
+          content_len = val_len;
+        }
+      }
     }
+    if (equiv_len == 12 && strncasecmp(equiv, "content-type", 12) == 0 &&
+        content != NULL) {
+      char *const s = charset_from_content(content, content_len);
+
+      if (s != NULL) {
+        return s;
+      }
+    }
+    i = j;
   }
   return NULL;
 }
@@ -841,71 +853,6 @@ static unsigned int nlz8(unsigned char x) {
     }                                           \
   } while(0)
 
-/* Sample. */
-#if 0
-
-int main(int argc, char **argv) {
-  int i;
-  int hex = 0;
-  
-#define READ_INT(DEST)                                  \
-  ( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1)      \
-    || (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
-  
-  for(i = 1 ; i < argc ; i++) {
-    unsigned int uc, from, to;
-    
-    if (strcmp(argv[i], "--hex") == 0) {
-      hex = 1;
-    }
-    else if (strcmp(argv[i], "--decimal") == 0) {
-      hex = 0;
-    }
-    else if (strcmp(argv[i], "--decode") == 0) {
-#define RD fgetc_unlocked(stdin)
-#define WR(C) do {                              \
-        if (C != -1) {                          \
-          printf("%04x\n", C);                  \
-        } else if (!feof(stdin)) {              \
-          fprintf(stderr, "read error\n");      \
-          exit(EXIT_FAILURE);                   \
-        }                                       \
-      } while(0)
-      while(!feof(stdin)) {
-        READ_UNICODE(RD, WR);
-      }
-#undef RD
-#undef WR
-    }
-    else if (strcmp(argv[i], "-range") == 0
-             && i + 2 < argc
-             && (++i, 1)
-             && READ_INT(from)
-             && (++i, 1)
-             && READ_INT(to)
-             ) {
-      unsigned int i;
-      for(i = from ; i < to ; i++) {
-#define EM(C) fputc_unlocked(C, stdout)
-        EMIT_UNICODE(i, EM);
-#undef EM
-      }
-    }
-    else if (READ_INT(uc)) {
-#define EM(C) fputc_unlocked(C, stdout)
-      EMIT_UNICODE(uc, EM);
-#undef EM
-    }
-    else {
-      return EXIT_FAILURE;
-    }
-  }
-  
-  return EXIT_SUCCESS;
-}
-
-#endif
-
 /* IDNA helpers. */
 #undef ADD_BYTE
 #undef INCREASE_CAPA
@@ -975,19 +922,19 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
                 /* Reader: can read bytes up to j */
 #define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
 
-                /* Writer: upon error, return FFFD (replacement character) */
-#define WR(C) do { \
-  if ((C) != -1) { \
-    /* copy character */ \
-    assertf(segOutputSize < segSize); \
-    segInt[segOutputSize++] = (C); \
-  } \
-  /* In case of error, abort. */ \
-  else { \
-    FREE_BUFFER(); \
-    return NULL; \
-  } \
-} while(0)
+                /* Writer: a malformed sequence abandons the encode (NULL) */
+#define WR(C)                                                                  \
+  do {                                                                         \
+    if ((C) != -1) {                                                           \
+      /* copy character */                                                     \
+      assertf(segOutputSize < segSize);                                        \
+      segInt[segOutputSize++] = (C);                                           \
+    } else {                                                                   \
+      freet(segInt);                                                           \
+      FREE_BUFFER();                                                           \
+      return NULL;                                                             \
+    }                                                                          \
+  } while (0)
 
                 /* Read/Write Unicode character. */
                 READ_UNICODE(RD, WR);
@@ -1205,17 +1152,45 @@ int hts_isStringUTF8(const char *s, size_t size) {
   const unsigned char *const data = (const unsigned char*) s;
   size_t i;
 
+  /* Strict RFC 3629: reject overlongs, surrogates, > U+10FFFF, 5/6-byte. */
   for(i = 0 ; i < size ; ) {
-    /* Reader: can read bytes up to j */
-#define RD ( i < size ? data[i++] : -1 )
+    const unsigned char c = data[i];
+    size_t len, k;
+    hts_UCS4 uc, min;
 
-    /* Writer: upon error, return FFFD (replacement character) */
-#define WR(C) if ((C) == -1) { return 0; }
+    if (c < 0x80) {
+      i++;
+      continue;
+    } else if (c >= 0xc2 && c <= 0xdf) {
+      len = 2;
+      min = 0x80;
+      uc = c & 0x1f;
+    } else if (c >= 0xe0 && c <= 0xef) {
+      len = 3;
+      min = 0x800;
+      uc = c & 0x0f;
+    } else if (c >= 0xf0 && c <= 0xf4) {
+      len = 4;
+      min = 0x10000;
+      uc = c & 0x07;
+    } else { /* continuation byte, overlong C0/C1, or F5..FF lead */
+      return 0;
+    }
+    if (size - i < len) {
+      return 0;
+    }
+    for (k = 1; k < len; k++) {
+      const unsigned char cc = data[i + k];
 
-    /* Read Unicode character. */
-    READ_UNICODE(RD, WR);
-#undef RD
-#undef WR
+      if ((cc & 0xc0) != 0x80) {
+        return 0;
+      }
+      uc = (uc << 6) | (cc & 0x3f);
+    }
+    if (uc < min || uc > 0x10FFFF || (uc >= 0xD800 && uc <= 0xDFFF)) {
+      return 0;
+    }
+    i += len;
   }
 
   return 1;

src/htscharset.h

@@ -76,7 +76,8 @@ extern char *hts_convertStringIDNAToUTF8(const char *s, size_t size);
 extern int hts_isStringIDNA(const char *s, size_t size);
 
 /**
- * Extract the charset from the HTML buffer "html"
+ * Extract the <meta> charset from the HTML buffer "html" (HTML5 charset= or
+ * legacy http-equiv form). Returns a malloc'ed string, or NULL if none.
  **/
 extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 
@@ -86,7 +87,8 @@ extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 extern int hts_isStringAscii(const char *s, size_t size);
 
 /**
- * Is the given string an UTF-8 string ?
+ * Is the given string valid UTF-8 ? Strict RFC 3629: overlong forms,
+ * surrogates, codepoints above U+10FFFF and 5/6-byte sequences are rejected.
  **/
 extern int hts_isStringUTF8(const char *s, size_t size);
 

src/htsconfig.h

@@ -54,10 +54,6 @@ Please visit our Website: http://www.httrack.com
 // extension par défaut pour fichiers n'en ayant pas
 #define DEFAULT_EXT ".html"
 #define DEFAULT_EXT_SHORT ".htm"
-// #define DEFAULT_BIN_EXT   ".bin"
-// #define DEFAULT_BIN_EXT_SHORT ".bin"
-// #define DEFAULT_EXT       ".txt"
-// #define DEFAULT_EXT_SHORT ".txt"
 
 // éviter les /nul, /con..
 #define HTS_OVERRIDE_DOS_FOLDERS 1
@@ -80,13 +76,6 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
-// gérer une table de hachage?
-// REMOVED
-// #define HTS_HASH 1
-
-// fast cache (build hash table)
-#define HTS_FAST_CACHE 1
-
 // le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
 // valide)
 #define GT_ENDS_COMMENT 1
@@ -97,27 +86,10 @@ Please visit our Website: http://www.httrack.com
 // always transform a '//' into a sigle '/'
 #define HTS_STRIP_DOUBLE_SLASH 0
 
-// case-sensitive pour les dossiers et fichiers (0/1)
-// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est
-// pas très utile..
-// ..et pas bcp respecté]
-// REMOVED
-// #define HTS_CASSE 0
-
-// Un fichier ayant une taille différente du content-length doit il être annulé?
-// SEE opt.tolerant and opt.http10
-// #define HTS_CL_IS_FATAL 0
-
 // une erreur supprime le fichier sur disque
 // (non fixé pour cause de retry)
 #define HTS_REMOVE_BAD_FILES 0
 
-// en cas de Range: xx- donnant un Content-length: xx
-// alors skipper le fichier, considéré comme transmis
-// #define HTS_SKIP_FULL_RANGE 1
-
-// nombre max de filtres que l'utilisateur peut fixer
-// #define HTS_FILTERSMAX 10000
 #define HTS_FILTERSINC 1000
 
 // connect non bloquant? (poll sur write)

src/htscore.c

@@ -119,46 +119,77 @@ hts_log_print(opt, LOG_INFO, "engine: end"); \
 RUN_CALLBACK0(opt, end); \
 }
 
-#define XH_extuninit do { \
-  HTMLCHECK_UNINIT \
-  hts_record_free(opt); \
-  if (filters && filters[0]) { \
-  freet(filters[0]); filters[0]=NULL; \
-  } \
-  if (filters) { \
-  freet(filters); filters=NULL; \
-  } \
-  back_delete_all(opt,&cache,sback); \
-  back_free(&sback); \
-  checkrobots_free(&robots);\
-  if (cache.use) { freet(cache.use); cache.use=NULL; } \
-  if (cache.dat) { fclose(cache.dat); cache.dat=NULL; }  \
-  if (cache.ndx) { fclose(cache.ndx); cache.ndx=NULL; } \
-  if (cache.zipOutput) { \
-    zipClose(cache.zipOutput, "Created by HTTrack Website Copier/"HTTRACK_VERSION); \
-    cache.zipOutput = NULL; \
-  } \
-  if (cache.zipInput) { \
-    unzClose(cache.zipInput); \
-    cache.zipInput = NULL; \
-  } \
-  if (cache.olddat) { fclose(cache.olddat); cache.olddat=NULL; } \
-  if (cache.lst) { fclose(cache.lst); cache.lst=opt->state.strc.lst=NULL; } \
-  if (cache.txt) { fclose(cache.txt); cache.txt=NULL; } \
-  if (opt->log != NULL) fflush(opt->log); \
-  if (makestat_fp) { fclose(makestat_fp); makestat_fp=NULL; } \
-  if (maketrack_fp){ fclose(maketrack_fp); maketrack_fp=NULL; } \
-  if (opt->accept_cookie) cookie_save(opt->cookie,fconcat(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),StringBuff(opt->path_log),"cookies.txt")); \
-  if (makeindex_fp) { fclose(makeindex_fp); makeindex_fp=NULL; } \
-  if (cache_hashtable) { coucal_delete(&cache_hashtable); } \
-  if (cache_tests)     { coucal_delete(&cache_tests); } \
-  if (template_header) { freet(template_header); template_header=NULL; } \
-  if (template_body)   { freet(template_body); template_body=NULL; } \
-  if (template_footer) { freet(template_footer); template_footer=NULL; } \
-  hash_free(&hash); \
-  clearCallbacks(&opt->state.callbacks); \
-  /*structcheck_init(-1);*/ \
-} while(0)
+#define XH_extuninit                                                           \
+  do {                                                                         \
+    HTMLCHECK_UNINIT                                                           \
+    hts_record_free(opt);                                                      \
+    if (filters && filters[0]) {                                               \
+      freet(filters[0]);                                                       \
+      filters[0] = NULL;                                                       \
+    }                                                                          \
+    if (filters) {                                                             \
+      freet(filters);                                                          \
+      filters = NULL;                                                          \
+    }                                                                          \
+    back_delete_all(opt, &cache, sback);                                       \
+    back_free(&sback);                                                         \
+    checkrobots_free(&robots);                                                 \
+    if (cache.zipOutput) {                                                     \
+      zipClose(cache.zipOutput,                                                \
+               "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \
+      cache.zipOutput = NULL;                                                  \
+    }                                                                          \
+    if (cache.zipInput) {                                                      \
+      unzClose(cache.zipInput);                                                \
+      cache.zipInput = NULL;                                                   \
+    }                                                                          \
+    if (cache.lst) {                                                           \
+      fclose(cache.lst);                                                       \
+      cache.lst = opt->state.strc.lst = NULL;                                  \
+    }                                                                          \
+    if (cache.txt) {                                                           \
+      fclose(cache.txt);                                                       \
+      cache.txt = NULL;                                                        \
+    }                                                                          \
+    if (opt->log != NULL)                                                      \
+      fflush(opt->log);                                                        \
+    if (makestat_fp) {                                                         \
+      fclose(makestat_fp);                                                     \
+      makestat_fp = NULL;                                                      \
+    }                                                                          \
+    if (maketrack_fp) {                                                        \
+      fclose(maketrack_fp);                                                    \
+      maketrack_fp = NULL;                                                     \
+    }                                                                          \
+    if (opt->accept_cookie)                                                    \
+      cookie_save(opt->cookie,                                                 \
+                  fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),           \
+                          StringBuff(opt->path_log), "cookies.txt"));          \
+    if (makeindex_fp) {                                                        \
+      fclose(makeindex_fp);                                                    \
+      makeindex_fp = NULL;                                                     \
+    }                                                                          \
+    if (cache_hashtable) {                                                     \
+      coucal_delete(&cache_hashtable);                                         \
+    }                                                                          \
+    if (cache_tests) {                                                         \
+      coucal_delete(&cache_tests);                                             \
+    }                                                                          \
+    if (template_header) {                                                     \
+      freet(template_header);                                                  \
+      template_header = NULL;                                                  \
+    }                                                                          \
+    if (template_body) {                                                       \
+      freet(template_body);                                                    \
+      template_body = NULL;                                                    \
+    }                                                                          \
+    if (template_footer) {                                                     \
+      freet(template_footer);                                                  \
+      template_footer = NULL;                                                  \
+    }                                                                          \
+    hash_free(&hash);                                                          \
+    clearCallbacks(&opt->state.callbacks);                                     \
+  } while (0)
 #define XH_uninit do { XH_extuninit; if (r.adr) { freet(r.adr); r.adr=NULL; } } while(0)
 
 struct lien_buffers {
@@ -441,32 +472,11 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
   *makeindex_done = 1;
 }
 
-/* Flush the parsed HTML output buffer to disk, skipping the rewrite when the
- * on-disk MD5 is unchanged. */
+/* Flush the parsed HTML output buffer to disk. */
 void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
                           FILE **fp, const char *ht_buff, size_t ht_len,
                           const char *adr, const char *fil, const char *save) {
-  char digest[32 + 2];
-  off_t fsize_old =
-      fsize(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), save));
-  int ok = 0;
-
-  digest[0] = '\0';
-  domd5mem(ht_buff, ht_len, digest, 1);
-  if (fsize_old == (off_t) ht_len) {
-    int mlen = 0;
-    char *mbuff;
-
-    cache_readdata(cache, "//[HTML-MD5]//", save, &mbuff, &mlen);
-    if (mlen)
-      mbuff[mlen] = '\0';
-    if ((mlen == 32) && (strcmp(((mbuff != NULL) ? mbuff : ""), digest) == 0)) {
-      ok = 1;
-      hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s", save);
-    }
-    freet(mbuff);
-  }
-  if (!ok) {
+  {
     file_notify(opt, adr, fil, save, 1, 1, r->notmodified);
     *fp = filecreate(&opt->state.strc, save);
     if (*fp) {
@@ -498,13 +508,7 @@ void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
       if (fcheck)
         hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
     }
-  } else {
-    file_notify(opt, adr, fil, save, 0, 0, r->notmodified);
-    filenote(&opt->state.strc, save, NULL);
   }
-  if (cache->ndx)
-    cache_writedata(cache->ndx, cache->dat, "//[HTML-MD5]//", save, digest,
-                    (int) strlen(digest));
 }
 
 /* does it look like XML ? (SVG et al.) */
@@ -523,7 +527,6 @@ int httpmirror(char *url1, httrackp * opt) {
   hash_struct *const hashptr = &hash;
   t_cookie BIGSTK cookie;       // gestion des cookies
 
-  //char* tab_alloc=NULL;
   int ptr;                      // pointeur actuel sur les liens
 
   //
@@ -534,15 +537,12 @@ int httpmirror(char *url1, httrackp * opt) {
   // pour les stats, nombre de fichiers & octets écrits
   LLint stat_fragment = 0;      // pour la fragmentation
 
-  //TStamp istat_timestart;   // départ pour calcul instantanné
-  //
   TStamp last_info_shell = 0;
   int info_shell = 0;
 
   // filtres
   char **filters = NULL;
 
-  //int filter_max=0;
   int filptr = 0;
 
   //
@@ -582,7 +582,6 @@ int httpmirror(char *url1, httrackp * opt) {
   // noter heure actuelle de départ en secondes
   memset(&HTS_STAT, 0, sizeof(HTS_STAT));
   HTS_STAT.stat_timestart = time_local();
-  //istat_timestart=stat_timestart;
   HTS_STAT.istat_timestart[0] = HTS_STAT.istat_timestart[1] = mtime_local();
   /* reset stats */
   HTS_STAT.HTS_TOTAL_RECV = 0;
@@ -616,9 +615,6 @@ int httpmirror(char *url1, httrackp * opt) {
   // initialiser usercommand
   usercommand(opt, opt->sys_com_exec, StringBuff(opt->sys_com), "", "", "");
 
-  // initialiser structcheck
-  // structcheck_init(1);
-
   // initialiser verif_backblue
   verif_backblue(opt, NULL);
   verif_external(opt, 0, 0);
@@ -638,9 +634,6 @@ int httpmirror(char *url1, httrackp * opt) {
                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_bin),
                  "templates/index-footer.html"), HTS_INDEX_FOOTER);
 
-  // initialiser mimedefs
-  //get_userhttptype(opt,1,StringBuff(opt->mimedefs),NULL);
-
   // Initialiser indexation
   if (opt->kindex)
     index_init(StringBuff(opt->path_html));
@@ -684,7 +677,6 @@ int httpmirror(char *url1, httrackp * opt) {
   opt->filters.filters = &filters;
   //
   opt->filters.filptr = &filptr;
-  //opt->filters.filter_max=&filter_max;
 
   // hash table
   opt->hash = &hash;
@@ -731,8 +723,7 @@ int httpmirror(char *url1, httrackp * opt) {
       else if (*a == '-')
         joker = 1;
 
-      if (joker) {              // joker ou filters
-        //char* p;
+      if (joker) { // joker ou filters
         char BIGSTK tempo[HTS_URLMAXSIZE * 2];
         int type;
         int plus = 0;
@@ -787,12 +778,11 @@ int httpmirror(char *url1, httrackp * opt) {
               XH_extuninit;
               return 0;
             }
-            //opt->filters.filters=filters;
           }
 
         }
 
-      } else {                  // adresse normale
+      } else { // adresse normale
         char BIGSTK url[HTS_URLMAXSIZE * 2];
 
         // prochaine adresse
@@ -813,6 +803,52 @@ int httpmirror(char *url1, httrackp * opt) {
       }
     }                           // while
 
+    /* --why: print which filter rule decides for this URL, then stop */
+    if (StringNotEmpty(opt->why_url)) {
+      char BIGSTK url[HTS_URLMAXSIZE * 2];
+      lien_adrfil af;
+
+      if (strstr(StringBuff(opt->why_url), ":/") == NULL)
+        snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
+      else
+        strcpybuff(url, StringBuff(opt->why_url));
+      if (ident_url_absolute(url, &af) < 0) {
+        printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
+      } else {
+        char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
+        int jok, jokDepth = 0;
+
+        /* the two forms the wizard tests */
+        strcpybuff(l, jump_identification_const(af.adr));
+        if (*af.fil != '/')
+          strcatbuff(l, "/");
+        strcatbuff(l, af.fil);
+        if (!link_has_authority(af.adr))
+          strcpybuff(lfull, "http://");
+        else
+          lfull[0] = '\0';
+        strcatbuff(lfull, af.adr);
+        if (*af.fil != '/')
+          strcatbuff(lfull, "/");
+        strcatbuff(lfull, af.fil);
+        jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
+                               NULL, &jokDepth);
+        if (jok > 0)
+          printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
+                 filters[jokDepth]);
+        else if (jok < 0)
+          printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
+                 filters[jokDepth]);
+        else
+          printf("%s: no filter rule matches; the wizard decides "
+                 "(same-site links are followed by default)" LF,
+                 url);
+      }
+      freet(primary);
+      XH_extuninit;
+      return 1;
+    }
+
     /* load URL file list */
     /* OPTIMIZED for fast load */
     if (StringNotEmpty(opt->filelist)) {
@@ -867,7 +903,6 @@ int httpmirror(char *url1, httrackp * opt) {
             htsbuff_cat(&primarybuff, "\n");
           }
         }
-        // fclose(fp);
         hts_log_print(opt, LOG_NOTICE, "%d links added from %s", n,
                       StringBuff(opt->filelist));
 
@@ -922,7 +957,6 @@ int httpmirror(char *url1, httrackp * opt) {
 #endif
 
   // backing
-  //soc_max=opt->maxsoc;
   if (opt->maxsoc > 0) {
 #if BDEBUG==2
     _CLRSCR;
@@ -1236,15 +1270,10 @@ int httpmirror(char *url1, httrackp * opt) {
       // tester r.adr
       if (!error) {
         // erreur, pas de fichier chargé:
-        if ((!r.adr) && (r.is_write == 0)
-            && (r.statuscode != 301)
-            && (r.statuscode != 302)
-            && (r.statuscode != 303)
-            && (r.statuscode != 307)
-            && (r.statuscode != 412)
-            && (r.statuscode != 416)
-          ) {
-          // error=1;
+        if ((!r.adr) && (r.is_write == 0) && (r.statuscode != 301) &&
+            (r.statuscode != 302) && (r.statuscode != 303) &&
+            (r.statuscode != 307) && (r.statuscode != 412) &&
+            (r.statuscode != 416)) {
 
           // peut être que le fichier était trop gros?
           if ((istoobig
@@ -1326,30 +1355,8 @@ int httpmirror(char *url1, httrackp * opt) {
                                          ||may_be_hypertext_mime(opt, r.contenttype, urlfil()))   /* Is real media, .. */
           ) {
 
-          /* Convert charset to UTF-8 - NOT! (what about links ? remote server side will have troubles with converted names) */
-          //if (r.adr != NULL && r.size != 0 && opt->convert_utf8) {
-          //  char *charset;
-          //  char *pos;
-          //  if (r.charset[0] != '\0') {
-          //    charset = strdup(r.charset);
-          //  } else {
-          //    charset = hts_getCharsetFromMeta(r.adr, r.size);
-          //  }
-          //  if (charset != NULL) {
-          //    char *const utf8 = hts_convertStringToUTF8(r.adr, r.size, charset);
-          //    /* Use new buffer */
-          //    if (utf8 != NULL) {
-          //      freet(r.adr);
-          //      r.size = strlen(utf8);
-          //      r.adr = utf8;
-          //      /* New UTF-8 charset */
-          //      r.charset[0] = '\0';
-          //      strcpy(r.charset, "utf-8");
-          //    }
-          //    /* Free charset */
-          //    free(charset);
-          //  }
-          //}
+          /* Convert charset to UTF-8 - NOT! (what about links ? remote server
+           * side will have troubles with converted names) */
 
           /* Check bogus chars */
           if ((r.adr) && (r.size)) {
@@ -1490,50 +1497,6 @@ int httpmirror(char *url1, httrackp * opt) {
 
         }
       }
-      // MOVED IN back_finalize()
-      //
-      // -------------------- 
-      // REAL MEDIA HACK
-      // Check if we have to load locally the file
-      // --------------------
-      //if (!error) {
-      //  if (r.statuscode == HTTP_OK) {    // OK (ou 304 en backing)
-      //    if (r.adr==NULL) {    // Written file
-      //      if (may_be_hypertext_mime(r.contenttype, urlfil())) {   // to parse!
-      //        LLint sz;
-      //        sz=fsize_utf8(savename());
-      //        if (sz>0) {   // ok, exists!
-      //          if (sz < 8192) {   // ok, small file --> to parse!
-      //            FILE* fp=FOPEN(savename(),"rb");
-      //            if (fp) {
-      //              r.adr=malloct(sz + 1);
-      //              if (r.adr) {
-      //                if (fread(r.adr,1,sz,fp) == sz) {
-      //                  r.size=sz;
-      //                                                r.adr[sz] = '\0';
-      //                                                r.is_write = 0;
-      //                } else {
-      //                  freet(r.adr);
-      //                  r.size=0;
-      //                  r.adr = NULL;
-      //                  r.statuscode=STATUSCODE_INVALID;
-      //                  strcpybuff(r.msg, ".RAM read error");
-      //                }
-      //                fclose(fp);
-      //                fp=NULL;
-      //                // remove (temporary) file!
-      //                remove(savename());
-      //              }
-      //              if (fp)
-      //                fclose(fp);
-      //            }
-      //          }
-      //        }
-      //      }
-      //    }
-      //  }
-      //}
-      // EN OF REAL MEDIA HACK
 
       // ---stockage en cache---
       // stocker dans le cache?
@@ -1654,24 +1617,6 @@ int httpmirror(char *url1, httrackp * opt) {
       }
 #endif
 
-      /* info: updated */
-      /*
-         if (ptr>0) {
-         // "mis à jour"
-         if ((!r.notmodified) && (opt->is_update) && (!store_errpage)) {    // page modifiée
-         if (strnotempty(savename())) {
-         HTS_STAT.stat_updated_files++;
-         //if ((opt->debug>0) && (opt->log!=NULL)) {
-         hts_log_print(opt, LOG_INFO, "File updated: %s%s",urladr(),urlfil());
-         }
-         } else {
-         if (!store_errpage) {
-         hts_log_print(opt, LOG_INFO, "File recorded: %s%s",urladr(),urlfil());
-         }
-         }
-         }
-       */
-
       // ------------------------------------------------------
       // traitement (parsing)
       // ------------------------------------------------------
@@ -1717,13 +1662,8 @@ int httpmirror(char *url1, httrackp * opt) {
                 free(charset);
             }
             /* Could not detect charset: could it be UTF-8 ? */
-            /* No, we can not do that: browsers do not do it 
+            /* No, we can not do that: browsers do not do it
                (and it would break links). */
-            //if (page_charset[0] == '\0') {
-            //  if (is_unicode_utf8(r.adr, r.size)) {
-            //    strcpy(page_charset, "utf-8");
-            //  }
-            //}
             /* Could not detect charset */
             if (page_charset[0] == '\0') {
               hts_log_print(opt, LOG_INFO,
@@ -1801,10 +1741,6 @@ int httpmirror(char *url1, httrackp * opt) {
               XH_uninit;
               return -1;
             }
-
-            // I'll have to segment this part
-// #include "htsparse.c"
-
           }
         }
         // Fin parsing HTML
@@ -1900,7 +1836,6 @@ int httpmirror(char *url1, httrackp * opt) {
              HTS_STAT.stat_files++;
              HTS_STAT.stat_bytes+=r.size;
            */
-          //printf("ok......\n");
         } else {
           // Si on doit sauver une page HTML sans la scanner, cela signifie que le niveau de
           // récursion nous en empêche
@@ -1969,10 +1904,11 @@ int httpmirror(char *url1, httrackp * opt) {
             }
           }
 
-          //printf("extern=%s\n",r.contenttype);
-
-          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!          
-          if (r.adr != NULL || r.size == 0) {
+          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
+          // An empty body must not overwrite the file when the transfer failed
+          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
+          // copy to 0 (#77 follow-up).
+          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {
@@ -1993,7 +1929,6 @@ int httpmirror(char *url1, httrackp * opt) {
                */
             }
           }
-
         }
 
         /* Parsing of other media types (java, ram..) */
@@ -2092,16 +2027,9 @@ int httpmirror(char *url1, httrackp * opt) {
           // prochain pass2
           while((ptr < opt->lien_tot) ? (!heap(ptr)->pass2) : 0)
             ptr++;
-
-          //printf("first link==%d\n");
-
         }
       }
     }
-    // copy abort state if necessary from outside
-    //if (!exit_xh && opt->state.exit_xh) {
-    //  exit_xh=opt->state.exit_xh;
-    //}
     // a-t-on dépassé le quota?
     if (!back_checkmirror(opt)) {
       ptr = opt->lien_tot;
@@ -2137,47 +2065,7 @@ int httpmirror(char *url1, httrackp * opt) {
     hts_log_print(opt, LOG_NOTICE,
                   "No data seems to have been transferred during this session! : restoring previous one!");
     XH_uninit;
-    if ((fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/old.dat")))
-        &&
-        (fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-           "hts-cache/old.ndx")))) {
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.dat"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.ndx"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.lst"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.txt"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.dat"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.ndx"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.lst"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.lst"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.txt"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.txt"));
-    }
+    hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
     opt->state.exit_xh = 2;     /* interrupted (no connection detected) */
     return 1;
   }
@@ -2298,13 +2186,11 @@ int httpmirror(char *url1, httrackp * opt) {
     char BIGSTK htstime[256];
     char BIGSTK infoupdated[256];
 
-    // int n=(int) (stat_loaded/(time_local()-HTS_STAT.stat_timestart));
     LLint n =
       (LLint) (HTS_STAT.HTS_TOTAL_RECV /
                (max(1, time_local() - HTS_STAT.stat_timestart)));
 
     sec2str(htstime, time_local() - HTS_STAT.stat_timestart);
-    //sprintf(finalInfo + strlen(finalInfo),LF"HTS-mirror complete in %s : %d links scanned, %d files written (%d bytes overall) [%d bytes received at %d bytes/sec]"LF,htstime,lien_tot-1,HTS_STAT.stat_files,stat_bytes,stat_loaded,n);
     infoupdated[0] = '\0';
     if (opt->is_update) {
       if (HTS_STAT.stat_updated_files > 0) {
@@ -2387,12 +2273,6 @@ int httpmirror(char *url1, httrackp * opt) {
 
 */
 int engine_stats(void) {
-#if 0
-  static FILE *debug_fp = NULL; /* ok */
-
-  if (!debug_fp)
-    debug_fp = fopen("esstat.txt", "wb");
-#endif
   HTS_STAT.stat_nsocket = HTS_STAT.stat_errors = HTS_STAT.nbk = 0;
   HTS_STAT.nb = 0;
   if (HTS_STAT.HTS_TOTAL_RECV > 2048) {
@@ -2403,10 +2283,6 @@ int engine_stats(void) {
       if ((cdif - HTS_STAT.istat_timestart[i]) >= 2000) {
         TStamp dif;
 
-#if 0
-        fprintf(debug_fp, "set timer %d\n", i);
-        fflush(debug_fp);
-#endif
         dif = cdif - HTS_STAT.istat_timestart[i];
         if ((TStamp) (dif / 1000) > 0) {
           LLint byt = (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[i]);
@@ -2425,10 +2301,6 @@ int engine_stats(void) {
     // timer #0 resync timer #1 when reaching 1 second limit
     if (HTS_STAT.istat_reference01 != HTS_STAT.istat_timestart[0]) {
       if ((cdif - HTS_STAT.istat_timestart[0]) >= 1000) {
-#if 0
-        fprintf(debug_fp, "resync timer 1\n");
-        fflush(debug_fp);
-#endif
         HTS_STAT.istat_bytes[1] = HTS_STAT.HTS_TOTAL_RECV;
         HTS_STAT.istat_timestart[1] = cdif;
         HTS_STAT.istat_reference01 = HTS_STAT.istat_timestart[0];
@@ -2449,7 +2321,6 @@ void host_ban(httrackp * opt, int ptr,
   lien_back *const back = sback->lnk;
   const int back_max = sback->count;
 
-  //int l;
   int i;
 
   if (host[0] == '!')
@@ -2506,9 +2377,7 @@ void host_ban(httrackp * opt, int ptr,
   }
 
   // effacer liens
-  //l=strlen(host);
-  for(i = 0; i < opt->lien_tot; i++) {
-    //if (heap(i)->adr_len==l) {    // même taille de chaîne
+  for (i = 0; i < opt->lien_tot; i++) {
     // Calcul de taille sécurisée
     if (heap(i)) {
       if (heap(i)->adr) {
@@ -2547,7 +2416,6 @@ void host_ban(httrackp * opt, int ptr,
                     "WARNING! HostCancel detected memory leaks [null at %d]",
                     i);
     }
-    //}
   }
 }
 
@@ -2892,6 +2760,9 @@ int check_fatal_io_errno(void) {
 #endif
 #ifdef EROFS
   case EROFS:                  /* Read-only file system */
+#endif
+#ifdef EDQUOT
+  case EDQUOT: /* Disk quota exceeded */
 #endif
     return 1;
     break;
@@ -2961,7 +2832,6 @@ FILE *fileappend(filenote_strc * strc, const char *s) {
   // noter lst
   filenote(strc, s, NULL);
 
-  // if (*s=='/') strcpybuff(fname,s+1); else strcpybuff(fname,s);    // pas de / (root!!) // ** SIIIIIII!!! à cause de -O <path>
   strcpybuff(fname, s);
 
 #if HTS_DOSNAME
@@ -3005,7 +2875,6 @@ int filecreateempty(filenote_strc * strc, const char *filename) {
 int filenote(filenote_strc * strc, const char *s, filecreate_params * params) {
   // gestion du fichier liste liste
   if (params) {
-    //filecreate_params* p = (filecreate_params*) params;
     strcpybuff(strc->path, params->path);
     strc->lst = params->lst;
     return 0;
@@ -3085,9 +2954,8 @@ void usercommand_exe(const char *cmd, const char *file) {
   }
 }
 
-static void postprocess_file(httrackp * opt, const char *save, const char *adr,
+static void postprocess_file(httrackp *opt, const char *save, const char *adr,
                              const char *fil) {
-  //int first = 0;
   /* MIME-html archive to build */
   if (opt != NULL && opt->mimehtml) {
     if (adr != NULL && strcmp(adr, "primary") == 0) {
@@ -3115,7 +2983,6 @@ static void postprocess_file(httrackp * opt, const char *save, const char *adr,
       }
 
       if (!opt->state.mimehtml_created) {
-        //first = 1;
         opt->state.mimefp =
           fopen(fconcat
                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
@@ -3259,28 +3126,6 @@ int fspc(httrackp * opt, FILE * fp, const char *type) {
   return 0;
 }
 
-// vérifier taux de transfert
-#if 0
-void check_rate(TStamp stat_timestart, int maxrate) {
-  // vérifier taux de transfert (pas trop grand?)
-  /*
-     if (maxrate>0) {
-     int r = (int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-stat_timestart));    // taux actuel de transfert
-     HTS_STAT.HTS_TOTAL_RECV_STATE=0;
-     if (r>maxrate) {    // taux>taux autorisé
-     int taux = (int) (((TStamp) (r - maxrate) * 100) / (TStamp) maxrate);
-     if (taux<15)
-     HTS_STAT.HTS_TOTAL_RECV_STATE=1;   // ralentir un peu (<15% dépassement)
-     else if (taux<50)
-     HTS_STAT.HTS_TOTAL_RECV_STATE=2;   // beaucoup (<50% dépassement)
-     else
-     HTS_STAT.HTS_TOTAL_RECV_STATE=3;   // énormément (>50% dépassement)
-     }
-     }
-   */
-}
-#endif
-
 // ---
 // sous routines liées au moteur et au backing
 
@@ -3290,21 +3135,8 @@ int backlinks_done(const struct_back * sback,
                    int ptr) {
   int n = 0;
 
-#if 0
-  int i;
-
-  //Links done and stored in cache
-  for(i = ptr + 1; i < lien_tot; i++) {
-    if (heap(i)) {
-      if (heap(i)->pass2 == -1) {
-        n++;
-      }
-    }
-  }
-#else
   // finalized in background
   n += HTS_STAT.stat_background;
-#endif
   n += back_done_incache(sback);
   return n;
 }
@@ -3371,6 +3203,41 @@ int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt) {
   return n;
 }
 
+/* One engine-loop tick: refresh the transfer stats and run the loop callback
+   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
+hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr) {
+  engine_stats();
+  HTS_STAT.stat_nsocket = back_nsoc(sback);
+  HTS_STAT.stat_errors = fspc(opt, NULL, "error");
+  HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
+  HTS_STAT.stat_infos = fspc(opt, NULL, "info");
+  HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
+  HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
+  return RUN_CALLBACK7(
+             opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
+             (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)
+             ? HTS_TRUE
+             : HTS_FALSE;
+}
+
+/* Single implementation of the historical WAIT_FOR_AVAILABLE_SOCKET macros. */
+hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
+                                      cache_back *cache, int ptr) {
+  const int prev = opt->state._hts_in_html_parsing;
+
+  while (back_pluggable_sockets_strict(sback, opt) <= 0) {
+    opt->state._hts_in_html_parsing = 6;
+    back_wait(sback, opt, cache, 0);
+    /* time limit (-E) exceeded: stop waiting for a socket (#481) */
+    if (!back_checkmirror(opt))
+      break;
+    if (!hts_loop_tick(sback, opt, -1, ptr))
+      return HTS_FALSE;
+  }
+  opt->state._hts_in_html_parsing = prev;
+  return HTS_TRUE;
+}
+
 int back_pluggable_sockets(struct_back * sback, httrackp * opt) {
   int n;
 
@@ -3406,8 +3273,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
     /* on a déja parcouru */
     if (p < cache->ptr_ant)
       p = cache->ptr_ant;
-    while(p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
-      //while((p<lien_tot) && (n>0) && (p < ptr+opt->maxcache_anticipate)) {
+    while (p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
       int ok = 1;
 
       // on ne met pas le fichier en backing si il doit être traité après ou s'il a déja été traité
@@ -3419,12 +3285,6 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
           ok = 0;
       }
 
-      // Why in hell did I do that ?
-      //if (ok && heap(p)->sav != NULL && heap(p)->sav[0] != '\0' 
-      //  && hash_read(opt->hash,heap(p)->sav,NULL,HASH_STRUCT_FILENAME ) >= 0)     // lookup in liens_record
-      //{
-      //  ok = 0;
-      //}
       if (heap(p)->sav == NULL || heap(p)->sav[0] == '\0'
           || hash_read(opt->hash, heap(p)->sav, NULL, HASH_STRUCT_FILENAME ) < 0) {
         ok = 0;
@@ -3453,7 +3313,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
         }
       }
       p++;
-    }                           // while
+    } // while
     /* sauver position dernière anticipation */
     cache->ptr_ant = p;
     cache->ptr_last = ptr;
@@ -3715,20 +3575,7 @@ HTSEXT_API hts_boolean hts_has_stopped(httrackp *opt) {
   return ended;
 }
 
-// régler en cours de route les paramètres réglables..
-// -1 : erreur
-//HTSEXT_API int hts_setopt(httrackp* set_opt) {
-//  if (set_opt) {
-//    httrackp* engine_opt=hts_declareoptbuffer(NULL);
-//    if (engine_opt) {
-//      //_hts_setopt=opt;
-//      copy_htsopt(set_opt,engine_opt);
-//    }
-//  }
-//  return 0;
-//}
 // ajout d'URL
-// -1 : erreur
 HTSEXT_API hts_boolean hts_addurl(httrackp *opt, char **url) {
   if (url)
     opt->state._hts_addurl = url;
@@ -3989,10 +3836,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
               heap_top()->link_import = 0; // pas mode import
 
               // écrire autres paramètres de la structure-lien
-              //if (meme_adresse)                                 
               heap_top()->premier = heap(ptr)->premier;
-              //else    // sinon l'objet père est le précédent lui même
-              //  heap_top()->premier=ptr;
 
               heap_top()->precedent = ptr;
               // noter la priorité
@@ -4003,9 +3847,6 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
               heap_top()->pass2 = max(pass_fix, numero_passe);
               heap_top()->retry = opt->retry;
 
-              //strcpybuff(heap_top()->adr,adr);
-              //strcpybuff(heap_top()->fil,fil);
-              //strcpybuff(heap_top()->sav,save); 
               hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s",
                             heap_top()->adr, heap_top()->fil,
                             heap_top()->sav);
@@ -4034,7 +3875,6 @@ void voidf(void) {
   a = "|                      HTTrack Website Copier   |" CRLF;
   a = "|Code:         Windows Interface Xavier Roche   |" CRLF;
   a = "|                    HTS/HTTrack Xavier Roche   |" CRLF;
-  a = "|                .class Parser Yann Philippot   |" CRLF;
   a = "|                                               |" CRLF;
   a = "|Tested on:                 Windows95,98,NT,2K  |" CRLF;
   a = "|                           Linux PC            |" CRLF;

src/htscore.h

@@ -193,8 +193,7 @@ struct cache_back {
   /* */
   int type;
   int ro;                   /**< read-only: no new cache is written */
-  FILE *dat, *ndx, *olddat; /**< new data, new index, old data files */
-  char *use;                /**< in-memory list of cached adr+fil keys */
+
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
   char lastmodified[256];
@@ -216,6 +215,7 @@ struct cache_back {
   int zipEntriesCapa;
   hts_boolean
       zipWriteFailed; /**< a cache write failed; stop touching the stream */
+  int zipWriteFailures; /**< consecutive entry write failures; reset on store */
 };
 
 #ifndef HTS_DEF_FWSTRUCT_hash_struct
@@ -287,12 +287,12 @@ struct filecreate_params {
 
 /* True if a new cache is being written (plain or zip backend). */
 HTS_STATIC int cache_writable(cache_back * cache) {
-  return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL));
+  return (cache != NULL && cache->zipOutput != NULL);
 }
 
 /* True if an old cache is available to read (plain or zip backend). */
 HTS_STATIC int cache_readable(cache_back * cache) {
-  return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL));
+  return (cache != NULL && cache->zipInput != NULL);
 }
 
 #endif
@@ -411,10 +411,6 @@ char *readfile_utf8(const char *fil);
 
 char *readfile_or(const char *fil, const char *defaultdata);
 
-#if 0
-void check_rate(TStamp stat_timestart, int maxrate);
-#endif
-
 /* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
    Not thread-safe; call from the single crawl loop. */
 
@@ -432,6 +428,15 @@ int back_pluggable_sockets(struct_back * sback, httrackp * opt);
 
 int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);
 
+/* One engine-loop tick: refresh the transfer stats and run the loop callback
+   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
+hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr);
+
+/* Wait until a test socket can be plugged, pumping transfers, stats and the
+   loop callback; gives up past the -E deadline. HTS_FALSE = callback abort. */
+hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
+                                      cache_back *cache, int ptr);
+
 /* Randomized inter-file pause target in [min_ms,max_ms] (#185), derived from a
    timestamp seed so it is stable within one gap and rerolls per launch. */
 int hts_pause_target_ms(TStamp seed, int min_ms, int max_ms);
@@ -457,11 +462,6 @@ int ask_continue(httrackp * opt);
 /* Number of decimal digits in n. */
 int nombre_digit(int n);
 
-// Java
-#if 0
-int hts_add_file(char *file, int file_position);
-#endif
-
 // Polling
 #if HTS_POLL
 int check_flot(T_SOC s);

src/htscoremain.c

@@ -145,7 +145,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   char *url = NULL;             // URLS séparées par un espace
   int url_sz = 65535;
 
-  //char url[65536];         // URLS séparées par un espace
   // the parametres
   int httrack_logmode = 3;      // ONE log file
 
@@ -290,12 +289,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         if (tmp_argc == 1) {    /* pas -P & co */
           if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */
             if (argv_url < 0)
-              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
-            //if (argv_url>=0) {
+              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
+                            // previous options
             argv_url++;
             if (!argv_firsturl)
               argv_firsturl = x_argv[x_argc - 1];
-            //}
           } else {
             if (strcmp(tmp_argv[0], "-h") == 0) {
               help(argv[0], !opt->quiet);
@@ -324,8 +322,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         } else if (tmp_argc == 2) {
           if ((strcmp(tmp_argv[0], "-%L") == 0)) {      // liste d'URLs
             if (argv_url < 0)
-              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
-            //if (argv_url>=0)
+              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
+                            // previous options
             argv_url++;         /* forcer */
           }
         }
@@ -403,15 +401,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     else
                       inQuote = 1;
                   } else if (!inQuote && !noDbl && argv[na][i] == ',') {
-                    //StringAddchar(path, '\0');
-                    //j = 0;
                     path = &opt->path_log;
                   } else {
                     StringAddchar(*path, argv[na][i]);
-                    //path[j++] = argv[na][i];
                   }
                 }
-                //path[j++] = '\0';
                 if (StringLength(opt->path_log) == 0) {
                   StringCopyS(opt->path_log, opt->path_html);
                 }
@@ -420,7 +414,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if (check_path(&opt->path_html, argv_firsturl)) {
                   opt->dir_topindex = 1;        // rebuilt top index
                 }
-                //printf("-->%s\n%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log));                
               }
               break;
             }                   // switch
@@ -539,74 +532,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             insert_after++;
           }
         }
-      } while(lastp != NULL);
-      //fclose(fp);
+      } while (lastp != NULL);
     }
   }
 
-  // Existence d'un cache - pas de new mais un old.. renommer
+  // No new cache but an old one? promote it
 #if DEBUG_STEPS
   printf("Checking cache\n");
 #endif
-  if (!fexist
-      (fconcat
-       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-       StringBuff(opt->path_log), "hts-cache/new.zip"))) {
-    if (fexist
-        (fconcat
-         (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-         StringBuff(opt->path_log), "hts-cache/old.zip"))) {
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.zip"));
-    }
-  } else
-    if ((!fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log), "hts-cache/new.dat")))
-        ||
-        (!fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log),
-           "hts-cache/new.ndx")))) {
-    if ((fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log), "hts-cache/old.dat")))
-        &&
-        (fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log),
-           "hts-cache/old.ndx")))) {
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/new.dat"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/new.ndx"));
-      //remove(fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.dat"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.ndx"));
-      //rename(fconcat(StringBuff(opt->path_log),"hts-cache/old.lst"),fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
-    }
-  }
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
 
   /* Interrupted mirror detected */
   if (!opt->quiet) {
@@ -774,24 +708,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             HTS_PANIC_PRINTF(s);
             htsmain_free();
             return -1;
-#else
-#if 0
-            char _args[8][256];
-            char *args[8];
-
-            printf("Cheking for updates...\n");
-            strcpybuff(_args[0], argv[0]);
-            strcpybuff(_args[1], "--get");
-            sprintf(_args[2], HTS_UPDATE_WEBSITE, 0, "");
-            strcpybuff(_args[3], "--quickinfo");
-            args[0] = _args[0];
-            args[1] = _args[1];
-            args[2] = _args[2];
-            args[3] = _args[3];
-            args[4] = NULL;
-            if (execvp(args[0], args) == -1) {
-            }
-#endif
 #endif
           }
           //
@@ -809,103 +725,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     }
   }
 
-  // Compter urls/jokers
-  /*
-     if (argv_url<=0) { 
-     int na;
-     argv_url=0;
-     for(na=1;na<argc;na++) {
-     if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) {
-     na++;    // sauter nom de proxy
-     } else if (!cmdl_opt(argv[na])) { 
-     argv_url++;   // un de plus       
-     } else if (strcmp(argv[na],"-h")==0) {
-     help(argv[0],!opt->quiet);
-     htsmain_free();
-     return 0;
-     } else {
-     if ((strchr(argv[na],'q')!=NULL))
-     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
-     if ((strchr(argv[na],'i')!=NULL)) {  // doit.log!
-     argv_url=0;
-     na=argc;
-     }
-     }
-     }
-     }  
-   */
-
-  // Ici on ajoute les arguments qui ont été appelés avant au cas où on récupère une session
-  // Exemple: httrack www.truc.fr -L0 puis ^C puis httrack sans URL : ajouter URL précédente
-  /*
-     if (argv_url==0) {
-     //if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) {  // il existe déja un cache précédent.. renommer
-     if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) {    // un cache est présent
-
-     x_argvblk=(char*) calloct(32768,1);
-
-     if (x_argvblk!=NULL) {
-     FILE* fp;
-     int x_argc;
-
-     //strcpybuff(x_argvblk,"httrack ");
-     fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb");
-     if (fp) {
-     linput(fp,x_argvblk+strlen(x_argvblk),8192);
-     fclose(fp); fp=NULL;
-     }
-
-     // calculer arguments selon derniers arguments
-     x_argv[0]=argv[0];
-     x_argc=1;
-     {
-     char* p=x_argvblk;
-     do {
-     x_argv[x_argc++]=p;
-     //p=strstr(p," ");
-     // exemple de chaine: "echo \"test\"" c:\a "\$0"
-     p=next_token(p,1);    // prochain token
-     if (p) {
-     *p=0;    // octet nul (tableau)
-     p++;
-     }            
-     } while(p!=NULL);
-     }
-     // recopier arguments actuels (pointeurs uniquement)
-     {
-     int na;
-     for(na=1;na<argc;na++) {
-     if (strcmp(argv[na],"-O") != 0)    // SAUF le path!
-     x_argv[x_argc++]=argv[na];
-     else
-     na++;
-     }
-     }
-     argc=x_argc;      // nouvel argc
-     argv=x_argv;      // nouvel argv
-     }
-
-     }
-     //}
-     }
-   */
-
-  // Vérifier quiet
-  /*
-     { 
-     int na;    
-     for(na=1;na<argc;na++) {
-     if (!cmdl_opt(argv[na])) { 
-     if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0))
-     na++;    // sauter nom de proxy
-     } else {
-     if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL))
-     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
-     }
-     }
-     }
-   */
-
   /* Engine self-tests: -#test lists them, -#test=NAME [args] runs one. Handled
      here, ahead of the no-URL usage gate below, so they need no dummy URL. */
   {
@@ -1012,8 +831,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   } else {                      // plus de 2 paramètres
     // un fichier log existe?
     if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-      StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // fichier lock?
-      //char s[32];
+                       StringBuff(opt->path_log),
+                       "hts-in_progress.lock"))) { // fichier lock?
 
       opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
       if (opt->quiet == 0) {
@@ -1044,11 +863,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           }
         }
       }
-    } else
-      if (fexist
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "index.html"))) {
-      //char s[32];
+    } else if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                              StringBuff(opt->path_html), "index.html"))) {
       opt->cache = HTS_CACHE_TEST_UPDATE;
       if (opt->quiet == 0) {
         if ((fexist
@@ -1143,7 +959,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             //
           case 'g':            // récupérer un (ou plusieurs) fichiers isolés
             opt->wizard = HTS_WIZARD_AUTO;
-            //opt->wizard=0;             // pas de wizard
             opt->cache = HTS_CACHE_NONE; // ni de cache
             opt->makeindex = 0; // ni d'index
             httrack_logmode = 1;        // erreurs à l'écran
@@ -1243,9 +1058,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             while(isdigit((unsigned char) *(com + 1)))
               com++;
             break;
-            //
-            //case 'A': opt->urlmode=1; break;
-            //case 'R': opt->urlmode=2; break;
           case 'K':
             opt->urlmode = HTS_URLMODE_ABSOLUTE;
             if (isdigit((unsigned char) *(com + 1))) {
@@ -1257,8 +1069,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               while(isdigit((unsigned char) *(com + 1)))
                 com++;
             }
-            //if (*(com+1)=='0') { opt->urlmode=2; com++; } break;
-            //
           case 'c':
             if (isdigit((unsigned char) *(com + 1))) {
               sscanf(com + 1, "%d", &opt->maxsoc);
@@ -1642,7 +1452,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF
                     ("Option %w needs to be followed by a blank space, and a module name");
-                  printf("Example: -%%w htsswf\n");
+                  printf("Example: -%%w httrack-plugin\n");
                   htsmain_free();
                   return -1;
                 } else {
@@ -1746,7 +1556,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   } else {
                     char *a;
 
-                    //char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs);
                     for(a = argv[na]; *a != '\0'; a++) {
                       if (*a == ';') {  /* next one */
                         StringAddchar(opt->mimedefs, '\n');
@@ -1994,6 +1803,23 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   StringCopy(opt->cookies_file, argv[na]);
                 }
                 break;
+              case 'Y': // why: explain the filter verdict for a URL, no crawl
+                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
+                  HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
+                  printf(
+                      "Example: --why \"http://www.example.com/file.zip\"\n");
+                  htsmain_free();
+                  return -1;
+                } else {
+                  na++;
+                  if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
+                    HTS_PANIC_PRINTF("why URL too long");
+                    htsmain_free();
+                    return -1;
+                  }
+                  StringCopy(opt->why_url, argv[na]);
+                }
+                break;
               case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF("Option pause needs a blank space and a "
@@ -2085,10 +1911,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   HTS_PANIC_PRINTF(s);
                   htsmain_free();
                   return -1;
-                }
-                break;
-
-                //case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break;
+              } break;
               }
             }
             break;
@@ -2130,18 +1953,20 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     char BIGSTK url[HTS_URLMAXSIZE * 2];
                     char linepos[256];
                     int pos;
-                    char *cacheNdx =
-                      readfile(fconcat
-                               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                                "hts-cache/new.ndx"));
+                    LLint cacheNdxLen = 0;
+                    char *cacheNdx = readfile2(
+                        fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                StringBuff(opt->path_log), "hts-cache/new.ndx"),
+                        &cacheNdxLen);
                     cache_init(&cache, opt);    /* load cache */
                     if (cacheNdx != NULL) {
                       char firstline[256];
                       char *a = cacheNdx;
+                      const char *const end = cacheNdx + cacheNdxLen;
 
                       a += cache_brstr(a, firstline, sizeof(firstline));
                       a += cache_brstr(a, firstline, sizeof(firstline));
-                      while(a != NULL) {
+                      while (a != NULL && a < end) {
                         a = strchr(a + 1, '\n');        /* start of line */
                         if (a) {
                           htsblk r;
@@ -2149,15 +1974,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                           /* */
                           a++;
                           /* read "host/file" */
-                          a += binput(a, afs.af.adr, HTS_URLMAXSIZE);
-                          a += binput(a, afs.af.fil, HTS_URLMAXSIZE);
+                          a += cache_binput(a, end, afs.af.adr, HTS_URLMAXSIZE);
+                          a += cache_binput(a, end, afs.af.fil, HTS_URLMAXSIZE);
                           url[0] = '\0';
                           if (!link_has_authority(afs.af.adr))
                             strcatbuff(url, "http://");
                           strcatbuff(url, afs.af.adr);
                           strcatbuff(url, afs.af.fil);
                           /* read position */
-                          a += binput(a, linepos, 200);
+                          a += cache_binput(a, end, linepos, 200);
                           sscanf(linepos, "%d", &pos);
                           if (!hasFilter
                               || (strjoker(url, filter, NULL, NULL) != NULL)
@@ -2332,15 +2157,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 return 0;
                 break;
               case '~':        /* internal lib test */
-                HTS_PANIC_PRINTF
-                  ("Option #~ is disabled for security reasons");
-                //Disabled because choke on GCC 4.3 (toni from links2linux.de)
-                //{
-                //  char thisIsATestYouShouldSeeAnError[12];
-                //  const char *const bufferOverflowTest = "0123456789012345678901234567890123456789";
-                //  strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest);
-                //  return 0;
-                //}
+                HTS_PANIC_PRINTF("Option #~ is disabled for security reasons");
                 break;
               case 'f':
                 opt->flush = 1;
@@ -2442,7 +2259,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               opt->proxy.active = 1;
               // Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
               a = argv[na] + strlen(argv[na]) - 1;
-              // a=strstr(argv[na],":");  // port
               while((a > argv[na]) && (*a != ':') && (*a != '@'))
                 a--;
               if (*a == ':') {  // un port est présent, <proxy>:port
@@ -2547,116 +2363,14 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
 #endif
 #endif
 
-  //printf("WARNING! This is *only* a beta-release of HTTrack\n");
   io_flush;
 
 #if DEBUG_STEPS
   printf("Cache & log settings\n");
 #endif
 
-  // on utilise le cache..
-  // en cas de présence des deux versions, garder la version la plus avancée,
-  // cad la version contenant le plus de fichiers  
-  if (opt->cache) {
-    if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // problemes..
-      if (fexist
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-            "hts-cache/new.dat"))) {
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.zip"))) {
-          if (fsize
-              (fconcat
-               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                "hts-cache/new.zip")) < 32768) {
-            if (fsize
-                (fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.zip")) > 65536) {
-              if (fsize
-                  (fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/old.zip")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                          StringBuff(opt->
-                                                                     path_log),
-                                                          "hts-cache/new.zip")))
-              {
-                remove(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/new.zip"));
-                rename(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                      StringBuff(opt->path_log),
-                                                      "hts-cache/new.zip"));
-              }
-            }
-          }
-        }
-      } else
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.dat"))
-            &&
-            fexist(fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/new.ndx"))) {
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.dat"))
-            &&
-            fexist(fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/old.ndx"))) {
-          // switcher si new<32Ko et old>65Ko (tailles arbitraires) ?
-          // ce cas est peut être une erreur ou un crash d'un miroir ancien, prendre
-          // alors l'ancien cache
-          if (fsize
-              (fconcat
-               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                "hts-cache/new.dat")) < 32768) {
-            if (fsize
-                (fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.dat")) > 65536) {
-              if (fsize
-                  (fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/old.dat")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                          StringBuff(opt->
-                                                                     path_log),
-                                                          "hts-cache/new.dat")))
-              {
-                remove(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/new.dat"));
-                remove(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/new.ndx"));
-                rename(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                      StringBuff(opt->path_log),
-                                                      "hts-cache/new.dat"));
-                rename(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                      StringBuff(opt->path_log),
-                                                      "hts-cache/new.ndx"));
-                //} else {  // ne rien faire
-                //  remove("hts-cache/old.dat");
-                //  remove("hts-cache/old.ndx");
-              }
-            }
-          }
-        }
-      }
-    }
-  }
+  // If both cache generations exist, keep the most complete one
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
   // Débuggage des en têtes
   if (_DEBUG_HEAD) {
     ioinfo =
@@ -2732,7 +2446,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     {
       FILE *fp = NULL;
 
-      //int n=0;
       char t[256];
 
       time_local_rfc822(t);     // faut bien que ca serve quelque part l'heure RFC1945 arf'
@@ -2766,9 +2479,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       }
 
       strcpybuff(n_lock,
-             fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                     "hts-in_progress.lock"));
-      //sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
+                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                         StringBuff(opt->path_log), "hts-in_progress.lock"));
       /*do {
          if (!n)
          sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
@@ -2844,8 +2556,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           fprintf(fp, LF);
           fclose(fp);
           fp = NULL;
-          //} else if (opt->debug>1) {
-          //  printf("! FileOpen error, \"%s\"\n",strerror(errno));
         }
       }
       // petit message dans le lock

src/htsdefines.h

@@ -69,11 +69,15 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
 typedef struct t_hts_callbackarg t_hts_callbackarg;
 #endif
 
-/* Marks a symbol an external wrapper module exports back to the engine
-   (dllexport on Windows, nothing elsewhere). */
+/* Marks a symbol an external wrapper module exports back to the engine.
+   Must override -fvisibility=hidden on ELF, or dlopen()ed plugins
+   hide their own hts_plug()/hts_unplug() entry points. */
 #ifndef EXTERNAL_FUNCTION
 #ifdef _WIN32
 #define EXTERNAL_FUNCTION __declspec(dllexport)
+#elif ((defined(__GNUC__) && (__GNUC__ >= 4)) ||                               \
+       (defined(HAVE_VISIBILITY) && HAVE_VISIBILITY))
+#define EXTERNAL_FUNCTION __attribute__((visibility("default")))
 #else
 #define EXTERNAL_FUNCTION
 #endif

src/htsdns_selftest.c

@@ -297,6 +297,26 @@ int dns_selftests(httrackp *opt) {
     IPV6_resolver = 0;
   }
 
+  /* "host:port" resolves as the bare host: the cache/connect path strips
+     ":port" before both the backend and the cache key (#181). */
+  mock_reset_calls();
+  {
+    SOCaddr a;
+    char ip[64];
+    const char *err = NULL;
+
+    /* cache miss: the backend is hit with the stripped host, not "host:port" */
+    CHECK(hts_dns_resolve2(opt, "v6only.test:8080", &a, &err) != NULL);
+    CHECK(mock_find("v6only.test")->calls == 1);
+    /* the bare host shares that one cache entry (stripped key) */
+    CHECK(hts_dns_resolve2(opt, "v6only.test", &a, &err) != NULL);
+    CHECK(mock_find("v6only.test")->calls == 1);
+    /* a port variant of an already-cached host also hits, right address */
+    CHECK(hts_dns_resolve2(opt, "v4only.test:1234", &a, &err) != NULL);
+    SOCaddr_inetntoa(ip, sizeof(ip), a);
+    CHECK(strcmp(ip, "1.2.3.4") == 0);
+  }
+
   /* newhttp_addr() must connect to the addr_index-th address, not always the
      first: this is what back_connect_next relies on to reach the fallback. */
   {

src/htsencoding.c

@@ -190,9 +190,9 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
         }
       }
     }
-    
-    /* copy */
-    if (j + 1 > max) {
+
+    /* reserve one byte for the trailing NUL written after the loop */
+    if (j + 1 >= max) {
       /* overflow */
       return -1;
     }
@@ -300,6 +300,11 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
 
           /* Was the character read successfully ? */
           if (nRead == utfBufferSize) {
+            /* the 'continue' below skips the NUL-reserve guard: re-check */
+            if (utfBufferJ + utfBufferSize >= max) {
+              return -1;
+            }
+
             /* Rollback write position to sequence start write position */
             j = utfBufferJ;
 
@@ -314,8 +319,8 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
       }
     }
 
-    /* Check for overflow */
-    if (j + 1 > max) {
+    /* reserve one byte for the trailing NUL written after the loop */
+    if (j + 1 >= max) {
       return -1;
     }
 

src/htsfilters.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsbase.h"
 #include "htslib.h"
 #include <ctype.h>
+#include <stdint.h>
 /* END specific definitions */
 
 // à partir d'un tableau de {"+*.toto","-*.zip","+*.tata"} définit si nom est autorisé
@@ -94,17 +95,127 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
   return verdict;
 }
 
-// supercomparateur joker (tm)
-// compare a et b (b=avec joker dedans), case insensitive [voir CI]
-// renvoi l'adresse de la première lettre de la chaine
-// (càd *[..]toto.. renvoi adresse de toto dans la chaine)
-// accepte les délires du genre www.*.*/ * / * truc*.*
-// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
-// * = toute lettre
-// --?-- : spécifique à HTTrack et aux ?
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
-                          int *size_flag) {
-  //int err=0;
+int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
+                     const char *nom2, LLint *size, int *size_flag,
+                     int *depth) {
+  int depth1 = 0, depth2 = 0;
+  int flag1 = 0, flag2 = 0;
+  LLint sz1 = 0, sz2 = 0;
+  int jok1, jok2, use1;
+
+  if (size) {
+    sz1 = *size;
+    sz2 = *size;
+  }
+  jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
+                     size_flag ? &flag1 : NULL, &depth1);
+  jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
+                     size_flag ? &flag2 : NULL, &depth2);
+  use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
+  if (size)
+    *size = use1 ? sz1 : sz2;
+  if (size_flag)
+    *size_flag = use1 ? flag1 : flag2;
+  if (depth)
+    *depth = use1 ? depth1 : depth2;
+  return use1 ? jok1 : jok2;
+}
+
+/* Real filters/URLs fit HTS_URLMAXSIZE*2; past it a hostile pattern recurses
+   O(len) deep or runs O(n^2*stars). Cap length (depth) and steps (work). */
+#define STRJOKER_MAXLEN (HTS_URLMAXSIZE * 2)
+#define STRJOKER_MAXSTEPS 2000000u
+
+/* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
+   pair keeps star-heavy patterns polynomial instead of exponential (#501). */
+typedef struct strjoker_memo {
+  const char *chaine0, *joker0; /* offsets are relative to these bases */
+  size_t stride;                /* strlen(joker0) + 1 */
+  unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
+  size_t *nsteps; /* shared work counter; NULL: unbounded (oracle) */
+} strjoker_memo;
+
+static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
+                                 const char *joker, LLint *size,
+                                 int *size_flag);
+
+/* A pair that failed once fails forever (*size is only written on the success
+   path), so record NULL results and cut the re-exploration. */
+static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
+                                const char *joker, LLint *size,
+                                int *size_flag) {
+  size_t bit = 0;
+  const char *adr;
+
+  if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
+    return NULL; /* work budget spent: fail the match safely */
+  if (memo->failed) {
+    bit = (size_t) (chaine - memo->chaine0) * memo->stride +
+          (size_t) (joker - memo->joker0);
+    if (memo->failed[bit >> 3] & (unsigned char) (1u << (bit & 7)))
+      return NULL;
+  }
+  adr = strjoker_impl(memo, chaine, joker, size, size_flag);
+  if (adr == NULL && memo->failed)
+    memo->failed[bit >> 3] |= (unsigned char) (1u << (bit & 7));
+  return adr;
+}
+
+/* Match chaine against joker with a shared work budget *nsteps (a strjokerfind
+   sweep passes one counter, bounding the whole scan). */
+static const char *strjoker_bounded(const char *chaine, const char *joker,
+                                    LLint *size, int *size_flag,
+                                    size_t *nsteps) {
+  strjoker_memo memo = {chaine, joker, 0, NULL, nsteps};
+  unsigned char stackbits[2048];
+  hts_boolean onheap = HTS_FALSE;
+  const char *adr;
+
+  if (chaine != NULL && joker != NULL) {
+    const size_t l1 = strlen(chaine), l2 = strlen(joker);
+
+    if (l1 > STRJOKER_MAXLEN || l2 > STRJOKER_MAXLEN)
+      return NULL; /* beyond any real filter/URL: bound depth+work */
+    memo.stride = l2 + 1;
+    if (l1 + 1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
+      const size_t bytes = ((l1 + 1) * memo.stride + 7) / 8;
+
+      if (bytes <= sizeof(stackbits)) {
+        memset(stackbits, 0, bytes);
+        memo.failed = stackbits;
+      } else {
+        memo.failed = (unsigned char *) calloct(bytes, 1);
+        onheap = memo.failed != NULL ? HTS_TRUE : HTS_FALSE;
+      }
+    }
+  }
+  adr = strjoker_rec(&memo, chaine, joker, size, size_flag);
+  if (onheap)
+    freet(memo.failed);
+  return adr;
+}
+
+// wildcard comparator: match chaine against joker (pattern), case-insensitive
+// returns the address of the first matched letter past any leading joker
+// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
+                                LLint *size, int *size_flag) {
+  size_t nsteps = 0;
+
+  return strjoker_bounded(chaine, joker, size, size_flag, &nsteps);
+}
+
+/* Test-only oracle: the same matcher without the failure memo. */
+const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
+                            int *size_flag) {
+  const strjoker_memo memo = {chaine, joker, 0, NULL};
+
+  return strjoker_rec(&memo, chaine, joker, size, size_flag);
+}
+
+static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
+                                 const char *joker, LLint *size,
+                                 int *size_flag) {
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -236,9 +347,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
                 for(j = (int) (unsigned char) joker[i];
                     j <= (int) (unsigned char) joker[i + 2]; j++)
                   pass[j] = 1;
-
               }
-              // else err=1;
               i += 3;
             } else { // 1 car, ex: *[ ]
               pass[(int) (unsigned char) joker[i]] = 1;
@@ -262,7 +371,6 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       for(i = 0; i < 256; i++)
         pass[i] = 1;            // tout autoriser
       jmp = 1;
-      ////if (joker[2]==LEFT) jmp=3;        // permet de recher *<crochet ouvrant>
     }
 
     {
@@ -281,7 +389,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
 
       // tester sans le joker (pas ()+ mais ()*)
       if (!unique) {
-        if ((adr = strjoker(chaine, joker + jmp, size, size_flag))) {
+        if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag))) {
           return adr;
         }
       }
@@ -290,10 +398,11 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       if (!unique)
         max = (int) strlen(chaine);
       else                      /* *(a) only match a (not aaaaa) */
-        max = 1;
+        max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
-          if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
+          if ((adr = strjoker_rec(memo, chaine + i + 1, joker + jmp, size,
+                                  size_flag))) {
             return adr;
           }
           i++;
@@ -303,7 +412,8 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
 
       // tester chaîne vide
       if (i != max + 2)         // avant c'est ok
-        if ((adr = strjoker(chaine + max, joker + jmp, size, size_flag)))
+        if ((adr = strjoker_rec(memo, chaine + max, joker + jmp, size,
+                                size_flag)))
           return adr;
 
       return NULL;              // perdu
@@ -325,7 +435,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
       // comparaison ok?
       if (ok) {
         // continuer la comparaison.
-        if (strjoker(chaine + jmp, joker + jmp, size, size_flag))
+        if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag))
           return chaine;        // retourner 1e lettre
       }
 
@@ -341,12 +451,18 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
 // d'un strcpy sur une variable ayant un nom en lettres et copiant une chaine de chiffres
 // ATTENTION!! Eviter les jokers en début, où gare au temps machine!
 const char *strjokerfind(const char *chaine, const char *joker) {
+  size_t nsteps = 0; /* one budget for the whole scan */
   const char *adr;
 
+  if (chaine != NULL && strlen(chaine) > STRJOKER_MAXLEN)
+    return NULL;
   while(*chaine) {
-    if ((adr = strjoker(chaine, joker, NULL, NULL))) {  // ok trouvé
+    if ((adr = strjoker_bounded(chaine, joker, NULL, NULL,
+                                &nsteps))) { // ok trouvé
       return adr;
     }
+    if (nsteps > STRJOKER_MAXSTEPS) // scan budget spent: no match
+      return NULL;
     chaine++;
   }
   return NULL;

src/htsfilters.h

@@ -41,8 +41,17 @@ Please visit our Website: http://www.httrack.com
 
 int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
                 int *size_flag, int *depth);
+/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
+   scheme); the match latest in the list wins, a "don't know" verdict defers.
+   Returns the merged verdict; the out-params carry the winner's values. */
+int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
+                     const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
+/* strjoker() without the failure memo (exponential worst case); test-only
+   oracle for the memoized matcher. */
+const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
+                            int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);
 #endif
 

src/htsftp.c

@@ -54,15 +54,9 @@ Please visit our Website: http://www.httrack.com
 #endif
 
 // ftp mode passif
-// #if HTS_INET6==0
 #define FTP_PASV 1
-// #else
-// no passive mode for v6
-// #define FTP_PASV 0
-// #endif
 
 #define FTP_DEBUG 0
-//#define FORK_DEBUG 0
 
 #if USE_BEGINTHREAD
 
@@ -128,6 +122,33 @@ void launch_ftp(FTPDownloadStruct * params) {
   return 0; \
   }
 
+/* Bounded split of a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
+void ftp_split_userpass(const char *src, const char *end, char *user,
+                        size_t user_size, char *pass, size_t pass_size) {
+  size_t n = 0;
+
+  assertf(user_size > 0 && pass_size > 0); /* the size-1 math underflows on 0 */
+
+  while (src[n] != '\0' && src[n] != ':') {
+    if (n < user_size - 1)
+      user[n] = src[n];
+    n++;
+  }
+  user[n < user_size ? n : user_size - 1] = '\0';
+  pass[0] = '\0';
+  if (src[n] == ':') { // password follows the colon
+    const size_t base = n + 1;
+    size_t k = 0;
+
+    while (&src[base + k + 1] < end && src[base + k] != '\0') {
+      if (k < pass_size - 1)
+        pass[k] = src[base + k];
+      k++;
+    }
+    pass[k < pass_size ? k : pass_size - 1] = '\0';
+  }
+}
+
 // la véritable fonction une fois lancées les routines thread/fork
 int run_launch_ftp(FTPDownloadStruct * pStruct) {
   lien_back *back = pStruct->pBack;
@@ -173,43 +194,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   while(*real_adr == '/')
     real_adr++;                 // sauter /
   if ((adr = jump_identification(real_adr)) != real_adr) {      // user
-    int i = -1;
-
-    pass[0] = '\0';
-    do {
-      i++;
-      user[i] = real_adr[i];
-    } while((real_adr[i] != ':') && (real_adr[i]));
-    user[i] = '\0';
-    if (real_adr[i] == ':') {   // pass
-      int j = -1;
-
-      i++;                      // oui on saute aussi le :
-      do {
-        j++;
-        pass[j] = real_adr[i + j];
-      } while(((&real_adr[i + j + 1]) < adr) && (real_adr[i + j]));
-      pass[j] = '\0';
-    }
+    ftp_split_userpass(real_adr, adr, user, sizeof(user), pass, sizeof(pass));
   }
   // Calculer RETR <nom>
   {
     char *a;
 
-#if 0
-    a = back->url_fil + strlen(back->url_fil) - 1;
-    while((a > back->url_fil) && (*a != '/'))
-      a--;
-    if (*a != '/') {
-      a = NULL;
-    }
-#else
     a = back->url_fil;
-#endif
     if (a != NULL && *a != '\0') {
-#if 0
-      a++;                      // sauter /
-#endif
       ftp_filename = a;
       if (strnotempty(a)) {
         char catbuff[CATBUFF_SIZE];
@@ -233,7 +225,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
       }
     } else {
       strcpybuff(back->r.msg, "Unexpected PORT error");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
     }
   }
@@ -267,7 +258,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
       snprintf(back->r.msg, sizeof(back->r.msg),
                "Unable to get server's address: %s", error);
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_NON_FATAL;
       _HALT_FTP return 0;
     }
@@ -280,20 +270,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
     if (soc_ctl == INVALID_SOCKET) {
       strcpybuff(back->r.msg, "Unable to create a socket");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
     }
 
     SOCaddr_initport(server, port);
-    // server.sin_port = htons((unsigned short int) port);
 
     // connexion (bloquante, on est en thread)
     strcpybuff(back->info, "connect");
 
     if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
       strcpybuff(back->r.msg, "Unable to connect to the server");
-      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
 #ifdef _WIN32
@@ -335,64 +322,22 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               // ok
             } else {
               strcpybuff(back->r.msg, "TYPE I error");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
-#if 0
-            // --CWD--
-            char *a;
-
-            a = back->url_fil + strlen(back->url_fil) - 1;
-            while((a > back->url_fil) && (*a != '/'))
-              a--;
-            if (*a == '/') {    // ok repéré
-              char BIGSTK target[1024];
-
-              target[0] = '\0';
-              strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
-              if (strnotempty(target) == 0)
-                strcatbuff(target, "/");
-              strcpybuff(back->info, "cwd");
-              snprintf(line, sizeof(line), "CWD %s", target);
-              send_line(soc_ctl, line);
-              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-              _CHECK_HALT_FTP;
-              if (line[0] == '2') {
-                send_line(soc_ctl, "TYPE I");
-                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-                _CHECK_HALT_FTP;
-                if (line[0] == '2') {
-                  // ok..
-                } else {
-                  strcpybuff(back->r.msg, "TYPE I error");
-                  // back->status=STATUS_FTP_READY;    // fini
-                  back->r.statuscode = STATUSCODE_INVALID;
-                }
-              } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
-                // back->status=STATUS_FTP_READY;    // fini
-                back->r.statuscode = STATUSCODE_INVALID;
-              }                 // sinon on est prêts
-            } else {
-              strcpybuff(back->r.msg, "Unexpected ftp error");
-              // back->status=STATUS_FTP_READY;    // fini
-              back->r.statuscode = STATUSCODE_INVALID;
-            }
-#endif
 
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
-          // back->status=STATUS_FTP_READY;    // fini
+          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
+                   linejmp(line));
           back->r.statuscode = STATUSCODE_INVALID;
         }
       } else {
-        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
-        // back->status=STATUS_FTP_READY;    // fini
+        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
+                 linejmp(line));
         back->r.statuscode = STATUSCODE_INVALID;
       }
 
@@ -459,8 +404,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
             // -- fin analyse de l'adresse IP et du port --
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
@@ -491,13 +436,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 }
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
-              // back->status=STATUS_FTP_READY;    // fini
+              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
+                       linejmp(line));
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         }
@@ -603,9 +548,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   deletesoc(soc_dat);
                   soc_dat = INVALID_SOCKET;
                   //
-                  snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
-                          linejmp(line));
-                  // back->status=STATUS_FTP_READY;    // fini
+                  snprintf(back->r.msg, sizeof(back->r.msg),
+                           "RETR command error: %s", linejmp(line));
                   back->r.statuscode = STATUSCODE_INVALID;
                 }               // sinon on est prêts
               } else {
@@ -616,23 +560,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 soc_dat = INVALID_SOCKET;
                 //
                 strcpybuff(back->r.msg, "Unable to connect");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }                 // sinon on est prêts
             } else {
               strcpybuff(back->r.msg, "Unable to create a socket");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }                   // sinon on est prêts
           } else {
             snprintf(back->r.msg, sizeof(back->r.msg),
                      "Unable to resolve IP %s: %s", adr_ip, error);
-            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
-          // back->status=STATUS_FTP_READY;    // fini
+          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
+                   linejmp(line));
           back->r.statuscode = STATUSCODE_INVALID;
         }                       // sinon on est prêts
 #else
@@ -653,17 +594,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               //T_SOC soc_dat;
               if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
                 strcpybuff(back->r.msg, "Unable to accept connection");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
-              // back->status=STATUS_FTP_READY;    // fini
+              snprintf(back->r.msg, sizeof(back->r.msg),
+                       "RETR command error: %s", linejmp(line));
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
+                     linejmp(line));
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -673,7 +613,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 #endif
         } else {
           strcpybuff(back->r.msg, "Unable to listen to a port");
-          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
 #endif
@@ -697,21 +636,18 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             int len = 1;
             int read_len = 1024;
 
-            //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
-
             while((len > 0) && (!stop_ftp(back))) {
               // attendre les données
               len = 1;          // pas d'erreur pour le moment
               switch (wait_socket_receive(soc_dat, timeout)) {
               case -1:
                 strcpybuff(back->r.msg, "FTP read error");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
               case 0:
-                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
-                // back->status=STATUS_FTP_READY;    // fini
+                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
+                         timeout);
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
@@ -733,17 +669,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                          }
                        */
                       strcpybuff(back->r.msg, "Write error");
-                      // back->status=STATUS_FTP_READY;    // fini
                       back->r.statuscode = STATUSCODE_INVALID;
                       len = 0;  // error
                     }
                   } else {
                     strcpybuff(back->r.msg, "Unexpected write error");
-                    // back->status=STATUS_FTP_READY;    // fini
                     back->r.statuscode = STATUSCODE_INVALID;
                   }
-                } else {        // Erreur ou terminé
-                  // back->status=STATUS_FTP_READY;    // fini
+                } else { // Erreur ou terminé
                   back->r.statuscode = 0;
                   if (back->r.totalsize > 0
                       && back->r.size != back->r.totalsize) {
@@ -752,7 +685,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   }
                 }
                 read_len = 1024;
-                //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
               }
             }
             if (back->r.fp) {
@@ -761,7 +693,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
           } else {
             strcpybuff(back->r.msg, "Unable to write file");
-            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -777,16 +708,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               get_ftp_line(soc_ctl, line, sizeof(line), timeout);
               if (line[0] == '2') {     // OK
                 strcpybuff(back->r.msg, "OK");
-                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = HTTP_OK;
               } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
-                // back->status=STATUS_FTP_READY;    // fini
+                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
+                         linejmp(line));
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
               strcpybuff(back->r.msg, "FTP read error");
-              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           }
@@ -812,7 +741,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     back->r.statuscode = HTTP_OK;
     strcpybuff(back->r.msg, "OK");
   }
-  // back->status=STATUS_FTP_READY;    // fini
   return 0;
 }
 
@@ -979,13 +907,11 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
       break;
     }
 
-    //HTS_TOTAL_RECV_CHECK(dummy);     // Diminuer au besoin si trop de données reçues
     switch (recv(soc, &b, 1, 0)) {
-      //case 0: break;    // pas encore --> erreur (on attend)!
     case 1:
       HTS_STAT.HTS_TOTAL_RECV += 1;     // compter flux entrant
-      if ((b != 10) && (b != 13))
-        data[i++] = b;
+      if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
+        data[i++] = b; // truncate hostile over-long reply lines
       break;
     default:
       if (ptrline)
@@ -1108,7 +1034,6 @@ int wait_socket_receive(T_SOC soc, int timeout) {
 int stop_ftp(lien_back * back) {
   if (back->stop_ftp) {
     strcpybuff(back->r.msg, "Cancelled by User");
-    // back->status=STATUS_FTP_READY;    // fini
     back->r.statuscode = STATUSCODE_INVALID;
     return 1;
   }

src/htsftp.h

@@ -70,6 +70,11 @@ int back_launch_ftp(FTPDownloadStruct * params);
 int run_launch_ftp(FTPDownloadStruct * params);
 int send_line(T_SOC soc, const char *data);
 int get_ftp_line(T_SOC soc, char *line, size_t line_size, int timeout);
+/* Split a "user[:pass]@" prefix (end = jump_identification result) into
+   bounded, NUL-terminated user/pass buffers, truncating to fit.
+   Both sizes must be nonzero. */
+void ftp_split_userpass(const char *src, const char *end, char *user,
+                        size_t user_size, char *pass, size_t pass_size);
 T_SOC get_datasocket(char *to_send, size_t to_send_size);
 int stop_ftp(lien_back * back);
 char *linejmp(char *line);

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-10"
-#define HTTRACK_VERSIONID "3.49.10"
+#define HTTRACK_VERSION "3.49-12"
+#define HTTRACK_VERSIONID "3.49.12"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 
@@ -56,6 +56,10 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
+// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
+#include <stdint.h>
+#include <inttypes.h>
+
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -111,29 +115,6 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
-#ifndef HTS_LONGLONG
-#ifdef SIZEOF_LONG_LONG
-#if SIZEOF_LONG_LONG == 8
-#define HTS_LONGLONG 1
-#endif
-#endif
-
-#ifndef HTS_LONGLONG
-#ifdef __sun
-#define HTS_LONGLONG 0
-#endif
-#ifdef __osf__
-#define HTS_LONGLONG 0
-#endif
-#ifdef __linux
-#define HTS_LONGLONG 1
-#endif
-#ifdef _WIN32
-#define HTS_LONGLONG 1
-#endif
-#endif
-#endif
-
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -326,59 +307,11 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-#ifndef HTS_LONGLONG
-#ifdef HTS_NO_64_BIT
-#define HTS_LONGLONG 0
-#else
-#define HTS_LONGLONG 1
-#endif
-#endif
-
-/* Wide integer types, chosen per platform.
-   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
-           plain int where 64-bit is unavailable).
-   TStamp: timestamp/duration in the same width (a double in the no-64-bit
-           fallback).
-   LLintP: the printf conversion for an LLint. */
-#if HTS_LONGLONG
-#ifdef LLINT_FORMAT
-typedef LLINT_TYPE LLint;
-
-typedef LLINT_TYPE TStamp;
-
-#define LLintP LLINT_FORMAT
-#else
-
-#ifdef _WIN32
-typedef __int64 LLint;
-
-typedef __int64 TStamp;
-
-#define LLintP "%I64d"
-#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
-       defined(__64BIT__))
-
-typedef long int LLint;
-
-typedef long int TStamp;
-
-#define LLintP "%ld"
-#else
-typedef long long int LLint;
-
-typedef long long int TStamp;
-
-#define LLintP "%lld"
-#endif
-
-#endif /* HTS_LONGLONG */
-
-#else
-typedef int LLint;
-
-#define LLintP "%d"
-typedef double TStamp;
-#endif
+/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
+typedef int64_t LLint;
+typedef int64_t TStamp;
+/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
+#define LLintP "%" PRId64
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain
@@ -422,9 +355,10 @@ typedef int T_SOC;
 #endif
 
 /* Permission bits for created folders and files (mkdir and chmod).
-   PROTECT_FOLDER is owner-only. With HTS_ACCESS set (the default) the ACCESS_
-   modes also grant group/other read; otherwise they stay owner-only. */
+   PROTECT_FOLDER/FILE are owner-only. With HTS_ACCESS set (the default) the
+   ACCESS_ modes also grant group/other read; otherwise they stay owner-only. */
 #define HTS_PROTECT_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
+#define HTS_PROTECT_FILE (S_IRUSR | S_IWUSR)
 
 #if HTS_ACCESS
 #define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)

src/htshelp.c

@@ -152,16 +152,6 @@ void help_wizard(httrackp * opt) {
 #define str (buffers->str)
 #define argv (buffers->argv)
 
-  //char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
-  //char *mainpath = (char *) malloct(256);
-  //char *projname = (char *) malloct(256);
-  //char *stropt = (char *) malloct(2048);        // options
-  //char *stropt2 = (char *) malloct(2048);       // options longues
-  //char *strwild = (char *) malloct(2048);       // wildcards
-  //char *cmd = (char *) malloct(4096);
-  //char *str = (char *) malloct(256);
-  //char **argv = (char **) malloct(256 * sizeof(char *));
-
   //
   char *a;
 
@@ -227,8 +217,6 @@ void help_wizard(httrackp * opt) {
   strcatbuff(stropt2, str);
   strcatbuff(stropt2, projname);
   strcatbuff(stropt2, "\" ");
-  // Créer si ce n'est fait un index.html 1er niveau
-  make_empty_index(str);
   //
   printf("\n");
   printf("Enter URLs (separated by commas or blank spaces) :");
@@ -349,8 +337,6 @@ void help_wizard(httrackp * opt) {
       }
       hts_main(argc, argv);
     }
-    //} else {
-    //  help("httrack",1);
   }
 
   /* Free buffers */
@@ -455,23 +441,6 @@ void help_catchurl(const char *dest_path) {
     printf("Unable to create a temporary proxy (no remaining port)\n");
 }
 
-// Créer un index.html vide
-void make_empty_index(const char *str) {
-#if 0
-  if (!fexist(fconcat(str, "index.html"))) {
-    FILE *fp = fopen(fconcat(str, "index.html"), "wb");
-
-    if (fp) {
-      fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
-      fprintf(fp,
-              "<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
-              CRLF);
-      fclose(fp);
-    }
-  }
-#endif
-}
-
 // mini-aide  (h: help)
 //           y
 void help(const char *app, int more) {
@@ -574,10 +543,11 @@ void help(const char *app, int more) {
   infomsg("Spider options:");
   infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
   infomsg(" %K  load extra cookies from a Netscape cookies.txt");
+  infomsg(" %Y  explain which filter rule accepts or rejects a URL, then exit");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
-  infomsg
-    ("  j *parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive)");
+  infomsg("  j *parse scripts (j0 don't parse, bitmask: |1 parse default, |4 "
+          "don't parse .js |8 don't be aggressive)");
   infomsg
     ("  sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules))");
   infomsg
@@ -601,8 +571,7 @@ void help(const char *app, int more) {
     ("     can also be used to force a specific file type: --assume foo.cgi=text/html");
   infomsg
     (" @iN internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only)");
-  infomsg
-    (" %w  disable a specific external mime module (-%w htsswf -%w htsjava)");
+  infomsg(" %w  disable a specific external mime module (-%w httrack-plugin)");
   infomsg("");
   infomsg("Browser ID:");
   infomsg
@@ -808,7 +777,4 @@ void help(const char *app, int more) {
   infomsg("[compiled: " HTS_PLATFORM_NAME "]");
 #endif
   infomsg(NULL);
-
-//  infomsg("  R  *relative links (e.g ../link)\n");
-//  infomsg("  A   absolute links (e.g /www.adr/link)\n");
 }

src/htshelp.h

@@ -45,7 +45,6 @@ typedef struct httrackp httrackp;
 
 void infomsg(const char *msg);
 void help(const char *app, int more);
-void make_empty_index(const char *str);
 void help_wizard(httrackp * opt);
 int help_query(const char *list, int def);
 void help_catchurl(const char *dest_path);

src/htsindex.c

@@ -142,8 +142,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
   char keyword[KEYW_LEN + 32];
   int i = 0;
 
-  //
-  //int WordIndexSize = 1024;
   coucal WordIndexHash = NULL;
   FILE *tmpfp = NULL;
 
@@ -178,8 +176,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
              || (strfield2(mime, "text/css"))
     ) {
     inscript = 1;
-    //} else if (strfield2(mime, "text/vnd.wap.wml")) {   // humm won't work in many cases
-    //  inscript=0;
   } else
     return 0;
 
@@ -275,8 +271,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
 
   // Process indexing for this page
   {
-    //FILE* fp=NULL;
-    //fp=fopen(concat(indexpath,"index.txt"),"ab");
     if (fp_tmpproject) {
       while(!feof(tmpfp)) {
         char line[KEYW_LEN + 32];
@@ -286,7 +280,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
           intptr_t e = 0;
 
           if (coucal_read(WordIndexHash, line, &e)) {
-            //if (e) {
             char BIGSTK savelst[HTS_URLMAXSIZE * 2];
 
             e++;                /* 0 means "once" */
@@ -300,11 +293,9 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
             fprintf(fp_tmpproject, "%s %d %s\n", line,
                     (int) (KEYW_SORT_MAXCOUNT - e), savelst);
             hts_primindex_size++;
-            //}
           }
         }
       }
-      //fclose(fp);
     }
   }
 
@@ -330,7 +321,6 @@ void index_finish(const char *indexpath, int mode) {
   off_t size = fpsize(fp_tmpproject);
 
   if (size > 0) {
-    //FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
     if (fp_tmpproject) {
       tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
       if (tab) {
@@ -398,8 +388,6 @@ void index_finish(const char *indexpath, int mode) {
                       if (total_hit) {
                         if (mode == 1)  // TEXT
                           fprintf(fp, "\t=%d\r\n", total_hit);
-                        //else                // HTML
-                        //  fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
                         if ((((total_hit * 1000) / hts_primindex_words) >=
                              KEYW_USELESS1K)
                             || (((total_line * 1000) / index) >=
@@ -416,8 +404,6 @@ void index_finish(const char *indexpath, int mode) {
                           if (mode == 1)        // TEXT
                             fprintf(fp, "\t(%d)\r\n",
                                     ((total_hit * 1000) / hts_primindex_words));
-                          //else                // HTML
-                          //  fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
                         }
                       }
                       if (mode == 1)    // TEXT

src/htsjava.c

@@ -1,516 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* ------------------------------------------------------------ */
-/* File: Java classes parser                                    */
-/* Author: Yann Philippot                                       */
-/* ------------------------------------------------------------ */
-
-/* Version: Oct/2000 */
-/* Fixed: problems with class structure (10/2000) */
-
-// htsjava.c - Parseur de classes java
-
-#ifdef HAVE_CONFIG_H
-#include "config.h"
-#endif
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/stat.h>
-#if ( defined(_WIN32) ||defined(HAVE_SYS_TYPES_H) )
-#include <sys/types.h>
-#endif
-#ifdef HAVE_UNISTD_H
-#include <unistd.h>
-#endif
-
-/* Standard httrack module includes */
-#include "httrack-library.h"
-#include "htsopt.h"
-#include "htsdefines.h"
-
-/* Module structures */
-#include "htsmodules.h"
-
-/* We link to libhttrack, we can use its functions */
-#include "httrack-library.h"
-
-/* This file */
-#include "htsjava.h"
-
-static int reverse_endian(void) {
-  int endian = 1;
-
-  return (*((char *) &endian) == 1);
-}
-
-/* big/little endian swap */
-#define hts_swap16(A) ( (((A) & 0xFF)<<8) | (((A) & 0xFF00)>>8) )
-#define hts_swap32(A) ( (( (hts_swap16(A)) & 0xFFFF)<<16) | (( (hts_swap16(A>>16)) & 0xFFFF)) )
-
-/* Static definitions */
-static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp, RESP_STRUCT,
-                             int *);
-static unsigned short int readshort(FILE * fp);
-static int tris(httrackp * opt, char *);
-static char *printname(char[1024], char[1024]);
-
-// ** HTS_xx sinon pas pris par VC++
-#define HTS_CLASS  7
-#define HTS_FIELDREF  9
-#define HTS_METHODREF  10
-#define HTS_STRING  8
-#define HTS_INTEGER  3
-#define HTS_FLOAT  4
-#define HTS_LONG  5
-#define HTS_DOUBLE  6
-#define HTS_INTERFACE  11
-#define HTS_NAMEANDTYPE  12
-#define HTS_ASCIZ  1
-#define HTS_UNICODE  2
-
-#define JAVADEBUG 0
-
-static const char *libName = "htsjava";
-
-#ifdef _WIN32
-#define strcasecmp(a,b) stricmp(a,b)
-#define strncasecmp(a,b,n) strnicmp(a,b,n)
-#endif
-
-static int detect_mime(htsmoduleStruct * str) {
-  const char *savename = str->filename;
-
-  if (savename) {
-    int len = (int) strlen(savename);
-
-    if (len > 6 && strcasecmp(savename + len - 6, ".class") == 0) {
-      return 1;
-    }
-  }
-  return 0;
-}
-
-static int hts_detect_java(t_hts_callbackarg * carg, httrackp * opt,
-                           htsmoduleStruct * str) {
-  /* Call parent functions if multiple callbacks are chained. */
-  if (CALLBACKARG_PREV_FUN(carg, detect) != NULL) {
-    if (CALLBACKARG_PREV_FUN(carg, detect)
-        (CALLBACKARG_PREV_CARG(carg), opt, str)) {
-      return 1;                 /* Found before us, let them have the priority */
-    }
-  }
-
-  /* Check MIME */
-  if (detect_mime(str)) {
-    str->wrapper_name = libName;        /* Our ID */
-    return 1;                   /* Known format, we take it */
-  }
-
-  return 0;                     /* Unknown format */
-}
-
-static off_t fsize(const char *s) {
-  STRUCT_STAT st;
-
-  if (STAT(s, &st) == 0 && S_ISREG(st.st_mode)) {
-    return st.st_size;
-  } else {
-    return -1;
-  }
-}
-
-static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
-                          htsmoduleStruct * str) {
-  /* The wrapper_name memebr has changed: not for us anymore */
-  if (str->wrapper_name == NULL || strcmp(str->wrapper_name, libName) != 0) {
-    /* Call parent functions if multiple callbacks are chained. */
-    if (CALLBACKARG_PREV_FUN(carg, parse) != NULL) {
-      return CALLBACKARG_PREV_FUN(carg, parse) (CALLBACKARG_PREV_CARG(carg),
-                                                opt, str);
-    }
-    strcpy(str->err_msg,
-           "unexpected error: bad wrapper_name and no previous wrapper");
-    return 0;                   /* Unexpected error */
-  } else {
-    if (detect_mime(str)) {
-
-      /* (Legacy code) */
-      char catbuff[CATBUFF_SIZE];
-      FILE *fpout;
-      JAVA_HEADER header;
-      RESP_STRUCT *tab;
-      const char *file = str->filename;
-
-      str->relativeToHtmlLink = 1;
-
-#if JAVADEBUG
-      printf("fopen\n");
-#endif
-      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
-        //fprintf(stderr, "Cannot open input file.\n");
-        sprintf(str->err_msg, "Unable to open file %s", file);
-        return 0;               // une erreur..
-      }
-#if JAVADEBUG
-      printf("fread\n");
-#endif
-      //if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) {   // pas complet..
-      if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
-        fclose(fpout);
-        sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
-                (LLint) fsize(file));
-        return 0;
-      }
-#if JAVADEBUG
-      printf("header\n");
-#endif
-      // tester en tête
-      if (reverse_endian()) {
-        header.magic = hts_swap32(header.magic);
-        header.count = hts_swap16(header.count);
-      }
-      if (header.magic != 0xCAFEBABE) {
-        sprintf(str->err_msg, "non java file");
-        if (fpout) {
-          fclose(fpout);
-          fpout = NULL;
-        }
-        return 0;
-      }
-
-      tab = (RESP_STRUCT *) calloc(header.count, sizeof(RESP_STRUCT));
-      if (!tab) {
-        sprintf(str->err_msg, "Unable to alloc %d bytes",
-                (int) sizeof(RESP_STRUCT));
-        if (fpout) {
-          fclose(fpout);
-          fpout = NULL;
-        }
-        return 0;               // erreur..
-      }
-#if JAVADEBUG
-      printf("calchead\n");
-#endif
-      {
-        int i;
-
-        for(i = 1; i < header.count; i++) {
-          int err = 0;          // ++    
-
-          tab[i] = readtable(str, fpout, tab[i], &err);
-          if (!err) {
-            if ((tab[i].type == HTS_LONG) || (tab[i].type == HTS_DOUBLE))
-              i++;              //2 element si double ou float
-          } else {              // ++ une erreur est survenue!
-            if (strnotempty(str->err_msg) == 0)
-              strcpy(str->err_msg, "Internal readtable error");
-            free(tab);
-            if (fpout) {
-              fclose(fpout);
-              fpout = NULL;
-            }
-            return 0;
-          }
-        }
-
-      }
-
-#if JAVADEBUG
-      printf("addfiles\n");
-#endif
-      {
-        //unsigned int acess;
-        unsigned int Class;
-        unsigned int SClass;
-        int i;
-
-        //acess = readshort(fpout);
-        Class = readshort(fpout);
-        SClass = readshort(fpout);
-
-        for(i = 1; i < header.count; i++) {
-
-          if (tab[i].type == HTS_CLASS) {
-
-            if ((tab[i].index1 < header.count) && (tab[i].index1 >= 0)) {
-
-              if ((tab[i].index1 != SClass) && (tab[i].index1 != Class)
-                  && (tab[tab[i].index1].name[0] != '[')) {
-
-                if (!strstr(tab[tab[i].index1].name, "java/")) {
-                  char BIGSTK tempo[1024];
-
-                  tempo[0] = '\0';
-
-                  snprintf(tempo, sizeof(tempo), "%s.class", tab[tab[i].index1].name);
-#if JAVADEBUG
-                  printf("add %s\n", tempo);
-#endif
-                  if (tab[tab[i].index1].file_position >= 0)
-                    str->addLink(str, tempo);   /* tab[tab[i].index1].file_position */
-                }
-
-              }
-            } else {
-              i = header.count; // exit 
-            }
-          }
-
-        }
-      }
-
-#if JAVADEBUG
-      printf("end\n");
-#endif
-      free(tab);
-      if (fpout) {
-        fclose(fpout);
-        fpout = NULL;
-      }
-      return 1;
-
-    } else {
-      strcpy(str->err_msg, "bad MIME type");
-    }
-  }
-  return 0;                     /* Error */
-}
-
-/*
-module entry point 
-*/
-EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
-EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
-  /* Plug callback functions */
-  CHAIN_FUNCTION(opt, detect, hts_detect_java, NULL);
-  CHAIN_FUNCTION(opt, parse, hts_parse_java, NULL);
-
-  return 1;                     /* success */
-}
-
-// error: !=0 si erreur fatale
-static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
-                             RESP_STRUCT trans, int *error) {
-  char rname[1024];
-  unsigned short int length;
-  int j;
-
-  *error = 0;                   // pas d'erreur
-  trans.file_position = -1;
-  trans.type = (int) (unsigned char) fgetc(fp);
-  switch (trans.type) {
-  case HTS_CLASS:
-    strcpy(trans.name, "Class");
-    trans.index1 = readshort(fp);
-    break;
-
-  case HTS_FIELDREF:
-    strcpy(trans.name, "Field Reference");
-    trans.index1 = readshort(fp);
-    readshort(fp);
-    break;
-
-  case HTS_METHODREF:
-    strcpy(trans.name, "Method Reference");
-    trans.index1 = readshort(fp);
-    readshort(fp);
-    break;
-
-  case HTS_INTERFACE:
-    strcpy(trans.name, "Interface Method Reference");
-    trans.index1 = readshort(fp);
-    readshort(fp);
-    break;
-  case HTS_NAMEANDTYPE:
-    strcpy(trans.name, "Name and Type");
-    trans.index1 = readshort(fp);
-    readshort(fp);
-    break;
-
-  case HTS_STRING:             // CONSTANT_String
-    strcpy(trans.name, "String");
-    trans.index1 = readshort(fp);
-    break;
-
-  case HTS_INTEGER:
-    strcpy(trans.name, "Integer");
-    for(j = 0; j < 4; j++)
-      fgetc(fp);
-    break;
-
-  case HTS_FLOAT:
-    strcpy(trans.name, "Float");
-    for(j = 0; j < 4; j++)
-      fgetc(fp);
-    break;
-
-  case HTS_LONG:
-    strcpy(trans.name, "Long");
-    for(j = 0; j < 8; j++)
-      fgetc(fp);
-    break;
-  case HTS_DOUBLE:
-    strcpy(trans.name, "Double");
-    for(j = 0; j < 8; j++)
-      fgetc(fp);
-    break;
-
-  case HTS_ASCIZ:
-  case HTS_UNICODE:
-
-    if (trans.type == HTS_ASCIZ)
-      strcpy(trans.name, "HTS_ASCIZ");
-    else
-      strcpy(trans.name, "HTS_UNICODE");
-
-    {
-      char BIGSTK buffer[1024];
-      char *p;
-
-      p = &buffer[0];
-
-      //fflush(fp); 
-      trans.file_position = ftell(fp);
-      length = readshort(fp);
-      if (length < HTS_URLMAXSIZE) {
-        // while ((length > 0) && (length<500)) {
-        while(length > 0) {
-          *p++ = fgetc(fp);
-
-          length--;
-        }
-        *p = '\0';
-
-        //#if JDEBUG
-        //      if(tris(buffer)==1) printf("%s\n ",buffer);
-        //      if(tris(buffer)==2)  printf("%s\n ",printname(buffer));
-        //#endif
-        if (tris(str->opt, buffer) == 1)
-          str->addLink(str, buffer);    /* trans.file_position */
-        else if (tris(str->opt, buffer) == 2)
-          str->addLink(str, printname(rname, buffer));
-
-        strcpy(trans.name, buffer);
-      } else {                  // gros pb
-        while((length > 0) && (!feof(fp))) {
-          fgetc(fp);
-          length--;
-        }
-        if (!feof(fp)) {
-          trans.type = -1;
-        } else {
-          sprintf(str->err_msg, "Internal structure error (ASCII)");
-          *error = 1;
-        }
-        return (trans);
-      }
-    }
-    break;
-  default:
-    // printf("Type inconnue\n");
-    // on arrête tout 
-    sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
-    *error = 1;
-    return (trans);
-    break;
-  }
-  return (trans);
-}
-
-static unsigned short int readshort(FILE * fp) {
-  unsigned short int valint;
-
-  if (fread(&valint, sizeof(valint), 1, fp) == 1) {
-    if (reverse_endian())
-      return hts_swap16(valint);
-    else
-      return valint;
-  } else {
-    return 0;
-  }
-
-}
-
-static int tris(httrackp * opt, char *buffer) {
-  char catbuff[CATBUFF_SIZE];
-
-  //
-  // Java
-  if ((buffer[0] == '[') && buffer[1] == 'L' && (!strstr(buffer, "java/")))
-    return 2;
-  if (strstr(buffer, ".gif") || strstr(buffer, ".jpg")
-      || strstr(buffer, ".jpeg") || strstr(buffer, ".au"))
-    return 1;
-  // Ajouts R.X: test type
-  // Autres fichiers
-  {
-    char type[256];
-
-    if (get_httptype_sized(opt, type, sizeof(type), buffer,
-                           0)) // recognized type
-      return 1;
-    // ajout RX 05/2001
-    else if (is_dyntype(get_ext(catbuff, sizeof(catbuff), buffer)))      // asp,cgi...
-      return 1;
-  }
-  return 0;
-}
-
-static char *printname(char rname[1024], char name[1024]) {
-  char *p;
-  char *p1;
-  size_t j;
-
-  rname[0] = '\0';
-  //
-
-  p = &name[0];
-
-  if (*p != '[') {
-    return rname;  // ""
-  }
-  p += 2;
-  //rname=(char*)calloct(strlen(name)+8,sizeof(char));
-  p1 = rname;
-  for(j = 0; name[j] != '\0'; j++, p++) {
-    if (*p == '/')
-      *p1 = '.';
-    if (*p == ';') {
-      *p1 = '\0';
-      strcat(rname, ".class");
-      return rname;
-    } else
-      *p1 = *p;
-    p1++;
-  }
-  p1 -= 3;
-  *p1 = '\0';
-  return rname;
-
-}

src/htsjava.vcproj

@@ -1,373 +0,0 @@
-<?xml version="1.0" encoding="Windows-1252"?>
-<VisualStudioProject
-	ProjectType="Visual C++"
-	Version="9,00"
-	Name="htsjava"
-	ProjectGUID="{2F353408-2C6C-4D31-9AC0-08BDC064DF64}"
-	RootNamespace="htsjava"
-	Keyword="Win32Proj"
-	TargetFrameworkVersion="131072"
-	>
-	<Platforms>
-		<Platform
-			Name="Win32"
-		/>
-		<Platform
-			Name="x64"
-		/>
-	</Platforms>
-	<ToolFiles>
-	</ToolFiles>
-	<Configurations>
-		<Configuration
-			Name="Debug|Win32"
-			OutputDirectory="C:\temp\htsjava\Debug"
-			IntermediateDirectory="C:\temp\htsjava\Debug"
-			ConfigurationType="2"
-			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
-			CharacterSet="2"
-			>
-			<Tool
-				Name="VCPreBuildEventTool"
-			/>
-			<Tool
-				Name="VCCustomBuildTool"
-			/>
-			<Tool
-				Name="VCXMLDataGeneratorTool"
-			/>
-			<Tool
-				Name="VCWebServiceProxyGeneratorTool"
-			/>
-			<Tool
-				Name="VCMIDLTool"
-			/>
-			<Tool
-				Name="VCCLCompilerTool"
-				Optimization="0"
-				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
-				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
-				MinimalRebuild="true"
-				BasicRuntimeChecks="3"
-				RuntimeLibrary="1"
-				UsePrecompiledHeader="0"
-				WarningLevel="3"
-				Detect64BitPortabilityProblems="true"
-				DebugInformationFormat="4"
-			/>
-			<Tool
-				Name="VCManagedResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCPreLinkEventTool"
-			/>
-			<Tool
-				Name="VCLinkerTool"
-				OutputFile="C:\temp\httrack\htsjava.dll"
-				LinkIncremental="2"
-				GenerateDebugInformation="true"
-				ProgramDatabaseFile="$(OutDir)/htsjava.pdb"
-				SubSystem="2"
-				RandomizedBaseAddress="1"
-				DataExecutionPrevention="0"
-				ImportLibrary="$(OutDir)/htsjava.lib"
-				TargetMachine="1"
-			/>
-			<Tool
-				Name="VCALinkTool"
-			/>
-			<Tool
-				Name="VCManifestTool"
-			/>
-			<Tool
-				Name="VCXDCMakeTool"
-			/>
-			<Tool
-				Name="VCBscMakeTool"
-			/>
-			<Tool
-				Name="VCFxCopTool"
-			/>
-			<Tool
-				Name="VCAppVerifierTool"
-			/>
-			<Tool
-				Name="VCPostBuildEventTool"
-			/>
-		</Configuration>
-		<Configuration
-			Name="Debug|x64"
-			OutputDirectory="$(SolutionDir)$(PlatformName)\$(ConfigurationName)"
-			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
-			ConfigurationType="2"
-			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
-			CharacterSet="2"
-			>
-			<Tool
-				Name="VCPreBuildEventTool"
-			/>
-			<Tool
-				Name="VCCustomBuildTool"
-			/>
-			<Tool
-				Name="VCXMLDataGeneratorTool"
-			/>
-			<Tool
-				Name="VCWebServiceProxyGeneratorTool"
-			/>
-			<Tool
-				Name="VCMIDLTool"
-				TargetEnvironment="3"
-			/>
-			<Tool
-				Name="VCCLCompilerTool"
-				Optimization="0"
-				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
-				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
-				MinimalRebuild="true"
-				BasicRuntimeChecks="3"
-				RuntimeLibrary="1"
-				UsePrecompiledHeader="0"
-				WarningLevel="3"
-				Detect64BitPortabilityProblems="true"
-				DebugInformationFormat="3"
-			/>
-			<Tool
-				Name="VCManagedResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCPreLinkEventTool"
-			/>
-			<Tool
-				Name="VCLinkerTool"
-				OutputFile="C:\temp\httrack64\htsjava.dll"
-				LinkIncremental="2"
-				GenerateDebugInformation="true"
-				ProgramDatabaseFile="$(OutDir)/htsjava.pdb"
-				SubSystem="2"
-				RandomizedBaseAddress="1"
-				DataExecutionPrevention="0"
-				ImportLibrary="$(OutDir)/htsjava.lib"
-				TargetMachine="17"
-			/>
-			<Tool
-				Name="VCALinkTool"
-			/>
-			<Tool
-				Name="VCManifestTool"
-			/>
-			<Tool
-				Name="VCXDCMakeTool"
-			/>
-			<Tool
-				Name="VCBscMakeTool"
-			/>
-			<Tool
-				Name="VCFxCopTool"
-			/>
-			<Tool
-				Name="VCAppVerifierTool"
-			/>
-			<Tool
-				Name="VCPostBuildEventTool"
-			/>
-		</Configuration>
-		<Configuration
-			Name="Release|Win32"
-			OutputDirectory="C:\temp\htsjava\Release"
-			IntermediateDirectory="C:\temp\htsjava\Release"
-			ConfigurationType="2"
-			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
-			CharacterSet="2"
-			>
-			<Tool
-				Name="VCPreBuildEventTool"
-			/>
-			<Tool
-				Name="VCCustomBuildTool"
-			/>
-			<Tool
-				Name="VCXMLDataGeneratorTool"
-			/>
-			<Tool
-				Name="VCWebServiceProxyGeneratorTool"
-			/>
-			<Tool
-				Name="VCMIDLTool"
-			/>
-			<Tool
-				Name="VCCLCompilerTool"
-				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
-				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
-				RuntimeLibrary="0"
-				UsePrecompiledHeader="0"
-				WarningLevel="3"
-				Detect64BitPortabilityProblems="true"
-				DebugInformationFormat="0"
-			/>
-			<Tool
-				Name="VCManagedResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCPreLinkEventTool"
-			/>
-			<Tool
-				Name="VCLinkerTool"
-				OutputFile="O:\HTTrack\httrack\htsjava.dll"
-				LinkIncremental="1"
-				GenerateDebugInformation="true"
-				ProgramDatabaseFile="O:\HTTrack\httrack\htsjava.pdb"
-				SubSystem="2"
-				OptimizeReferences="2"
-				EnableCOMDATFolding="2"
-				RandomizedBaseAddress="1"
-				DataExecutionPrevention="0"
-				ImportLibrary="$(OutDir)/htsjava.lib"
-				TargetMachine="1"
-			/>
-			<Tool
-				Name="VCALinkTool"
-			/>
-			<Tool
-				Name="VCManifestTool"
-			/>
-			<Tool
-				Name="VCXDCMakeTool"
-			/>
-			<Tool
-				Name="VCBscMakeTool"
-			/>
-			<Tool
-				Name="VCFxCopTool"
-			/>
-			<Tool
-				Name="VCAppVerifierTool"
-			/>
-			<Tool
-				Name="VCPostBuildEventTool"
-				CommandLine="&quot;C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\signtool.exe&quot; sign /f &quot;Z:\Mes Documents\certificats\httrack-com\httrack-signing-certificate.pfx&quot; /t http://timestamp.verisign.com/scripts/timstamp.dll $(TargetPath)&#x0D;&#x0A;"
-			/>
-		</Configuration>
-		<Configuration
-			Name="Release|x64"
-			OutputDirectory="$(SolutionDir)$(PlatformName)\$(ConfigurationName)"
-			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
-			ConfigurationType="2"
-			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
-			CharacterSet="2"
-			>
-			<Tool
-				Name="VCPreBuildEventTool"
-			/>
-			<Tool
-				Name="VCCustomBuildTool"
-			/>
-			<Tool
-				Name="VCXMLDataGeneratorTool"
-			/>
-			<Tool
-				Name="VCWebServiceProxyGeneratorTool"
-			/>
-			<Tool
-				Name="VCMIDLTool"
-				TargetEnvironment="3"
-			/>
-			<Tool
-				Name="VCCLCompilerTool"
-				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
-				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
-				RuntimeLibrary="0"
-				UsePrecompiledHeader="0"
-				WarningLevel="3"
-				Detect64BitPortabilityProblems="true"
-				DebugInformationFormat="0"
-			/>
-			<Tool
-				Name="VCManagedResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCResourceCompilerTool"
-			/>
-			<Tool
-				Name="VCPreLinkEventTool"
-			/>
-			<Tool
-				Name="VCLinkerTool"
-				OutputFile="O:\HTTrack\httrack\x64\htsjava.dll"
-				LinkIncremental="1"
-				GenerateDebugInformation="true"
-				ProgramDatabaseFile="O:\HTTrack\httrack\x64\htsjava.pdb"
-				SubSystem="2"
-				OptimizeReferences="2"
-				EnableCOMDATFolding="2"
-				RandomizedBaseAddress="1"
-				DataExecutionPrevention="0"
-				ImportLibrary="$(OutDir)/htsjava.lib"
-				TargetMachine="17"
-			/>
-			<Tool
-				Name="VCALinkTool"
-			/>
-			<Tool
-				Name="VCManifestTool"
-			/>
-			<Tool
-				Name="VCXDCMakeTool"
-			/>
-			<Tool
-				Name="VCBscMakeTool"
-			/>
-			<Tool
-				Name="VCFxCopTool"
-			/>
-			<Tool
-				Name="VCAppVerifierTool"
-			/>
-			<Tool
-				Name="VCPostBuildEventTool"
-				CommandLine="&quot;C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\signtool.exe&quot; sign /f &quot;Z:\Mes Documents\certificats\httrack-com\httrack-signing-certificate.pfx&quot; /t http://timestamp.verisign.com/scripts/timstamp.dll $(TargetPath)&#x0D;&#x0A;"
-			/>
-		</Configuration>
-	</Configurations>
-	<References>
-	</References>
-	<Files>
-		<Filter
-			Name="Source Files"
-			Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
-			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
-			>
-			<File
-				RelativePath="htsjava.c"
-				>
-			</File>
-		</Filter>
-		<Filter
-			Name="Header Files"
-			Filter="h;hpp;hxx;hm;inl;inc;xsd"
-			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
-			>
-			<File
-				RelativePath="htsjava.h"
-				>
-			</File>
-		</Filter>
-		<Filter
-			Name="Resource Files"
-			Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
-			UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
-			>
-		</Filter>
-	</Files>
-	<Globals>
-	</Globals>
-</VisualStudioProject>

src/htslib.c

@@ -598,56 +598,27 @@ static const char *hts_mime_modern[][2] = {
 
 // Reserved (RFC2396)
 #define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
-#define CHAR_RESERVED(c)  ( CIS(c,';') \
-                         || CIS(c,'/') \
-                         || CIS(c,'?') \
-                         || CIS(c,':') \
-                         || CIS(c,'@') \
-                         || CIS(c,'&') \
-                         || CIS(c,'=') \
-                         || CIS(c,'+') \
-                         || CIS(c,'$') \
-                         || CIS(c,',') )
-//#define CHAR_RESERVED(c)  ( strchr(";/?:@&=+$,",(unsigned char)(c)) != 0 )
+#define CHAR_RESERVED(c)                                                       \
+  (CIS(c, ';') || CIS(c, '/') || CIS(c, '?') || CIS(c, ':') || CIS(c, '@') ||  \
+   CIS(c, '&') || CIS(c, '=') || CIS(c, '+') || CIS(c, '$') || CIS(c, ','))
 // Delimiters (RFC2396)
-#define CHAR_DELIM(c)     ( CIS(c,'<') \
-                         || CIS(c,'>') \
-                         || CIS(c,'#') \
-                         || CIS(c,'%') \
-                         || CIS(c,'\"') )
-//#define CHAR_DELIM(c)     ( strchr("<>#%\"",(unsigned char)(c)) != 0 )
+#define CHAR_DELIM(c)                                                          \
+  (CIS(c, '<') || CIS(c, '>') || CIS(c, '#') || CIS(c, '%') || CIS(c, '\"'))
 // Unwise (RFC2396)
-#define CHAR_UNWISE(c)    ( CIS(c,'{') \
-                         || CIS(c,'}') \
-                         || CIS(c,'|') \
-                         || CIS(c,'\\') \
-                         || CIS(c,'^') \
-                         || CIS(c,'[') \
-                         || CIS(c,']') \
-                         || CIS(c,'`') )
-//#define CHAR_UNWISE(c)    ( strchr("{}|\\^[]`",(unsigned char)(c)) != 0 )
+#define CHAR_UNWISE(c)                                                         \
+  (CIS(c, '{') || CIS(c, '}') || CIS(c, '|') || CIS(c, '\\') || CIS(c, '^') || \
+   CIS(c, '[') || CIS(c, ']') || CIS(c, '`'))
 // Special (escape chars) (RFC2396 + >127 )
 #define CHAR_LOW(c)       ( ((unsigned char)(c) <= 31) )
 #define CHAR_HIG(c)       ( ((unsigned char)(c) >= 127) )
 #define CHAR_SPECIAL(c)   ( CHAR_LOW(c) || CHAR_HIG(c) )
 // We try to avoid them and encode them instead
-#define CHAR_XXAVOID(c)   ( CIS(c,' ') \
-                         || CIS(c,'*') \
-                         || CIS(c,'\'') \
-                         || CIS(c,'\"') \
-                         || CIS(c,'&') \
-                         || CIS(c,'!') )
-//#define CHAR_XXAVOID(c)   ( strchr(" *'\"!",(unsigned char)(c)) != 0 )
-#define CHAR_MARK(c)      ( CIS(c,'-') \
-                         || CIS(c,'_') \
-                         || CIS(c,'.') \
-                         || CIS(c,'!') \
-                         || CIS(c,'~') \
-                         || CIS(c,'*') \
-                         || CIS(c,'\'') \
-                         || CIS(c,'(') \
-                         || CIS(c,')') )
-//#define CHAR_MARK(c)      ( strchr("-_.!~*'()",(unsigned char)(c)) != 0 )
+#define CHAR_XXAVOID(c)                                                        \
+  (CIS(c, ' ') || CIS(c, '*') || CIS(c, '\'') || CIS(c, '\"') ||               \
+   CIS(c, '&') || CIS(c, '!'))
+#define CHAR_MARK(c)                                                           \
+  (CIS(c, '-') || CIS(c, '_') || CIS(c, '.') || CIS(c, '!') || CIS(c, '~') ||  \
+   CIS(c, '*') || CIS(c, '\'') || CIS(c, '(') || CIS(c, ')'))
 
 // conversion éventuelle / vers antislash
 #ifdef _WIN32
@@ -841,15 +812,12 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
 // treat: traiter header?
 // waitconnect: attendre le connect()
 // note: dans retour, on met les params du proxy
-T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
-                  const char *xsend, const char *adr, const char *fil, htsblk * retour) {
-  //htsblk retour;
-  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
+T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
+                  const char *xsend, const char *adr, const char *fil,
+                  htsblk *retour) {
   T_SOC soc = INVALID_SOCKET;
   char BIGSTK tempo_fil[HTS_URLMAXSIZE * 2];
 
-  //char *p,*q;
-
   // retour prédéfini: erreur
   if (retour) {
     retour->adr = NULL;
@@ -934,7 +902,6 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
         strcpybuff(retour->msg, "Unable to open local file");
       else {
         // Note: On passe par un FILE* (plus propre)
-        //soc=open(fil,O_RDONLY,0);    // en lecture seule!
         retour->fp = FOPEN(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), 
           unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil)), "rb");      // ouvrir
         if (retour->fp == NULL)
@@ -1011,17 +978,10 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
 
         } while(strnotempty(rcvd));
 
-        //rcvsize=-1;    // forCER CHARGEMENT INCONNU
-
-        //if (retour)
-        //  retour->totalsize=rcvsize;
-
-      } else {                  // si GET, on recevra l'en tête APRES
-        //rcvsize=-1;    // on ne connait pas la taille de l'en-tête
+      } else { // si GET, on recevra l'en tête APRES
         if (retour)
           retour->totalsize = -1;
       }
-
     }
 
   }
@@ -1099,10 +1059,10 @@ static int append_cookie_header(buff_struct *bstr, t_cookie *cookie,
   return cook;
 }
 
-/* Self-test entry for append_cookie_header(): build the request Cookie line
-   into dst (always NUL-terminated). Returns the number of cookies emitted. */
-int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
-                                const char *path, char *dst, size_t dst_size) {
+/* Build the request Cookie line for domain/path into dst (always
+   NUL-terminated). Returns the number of cookies emitted. */
+int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
+                       char *dst, size_t dst_size) {
   buff_struct bstr = {dst, dst_size, 0};
 
   assertf(dst != NULL && dst_size > 0);
@@ -1118,19 +1078,12 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
   char BIGSTK buffer_head_request[16384];
   buff_struct bstr = { buffer_head_request, sizeof(buffer_head_request), 0 };
 
-  //int use_11=0;     // HTTP 1.1 utilisé
   int direct_url = 0;           // ne pas analyser l'url (exemple: ftp://)
   const char *search_tag = NULL;
 
   // Initialize buffer
   buffer_head_request[0] = '\0';
 
-  // header Date
-  //strcatbuff(buff,"Date: ");
-  //time_gmt_rfc822(buff);    // obtenir l'heure au format rfc822
-  //sendc("\n");
-  //strcatbuff(buff,buff);
-
   // possibilité non documentée: >post: et >postfile:
   // si présence d'un tag >post: alors executer un POST
   // exemple: http://www.example.com/test.cgi?foo>post:posteddata=10&foo=5
@@ -1149,7 +1102,8 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
           char BIGSTK protocol[256], url[HTS_URLMAXSIZE * 2], method[256];
 
           linput(fp, line, 1000);
-          if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
+          /* widths bound method[256], url[HTS_URLMAXSIZE*2], protocol[256] */
+          if (sscanf(line, "%255s %2047s %255s", method, url, protocol) == 3) {
             size_t ret;
             // selon que l'on a ou pas un proxy
             if (retour->req.proxy.active) {
@@ -1223,11 +1177,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
     }
 
     // protocole
-    if (!retour->req.http11) {  // forcer HTTP/1.0
-      //use_11=0;
+    if (!retour->req.http11) { // forcer HTTP/1.0
       print_buffer(&bstr, " HTTP/1.0\x0d\x0a");
-    } else {                    // Requète 1.1
-      //use_11=1;
+    } else { // Requète 1.1
       print_buffer(&bstr, " HTTP/1.1\x0d\x0a");
     }
 
@@ -1363,9 +1315,7 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
           print_buffer(&bstr, "Authorization: Basic %s"H_CRLF, autorisation);
         }
       }
-
     }
-    //strcatbuff(buff,"Accept-Charset: iso-8859-1,*,utf-8\n");
 
     // Custom header(s)
     if (strnotempty(retour->req.headers)) {
@@ -1519,14 +1469,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a += strlen("filename=");
         while(is_space(*a))
           a++;
-        //a=strchr(a,'"');
         if (a) {
           char *c = NULL;
 
-          //a++;      /* jump " */
           while((c = strchr(a, '/')))   /* skip all / (see RFC2616) */
             a = c + 1;
-          //b=strchr(a+1,'"');
           b = a + strlen(a) - 1;
           while(is_space(*b))
             b--;
@@ -1543,16 +1490,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
   } else if ((p = strfield(rcvd, "Last-Modified:")) != 0) {
     while(is_realspace(*(rcvd + p)))
       p++;                      // sauter espaces
-    if ((int) strlen(rcvd + p) < 64) {  // pas trop long?
-      //struct tm* tm_time=convert_time_rfc822(rcvd+p);
+    if ((int) strlen(rcvd + p) < 64) { // pas trop long?
       strcpybuff(retour->lastmodified, rcvd + p);
     }
   } else if ((p = strfield(rcvd, "Date:")) != 0) {
     if (strnotempty(retour->lastmodified) == 0) {       /* pas encore de last-modified */
       while(is_realspace(*(rcvd + p)))
         p++;                    // sauter espaces
-      if ((int) strlen(rcvd + p) < 64) {        // pas trop long?
-        //struct tm* tm_time=convert_time_rfc822(rcvd+p);
+      if ((int) strlen(rcvd + p) < 64) { // pas trop long?
         strcpybuff(retour->lastmodified, rcvd + p);
       }
     }
@@ -1565,14 +1510,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       else                      // erreur.. ignorer
         retour->etag[0] = '\0';
     }
-  }
-  // else if ((p=strfield(rcvd,"Transfer-Encoding: chunked"))!=0) {  // chunk!
-  else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) {   // chunk!
+  } else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
     while(is_realspace(*(rcvd + p)))
       p++;                      // sauter espaces
     if (strfield(rcvd + p, "chunked")) {
-      retour->is_chunk = 1;     // chunked
-      //retour->http11=2;     // chunked
+      retour->is_chunk = 1; // chunked
 #if HDEBUG
       printf("ok, Transfer-Encoding: détecté\n");
 #endif
@@ -1620,7 +1562,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       }
       // An empty/whitespace Content-Type value yields no token: keep the
       // sentinel default rather than reading an uninitialized tempo.
-      if (sscanf(rcvd + p, "%s", tempo) == 1) {
+      if (sscanf(rcvd + p, "%1099s", tempo) == 1) { // tempo[1100], server data
         if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
           strcpybuff(retour->contenttype, tempo);
         else
@@ -1644,7 +1586,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a = strchr(rcvd + p, '/');
         if (a != NULL) {
           a++;
-          if (sscanf(a, LLintP, &retour->crange) == 1) {
+          if (sscanf(a, LLintP, &retour->crange) == 1 && retour->crange >= 0) {
             retour->crange_start = 0;
             retour->crange_end = retour->crange - 1;
           } else {
@@ -1652,6 +1594,12 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           }
         }
       }
+      // A valid Content-Range has no negative field; reject hostile values so
+      // the crange +/- 1 arithmetic downstream cannot sign-overflow (UB).
+      if (retour->crange_start < 0 || retour->crange_end < 0 ||
+          retour->crange < 0) {
+        retour->crange_start = retour->crange_end = retour->crange = 0;
+      }
     }
   } else if ((p = strfield(rcvd, "Connection:")) != 0) {
     char *a = rcvd + p;
@@ -1715,11 +1663,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         if (a)
           *a = '\0';
       }
-      sscanf(a, "%s", tempo);
-      if (strlen(tempo) < 64)   // pas trop long!!
+      // Bound to tempo[1100]; no token => leave empty, don't read uninit tempo.
+      if (sscanf(a, "%1099s", tempo) == 1 && strlen(tempo) < 64)
         strcpybuff(retour->contentencoding, tempo);
       else
-        retour->contentencoding[0] = '\0';      // erreur
+        retour->contentencoding[0] = '\0';
 #if HTS_USEZLIB
       /* Check known encodings */
       if (retour->contentencoding[0]) {
@@ -1749,7 +1697,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           retour->location[0] = '\0';
       }
     }
-  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) && (cookie)) {  // ohh un cookie
+  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) &&
+             (cookie)) {        // ohh un cookie
     char *a = rcvd + p;         // pointeur
     char domain[256];           // domaine cookie (.netscape.com)
     char path[256];             // chemin (/)
@@ -1759,6 +1708,9 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
 #if DEBUG_COOK
     printf("set-cookie detected\n");
 #endif
+    /* Over-long host overflows the fail-safe domain[] copy (abort); drop it. */
+    if (adr && strlen(jump_identification_const(adr)) >= sizeof(domain))
+      return;
     while(*a) {
       char *token_st, *token_end;
       char *value_st, *value_end;
@@ -1793,10 +1745,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
             a++;                // sauter espaces
           value_st = a;
           while((*a != ';') && (*a))
-            a++;                // prochain ;
-          //while( ((*a!='"') || (*(a-1)=='\\')) && (*a)) a++;    // prochain " (et pas \")
+            a++; // prochain ;
           value_end = a;
-          //if (*a==';') {  // finit par un ;
           // vérifier débordements
           if ((((int) (token_end - token_st)) < 200)
               && (((int) (value_end - value_st)) < 8000)
@@ -2057,6 +2007,15 @@ LLint http_xfread1(htsblk * r, int bufl) {
 
   if (bufl > 0) {
     if (!r->is_write) {         // stocker en mémoire
+      // In-memory content must fit a 32-bit index (allocs below add 1, reads
+      // use int offsets): reject a hostile Content-Length or endless stream.
+      const LLint inmem_want =
+          (r->totalsize >= 0) ? r->totalsize : (r->size + bufl);
+      if (inmem_want >= INT32_MAX) {
+        r->statuscode = STATUSCODE_INVALID;
+        strcpybuff(r->msg, "In-memory content too large");
+        return READ_ERROR;
+      }
       if (r->totalsize >= 0) {  // totalsize déterminé ET ALLOUE
         if (r->adr == NULL) {
           r->adr = (char *) malloct((size_t) r->totalsize + 1);
@@ -2138,8 +2097,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
             nl = READ_ERROR;
           }
         }
-        //if ((nl < 0) || ((r->totalsize>0) && (r->size >= r->totalsize)))
-        //  nl=-1;  // break
 
         // libérer bloc tempo
         freet(buff);
@@ -2227,9 +2184,6 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
   T_SOC soc;
   htsblk retour;
 
-  //int rcvsize=-1;
-  //char* rcv=NULL;    // adresse de retour
-  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
   TStamp tl;
   int timeout = 30;             // timeout pour un check (arbitraire) // **
 
@@ -2238,11 +2192,8 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
 
   loc[0] = '\0';
   hts_init_htsblk(&retour);
-  //memset(&retour, 0, sizeof(htsblk));    // effacer
   retour.location = loc;        // si non nul, contiendra l'adresse véritable en cas de moved xx
 
-  //soc=http_fopen(adr,fil,&retour,NULL);  // ouvrir, + header
-
   // on ouvre en head, et on traite l'en tête
   soc = http_xfopen(opt, 1, 0, 1, NULL, adr, fil, &retour);     // ouvrir HEAD, + envoi header
 
@@ -2564,6 +2515,10 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
   // effacer adrfil->adr et adrfil->fil
   adrfil->adr[0] = adrfil->fil[0] = '\0';
 
+  // Reject an over-long URL: a root-relative path is copied whole into fil[].
+  if (strlen(url) >= HTS_URLMAXSIZE * 2)
+    return -1;
+
 #if HDEBUG
   printf("protocol: %s\n", url);
 #endif
@@ -2618,12 +2573,16 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
 
     // chemin www... trop long!!
     if ((((int) (q - p))) > HTS_URLMAXSIZE) {
-      //strcpybuff(retour.msg,"Path too long");
       return -1;                // erreur
     }
     // recopier adrfil->adresse www..
     strncatbuff(adrfil->adr, p, ((int) (q - p)));
     // *( adrfil->adr+( ((int) q) - ((int) p) ) )=0;  // faut arrêter la fumette!
+
+    // fil[] holds the path plus a possible leading '/' the top strlen() misses.
+    if (strlen(q) >= sizeof(adrfil->fil) - (q[0] != '/' ? 1 : 0))
+      return -1;
+
     // recopier chemin /pub/..
     if (q[0] != '/')            // page par défaut (/)
       strcatbuff(adrfil->fil, "/");
@@ -2640,7 +2599,7 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
     if (*p == '/' || *p == '\\') {      /* adrfil->file:///.. */
       strcatbuff(adrfil->fil, p);       // fichier local ; adrfil->adr="#"
     } else {
-      if (p[1] != ':') {
+      if (*p == '\0' || p[1] != ':') {  /* empty path: not a DOS drive letter */
         strcatbuff(adrfil->fil, "//");  /* adrfil->file://server/foo */
         strcatbuff(adrfil->fil, p);
       } else {
@@ -2804,7 +2763,6 @@ void deletesoc_r(htsblk * r) {
 #if HTS_USEOPENSSL
   if (r->ssl_con) {
     SSL_shutdown(r->ssl_con);
-    // SSL_CTX_set_quiet_shutdown(r->ssl_con->ctx, 1);
     SSL_free(r->ssl_con);
     r->ssl_con = NULL;
   }
@@ -3103,12 +3061,11 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
 /* See http://physics.nist.gov/cuu/Units/binary.html */
 #define ToLLint(a) ((LLint)(a))
 #define ToLLintKiB (ToLLint(1024))
-#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
-#ifdef HTS_LONGLONG
-#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#endif
+#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
+#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintPiB                                                             \
+  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
 HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
   if (n < ToLLintKiB) {
     sprintf(strc->buff1, "%d", (int) (LLint) n);
@@ -3117,9 +3074,7 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
             (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
     strcpybuff(strc->buff2, "KiB");
-  }
-#ifdef HTS_LONGLONG
-  else if (n < ToLLintGiB) {
+  } else if (n < ToLLintGiB) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
             (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
     strcpybuff(strc->buff2, "MiB");
@@ -3136,13 +3091,6 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
             (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
     strcpybuff(strc->buff2, "PiB");
   }
-#else
-  else {
-    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
-            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
-    strcpybuff(strc->buff2, "MiB");
-  }
-#endif
   strc->buffadr[0] = strc->buff1;
   strc->buffadr[1] = strc->buff2;
   return strc->buffadr;
@@ -3193,7 +3141,6 @@ int finput(T_SOC fd, char *s, int max) {
   int j = 0;
 
   do {
-    //c=fgetc(fp);
     if (read((int) fd, &c, 1) <= 0) {
       c = 0;
     }
@@ -3209,7 +3156,7 @@ int finput(T_SOC fd, char *s, int max) {
         break;
       }
     }
-  } while((c != 0) && (j < max - 1));
+  } while ((c != 0) && (j < max - 1));
   s[j] = '\0';
   return j;
 }
@@ -4868,7 +4815,6 @@ LLint check_downloadable_bytes(int rate) {
 
     time_now = mtime_local();
     elapsed_useconds = time_now - HTS_STAT.istat_timestart[id_timer];
-    // NO totally stupid - elapsed_useconds+=1000;      // for the next second, too
     bytes_transferred_during_period =
       (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[id_timer]);
 
@@ -4881,32 +4827,6 @@ LLint check_downloadable_bytes(int rate) {
     return TAILLE_BUFFER;
 }
 
-//
-// 0 : OK
-// 1 : slow down
-#if 0
-int HTS_TOTAL_RECV_CHECK(int var) {
-  if (HTS_STAT.HTS_TOTAL_RECV_STATE)
-    return 1;
-  /*
-     {
-     if (HTS_STAT.HTS_TOTAL_RECV_STATE==3) { 
-     var = min(var,32); 
-     Sleep(250); 
-     } else if (HTS_STAT.HTS_TOTAL_RECV_STATE==2) { 
-     var = min(var,256); 
-     Sleep(100); 
-     } else { 
-     var/=2; 
-     if (var<=0) var=1; 
-     Sleep(50); 
-     } 
-     }
-   */
-  return 0;
-}
-#endif
-
 // Lecture dans buff de size octets au maximum en utilisant la socket r (structure htsblk)
 // returns: 
 // >0 : data received
@@ -4915,7 +4835,6 @@ int HTS_TOTAL_RECV_CHECK(int var) {
 int hts_read(htsblk * r, char *buff, int size) {
   int retour;
 
-  //  return read(soc,buff,size);
   if (r->is_file) {
 #if HTS_WIDE_DEBUG
     DEBUG_W("read(%p, %d, %d)\n" _(void *)buff _(int) size _(int) r->fp);
@@ -4933,7 +4852,6 @@ int hts_read(htsblk * r, char *buff, int size) {
     if (r->soc == INVALID_SOCKET)
       printf("!!WIDE_DEBUG ERROR, soc==INVALID hts_read\n");
 #endif
-    //HTS_TOTAL_RECV_CHECK(size);         // Diminuer au besoin si trop de données reçues
 #if HTS_USEOPENSSL
     if (r->ssl) {
       retour = SSL_read(r->ssl_con, buff, size);
@@ -5368,18 +5286,13 @@ SOCaddr* hts_dns_resolve(httrackp * opt, const char *_iadr, SOCaddr *const addr)
 
 // --- Tracage des mallocs() ---
 #ifdef HTS_TRACE_MALLOC
-//#define htsLocker(A, N) htsLocker(A, N)
 #define htsLocker(A, N) do {} while(0)
 static mlink trmalloc = { NULL, 0, 0, NULL };
 
 static int trmalloc_id = 0;
 static htsmutex *mallocMutex = NULL;
-static void hts_meminit(void) {
-  //if (mallocMutex == NULL) {
-  //  mallocMutex = calloc(sizeof(*mallocMutex), 1);
-  //  htsLocker(mallocMutex, -999);
-  //}
-}
+
+static void hts_meminit(void) {}
 void *hts_malloc(size_t len) {
   void *adr;
 
@@ -5463,7 +5376,6 @@ void hts_free(void *adr) {
               htsboundary);
       lnk->next = lnk->next->next;
       free((void *) blk_free);
-      //blk_free->id=-1;
       free((char *) adr - bsize);
       htsLocker(mallocMutex, 0);
       return;
@@ -5532,7 +5444,6 @@ mlink *hts_find(char *adr) {
 
     if (depl < 0)
       depl = -depl;
-    //assertf(depl < 512000);   /* near the stack frame.. doesn't look like malloc but stack variable */
     return NULL;
   }
 }
@@ -5588,8 +5499,7 @@ int ftp_available(void) {
 }
 #else
 int ftp_available(void) {
-  return 1;                     // ok!
-  //return 0;   // SOUS UNIX, PROBLEMESs
+  return 1; // ok!
 }
 #endif
 
@@ -5803,7 +5713,6 @@ HTSEXT_API int hts_init(void) {
       assertf("OpenSSL version seems vulnerable to heartbleed bug (CVE-2014-0160)" == NULL);
     }
 
-    // OpenSSL_add_all_algorithms();
     openssl_ctx = SSL_CTX_new(method);
     if (!openssl_ctx) {
       fprintf(stderr, "fatal: unable to initialize TLS: SSL_CTX_new()\n");
@@ -6017,15 +5926,7 @@ int multipleStringMatch(const char *s, const char *match) {
 }
 
 HTSEXT_API httrackp *hts_create_opt(void) {
-#if ( defined(_WIN32) || defined(__ANDROID__) )
-  static const char *defaultModules[] = {
-    "htsswf", "htsjava", "httrack-plugin", NULL
-  };
-#else
-  static const char *defaultModules[] = {
-    "libhtsswf.so.1", "libhtsjava.so.2", "httrack-plugin", NULL
-  };
-#endif
+  static const char *defaultModules[] = {"httrack-plugin", NULL};
   httrackp *opt = malloc(sizeof(httrackp));
 
   /* default options */
@@ -6108,6 +6009,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
   StringCopy(opt->strip_query, "");
   StringCopy(opt->cookies_file, "");
+  StringCopy(opt->why_url, "");
   opt->pause_min_ms = 0;
   opt->pause_max_ms = 0;
   opt->ftp_proxy = HTS_TRUE;
@@ -6123,7 +6025,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->log = stdout;
   opt->errlog = stderr;
   opt->flush = HTS_TRUE;
-  // opt->aff_progress=0;
   opt->keyboard = HTS_FALSE;
   //
   StringCopy(opt->path_html, "");
@@ -6133,8 +6034,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   //
   opt->maxlink = 100000;        // 100,000 liens max par défaut
   opt->maxfilter = 200;         // 200 filtres max par défaut
-  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT --
-  //opt->maxcache_anticipate=256;  // maximum de liens à anticiper
+  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON
+                                // PARAMETRABLE POUR L'INSTANT --
   opt->maxtime = -1;            // temps max en secondes
   opt->maxrate = 100000;        // taux maxi
   opt->maxconn = 5.0;           // nombre connexions/s
@@ -6256,6 +6157,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
     StringFree(opt->mod_blacklist);
     StringFree(opt->strip_query);
     StringFree(opt->cookies_file);
+    StringFree(opt->why_url);
 
     StringFree(opt->path_html);
     StringFree(opt->path_html_utf8);
@@ -6287,11 +6189,9 @@ const hts_stat_struct* hts_get_stats(httrackp * opt) {
 }
 
 // defaut wrappers
-static void __cdecl htsdefault_init(t_hts_callbackarg * carg) {
-}
-static void __cdecl htsdefault_uninit(t_hts_callbackarg * carg) {
-  // hts_freevar();
-}
+static void __cdecl htsdefault_init(t_hts_callbackarg *carg) {}
+
+static void __cdecl htsdefault_uninit(t_hts_callbackarg *carg) {}
 static int __cdecl htsdefault_start(t_hts_callbackarg * carg, httrackp * opt) {
   return 1;
 }

src/htslib.h

@@ -171,7 +171,6 @@ HTSEXT_API const char* hts_version(void);
 // fonctions unix/winsock
 int hts_read(htsblk * r, char *buff, int size);
 
-//int HTS_TOTAL_RECV_CHECK(int var);
 LLint check_downloadable_bytes(int rate);
 
 HTSEXT_API int hts_uninit_module(void);
@@ -185,12 +184,11 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
                   const char *referer_adr, const char *referer_fil,
                   htsblk * retour);
 /* Build the request "Cookie:" header line for stored cookies matching
-   domain/path into dst (NUL-terminated). Exposed for the -#Q self-test;
-   wraps the same logic http_sendhead() uses. Returns cookies emitted. */
-int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
-                                const char *path, char *dst, size_t dst_size);
+   domain/path into dst (NUL-terminated), wrapping the logic http_sendhead()
+   uses. Returns cookies emitted. */
+int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
+                       char *dst, size_t dst_size);
 
-//int newhttp(char* iadr,char* err=NULL);
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
               int waitconnect);
 /* Like newhttp(), but connect to the addr_index-th resolved address of the host

src/htsmodules.c

@@ -55,12 +55,6 @@ extern int fspc(httrackp * opt, FILE * fp, const char *type);
 
 /* >>> Put all modules variables here */
 
-#if 0
-t_gzopen gzopen = NULL;
-t_gzread gzread = NULL;
-t_gzclose gzclose = NULL;
-#endif
-
 int V6_is_available = HTS_INET6;
 
 static char WHAT_is_available[64] = "";
@@ -112,18 +106,6 @@ int hts_parse_externals(htsmoduleStruct * str) {
   return -1;
 }
 
-//static void addCallback(htscallbacks* chain, void* moduleHandle, htscallbacksfncptr exitFnc) {
-//  while(chain->next != NULL) {
-//    chain = chain->next;
-//  }
-//  chain->next = calloct(1, sizeof(htscallbacks));
-//  assertf(chain->next != NULL);
-//  chain = chain->next;
-//  memset(chain, 0, sizeof(*chain));
-//  chain->exitFnc = exitFnc;
-//  chain->moduleHandle = moduleHandle;
-//}
-
 void clearCallbacks(htscallbacks * chain_);
 void clearCallbacks(htscallbacks * chain_) {
   htscallbacks *chain;

src/htsname.c

@@ -41,6 +41,10 @@ Please visit our Website: http://www.httrack.com
 #include "htstools.h"
 #include "htscharset.h"
 #include "htsencoding.h"
+#include "htssniff.h"
+#if HTS_USEZLIB
+#include "htszlib.h"
+#endif
 #include <ctype.h>
 
 #define ADD_STANDARD_PATH \
@@ -70,31 +74,6 @@ static const char *hts_tbdev[] = {
   ""
 };
 
-#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET() do { \
-  int prev = opt->state._hts_in_html_parsing; \
-  while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
-    opt->state. _hts_in_html_parsing = 6; \
-    /* Wait .. */ \
-    back_wait(sback,opt,cache,0); \
-    /* Transfer rate */ \
-    engine_stats(); \
-    /* Refresh various stats */ \
-    HTS_STAT.stat_nsocket=back_nsoc(sback); \
-    HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
-    HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
-    HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
-    HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
-    HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
-    /* Check */ \
-    { \
-      if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,-1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
-        return -1; \
-      } \
-    } \
-  } \
-  opt->state._hts_in_html_parsing = prev; \
-} while(0)
-
 /* Strip all // */
 static void cleanDoubleSlash(char *s) {
   int i, j;
@@ -138,37 +117,191 @@ static void cleanEndingSpaceOrDot(char *s) {
   }
 }
 
-/* Should the wire Content-Type override the URL's own extension when naming the
-   saved file? True when the type is patchable (may_unknown2) and either the URL
-   extension implies no specific type or the server declared a disagreeing one.
-   A URL extension mapping to a specific non-HTML type is kept only when the
-   server declared NO type (the HTS_UNKNOWN_MIME sentinel; the #267 mangle
-   guard): a typeless .png stays .png, but a .pdf explicitly served as text/html
-   is named .html. The sentinel rides the cache, so updates stay consistent. */
-static int wire_patches_ext(httrackp *opt, const char *wiremime,
-                            const char *file) {
-  char urlmime[256];
+/* Wire Content-Type vs URL extension: a patchable wire type wins over an
+   unspecific ext, the HTS_UNKNOWN_MIME sentinel keeps a specific non-HTML ext
+   (#267 guard), a declared disagreement is CONTESTED (sniffed below). */
+typedef enum wire_verdict {
+  WIRE_KEEPS_EXT,
+  WIRE_WINS,
+  WIRE_CONTESTED
+} wire_verdict;
 
+static wire_verdict wire_ext_verdict(httrackp *opt, const char *wiremime,
+                                     const char *file, char *urlmime,
+                                     size_t urlmime_size) {
   if (may_unknown2(opt, wiremime, file))
-    return 0; /* type kept verbatim (keep-list / bogus-multiple) */
+    return WIRE_KEEPS_EXT; /* type kept verbatim (keep-list / bogus-multiple) */
   urlmime[0] = '\0';
   /* type implied by the URL extension, only when confidently known (flag 0) */
-  if (!get_httptype_sized(opt, urlmime, sizeof(urlmime), file, 0))
-    return 1; /* URL ext implies no known type: trust the wire type */
+  if (!get_httptype_sized(opt, urlmime, urlmime_size, file, 0))
+    return WIRE_WINS; /* URL ext implies no known type */
   if (strfield2(wiremime, urlmime))
-    return 0; /* wire agrees with the ext: keep it (no .htm->.html churn) */
-  /* wire disagrees with a specific non-HTML URL ext. Keep the ext only when
-     the server declared no type (the sentinel); an explicitly declared type,
-     even text/html, is trusted, so a binary-looking URL that really serves
-     HTML (login/error interstitial, soft-404) is named .html. */
+    return WIRE_KEEPS_EXT; /* agreement (no .htm->.html churn) */
   if (!is_hypertext_mime(opt, urlmime, file) &&
       strfield2(wiremime, HTS_UNKNOWN_MIME))
+    return WIRE_KEEPS_EXT; /* no declared type */
+  return WIRE_CONTESTED;
+}
+
+/* Optional evidence for a contested wire-vs-ext verdict. */
+typedef struct sniff_src {
+  struct_back *sback;       /* live backing (looked up by adr/fil) */
+  const lien_back *headers; /* snapshot: r.adr, else the url_sav file */
+  const char *adr, *fil;
+  const char *prev_save; /* previous run's save name (cache X-Save) */
+} sniff_src;
+
+#if HTS_USEZLIB
+/* Inflate the head of a gzip/zlib stream; 0 when undecodable. */
+static size_t sniff_inflate_head(const void *in, size_t in_len, void *out,
+                                 size_t out_len) {
+  z_stream zs;
+  size_t n = 0;
+  int err;
+
+  memset(&zs, 0, sizeof(zs));
+  if (inflateInit2(&zs, 47) != Z_OK) /* 47: gzip or zlib, autodetected */
     return 0;
+  zs.next_in = (const Bytef *) in;
+  zs.avail_in = (uInt) in_len;
+  zs.next_out = (Bytef *) out;
+  zs.avail_out = (uInt) out_len;
+  err = inflate(&zs, Z_SYNC_FLUSH);
+  if (err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR)
+    n = out_len - zs.avail_out;
+  inflateEnd(&zs);
+  return n;
+}
+#endif
+
+static size_t sniff_read_head(const char *path, void *buf, size_t len) {
+  char catbuff[CATBUFF_SIZE];
+  FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "rb");
+  size_t n = 0;
+
+  if (fp != NULL) {
+    n = fread(buf, 1, len, fp);
+    fclose(fp);
+  }
+  return n;
+}
+
+/* Body head of one slot: memory, else its flushed on-disk file (url_sav, or
+   tmpfile for a compressed stream); inflated so the sniff sees the final body.
+ */
+static size_t sniff_slot_head(const lien_back *slot, void *buf, size_t len) {
+  const htsblk *const r = &slot->r;
+  size_t n = 0;
+
+  if (r->adr != NULL && r->size > 0) {
+    n = (size_t) r->size < len ? (size_t) r->size : len;
+    memcpy(buf, r->adr, n);
+  } else {
+    if (r->out != NULL)
+      fflush(r->out);
+    if (slot->url_sav[0] != '\0')
+      n = sniff_read_head(slot->url_sav, buf, len);
+    if (n == 0 && slot->tmpfile != NULL && slot->tmpfile[0] != '\0')
+      n = sniff_read_head(slot->tmpfile, buf, len);
+  }
+  if (n > 0 && r->compressed) {
+#if HTS_USEZLIB
+    unsigned char raw[HTS_SNIFF_LEN];
+
+    if (n > sizeof(raw))
+      n = sizeof(raw);
+    memcpy(raw, buf, n);
+    n = sniff_inflate_head(raw, n, buf, len);
+#else
+    n = 0;
+#endif
+  }
+  return n;
+}
+
+/* Up to len leading body bytes; 0 when unavailable, and always in
+   non-delayed mode (its HEAD-probe first run couldn't sniff either). */
+static size_t sniff_body_head(httrackp *opt, const sniff_src *src, void *buf,
+                              size_t len) {
+  size_t n = 0;
+
+  if (src == NULL || opt->savename_delayed == HTS_SAVENAME_DELAYED_NONE)
+    return 0;
+  /* live backing slot: a snapshot (back_copy_static) loses r.adr/r.out */
+  if (src->sback != NULL && src->adr != NULL && src->fil != NULL) {
+    const int b = back_index(opt, src->sback, src->adr, src->fil, NULL);
+
+    if (b >= 0)
+      n = sniff_slot_head(&src->sback->lnk[b], buf, len);
+  }
+  if (n == 0 && src->headers != NULL)
+    n = sniff_slot_head(src->headers, buf, len);
+  return n;
+}
+
+/* Contested verdicts: magic proving the URL ext keeps it, else wire wins. */
+static int wire_patches_ext(httrackp *opt, const sniff_src *src,
+                            const char *wiremime, const char *file) {
+  char urlmime[256];
+
+  switch (wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime))) {
+  case WIRE_KEEPS_EXT:
+    return 0;
+  case WIRE_WINS:
+    return 1;
+  case WIRE_CONTESTED:
+    break;
+  }
+  if (src != NULL) {
+    if (hts_sniff_mime_known(urlmime)) {
+      unsigned char head[HTS_SNIFF_LEN];
+      const size_t n = sniff_body_head(opt, src, head, sizeof(head));
+
+      if (n > 0)
+        return hts_sniff_mime_consistent(head, n, urlmime) ? 0 : 1;
+    }
+    /* no bytes: reproduce the previous run's verdict (cached X-Save name) */
+    if (src->prev_save != NULL && src->prev_save[0] != '\0') {
+      char prevmime[256];
+
+      prevmime[0] = '\0';
+      if (get_httptype_sized(opt, prevmime, sizeof(prevmime), src->prev_save,
+                             0) &&
+          strfield2(prevmime, urlmime))
+        return 0;
+    }
+  }
   return 1;
 }
 
-// forme le nom du fichier à sauver (save) à partir de fil et adr
-// système intelligent, qui renomme en cas de besoin (exemple: deux INDEX.HTML et index.html)
+int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime,
+                         const char *file) {
+  char urlmime[256];
+
+  return wiremime != NULL && strnotempty(wiremime) &&
+         wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime)) ==
+             WIRE_CONTESTED &&
+         hts_sniff_mime_known(urlmime);
+}
+
+/* Wire-metadata name change: a Content-Disposition filename wins (returns 2),
+   else the declared type's ext when wire_patches_ext() allows (returns 1),
+   else 0. ext receives the new extension or replacement filename. */
+static int resolve_extension(httrackp *opt, const sniff_src *src,
+                             const char *cdispo, const char *contenttype,
+                             const char *fil, char *ext, size_t ext_size) {
+  if (strnotempty(cdispo)) {
+    strlcpybuff(ext, cdispo, ext_size);
+    return 2;
+  }
+  if (wire_patches_ext(opt, src, contenttype, fil) &&
+      give_mimext(ext, ext_size, contenttype))
+    return 1;
+  return 0;
+}
+
+// Build the local save name (save) from adr/fil; renames on collision
+// (e.g. INDEX.HTML vs index.html).
 int url_savename(lien_adrfilsave *const afs,
                  lien_adrfil *const former,
                  const char *referer_adr, const char *referer_fil, 
@@ -405,45 +538,30 @@ int url_savename(lien_adrfilsave *const afs,
 
   // si option check_type activée
   if (is_html < 0 && opt->check_type && !ext_chg) {
-    int ishtest = 0;
-
     if (protocol != PROTOCOL_FILE
         && protocol != PROTOCOL_FTP
       ) {
       // tester type avec requète HEAD si on ne connait pas le type du fichier
       if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/')))   // slash doit être html?
         if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
-            (ishtest = ishtml(opt, fil)) <
-                0) { // unsure whether it's html or a file
+            ishtml(opt, fil) < 0) { // unsure whether it's html or a file
           // lire dans le cache
-          htsblk r = cache_read_including_broken(opt, cache, adr, fil); // test uniquement
+          char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
+          htsblk r;
 
-          if (r.statuscode != -1) {     // pas d'erreur de lecture cache
-            char s[32];
+          previous_save[0] = '\0';
+          r = cache_read_including_broken(opt, cache, adr, fil,
+                                          previous_save); // test uniquement
 
-            s[0] = '\0';
+          if (r.statuscode != -1) { // cache entry read OK
             hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s",
                           adr_complete, fil_complete);
             if (!HTTP_IS_REDIRECT(r.statuscode)) {
-              if (strnotempty(r.cdispo)) {        /* filename given */
-                ext_chg = 2;      /* change filename */
-                strcpybuff(ext, r.cdispo);
-              } else if (wire_patches_ext(opt, r.contenttype, fil)) {
-                if (give_mimext(s, sizeof(s),
-                                r.contenttype)) { // recognized extension
-                  ext_chg = 1;
-                  strcpybuff(ext, s);
-                }
-              }
+              const sniff_src src = {sback, NULL, adr, fil, previous_save};
+
+              ext_chg = resolve_extension(opt, &src, r.cdispo, r.contenttype,
+                                          fil, ext, sizeof(ext));
             }
-#ifdef DEFAULT_BIN_EXT
-            // no extension and potentially bogus
-            else if (ishtest == -2) {
-              ext_chg = 1;
-              strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-            }
-#endif
-            //
           } else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD &&
                      is_userknowntype(opt, fil)) { /* PATCH BY BRIAN SCHRÖDER.
                               Lookup mimetype not only by extension,
@@ -467,22 +585,13 @@ int url_savename(lien_adrfilsave *const afs,
           // fail later
           else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_NONE &&
                    !opt->state.stop) {
-            // Check if the file is ready in backing. We basically take the same logic as later.
-            // FIXME: we should cleanup and factorize this unholy mess
+            // Check if the file is ready in backing.
             if (headers != NULL && headers->status >= 0 && !is_redirect) {
-              if (strnotempty(headers->r.cdispo)) {        /* filename given */
-                ext_chg = 2;      /* change filename */
-                strcpybuff(ext, headers->r.cdispo);
-              } else if (wire_patches_ext(opt, headers->r.contenttype,
-                                          headers->url_fil)) {
-                char s[16];
-                if (give_mimext(
-                        s, sizeof(s),
-                        headers->r.contenttype)) { // recognized extension
-                  ext_chg = 1;
-                  strcpybuff(ext, s);
-                }
-              }
+              const sniff_src src = {sback, headers, adr, fil, NULL};
+
+              ext_chg = resolve_extension(opt, &src, headers->r.cdispo,
+                                          headers->r.contenttype,
+                                          headers->url_fil, ext, sizeof(ext));
             }
             else if (mime_type != NULL) {
               ext[0] = '\0';
@@ -500,13 +609,6 @@ int url_savename(lien_adrfilsave *const afs,
                   if (!may_unknown2(opt, mime_type, fil)) {
                     ext_chg = 1;
                   }
-#ifdef DEFAULT_BIN_EXT
-                  // no extension and potentially bogus
-                  else if (ishtml(opt, fil) == -2) {
-                    ext_chg = 1;
-                    strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-                  }
-#endif
                 } else {
                   ext_chg = 0;
                 }
@@ -525,11 +627,10 @@ int url_savename(lien_adrfilsave *const afs,
             int has_been_moved = 0;
             lien_adrfil current;
 
-            /* Ensure we don't use too many sockets by using a "testing" one
-               If we have only 1 simultaneous connection authorized, wait for pending download
-               Wait for an available slot 
+            /* Wait for an available test slot, honoring the connection limits
              */
-            URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
+            if (!hts_wait_available_socket(sback, opt, cache, ptr))
+              return -1;
 
             /* Rock'in */
             current.adr[0] = current.fil[0] = '\0';
@@ -559,24 +660,11 @@ int url_savename(lien_adrfilsave *const afs,
                   if (ptr >= 0) {
                     back_fillmax(sback, opt, cache, ptr, numero_passe);
                   }
-                  // on est obligé d'appeler le shell pour le refresh..
-                  // Transfer rate
-                  engine_stats();
-
-                  // Refresh various stats
-                  HTS_STAT.stat_nsocket = back_nsoc(sback);
-                  HTS_STAT.stat_errors = fspc(opt, NULL, "error");
-                  HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
-                  HTS_STAT.stat_infos = fspc(opt, NULL, "info");
-                  HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
-                  HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
-
-                  if (!RUN_CALLBACK7
-                      (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
-                       (int) (time_local() - HTS_STAT.stat_timestart),
-                       &HTS_STAT)) {
+                  if (!hts_loop_tick(sback, opt, b, ptr)) {
                     return -1;
-                  } else if (opt->state._hts_cancel || !back_checkmirror(opt)) {        // cancel 2 ou 1 (cancel parsing)
+                  } else if (opt->state._hts_cancel ||
+                             !back_checkmirror(
+                                 opt)) { // cancel level 2 or 1 (cancel parsing)
                     back_delete(opt, cache, sback, b);  // cancel test
                     stop_looping = 1;
                   }
@@ -641,8 +729,9 @@ int url_savename(lien_adrfilsave *const afs,
                                                 "Loop with HEAD request (during prefetch) at %s%s",
                                                 current.adr, current.fil);
                                 }
-                                // Ajouter
-                                URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
+                                if (!hts_wait_available_socket(sback, opt,
+                                                               cache, ptr))
+                                  return -1;
                                 if (back_add(sback, opt, cache, moved.adr, moved.fil, methode, referer_adr, referer_fil, 1) != -1) {        // OK
                                   hts_log_print(opt, LOG_DEBUG,
                                                 "(during prefetch) %s (%d) to link %s at %s%s",
@@ -696,30 +785,10 @@ int url_savename(lien_adrfilsave *const afs,
                     // libérer emplacement backing
                   }
 
-                  {             // pas d'erreur, changer type?
-                    char s[16];
-
-                    s[0] = '\0';
-                    if (strnotempty(back[b].r.cdispo)) {        /* filename given */
-                      ext_chg = 2;      /* change filename */
-                      strcpybuff(ext, back[b].r.cdispo);
-                    } else if (wire_patches_ext(opt, back[b].r.contenttype,
-                                                back[b].url_fil)) {
-                      if (give_mimext(
-                              s, sizeof(s),
-                              back[b].r.contenttype)) { // recognized extension
-                        ext_chg = 1;
-                        strcpybuff(ext, s);
-                      }
-                    }
-#ifdef DEFAULT_BIN_EXT
-                    // no extension and potentially bogus
-                    else if (ishtest == -2) {
-                      ext_chg = 1;
-                      strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-                    }
-#endif
-                  }
+                  // no error: change the type?
+                  ext_chg = resolve_extension(
+                      opt, NULL, back[b].r.cdispo, back[b].r.contenttype,
+                      back[b].url_fil, ext, sizeof(ext));
                 }
                 // FIN Si non déplacé, forcer type?
 
@@ -1263,8 +1332,7 @@ int url_savename(lien_adrfilsave *const afs,
 
     while((a > afs->save) && (*a != '.') && (*a != '/'))
       a--;
-    if (*a != '.') {            // agh pas de point
-      //strcatbuff(save,".none");                 // a éviter
+    if (*a != '.') {                         // agh pas de point
       strcatbuff(afs->save, ".html");        // préférable!
       hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",
                     adr_complete, fil_complete, afs->save);
@@ -1402,17 +1470,6 @@ int url_savename(lien_adrfilsave *const afs,
   fil_simplifie(afs->save);
 
   /* convert name to UTF-8 ? Note: already done while parsing. */
-  //if (charset != NULL && charset[0] != '\0') {
-  //  char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
-
-  //  if (s != NULL) {
-  //    hts_log_print(opt, LOG_DEBUG,
-  //                  "engine: save-name: charset conversion from '%s' to '%s' using charset '%s'",
-  //                  save, s, charset);
-  //    strcpybuff(save, s);
-  //    free(s);
-  //  }
-  //}
 
   /* callback */
   RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr,
@@ -1560,10 +1617,6 @@ int url_savename(lien_adrfilsave *const afs,
         int sameAdr = (strfield2(heap(i)->adr, normadr) != 0);
         int sameFil;
 
-        // NO - URL hack is only for stripping // and www.
-        //if (opt->urlhack != 0)
-        //  sameFil = ( strfield2(heap(i)->fil, normfil) != 0);
-        //else
         sameFil = (strcmp(heap(i)->fil, normfil) == 0);
         if (sameAdr && sameFil) {       // ok c'est le même lien, adresse déja définie
           /* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */
@@ -1629,17 +1682,13 @@ int url_savename(lien_adrfilsave *const afs,
 
           strcpybuff(afs->save, tempo);
 
-          //printf("switched: %s\n",save);
-
         }                       // if
       }
 #if DEBUG_SAVENAME
       printf("\nEnd search, %s\n", fil_complete);
 #endif
-    } while(!nom_ok);
-
+    } while (!nom_ok);
   }
-  //printf("'%s' %s %s\n",save,adr,fil);
 
   return 0;
 }

src/htsname.h

@@ -100,6 +100,8 @@ void standard_name(char *b, size_t bsize, const char *dot_pos,
                    const char *nom_pos, const char *fil_complete,
                    int short_ver);
 void url_savename_addstr(char *d, const char *s);
+/* Contested wire-vs-ext verdict that a body sniff could settle (htssniff.h). */
+int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime, const char *file);
 char *url_md5(char *digest_buffer, const char *fil_complete);
 void url_savename_refname(const char *adr, const char *fil, char *filename);
 char *url_savename_refname_fullpath(httrackp * opt, const char *adr,

src/htsopt.h

@@ -105,7 +105,6 @@ typedef struct htsfilters htsfilters;
 struct htsfilters {
   char ***filters; /**< pointer to the +/-pattern filter array */
   int *filptr;     /**< pointer to the current filter count */
-  // int*    filter_max;
 };
 
 /* User callbacks chain */
@@ -440,7 +439,6 @@ struct httrackp {
   float maxconn;          /**< max connections per second */
   int waittime;           /**< scheduled start time (wall-clock seconds) */
   hts_cachemode cache;    /**< cache generation mode */
-  // int aff_progress;     // progress bar
   hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
   t_proxy proxy;     /**< proxy configuration */
   hts_savename_83
@@ -492,7 +490,6 @@ struct httrackp {
   hts_verbosedisplay verbosedisplay; /**< animated text progress display */
   String footer; /**< footer/info line injected into pages */
   int maxcache;  /**< in-memory cache backing limit (bytes) */
-  // int maxcache_anticipate; // maximum links to anticipate (upper bound)
   hts_boolean ftp_proxy;    /**< use the HTTP proxy for FTP too */
   String filelist;          /**< file listing URLs to include */
   String urllist;           /**< file listing filters to include */
@@ -539,6 +536,8 @@ struct httrackp {
                                  (--cookies-file) */
   int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
   int pause_max_ms; /**< inter-file pause upper bound, ms */
+  String why_url;   /**< URL to diagnose (--why): print the deciding filter rule
+                         and exit without crawling */
 };
 
 /* Running statistics for a mirror. */
@@ -549,8 +548,6 @@ typedef struct hts_stat_struct hts_stat_struct;
 struct hts_stat_struct {
   LLint HTS_TOTAL_RECV; /**< total bytes received from the network */
   LLint stat_bytes;     /**< total bytes written to disk */
-  // int HTS_TOTAL_RECV_STATE;  // status: 0 ok 1: slow down a little 2: slow
-  // down 3: a lot
   TStamp stat_timestart; /**< mirror start time */
   //
   LLint total_packed;    /**< compressed bytes received (on the wire) */
@@ -682,7 +679,6 @@ struct lien_url {
   int depth;        /**< remaining allowed depth; >0 strong, 0 weak */
   int pass2;        /**< second-pass marker; -1 means handled in background */
   char link_import; /**< imported after a move; skip the usual up/down rules */
-  // int moved;          // pointer to moved
   int retry;    /**< remaining retries */
   int testmode; /**< test only: send just a HEAD */
 };
@@ -727,9 +723,6 @@ struct lien_back {
   LLint chunk_size; /**< size of the chunk being loaded */
   LLint chunk_blocksize; /**< data size declared by the chunk */
   LLint compressed_size; /**< compressed size (stats only) */
-  //
-  // int links_index;        // to access liens[links_index]
-  //
   char info[256]; /**< status text, e.g. for FTP */
   int stop_ftp;   /**< stop flag for FTP */
   int finalized;  /**< finalized (memory optimization) */

src/htsparse.h

@@ -66,7 +66,6 @@ struct htsmoduleStructExtended {
   char ***filters_;
   robots_wizard *robots_;
   hash_struct *hash_;
-  //int *lien_max_;
 
   /* Base & codebase */
   char *base;
@@ -116,6 +115,19 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre);
 int hts_mirror_check_moved(htsmoduleStruct * str,
                            htsmoduleStructExtended * stre);
 
+/*
+  Non-zero if a redirect (cur_adr,cur_fil)->(moved_adr,moved_fil) saves to the
+  same local file, so it must be followed rather than turned into a
+  self-pointing "moved" stub (#159). Mirrors the savename: scheme+userinfo
+  stripped, www kept (www dedup is the crawl layer's job), path
+  slash/query-normalized per the URL-hack flags. Not hash_url_equals: that keys
+  on the dedup hash, which folds www and never collapses http<->https.
+*/
+hts_boolean hts_redirect_same_savefile(httrackp *opt, const char *cur_adr,
+                                       const char *cur_fil,
+                                       const char *moved_adr,
+                                       const char *moved_fil);
+
 /*
   Process user intercations: pause, add link, delete link..
 */
@@ -162,27 +174,4 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
   /* Apply changes */ \
   * str->ptr_ = ptr
 
-#define WAIT_FOR_AVAILABLE_SOCKET() do { \
-  int prev = opt->state._hts_in_html_parsing; \
-  while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
-    opt->state._hts_in_html_parsing = 6; \
-    /* Wait .. */ \
-    back_wait(sback,opt,cache,0); \
-    /* Transfer rate */ \
-    engine_stats(); \
-    /* Refresh various stats */ \
-    HTS_STAT.stat_nsocket=back_nsoc(sback); \
-    HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
-    HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
-    HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
-    HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
-    HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
-    /* Check */ \
-    if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count, -1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
-      return -1; \
-    } \
-  } \
-  opt->state._hts_in_html_parsing = prev; \
-} while(0)
-
 #endif

src/htsserver.c

@@ -265,7 +265,6 @@ T_SOC smallserver_init(int *port, char *adr) {
       SOCaddr_initport(server, *port);
       if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
         if (listen(soc, 10) >= 0) {
-          // SOCaddr_inetntoa(adr, 128, server2);
           strcpy(adr, h_loc);
         } else {
 #ifdef _WIN32
@@ -439,7 +438,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
         meth = 10;
       } else {
 #ifdef _DEBUG
-        // assert(FALSE);
 #endif
       }
       if (meth) {
@@ -1360,7 +1358,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
 
             StringCat(headers, error_hdr);
             StringCat(output, error);
-            //assert(file == NULL);
           }
         }
       } else {
@@ -1388,12 +1385,10 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                   != StringLength(output)))
         ) {
 #ifdef _DEBUG
-        //assert(FALSE);
 #endif
       }
     } else {
 #ifdef _DEBUG
-      // assert(FALSE);
 #endif
     }
 
@@ -1540,8 +1535,7 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
             } while(strnotempty(test));
           }
 
-          if (!strnotempty(test)) {     // éviter doublons
-            // conv_printf(key,key);
+          if (!strnotempty(test)) { // éviter doublons
             const size_t len = strlen(intkey);
             char *const buff = (char *) malloc(len + 1);
 
@@ -1632,8 +1626,6 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
                     intkey = "";
                 } else {
                   if (loops > 0) {
-                    //err_msg += intkey;
-                    //err_msg += " ";
                   }
                 }
               }
@@ -1740,13 +1732,7 @@ static void LANG_DELETE(void) {
 }
 
 // sélection de la langue
-static void LANG_INIT(const char *path) {
-  //CWinApp* pApp = AfxGetApp();
-  //if (pApp) {
-  /* pApp->GetProfileInt("Language","IntId",0); */
-  LANG_T(path, 0 /*pApp->GetProfileInt("Language","IntId",0) */ );
-  //}
-}
+static void LANG_INIT(const char *path) { LANG_T(path, 0); }
 
 static int LANG_T(const char *path, int l) {
   if (l >= 0) {