Replace single-letter -# self-tests with a named -#test=NAME registry

The hidden engine self-tests had accreted into a grab-bag of arbitrary
single-letter/-digit -# arms (-#0, -#A, -#W, ...) buried in the htscoremain.c
option switch, with no mnemonics and stale --help text. Collapse them into one
registry: -#test lists every test with a usage hint and one-line description,
and -#test=NAME [args] runs one.

The handlers and the two helpers they used (basic_selftests,
string_safety_selftests) move to a new htsselftest.c keyed by a
{name, args, desc, fn} table; htscoremain.c keeps only a small dispatch that
runs ahead of the no-URL usage gate, so a bare -#test (or an arg-less test like
copyopt/dns/cookies) no longer needs a dummy URL token to be reached. The
genuine debug knobs (-#L, -#C, -#R, -#h, ...) stay as letters in the switch;
only the unit self-tests, whose sole callers are tests/01_engine-*.test, are
renamed, so this is internal-only with no compatibility surface. Behavior is
preserved: each test prints the same result line and exit code, which the
existing assertions pin. Three now-unused includes (htscache_selftest.h,
htsdns_selftest.h, htsencoding.h) drop out of htscoremain.c.

Tests: the engine tests move to -#test=NAME; 01_engine-hashtable now asserts its
success line (not just exit code) so a misrouted registry row can't pass, and a
new 01_engine-selftest-dispatch covers the bare-list and unknown-name paths.

The --help/man "guru options" list now points at -#test instead of enumerating
a stale subset. The lone vestigial alias --debug-testfilters still resolves to
the removed -#0 (it was already non-functional: param1 supplies one argument,
-#0 required two); it is left untouched because editing that array forces
clang-format to reflow the whole untouched table.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -55,54 +55,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
-
-      - name: Print the test log on failure
-        if: failure()
-        run: cat tests/test-suite.log 2>/dev/null || true
-
-  # Reproduce the Debian buildds: they build in a minimal chroot with no
-  # python3, so the local-server tests must SKIP (exit 77), not fail. GitHub
-  # runners ship python3, so every other job hides this path; here we remove it
-  # before `make check`. This is the guard that would have caught the 3.49.10-1
-  # FTBFS (28_local-pause failed instead of skipping when python3 was absent).
-  buildd-no-python3:
-    name: build (no python3, Debian buildd)
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev
-
-      - name: Configure
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure
-
-      - name: Build
-        run: make -j"$(nproc)"
-
-      - name: Test without python3
-        run: |
-          set -euo pipefail
-          # Hide every python3* so `command -v python3` fails like it does in the
-          # buildd chroot; masking with /bin/false would still resolve.
-          sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
-            -exec mv {} {}.hidden \;
-          ! command -v python3
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -135,52 +88,13 @@ jobs:
       - name: Build
         run: make -j"$(sysctl -n hw.ncpu)"
 
-      - name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
-        # 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
-        # instantly like Linux; alias them onto lo0 so they don't stall to timeout.
-        run: |
-          set -euo pipefail
-          sudo ifconfig lo0 alias 127.0.0.2 up
-          sudo ifconfig lo0 alias 127.0.0.3 up
-
       - name: Test
-        run: |
-          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
-  # Runtime smoke of the WebHTTrack launcher on macOS: it carries a Darwin-only
-  # browser path (open -W) and nothing else exercises htsserver. Install into a
-  # temp prefix, then check webhttrack brings up htsserver and serves the UI.
-  webhttrack-macos:
-    name: webhttrack smoke (macOS arm64)
-    runs-on: macos-14
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          brew install autoconf automake libtool autoconf-archive
-
-      - name: Build and install into a temp prefix
-        run: |
-          set -euo pipefail
-          ssl="$(brew --prefix openssl@3)"
-          autoreconf -fi
-          ./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib" \
-            --prefix="$RUNNER_TEMP/inst"
-          make -j"$(sysctl -n hw.ncpu)"
-          make install
-
-      - name: Smoke-test webhttrack
-        run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
-
   # Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
   # -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
   # and bounds math can truncate or wrap in ways 64-bit never reveals (the axis
@@ -213,9 +127,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -270,100 +182,12 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
-  # MemorySanitizer catches reads of uninitialized memory (#143's stack-garbage
-  # size filter) that ASan/UBSan miss. It flags any byte an uninstrumented lib
-  # wrote, so the job stays in our own code: offline self-tests only, no openssl
-  # (--disable-https), no zlib cache tests, static (the runtime is not in .so's).
-  msan:
-    name: msan (MemorySanitizer, clang)
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential clang autoconf automake libtool autoconf-archive \
-            zlib1g-dev
-
-      - name: Configure (MSan, static, no https)
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure CC=clang \
-            CFLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
-            LDFLAGS="-fsanitize=memory" \
-            --disable-https --disable-shared --enable-static
-
-      - name: Build
-        run: make -j"$(nproc)"
-
-      - name: Test (offline self-tests under MSan)
-        env:
-          MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
-        run: |
-          set -euo pipefail
-          # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
-          # skipped here (uninstrumented libz floods MSan with false positives).
-          tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs" TESTS="$tests"
-
-      - name: Print the test log on failure
-        if: failure()
-        run: cat tests/test-suite.log 2>/dev/null || true
-
-  # libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
-  # parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
-  # (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
-  # catastrophic backtracking -- and pins the regression seeds for the bugs the
-  # fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
-  fuzz:
-    name: fuzz (libFuzzer corpus replay, clang)
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential clang autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev
-
-      - name: Configure (fuzzers, static)
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure CC=clang \
-            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
-            LDFLAGS="-fsanitize=address,undefined" \
-            --enable-fuzzers --disable-shared
-
-      - name: Build
-        run: make -j"$(nproc)"
-
-      - name: Replay corpora
-        env:
-          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
-          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: bash fuzz/run-fuzzers.sh fuzz check
-
   # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
   # the configuration users on minimal systems build, and one libssl is not even
   # installed here so configure cannot silently re-enable it. The matrix above
@@ -394,9 +218,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()

.github/workflows/codeql.yml

@@ -1,63 +0,0 @@
-# CodeQL static analysis (C). The security-extended suite covers the classes
-# this codebase actually fights: overflows, tainted-size allocs, format bugs.
-name: CodeQL
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-  schedule:
-    # Weekly re-scan of master so new/updated queries land without a push.
-    - cron: "17 4 * * 1"
-
-permissions:
-  contents: read
-
-concurrency:
-  group: codeql-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: analyze (c-cpp)
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: read
-      # Upload findings to the repo's code-scanning dashboard.
-      security-events: write
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev
-
-      - uses: github/codeql-action/init@v4
-        with:
-          languages: c-cpp
-          build-mode: manual
-          queries: security-extended
-          # fopen's umask-controlled 0666 is intended for mirror/cache/log
-          # output; the one credential file (cookies.txt) is kept 0600 on Unix.
-          config: |
-            query-filters:
-              - exclude:
-                  id: cpp/world-writable-file-creation
-
-      # Manual build: CodeQL traces the compiler, so build exactly what ships.
-      - name: Build
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure
-          make -j"$(nproc)"
-
-      - uses: github/codeql-action/analyze@v4
-        with:
-          category: "/language:c-cpp"

.gitignore

@@ -34,12 +34,8 @@ Makefile
 *.so.*
 *.a
 
-# make dist output; dist/ holds httrack-gh-release staging artifacts.
+# make dist output.
 /httrack-*.tar.gz
-/dist/
 
 # Editor / autotools backup files.
 *~
-
-# Python bytecode (tests/local-server.py).
-__pycache__/

AGENTS.md

@@ -6,19 +6,8 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 ## Build & test
 - Fresh clone first: `git submodule update --init src/coucal`
 - `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
-  automake, libtool), then `bash configure && make -j"$(nproc)" && make check
-  -j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
-  automake's parallel harness and each crawl test binds its own ephemeral-port
-  server, so `-j` never contends and a multi-minute serial run drops to
-  seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
-  automatically; only a test slower than the current longest raises the floor.
-  On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
-  spend much of their wall time asleep (server trickles, httrack self-pacing),
-  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
-  platform, macOS included: the test server raises its listen backlog
-  (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
-  `-c16` bigcrawl the way Python's default backlog of 5 did.
-  Or run `sh build.sh` to do bootstrap + configure + make in one shot.
+  automake, libtool), then `bash configure && make && make check`. Or run
+  `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

CONTRIBUTING.md

@@ -39,10 +39,6 @@ Welcome, and nothing to disclose. Two rules:
 
 The sign-off covers AI-assisted code too.
 
-## Translations
-
-Interface strings live in [`lang/`](lang/). See [lang/README.md](lang/README.md) for the file format and how to add or update a language.
-
 ## Bugs
 
 Open an issue with the version, OS, command used, and expected vs actual result.

Makefile.am

@@ -1,5 +1,5 @@
 
-SUBDIRS = src man m4 libtest templates lang html tests fuzz
+SUBDIRS = src man m4 libtest templates lang html tests
 
 ACLOCAL_AMFLAGS = -I m4
 

README

@@ -40,6 +40,7 @@ These are the principals limits of HTTrack for that moment. Note that we did not
 that would have solved them.
 
 - Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
+- Some java classes may not find some files on them (class included)
 - Cgi-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like -*cgi-bin*
 
 

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.9], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,11 +29,10 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
-# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
-# so it stays soname .so.3; bump revision.
-# (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:4:0"
+# 3:1:0: 3.49.9 changed code but not the exported interface vs 3.49.8 (same 164
+# symbols, no struct-layout change), so bump revision only. (3:0:0 was the htsblk
+# mime-buffer widening, an ABI break that moved the soname .so.2 -> .so.3.)
+VERSION_INFO="3:1:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 
@@ -92,8 +91,7 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
 AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
 AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
 AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
-AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
-	[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
+AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
 AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
 AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
 AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -101,16 +99,9 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
 AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
 AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
 
-# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
-# sanitizer builds, whose interceptors want the unfortified calls.
-case "$CFLAGS" in
-*-fsanitize=*) ;;
-*) AX_ADD_FORTIFY_SOURCE ;;
-esac
-
 # Force libc back into DT_NEEDED for libraries that reach it only through
-# libhttrack (the libtest callbacks), but only with a GNU-style linker; Apple
-# ld rejects these flags and links libSystem unconditionally.
+# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
+# linker; Apple ld rejects these flags and links libSystem unconditionally.
 AX_CHECK_LINK_FLAG([-Wl,--push-state,--no-as-needed,-lc,--pop-state],
 	[LIBC_FORCE_LINK="-Wl,--push-state,--no-as-needed,-lc,--pop-state"])
 AC_SUBST([LIBC_FORCE_LINK])
@@ -308,37 +299,6 @@ AC_ARG_ENABLE([online-unit-tests],
 	])
 AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
 
-## libFuzzer harnesses (fuzz/); requires clang
-AC_MSG_CHECKING(whether to build fuzzers)
-AC_ARG_ENABLE([fuzzers],
-	[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
-	[
-	case "${enableval}" in
-	no|yes)
-		fuzzers=$enableval
-		AC_MSG_RESULT($enableval)
-		;;
-	*)
-		AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
-		;;
-	esac
-	],
-	[
-	fuzzers=no
-	AC_MSG_RESULT(no)
-	])
-if test x"$fuzzers" = x"yes"; then
-	# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
-	AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
-		[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
-		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
-	# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
-	if test x"$enable_shared" != x"no"; then
-		AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
-	fi
-fi
-AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
-
 # Final output
 AC_CONFIG_FILES([
 Makefile
@@ -350,6 +310,5 @@ lang/Makefile
 html/Makefile
 libtest/Makefile
 tests/Makefile
-fuzz/Makefile
 ])
 AC_OUTPUT

debian/changelog

@@ -1,47 +1,3 @@
-httrack (3.49.12-1) unstable; urgency=medium
-
-  * New upstream release: security and crawl-correctness fixes (remote stack
-    overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
-    world-readable cookies.txt, filter-pattern denial of service) plus a new
-    --why filter diagnostic; full list in history.txt.
-  * Build with _FORTIFY_SOURCE and -fstack-protector-strong.
-
- -- Xavier Roche <xavier@debian.org>  Fri, 10 Jul 2026 21:11:09 +0200
-
-httrack (3.49.11-1) unstable; urgency=medium
-
-  * New upstream release: crawl correctness and security fixes (network-facing
-    buffer overflows, file-type detection, redirect handling) and modernized
-    web defaults; full list in history.txt.
-  * Add DEP-12 upstream metadata (#466).
-  * Bump debhelper compat to 14 (#466).
-  * Drop the redundant Priority field and update the NMU lintian override to
-    the current tag names (#466).
-
- -- Xavier Roche <xavier@debian.org>  Sun, 05 Jul 2026 00:03:18 +0200
-
-httrack (3.49.10-2) unstable; urgency=medium
-
-  * Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
-    absent (the local-server tests need python3, which the buildds lack). Add
-    patches/skip-local-pause-test-without-python3.patch to guard the test on
-    python3 up front, like its siblings, so it skips cleanly.
-
- -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 20:18:46 +0200
-
-httrack (3.49.10-1) unstable; urgency=medium
-
-  * New upstream release: new download-pacing and URL-handling options plus a
-    batch of crawl and robustness fixes (full list in history.txt).
-  * Rewrite debian/copyright in machine-readable DEP-5 format, crediting the
-    bundled minizip, md5 and coucal sources (#415).
-  * Lead the webhttrack browser dependency with chromium so httrack is not
-    dragged into the firefox-esr autoremoval cascade (#436).
-  * Override the embedded-library lint for the bundled minizip (#419).
-  * Bump Standards-Version to 4.7.4 (no changes required).
-
- -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 14:01:53 +0200
-
 httrack (3.49.9-1) unstable; urgency=medium
 
   * New upstream release: Content-Type and file-type detection fixes (trust a

debian/control

@@ -1,8 +1,9 @@
 Source: httrack
 Section: web
+Priority: optional
 Maintainer: Xavier Roche <roche@httrack.com>
-Standards-Version: 4.7.4
-Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
+Standards-Version: 4.7.0
+Build-Depends: debhelper-compat (= 13), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
 Rules-Requires-Root: no
 Homepage: http://www.httrack.com
 Vcs-Git: https://github.com/xroche/httrack.git
@@ -29,7 +30,7 @@ Description: Copy websites to your computer (Offline browser)
 Package: webhttrack
 Architecture: any
 Multi-Arch: foreign
-Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, chromium | firefox-esr | www-browser
+Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, firefox-esr | chromium | www-browser
 Replaces: webhttrack-common (<< 3.43.9-2)
 Breaks: webhttrack-common (<< 3.43.9-2)
 Suggests: httrack, httrack-doc

debian/copyright

@@ -7,8 +7,9 @@ Files: *
 Copyright: 1998-2026 Xavier Roche and other contributors
 License: GPL-3+
 Comment:
- htsbasenet.h links against the system OpenSSL library (originally by Eric
- Young); no OpenSSL/SSLeay code is bundled here.
+ The engine includes contributions from Yann Philippot (src/htsjava.c,
+ src/htsjava.h). htsbasenet.h links against the system OpenSSL library
+ (originally by Eric Young); no OpenSSL/SSLeay code is bundled here.
 
 Files: src/minizip/*
 Copyright: 1998-2010 Gilles Vollant

debian/libhttrack-dev.files

@@ -1,4 +1,5 @@
 usr/include/httrack
 usr/lib/*/libhttrack.{so}
+usr/lib/*/libhtsjava.{so}
 usr/share/httrack/libtest
 usr/lib/*/httrack/libtest

debian/libhttrack3.files

@@ -1,2 +1,3 @@
 usr/lib/*/libhttrack.so.3*
+usr/lib/*/libhtsjava.so.3*
 usr/share/httrack/templates

debian/source/lintian-overrides

@@ -1,6 +1,4 @@
-# Maintainer uploads sign the changelog as xavier@debian.org while the control
-# Maintainer is roche@httrack.com; lintian reads the address mismatch as an NMU.
-httrack source: no-nmu-in-changelog
+httrack source: changelog-should-mention-nmu
 httrack source: source-nmu-has-incorrect-version-number
 
 # The bundled HTML pages are the genuine upstream documentation taken from

debian/upstream/metadata

@@ -1,6 +0,0 @@
----
-Repository: https://github.com/xroche/httrack.git
-Repository-Browse: https://github.com/xroche/httrack
-Bug-Database: https://github.com/xroche/httrack/issues
-Bug-Submit: https://github.com/xroche/httrack/issues/new
-Contact: Xavier Roche <roche@httrack.com>

fuzz/Makefile.am

@@ -1,45 +0,0 @@
-# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
-if FUZZERS
-noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
-	fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx
-endif
-
-AM_CPPFLAGS = \
-	@DEFAULT_CFLAGS@ \
-	@THREADS_CFLAGS@ \
-	@V6_FLAG@ \
-	@LFS_FLAG@ \
-	-I$(top_srcdir)/src \
-	-I$(top_srcdir)/src/coucal
-
-# Static-link libhttrack.la: the internal symbols are hidden in the .so.
-AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
-LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
-
-fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
-fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
-fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
-fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
-fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
-fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
-fuzz_url_SOURCES = fuzz-url.c fuzz.h
-fuzz_header_SOURCES = fuzz-header.c fuzz.h
-fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
-
-# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
-EXTRA_DIST = README.md run-fuzzers.sh \
-	corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
-	corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
-	corpus/idna/idna.txt corpus/idna/unicode.txt \
-	corpus/idna/regress-multilabel-leak.txt \
-	corpus/entities/entities.txt \
-	corpus/unescape/percent.txt \
-	corpus/filters/filter.bin corpus/filters/filter-size.bin \
-	corpus/filters/regress-empty-subject-unique.bin \
-	corpus/filters/redos-star-classes.bin \
-	corpus/url/http-url.txt corpus/url/relative-path.txt \
-	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
-	corpus/header/full-response.txt corpus/header/redirect.txt \
-	corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
-	corpus/cachendx/regress-overadvance.bin \
-	corpus/cachendx/regress-truncated-entry.bin

fuzz/README.md

@@ -1,15 +0,0 @@
-# Fuzzing httrack
-
-libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
-
-```sh
-./bootstrap
-mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
-CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
-  LDFLAGS="-fsanitize=address,undefined" \
-  bash /path/to/httrack/configure --enable-fuzzers --disable-shared
-make
-bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60   # 60s per target
-```
-
-Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.

fuzz/corpus/cachendx/new-format.txt

@@ -1,7 +0,0 @@
-8
-CACHE-1.1
-28
-Mon, 01 Jan 2024 00:00:00 GMT
-www.example.com
-/index.html
-123

fuzz/corpus/cachendx/old-format.txt

@@ -1,5 +0,0 @@
-3
-1.0
-www.example.com
-/page
-5

fuzz/corpus/cachendx/regress-overadvance.bin

@@ -1,2 +0,0 @@
-32768
-CACHE-1.1

fuzz/corpus/cachendx/regress-truncated-entry.bin

@@ -1,5 +0,0 @@
-8
-CACHE-1.1
-1
-x
-www.example.com

fuzz/corpus/charset/latin1.txt

@@ -1 +0,0 @@
-
café naďve ¤

fuzz/corpus/charset/sjis.txt

@@ -1 +0,0 @@
-コンピュータ

fuzz/corpus/entities/entities.txt

@@ -1 +0,0 @@
-&<>A☃é&notarealentity;�

fuzz/corpus/filters/regress-empty-subject-unique.bin

@@ -1 +0,0 @@
-**((

fuzz/corpus/header/full-response.txt

@@ -1,10 +0,0 @@
-HTTP/1.1 200 OK
-Content-Type: text/html; charset=utf-8
-Content-Length: 1234
-Content-Encoding: gzip
-Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
-Etag: "abc"
-Location: http://example.com/x
-Set-Cookie: ID=42; path=/; domain=.example.com
-Content-Range: bytes 0-99/100
-Transfer-Encoding: chunked

fuzz/corpus/header/redirect.txt

@@ -1,3 +0,0 @@
-HTTP/1.0 301 Moved
-Location: /elsewhere
-Content-Disposition: attachment; filename="a.pdf"

fuzz/corpus/idna/idna.txt

@@ -1 +0,0 @@
-xn--bcher-kva.example.com

fuzz/corpus/idna/regress-multilabel-leak.txt

@@ -1 +0,0 @@
-büchev.例å­bücheple

fuzz/corpus/idna/unicode.txt

@@ -1 +0,0 @@
-bücher.例子.example

fuzz/corpus/meta/meta-charset.html

@@ -1 +0,0 @@
-<html><head><meta charset="utf-8"></head><body>x</body></html>

fuzz/corpus/meta/meta-content-first.html

@@ -1 +0,0 @@
-<meta content="text/html; charset=gb2312" http-equiv="content-type">

fuzz/corpus/meta/meta-http-equiv.html

@@ -1 +0,0 @@
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">

fuzz/corpus/meta/meta-truncated.html

@@ -1 +0,0 @@
-<meta charset='utf-8

fuzz/corpus/meta/meta-unquoted.html

@@ -1 +0,0 @@
-<meta   charset = utf-8 />

fuzz/corpus/unescape/percent.txt

@@ -1 +0,0 @@
-%41%zz%%20%c3%a9+%2e%2e%2f

fuzz/corpus/url/http-url.txt

@@ -1 +0,0 @@
-http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag

fuzz/corpus/url/regress-file-empty-path.txt

@@ -1 +0,0 @@
-file://

fuzz/corpus/url/regress-long-path-abort.txt

@@ -1 +0,0 @@
-ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../

fuzz/corpus/url/relative-path.txt

@@ -1 +0,0 @@
-ftp://ftp.example.com/pub/../file.txt

fuzz/fuzz-cachendx.c

@@ -1,65 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
-   walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
-   Mirrors the -#C cache-listing scan (htscoremain.c). */
-#include "fuzz.h"
-#include "htscache.h"
-#include "htslib.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  const char *const end = buf + size;
-  char firstline[256];
-  char *a = buf;
-
-  /* header: two length-prefixed fields (version, last-modified) */
-  a += cache_brstr(a, firstline, sizeof(firstline));
-  a += cache_brstr(a, firstline, sizeof(firstline));
-
-  /* body: newline-delimited host/file/position triples; the length-prefixed
-     scan must stay inside the buffer */
-  while (a != NULL && a < end) {
-    char BIGSTK line[HTS_URLMAXSIZE * 2];
-    char linepos[256];
-    int pos;
-
-    a = strchr(a + 1, '\n');
-    if (a == NULL)
-      break;
-    a++;
-    a += cache_binput(a, end, line, HTS_URLMAXSIZE);
-    a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
-    a += cache_binput(a, end, linepos, 200);
-    sscanf(linepos, "%d", &pos);
-    (void) pos;
-  }
-
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-charset.c

@@ -1,84 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
-   UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
-#include "fuzz.h"
-#include "htscharset.h"
-
-static const char *const charsets[] = {
-    "utf-8",    "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
-    "us-ascii", "shift_jis",  "euc-jp",     "iso-2022-jp", "gb2312",
-    "big5",     "euc-kr",     "koi8-r",     "utf-16",      "unknown-charset",
-};
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  const char *charset;
-  char *s;
-
-  if (size == 0)
-    return 0;
-  charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
-  data++, size--;
-  s = fuzz_strdup(data, size);
-
-  {
-    char *utf8 = hts_convertStringToUTF8(s, size, charset);
-    freet(utf8);
-  }
-  {
-    char *enc = hts_convertStringFromUTF8(s, size, charset);
-    freet(enc);
-  }
-  {
-    size_t nChars = 0;
-    hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
-
-    if (ucs != NULL) {
-      char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
-      freet(back);
-      freet(ucs);
-    }
-  }
-  {
-    size_t i = 0;
-
-    while (i < size) {
-      hts_UCS4 uc = 0;
-      const size_t nr = hts_readUTF8(s + i, size - i, &uc);
-      char out[8];
-
-      if (nr == 0)
-        break;
-      hts_writeUTF8(uc, out, sizeof(out));
-      i += nr;
-    }
-  }
-
-  freet(s);
-  return 0;
-}

fuzz/fuzz-entities.c

@@ -1,51 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
-   destination size, so truncation bounds get exercised too. */
-#include "fuzz.h"
-#include "htsencoding.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  static const size_t dsizes[] = {1, 2, 8, 64, 4096};
-  size_t dsize;
-  char *src, *dest;
-
-  if (size == 0)
-    return 0;
-  dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
-  data++, size--;
-  src = fuzz_strdup(data, size);
-  dest = malloct(dsize);
-
-  (void) hts_unescapeEntities(src, dest, dsize);
-  (void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
-
-  freet(dest);
-  freet(src);
-  return 0;
-}

fuzz/fuzz-filters.c

@@ -1,65 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
-   here). Input splits on the first NUL: pattern, then subject string. */
-#include "fuzz.h"
-#include "htsfilters.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  const char *joker = buf;
-  const uint8_t *sep = memchr(data, '\0', size);
-  /* subject in its own allocation so ASan bounds it apart from the pattern */
-  char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
-                          : fuzz_strdup(data + size, 0);
-
-  (void) strjoker(nom, joker, NULL, NULL);
-  {
-    LLint sz = (LLint) size;
-    int size_flag = 0;
-
-    (void) strjoker(nom, joker, &sz, &size_flag);
-  }
-  (void) strjokerfind(nom, joker);
-  {
-    char *filter = malloct(strlen(joker) + 2);
-    char *filters[1];
-    LLint sz = (LLint) size;
-    int size_flag = 0, depth = 0;
-
-    filter[0] = '-';
-    memcpy(filter + 1, joker, strlen(joker) + 1);
-    filters[0] = filter;
-    (void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
-    freet(filter);
-  }
-
-  freet(nom);
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-header.c

@@ -1,74 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
-   status line, treathead on each following header. Both consume raw bytes
-   off the wire and copy fields into fixed htsblk buffers; treathead also
-   mutates its line in place and drives cookie parsing. */
-#include "fuzz.h"
-#include "htslib.h"
-#include "htsbauth.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  htsblk r;
-  t_cookie *cookie = calloct(1, sizeof(*cookie));
-  char *line = malloct(size + 1);
-  char *p = buf;
-  int first = 1;
-
-  memset(&r, 0, sizeof(r));
-  cookie->max_len = (int) sizeof(cookie->data);
-  r.location = malloct(HTS_URLMAXSIZE * 2);
-  r.location[0] = '\0';
-
-  /* feed one header line at a time, as the receive loop does */
-  while (p != NULL && *p != '\0') {
-    char *nl = strchr(p, '\n');
-    size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
-    size_t i, len = 0;
-
-    /* binput drops every '\r' on the wire; mirror it */
-    for (i = 0; i < n; i++)
-      if (p[i] != '\r')
-        line[len++] = p[i];
-    line[len] = '\0';
-    if (first) {
-      treatfirstline(&r, line);
-      first = 0;
-    } else {
-      treathead(cookie, "www.example.com", "/", &r, line);
-    }
-    p = (nl != NULL) ? nl + 1 : NULL;
-  }
-
-  freet(r.location);
-  freet(line);
-  freet(cookie);
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-idna.c

@@ -1,47 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
-#include "fuzz.h"
-#include "htscharset.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *s = fuzz_strdup(data, size);
-
-  {
-    char *idna = hts_convertStringUTF8ToIDNA(s, size);
-    freet(idna);
-  }
-  {
-    char *utf8 = hts_convertStringIDNAToUTF8(s, size);
-    freet(utf8);
-  }
-  (void) hts_isStringIDNA(s, size);
-
-  freet(s);
-  return 0;
-}

fuzz/fuzz-meta.c

@@ -1,40 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
-   <meta> charset declaration. */
-#include "fuzz.h"
-#include "htscharset.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *html = fuzz_strdup(data, size);
-  char *charset = hts_getCharsetFromMeta(html, size);
-
-  freet(charset);
-  freet(html);
-  return 0;
-}

fuzz/fuzz-unescape.c

@@ -1,53 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
-   output buffer size, so the bounded-copy contract is exercised. */
-#include "fuzz.h"
-#include "httrack-library.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  static const size_t bsizes[] = {1, 2, 16, 256, 8192};
-  size_t bsize;
-  char *s, *catbuff;
-
-  if (size == 0)
-    return 0;
-  bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
-  data++, size--;
-  s = fuzz_strdup(data, size);
-  catbuff = malloct(bsize);
-
-  (void) unescape_http(catbuff, bsize, s);
-  (void) unescape_http_unharm(catbuff, bsize, s, 0);
-  (void) unescape_http_unharm(catbuff, bsize, s, 1);
-  unescape_amp(s);
-
-  freet(catbuff);
-  freet(s);
-  return 0;
-}

fuzz/fuzz-url.c

@@ -1,49 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
-   is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
-   in place. */
-#include "fuzz.h"
-#include "htscore.h"
-#include "htslib.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *s = fuzz_strdup(data, size);
-  lien_adrfil *af = calloct(1, sizeof(*af));
-  /* fil_simplifie rewrites in place and may grow an empty path to "./" */
-  char *path = malloct(size + 3);
-
-  (void) ident_url_absolute(s, af);
-  memcpy(path, s, size + 1);
-  fil_simplifie(path);
-
-  freet(path);
-  freet(af);
-  freet(s);
-  return 0;
-}

fuzz/run-fuzzers.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-# Drive every built harness against its seed corpus.
-#   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
-#   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
-# Replay is crash/leak-only and never mutates; the per-unit -timeout guards
-# both modes against pathological slowdowns (e.g. pre-#501 strjoker).
-set -euo pipefail
-
-srcdir=$(cd "$(dirname "$0")" && pwd)
-bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
-mode=${2:-20}
-
-status=0
-for f in "$bld"/fuzz-*; do
-    if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
-    case "$f" in *.o | *.c | *.dSYM) continue ;; esac
-    name=$(basename "$f")
-    corpus="$srcdir/corpus/${name#fuzz-}"
-    if [ "$mode" = "check" ]; then
-        echo "=== $name (replay) ==="
-        [ -d "$corpus" ] || continue
-        if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
-            echo "*** $name FAILED on its corpus" >&2
-            status=1
-        fi
-        continue
-    fi
-    work=$(mktemp -d)
-    args=("$work")
-    [ -d "$corpus" ] && args+=("$corpus")
-    echo "=== $name (${mode}s) ==="
-    if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
-        -artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
-        echo "*** $name FAILED; artifacts:" >&2
-        ls -l "$work" >&2
-        status=1
-    else
-        rm -rf "$work"
-    fi
-done
-exit $status

history.txt

@@ -4,59 +4,7 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
-3.49-12
-+ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
-+ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
-+ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
-+ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
-+ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
-+ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
-+ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
-+ Fixed: cookies.txt was created world-readable (#511)
-+ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
-+ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
-+ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
-+ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
-+ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
-+ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
-+ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
-
-3.49-11
-+ New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
-+ New: advertise deflate in Accept-Encoding and decode deflate responses (#450)
-+ New: follow <source> and <track> media elements as embedded links (#451)
-+ New: added modern web MIME types to the type/extension table (#448)
-+ Fixed: enforce the -E time limit during a slow transfer instead of only between files (#481)
-+ Fixed: sniff the leading bytes of a download so a misdeclared Content-Type no longer renames a correct URL extension
-+ Fixed: fast transfers could be saved under their temporary .delayed placeholder name (#5, #107)
-+ Fixed: follow a redirect that maps to the same saved file instead of writing a self-pointing stub (#159)
-+ Fixed: several network-facing buffer overflows in the FTP, Java and HTML parsers
-+ Fixed: the htsjava plugin could not be loaded (hidden entry points, stale library name)
-+ Fixed: HTML-escape truncation and a cache-buffer leak in the parser
-+ Changed: modernized the default User-Agent to an honest HTTrack identifier (#449)
-+ Changed: decode the full WHATWG set of HTML named character references (#443)
-+ Changed: refreshed stale HTTP status, proxy-port and TLS-floor constants (#453)
-+ Changed: multiple internal hardening, build, test and CI improvements
-
-3.49-10
-+ New: --cookies-file to preload a Netscape cookies.txt before crawling (#215)
-+ New: --pause to space out file downloads by a random delay (#185)
-+ New: --strip-query to drop selected query keys from the dedup naming (#112)
-+ Changed: split the -%u URL hacks into independent --keep-www-prefix, --keep-double-slashes and --keep-query-order toggles (#271)
-+ Fixed: follow a redirect Location after dropping its #fragment, instead of requesting the fragment and polluting the saved name (#204)
-+ Fixed: escaped brackets inside a *[...] filter character class (#148)
-+ Fixed: honor the server's Content-Range when resuming a partial download, instead of appending overlapping bytes (#198)
-+ Fixed: abort the download as soon as the response type is excluded by -mime:, instead of fetching then discarding the body (#58)
-+ Fixed: keep size-based filter rules neutral until the file size is known (#143)
-+ Fixed: stop the mirror with a clean fatal error on a cache write failure, instead of crashing (#174, #219)
-+ Fixed: stop the 412/416 partial re-get loop on --continue and --update (#206)
-+ Fixed: keep an unrecognized URL tail instead of mangling it to .html (#115)
-+ Fixed: honor --tolerant (-%B) on a broken Content-Length, and fix an out-of-bounds read it exposed (#32, #41)
-+ Fixed: fall back to the next resolved address when a connection fails or stalls, instead of hanging on a dead IPv6 address
-+ Fixed: report why a -%L URL list could not be loaded (#49)
-+ Changed: multiple internal hardening, build and CI improvements
-
-.49-9
+3.49-9
 + Fixed: file-type detection from the Content-Type header: trust a declared type over a binary URL extension, honor --assume under the delayed type check, and keep a known extension against a bogus or empty Content-Type (#267, #29, #56)
 + Fixed: an uninitialized-buffer read when the Content-Type is empty (#411)
 + Fixed: restored C++ source-compatibility of the installed headers so reverse dependencies (httraqt) build again (#413)

html/faq.html

@@ -609,9 +609,10 @@ most mirrors do work. We still are working to improve the mirror quality of HTTr
 <br>
 
 </a><a NAME="Q2b">Q: <strong>Some Java applets do not work properly!</strong><br>
-A: <em>Java applets may not work in some cases, because HTTrack does not parse compiled class files to find
-the resources they load. Java applets often need to be online anyway, since remote files are fetched directly
-at runtime.<br>
+A: <em>Java applets may not work in some cases, for example if HTTrack failed to detect all included classes
+or files called within the class file. Sometimes, Java applets need to be online, because remote files are
+directly caught. Finally, the site structure can be incompatible with the class (always try to keep the original site structure
+when you want to get Java classes)<br>
 If there is no way to make some classes work properly, you can exclude them with the filters. 
 They will be available, but only online.
 </em>

html/fcguide.html

@@ -243,7 +243,7 @@ Build options:
 Spider options:
   bN accept cookies in cookies.txt (0=do not accept,* 1=accept) (--cookies[=N])
   u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (--check-type[=N])
-  j *parse scripts (j0 don't parse) (--parse-java[=N])
+  j *parse Java Classes (j0 don't parse) (--parse-java[=N])
   sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) (--robots[=N])
  %h  force HTTP/1.0 requests (reduce update features, only for old servers or proxies) (--http-10)
  %B  tolerant requests (accept bogus responses on some servers, but not standard!) (--tolerant)
@@ -417,7 +417,7 @@ site. Specifically, the defauls are:
   X *purge old files after update (X0 keep delete)
   bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
   u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
-  j *parse scripts (j0 don't parse)
+  j *parse Java Classes (j0 don't parse)
   sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
   C  create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)
   f *log file mode
@@ -497,9 +497,9 @@ about your browser retaining them.
 <p align=justify> This causes different document types to be analyzed
 differently.
 
-<pre><b><i>  j *parse scripts (j0 don't parse) </i></b></pre>
+<pre><b><i>  j *parse Java Classes (j0 don't parse) </i></b></pre>
 
-<p align=justify> This causes scripts to be parsed looking for
+<p align=justify> This causes Java class files to be parsed looking for
 URLs.
 
 <pre><b><i>  sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) </i></b></pre>
@@ -1178,7 +1178,7 @@ accepted and sent back in order to allow access.
 Spider options:
  bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
  u   check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
- j   *parse scripts (j0 don't parse)
+ j   *parse Java Classes (j0 don't parse)
  sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
  %h  force HTTP/1.0 requests (reduce update features, only for old servers or proxies)
  %B  tolerant requests (accept bogus responses on some servers, but not standard!)
@@ -2451,10 +2451,12 @@ work.  We still are working to improve the mirror quality of HTTrack.
 
 <p align=justify><b>Q: Some Java applets do not work properly! </b>
 
-<p align=justify> A: Java applets may not work in some cases, because
-HTTrack does not parse compiled class files to find the resources they
-load.  Java applets often need to be online anyway, since remote files
-are fetched directly at runtime.
+<p align=justify> A: Java applets may not work in some cases, for
+example if HTTrack failed to detect all included classes or files called
+within the class file.  Sometimes, Java applets need to be online,
+because remote files are directly caught.  Finally, the site structure
+can be incompatible with the class (always try to keep the original site
+structure when you want to get Java classes)
 
 <p align=justify> If there is no way to make some classes work properly,
 you can exclude them with the filters.  They will be available, but only

html/filters.html

@@ -247,7 +247,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
         <td>the \ character</td>
       </tr>
       <tr>
-        <td nowrap><tt>*[\[,\]]</tt></td>
+        <td nowrap><tt>*[\[\]]</tt></td>
         <td>the [ or ] character</td>
       </tr>
       <tr>

html/httrack.man.html

@@ -764,9 +764,9 @@ check, * u1 check but /, u2 check always)
 <td width="78%">
 
 
-<p>*parse scripts (j0 don t parse, bitmask: |1 parse
-default, |4 don t parse .js |8 don t be aggressive)
-(--parse-java[=N])</p></td></tr>
+<p>*parse Java Classes (j0 don t parse, bitmask: |1 parse
+default, |2 don t parse .class |4 don t parse .js |8 don t
+be aggressive) (--parse-java[=N])</p></td></tr>
 <tr valign="top" align="left">
 <td width="11%"></td>
 <td width="6%">
@@ -889,8 +889,8 @@ only) (--protocol[=N])</p></td></tr>
 <td width="78%">
 
 
-<p>disable a specific external mime module (-%w
-httrack-plugin) (--disable-module &lt;param&gt;)</p></td></tr>
+<p>disable a specific external mime module (-%w htsswf -%w
+htsjava) (--disable-module &lt;param&gt;)</p></td></tr>
 </table>
 
 <p style="margin-left:11%; margin-top: 1em"><b>Browser
@@ -2175,6 +2175,10 @@ Several scripts generating complex filenames may not find
 them (ex:
 img.src=&rsquo;image&rsquo;+a+Mobj.dst+&rsquo;.gif&rsquo;)</small></p>
 
+<p style="margin-left:11%; margin-top: 1em"><small>- Some
+java classes may not find some files on them (class
+included)</small></p>
+
 <p style="margin-left:11%; margin-top: 1em"><small>-
 Cgi-bin links may not work properly in some cases
 (parameters needed). To avoid them: use filters like

html/options.html

@@ -288,8 +288,8 @@ This option define whether the engine has to abandon a host if a timeout/"too sl
 &P extended parsing, attempt to parse all links (even in unknown tags or Javascript)
 This option activates the extended parsing, that attempt to find links in unknown Html code/javascript
 
-j *parse scripts (j0 don't parse)
-This option defines whether the engine has to parse scripts or not to catch included files
+j *parse Java Classes (j0 don't parse)
+This option define whether the engine has to parse java files or not to catch included files
 
 I *make an index (I0 don't make)
 This option define whether the engine has to generate an index.html on the top directory

html/plug.html

@@ -103,219 +103,220 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
 
 <br>
 
-You can write external functions to be plugged in the httrack library very easily. 
-We'll see there some examples.
+You can write external functions to be plugged in the httrack library very easily. 
+We'll see there some examples.
 
 <br><br>
-
-The <tt>httrack</tt> commandline tool allows (since the 3.30 release) to plug external functions to various callbacks defined in httrack.
-The 3.41 release introduces a cleaned up verion of callbacks, with two major changes:
-<ul>
-<li>Cleaned up function prototypes, with two arguments always passed (the caller carg structure, and the httrackp* object), convenient to pass an user-defined pointer (see <tt>CALLBACKARG_USERDEF(carg)</tt>)</li>
-<li>The httrackp* option structure can be directly accessed to plug callbacks (no need to give the callback name and function name in the commandline!)</li>
-<li>The callback plug is made through the CHAIN_FUNCTION() helper, allowing to chain multiple callbacks of the same type (the callbacks MUST preserve the chain by calling ancestors)</li>
-</ul>
-
-<br>
-References: 
-<ul>
-<li>the <tt>httrack-library.h</tt> prototype file
-<br />
+
+The <tt>httrack</tt> commandline tool allows (since the 3.30 release) to plug external functions to various callbacks defined in httrack.
+The 3.41 release introduces a cleaned up verion of callbacks, with two major changes:
+<ul>
+<li>Cleaned up function prototypes, with two arguments always passed (the caller carg structure, and the httrackp* object), convenient to pass an user-defined pointer (see <tt>CALLBACKARG_USERDEF(carg)</tt>)</li>
+<li>The httrackp* option structure can be directly accessed to plug callbacks (no need to give the callback name and function name in the commandline!)</li>
+<li>The callback plug is made through the CHAIN_FUNCTION() helper, allowing to chain multiple callbacks of the same type (the callbacks MUST preserve the chain by calling ancestors)</li>
+</ul>
+
+<br>
+References: 
+<ul>
+<li>the <tt>httrack-library.h</tt> prototype file
+<br />
 Note: the <i>Initialization</i>, <i>Main functions</i>, <i>Options handling</i> and <i>Wrapper functions</i> sections are generally the only ones to be considered.
 </li>
-<li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
-<li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
-<li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
-<li>the example given at the end of this document</li>
-</ul>
-
-<br />
-Below the list of functions to be defined in the module (plugin).<br />
-<br />
-
-<table width="100%">
-<tr><td><b><i>module function name</i></b></td><td><b>function description</b></td><td><b>function signature</b></td></tr>
-<tr><td background="img/fade.gif"><i>hts_plug</i></td><td background="img/fade.gif">
-The module entry point. The opt structure can be used to plug callbacks, using the CHAIN_FUNCTION() macro helper. The argv optional argument is the one passed in the commandline as --wrapper parameter.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)<br />
-
-<br />
-Wrappers can be plugged inside hts_plug() using:<br />
-<tt>
-CHAIN_FUNCTION(opt, &lt;callback name&gt;, &lt;our callback function name&gt;, &lt;our callback function optional custom pointer argument&gt;);
-</tt>
-<br />
-
-<br />
-Example:
-<br />
-<tt>
-CHAIN_FUNCTION(opt, check_html, process, userdef);
-</tt>
-<br />
-
-</td><td background="img/fade.gif"><tt>extern int hts_plug(httrackp *opt, const char* argv);</tt></td></tr>
-
-<!-- -->
-
-<tr><td background="img/fade.gif"><i>hts_unplug</i></td><td background="img/fade.gif">
-The module exit point. To free allocated resources without using global variables, use the uninit callback (see below)</td><td background="img/fade.gif"><tt>extern int hts_unplug(httrackp *opt);</tt></td></tr>
-
-</table>
-
-
-<br />
-Note that all callbacks (except init and uninit) take as first two argument:
-<ul>
-<li>the t_hts_callbackarg structure<br />
-this structure holds the callback chain (parent callbacks defined before the current callback) pointers, and the user-defined pointer ; see <tt>CALLBACKARG_USERDEF(carg)</tt>)
-</li>
-<li>the httrackp structure<br />
-this structure, holding all current httrack options and mirror state, can be read or mofidied
-</li>
-</ul>
-
-<br />
-Below the list of callbacks, and associated external wrappers.
-
-<table width="100%">
-<tr><td><b><i>callback name</i></b></td><td><b>callback description</b></td><td><b>callback function signature</b></td></tr>
-
-<tr><td background="img/fade.gif"><i>init</i></td><td background="img/fade.gif">Note: the use the "start" callback is advised. Called during initialization.<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>uninit</i></td><td background="img/fade.gif">Note: the use os the "end" callback is advised.<br />Called during un-initialization<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>start</i></td><td background="img/fade.gif">Called when the mirror starts. The <tt>opt</tt> structure passed lists all options defined for this mirror. You may modify the <tt>opt</tt> structure to fit your needs.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>end</i></td><td background="img/fade.gif">Called when the mirror ends<br>return value: 1 upon success, 0 upon error (the mirror will then be considered aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>chopt</i></td><td background="img/fade.gif">Called when options are to be changed. The <tt>opt</tt> structure passed lists all options, updated to take account of recent changes<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>preprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is to be parsed (original, not yet modified document). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>postprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is parsed and transformed (links rewritten). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_html</i></td><td background="img/fade.gif">Called when a document (which may not be an html document) is to be parsed. The <tt>html</tt> address points to the document data, of lenth <tt>len</tt>. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the parsing can be processed, 0 if the file must be skipped without being parsed</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question. The <tt>question</tt> string contains the question for the (human) user<br>return value: the string answer ("" for default reply)</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query2</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query3</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>loop</i></td><td background="img/fade.gif">Called periodically (informational, to display statistics)<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back, int back_max, int back_index, int lien_tot, int lien_ntot, int stat_time, hts_stat_struct* stats);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_link</i></td><td background="img/fade.gif">Called when a link has to be tested. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, int status);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_mime</i></td><td background="img/fade.gif">Called when a link download has begun, and needs to be tested against its MIME type. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested, and the <tt>mime</tt> string contains the link type being processed. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, const char* mime, int status);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>pause</i></td><td background="img/fade.gif">Called when the engine must pause. When the <tt>lockfile</tt> passed is deleted, the function can return<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* lockfile);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>filesave</i></td><td background="img/fade.gif">Called when a file is to be saved on disk<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>filesave2</i></td><td background="img/fade.gif">Called when a file is to be saved or checked on disk<br>The hostname, filename and local filename are given. Two additional flags tells if the local file is new (is_new), if the local file is to be modified (is_modified), and if the file was not updated remotely (not_updated).<br>(!is_new && !is_modified): the file is up-to-date, and will not be modified<br>(is_new && is_modified): a new file will be written (or an updated file is being written)<br>(!is_new && is_modified): a file is being updated (append)<br>(is_new && !is_modified): an empty file will be written ("do not recatch locally erased files")<br>not_updated: the file was not re-downloaded because it was up-to-date (no data transferred again)<br><br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* hostname, const char* filename, const char* localfile, int is_new, int is_modified, int not_updated);</tt></td></tr>
-
-<tr><td background="img/fade.gif"><i>linkdetected</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>linkdetected2</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link, const const char* tag_start);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>xfrstatus</i></td><td background="img/fade.gif">Called when a file has been processed (downloaded, updated, or error)<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>savename</i></td><td background="img/fade.gif">Called when a local filename has to be processed. The <tt>adr_complete</tt> and <tt>fil_complete</tt> are the address and URI of the file being saved ; the <tt>referer_adr</tt> and <tt>referer_fil</tt> are the address and URI of the referer link. The <tt>save</tt> string contains the local filename being used. You may modifiy the <tt>save</tt> string to fit your needs, up to 1024 bytes (note: filename collisions, if any, will be handled by the engine by renaming the file into file-2.ext, file-3.ext ..).<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>sendhead</i></td><td background="img/fade.gif">Called when HTTP headers are to be sent to the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>outgoing</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>receivehead</i></td><td background="img/fade.gif">Called when HTTP headers are recevived from the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>incoming</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);</tt></td></tr>
-
-<tr><td background="img/fade.gif"><i>detect</i></td><td background="img/fade.gif">Called when an unknown document is to be parsed. The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the type is known and can be parsed, 0 if the document type is unknown</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>parse</i></td><td background="img/fade.gif">The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the document was successfully parsed, 0 if an error occurred</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
-
-
-</table>
+<li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
+<li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
+<li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
+<li>the <tt>htsjava.c</tt> source file (the java class plugin ; overrides 'detect' and 'parse')</li>
+<li>the example given at the end of this document</li>
+</ul>
+
+<br />
+Below the list of functions to be defined in the module (plugin).<br />
+<br />
+
+<table width="100%">
+<tr><td><b><i>module function name</i></b></td><td><b>function description</b></td><td><b>function signature</b></td></tr>
+<tr><td background="img/fade.gif"><i>hts_plug</i></td><td background="img/fade.gif">
+The module entry point. The opt structure can be used to plug callbacks, using the CHAIN_FUNCTION() macro helper. The argv optional argument is the one passed in the commandline as --wrapper parameter.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)<br />
+
+<br />
+Wrappers can be plugged inside hts_plug() using:<br />
+<tt>
+CHAIN_FUNCTION(opt, &lt;callback name&gt;, &lt;our callback function name&gt;, &lt;our callback function optional custom pointer argument&gt;);
+</tt>
+<br />
+
+<br />
+Example:
+<br />
+<tt>
+CHAIN_FUNCTION(opt, check_html, process, userdef);
+</tt>
+<br />
+
+</td><td background="img/fade.gif"><tt>extern int hts_plug(httrackp *opt, const char* argv);</tt></td></tr>
+
+<!-- -->
+
+<tr><td background="img/fade.gif"><i>hts_unplug</i></td><td background="img/fade.gif">
+The module exit point. To free allocated resources without using global variables, use the uninit callback (see below)</td><td background="img/fade.gif"><tt>extern int hts_unplug(httrackp *opt);</tt></td></tr>
+
+</table>
+
+
+<br />
+Note that all callbacks (except init and uninit) take as first two argument:
+<ul>
+<li>the t_hts_callbackarg structure<br />
+this structure holds the callback chain (parent callbacks defined before the current callback) pointers, and the user-defined pointer ; see <tt>CALLBACKARG_USERDEF(carg)</tt>)
+</li>
+<li>the httrackp structure<br />
+this structure, holding all current httrack options and mirror state, can be read or mofidied
+</li>
+</ul>
+
+<br />
+Below the list of callbacks, and associated external wrappers.
+
+<table width="100%">
+<tr><td><b><i>callback name</i></b></td><td><b>callback description</b></td><td><b>callback function signature</b></td></tr>
+
+<tr><td background="img/fade.gif"><i>init</i></td><td background="img/fade.gif">Note: the use the "start" callback is advised. Called during initialization.<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>uninit</i></td><td background="img/fade.gif">Note: the use os the "end" callback is advised.<br />Called during un-initialization<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>start</i></td><td background="img/fade.gif">Called when the mirror starts. The <tt>opt</tt> structure passed lists all options defined for this mirror. You may modify the <tt>opt</tt> structure to fit your needs.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>end</i></td><td background="img/fade.gif">Called when the mirror ends<br>return value: 1 upon success, 0 upon error (the mirror will then be considered aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>chopt</i></td><td background="img/fade.gif">Called when options are to be changed. The <tt>opt</tt> structure passed lists all options, updated to take account of recent changes<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>preprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is to be parsed (original, not yet modified document). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>postprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is parsed and transformed (links rewritten). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_html</i></td><td background="img/fade.gif">Called when a document (which may not be an html document) is to be parsed. The <tt>html</tt> address points to the document data, of lenth <tt>len</tt>. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the parsing can be processed, 0 if the file must be skipped without being parsed</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question. The <tt>question</tt> string contains the question for the (human) user<br>return value: the string answer ("" for default reply)</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query2</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query3</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>loop</i></td><td background="img/fade.gif">Called periodically (informational, to display statistics)<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back, int back_max, int back_index, int lien_tot, int lien_ntot, int stat_time, hts_stat_struct* stats);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_link</i></td><td background="img/fade.gif">Called when a link has to be tested. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, int status);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_mime</i></td><td background="img/fade.gif">Called when a link download has begun, and needs to be tested against its MIME type. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested, and the <tt>mime</tt> string contains the link type being processed. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, const char* mime, int status);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>pause</i></td><td background="img/fade.gif">Called when the engine must pause. When the <tt>lockfile</tt> passed is deleted, the function can return<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* lockfile);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>filesave</i></td><td background="img/fade.gif">Called when a file is to be saved on disk<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>filesave2</i></td><td background="img/fade.gif">Called when a file is to be saved or checked on disk<br>The hostname, filename and local filename are given. Two additional flags tells if the local file is new (is_new), if the local file is to be modified (is_modified), and if the file was not updated remotely (not_updated).<br>(!is_new && !is_modified): the file is up-to-date, and will not be modified<br>(is_new && is_modified): a new file will be written (or an updated file is being written)<br>(!is_new && is_modified): a file is being updated (append)<br>(is_new && !is_modified): an empty file will be written ("do not recatch locally erased files")<br>not_updated: the file was not re-downloaded because it was up-to-date (no data transferred again)<br><br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* hostname, const char* filename, const char* localfile, int is_new, int is_modified, int not_updated);</tt></td></tr>
+
+<tr><td background="img/fade.gif"><i>linkdetected</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>linkdetected2</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link, const const char* tag_start);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>xfrstatus</i></td><td background="img/fade.gif">Called when a file has been processed (downloaded, updated, or error)<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>savename</i></td><td background="img/fade.gif">Called when a local filename has to be processed. The <tt>adr_complete</tt> and <tt>fil_complete</tt> are the address and URI of the file being saved ; the <tt>referer_adr</tt> and <tt>referer_fil</tt> are the address and URI of the referer link. The <tt>save</tt> string contains the local filename being used. You may modifiy the <tt>save</tt> string to fit your needs, up to 1024 bytes (note: filename collisions, if any, will be handled by the engine by renaming the file into file-2.ext, file-3.ext ..).<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>sendhead</i></td><td background="img/fade.gif">Called when HTTP headers are to be sent to the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>outgoing</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>receivehead</i></td><td background="img/fade.gif">Called when HTTP headers are recevived from the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>incoming</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);</tt></td></tr>
+
+<tr><td background="img/fade.gif"><i>detect</i></td><td background="img/fade.gif">Called when an unknown document is to be parsed. The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the type is known and can be parsed, 0 if the document type is unknown</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>parse</i></td><td background="img/fade.gif">The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the document was successfully parsed, 0 if an error occurred</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
+
+
+</table>
 
 <br><br>
-Note: the optional libhttrack-plugin module (libhttrack-plugin.dll or libhttrack-plugin.so), if found in the library environment, is loaded automatically, and its <tt>hts_plug()</tt> function being called.<br />
-
-
-<br />
-An example is generally more efficient than anything else, so let's write our first module, aimed to stupidely print all parsed html files:
-<table width="100%" border="2">
-<tr><td>
-<pre>
-/* system includes */
-#include &lt;stdio.h&gt;
-#include &lt;stdlib.h&gt;
-#include &lt;string.h&gt;
-
-/* standard httrack module includes */
-#include "httrack-library.h"
-#include "htsopt.h"
-#include "htsdefines.h"
-
-/* local function called as "check_html" callback */
-static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt, 
-                        /*other parameters are callback-specific*/
-                        char* html, int len, const char* url_address, const char* url_file) {
-  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
-
-  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
-  if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
-    if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
-                                                html, len, url_address, url_file)) {
-        return 0;  /* abort */
-      }
-  }
-
-  printf("file %s%s content: %s\n", url_address, url_file, html);
-  return 1;  /* success */
-}
-
-/* local function called as "end" callback */
-static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
-  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
-
-  /* processing */
-  fprintf(stderr, "That's all, folks!\n");
-
-  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
-  if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
-    /* status is ok on our side, return other callabck's status */
-    return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
-  }
-
-  return 1;  /* success */
-}
-
-/*
-module entry point
-the function name and prototype MUST match this prototype
-*/
-EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
-  /* optional argument passed in the commandline we won't be using here */
-  const char *arg = strchr(argv, ',');
-  if (arg != NULL)
-    arg++;
-
-  /* plug callback functions */
-  CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
-  CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
-
-  return 1;  /* success */
-}
-
-/*
-module exit point
-the function name and prototype MUST match this prototype
-*/
-EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
-  fprintf(stder, "Module unplugged");
-
-  return 1;  /* success */
-}
-</pre>
-</td></tr></table>
-
-<br />
-Compile this file ; for example:
-<br />
-<tt>
-gcc -O -g3 -shared -o mylibrary.so myexample.c
-</tt>
-<br />
-and plug the module using the commandline ; for example:
-<br />
-<tt>
-httrack --wrapper mylibrary http://www.example.com
-</tt>
-<br />
-or, if some parameters are desired:
-<br />
-<tt>
-httrack --wrapper mylibrary,myparameter-string http://www.example.com
-</tt>
-<br />
-(the "myparameter-string" string will be available in the 'arg' parameter passed to the hts_plug entry point)
-<br />
-
+Note: the optional libhttrack-plugin module (libhttrack-plugin.dll or libhttrack-plugin.so), if found in the library environment, is loaded automatically, and its <tt>hts_plug()</tt> function being called.<br />
+
 
+<br />
+An example is generally more efficient than anything else, so let's write our first module, aimed to stupidely print all parsed html files:
+<table width="100%" border="2">
+<tr><td>
+<pre>
+/* system includes */
+#include &lt;stdio.h&gt;
+#include &lt;stdlib.h&gt;
+#include &lt;string.h&gt;
+
+/* standard httrack module includes */
+#include "httrack-library.h"
+#include "htsopt.h"
+#include "htsdefines.h"
+
+/* local function called as "check_html" callback */
+static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt, 
+                        /*other parameters are callback-specific*/
+                        char* html, int len, const char* url_address, const char* url_file) {
+  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
+
+  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
+  if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
+    if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
+                                                html, len, url_address, url_file)) {
+        return 0;  /* abort */
+      }
+  }
+
+  printf("file %s%s content: %s\n", url_address, url_file, html);
+  return 1;  /* success */
+}
+
+/* local function called as "end" callback */
+static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
+  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
+
+  /* processing */
+  fprintf(stderr, "That's all, folks!\n");
+
+  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
+  if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
+    /* status is ok on our side, return other callabck's status */
+    return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
+  }
+
+  return 1;  /* success */
+}
+
+/*
+module entry point
+the function name and prototype MUST match this prototype
+*/
+EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
+  /* optional argument passed in the commandline we won't be using here */
+  const char *arg = strchr(argv, ',');
+  if (arg != NULL)
+    arg++;
+
+  /* plug callback functions */
+  CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
+  CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
+
+  return 1;  /* success */
+}
+
+/*
+module exit point
+the function name and prototype MUST match this prototype
+*/
+EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
+  fprintf(stder, "Module unplugged");
+
+  return 1;  /* success */
+}
+</pre>
+</td></tr></table>
+
+<br />
+Compile this file ; for example:
+<br />
+<tt>
+gcc -O -g3 -shared -o mylibrary.so myexample.c
+</tt>
+<br />
+and plug the module using the commandline ; for example:
+<br />
+<tt>
+httrack --wrapper mylibrary http://www.example.com
+</tt>
+<br />
+or, if some parameters are desired:
+<br />
+<tt>
+httrack --wrapper mylibrary,myparameter-string http://www.example.com
+</tt>
+<br />
+(the "myparameter-string" string will be available in the 'arg' parameter passed to the hts_plug entry point)
+<br />
+
+
 <br><br>
 
 <!-- ==================== Start epilogue ==================== -->

html/step9_opt6.html

@@ -118,8 +118,8 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
   <br>Avoid "never", because the local mirror could be bogus
   </small><br><br>
 <!-- -->
-  <li>Parse scripts</li>
-  <br><small>Must the engine parse scripts to seek included filenames?
+  <li>Parse java files</li>
+  <br><small>Must the engine parse .java files (java classes) to seek included filenames?
   <br>It is checked by default
   </small><br><br>
 <!-- -->

lang/English.txt

@@ -295,7 +295,7 @@ Max Depth
 Maximum external depth:
 Maximum external depth:
 Filters (refuse/accept links) :
-Filters (refuse/accept links):
+Filters (refuse/accept links) :
 Paths
 Paths
 Save prefs

lang/README.md

@@ -1,37 +0,0 @@
-# Translating HTTrack
-
-Interface strings live here, one `.txt` file per language. `English.txt` is the reference: every other file maps each English string to its translation.
-
-## File format
-
-Plain text, entries in consecutive pairs of lines:
-
-```
-<English string>
-<translation>
-```
-
-The first line of a pair is the lookup key and must stay identical to the one in `English.txt`; translate only the second line. Missing entries fall back to the English text at runtime, so a partial translation works.
-
-Preserve any `\r\n`, `\t` and `printf` placeholders (`%s`, `%d`, ...) in the translation.
-
-A few `LANGUAGE_*` entries at the top describe the file itself:
-
-| Key | Meaning |
-| --- | --- |
-| `LANGUAGE_NAME` | Name shown in the language picker, in its own language (`Deutsch`, not `German`) |
-| `LANGUAGE_ISO` | ISO 639 code, with region if needed (`de`, `pt_BR`) |
-| `LANGUAGE_CHARSET` | Encoding the file is saved in (`ISO-8859-1`, `windows-1251`, `UTF-8`, ...) |
-| `LANGUAGE_AUTHOR` | Your name and contact |
-| `LANGUAGE_WINDOWSID` | Windows locale name used by WinHTTrack (`German (Standard)`) |
-
-Save the file in exactly its declared `LANGUAGE_CHARSET`; an editor that rewrites it as UTF-8 will corrupt the non-ASCII bytes.
-
-## Adding or updating a language
-
-1. Copy `English.txt` to `<Language>.txt`, or edit the existing file.
-2. Translate each second line; leave the English keys untouched.
-3. Fill in the `LANGUAGE_*` header for a new file.
-4. Open a pull request, or attach the file to a GitHub issue.
-
-When new strings land in `English.txt` they show up untranslated (as English) until a translator fills them in.

m4/ax_add_fortify_source.m4

@@ -1,119 +0,0 @@
-# ===========================================================================
-#  https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_ADD_FORTIFY_SOURCE
-#
-# DESCRIPTION
-#
-#   Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
-#   redefinition warnings, other cpp warnings or linker. Some distributions
-#   (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
-#   their compilers, leading to unnecessary warnings in the form of
-#
-#     <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
-#     <built-in>: note: this is the location of the previous definition
-#
-#   which is a problem if -Werror is enabled. This macro checks whether
-#   _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
-#   to CPPFLAGS.
-#
-#   Newer mingw-w64 msys2 package comes with a bug in
-#   headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
-#   and would need -lssp or -fstack-protector.  See
-#   https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
-#   link it.
-#
-# LICENSE
-#
-#   Copyright (c) 2017 David Seifert <soap@gentoo.org>
-#   Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved.  This file is offered as-is, without any
-#   warranty.
-
-#serial 10
-
-AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
-    ac_save_cflags=$CFLAGS
-    ac_cwerror_flag=yes
-    AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
-    ax_add_fortify_3_failed=
-    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],
-            [[
-                #ifndef _FORTIFY_SOURCE
-                    return 0;
-                #else
-                    _FORTIFY_SOURCE_already_defined;
-                #endif
-            ]]
-        )],
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([[
-                #define _FORTIFY_SOURCE 3
-                #include <string.h>
-                int main(void) {
-                    char *s = " ";
-                    strcpy(s, "x");
-                    return strlen(s)-1;
-                }
-              ]]
-            )],
-            [
-              AC_MSG_RESULT([yes])
-              CFLAGS=$ac_save_cflags
-              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
-            ], [
-              AC_MSG_RESULT([no])
-              ax_add_fortify_3_failed=1
-            ],
-        ),
-        [
-          AC_MSG_RESULT([no])
-          ax_add_fortify_3_failed=1
-        ])
-    if test -n "$ax_add_fortify_3_failed"
-    then
-    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],
-            [[
-                #ifndef _FORTIFY_SOURCE
-                    return 0;
-                #else
-                    _FORTIFY_SOURCE_already_defined;
-                #endif
-            ]]
-        )],
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([[
-                #define _FORTIFY_SOURCE 2
-                #include <string.h>
-                int main(void) {
-                    char *s = " ";
-                    strcpy(s, "x");
-                    return strlen(s)-1;
-                }
-              ]]
-            )],
-            [
-              AC_MSG_RESULT([yes])
-              CFLAGS=$ac_save_cflags
-              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
-            ], [
-              AC_MSG_RESULT([no])
-              CFLAGS=$ac_save_cflags
-            ],
-        ),
-        [
-          AC_MSG_RESULT([no])
-          CFLAGS=$ac_save_cflags
-        ])
-    fi
-])

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "12 July 2026" "httrack website copier"
+.TH httrack 1 "26 June 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -24,7 +24,6 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-EN, \-\-max\-time[=N]\fR ]
 [ \fB\-AN, \-\-max\-rate[=N]\fR ]
 [ \fB\-%cN, \-\-connection\-per\-second[=N]\fR ]
-[ \fB\-%G, \-\-pause\fR ]
 [ \fB\-GN, \-\-max\-pause[=N]\fR ]
 [ \fB\-cN, \-\-sockets[=N]\fR ]
 [ \fB\-TN, \-\-timeout[=N]\fR ]
@@ -44,14 +43,11 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-x, \-\-replace\-external\fR ]
 [ \fB\-%x, \-\-disable\-passwords\fR ]
 [ \fB\-%q, \-\-include\-query\-string\fR ]
-[ \fB\-%g, \-\-strip\-query\fR ]
 [ \fB\-o, \-\-generate\-errors\fR ]
 [ \fB\-X, \-\-purge\-old[=N]\fR ]
 [ \fB\-%p, \-\-preserve\fR ]
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
-[ \fB\-%K, \-\-cookies\-file\fR ]
-[ \fB\-%Y, \-\-why\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -157,8 +153,6 @@ maximum mirror time in seconds (60=1 minute, 3600=1 hour) (\-\-max\-time[=N])
 maximum transfer rate in bytes/seconds (1000=1KB/s max) (\-\-max\-rate[=N])
 .IP \-%cN
 maximum number of connections/seconds (*%c10) (\-\-connection\-per\-second[=N])
-.IP \-%G
-random pause of MIN[:MAX] seconds between files (e.g. %G5:10) (\-\-pause <param>)
 .IP \-GN
 pause transfer if N bytes reached, and wait until lock file is deleted (\-\-max\-pause[=N])
 .SS Flow control:
@@ -204,8 +198,6 @@ replace external html links by error pages (\-\-replace\-external)
 do not include any password for external password protected websites (%x0 include) (\-\-disable\-passwords)
 .IP \-%q
 *include query string for local files (useless, for information purpose only) (%q0 don't include) (\-\-include\-query\-string)
-.IP \-%g
-strip query keys for dedup ([host/pattern=]key1,key2,...) (\-\-strip\-query <param>)
 .IP \-o
 *generate output html file in case of error (404..) (o0 don't generate) (\-\-generate\-errors)
 .IP \-X
@@ -217,14 +209,10 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 .SS Spider options:
 .IP \-bN
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
-.IP \-%K
-load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
-.IP \-%Y
-explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j
-*parse scripts (j0 don't parse, bitmask: |1 parse default, |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
+*parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
 .IP \-sN
 follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules)) (\-\-robots[=N])
 .IP \-%h
@@ -237,8 +225,6 @@ tolerant requests (accept bogus responses on some servers, but not standard!) (\
 update hacks: various hacks to limit re\-transfers when updating (identical size, bogus response..) (\-\-updatehack)
 .IP \-%u
 url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..) (\-\-urlhack)
-.br
-opt out of one url\-hack part: \-\-keep\-www\-prefix (www.foo.com<>foo.com), \-\-keep\-double\-slashes (//), \-\-keep\-query\-order (?b&a)
 .IP \-%A
 assume that a type (cgi,asp..) is always linked with a mime type (\-%A php3,cgi=text/html;dat,bin=application/x\-zip) (\-\-assume <param>)
 .br
@@ -248,7 +234,7 @@ can also be used to force a specific file type: \-\-assume foo.cgi=text/html
 .IP \-@iN
 internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only) (\-\-protocol[=N])
 .IP \-%w
-disable a specific external mime module (\-%w httrack\-plugin) (\-\-disable\-module <param>)
+disable a specific external mime module (\-%w htsswf \-%w htsjava) (\-\-disable\-module <param>)
 .SS Browser ID:
 .IP \-F
 user\-agent field sent in HTTP headers (\-F "user\-agent name") (\-\-user\-agent <param>)
@@ -514,6 +500,8 @@ that would have solved them.
 .SM
 \- Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
 .SM
+\- Some java classes may not find some files on them (class included)
+.SM
 \- Cgi\-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like \-*cgi\-bin*
 .SH BUGS
 Please reports bugs to

src/Makefile.am

@@ -44,7 +44,7 @@ httrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 proxytrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 
-lib_LTLIBRARIES = libhttrack.la
+lib_LTLIBRARIES = libhttrack.la libhtsjava.la
 
 htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h
 proxytrack_SOURCES = proxy/main.c \
@@ -62,7 +62,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htsname.c htsrobots.c htstools.c htswizard.c \
 	htsalias.c htsthread.c htsindex.c htsbauth.c \
 	htsmd5.c htszlib.c htswrap.c htsconcat.c \
-	htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
+	htsmodules.c htscharset.c punycode.c htsencoding.c \
 	md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
 	hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
@@ -70,7 +70,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h  \
 	htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
 	htshelp.h htsindex.h htslib.h htsmd5.h \
-	htsmodules.h htsname.h htsnet.h htssniff.h \
+	htsmodules.h htsname.h htsnet.h \
 	htsopt.h htsrobots.h htsthread.h  \
 	htstools.h htswizard.h htswrap.h htszlib.h  \
 	htsstrings.h htsarrays.h httrack-library.h \
@@ -82,6 +82,14 @@ libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(OPENSSL_LIBS) $(DL_LIBS) $
 libhttrack_la_CFLAGS = $(AM_CFLAGS) -DLIBHTTRACK_EXPORTS -DZLIB_CONST
 libhttrack_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO)
 
+libhtsjava_la_SOURCES = htsjava.c htsjava.h
+libhtsjava_la_LIBADD = $(THREADS_LIBS) $(DL_LIBS) libhttrack.la
+# This thin JNI wrapper reaches libc only through libhttrack, so the direct
+# libc edge is dropped from DT_NEEDED (library-not-linked-against-libc). Force
+# libc back as a dependency; configure gates the flag since only a GNU-style
+# linker accepts it (LIBC_FORCE_LINK is empty on e.g. macOS).
+libhtsjava_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO) $(LIBC_FORCE_LINK)
+
 EXTRA_DIST = httrack.h webhttrack \
 		coucal/murmurhash3.h.diff \
 		coucal/murmurhash3.h.orig \
@@ -111,6 +119,7 @@ EXTRA_DIST = httrack.h webhttrack \
 		coucal/README.md \
 		coucal/sample.c \
 		coucal/tests.c \
+		htsjava.vcproj \
 		httrack.dsp httrack.dsw httrack.vcproj \
 		libhttrack.dsp libhttrack.dsw libhttrack.vcproj \
 		webhttrack.dsp webhttrack.dsw webhttrack.vcproj

src/htsalias.c

@@ -60,9 +60,6 @@ Please visit our Website: http://www.httrack.com
   param1 : this option must be alone, and needs one distinct parameter (-P <path>)
   param0 : this option must be alone, but the parameter should be put together (+*.gif)
 */
-/* clang-format off: hand-aligned table; clang-format reflows the whole
-   initializer (2->4 space) on any edit, churning every untouched row. */
-/* clang-format off */
 const char *hts_optalias[][4] = {
   /*   {"","","",""}, */
   {"path", "-O", "param1", "output path"},
@@ -110,14 +107,6 @@ const char *hts_optalias[][4] = {
   {"disable-passwords", "-%x", "single", ""}, {"disable-password", "-%x",
                                                "single", ""},
   {"include-query-string", "-%q", "single", ""},
-  {"strip-query", "-%g", "param1",
-   "strip [host/pattern=]key1,key2,... from URLs"},
-  {"cookies-file", "-%K", "param1",
-   "load extra cookies from a Netscape cookies.txt"},
-  {"why", "-%Y", "param1",
-   "explain which filter rule accepts or rejects a URL, then exit"},
-  {"pause", "-%G", "param1",
-   "random pause of MIN[:MAX] seconds between files"},
   {"generate-errors", "-o", "single", ""},
   {"do-not-generate-errors", "-o0", "single", ""},
   {"purge-old", "-X", "param", ""},
@@ -134,9 +123,6 @@ const char *hts_optalias[][4] = {
   {"tolerant", "-%B", "single", ""},
   {"updatehack", "-%s", "single", ""}, {"sizehack", "-%s", "single", ""},
   {"urlhack", "-%u", "single", ""},
-  {"keep-www-prefix", "-%j", "single", ""},
-  {"keep-double-slashes", "-%o", "single", ""},
-  {"keep-query-order", "-%y", "single", ""},
   {"user-agent", "-F", "param1", "user-agent identity"},
   {"referer", "-%R", "param1", "default referer URL"},
   {"from", "-%E", "param1", "from email address"},
@@ -255,7 +241,6 @@ const char *hts_optalias[][4] = {
 
   {"", "", "", ""}
 };
-/* clang-format on */
 
 /* 
   Check for alias in command-line 
@@ -285,6 +270,7 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
       char *position;
       int need_param = 1;
 
+      //int return_param=0;
       int pos;
 
       command[0] = param[0] = addcommand[0] = '\0';
@@ -362,6 +348,7 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
               strlcatbuff(return_argv[0], "0", return_argv_size);
             else if (strcmp(param, "on") == 0) {
               // on is the default
+              // strcatbuff(return_argv[0],"1");
             } else
               strlcatbuff(return_argv[0], param, return_argv_size);
           }

src/htsback.h

@@ -113,12 +113,6 @@ void back_set_locked(struct_back * sback, const int p);
 void back_set_unlocked(struct_back * sback, const int p);
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p);
-/* Discard back's on-disk .delayed placeholder and its refname. */
-void back_delayed_discard(httrackp *opt, lien_back *back);
-/* Move back's .delayed placeholder (and open stream) to newname;
-   HTS_FALSE = file lost, slot flagged in error. */
-hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
-                                const char *newname);
 void back_index_unlock(struct_back * sback, const int p);
 int back_clear_entry(lien_back * back);
 int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
@@ -138,11 +132,10 @@ LLint back_transferred(LLint add, struct_back * sback);
 
 // hostback
 #if HTS_XGETHOST
+void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
-/* Enforce -M/-E quotas: smooth-stops when reached; returns 0 once the -M cap
-   or -E deadline overruns its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htsbasenet.h

@@ -55,6 +55,8 @@ Please visit our Website: http://www.httrack.com
 // inline function in Wspiapi.h.
 #include <ws2tcpip.h>
 #include <Wspiapi.h>
+// #include <winsock2.h>
+// #include <tpipv6.h>
 
 #endif
 
@@ -127,8 +129,6 @@ typedef enum HTTPStatusCode {
   HTTP_UNSUPPORTED_MEDIA_TYPE = 415,
   HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416,
   HTTP_EXPECTATION_FAILED = 417,
-  HTTP_TOO_MANY_REQUESTS = 429,
-  HTTP_UNAVAILABLE_FOR_LEGAL_REASONS = 451,
   HTTP_INTERNAL_SERVER_ERROR = 500,
   HTTP_NOT_IMPLEMENTED = 501,
   HTTP_BAD_GATEWAY = 502,
@@ -146,8 +146,7 @@ typedef enum BackStatusCode {
   STATUSCODE_NON_FATAL = -5,
   STATUSCODE_SSL_HANDSHAKE = -6,
   STATUSCODE_TOO_BIG = -7,
-  STATUSCODE_TEST_OK = -10,
-  STATUSCODE_EXCLUDED = -11 /* aborted: MIME excluded by a -mime: filter */
+  STATUSCODE_TEST_OK = -10
 } BackStatusCode;
 
 /** HTTrack status ('status' member of of 'lien_back') **/

src/htsbasiccharsets.sh

@@ -3,12 +3,12 @@
 
 # Change this to download files
 if false; then
-    echo "mget https://www.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
-    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
-    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
-    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
-    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
-    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
     rm -f CP932.TXT CP936.TXT CP949.TXT CP950.TXT
 fi
 

src/htsbauth.c

@@ -107,6 +107,7 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
 #if DEBUG_COOK
   printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
          cook_name, cook_value, domain, path);
+  //printf(">>>cook: %s<<<\n",cookie->data);
 #endif
   return 0;
 }
@@ -207,6 +208,8 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   char catbuff[CATBUFF_SIZE];
   char buffer[8192];
 
+  //  cookie->data[0]='\0';
+
   // Fusionner d'abord les éventuels cookies IE
 #ifdef _WIN32
   {
@@ -307,25 +310,14 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   return -1;
 }
 
-/* Write cookies.txt; returns 0 on success. The jar holds live session
-   cookies, so keep it owner-only on Unix (Windows inherits folder ACLs). */
+// écrire cookies.txt
+// !=0 : erreur
 int cookie_save(t_cookie * cookie, const char *name) {
   char catbuff[CATBUFF_SIZE];
 
   if (strnotempty(cookie->data)) {
     char BIGSTK line[8192];
-#ifdef _WIN32
     FILE *fp = fopen(fconv(catbuff, sizeof(catbuff), name), "wb");
-#else
-    const int fd = open(fconv(catbuff, sizeof(catbuff), name),
-                        O_WRONLY | O_CREAT | O_TRUNC, HTS_PROTECT_FILE);
-    FILE *fp = fd != -1 ? fdopen(fd, "wb") : NULL;
-
-    if (fd != -1 && fp == NULL)
-      close(fd);    /* fdopen failed: don't leak the descriptor */
-    if (fp != NULL) /* O_CREAT's mode skips pre-existing jars: tighten those */
-      (void) fchmod(fd, HTS_PROTECT_FILE);
-#endif
 
     if (fp) {
       char *a = cookie->data;

src/htscache.h

@@ -66,11 +66,8 @@ htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr,
                   const char *fil, const char *save, char *location);
 htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, const char *save, char *location);
-/* Like cache_read, but also yields entries whose transfer broke; return_save
-   (optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name. */
-htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
-                                   const char *adr, const char *fil,
-                                   char *return_save);
+htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
+                                   const char *adr, const char *fil);
 htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr,
                     const char *fil, const char *save, char *location,
                     char *return_save, int readonly);
@@ -78,22 +75,14 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, htsblk * r);
 void cache_init(cache_back * cache, httrackp * opt);
 
-/* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
-typedef enum {
-  CACHE_RECONCILE_PROMOTE,     /* no new cache: promote the old generation */
-  CACHE_RECONCILE_INTERRUPTED, /* aborted run: keep the larger generation */
-  CACHE_RECONCILE_ROLLBACK     /* nothing transferred: restore the old one */
-} hts_cache_reconcile_mode;
-
-/* Reconcile the on-disk cache generations according to mode; a no-op when
-   the involved files are absent. */
-void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
+int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
+                    const char *str2, char *outbuff, int len);
+int cache_readdata(cache_back * cache, const char *str1, const char *str2,
+                   char **inbuff, int *len);
 
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);
-/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
-int cache_binput(char *adr, const char *end, char *s, int max);
 int cache_brint(char *adr, int *i);
 void cache_rint(FILE * fp, int *i);
 void cache_rLLint(FILE * fp, LLint * i);

src/htscache_selftest.c

@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
 #include "htszlib.h"
 
 #include <errno.h>
-#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -78,6 +77,14 @@ static void selftest_open_for_read(cache_back *cache, httrackp *opt) {
 }
 
 static void selftest_close(cache_back *cache) {
+  if (cache->dat != NULL) {
+    fclose(cache->dat);
+    cache->dat = NULL;
+  }
+  if (cache->ndx != NULL) {
+    fclose(cache->ndx);
+    cache->ndx = NULL;
+  }
   if (cache->zipOutput != NULL) {
     zipClose(cache->zipOutput,
              "Created by HTTrack Website Copier (cache self-test)");
@@ -314,7 +321,6 @@ typedef struct {
   size_t budget;  /**< bytes allowed through before writes start failing */
   int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
   int writes;     /**< zwrite call count, to detect re-entry into the stream */
-  int fail_once;  /**< recover (unlimited budget) after the first failure */
 } writefail_inject;
 
 /* zwrite that copies until the budget runs out, then fails with inj->fail_errno
@@ -329,8 +335,6 @@ static uLong selftest_failing_zwrite(voidpf opaque, voidpf stream,
     inj->budget -= (size_t) size;
     return (uLong) fwrite(buf, 1, (size_t) size, (FILE *) stream);
   }
-  if (inj->fail_once)
-    inj->budget = (size_t) -1; /* the backend recovers after this failure */
   errno = inj->fail_errno;
   return 0; /* short write -> the minizip op returns an error */
 }
@@ -369,50 +373,9 @@ static void writefail_store(httrackp *opt, cache_back *cache, const char *fil,
   freet(bodycopy);
 }
 
-/* Store an entry claiming a >2GB body; the degrade path never reads data. */
-static void writefail_store_oversized(httrackp *opt, cache_back *cache,
-                                      const char *fil, int is_write) {
-  htsblk r;
-  char locbuf[4];
-
-  hts_init_htsblk(&r);
-  r.statuscode = 200;
-  r.size = (LLint) INT_MAX + 1;
-  strcpybuff(r.msg, "OK");
-  strcpybuff(r.contenttype, "application/octet-stream");
-  locbuf[0] = '\0';
-  r.location = locbuf;
-  r.is_write = (short int) is_write;
-  cache_add(opt, cache, &r, "example.com", fil, "example.com/big.bin", 1, NULL);
-}
-
-/* Read back `entryname`: extra field (cached headers) and body. Returns the
-   body length, or -1 if the entry is absent or unreadable. */
-static int writefail_read_entry(const char *path, const char *entryname,
-                                char *extra, size_t extralen, char *body,
-                                size_t bodylen) {
-  unzFile z = unzOpen(path);
-  int n = -1;
-
-  if (z == NULL)
-    return -1;
-  if (unzLocateFile(z, entryname, 1) == UNZ_OK &&
-      unzOpenCurrentFile(z) == UNZ_OK) {
-    const int elen = unzGetLocalExtrafield(z, extra, (unsigned) (extralen - 1));
-
-    if (elen >= 0) {
-      extra[elen] = '\0';
-      n = unzReadCurrentFile(z, body, (unsigned) bodylen);
-    }
-    unzCloseCurrentFile(z);
-  }
-  unzClose(z);
-  return n;
-}
-
-/* Cache write-failure policy (#174/#219): fatal errno or a failure streak
-   stops the mirror (exit_xh=-1, no crash); isolated/oversized drops the entry.
- */
+/* #174/#219: a failing cache write used to crash via assertf(); it must instead
+   stop the mirror (exit_xh = -1) without crashing. Assert that, plus the cache
+   is flagged and a sibling write doesn't re-enter the broken stream. */
 int cache_write_failure_selftest(httrackp *opt, const char *dir) {
   int fail = 0;
   char path[HTS_URLMAXSIZE];
@@ -425,8 +388,9 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
   gen_body(body, body_len, 1 /* incompressible */);
   fconcat(path, sizeof(path), dir, "/wfail.zip");
 
-  /* phase 0: fatal errno (ENOSPC) aborts at once; phase 1: persistent EIO
-     drops entries until the streak caps out, then aborts. */
+  /* phase 0: fail on the body write, fatal errno (ENOSPC, the disk-full
+     branch). phase 1: fail on the open, non-fatal errno (EIO, dropped-share
+     branch). Both must abort the mirror. */
   for (phase = 0; phase < 2; phase++) {
     cache_back cache;
     writefail_inject inj;
@@ -435,7 +399,6 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     inj.budget = (phase == 0) ? 4096 : 0;
     inj.fail_errno = (phase == 0) ? ENOSPC : EIO;
     inj.writes = 0;
-    inj.fail_once = 0;
     memset(&cache, 0, sizeof(cache));
     cache.type = 1;
     cache.log = stderr;
@@ -449,25 +412,7 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     }
 
     opt->state.exit_xh = 0; /* clear; the failing write must set it to -1 */
-    if (phase == 0) {
-      writefail_store(opt, &cache, "/blob.bin", body, body_len);
-    } else {
-      /* the abort must land exactly on the 8th consecutive failure */
-      int i;
-
-      for (i = 0; i < 7; i++) {
-        char fil[32];
-
-        snprintf(fil, sizeof(fil), "/b%d.bin", i);
-        writefail_store(opt, &cache, fil, body, 16);
-      }
-      if (cache.zipWriteFailed) {
-        fprintf(stderr, "cache-writefail: phase 1: aborted before the "
-                        "8th consecutive failure\n");
-        fail++;
-      }
-      writefail_store(opt, &cache, "/b7.bin", body, 16);
-    }
+    writefail_store(opt, &cache, "/blob.bin", body, body_len);
     if (!cache.zipWriteFailed) {
       fprintf(stderr, "cache-writefail: phase %d: write error not caught\n",
               phase);
@@ -498,136 +443,6 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     }
   }
 
-  /* failures with successes in between reset the streak: never aborts */
-  {
-    cache_back cache;
-    writefail_inject inj;
-    int i;
-
-    inj.budget = (size_t) -1;
-    inj.fail_errno = EIO;
-    inj.writes = 0;
-    inj.fail_once = 0;
-    memset(&cache, 0, sizeof(cache));
-    cache.type = 1;
-    cache.log = stderr;
-    cache.errlog = stderr;
-    cache.hashtable = coucal_new(0);
-    cache.zipOutput = selftest_open_failing_zip(path, &inj);
-    opt->state.exit_xh = 0;
-
-    for (i = 0; i < 10; i++) {
-      char fil[32];
-
-      inj.budget = 0; /* this store fails */
-      snprintf(fil, sizeof(fil), "/s%d.bin", i);
-      writefail_store(opt, &cache, fil, body, 16);
-      inj.budget = (size_t) -1; /* this one succeeds and resets the streak */
-      snprintf(fil, sizeof(fil), "/ok%d.bin", i);
-      writefail_store(opt, &cache, fil, body, 16);
-    }
-    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
-      fprintf(stderr,
-              "cache-writefail: scattered: non-consecutive failures aborted "
-              "the mirror (flagged=%d, exit_xh=%d)\n",
-              (int) cache.zipWriteFailed, opt->state.exit_xh);
-      fail++;
-    }
-    zipClose(cache.zipOutput, NULL);
-    cache.zipOutput = NULL;
-  }
-
-  /* isolated failure: only that entry drops; a later sibling round-trips */
-  {
-    cache_back cache;
-    writefail_inject inj;
-    char extra[8192];
-    char rbody[64];
-    int n;
-
-    inj.budget = 4096;
-    inj.fail_errno = EIO;
-    inj.writes = 0;
-    inj.fail_once = 1;
-    memset(&cache, 0, sizeof(cache));
-    cache.type = 1;
-    cache.log = stderr;
-    cache.errlog = stderr;
-    cache.hashtable = coucal_new(0);
-    cache.zipOutput = selftest_open_failing_zip(path, &inj);
-    opt->state.exit_xh = 0;
-
-    writefail_store(opt, &cache, "/blob.bin", body, body_len);
-    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
-      fprintf(stderr,
-              "cache-writefail: skip: isolated failure aborted the mirror "
-              "(flagged=%d, exit_xh=%d)\n",
-              (int) cache.zipWriteFailed, opt->state.exit_xh);
-      fail++;
-    }
-    writefail_store(opt, &cache, "/blob2.bin", body, 16);
-    zipClose(cache.zipOutput, NULL);
-    cache.zipOutput = NULL;
-    n = writefail_read_entry(path, "http://example.com/blob2.bin", extra,
-                             sizeof(extra), rbody, sizeof(rbody));
-    if (n != 16 || memcmp(rbody, body, 16) != 0) {
-      fprintf(stderr,
-              "cache-writefail: skip: sibling entry lost after a skipped "
-              "entry (%d)\n",
-              n);
-      fail++;
-    }
-  }
-
-  /* >2GB bodies: in-memory drops the entry, on-disk degrades to headers-only */
-  {
-    cache_back cache;
-    writefail_inject inj;
-    char extra[8192];
-    char rbody[64];
-    int n;
-
-    inj.budget = (size_t) -1; /* no injected failure */
-    inj.fail_errno = 0;
-    inj.writes = 0;
-    inj.fail_once = 0;
-    memset(&cache, 0, sizeof(cache));
-    cache.type = 1;
-    cache.log = stderr;
-    cache.errlog = stderr;
-    cache.hashtable = coucal_new(0);
-    cache.zipOutput = selftest_open_failing_zip(path, &inj);
-    opt->state.exit_xh = 0;
-
-    writefail_store_oversized(opt, &cache, "/bigmem.bin", 0 /* in-memory */);
-    writefail_store_oversized(opt, &cache, "/bigdisk.bin", 1 /* on-disk */);
-    zipClose(cache.zipOutput, NULL);
-    cache.zipOutput = NULL;
-
-    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
-      fprintf(stderr,
-              "cache-writefail: oversize: mirror aborted (flagged=%d, "
-              "exit_xh=%d)\n",
-              (int) cache.zipWriteFailed, opt->state.exit_xh);
-      fail++;
-    }
-    if (writefail_read_entry(path, "http://example.com/bigmem.bin", extra,
-                             sizeof(extra), rbody, sizeof(rbody)) >= 0) {
-      fprintf(stderr,
-              "cache-writefail: oversize: in-memory entry was stored\n");
-      fail++;
-    }
-    n = writefail_read_entry(path, "http://example.com/bigdisk.bin", extra,
-                             sizeof(extra), rbody, sizeof(rbody));
-    if (n != 0 || strstr(extra, "X-In-Cache: 0") == NULL) {
-      fprintf(stderr,
-              "cache-writefail: oversize: on-disk entry not stored "
-              "headers-only (%d)\n",
-              n);
-      fail++;
-    }
-  }
-
   freet(body);
   return fail;
 }
@@ -901,583 +716,3 @@ int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
 
   return failures;
 }
-
-/* --- hts_cache_reconcile() policies -------------------------------------- */
-
-/* All reconcile inputs/outputs, wiped between cases. */
-static const char *const reconcile_files[] = {
-    "hts-cache/new.zip", "hts-cache/old.zip",   "hts-cache/new.dat",
-    "hts-cache/old.dat", "hts-cache/new.ndx",   "hts-cache/old.ndx",
-    "hts-cache/new.lst", "hts-cache/old.lst",   "hts-cache/new.txt",
-    "hts-cache/old.txt", "hts-in_progress.lock"};
-
-static char *reconcile_st_path(httrackp *opt, const char *name) {
-  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                 StringBuff(opt->path_log), name);
-}
-
-static void reconcile_wipe(httrackp *opt) {
-  size_t i;
-
-  for (i = 0; i < sizeof(reconcile_files) / sizeof(reconcile_files[0]); i++)
-    remove(reconcile_st_path(opt, reconcile_files[i]));
-}
-
-/* Create a filler file of exactly `size` bytes. */
-static void reconcile_put(httrackp *opt, const char *name, size_t size) {
-  FILE *const fp = fopen(reconcile_st_path(opt, name), "wb");
-  static const char filler[1024] = {'x'};
-
-  assertf(fp != NULL);
-  while (size > 0) {
-    const size_t n = size > sizeof(filler) ? sizeof(filler) : size;
-
-    assertf(fwrite(filler, 1, n, fp) == n);
-    size -= n;
-  }
-  fclose(fp);
-}
-
-/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
-static int reconcile_expect(httrackp *opt, const char *name, off_t size,
-                            const char *what) {
-  const off_t got = fsize(reconcile_st_path(opt, name));
-
-  if (got != size) {
-    fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
-            name, (int) got, (int) size);
-    return 1;
-  }
-  return 0;
-}
-
-int cache_reconcile_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-
-  /* around the interrupted-run thresholds (new < 32768, old > 65536) */
-  static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
-
-  golden_setup(opt, dir);
-#ifdef _WIN32
-  mkdir(reconcile_st_path(opt, "hts-cache"));
-#else
-  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
-#endif
-
-  /* PROMOTE: a zip old generation replaces a missing new one */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "promote-zip");
-  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "promote-zip");
-
-  /* PROMOTE: an existing new.zip is left alone */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", TINY, "promote-zip-noop");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
-
-  /* PROMOTE: the removed pre-3.31 legacy pair is left alone */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/old.dat", SOLID);
-  reconcile_put(opt, "hts-cache/old.ndx", TINY);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", TINY, "promote-dat");
-
-  /* INTERRUPTED: no lock file, no action */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-nolock");
-
-  /* INTERRUPTED: an absent new.zip must NOT promote old.zip (fsize(-1) would
-     spuriously pass "< TINY"); leave the solid old generation for ROLLBACK */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-in_progress.lock", 0);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", -1, "interrupted-nonew");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "interrupted-nonew");
-
-  /* INTERRUPTED: stalled tiny new.zip loses to a solid old.zip (was dead for
-     zip caches: the arm was gated on a legacy new.dat) */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-in_progress.lock", 0);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", SOLID, "interrupted-zip");
-  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "interrupted-zip");
-
-  /* INTERRUPTED: old below the confidence threshold, keep new */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-in_progress.lock", 0);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  reconcile_put(opt, "hts-cache/old.zip", MID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-smallold");
-
-  /* INTERRUPTED: new big enough to trust, keep it */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-in_progress.lock", 0);
-  reconcile_put(opt, "hts-cache/new.zip", MID);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
-
-  /* INTERRUPTED: the removed pre-3.31 legacy pair is left alone */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-in_progress.lock", 0);
-  reconcile_put(opt, "hts-cache/new.dat", TINY);
-  reconcile_put(opt, "hts-cache/new.ndx", TINY);
-  reconcile_put(opt, "hts-cache/old.dat", SOLID);
-  reconcile_put(opt, "hts-cache/old.ndx", MID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.dat", TINY, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.dat", SOLID, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.ndx", MID, "interrupted-dat");
-
-  /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
-     its only good generation here) */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "rollback-zip");
-  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "rollback-zip");
-
-  /* ROLLBACK: sidecars are restored regardless of format */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.lst", TINY);
-  reconcile_put(opt, "hts-cache/old.lst", MID);
-  reconcile_put(opt, "hts-cache/old.txt", MID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
-  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
-
-  /* ROLLBACK: sidecars restored, the removed pre-3.31 pair left alone */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.dat", TINY);
-  reconcile_put(opt, "hts-cache/old.dat", SOLID);
-  reconcile_put(opt, "hts-cache/old.ndx", MID);
-  reconcile_put(opt, "hts-cache/old.lst", MID);
-  reconcile_put(opt, "hts-cache/old.txt", MID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", TINY, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", MID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
-
-  /* ROLLBACK: nothing to restore, the new generation stays */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.zip", TINY, "rollback-noop");
-
-  reconcile_wipe(opt);
-  return failures;
-}
-
-/* The pre-3.31 .dat/.ndx import was removed: cache_init must refuse the
-   legacy pair, leave the files in place, and not flag an update run. */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-  int variant;
-
-  selftest_tag = "cache-legacy";
-  golden_setup(opt, dir);
-#ifdef _WIN32
-  mkdir(reconcile_st_path(opt, "hts-cache"));
-#else
-  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
-#endif
-
-  /* variant 0: old.dat/old.ndx (--update layout); 1: new.dat/new.ndx (ro) */
-  for (variant = 0; variant < 2; variant++) {
-    cache_back cache;
-    const char *const dat =
-        variant == 0 ? "hts-cache/old.dat" : "hts-cache/new.dat";
-    const char *const ndx =
-        variant == 0 ? "hts-cache/old.ndx" : "hts-cache/new.ndx";
-
-    reconcile_wipe(opt);
-    reconcile_put(opt, dat, 4096);
-    reconcile_put(opt, ndx, 4096);
-    opt->is_update = 0;
-    selftest_open(&cache, opt, /*ro*/ variant == 1);
-    if (cache_readable(&cache)) {
-      printf("cache-legacy: variant %d imported a removed format\n", variant);
-      failures++;
-    }
-    if (coucal_nitems(cache.hashtable) != 0) { /* no phantom index entries */
-      printf("cache-legacy: variant %d imported index entries\n", variant);
-      failures++;
-    }
-    if (opt->is_update) {
-      printf("cache-legacy: variant %d flagged an update\n", variant);
-      failures++;
-    }
-    if (fsize(reconcile_st_path(opt, dat)) != 4096 ||
-        fsize(reconcile_st_path(opt, ndx)) != 4096) {
-      printf("cache-legacy: variant %d touched the legacy files\n", variant);
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  /* a healthy zip must win over stray legacy files, with no refusal */
-  {
-    cache_back cache;
-    const char *const body = "<html>zip wins</html>";
-
-    reconcile_wipe(opt);
-    selftest_open_for_write(&cache, opt);
-    store_entry(opt, &cache, "example.com", "/", "example.com/index.html", 200,
-                "OK", "text/html", "utf-8", "", "", "", "", body, strlen(body));
-    selftest_close(&cache);
-    reconcile_put(opt, "hts-cache/old.ndx", 4096);
-    reconcile_put(opt, "hts-cache/old.dat", 4096);
-    selftest_open_for_read(&cache, opt);
-    if (!cache_readable(&cache)) {
-      printf("cache-legacy: stray legacy files shadowed the zip cache\n");
-      failures++;
-    }
-    failures +=
-        check_entry(opt, &cache, "example.com", "/", 200, "OK", "text/html",
-                    "utf-8", "", "", "", "", body, strlen(body));
-    if (fsize(reconcile_st_path(opt, "hts-cache/old.ndx")) != 4096 ||
-        fsize(reconcile_st_path(opt, "hts-cache/old.dat")) != 4096) {
-      printf("cache-legacy: zip-wins pass touched the legacy files\n");
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  reconcile_wipe(opt);
-  return failures;
-}
-
-/* --- read-side corruption injection --------------------------------------- */
-
-/* canary read back intact after each corruption; victim gets the byte surgery
- */
-#define CORRUPT_ADR "corrupt.example.com"
-static char corrupt_body_a[33 + 1];
-static char corrupt_body_b[44 + 1];
-
-/* Write a fresh two-entry cache: /canary.html then /victim.html. */
-static void corrupt_build(httrackp *opt) {
-  cache_back cache;
-
-  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
-  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
-  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
-  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  selftest_open_for_write(&cache, opt);
-  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
-              strlen(corrupt_body_a));
-  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
-              strlen(corrupt_body_b));
-  selftest_close(&cache);
-}
-
-/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
-   is later overwritten with a forged oversized X-Size (same byte length). */
-static void corrupt_build_etag(httrackp *opt) {
-  cache_back cache;
-
-  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
-  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
-  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
-  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  selftest_open_for_write(&cache, opt);
-  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
-              strlen(corrupt_body_a));
-  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
-              "OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
-              corrupt_body_b, strlen(corrupt_body_b));
-  selftest_close(&cache);
-}
-
-/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
-   body on disk): the shape every non-html file is stored with. */
-static void corrupt_build_disk(httrackp *opt) {
-  cache_back cache;
-  htsblk w;
-  char locw[4];
-  char BIGSTK save[HTS_URLMAXSIZE * 2];
-  char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
-  char *path;
-  FILE *fp;
-
-  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
-  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
-  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
-          CORRUPT_ADR "/victim.bin");
-  selftest_open_for_write(&cache, opt);
-  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
-              strlen(corrupt_body_a));
-  hts_init_htsblk(&w);
-  w.statuscode = 200;
-  w.size = (LLint) sizeof(corrupt_body_b) - 1;
-  strcpybuff(w.msg, "OK");
-  strcpybuff(w.contenttype, "application/octet-stream");
-  strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
-  locw[0] = '\0';
-  w.location = locw;
-  w.is_write = 0;
-  cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
-            0 /* all_in_cache */, NULL);
-  selftest_close(&cache);
-  /* the reader only checks this file exists; it never reads it here */
-  path = fconv(catbuff, sizeof(catbuff), save);
-  (void) structcheck(path);
-  fp = FOPEN(path, "wb");
-  assertf(fp != NULL);
-  fclose(fp);
-}
-
-/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
-static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
-                          const char *rep, size_t nth, size_t total) {
-  LLint fsz = 0;
-  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
-  const size_t n = (size_t) fsz;
-  size_t k, hits = 0, at = 0;
-  FILE *fp;
-
-  assertf(data != NULL);
-  for (k = 0; k + patlen <= n; k++) {
-    if (memcmp(data + k, pat, patlen) == 0) {
-      hits++;
-      if (hits == nth)
-        at = k;
-    }
-  }
-  assertf(hits == total);
-  memcpy(data + at, rep, patlen);
-  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
-  assertf(fp != NULL);
-  assertf(fwrite(data, 1, n, fp) == n);
-  fclose(fp);
-  freet(data);
-}
-
-/* Garbage the first bytes of the victim's deflated data (2nd local header). */
-static void corrupt_victim_body(httrackp *opt) {
-  LLint fsz = 0;
-  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
-  const size_t n = (size_t) fsz;
-  size_t k, hits = 0, off = 0;
-  FILE *fp;
-
-  assertf(data != NULL);
-  for (k = 0; k + 4 <= n; k++) {
-    if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
-      const size_t namelen =
-          (unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
-      const size_t extralen =
-          (unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
-
-      off = k + 30 + namelen + extralen;
-    }
-  }
-  assertf(hits == 2);
-  assertf(off != 0 && off + 4 <= n);
-  memset(data + off, 0xFF, 4);
-  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
-  assertf(fp != NULL);
-  assertf(fwrite(data, 1, n, fp) == n);
-  fclose(fp);
-  freet(data);
-}
-
-/* Read the corrupt /victim.html and, in the SAME read session, the intact
-   /canary.html: the victim must be rejected (wantmsg pins which path) and the
-   canary must still decode byte-exact, proving one bad entry never taints a
-   sibling read. */
-static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
-                                     const char *wantmsg, const char *what) {
-  cache_back cache;
-  htsblk v, c;
-  char BIGSTK lv[HTS_URLMAXSIZE * 2];
-  char BIGSTK lc[HTS_URLMAXSIZE * 2];
-  int fail = 0;
-
-  selftest_open_for_read(&cache, opt);
-  lv[0] = lc[0] = '\0';
-  v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
-  if (v.statuscode != STATUSCODE_INVALID) {
-    fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
-            selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
-    fail++;
-  }
-  if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
-    fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
-            selftest_tag, what, v.msg, wantmsg);
-    fail++;
-  }
-  c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
-  if (c.statuscode != 200 || c.adr == NULL ||
-      c.size != (LLint) strlen(corrupt_body_a) ||
-      memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
-    fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
-            c.statuscode);
-    fail++;
-  }
-  if (v.adr != NULL)
-    freet(v.adr);
-  if (c.adr != NULL)
-    freet(c.adr);
-  selftest_close(&cache);
-  return fail;
-}
-
-static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
-                                 const char *what) {
-  return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
-}
-
-/* Headers-only probe of the disk victim: must parse OK with the size kept. */
-static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
-                                      const char *what) {
-  cache_back cache;
-  htsblk v;
-  char BIGSTK lv[HTS_URLMAXSIZE * 2];
-  int fail = 0;
-
-  selftest_open_for_read(&cache, opt);
-  lv[0] = '\0';
-  v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
-  if (v.statuscode != 200 || v.size != wantsize) {
-    fprintf(stderr,
-            "%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
-            selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
-    fail++;
-  }
-  if (v.adr != NULL)
-    freet(v.adr);
-  selftest_close(&cache);
-  return fail;
-}
-
-/* One zip corruption case: build, patch, then check victim+canary in-session.
- */
-static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
-                            size_t nth, size_t total, const char *wantmsg,
-                            const char *what) {
-  corrupt_build(opt);
-  corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
-  return corrupt_expect_victim(opt, wantmsg, what);
-}
-
-int cache_corruption_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-
-  selftest_tag = "cache-corrupt";
-  golden_setup(opt, dir);
-
-  failures +=
-      corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
-                       "Cache Read Error : Read Data", "oversized X-Size");
-  failures +=
-      corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
-                       "Cache Read Error : Bad Size", "negative X-Size");
-  /* both entries carry the line; the victim's is the second */
-  failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
-                               "Previous cache file not found (empty filename)",
-                               "blanked X-In-Cache");
-  /* smashed local file header: the entry is dropped at index load */
-  failures +=
-      corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
-                       "File Cache Entry Not Found", "smashed local header");
-
-  corrupt_build(opt);
-  corrupt_victim_body(opt);
-  failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
-                                    "garbled deflate stream");
-
-  /* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
-     truncates negative in the (int) cast the malloc uses: a wraparound alloc.
-     cache_add asserts size fits an int, so such a value only reaches the reader
-     from a corrupt/foreign cache; inject it by overwriting the victim's long
-     Etag line with a same-length forged X-Size line (the parser keeps the last
-     X-Size it sees), keeping the zip byte-length and offsets intact. */
-  corrupt_build_etag(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: 2147483648AAAAAAAA", 1, 1);
-  failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
-                                    "X-Size above INT_MAX");
-
-  /* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
-     is how every >2GB non-html file is stored. It must survive a header probe
-     (or every update re-fetches the file); an in-memory read still rejects. */
-  corrupt_build_disk(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: 2147483648AAAAAAAA", 1, 1);
-  failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
-                                         "headers-only X-Size above INT_MAX");
-  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
-                                        "Cache Read Error : Bad Size",
-                                        "in-memory X-Size above INT_MAX");
-
-  /* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
-  corrupt_build_disk(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: 2147483647AAAAAAAA", 1, 1);
-  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
-                                        "Cache Read Error : Bad Size",
-                                        "in-memory X-Size at INT_MAX");
-
-  /* the negative check must stay global, headers-only included */
-  corrupt_build_disk(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: -2147483648AAAAAAA", 1, 1);
-  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
-                                        "Cache Read Error : Bad Size",
-                                        "headers-only negative X-Size");
-
-  return failures;
-}

src/htscache_selftest.h

@@ -52,23 +52,10 @@ int cache_selftests(httrackp *opt, const char *dir);
    committed file, never by the test). Returns the failed-check count. */
 int cache_golden_selftest(httrackp *opt, const char *dir, int regen);
 
-/* Cache write-failure policy (#174/#219): abort on fatal errno or a streak,
-   drop just the entry otherwise. Returns the failed-check count. */
+/* #174/#219: assert a failing cache write aborts the mirror cleanly instead of
+   crashing. Returns the failed-check count. */
 int cache_write_failure_selftest(httrackp *opt, const char *dir);
 
-/* Exercise the hts_cache_reconcile() generation policies on file fixtures
-   under <dir>. Returns the failed-check count. */
-int cache_reconcile_selftest(httrackp *opt, const char *dir);
-
-/* Verify cache_init refuses a pre-3.31 .dat/.ndx cache without touching it.
-   Returns the number of failed checks (0 = pass). */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir);
-
-/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
-   under <dir> and assert every case degrades to STATUSCODE_INVALID without
-   tainting a sibling entry. */
-int cache_corruption_selftest(httrackp *opt, const char *dir);
-
 #endif
 
 #endif

src/htscatchurl.c

@@ -64,7 +64,7 @@ Please visit our Website: http://www.httrack.com
 // catch_url_init(&port,&return_host);
 HTSEXT_API T_SOC catch_url_init_std(int *port_prox, char *adr_prox) {
   T_SOC soc;
-  int try_to_listen_to[] = {8080, 3128, 80, 81, 82, 8081, 3129, 0, -1};
+  int try_to_listen_to[] = { 8080, 3128, 80, 81, 82, 8081, 3129, 31337, 0, -1 };
   int i = 0;
 
   do {
@@ -175,9 +175,7 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
       //
       socinput(soc, line, 1000);
       if (strnotempty(line)) {
-        /* widths bound the caller buffers: method[32], url[HTS_URLMAXSIZE*2],
-           protocol[256] */
-        if (sscanf(line, "%31s %2047s %255s", method, url, protocol) == 3) {
+        if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
           lien_adrfil af;
 
           // méthode en majuscule
@@ -197,6 +195,7 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
             htsblk blkretour;
 
             hts_init_htsblk(&blkretour);
+            //memset(&blkretour, 0, sizeof(htsblk));    // effacer
             blkretour.location = loc;   // si non nul, contiendra l'adresse véritable en cas de moved xx
             // Lire en têtes restants
             sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
@@ -206,8 +205,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
               strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
               strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
             }
-            // CR/LF final de l'en tête inutile car déja placé via la ligne vide
-            // juste au dessus
+            // CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
+            //strcatbuff(data,"\r\n");
             if (blkretour.totalsize > 0) {
               int len = (int) min(blkretour.totalsize, 32000);
               int pos = (int) strlen(data);

src/htscharset.c

@@ -84,7 +84,7 @@ static int hts_equalsAlphanum(const char *a, const char *b) {
 
 /* Copy the memory region [s .. s + size - 1 ] as a \0-terminated string. */
 static char *hts_stringMemCopy(const char *s, size_t size) {
-  char *dest = malloct(size + 1);
+  char *dest = malloc(size + 1);
 
   if (dest != NULL) {
     memcpy(dest, s, size);
@@ -549,6 +549,42 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
 
 #endif
 
+HTS_STATIC char *hts_getCharsetFromContentType(const char *mime) {
+  /* text/html; charset=utf-8 */
+  const char *const charset = "charset";
+  char *pos = strstr(mime, charset);
+
+  if (pos != NULL) {
+    /* Skip spaces */
+    int eq = 0;
+
+    for(pos += strlen(charset);
+        *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\''; pos++) {
+      if (*pos == '=') {
+        eq = 1;
+      }
+    }
+    if (eq == 1) {
+      int len;
+
+      for(len = 0;
+          pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'';
+          pos++) ;
+      if (len != 0) {
+        char *const s = malloc(len + 1);
+        int i;
+
+        for(i = 0; i < len; i++) {
+          s[i] = pos[i];
+        }
+        s[len] = '\0';
+        return s;
+      }
+    }
+  }
+  return NULL;
+}
+
 #ifdef _WIN32
 #define strcasecmp(a,b) stricmp(a,b)
 #define strncasecmp(a,b,n) strnicmp(a,b,n)
@@ -608,106 +644,58 @@ int hts_isCharsetUTF8(const char *charset) {
          || strcasecmp(charset, "utf8") == 0 );
 }
 
-/* Extract X from a "text/html; charset=X" content= attribute value. */
-static char *charset_from_content(const char *s, size_t len) {
-  size_t i;
-
-  for (i = 0; i + 7 < len; i++) {
-    if ((i == 0 || is_space(s[i - 1]) || s[i - 1] == ';') &&
-        strncasecmp(&s[i], "charset", 7) == 0 && is_space_or_equal(s[i + 7])) {
-      size_t j, val;
-
-      for (j = i + 7; j < len && is_space_or_equal_or_quote(s[j]); j++)
-        ;
-      for (val = j; j < len && s[j] != '"' && s[j] != '\'' && s[j] != ';' &&
-                    !is_space(s[j]);
-           j++)
-        ;
-      if (j != val) {
-        return hts_stringMemCopy(&s[val], j - val);
-      }
-    }
-  }
-  return NULL;
-}
-
 char *hts_getCharsetFromMeta(const char *html, size_t size) {
-  size_t i;
+  int i;
 
-  /* HTML5 <meta charset=X> or legacy
-     <meta http-equiv="Content-Type" content="text/html; charset=X">,
-     attributes in any order; first resolvable tag wins. */
-  for (i = 0; i + 6 < size; i++) {
-    size_t j;
-    const char *equiv = NULL, *content = NULL;
-    size_t equiv_len = 0, content_len = 0;
+  /* <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8" > */
+  for(i = 0; i < size; i++) {
+    if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0
+        && is_space(html[i + 5])) {
+      /* Skip spaces */
+      for(i += 5; is_space(html[i]); i++) ;
+      if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0
+          && is_space_or_equal(html[i + 10])) {
+        for(i += 10; is_space_or_equal_or_quote(html[i]); i++) ;
+        if (strncasecmp(&html[i], "CONTENT-TYPE", 12) == 0) {
+          for(i += 12; is_space_or_equal_or_quote(html[i]); i++) ;
+          if (strncasecmp(&html[i], "CONTENT", 7) == 0
+              && is_space_or_equal(html[i + 7])) {
+            for(i += 7; is_space_or_equal_or_quote(html[i]); i++) ;
+            /* Skip content-type */
+            for(;
+                i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'';
+                i++) ;
+            /* Expect charset attribute here */
+            if (html[i] == ';') {
+              for(i++; is_space(html[i]); i++) ;
+              /* Look for charset */
+              if (strncasecmp(&html[i], "charset", 7) == 0
+                  && is_space_or_equal(html[i + 7])) {
+                int len;
 
-    if (html[i] != '<' || strncasecmp(&html[i + 1], "meta", 4) != 0 ||
-        !is_space(html[i + 5])) {
-      continue;
-    }
-    /* Attribute scan, strictly bounded by size. */
-    for (j = i + 6; j < size && html[j] != '>';) {
-      size_t name, name_len, val = 0, val_len = 0;
+                for(i += 7; is_space_or_equal(html[i]) || html[i] == '\'';
+                    i++) ;
+                /* Charset */
+                for(len = 0;
+                    i + len < size && html[i + len] != '"'
+                    && html[i + len] != '\'' && html[i + len] != ' '; len++) ;
+                /* No error ? */
+                if (len != 0 && i < size) {
+                  char *const s = malloc(len + 1);
+                  int j;
 
-      if (is_space(html[j]) || html[j] == '/') {
-        j++;
-        continue;
-      }
-      for (name = j; j < size && html[j] != '=' && html[j] != '>' &&
-                     html[j] != '/' && !is_space(html[j]);
-           j++)
-        ;
-      name_len = j - name;
-      for (; j < size && is_space(html[j]); j++)
-        ;
-      if (j < size && html[j] == '=') {
-        for (j++; j < size && is_space(html[j]); j++)
-          ;
-        if (j < size && (html[j] == '"' || html[j] == '\'')) {
-          const char quote = html[j++];
-
-          for (val = j; j < size && html[j] != quote; j++)
-            ;
-          if (j >= size) /* unterminated quote: drop the tag */
-            break;
-          val_len = j++ - val;
-        } else {
-          /* unquoted: ends at whitespace or '>' only (HTML5 prescan); '/'
-             belongs to the value except as the tail of a self-close */
-          for (val = j; j < size && !is_space(html[j]) && html[j] != '>'; j++)
-            ;
-          val_len = j - val;
-          if (val_len != 0 && j < size && html[j] == '>' &&
-              html[val + val_len - 1] == '/') {
-            val_len--;
+                  for(j = 0; j < len; j++) {
+                    s[j] = html[i + j];
+                  }
+                  s[len] = '\0';
+                  return s;
+                }
+              }
+            }
           }
         }
       }
-      /* first occurrence of each attribute wins, as in browsers */
-      if (val_len != 0) {
-        if (name_len == 7 && strncasecmp(&html[name], "charset", 7) == 0) {
-          return hts_stringMemCopy(&html[val], val_len);
-        } else if (equiv == NULL && name_len == 10 &&
-                   strncasecmp(&html[name], "http-equiv", 10) == 0) {
-          equiv = &html[val];
-          equiv_len = val_len;
-        } else if (content == NULL && name_len == 7 &&
-                   strncasecmp(&html[name], "content", 7) == 0) {
-          content = &html[val];
-          content_len = val_len;
-        }
-      }
     }
-    if (equiv_len == 12 && strncasecmp(equiv, "content-type", 12) == 0 &&
-        content != NULL) {
-      char *const s = charset_from_content(content, content_len);
-
-      if (s != NULL) {
-        return s;
-      }
-    }
-    i = j;
   }
   return NULL;
 }
@@ -853,6 +841,71 @@ static unsigned int nlz8(unsigned char x) {
     }                                           \
   } while(0)
 
+/* Sample. */
+#if 0
+
+int main(int argc, char **argv) {
+  int i;
+  int hex = 0;
+  
+#define READ_INT(DEST)                                  \
+  ( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1)      \
+    || (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
+  
+  for(i = 1 ; i < argc ; i++) {
+    unsigned int uc, from, to;
+    
+    if (strcmp(argv[i], "--hex") == 0) {
+      hex = 1;
+    }
+    else if (strcmp(argv[i], "--decimal") == 0) {
+      hex = 0;
+    }
+    else if (strcmp(argv[i], "--decode") == 0) {
+#define RD fgetc_unlocked(stdin)
+#define WR(C) do {                              \
+        if (C != -1) {                          \
+          printf("%04x\n", C);                  \
+        } else if (!feof(stdin)) {              \
+          fprintf(stderr, "read error\n");      \
+          exit(EXIT_FAILURE);                   \
+        }                                       \
+      } while(0)
+      while(!feof(stdin)) {
+        READ_UNICODE(RD, WR);
+      }
+#undef RD
+#undef WR
+    }
+    else if (strcmp(argv[i], "-range") == 0
+             && i + 2 < argc
+             && (++i, 1)
+             && READ_INT(from)
+             && (++i, 1)
+             && READ_INT(to)
+             ) {
+      unsigned int i;
+      for(i = from ; i < to ; i++) {
+#define EM(C) fputc_unlocked(C, stdout)
+        EMIT_UNICODE(i, EM);
+#undef EM
+      }
+    }
+    else if (READ_INT(uc)) {
+#define EM(C) fputc_unlocked(C, stdout)
+      EMIT_UNICODE(uc, EM);
+#undef EM
+    }
+    else {
+      return EXIT_FAILURE;
+    }
+  }
+  
+  return EXIT_SUCCESS;
+}
+
+#endif
+
 /* IDNA helpers. */
 #undef ADD_BYTE
 #undef INCREASE_CAPA
@@ -922,19 +975,19 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
                 /* Reader: can read bytes up to j */
 #define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
 
-                /* Writer: a malformed sequence abandons the encode (NULL) */
-#define WR(C)                                                                  \
-  do {                                                                         \
-    if ((C) != -1) {                                                           \
-      /* copy character */                                                     \
-      assertf(segOutputSize < segSize);                                        \
-      segInt[segOutputSize++] = (C);                                           \
-    } else {                                                                   \
-      freet(segInt);                                                           \
-      FREE_BUFFER();                                                           \
-      return NULL;                                                             \
-    }                                                                          \
-  } while (0)
+                /* Writer: upon error, return FFFD (replacement character) */
+#define WR(C) do { \
+  if ((C) != -1) { \
+    /* copy character */ \
+    assertf(segOutputSize < segSize); \
+    segInt[segOutputSize++] = (C); \
+  } \
+  /* In case of error, abort. */ \
+  else { \
+    FREE_BUFFER(); \
+    return NULL; \
+  } \
+} while(0)
 
                 /* Read/Write Unicode character. */
                 READ_UNICODE(RD, WR);
@@ -1152,45 +1205,17 @@ int hts_isStringUTF8(const char *s, size_t size) {
   const unsigned char *const data = (const unsigned char*) s;
   size_t i;
 
-  /* Strict RFC 3629: reject overlongs, surrogates, > U+10FFFF, 5/6-byte. */
   for(i = 0 ; i < size ; ) {
-    const unsigned char c = data[i];
-    size_t len, k;
-    hts_UCS4 uc, min;
+    /* Reader: can read bytes up to j */
+#define RD ( i < size ? data[i++] : -1 )
 
-    if (c < 0x80) {
-      i++;
-      continue;
-    } else if (c >= 0xc2 && c <= 0xdf) {
-      len = 2;
-      min = 0x80;
-      uc = c & 0x1f;
-    } else if (c >= 0xe0 && c <= 0xef) {
-      len = 3;
-      min = 0x800;
-      uc = c & 0x0f;
-    } else if (c >= 0xf0 && c <= 0xf4) {
-      len = 4;
-      min = 0x10000;
-      uc = c & 0x07;
-    } else { /* continuation byte, overlong C0/C1, or F5..FF lead */
-      return 0;
-    }
-    if (size - i < len) {
-      return 0;
-    }
-    for (k = 1; k < len; k++) {
-      const unsigned char cc = data[i + k];
+    /* Writer: upon error, return FFFD (replacement character) */
+#define WR(C) if ((C) == -1) { return 0; }
 
-      if ((cc & 0xc0) != 0x80) {
-        return 0;
-      }
-      uc = (uc << 6) | (cc & 0x3f);
-    }
-    if (uc < min || uc > 0x10FFFF || (uc >= 0xD800 && uc <= 0xDFFF)) {
-      return 0;
-    }
-    i += len;
+    /* Read Unicode character. */
+    READ_UNICODE(RD, WR);
+#undef RD
+#undef WR
   }
 
   return 1;

src/htscharset.h

@@ -76,8 +76,7 @@ extern char *hts_convertStringIDNAToUTF8(const char *s, size_t size);
 extern int hts_isStringIDNA(const char *s, size_t size);
 
 /**
- * Extract the <meta> charset from the HTML buffer "html" (HTML5 charset= or
- * legacy http-equiv form). Returns a malloc'ed string, or NULL if none.
+ * Extract the charset from the HTML buffer "html"
  **/
 extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 
@@ -87,8 +86,7 @@ extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 extern int hts_isStringAscii(const char *s, size_t size);
 
 /**
- * Is the given string valid UTF-8 ? Strict RFC 3629: overlong forms,
- * surrogates, codepoints above U+10FFFF and 5/6-byte sequences are rejected.
+ * Is the given string an UTF-8 string ?
  **/
 extern int hts_isStringUTF8(const char *s, size_t size);
 

src/htsconfig.h

@@ -54,6 +54,10 @@ Please visit our Website: http://www.httrack.com
 // extension par défaut pour fichiers n'en ayant pas
 #define DEFAULT_EXT ".html"
 #define DEFAULT_EXT_SHORT ".htm"
+// #define DEFAULT_BIN_EXT   ".bin"
+// #define DEFAULT_BIN_EXT_SHORT ".bin"
+// #define DEFAULT_EXT       ".txt"
+// #define DEFAULT_EXT_SHORT ".txt"
 
 // éviter les /nul, /con..
 #define HTS_OVERRIDE_DOS_FOLDERS 1
@@ -76,6 +80,13 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
+// gérer une table de hachage?
+// REMOVED
+// #define HTS_HASH 1
+
+// fast cache (build hash table)
+#define HTS_FAST_CACHE 1
+
 // le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
 // valide)
 #define GT_ENDS_COMMENT 1
@@ -86,10 +97,27 @@ Please visit our Website: http://www.httrack.com
 // always transform a '//' into a sigle '/'
 #define HTS_STRIP_DOUBLE_SLASH 0
 
+// case-sensitive pour les dossiers et fichiers (0/1)
+// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est
+// pas très utile..
+// ..et pas bcp respecté]
+// REMOVED
+// #define HTS_CASSE 0
+
+// Un fichier ayant une taille différente du content-length doit il être annulé?
+// SEE opt.tolerant and opt.http10
+// #define HTS_CL_IS_FATAL 0
+
 // une erreur supprime le fichier sur disque
 // (non fixé pour cause de retry)
 #define HTS_REMOVE_BAD_FILES 0
 
+// en cas de Range: xx- donnant un Content-length: xx
+// alors skipper le fichier, considéré comme transmis
+// #define HTS_SKIP_FULL_RANGE 1
+
+// nombre max de filtres que l'utilisateur peut fixer
+// #define HTS_FILTERSMAX 10000
 #define HTS_FILTERSINC 1000
 
 // connect non bloquant? (poll sur write)

src/htscore.h

@@ -193,7 +193,8 @@ struct cache_back {
   /* */
   int type;
   int ro;                   /**< read-only: no new cache is written */
-
+  FILE *dat, *ndx, *olddat; /**< new data, new index, old data files */
+  char *use;                /**< in-memory list of cached adr+fil keys */
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
   char lastmodified[256];
@@ -215,7 +216,6 @@ struct cache_back {
   int zipEntriesCapa;
   hts_boolean
       zipWriteFailed; /**< a cache write failed; stop touching the stream */
-  int zipWriteFailures; /**< consecutive entry write failures; reset on store */
 };
 
 #ifndef HTS_DEF_FWSTRUCT_hash_struct
@@ -234,12 +234,8 @@ struct hash_struct {
   coucal adrfil;
   /* former address+path -> link index (renamed/moved entries) */
   coucal former_adrfil;
-  /* effective urlhack sub-flags: www.==host / // collapse / query-arg sort */
-  hts_boolean norm_host;
-  hts_boolean norm_slash;
-  hts_boolean norm_query;
-  /* query-strip keys (not owned); set from opt->strip_query at hash_init */
-  const char *strip_query;
+  /* scratch buffers reused across lookups (not reentrant) */
+  int normalized;
   char normfil[HTS_URLMAXSIZE * 2];
   char normfil2[HTS_URLMAXSIZE * 2];
   char catbuff[CATBUFF_SIZE];
@@ -287,12 +283,12 @@ struct filecreate_params {
 
 /* True if a new cache is being written (plain or zip backend). */
 HTS_STATIC int cache_writable(cache_back * cache) {
-  return (cache != NULL && cache->zipOutput != NULL);
+  return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL));
 }
 
 /* True if an old cache is available to read (plain or zip backend). */
 HTS_STATIC int cache_readable(cache_back * cache) {
-  return (cache != NULL && cache->zipInput != NULL);
+  return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL));
 }
 
 #endif
@@ -362,42 +358,12 @@ void usercommand(httrackp * opt, int exe, const char *cmd, const char *file,
 
 void usercommand_exe(const char *cmd, const char *file);
 
-// Finish the makeindex index.html (footer + refresh meta), run usercommand.
-// Updates *makeindex_done/*makeindex_fp in place; adr/fil are the mode strings.
-void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
-                          FILE **makeindex_fp, int makeindex_links,
-                          const char *makeindex_firstlink,
-                          const char *template_footer, const char *adr,
-                          const char *fil);
-
-// Flush ht_buff[0..ht_len] to save on disk (skip if MD5 unchanged); *fp
-// closed+NULLed on write. Precondition: ht_len>0.
-void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
-                          FILE **fp, const char *ht_buff, size_t ht_len,
-                          const char *adr, const char *fil, const char *save);
-
 int filters_init(char ***ptrfilters, int maxfilter, int filterinc);
 
 int fspc(httrackp * opt, FILE * fp, const char *type);
 
 char *next_token(char *p, int flag);
 
-/* Like fil_normalized(), but first drops query keys in STRIP (comma-separated,
-   "*" = all); STRIP NULL/empty behaves exactly like fil_normalized(). */
-char *fil_normalized_filtered(const char *source, char *dest,
-                              const char *strip);
-
-/* As fil_normalized_filtered(), but DO_SLASH/DO_QUERY gate the // collapse and
-   the query-argument sort independently (the urlhack sub-flags). */
-char *fil_normalized_filtered_ex(const char *source, char *dest,
-                                 const char *strip, int do_slash, int do_query);
-
-/* For URL ADR/FIL, return (in DEST) the comma keylist to strip from the
-   '\n'-separated "[pattern=]keys" RULES (patterns matched on host/path via
-   strjoker, last wins); NULL if none match. Feeds fil_normalized_filtered(). */
-const char *hts_query_strip_keys(const char *rules, const char *adr,
-                                 const char *fil, char *dest, size_t destsize);
-
 /* Read a whole file into a freshly malloc'd, NUL-terminated buffer; the caller
    owns it and must release it with freet(). Return NULL on missing/unreadable
    file (readfile_or substitutes defaultdata instead). The byte content is NOT
@@ -411,6 +377,10 @@ char *readfile_utf8(const char *fil);
 
 char *readfile_or(const char *fil, const char *defaultdata);
 
+#if 0
+void check_rate(TStamp stat_timestart, int maxrate);
+#endif
+
 /* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
    Not thread-safe; call from the single crawl loop. */
 
@@ -428,19 +398,6 @@ int back_pluggable_sockets(struct_back * sback, httrackp * opt);
 
 int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);
 
-/* One engine-loop tick: refresh the transfer stats and run the loop callback
-   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
-hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr);
-
-/* Wait until a test socket can be plugged, pumping transfers, stats and the
-   loop callback; gives up past the -E deadline. HTS_FALSE = callback abort. */
-hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
-                                      cache_back *cache, int ptr);
-
-/* Randomized inter-file pause target in [min_ms,max_ms] (#185), derived from a
-   timestamp seed so it is stable within one gap and rerolls per launch. */
-int hts_pause_target_ms(TStamp seed, int min_ms, int max_ms);
-
 /* Schedule more links from the heap into free slots. Returns the number queued,
    or <=0 if none could be added (no free slot / paused / stopped). */
 int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
@@ -462,6 +419,11 @@ int ask_continue(httrackp * opt);
 /* Number of decimal digits in n. */
 int nombre_digit(int n);
 
+// Java
+#if 0
+int hts_add_file(char *file, int file_position);
+#endif
+
 // Polling
 #if HTS_POLL
 int check_flot(T_SOC s);
@@ -484,8 +446,4 @@ void voidf(void);
 /* HTML marker comment marking where the top index is spliced. */
 #define HTS_TOPINDEX "TOP_INDEX_HTTRACK"
 
-/* Worst-case byte expansion HT_ADD_HTMLESCAPED* must reserve per escaper. */
-#define HTS_HTMLESCAPE_MAXEXP 5      /* escape_for_html_print: '&'->"&amp;" */
-#define HTS_HTMLESCAPE_FULL_MAXEXP 6 /* _full: high byte->"&#xHH;" */
-
 #endif

src/htscoremain.c

@@ -145,6 +145,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   char *url = NULL;             // URLS séparées par un espace
   int url_sz = 65535;
 
+  //char url[65536];         // URLS séparées par un espace
   // the parametres
   int httrack_logmode = 3;      // ONE log file
 
@@ -289,11 +290,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         if (tmp_argc == 1) {    /* pas -P & co */
           if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */
             if (argv_url < 0)
-              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
-                            // previous options
+              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
+            //if (argv_url>=0) {
             argv_url++;
             if (!argv_firsturl)
               argv_firsturl = x_argv[x_argc - 1];
+            //}
           } else {
             if (strcmp(tmp_argv[0], "-h") == 0) {
               help(argv[0], !opt->quiet);
@@ -322,8 +324,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         } else if (tmp_argc == 2) {
           if ((strcmp(tmp_argv[0], "-%L") == 0)) {      // liste d'URLs
             if (argv_url < 0)
-              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
-                            // previous options
+              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
+            //if (argv_url>=0)
             argv_url++;         /* forcer */
           }
         }
@@ -401,11 +403,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     else
                       inQuote = 1;
                   } else if (!inQuote && !noDbl && argv[na][i] == ',') {
+                    //StringAddchar(path, '\0');
+                    //j = 0;
                     path = &opt->path_log;
                   } else {
                     StringAddchar(*path, argv[na][i]);
+                    //path[j++] = argv[na][i];
                   }
                 }
+                //path[j++] = '\0';
                 if (StringLength(opt->path_log) == 0) {
                   StringCopyS(opt->path_log, opt->path_html);
                 }
@@ -414,6 +420,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if (check_path(&opt->path_html, argv_firsturl)) {
                   opt->dir_topindex = 1;        // rebuilt top index
                 }
+                //printf("-->%s\n%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log));                
               }
               break;
             }                   // switch
@@ -532,15 +539,74 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             insert_after++;
           }
         }
-      } while (lastp != NULL);
+      } while(lastp != NULL);
+      //fclose(fp);
     }
   }
 
-  // No new cache but an old one? promote it
+  // Existence d'un cache - pas de new mais un old.. renommer
 #if DEBUG_STEPS
   printf("Checking cache\n");
 #endif
-  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  if (!fexist
+      (fconcat
+       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+       StringBuff(opt->path_log), "hts-cache/new.zip"))) {
+    if (fexist
+        (fconcat
+         (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+         StringBuff(opt->path_log), "hts-cache/old.zip"))) {
+      rename(fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+             StringBuff(opt->path_log),
+              "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                            StringBuff(opt->path_log),
+                                            "hts-cache/new.zip"));
+    }
+  } else
+    if ((!fexist
+         (fconcat
+          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+          StringBuff(opt->path_log), "hts-cache/new.dat")))
+        ||
+        (!fexist
+         (fconcat
+          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+          StringBuff(opt->path_log),
+           "hts-cache/new.ndx")))) {
+    if ((fexist
+         (fconcat
+          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+          StringBuff(opt->path_log), "hts-cache/old.dat")))
+        &&
+        (fexist
+         (fconcat
+          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+          StringBuff(opt->path_log),
+           "hts-cache/old.ndx")))) {
+      remove(fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+             StringBuff(opt->path_log),
+              "hts-cache/new.dat"));
+      remove(fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+             StringBuff(opt->path_log),
+              "hts-cache/new.ndx"));
+      //remove(fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
+      rename(fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+             StringBuff(opt->path_log),
+              "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                            StringBuff(opt->path_log),
+                                            "hts-cache/new.dat"));
+      rename(fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+              "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                            StringBuff(opt->path_log),
+                                            "hts-cache/new.ndx"));
+      //rename(fconcat(StringBuff(opt->path_log),"hts-cache/old.lst"),fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
+    }
+  }
 
   /* Interrupted mirror detected */
   if (!opt->quiet) {
@@ -708,6 +774,24 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             HTS_PANIC_PRINTF(s);
             htsmain_free();
             return -1;
+#else
+#if 0
+            char _args[8][256];
+            char *args[8];
+
+            printf("Cheking for updates...\n");
+            strcpybuff(_args[0], argv[0]);
+            strcpybuff(_args[1], "--get");
+            sprintf(_args[2], HTS_UPDATE_WEBSITE, 0, "");
+            strcpybuff(_args[3], "--quickinfo");
+            args[0] = _args[0];
+            args[1] = _args[1];
+            args[2] = _args[2];
+            args[3] = _args[3];
+            args[4] = NULL;
+            if (execvp(args[0], args) == -1) {
+            }
+#endif
 #endif
           }
           //
@@ -725,6 +809,103 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     }
   }
 
+  // Compter urls/jokers
+  /*
+     if (argv_url<=0) { 
+     int na;
+     argv_url=0;
+     for(na=1;na<argc;na++) {
+     if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) {
+     na++;    // sauter nom de proxy
+     } else if (!cmdl_opt(argv[na])) { 
+     argv_url++;   // un de plus       
+     } else if (strcmp(argv[na],"-h")==0) {
+     help(argv[0],!opt->quiet);
+     htsmain_free();
+     return 0;
+     } else {
+     if ((strchr(argv[na],'q')!=NULL))
+     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
+     if ((strchr(argv[na],'i')!=NULL)) {  // doit.log!
+     argv_url=0;
+     na=argc;
+     }
+     }
+     }
+     }  
+   */
+
+  // Ici on ajoute les arguments qui ont été appelés avant au cas où on récupère une session
+  // Exemple: httrack www.truc.fr -L0 puis ^C puis httrack sans URL : ajouter URL précédente
+  /*
+     if (argv_url==0) {
+     //if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) {  // il existe déja un cache précédent.. renommer
+     if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) {    // un cache est présent
+
+     x_argvblk=(char*) calloct(32768,1);
+
+     if (x_argvblk!=NULL) {
+     FILE* fp;
+     int x_argc;
+
+     //strcpybuff(x_argvblk,"httrack ");
+     fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb");
+     if (fp) {
+     linput(fp,x_argvblk+strlen(x_argvblk),8192);
+     fclose(fp); fp=NULL;
+     }
+
+     // calculer arguments selon derniers arguments
+     x_argv[0]=argv[0];
+     x_argc=1;
+     {
+     char* p=x_argvblk;
+     do {
+     x_argv[x_argc++]=p;
+     //p=strstr(p," ");
+     // exemple de chaine: "echo \"test\"" c:\a "\$0"
+     p=next_token(p,1);    // prochain token
+     if (p) {
+     *p=0;    // octet nul (tableau)
+     p++;
+     }            
+     } while(p!=NULL);
+     }
+     // recopier arguments actuels (pointeurs uniquement)
+     {
+     int na;
+     for(na=1;na<argc;na++) {
+     if (strcmp(argv[na],"-O") != 0)    // SAUF le path!
+     x_argv[x_argc++]=argv[na];
+     else
+     na++;
+     }
+     }
+     argc=x_argc;      // nouvel argc
+     argv=x_argv;      // nouvel argv
+     }
+
+     }
+     //}
+     }
+   */
+
+  // Vérifier quiet
+  /*
+     { 
+     int na;    
+     for(na=1;na<argc;na++) {
+     if (!cmdl_opt(argv[na])) { 
+     if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0))
+     na++;    // sauter nom de proxy
+     } else {
+     if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL))
+     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
+     }
+     }
+     }
+   */
+
   /* Engine self-tests: -#test lists them, -#test=NAME [args] runs one. Handled
      here, ahead of the no-URL usage gate below, so they need no dummy URL. */
   {
@@ -831,8 +1012,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   } else {                      // plus de 2 paramètres
     // un fichier log existe?
     if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                       StringBuff(opt->path_log),
-                       "hts-in_progress.lock"))) { // fichier lock?
+      StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // fichier lock?
+      //char s[32];
 
       opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
       if (opt->quiet == 0) {
@@ -863,8 +1044,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           }
         }
       }
-    } else if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                              StringBuff(opt->path_html), "index.html"))) {
+    } else
+      if (fexist
+          (fconcat
+           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "index.html"))) {
+      //char s[32];
       opt->cache = HTS_CACHE_TEST_UPDATE;
       if (opt->quiet == 0) {
         if ((fexist
@@ -959,6 +1143,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             //
           case 'g':            // récupérer un (ou plusieurs) fichiers isolés
             opt->wizard = HTS_WIZARD_AUTO;
+            //opt->wizard=0;             // pas de wizard
             opt->cache = HTS_CACHE_NONE; // ni de cache
             opt->makeindex = 0; // ni d'index
             httrack_logmode = 1;        // erreurs à l'écran
@@ -1058,6 +1243,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             while(isdigit((unsigned char) *(com + 1)))
               com++;
             break;
+            //
+            //case 'A': opt->urlmode=1; break;
+            //case 'R': opt->urlmode=2; break;
           case 'K':
             opt->urlmode = HTS_URLMODE_ABSOLUTE;
             if (isdigit((unsigned char) *(com + 1))) {
@@ -1069,6 +1257,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               while(isdigit((unsigned char) *(com + 1)))
                 com++;
             }
+            //if (*(com+1)=='0') { opt->urlmode=2; com++; } break;
+            //
           case 'c':
             if (isdigit((unsigned char) *(com + 1))) {
               sscanf(com + 1, "%d", &opt->maxsoc);
@@ -1380,30 +1570,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   com++;
                 }
                 break;          // url hack
-              case 'j':
-                opt->no_www_dedup =
-                    HTS_TRUE; // --keep-www-prefix: keep www.X != X
-                if (*(com + 1) == '0') {
-                  opt->no_www_dedup = HTS_FALSE;
-                  com++;
-                }
-                break;
-              case 'o':
-                opt->no_slash_dedup =
-                    HTS_TRUE; // --keep-double-slashes: keep //
-                if (*(com + 1) == '0') {
-                  opt->no_slash_dedup = HTS_FALSE;
-                  com++;
-                }
-                break;
-              case 'y':
-                opt->no_query_dedup =
-                    HTS_TRUE; // --keep-query-order: keep ?b&a order
-                if (*(com + 1) == '0') {
-                  opt->no_query_dedup = HTS_FALSE;
-                  com++;
-                }
-                break;
               case 'v':
                 opt->verbosedisplay = HTS_VERBOSE_FULL;
                 if (isdigit((unsigned char) *(com + 1))) {
@@ -1452,7 +1618,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF
                     ("Option %w needs to be followed by a blank space, and a module name");
-                  printf("Example: -%%w httrack-plugin\n");
+                  printf("Example: -%%w htsswf\n");
                   htsmain_free();
                   return -1;
                 } else {
@@ -1556,6 +1722,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   } else {
                     char *a;
 
+                    //char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs);
                     for(a = argv[na]; *a != '\0'; a++) {
                       if (*a == ';') {  /* next one */
                         StringAddchar(opt->mimedefs, '\n');
@@ -1770,83 +1937,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 }
                 break;
 
-              case 'g': // strip-query: accumulate "[pattern=]keys" entries
-                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
-                  HTS_PANIC_PRINTF("Option strip-query needs a blank space and "
-                                   "[host/pattern=]key1,key2,...");
-                  printf("Example: --strip-query "
-                         "\"www.example.com/*=utm_source,sid\"\n");
-                  htsmain_free();
-                  return -1;
-                } else {
-                  na++;
-                  if (StringNotEmpty(opt->strip_query))
-                    StringCat(opt->strip_query, "\n");
-                  StringCat(opt->strip_query, argv[na]);
-                }
-                break;
-              case 'K': // cookies-file: extra Netscape cookies.txt to preload
-                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
-                  HTS_PANIC_PRINTF(
-                      "Option cookies-file needs a blank space and "
-                      "a cookies.txt path");
-                  printf("Example: --cookies-file \"/home/me/cookies.txt\"\n");
-                  htsmain_free();
-                  return -1;
-                } else {
-                  na++;
-                  if (strlen(argv[na]) >= 1024) {
-                    HTS_PANIC_PRINTF("Cookie file path too long");
-                    htsmain_free();
-                    return -1;
-                  }
-                  StringCopy(opt->cookies_file, argv[na]);
-                }
-                break;
-              case 'Y': // why: explain the filter verdict for a URL, no crawl
-                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
-                  HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
-                  printf(
-                      "Example: --why \"http://www.example.com/file.zip\"\n");
-                  htsmain_free();
-                  return -1;
-                } else {
-                  na++;
-                  if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
-                    HTS_PANIC_PRINTF("why URL too long");
-                    htsmain_free();
-                    return -1;
-                  }
-                  StringCopy(opt->why_url, argv[na]);
-                }
-                break;
-              case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
-                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
-                  HTS_PANIC_PRINTF("Option pause needs a blank space and a "
-                                   "delay in seconds (MIN[:MAX])");
-                  printf("Example: --pause 5:10\n");
-                  htsmain_free();
-                  return -1;
-                } else {
-                  double pmin = 0, pmax = 0;
-                  int nf;
-
-                  na++;
-                  nf = sscanf(argv[na], "%lf:%lf", &pmin, &pmax);
-                  if (nf < 2)
-                    pmax = pmin; /* a single value means a fixed delay */
-                  /* positive-form bounds: NaN fails every comparison, so this
-                     rejects it before the undefined (int)(NaN*1000) cast */
-                  if (nf < 1 || !(pmin >= 0 && pmax >= pmin && pmax <= 86400)) {
-                    HTS_PANIC_PRINTF("Invalid --pause range (expected "
-                                     "MIN[:MAX] seconds, 0<=MIN<=MAX<=86400)");
-                    htsmain_free();
-                    return -1;
-                  }
-                  opt->pause_min_ms = (int) (pmin * 1000.0);
-                  opt->pause_max_ms = (int) (pmax * 1000.0);
-                }
-                break;
               case 't':        /* do not change type (ending) of filenames according to the MIME type */
                 opt->no_type_change = 1;
                 if (*(com+1)=='0') { opt->no_type_change = 0; com++; }
@@ -1911,7 +2001,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   HTS_PANIC_PRINTF(s);
                   htsmain_free();
                   return -1;
-              } break;
+                }
+                break;
+
+                //case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break;
               }
             }
             break;
@@ -1953,20 +2046,18 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     char BIGSTK url[HTS_URLMAXSIZE * 2];
                     char linepos[256];
                     int pos;
-                    LLint cacheNdxLen = 0;
-                    char *cacheNdx = readfile2(
-                        fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                StringBuff(opt->path_log), "hts-cache/new.ndx"),
-                        &cacheNdxLen);
+                    char *cacheNdx =
+                      readfile(fconcat
+                               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                                "hts-cache/new.ndx"));
                     cache_init(&cache, opt);    /* load cache */
                     if (cacheNdx != NULL) {
                       char firstline[256];
                       char *a = cacheNdx;
-                      const char *const end = cacheNdx + cacheNdxLen;
 
                       a += cache_brstr(a, firstline, sizeof(firstline));
                       a += cache_brstr(a, firstline, sizeof(firstline));
-                      while (a != NULL && a < end) {
+                      while(a != NULL) {
                         a = strchr(a + 1, '\n');        /* start of line */
                         if (a) {
                           htsblk r;
@@ -1974,15 +2065,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                           /* */
                           a++;
                           /* read "host/file" */
-                          a += cache_binput(a, end, afs.af.adr, HTS_URLMAXSIZE);
-                          a += cache_binput(a, end, afs.af.fil, HTS_URLMAXSIZE);
+                          a += binput(a, afs.af.adr, HTS_URLMAXSIZE);
+                          a += binput(a, afs.af.fil, HTS_URLMAXSIZE);
                           url[0] = '\0';
                           if (!link_has_authority(afs.af.adr))
                             strcatbuff(url, "http://");
                           strcatbuff(url, afs.af.adr);
                           strcatbuff(url, afs.af.fil);
                           /* read position */
-                          a += cache_binput(a, end, linepos, 200);
+                          a += binput(a, linepos, 200);
                           sscanf(linepos, "%d", &pos);
                           if (!hasFilter
                               || (strjoker(url, filter, NULL, NULL) != NULL)
@@ -2157,7 +2248,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 return 0;
                 break;
               case '~':        /* internal lib test */
-                HTS_PANIC_PRINTF("Option #~ is disabled for security reasons");
+                HTS_PANIC_PRINTF
+                  ("Option #~ is disabled for security reasons");
+                //Disabled because choke on GCC 4.3 (toni from links2linux.de)
+                //{
+                //  char thisIsATestYouShouldSeeAnError[12];
+                //  const char *const bufferOverflowTest = "0123456789012345678901234567890123456789";
+                //  strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest);
+                //  return 0;
+                //}
                 break;
               case 'f':
                 opt->flush = 1;
@@ -2259,6 +2358,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               opt->proxy.active = 1;
               // Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
               a = argv[na] + strlen(argv[na]) - 1;
+              // a=strstr(argv[na],":");  // port
               while((a > argv[na]) && (*a != ':') && (*a != '@'))
                 a--;
               if (*a == ':') {  // un port est présent, <proxy>:port
@@ -2363,14 +2463,116 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
 #endif
 #endif
 
+  //printf("WARNING! This is *only* a beta-release of HTTrack\n");
   io_flush;
 
 #if DEBUG_STEPS
   printf("Cache & log settings\n");
 #endif
 
-  // If both cache generations exist, keep the most complete one
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  // on utilise le cache..
+  // en cas de présence des deux versions, garder la version la plus avancée,
+  // cad la version contenant le plus de fichiers  
+  if (opt->cache) {
+    if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // problemes..
+      if (fexist
+          (fconcat
+           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+            "hts-cache/new.dat"))) {
+        if (fexist
+            (fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+              "hts-cache/old.zip"))) {
+          if (fsize
+              (fconcat
+               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                "hts-cache/new.zip")) < 32768) {
+            if (fsize
+                (fconcat
+                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                  "hts-cache/old.zip")) > 65536) {
+              if (fsize
+                  (fconcat
+                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                    "hts-cache/old.zip")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                          StringBuff(opt->
+                                                                     path_log),
+                                                          "hts-cache/new.zip")))
+              {
+                remove(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/new.zip"));
+                rename(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                      StringBuff(opt->path_log),
+                                                      "hts-cache/new.zip"));
+              }
+            }
+          }
+        }
+      } else
+        if (fexist
+            (fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+              "hts-cache/new.dat"))
+            &&
+            fexist(fconcat
+                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                    "hts-cache/new.ndx"))) {
+        if (fexist
+            (fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+              "hts-cache/old.dat"))
+            &&
+            fexist(fconcat
+                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                    "hts-cache/old.ndx"))) {
+          // switcher si new<32Ko et old>65Ko (tailles arbitraires) ?
+          // ce cas est peut être une erreur ou un crash d'un miroir ancien, prendre
+          // alors l'ancien cache
+          if (fsize
+              (fconcat
+               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                "hts-cache/new.dat")) < 32768) {
+            if (fsize
+                (fconcat
+                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                  "hts-cache/old.dat")) > 65536) {
+              if (fsize
+                  (fconcat
+                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                    "hts-cache/old.dat")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                          StringBuff(opt->
+                                                                     path_log),
+                                                          "hts-cache/new.dat")))
+              {
+                remove(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/new.dat"));
+                remove(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/new.ndx"));
+                rename(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                      StringBuff(opt->path_log),
+                                                      "hts-cache/new.dat"));
+                rename(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                      StringBuff(opt->path_log),
+                                                      "hts-cache/new.ndx"));
+                //} else {  // ne rien faire
+                //  remove("hts-cache/old.dat");
+                //  remove("hts-cache/old.ndx");
+              }
+            }
+          }
+        }
+      }
+    }
+  }
   // Débuggage des en têtes
   if (_DEBUG_HEAD) {
     ioinfo =
@@ -2446,6 +2648,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     {
       FILE *fp = NULL;
 
+      //int n=0;
       char t[256];
 
       time_local_rfc822(t);     // faut bien que ca serve quelque part l'heure RFC1945 arf'
@@ -2479,8 +2682,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       }
 
       strcpybuff(n_lock,
-                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                         StringBuff(opt->path_log), "hts-in_progress.lock"));
+             fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                     "hts-in_progress.lock"));
+      //sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
       /*do {
          if (!n)
          sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
@@ -2556,6 +2760,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           fprintf(fp, LF);
           fclose(fp);
           fp = NULL;
+          //} else if (opt->debug>1) {
+          //  printf("! FileOpen error, \"%s\"\n",strerror(errno));
         }
       }
       // petit message dans le lock

src/htsdefines.h

@@ -69,15 +69,11 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
 typedef struct t_hts_callbackarg t_hts_callbackarg;
 #endif
 
-/* Marks a symbol an external wrapper module exports back to the engine.
-   Must override -fvisibility=hidden on ELF, or dlopen()ed plugins
-   hide their own hts_plug()/hts_unplug() entry points. */
+/* Marks a symbol an external wrapper module exports back to the engine
+   (dllexport on Windows, nothing elsewhere). */
 #ifndef EXTERNAL_FUNCTION
 #ifdef _WIN32
 #define EXTERNAL_FUNCTION __declspec(dllexport)
-#elif ((defined(__GNUC__) && (__GNUC__ >= 4)) ||                               \
-       (defined(HAVE_VISIBILITY) && HAVE_VISIBILITY))
-#define EXTERNAL_FUNCTION __attribute__((visibility("default")))
 #else
 #define EXTERNAL_FUNCTION
 #endif

src/htsdns_selftest.c

@@ -297,26 +297,6 @@ int dns_selftests(httrackp *opt) {
     IPV6_resolver = 0;
   }
 
-  /* "host:port" resolves as the bare host: the cache/connect path strips
-     ":port" before both the backend and the cache key (#181). */
-  mock_reset_calls();
-  {
-    SOCaddr a;
-    char ip[64];
-    const char *err = NULL;
-
-    /* cache miss: the backend is hit with the stripped host, not "host:port" */
-    CHECK(hts_dns_resolve2(opt, "v6only.test:8080", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* the bare host shares that one cache entry (stripped key) */
-    CHECK(hts_dns_resolve2(opt, "v6only.test", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* a port variant of an already-cached host also hits, right address */
-    CHECK(hts_dns_resolve2(opt, "v4only.test:1234", &a, &err) != NULL);
-    SOCaddr_inetntoa(ip, sizeof(ip), a);
-    CHECK(strcmp(ip, "1.2.3.4") == 0);
-  }
-
   /* newhttp_addr() must connect to the addr_index-th address, not always the
      first: this is what back_connect_next relies on to reach the fallback. */
   {

src/htsencoding.c

@@ -30,14 +30,12 @@ Please visit our Website: http://www.httrack.com
 /* Author: Xavier Roche                                         */
 /* ------------------------------------------------------------ */
 
-#include <stdint.h>
-
 #include "htscharset.h"
 #include "htsencoding.h"
 #include "htssafe.h"
 
-/* static int decode_entity(const uint64_t hash, const size_t len);
- */
+/* static int decode_entity(const unsigned int hash, const size_t len);
+*/
 #include "htsentities.h"
 
 /* hexadecimal conversion */
@@ -52,31 +50,30 @@ static int get_hex_value(char c) {
     return -1;
 }
 
-/* 64-bit FNV-1a; must match htsentities.sh, which keys the entity table on it.
- */
-#define HASH_INIT 0xcbf29ce484222325ULL
-#define HASH_PRIME 0x100000001b3ULL
-#define HASH_ADD(HASH, C)                                                      \
-  do {                                                                         \
-    (HASH) ^= (unsigned char) (C);                                             \
-    (HASH) *= HASH_PRIME;                                                      \
-  } while (0)
+/* Numerical Recipes,
+   see <http://en.wikipedia.org/wiki/Linear_congruential_generator> */
+#define HASH_PRIME ( 1664525 )
+#define HASH_CONST ( 1013904223 )
+#define HASH_ADD(HASH, C) do {                  \
+    (HASH) *= HASH_PRIME;                       \
+    (HASH) += HASH_CONST;                       \
+    (HASH) += (C);                              \
+  } while(0)
 
 int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t max, const char *charset) {
   size_t i, j, ampStart, ampStartDest;
   int uc;
   int hex;
-  uint64_t hash;
+  unsigned int hash;
 
   assertf(max != 0);
-  for (i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0, uc = -1, hex = 0,
-      hash = HASH_INIT;
-       src[i] != '\0'; i++) {
+  for(i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0,
+        uc = -1, hex = 0, hash = 0 ; src[i] != '\0' ; i++) {
     /* start of entity */
     if (src[i] == '&') {
       ampStart = i;
       ampStartDest = j;
-      hash = HASH_INIT;
+      hash = 0;
       uc = -1;
     }
     /* inside a potential entity */
@@ -177,11 +174,14 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
       }
       /* alphanumerical entity */
       else {
-        /* alphanum, capped at the longest name
-         * '&CounterClockwiseContourIntegral;' (31) */
-        if (i <= ampStart + 31 && ((src[i] >= '0' && src[i] <= '9') ||
-                                   (src[i] >= 'A' && src[i] <= 'Z') ||
-                                   (src[i] >= 'a' && src[i] <= 'z'))) {
+        /* alphanum and not too far ('&thetasym;' is the longest) */
+        if (i <= ampStart + 10 &&
+            (
+             (src[i] >= '0' && src[i] <= '9')
+             || (src[i] >= 'A' && src[i] <= 'Z')
+             || (src[i] >= 'a' && src[i] <= 'z')
+             )
+            ) {
           /* compute hash */
           HASH_ADD(hash, (unsigned char) src[i]);
         } else {
@@ -190,9 +190,9 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
         }
       }
     }
-
-    /* reserve one byte for the trailing NUL written after the loop */
-    if (j + 1 >= max) {
+    
+    /* copy */
+    if (j + 1 > max) {
       /* overflow */
       return -1;
     }
@@ -300,11 +300,6 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
 
           /* Was the character read successfully ? */
           if (nRead == utfBufferSize) {
-            /* the 'continue' below skips the NUL-reserve guard: re-check */
-            if (utfBufferJ + utfBufferSize >= max) {
-              return -1;
-            }
-
             /* Rollback write position to sequence start write position */
             j = utfBufferJ;
 
@@ -319,8 +314,8 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
       }
     }
 
-    /* reserve one byte for the trailing NUL written after the loop */
-    if (j + 1 >= max) {
+    /* Check for overflow */
+    if (j + 1 > max) {
       return -1;
     }
 

src/htsentities.sh

@@ -1,92 +1,75 @@
 #!/bin/bash
 #
-# Regenerate htsentities.h from the WHATWG named character references.
 
-set -euo pipefail
-
-src=entities.json
-url=https://html.spec.whatwg.org/entities.json
+src=html40.txt
+url=http://www.w3.org/TR/1998/REC-html40-19980424/html40.txt
 dest=htsentities.h
 
-# 64-bit FNV-1a of $1, printed as a C constant. Must match the hash in
-# htsencoding.c. The offset basis is stored as its wrapped (signed) bit pattern;
-# bash arithmetic is 64-bit two's complement, so the result is bit-exact.
-fnv1a() {
-    local s=$1 i c h=$((0xcbf29ce484222325))
-    for ((i = 0; i < ${#s}; i++)); do
-        printf -v c '%d' "'${s:i:1}"
-        h=$(((h ^ (c & 0xff)) * 0x100000001b3))
-    done
-    printf '0x%016xULL' "$h"
-}
+(
+    cat <<EOF
+/*
+  -- ${dest} --
+  FILE GENERATED BY $0, DO NOT MODIFY
 
-if [ ! -f "$src" ]; then
-    curl -fsS "$url" -o "$src"
-fi
+  We compute the LCG hash
+  (see <http://en.wikipedia.org/wiki/Linear_congruential_generator>)
+  for each entity. We should in theory check using strncmp() that we
+  actually have the correct entity, but this is actually statistically
+  not needed.
 
-# Keep ';'-terminated single-codepoint names; the ~93 multi-codepoint refs can't
-# fit decode_entity's single-codepoint return and are skipped (left verbatim).
-pairs=$(jq -r '
-    to_entries
-    | map(select((.key | endswith(";")) and (.value.codepoints | length == 1)))
-    | sort_by(.key)
-    | .[] | "\(.key | ltrimstr("&") | rtrimstr(";"))\t\(.value.codepoints[0])"' "$src")
+  We may want to do better, but we expect the hash function to be uniform, and
+  let the compiler be smart enough to optimize the switch (for example by
+  checking in log2() intervals)
+  
+  This code has been generated using the evil $0 script.
+*/
 
-# Skipped multi-codepoint names, kept to prove none aliases an emitted hash.
-skipped=$(jq -r '
-    to_entries
-    | map(select((.key | endswith(";")) and (.value.codepoints | length > 1)))
-    | .[] | .key | ltrimstr("&") | rtrimstr(";")' "$src")
-
-cases=""
-emit_hashes=""
-while IFS=$'\t' read -r name cp; do
-    hash=$(fnv1a "$name")
-    cases+="    /* $name */"$'\n'
-    cases+="  case $hash:"$'\n'
-    cases+="    if (len == ${#name}) {"$'\n'
-    cases+="      return $cp;"$'\n'
-    cases+="    }"$'\n'
-    cases+="    break;"$'\n'
-    emit_hashes+="$hash"$'\n'
-done <<<"$pairs"
-
-skip_hashes=""
-while IFS= read -r name; do
-    [ -n "$name" ] && skip_hashes+="$(fnv1a "$name")"$'\n'
-done <<<"$skipped"
-
-# The switch keys on the hash alone, so the dispatch is correct only while every
-# emitted name hashes uniquely; prove it here, no runtime name compare needed.
-dups=$(printf '%s' "$emit_hashes" | sort | uniq -d || true)
-if [ -n "$dups" ]; then
-    echo "FATAL: two entity names share a hash (duplicate switch case); change the hash:" >&2
-    echo "$dups" >&2
-    exit 1
-fi
-# A skipped name colliding with an emitted hash would mis-decode instead of
-# staying verbatim; forbid that too.
-aliased=$(comm -12 <(printf '%s' "$emit_hashes" | sort -u) <(printf '%s' "$skip_hashes" | sort -u) || true)
-if [ -n "$aliased" ]; then
-    echo "FATAL: a skipped multi-codepoint name aliases an emitted hash:" >&2
-    echo "$aliased" >&2
-    exit 1
-fi
-
-cat >"$dest" <<EOF
-/* GENERATED by $0 from the WHATWG named character references
-   (${url}). DO NOT EDIT.
-   Dispatch keys on a 64-bit FNV-1a hash of the entity name; the generator
-   aborts on any hash collision, so no runtime name compare is needed. */
-
-#include <stdint.h>
-
-static int decode_entity(const uint64_t hash, const size_t len) {
+static int decode_entity(const unsigned int hash, const size_t len) {
   switch(hash) {
-${cases}  }
+EOF
+    (
+        if test -f ${src}; then
+            cat ${src}
+        else
+            GET "${url}"
+        fi
+    ) |
+        grep -E '^<!ENTITY [a-zA-Z0-9_]' |
+        sed \
+            -e 's/<!ENTITY //' -e "s/[[:space:]][[:space:]]*/ /g" \
+            -e 's/-->$//' \
+            -e 's/\([^ ]*\) CDATA "&#\([^\"]*\);" -- \(.*\)/\1 \2 \3/' |
+        (
+            read -r A
+            while test -n "$A"; do
+                ent="${A%% *}"
+                code=$(echo "$A" | cut -f2 -d' ')
+                # compute hash
+                hash=0
+                i=0
+                a=1664525
+                c=1013904223
+                m="$((1 << 32))"
+                while test "$i" -lt ${#ent}; do
+                    d="$(echo -n "${ent:${i}:1}" | hexdump -v -e '/1 "%d"')"
+                    hash="$((((hash * a) % (m) + d + c) % (m)))"
+                    i=$((i + 1))
+                done
+                echo -e "    /* $A */"
+                echo -e "  case ${hash}u:"
+                echo -e "    if (len == ${#ent} /* && strncmp(ent, \"${ent}\") == 0 */) {"
+                echo -e "      return ${code};"
+                echo -e "    }"
+                echo -e "    break;"
+
+                # next
+                read -r A
+            done
+        )
+    cat <<EOF
+  }
   /* unknown */
   return -1;
 }
 EOF
-
-echo "wrote $dest ($(grep -c '^  case ' "$dest") entities)" >&2
+) >${dest}

src/htsfilters.c

@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
 #include "htsbase.h"
 #include "htslib.h"
 #include <ctype.h>
-#include <stdint.h>
 /* END specific definitions */
 
 // à partir d'un tableau de {"+*.toto","-*.zip","+*.tata"} définit si nom est autorisé
@@ -77,8 +76,7 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
     }
     if (size)
       sz = *size;
-    /* size unknown (scan time): no size pointer => size tests stay neutral */
-    if (strjoker(nom, filters[i] + filteroffs, size ? &sz : NULL, size_flag)) {
+    if (strjoker(nom, filters[i] + filteroffs, &sz, size_flag)) {       // reconnu
       if (size)
         if (sz != *size)
           sizelimit = sz;
@@ -95,127 +93,17 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
   return verdict;
 }
 
-int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
-                     const char *nom2, LLint *size, int *size_flag,
-                     int *depth) {
-  int depth1 = 0, depth2 = 0;
-  int flag1 = 0, flag2 = 0;
-  LLint sz1 = 0, sz2 = 0;
-  int jok1, jok2, use1;
-
-  if (size) {
-    sz1 = *size;
-    sz2 = *size;
-  }
-  jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
-                     size_flag ? &flag1 : NULL, &depth1);
-  jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
-                     size_flag ? &flag2 : NULL, &depth2);
-  use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
-  if (size)
-    *size = use1 ? sz1 : sz2;
-  if (size_flag)
-    *size_flag = use1 ? flag1 : flag2;
-  if (depth)
-    *depth = use1 ? depth1 : depth2;
-  return use1 ? jok1 : jok2;
-}
-
-/* Real filters/URLs fit HTS_URLMAXSIZE*2; past it a hostile pattern recurses
-   O(len) deep or runs O(n^2*stars). Cap length (depth) and steps (work). */
-#define STRJOKER_MAXLEN (HTS_URLMAXSIZE * 2)
-#define STRJOKER_MAXSTEPS 2000000u
-
-/* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
-   pair keeps star-heavy patterns polynomial instead of exponential (#501). */
-typedef struct strjoker_memo {
-  const char *chaine0, *joker0; /* offsets are relative to these bases */
-  size_t stride;                /* strlen(joker0) + 1 */
-  unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
-  size_t *nsteps; /* shared work counter; NULL: unbounded (oracle) */
-} strjoker_memo;
-
-static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
-                                 const char *joker, LLint *size,
-                                 int *size_flag);
-
-/* A pair that failed once fails forever (*size is only written on the success
-   path), so record NULL results and cut the re-exploration. */
-static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
-                                const char *joker, LLint *size,
-                                int *size_flag) {
-  size_t bit = 0;
-  const char *adr;
-
-  if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
-    return NULL; /* work budget spent: fail the match safely */
-  if (memo->failed) {
-    bit = (size_t) (chaine - memo->chaine0) * memo->stride +
-          (size_t) (joker - memo->joker0);
-    if (memo->failed[bit >> 3] & (unsigned char) (1u << (bit & 7)))
-      return NULL;
-  }
-  adr = strjoker_impl(memo, chaine, joker, size, size_flag);
-  if (adr == NULL && memo->failed)
-    memo->failed[bit >> 3] |= (unsigned char) (1u << (bit & 7));
-  return adr;
-}
-
-/* Match chaine against joker with a shared work budget *nsteps (a strjokerfind
-   sweep passes one counter, bounding the whole scan). */
-static const char *strjoker_bounded(const char *chaine, const char *joker,
-                                    LLint *size, int *size_flag,
-                                    size_t *nsteps) {
-  strjoker_memo memo = {chaine, joker, 0, NULL, nsteps};
-  unsigned char stackbits[2048];
-  hts_boolean onheap = HTS_FALSE;
-  const char *adr;
-
-  if (chaine != NULL && joker != NULL) {
-    const size_t l1 = strlen(chaine), l2 = strlen(joker);
-
-    if (l1 > STRJOKER_MAXLEN || l2 > STRJOKER_MAXLEN)
-      return NULL; /* beyond any real filter/URL: bound depth+work */
-    memo.stride = l2 + 1;
-    if (l1 + 1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
-      const size_t bytes = ((l1 + 1) * memo.stride + 7) / 8;
-
-      if (bytes <= sizeof(stackbits)) {
-        memset(stackbits, 0, bytes);
-        memo.failed = stackbits;
-      } else {
-        memo.failed = (unsigned char *) calloct(bytes, 1);
-        onheap = memo.failed != NULL ? HTS_TRUE : HTS_FALSE;
-      }
-    }
-  }
-  adr = strjoker_rec(&memo, chaine, joker, size, size_flag);
-  if (onheap)
-    freet(memo.failed);
-  return adr;
-}
-
-// wildcard comparator: match chaine against joker (pattern), case-insensitive
-// returns the address of the first matched letter past any leading joker
-// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
-                                LLint *size, int *size_flag) {
-  size_t nsteps = 0;
-
-  return strjoker_bounded(chaine, joker, size, size_flag, &nsteps);
-}
-
-/* Test-only oracle: the same matcher without the failure memo. */
-const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
-                            int *size_flag) {
-  const strjoker_memo memo = {chaine, joker, 0, NULL};
-
-  return strjoker_rec(&memo, chaine, joker, size, size_flag);
-}
-
-static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
-                                 const char *joker, LLint *size,
-                                 int *size_flag) {
+// supercomparateur joker (tm)
+// compare a et b (b=avec joker dedans), case insensitive [voir CI]
+// renvoi l'adresse de la première lettre de la chaine
+// (càd *[..]toto.. renvoi adresse de toto dans la chaine)
+// accepte les délires du genre www.*.*/ * / * truc*.*
+// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
+// * = toute lettre
+// --?-- : spécifique à HTTrack et aux ?
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
+                          int *size_flag) {
+  //int err=0;
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -304,12 +192,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
           int len = (int) strlen(joker);
 
           while((joker[i] != RIGHT) && (joker[i]) && (i < len)) {
-            // '\' escapes the next char as a literal member, e.g. *[\[\]]
-            if (joker[i] == '\\' && joker[i + 1] != '\0') {
-              i++;
-              pass[(int) (unsigned char) joker[i]] = 1;
-              i++;
-            } else if ((joker[i] == '<') || (joker[i] == '>')) { // *[<10]
+            if ((joker[i] == '<') || (joker[i] == '>')) {       // *[<10]
               int lsize = 0;
               int lverdict;
 
@@ -337,9 +220,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
                 while(isdigit((unsigned char) joker[i]))
                   i++;
               }
-            } else if (joker[i + 1] == '-' && joker[i + 2] != '\0') {
-              // range *[A-Z]; the '\0' guard rejects a truncated *[a- (else
-              // i+=3 overshoots the NUL)
+            } else if (joker[i + 1] == '-') {   // 2 car, ex: *[A-Z]
               if ((int) (unsigned char) joker[i + 2] >
                   (int) (unsigned char) joker[i]) {
                 int j;
@@ -347,9 +228,14 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
                 for(j = (int) (unsigned char) joker[i];
                     j <= (int) (unsigned char) joker[i + 2]; j++)
                   pass[j] = 1;
+
               }
+              // else err=1;
               i += 3;
-            } else { // 1 car, ex: *[ ]
+            } else {            // 1 car, ex: *[ ]
+              if (joker[i + 2] == '\\' && joker[i + 3] != 0) {  // escaped char, such as *[\[] or *[\]]
+                i++;
+              }
               pass[(int) (unsigned char) joker[i]] = 1;
               i++;
             }
@@ -371,6 +257,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       for(i = 0; i < 256; i++)
         pass[i] = 1;            // tout autoriser
       jmp = 1;
+      ////if (joker[2]==LEFT) jmp=3;        // permet de recher *<crochet ouvrant>
     }
 
     {
@@ -389,7 +276,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 
       // tester sans le joker (pas ()+ mais ()*)
       if (!unique) {
-        if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag))) {
+        if ((adr = strjoker(chaine, joker + jmp, size, size_flag))) {
           return adr;
         }
       }
@@ -398,11 +285,10 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       if (!unique)
         max = (int) strlen(chaine);
       else                      /* *(a) only match a (not aaaaa) */
-        max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
+        max = 1;
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
-          if ((adr = strjoker_rec(memo, chaine + i + 1, joker + jmp, size,
-                                  size_flag))) {
+          if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
             return adr;
           }
           i++;
@@ -412,8 +298,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 
       // tester chaîne vide
       if (i != max + 2)         // avant c'est ok
-        if ((adr = strjoker_rec(memo, chaine + max, joker + jmp, size,
-                                size_flag)))
+        if ((adr = strjoker(chaine + max, joker + jmp, size, size_flag)))
           return adr;
 
       return NULL;              // perdu
@@ -435,7 +320,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       // comparaison ok?
       if (ok) {
         // continuer la comparaison.
-        if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag))
+        if (strjoker(chaine + jmp, joker + jmp, size, size_flag))
           return chaine;        // retourner 1e lettre
       }
 
@@ -451,18 +336,12 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 // d'un strcpy sur une variable ayant un nom en lettres et copiant une chaine de chiffres
 // ATTENTION!! Eviter les jokers en début, où gare au temps machine!
 const char *strjokerfind(const char *chaine, const char *joker) {
-  size_t nsteps = 0; /* one budget for the whole scan */
   const char *adr;
 
-  if (chaine != NULL && strlen(chaine) > STRJOKER_MAXLEN)
-    return NULL;
   while(*chaine) {
-    if ((adr = strjoker_bounded(chaine, joker, NULL, NULL,
-                                &nsteps))) { // ok trouvé
+    if ((adr = strjoker(chaine, joker, NULL, NULL))) {  // ok trouvé
       return adr;
     }
-    if (nsteps > STRJOKER_MAXSTEPS) // scan budget spent: no match
-      return NULL;
     chaine++;
   }
   return NULL;

src/htsfilters.h

@@ -41,17 +41,8 @@ Please visit our Website: http://www.httrack.com
 
 int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
                 int *size_flag, int *depth);
-/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
-   scheme); the match latest in the list wins, a "don't know" verdict defers.
-   Returns the merged verdict; the out-params carry the winner's values. */
-int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
-                     const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
-/* strjoker() without the failure memo (exponential worst case); test-only
-   oracle for the memoized matcher. */
-const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
-                            int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);
 #endif
 

src/htsftp.c

@@ -54,9 +54,15 @@ Please visit our Website: http://www.httrack.com
 #endif
 
 // ftp mode passif
+// #if HTS_INET6==0
 #define FTP_PASV 1
+// #else
+// no passive mode for v6
+// #define FTP_PASV 0
+// #endif
 
 #define FTP_DEBUG 0
+//#define FORK_DEBUG 0
 
 #if USE_BEGINTHREAD
 
@@ -122,33 +128,6 @@ void launch_ftp(FTPDownloadStruct * params) {
   return 0; \
   }
 
-/* Bounded split of a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
-void ftp_split_userpass(const char *src, const char *end, char *user,
-                        size_t user_size, char *pass, size_t pass_size) {
-  size_t n = 0;
-
-  assertf(user_size > 0 && pass_size > 0); /* the size-1 math underflows on 0 */
-
-  while (src[n] != '\0' && src[n] != ':') {
-    if (n < user_size - 1)
-      user[n] = src[n];
-    n++;
-  }
-  user[n < user_size ? n : user_size - 1] = '\0';
-  pass[0] = '\0';
-  if (src[n] == ':') { // password follows the colon
-    const size_t base = n + 1;
-    size_t k = 0;
-
-    while (&src[base + k + 1] < end && src[base + k] != '\0') {
-      if (k < pass_size - 1)
-        pass[k] = src[base + k];
-      k++;
-    }
-    pass[k < pass_size ? k : pass_size - 1] = '\0';
-  }
-}
-
 // la véritable fonction une fois lancées les routines thread/fork
 int run_launch_ftp(FTPDownloadStruct * pStruct) {
   lien_back *back = pStruct->pBack;
@@ -194,14 +173,43 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   while(*real_adr == '/')
     real_adr++;                 // sauter /
   if ((adr = jump_identification(real_adr)) != real_adr) {      // user
-    ftp_split_userpass(real_adr, adr, user, sizeof(user), pass, sizeof(pass));
+    int i = -1;
+
+    pass[0] = '\0';
+    do {
+      i++;
+      user[i] = real_adr[i];
+    } while((real_adr[i] != ':') && (real_adr[i]));
+    user[i] = '\0';
+    if (real_adr[i] == ':') {   // pass
+      int j = -1;
+
+      i++;                      // oui on saute aussi le :
+      do {
+        j++;
+        pass[j] = real_adr[i + j];
+      } while(((&real_adr[i + j + 1]) < adr) && (real_adr[i + j]));
+      pass[j] = '\0';
+    }
   }
   // Calculer RETR <nom>
   {
     char *a;
 
+#if 0
+    a = back->url_fil + strlen(back->url_fil) - 1;
+    while((a > back->url_fil) && (*a != '/'))
+      a--;
+    if (*a != '/') {
+      a = NULL;
+    }
+#else
     a = back->url_fil;
+#endif
     if (a != NULL && *a != '\0') {
+#if 0
+      a++;                      // sauter /
+#endif
       ftp_filename = a;
       if (strnotempty(a)) {
         char catbuff[CATBUFF_SIZE];
@@ -225,6 +233,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
       }
     } else {
       strcpybuff(back->r.msg, "Unexpected PORT error");
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
     }
   }
@@ -258,6 +267,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
       snprintf(back->r.msg, sizeof(back->r.msg),
                "Unable to get server's address: %s", error);
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_NON_FATAL;
       _HALT_FTP return 0;
     }
@@ -270,17 +280,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
     if (soc_ctl == INVALID_SOCKET) {
       strcpybuff(back->r.msg, "Unable to create a socket");
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
     }
 
     SOCaddr_initport(server, port);
+    // server.sin_port = htons((unsigned short int) port);
 
     // connexion (bloquante, on est en thread)
     strcpybuff(back->info, "connect");
 
     if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
       strcpybuff(back->r.msg, "Unable to connect to the server");
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
 #ifdef _WIN32
@@ -322,22 +335,64 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               // ok
             } else {
               strcpybuff(back->r.msg, "TYPE I error");
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
+#if 0
+            // --CWD--
+            char *a;
+
+            a = back->url_fil + strlen(back->url_fil) - 1;
+            while((a > back->url_fil) && (*a != '/'))
+              a--;
+            if (*a == '/') {    // ok repéré
+              char BIGSTK target[1024];
+
+              target[0] = '\0';
+              strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
+              if (strnotempty(target) == 0)
+                strcatbuff(target, "/");
+              strcpybuff(back->info, "cwd");
+              snprintf(line, sizeof(line), "CWD %s", target);
+              send_line(soc_ctl, line);
+              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+              _CHECK_HALT_FTP;
+              if (line[0] == '2') {
+                send_line(soc_ctl, "TYPE I");
+                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+                _CHECK_HALT_FTP;
+                if (line[0] == '2') {
+                  // ok..
+                } else {
+                  strcpybuff(back->r.msg, "TYPE I error");
+                  // back->status=STATUS_FTP_READY;    // fini
+                  back->r.statuscode = STATUSCODE_INVALID;
+                }
+              } else {
+                snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
+                // back->status=STATUS_FTP_READY;    // fini
+                back->r.statuscode = STATUSCODE_INVALID;
+              }                 // sinon on est prêts
+            } else {
+              strcpybuff(back->r.msg, "Unexpected ftp error");
+              // back->status=STATUS_FTP_READY;    // fini
+              back->r.statuscode = STATUSCODE_INVALID;
+            }
+#endif
 
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
-                     linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
-                   linejmp(line));
+          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
+          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
       } else {
-        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
-                 linejmp(line));
+        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
+        // back->status=STATUS_FTP_READY;    // fini
         back->r.statuscode = STATUSCODE_INVALID;
       }
 
@@ -404,8 +459,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
             // -- fin analyse de l'adresse IP et du port --
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
-                     linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
@@ -436,13 +491,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 }
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
-                       linejmp(line));
+              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
-                     linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         }
@@ -548,8 +603,9 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   deletesoc(soc_dat);
                   soc_dat = INVALID_SOCKET;
                   //
-                  snprintf(back->r.msg, sizeof(back->r.msg),
-                           "RETR command error: %s", linejmp(line));
+                  snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
+                          linejmp(line));
+                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = STATUSCODE_INVALID;
                 }               // sinon on est prêts
               } else {
@@ -560,20 +616,23 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 soc_dat = INVALID_SOCKET;
                 //
                 strcpybuff(back->r.msg, "Unable to connect");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }                 // sinon on est prêts
             } else {
               strcpybuff(back->r.msg, "Unable to create a socket");
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }                   // sinon on est prêts
           } else {
             snprintf(back->r.msg, sizeof(back->r.msg),
                      "Unable to resolve IP %s: %s", adr_ip, error);
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
-                   linejmp(line));
+          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
+          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }                       // sinon on est prêts
 #else
@@ -594,16 +653,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               //T_SOC soc_dat;
               if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
                 strcpybuff(back->r.msg, "Unable to accept connection");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg),
-                       "RETR command error: %s", linejmp(line));
+              snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
-                     linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -613,6 +673,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 #endif
         } else {
           strcpybuff(back->r.msg, "Unable to listen to a port");
+          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
 #endif
@@ -636,18 +697,21 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             int len = 1;
             int read_len = 1024;
 
+            //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
+
             while((len > 0) && (!stop_ftp(back))) {
               // attendre les données
               len = 1;          // pas d'erreur pour le moment
               switch (wait_socket_receive(soc_dat, timeout)) {
               case -1:
                 strcpybuff(back->r.msg, "FTP read error");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
               case 0:
-                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
-                         timeout);
+                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
@@ -669,14 +733,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                          }
                        */
                       strcpybuff(back->r.msg, "Write error");
+                      // back->status=STATUS_FTP_READY;    // fini
                       back->r.statuscode = STATUSCODE_INVALID;
                       len = 0;  // error
                     }
                   } else {
                     strcpybuff(back->r.msg, "Unexpected write error");
+                    // back->status=STATUS_FTP_READY;    // fini
                     back->r.statuscode = STATUSCODE_INVALID;
                   }
-                } else { // Erreur ou terminé
+                } else {        // Erreur ou terminé
+                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = 0;
                   if (back->r.totalsize > 0
                       && back->r.size != back->r.totalsize) {
@@ -685,6 +752,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   }
                 }
                 read_len = 1024;
+                //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
               }
             }
             if (back->r.fp) {
@@ -693,6 +761,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
           } else {
             strcpybuff(back->r.msg, "Unable to write file");
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -708,14 +777,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               get_ftp_line(soc_ctl, line, sizeof(line), timeout);
               if (line[0] == '2') {     // OK
                 strcpybuff(back->r.msg, "OK");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = HTTP_OK;
               } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
-                         linejmp(line));
+                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
               strcpybuff(back->r.msg, "FTP read error");
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           }
@@ -741,6 +812,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     back->r.statuscode = HTTP_OK;
     strcpybuff(back->r.msg, "OK");
   }
+  // back->status=STATUS_FTP_READY;    // fini
   return 0;
 }
 
@@ -907,11 +979,13 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
       break;
     }
 
+    //HTS_TOTAL_RECV_CHECK(dummy);     // Diminuer au besoin si trop de données reçues
     switch (recv(soc, &b, 1, 0)) {
+      //case 0: break;    // pas encore --> erreur (on attend)!
     case 1:
       HTS_STAT.HTS_TOTAL_RECV += 1;     // compter flux entrant
-      if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
-        data[i++] = b; // truncate hostile over-long reply lines
+      if ((b != 10) && (b != 13))
+        data[i++] = b;
       break;
     default:
       if (ptrline)
@@ -1034,6 +1108,7 @@ int wait_socket_receive(T_SOC soc, int timeout) {
 int stop_ftp(lien_back * back) {
   if (back->stop_ftp) {
     strcpybuff(back->r.msg, "Cancelled by User");
+    // back->status=STATUS_FTP_READY;    // fini
     back->r.statuscode = STATUSCODE_INVALID;
     return 1;
   }

src/htsftp.h

@@ -70,11 +70,6 @@ int back_launch_ftp(FTPDownloadStruct * params);
 int run_launch_ftp(FTPDownloadStruct * params);
 int send_line(T_SOC soc, const char *data);
 int get_ftp_line(T_SOC soc, char *line, size_t line_size, int timeout);
-/* Split a "user[:pass]@" prefix (end = jump_identification result) into
-   bounded, NUL-terminated user/pass buffers, truncating to fit.
-   Both sizes must be nonzero. */
-void ftp_split_userpass(const char *src, const char *end, char *user,
-                        size_t user_size, char *pass, size_t pass_size);
 T_SOC get_datasocket(char *to_send, size_t to_send_size);
 int stop_ftp(lien_back * back);
 char *linejmp(char *line);

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-12"
-#define HTTRACK_VERSIONID "3.49.12"
+#define HTTRACK_VERSION "3.49-9"
+#define HTTRACK_VERSIONID "3.49.9"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 
@@ -56,10 +56,6 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
-// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
-#include <stdint.h>
-#include <inttypes.h>
-
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -115,6 +111,29 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
+#ifndef HTS_LONGLONG
+#ifdef SIZEOF_LONG_LONG
+#if SIZEOF_LONG_LONG == 8
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+#ifndef HTS_LONGLONG
+#ifdef __sun
+#define HTS_LONGLONG 0
+#endif
+#ifdef __osf__
+#define HTS_LONGLONG 0
+#endif
+#ifdef __linux
+#define HTS_LONGLONG 1
+#endif
+#ifdef _WIN32
+#define HTS_LONGLONG 1
+#endif
+#endif
+#endif
+
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -210,10 +229,6 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DEFAULT_FOOTER                                                     \
   "<!-- Mirrored from %s%s by HTTrack Website Copier/" HTTRACK_AFF_VERSION     \
   " " HTTRACK_AFF_AUTHORS ", %s -->"
-/* Honest crawler User-Agent; no fake OS/browser to go stale. */
-#define HTS_DEFAULT_USER_AGENT                                                 \
-  "Mozilla/5.0 (compatible; HTTrack/" HTTRACK_AFF_VERSION                      \
-  "; +https://www.httrack.com/)"
 #define HTTRACK_WEB "http://www.httrack.com"
 #define HTS_UPDATE_WEBSITE                                                     \
   "http://www.httrack.com/"                                                    \
@@ -307,11 +322,59 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
-typedef int64_t LLint;
-typedef int64_t TStamp;
-/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
-#define LLintP "%" PRId64
+#ifndef HTS_LONGLONG
+#ifdef HTS_NO_64_BIT
+#define HTS_LONGLONG 0
+#else
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+/* Wide integer types, chosen per platform.
+   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
+           plain int where 64-bit is unavailable).
+   TStamp: timestamp/duration in the same width (a double in the no-64-bit
+           fallback).
+   LLintP: the printf conversion for an LLint. */
+#if HTS_LONGLONG
+#ifdef LLINT_FORMAT
+typedef LLINT_TYPE LLint;
+
+typedef LLINT_TYPE TStamp;
+
+#define LLintP LLINT_FORMAT
+#else
+
+#ifdef _WIN32
+typedef __int64 LLint;
+
+typedef __int64 TStamp;
+
+#define LLintP "%I64d"
+#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
+       defined(__64BIT__))
+
+typedef long int LLint;
+
+typedef long int TStamp;
+
+#define LLintP "%ld"
+#else
+typedef long long int LLint;
+
+typedef long long int TStamp;
+
+#define LLintP "%lld"
+#endif
+
+#endif /* HTS_LONGLONG */
+
+#else
+typedef int LLint;
+
+#define LLintP "%d"
+typedef double TStamp;
+#endif
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain
@@ -355,10 +418,9 @@ typedef int T_SOC;
 #endif
 
 /* Permission bits for created folders and files (mkdir and chmod).
-   PROTECT_FOLDER/FILE are owner-only. With HTS_ACCESS set (the default) the
-   ACCESS_ modes also grant group/other read; otherwise they stay owner-only. */
+   PROTECT_FOLDER is owner-only. With HTS_ACCESS set (the default) the ACCESS_
+   modes also grant group/other read; otherwise they stay owner-only. */
 #define HTS_PROTECT_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
-#define HTS_PROTECT_FILE (S_IRUSR | S_IWUSR)
 
 #if HTS_ACCESS
 #define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)

src/htshash.c

@@ -106,10 +106,10 @@ static coucal_hashkeys key_adrfil_hashes_generic(void *arg,
   const lien_url*const lien = (const lien_url*) value;
   const char *const adr = !former ? lien->adr : lien->former_adr;
   const char *const fil = !former ? lien->fil : lien->former_fil;
-  const char *const adr_norm =
-      adr != NULL ? (hash->norm_host ? jump_normalized_const(adr)
-                                     : jump_identification_const(adr))
-                  : NULL;
+  const char *const adr_norm = adr != NULL ? 
+    ( hash->normalized  ? jump_normalized_const(adr)
+                        : jump_identification_const(adr) )
+    : NULL;
 
   // copy address
   assertf(adr_norm != NULL);
@@ -117,18 +117,10 @@ static coucal_hashkeys key_adrfil_hashes_generic(void *arg,
 
   // copy link
   assertf(fil != NULL);
-  {
-    /* resolve the per-URL strip keys; strip applies even when urlhack is off */
-    char BIGSTK keybuf[HTS_URLMAXSIZE];
-    const char *const keys = hts_query_strip_keys(hash->strip_query, adr, fil,
-                                                  keybuf, sizeof(keybuf));
-
-    if (hash->norm_slash || hash->norm_query || keys != NULL) {
-      fil_normalized_filtered_ex(fil, &hash->normfil[strlen(hash->normfil)],
-                                 keys, hash->norm_slash, hash->norm_query);
-    } else {
-      strcpy(&hash->normfil[strlen(hash->normfil)], fil);
-    }
+  if (hash->normalized) {
+    fil_normalized(fil, &hash->normfil[strlen(hash->normfil)]);
+  } else {
+    strcpy(&hash->normfil[strlen(hash->normfil)], fil);
   }
 
   // hash
@@ -140,7 +132,8 @@ static int key_adrfil_equals_generic(void *arg,
                                      coucal_key_const a_,
                                      coucal_key_const b_, 
                                      const int former) {
-  hash_struct *const hash = (hash_struct *) arg;
+  hash_struct *const hash = (hash_struct*) arg;
+  const int normalized = hash->normalized;
   const lien_url*const a = (const lien_url*) a_;
   const lien_url*const b = (const lien_url*) b_;
   const char *const a_adr = !former ? a->adr : a->former_adr;
@@ -157,10 +150,10 @@ static int key_adrfil_equals_generic(void *arg,
   assertf(b_fil != NULL);
 
   // skip scheme and authentication to the domain (possibly without www.)
-  ja = hash->norm_host ? jump_normalized_const(a_adr)
-                       : jump_identification_const(a_adr);
-  jb = hash->norm_host ? jump_normalized_const(b_adr)
-                       : jump_identification_const(b_adr);
+  ja = normalized
+    ? jump_normalized_const(a_adr) : jump_identification_const(a_adr);
+  jb = normalized
+    ? jump_normalized_const(b_adr) : jump_identification_const(b_adr);
   assertf(ja != NULL);
   assertf(jb != NULL);
   if (strcasecmp(ja, jb) != 0) {
@@ -168,23 +161,12 @@ static int key_adrfil_equals_generic(void *arg,
   }
 
   // now compare pathes
-  {
-    char BIGSTK ka[HTS_URLMAXSIZE], kb[HTS_URLMAXSIZE];
-    const char *const keysa =
-        hts_query_strip_keys(hash->strip_query, a_adr, a_fil, ka, sizeof(ka));
-    const char *const keysb =
-        hts_query_strip_keys(hash->strip_query, b_adr, b_fil, kb, sizeof(kb));
-
-    if (hash->norm_slash || hash->norm_query || keysa != NULL ||
-        keysb != NULL) {
-      fil_normalized_filtered_ex(a_fil, hash->normfil, keysa, hash->norm_slash,
-                                 hash->norm_query);
-      fil_normalized_filtered_ex(b_fil, hash->normfil2, keysb, hash->norm_slash,
-                                 hash->norm_query);
-      return strcmp(hash->normfil, hash->normfil2) == 0;
-    } else {
-      return strcmp(a_fil, b_fil) == 0;
-    }
+  if (normalized) {
+    fil_normalized(a_fil, hash->normfil);
+    fil_normalized(b_fil, hash->normfil2);
+    return strcmp(hash->normfil, hash->normfil2) == 0;
+  } else {
+    return strcmp(a_fil, b_fil) == 0;
   }
 }
 
@@ -240,17 +222,11 @@ static int key_former_adrfil_equals(void *arg,
   return key_adrfil_equals_generic(arg, a, b, 1);
 }
 
-void hash_init(httrackp *opt, hash_struct *hash, hts_boolean normalized) {
+void hash_init(httrackp *opt, hash_struct * hash, int normalized) {
   hash->sav = coucal_new(0);
   hash->adrfil = coucal_new(0);
   hash->former_adrfil = coucal_new(0);
-  /* urlhack is the umbrella; per-feature negatives opt out of each part */
-  hash->norm_host = normalized && !opt->no_www_dedup;
-  hash->norm_slash = normalized && !opt->no_slash_dedup;
-  hash->norm_query = normalized && !opt->no_query_dedup;
-  /* snapshot the query-strip list (not owned; valid for the hash lifetime) */
-  hash->strip_query =
-      StringNotEmpty(opt->strip_query) ? StringBuff(opt->strip_query) : NULL;
+  hash->normalized = normalized;
 
   hts_set_hash_handler(hash->sav, opt);
   hts_set_hash_handler(hash->adrfil, opt);
@@ -306,26 +282,6 @@ void hash_free(hash_struct *hash) {
   }
 }
 
-/* Test helper: do the two URLs dedupe to the same key under opt's urlhack
-   flags? Exercises the live hash compare (norm_host/slash/query resolution). */
-hts_boolean hash_url_equals(httrackp *opt, const char *adra, const char *fila,
-                            const char *adrb, const char *filb) {
-  hash_struct hash;
-  lien_url la, lb;
-  hts_boolean eq;
-
-  memset(&la, 0, sizeof(la));
-  memset(&lb, 0, sizeof(lb));
-  la.adr = key_duphandler(NULL, adra);
-  la.fil = key_duphandler(NULL, fila);
-  lb.adr = key_duphandler(NULL, adrb);
-  lb.fil = key_duphandler(NULL, filb);
-  hash_init(opt, &hash, opt->urlhack);
-  eq = key_adrfil_equals(&hash, &la, &lb);
-  hash_free(&hash);
-  return eq;
-}
-
 // retour: position ou -1 si non trouvé
 int hash_read(const hash_struct * hash, const char *nom1, const char *nom2,
               hash_struct_type type) {

src/htshash.h

@@ -51,12 +51,8 @@ typedef enum hash_struct_type {
 } hash_struct_type;
 
 // tables de hachage
-void hash_init(httrackp *opt, hash_struct *hash, hts_boolean normalized);
+void hash_init(httrackp *opt, hash_struct *hash, int normalized);
 void hash_free(hash_struct *hash);
-/* Test helper: HTS_TRUE if the two URLs dedupe together under opt's urlhack
-   flags. */
-hts_boolean hash_url_equals(httrackp *opt, const char *adra, const char *fila,
-                            const char *adrb, const char *filb);
 int hash_read(const hash_struct * hash, const char *nom1, const char *nom2,
               hash_struct_type type);
 void hash_write(hash_struct * hash, size_t lpos);

src/htshelp.c

@@ -152,6 +152,16 @@ void help_wizard(httrackp * opt) {
 #define str (buffers->str)
 #define argv (buffers->argv)
 
+  //char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
+  //char *mainpath = (char *) malloct(256);
+  //char *projname = (char *) malloct(256);
+  //char *stropt = (char *) malloct(2048);        // options
+  //char *stropt2 = (char *) malloct(2048);       // options longues
+  //char *strwild = (char *) malloct(2048);       // wildcards
+  //char *cmd = (char *) malloct(4096);
+  //char *str = (char *) malloct(256);
+  //char **argv = (char **) malloct(256 * sizeof(char *));
+
   //
   char *a;
 
@@ -217,6 +227,8 @@ void help_wizard(httrackp * opt) {
   strcatbuff(stropt2, str);
   strcatbuff(stropt2, projname);
   strcatbuff(stropt2, "\" ");
+  // Créer si ce n'est fait un index.html 1er niveau
+  make_empty_index(str);
   //
   printf("\n");
   printf("Enter URLs (separated by commas or blank spaces) :");
@@ -337,6 +349,8 @@ void help_wizard(httrackp * opt) {
       }
       hts_main(argc, argv);
     }
+    //} else {
+    //  help("httrack",1);
   }
 
   /* Free buffers */
@@ -441,6 +455,23 @@ void help_catchurl(const char *dest_path) {
     printf("Unable to create a temporary proxy (no remaining port)\n");
 }
 
+// Créer un index.html vide
+void make_empty_index(const char *str) {
+#if 0
+  if (!fexist(fconcat(str, "index.html"))) {
+    FILE *fp = fopen(fconcat(str, "index.html"), "wb");
+
+    if (fp) {
+      fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
+      fprintf(fp,
+              "<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
+              CRLF);
+      fclose(fp);
+    }
+  }
+#endif
+}
+
 // mini-aide  (h: help)
 //           y
 void help(const char *app, int more) {
@@ -490,7 +521,6 @@ void help(const char *app, int more) {
   infomsg("  EN maximum mirror time in seconds (60=1 minute, 3600=1 hour)");
   infomsg("  AN maximum transfer rate in bytes/seconds (1000=1KB/s max)");
   infomsg(" %cN maximum number of connections/seconds (*%c10)");
-  infomsg(" %G  random pause of MIN[:MAX] seconds between files (e.g. %G5:10)");
   infomsg
     ("  GN pause transfer if N bytes reached, and wait until lock file is deleted");
   infomsg("");
@@ -533,7 +563,6 @@ void help(const char *app, int more) {
     (" %x  do not include any password for external password protected websites (%x0 include)");
   infomsg
     (" %q *include query string for local files (useless, for information purpose only) (%q0 don't include)");
-  infomsg(" %g  strip query keys for dedup ([host/pattern=]key1,key2,...)");
   infomsg
     ("  o *generate output html file in case of error (404..) (o0 don't generate)");
   infomsg("  X *purge old files after update (X0 keep delete)");
@@ -542,12 +571,10 @@ void help(const char *app, int more) {
   infomsg("");
   infomsg("Spider options:");
   infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
-  infomsg(" %K  load extra cookies from a Netscape cookies.txt");
-  infomsg(" %Y  explain which filter rule accepts or rejects a URL, then exit");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
-  infomsg("  j *parse scripts (j0 don't parse, bitmask: |1 parse default, |4 "
-          "don't parse .js |8 don't be aggressive)");
+  infomsg
+    ("  j *parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive)");
   infomsg
     ("  sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules))");
   infomsg
@@ -560,9 +587,6 @@ void help(const char *app, int more) {
     (" %s  update hacks: various hacks to limit re-transfers when updating (identical size, bogus response..)");
   infomsg
     (" %u  url hacks: various hacks to limit duplicate URLs (strip //, www.foo.com==foo.com..)");
-  infomsg("     opt out of one url-hack part: --keep-www-prefix "
-          "(www.foo.com<>foo.com), --keep-double-slashes (//), "
-          "--keep-query-order (?b&a)");
   infomsg
     (" %A  assume that a type (cgi,asp..) is always linked with a mime type (-%A php3,cgi=text/html;dat,bin=application/x-zip)");
   infomsg("     shortcut: '--assume standard' is equivalent to -%A "
@@ -571,7 +595,8 @@ void help(const char *app, int more) {
     ("     can also be used to force a specific file type: --assume foo.cgi=text/html");
   infomsg
     (" @iN internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only)");
-  infomsg(" %w  disable a specific external mime module (-%w httrack-plugin)");
+  infomsg
+    (" %w  disable a specific external mime module (-%w htsswf -%w htsjava)");
   infomsg("");
   infomsg("Browser ID:");
   infomsg
@@ -777,4 +802,7 @@ void help(const char *app, int more) {
   infomsg("[compiled: " HTS_PLATFORM_NAME "]");
 #endif
   infomsg(NULL);
+
+//  infomsg("  R  *relative links (e.g ../link)\n");
+//  infomsg("  A   absolute links (e.g /www.adr/link)\n");
 }

src/htshelp.h

@@ -45,6 +45,7 @@ typedef struct httrackp httrackp;
 
 void infomsg(const char *msg);
 void help(const char *app, int more);
+void make_empty_index(const char *str);
 void help_wizard(httrackp * opt);
 int help_query(const char *list, int def);
 void help_catchurl(const char *dest_path);

src/htsindex.c

@@ -142,6 +142,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
   char keyword[KEYW_LEN + 32];
   int i = 0;
 
+  //
+  //int WordIndexSize = 1024;
   coucal WordIndexHash = NULL;
   FILE *tmpfp = NULL;
 
@@ -176,6 +178,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
              || (strfield2(mime, "text/css"))
     ) {
     inscript = 1;
+    //} else if (strfield2(mime, "text/vnd.wap.wml")) {   // humm won't work in many cases
+    //  inscript=0;
   } else
     return 0;
 
@@ -271,6 +275,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
 
   // Process indexing for this page
   {
+    //FILE* fp=NULL;
+    //fp=fopen(concat(indexpath,"index.txt"),"ab");
     if (fp_tmpproject) {
       while(!feof(tmpfp)) {
         char line[KEYW_LEN + 32];
@@ -280,6 +286,7 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
           intptr_t e = 0;
 
           if (coucal_read(WordIndexHash, line, &e)) {
+            //if (e) {
             char BIGSTK savelst[HTS_URLMAXSIZE * 2];
 
             e++;                /* 0 means "once" */
@@ -293,9 +300,11 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
             fprintf(fp_tmpproject, "%s %d %s\n", line,
                     (int) (KEYW_SORT_MAXCOUNT - e), savelst);
             hts_primindex_size++;
+            //}
           }
         }
       }
+      //fclose(fp);
     }
   }
 
@@ -321,6 +330,7 @@ void index_finish(const char *indexpath, int mode) {
   off_t size = fpsize(fp_tmpproject);
 
   if (size > 0) {
+    //FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
     if (fp_tmpproject) {
       tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
       if (tab) {
@@ -388,6 +398,8 @@ void index_finish(const char *indexpath, int mode) {
                       if (total_hit) {
                         if (mode == 1)  // TEXT
                           fprintf(fp, "\t=%d\r\n", total_hit);
+                        //else                // HTML
+                        //  fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
                         if ((((total_hit * 1000) / hts_primindex_words) >=
                              KEYW_USELESS1K)
                             || (((total_line * 1000) / index) >=
@@ -404,6 +416,8 @@ void index_finish(const char *indexpath, int mode) {
                           if (mode == 1)        // TEXT
                             fprintf(fp, "\t(%d)\r\n",
                                     ((total_hit * 1000) / hts_primindex_words));
+                          //else                // HTML
+                          //  fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
                         }
                       }
                       if (mode == 1)    // TEXT

src/htsjava.c

@@ -0,0 +1,516 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: Java classes parser                                    */
+/* Author: Yann Philippot                                       */
+/* ------------------------------------------------------------ */
+
+/* Version: Oct/2000 */
+/* Fixed: problems with class structure (10/2000) */
+
+// htsjava.c - Parseur de classes java
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#if ( defined(_WIN32) ||defined(HAVE_SYS_TYPES_H) )
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Standard httrack module includes */
+#include "httrack-library.h"
+#include "htsopt.h"
+#include "htsdefines.h"
+
+/* Module structures */
+#include "htsmodules.h"
+
+/* We link to libhttrack, we can use its functions */
+#include "httrack-library.h"
+
+/* This file */
+#include "htsjava.h"
+
+static int reverse_endian(void) {
+  int endian = 1;
+
+  return (*((char *) &endian) == 1);
+}
+
+/* big/little endian swap */
+#define hts_swap16(A) ( (((A) & 0xFF)<<8) | (((A) & 0xFF00)>>8) )
+#define hts_swap32(A) ( (( (hts_swap16(A)) & 0xFFFF)<<16) | (( (hts_swap16(A>>16)) & 0xFFFF)) )
+
+/* Static definitions */
+static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp, RESP_STRUCT,
+                             int *);
+static unsigned short int readshort(FILE * fp);
+static int tris(httrackp * opt, char *);
+static char *printname(char[1024], char[1024]);
+
+// ** HTS_xx sinon pas pris par VC++
+#define HTS_CLASS  7
+#define HTS_FIELDREF  9
+#define HTS_METHODREF  10
+#define HTS_STRING  8
+#define HTS_INTEGER  3
+#define HTS_FLOAT  4
+#define HTS_LONG  5
+#define HTS_DOUBLE  6
+#define HTS_INTERFACE  11
+#define HTS_NAMEANDTYPE  12
+#define HTS_ASCIZ  1
+#define HTS_UNICODE  2
+
+#define JAVADEBUG 0
+
+static const char *libName = "htsjava";
+
+#ifdef _WIN32
+#define strcasecmp(a,b) stricmp(a,b)
+#define strncasecmp(a,b,n) strnicmp(a,b,n)
+#endif
+
+static int detect_mime(htsmoduleStruct * str) {
+  const char *savename = str->filename;
+
+  if (savename) {
+    int len = (int) strlen(savename);
+
+    if (len > 6 && strcasecmp(savename + len - 6, ".class") == 0) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+static int hts_detect_java(t_hts_callbackarg * carg, httrackp * opt,
+                           htsmoduleStruct * str) {
+  /* Call parent functions if multiple callbacks are chained. */
+  if (CALLBACKARG_PREV_FUN(carg, detect) != NULL) {
+    if (CALLBACKARG_PREV_FUN(carg, detect)
+        (CALLBACKARG_PREV_CARG(carg), opt, str)) {
+      return 1;                 /* Found before us, let them have the priority */
+    }
+  }
+
+  /* Check MIME */
+  if (detect_mime(str)) {
+    str->wrapper_name = libName;        /* Our ID */
+    return 1;                   /* Known format, we take it */
+  }
+
+  return 0;                     /* Unknown format */
+}
+
+static off_t fsize(const char *s) {
+  STRUCT_STAT st;
+
+  if (STAT(s, &st) == 0 && S_ISREG(st.st_mode)) {
+    return st.st_size;
+  } else {
+    return -1;
+  }
+}
+
+static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
+                          htsmoduleStruct * str) {
+  /* The wrapper_name memebr has changed: not for us anymore */
+  if (str->wrapper_name == NULL || strcmp(str->wrapper_name, libName) != 0) {
+    /* Call parent functions if multiple callbacks are chained. */
+    if (CALLBACKARG_PREV_FUN(carg, parse) != NULL) {
+      return CALLBACKARG_PREV_FUN(carg, parse) (CALLBACKARG_PREV_CARG(carg),
+                                                opt, str);
+    }
+    strcpy(str->err_msg,
+           "unexpected error: bad wrapper_name and no previous wrapper");
+    return 0;                   /* Unexpected error */
+  } else {
+    if (detect_mime(str)) {
+
+      /* (Legacy code) */
+      char catbuff[CATBUFF_SIZE];
+      FILE *fpout;
+      JAVA_HEADER header;
+      RESP_STRUCT *tab;
+      const char *file = str->filename;
+
+      str->relativeToHtmlLink = 1;
+
+#if JAVADEBUG
+      printf("fopen\n");
+#endif
+      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
+        //fprintf(stderr, "Cannot open input file.\n");
+        sprintf(str->err_msg, "Unable to open file %s", file);
+        return 0;               // une erreur..
+      }
+#if JAVADEBUG
+      printf("fread\n");
+#endif
+      //if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) {   // pas complet..
+      if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
+        fclose(fpout);
+        sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
+                (LLint) fsize(file));
+        return 0;
+      }
+#if JAVADEBUG
+      printf("header\n");
+#endif
+      // tester en tête
+      if (reverse_endian()) {
+        header.magic = hts_swap32(header.magic);
+        header.count = hts_swap16(header.count);
+      }
+      if (header.magic != 0xCAFEBABE) {
+        sprintf(str->err_msg, "non java file");
+        if (fpout) {
+          fclose(fpout);
+          fpout = NULL;
+        }
+        return 0;
+      }
+
+      tab = (RESP_STRUCT *) calloc(header.count, sizeof(RESP_STRUCT));
+      if (!tab) {
+        sprintf(str->err_msg, "Unable to alloc %d bytes",
+                (int) sizeof(RESP_STRUCT));
+        if (fpout) {
+          fclose(fpout);
+          fpout = NULL;
+        }
+        return 0;               // erreur..
+      }
+#if JAVADEBUG
+      printf("calchead\n");
+#endif
+      {
+        int i;
+
+        for(i = 1; i < header.count; i++) {
+          int err = 0;          // ++    
+
+          tab[i] = readtable(str, fpout, tab[i], &err);
+          if (!err) {
+            if ((tab[i].type == HTS_LONG) || (tab[i].type == HTS_DOUBLE))
+              i++;              //2 element si double ou float
+          } else {              // ++ une erreur est survenue!
+            if (strnotempty(str->err_msg) == 0)
+              strcpy(str->err_msg, "Internal readtable error");
+            free(tab);
+            if (fpout) {
+              fclose(fpout);
+              fpout = NULL;
+            }
+            return 0;
+          }
+        }
+
+      }
+
+#if JAVADEBUG
+      printf("addfiles\n");
+#endif
+      {
+        //unsigned int acess;
+        unsigned int Class;
+        unsigned int SClass;
+        int i;
+
+        //acess = readshort(fpout);
+        Class = readshort(fpout);
+        SClass = readshort(fpout);
+
+        for(i = 1; i < header.count; i++) {
+
+          if (tab[i].type == HTS_CLASS) {
+
+            if ((tab[i].index1 < header.count) && (tab[i].index1 >= 0)) {
+
+              if ((tab[i].index1 != SClass) && (tab[i].index1 != Class)
+                  && (tab[tab[i].index1].name[0] != '[')) {
+
+                if (!strstr(tab[tab[i].index1].name, "java/")) {
+                  char BIGSTK tempo[1024];
+
+                  tempo[0] = '\0';
+
+                  snprintf(tempo, sizeof(tempo), "%s.class", tab[tab[i].index1].name);
+#if JAVADEBUG
+                  printf("add %s\n", tempo);
+#endif
+                  if (tab[tab[i].index1].file_position >= 0)
+                    str->addLink(str, tempo);   /* tab[tab[i].index1].file_position */
+                }
+
+              }
+            } else {
+              i = header.count; // exit 
+            }
+          }
+
+        }
+      }
+
+#if JAVADEBUG
+      printf("end\n");
+#endif
+      free(tab);
+      if (fpout) {
+        fclose(fpout);
+        fpout = NULL;
+      }
+      return 1;
+
+    } else {
+      strcpy(str->err_msg, "bad MIME type");
+    }
+  }
+  return 0;                     /* Error */
+}
+
+/*
+module entry point 
+*/
+EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
+EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
+  /* Plug callback functions */
+  CHAIN_FUNCTION(opt, detect, hts_detect_java, NULL);
+  CHAIN_FUNCTION(opt, parse, hts_parse_java, NULL);
+
+  return 1;                     /* success */
+}
+
+// error: !=0 si erreur fatale
+static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
+                             RESP_STRUCT trans, int *error) {
+  char rname[1024];
+  unsigned short int length;
+  int j;
+
+  *error = 0;                   // pas d'erreur
+  trans.file_position = -1;
+  trans.type = (int) (unsigned char) fgetc(fp);
+  switch (trans.type) {
+  case HTS_CLASS:
+    strcpy(trans.name, "Class");
+    trans.index1 = readshort(fp);
+    break;
+
+  case HTS_FIELDREF:
+    strcpy(trans.name, "Field Reference");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+
+  case HTS_METHODREF:
+    strcpy(trans.name, "Method Reference");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+
+  case HTS_INTERFACE:
+    strcpy(trans.name, "Interface Method Reference");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+  case HTS_NAMEANDTYPE:
+    strcpy(trans.name, "Name and Type");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+
+  case HTS_STRING:             // CONSTANT_String
+    strcpy(trans.name, "String");
+    trans.index1 = readshort(fp);
+    break;
+
+  case HTS_INTEGER:
+    strcpy(trans.name, "Integer");
+    for(j = 0; j < 4; j++)
+      fgetc(fp);
+    break;
+
+  case HTS_FLOAT:
+    strcpy(trans.name, "Float");
+    for(j = 0; j < 4; j++)
+      fgetc(fp);
+    break;
+
+  case HTS_LONG:
+    strcpy(trans.name, "Long");
+    for(j = 0; j < 8; j++)
+      fgetc(fp);
+    break;
+  case HTS_DOUBLE:
+    strcpy(trans.name, "Double");
+    for(j = 0; j < 8; j++)
+      fgetc(fp);
+    break;
+
+  case HTS_ASCIZ:
+  case HTS_UNICODE:
+
+    if (trans.type == HTS_ASCIZ)
+      strcpy(trans.name, "HTS_ASCIZ");
+    else
+      strcpy(trans.name, "HTS_UNICODE");
+
+    {
+      char BIGSTK buffer[1024];
+      char *p;
+
+      p = &buffer[0];
+
+      //fflush(fp); 
+      trans.file_position = ftell(fp);
+      length = readshort(fp);
+      if (length < HTS_URLMAXSIZE) {
+        // while ((length > 0) && (length<500)) {
+        while(length > 0) {
+          *p++ = fgetc(fp);
+
+          length--;
+        }
+        *p = '\0';
+
+        //#if JDEBUG
+        //      if(tris(buffer)==1) printf("%s\n ",buffer);
+        //      if(tris(buffer)==2)  printf("%s\n ",printname(buffer));
+        //#endif
+        if (tris(str->opt, buffer) == 1)
+          str->addLink(str, buffer);    /* trans.file_position */
+        else if (tris(str->opt, buffer) == 2)
+          str->addLink(str, printname(rname, buffer));
+
+        strcpy(trans.name, buffer);
+      } else {                  // gros pb
+        while((length > 0) && (!feof(fp))) {
+          fgetc(fp);
+          length--;
+        }
+        if (!feof(fp)) {
+          trans.type = -1;
+        } else {
+          sprintf(str->err_msg, "Internal structure error (ASCII)");
+          *error = 1;
+        }
+        return (trans);
+      }
+    }
+    break;
+  default:
+    // printf("Type inconnue\n");
+    // on arrête tout 
+    sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
+    *error = 1;
+    return (trans);
+    break;
+  }
+  return (trans);
+}
+
+static unsigned short int readshort(FILE * fp) {
+  unsigned short int valint;
+
+  if (fread(&valint, sizeof(valint), 1, fp) == 1) {
+    if (reverse_endian())
+      return hts_swap16(valint);
+    else
+      return valint;
+  } else {
+    return 0;
+  }
+
+}
+
+static int tris(httrackp * opt, char *buffer) {
+  char catbuff[CATBUFF_SIZE];
+
+  //
+  // Java
+  if ((buffer[0] == '[') && buffer[1] == 'L' && (!strstr(buffer, "java/")))
+    return 2;
+  if (strstr(buffer, ".gif") || strstr(buffer, ".jpg")
+      || strstr(buffer, ".jpeg") || strstr(buffer, ".au"))
+    return 1;
+  // Ajouts R.X: test type
+  // Autres fichiers
+  {
+    char type[256];
+
+    if (get_httptype_sized(opt, type, sizeof(type), buffer,
+                           0)) // recognized type
+      return 1;
+    // ajout RX 05/2001
+    else if (is_dyntype(get_ext(catbuff, sizeof(catbuff), buffer)))      // asp,cgi...
+      return 1;
+  }
+  return 0;
+}
+
+static char *printname(char rname[1024], char name[1024]) {
+  char *p;
+  char *p1;
+  size_t j;
+
+  rname[0] = '\0';
+  //
+
+  p = &name[0];
+
+  if (*p != '[') {
+    return rname;  // ""
+  }
+  p += 2;
+  //rname=(char*)calloct(strlen(name)+8,sizeof(char));
+  p1 = rname;
+  for(j = 0; name[j] != '\0'; j++, p++) {
+    if (*p == '/')
+      *p1 = '.';
+    if (*p == ';') {
+      *p1 = '\0';
+      strcat(rname, ".class");
+      return rname;
+    } else
+      *p1 = *p;
+    p1++;
+  }
+  p1 -= 3;
+  *p1 = '\0';
+  return rname;
+
+}