Split -%u URL Hacks into independent www/slash/query toggles (#271)

-%u (--urlhack) bundled three dedup normalizations under one switch:
www.host == host, redundant // collapse, and query-argument reordering.
A mirror that needed one but not another (e.g. keep www. distinct) had to
turn the whole umbrella off. Add three opt-out sub-options, defaulting to
the umbrella so existing -%u/-%u0 behavior is unchanged:

  --keep-www-prefix      keep www.foo.com distinct from foo.com   (-%j)
  --keep-double-slashes  keep redundant // in the path            (-%o)
  --keep-query-order     keep query-argument order significant    (-%y)

The split is resolved once in hash_init() into norm_host/norm_slash/
norm_query and threaded through the dedup hash (htshash.c), the savename
lookup key (htsname.c) and the redirect-loop diagnostic (htsparse.c) so
all three stay consistent. fil_normalized() gains an internal
fil_normalized_ex(do_slash, do_query) core; the public
fil_normalized()/fil_normalized_filtered() keep their signatures.

Normalization (slash/query) now follows urlhack and its sub-flags
uniformly, while --strip-query stays orthogonal. So with urlhack off,
strip-query strips keys without sorting the remainder; the url_savename
urlhack-off branch is moved to the same do_slash=0/do_query=0 normalizer
the hash uses, so a URL is always looked up under the key it was stored
with (a self-lookup mismatch this otherwise introduced).

http/https are always merged in the dedup key (the scheme is stripped
regardless of -%u), so that part of the request needs no toggle.

The opt-outs are spelled positively (--keep-*) because httrack's generic
--no<opt> prefix only appends the disabling "0" for parametered options,
not "single" booleans, so --nowww-dedup would silently no-op.

opt grows three hts_boolean fields appended at the struct tail (offsets
stable, no soname bump, matching the strip_query addition in #112).

Tested by a -#test=urlhack engine self-test (hash_url_equals over each
flag combination) plus a -%u0 + --strip-query crawl case exercising the
urlhack-off savename branch.

Closes #271

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -55,54 +55,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
+        run: make check
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
-
-      - name: Print the test log on failure
-        if: failure()
-        run: cat tests/test-suite.log 2>/dev/null || true
-
-  # Reproduce the Debian buildds: they build in a minimal chroot with no
-  # python3, so the local-server tests must SKIP (exit 77), not fail. GitHub
-  # runners ship python3, so every other job hides this path; here we remove it
-  # before `make check`. This is the guard that would have caught the 3.49.10-1
-  # FTBFS (28_local-pause failed instead of skipping when python3 was absent).
-  buildd-no-python3:
-    name: build (no python3, Debian buildd)
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev
-
-      - name: Configure
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure
-
-      - name: Build
-        run: make -j"$(nproc)"
-
-      - name: Test without python3
-        run: |
-          set -euo pipefail
-          # Hide every python3* so `command -v python3` fails like it does in the
-          # buildd chroot; masking with /bin/false would still resolve.
-          sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
-            -exec mv {} {}.hidden \;
-          ! command -v python3
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -135,52 +88,13 @@ jobs:
       - name: Build
         run: make -j"$(sysctl -n hw.ncpu)"
 
-      - name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
-        # 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
-        # instantly like Linux; alias them onto lo0 so they don't stall to timeout.
-        run: |
-          set -euo pipefail
-          sudo ifconfig lo0 alias 127.0.0.2 up
-          sudo ifconfig lo0 alias 127.0.0.3 up
-
       - name: Test
-        run: |
+        run: make check
-          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
-  # Runtime smoke of the WebHTTrack launcher on macOS: it carries a Darwin-only
-  # browser path (open -W) and nothing else exercises htsserver. Install into a
-  # temp prefix, then check webhttrack brings up htsserver and serves the UI.
-  webhttrack-macos:
-    name: webhttrack smoke (macOS arm64)
-    runs-on: macos-14
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          brew install autoconf automake libtool autoconf-archive
-
-      - name: Build and install into a temp prefix
-        run: |
-          set -euo pipefail
-          ssl="$(brew --prefix openssl@3)"
-          autoreconf -fi
-          ./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib" \
-            --prefix="$RUNNER_TEMP/inst"
-          make -j"$(sysctl -n hw.ncpu)"
-          make install
-
-      - name: Smoke-test webhttrack
-        run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
-
   # Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
   # -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
   # and bounds math can truncate or wrap in ways 64-bit never reveals (the axis
@@ -213,9 +127,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
+        run: make check
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -270,9 +182,7 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: |
+        run: make check
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
@@ -315,55 +225,14 @@ jobs:
           MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
         run: |
           set -euo pipefail
-          # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
+          # Engine self-tests only; the cache trio pulls in uninstrumented zlib.
-          # skipped here (uninstrumented libz floods MSan with false positives).
+          tests="$(cd tests && ls 01_engine-*.test | grep -v -- '-cache' | tr '\n' ' ')"
-          tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
+          make check TESTS="$tests"
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs" TESTS="$tests"
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
-  # libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
-  # parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
-  # (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
-  # catastrophic backtracking -- and pins the regression seeds for the bugs the
-  # fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
-  fuzz:
-    name: fuzz (libFuzzer corpus replay, clang)
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential clang autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev
-
-      - name: Configure (fuzzers, static)
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure CC=clang \
-            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
-            LDFLAGS="-fsanitize=address,undefined" \
-            --enable-fuzzers --disable-shared
-
-      - name: Build
-        run: make -j"$(nproc)"
-
-      - name: Replay corpora
-        env:
-          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
-          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: bash fuzz/run-fuzzers.sh fuzz check
-
   # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
   # the configuration users on minimal systems build, and one libssl is not even
   # installed here so configure cannot silently re-enable it. The matrix above
@@ -394,9 +263,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
+        run: make check
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()

.github/workflows/codeql.yml

@@ -1,63 +0,0 @@
-# CodeQL static analysis (C). The security-extended suite covers the classes
-# this codebase actually fights: overflows, tainted-size allocs, format bugs.
-name: CodeQL
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-  schedule:
-    # Weekly re-scan of master so new/updated queries land without a push.
-    - cron: "17 4 * * 1"
-
-permissions:
-  contents: read
-
-concurrency:
-  group: codeql-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: analyze (c-cpp)
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: read
-      # Upload findings to the repo's code-scanning dashboard.
-      security-events: write
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev
-
-      - uses: github/codeql-action/init@v4
-        with:
-          languages: c-cpp
-          build-mode: manual
-          queries: security-extended
-          # fopen's umask-controlled 0666 is intended for mirror/cache/log
-          # output; the one credential file (cookies.txt) is kept 0600 on Unix.
-          config: |
-            query-filters:
-              - exclude:
-                  id: cpp/world-writable-file-creation
-
-      # Manual build: CodeQL traces the compiler, so build exactly what ships.
-      - name: Build
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure
-          make -j"$(nproc)"
-
-      - uses: github/codeql-action/analyze@v4
-        with:
-          category: "/language:c-cpp"

.gitignore

@@ -34,12 +34,8 @@ Makefile
 *.so.*
 *.a
 
-# make dist output; dist/ holds httrack-gh-release staging artifacts.
+# make dist output.
 /httrack-*.tar.gz
-/dist/
 
 # Editor / autotools backup files.
 *~
-
-# Python bytecode (tests/local-server.py).
-__pycache__/

AGENTS.md

@@ -6,19 +6,8 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 ## Build & test
 - Fresh clone first: `git submodule update --init src/coucal`
 - `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
-  automake, libtool), then `bash configure && make -j"$(nproc)" && make check
+  automake, libtool), then `bash configure && make && make check`. Or run
-  -j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
+  `sh build.sh` to do bootstrap + configure + make in one shot.
-  automake's parallel harness and each crawl test binds its own ephemeral-port
-  server, so `-j` never contends and a multi-minute serial run drops to
-  seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
-  automatically; only a test slower than the current longest raises the floor.
-  On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
-  spend much of their wall time asleep (server trickles, httrack self-pacing),
-  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
-  platform, macOS included: the test server raises its listen backlog
-  (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
-  `-c16` bigcrawl the way Python's default backlog of 5 did.
-  Or run `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

CONTRIBUTING.md

@@ -39,10 +39,6 @@ Welcome, and nothing to disclose. Two rules:
 
 The sign-off covers AI-assisted code too.
 
-## Translations
-
-Interface strings live in [`lang/`](lang/). See [lang/README.md](lang/README.md) for the file format and how to add or update a language.
-
 ## Bugs
 
 Open an issue with the version, OS, command used, and expected vs actual result.

Makefile.am

@@ -1,5 +1,5 @@
 
-SUBDIRS = src man m4 libtest templates lang html tests fuzz
+SUBDIRS = src man m4 libtest templates lang html tests
 
 ACLOCAL_AMFLAGS = -I m4
 

README

@@ -40,6 +40,7 @@ These are the principals limits of HTTrack for that moment. Note that we did not
 that would have solved them.
 
 - Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
+- Some java classes may not find some files on them (class included)
 - Cgi-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like -*cgi-bin*
 
 

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.9], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,11 +29,10 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
+# 3:1:0: 3.49.9 changed code but not the exported interface vs 3.49.8 (same 164
-# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
+# symbols, no struct-layout change), so bump revision only. (3:0:0 was the htsblk
-# so it stays soname .so.3; bump revision.
+# mime-buffer widening, an ABI break that moved the soname .so.2 -> .so.3.)
-# (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
+VERSION_INFO="3:1:0"
-VERSION_INFO="3:4:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 
@@ -92,8 +91,7 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
 AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
 AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
 AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
-AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
+AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
-	[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
 AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
 AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
 AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -101,16 +99,9 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
 AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
 AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
 
-# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
-# sanitizer builds, whose interceptors want the unfortified calls.
-case "$CFLAGS" in
-*-fsanitize=*) ;;
-*) AX_ADD_FORTIFY_SOURCE ;;
-esac
-
 # Force libc back into DT_NEEDED for libraries that reach it only through
-# libhttrack (the libtest callbacks), but only with a GNU-style linker; Apple
+# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
-# ld rejects these flags and links libSystem unconditionally.
+# linker; Apple ld rejects these flags and links libSystem unconditionally.
 AX_CHECK_LINK_FLAG([-Wl,--push-state,--no-as-needed,-lc,--pop-state],
 	[LIBC_FORCE_LINK="-Wl,--push-state,--no-as-needed,-lc,--pop-state"])
 AC_SUBST([LIBC_FORCE_LINK])
@@ -308,37 +299,6 @@ AC_ARG_ENABLE([online-unit-tests],
 	])
 AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
 
-## libFuzzer harnesses (fuzz/); requires clang
-AC_MSG_CHECKING(whether to build fuzzers)
-AC_ARG_ENABLE([fuzzers],
-	[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
-	[
-	case "${enableval}" in
-	no|yes)
-		fuzzers=$enableval
-		AC_MSG_RESULT($enableval)
-		;;
-	*)
-		AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
-		;;
-	esac
-	],
-	[
-	fuzzers=no
-	AC_MSG_RESULT(no)
-	])
-if test x"$fuzzers" = x"yes"; then
-	# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
-	AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
-		[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
-		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
-	# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
-	if test x"$enable_shared" != x"no"; then
-		AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
-	fi
-fi
-AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
-
 # Final output
 AC_CONFIG_FILES([
 Makefile
@@ -350,6 +310,5 @@ lang/Makefile
 html/Makefile
 libtest/Makefile
 tests/Makefile
-fuzz/Makefile
 ])
 AC_OUTPUT

debian/changelog

@@ -1,47 +1,3 @@
-httrack (3.49.12-1) unstable; urgency=medium
-
-  * New upstream release: security and crawl-correctness fixes (remote stack
-    overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
-    world-readable cookies.txt, filter-pattern denial of service) plus a new
-    --why filter diagnostic; full list in history.txt.
-  * Build with _FORTIFY_SOURCE and -fstack-protector-strong.
-
- -- Xavier Roche <xavier@debian.org>  Fri, 10 Jul 2026 21:11:09 +0200
-
-httrack (3.49.11-1) unstable; urgency=medium
-
-  * New upstream release: crawl correctness and security fixes (network-facing
-    buffer overflows, file-type detection, redirect handling) and modernized
-    web defaults; full list in history.txt.
-  * Add DEP-12 upstream metadata (#466).
-  * Bump debhelper compat to 14 (#466).
-  * Drop the redundant Priority field and update the NMU lintian override to
-    the current tag names (#466).
-
- -- Xavier Roche <xavier@debian.org>  Sun, 05 Jul 2026 00:03:18 +0200
-
-httrack (3.49.10-2) unstable; urgency=medium
-
-  * Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
-    absent (the local-server tests need python3, which the buildds lack). Add
-    patches/skip-local-pause-test-without-python3.patch to guard the test on
-    python3 up front, like its siblings, so it skips cleanly.
-
- -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 20:18:46 +0200
-
-httrack (3.49.10-1) unstable; urgency=medium
-
-  * New upstream release: new download-pacing and URL-handling options plus a
-    batch of crawl and robustness fixes (full list in history.txt).
-  * Rewrite debian/copyright in machine-readable DEP-5 format, crediting the
-    bundled minizip, md5 and coucal sources (#415).
-  * Lead the webhttrack browser dependency with chromium so httrack is not
-    dragged into the firefox-esr autoremoval cascade (#436).
-  * Override the embedded-library lint for the bundled minizip (#419).
-  * Bump Standards-Version to 4.7.4 (no changes required).
-
- -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 14:01:53 +0200
-
 httrack (3.49.9-1) unstable; urgency=medium
 
   * New upstream release: Content-Type and file-type detection fixes (trust a

debian/control

@@ -1,8 +1,9 @@
 Source: httrack
 Section: web
+Priority: optional
 Maintainer: Xavier Roche <roche@httrack.com>
-Standards-Version: 4.7.4
+Standards-Version: 4.7.0
-Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
+Build-Depends: debhelper-compat (= 13), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
 Rules-Requires-Root: no
 Homepage: http://www.httrack.com
 Vcs-Git: https://github.com/xroche/httrack.git
@@ -29,7 +30,7 @@ Description: Copy websites to your computer (Offline browser)
 Package: webhttrack
 Architecture: any
 Multi-Arch: foreign
-Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, chromium | firefox-esr | www-browser
+Depends: ${misc:Depends}, ${shlibs:Depends}, webhttrack-common, sensible-utils, firefox-esr | chromium | www-browser
 Replaces: webhttrack-common (<< 3.43.9-2)
 Breaks: webhttrack-common (<< 3.43.9-2)
 Suggests: httrack, httrack-doc

debian/copyright

@@ -7,8 +7,9 @@ Files: *
 Copyright: 1998-2026 Xavier Roche and other contributors
 License: GPL-3+
 Comment:
- htsbasenet.h links against the system OpenSSL library (originally by Eric
+ The engine includes contributions from Yann Philippot (src/htsjava.c,
- Young); no OpenSSL/SSLeay code is bundled here.
+ src/htsjava.h). htsbasenet.h links against the system OpenSSL library
+ (originally by Eric Young); no OpenSSL/SSLeay code is bundled here.
 
 Files: src/minizip/*
 Copyright: 1998-2010 Gilles Vollant

debian/libhttrack-dev.files

@@ -1,4 +1,5 @@
 usr/include/httrack
 usr/lib/*/libhttrack.{so}
+usr/lib/*/libhtsjava.{so}
 usr/share/httrack/libtest
 usr/lib/*/httrack/libtest

debian/libhttrack3.files

@@ -1,2 +1,3 @@
 usr/lib/*/libhttrack.so.3*
+usr/lib/*/libhtsjava.so.3*
 usr/share/httrack/templates

debian/source/lintian-overrides

@@ -1,6 +1,4 @@
-# Maintainer uploads sign the changelog as xavier@debian.org while the control
+httrack source: changelog-should-mention-nmu
-# Maintainer is roche@httrack.com; lintian reads the address mismatch as an NMU.
-httrack source: no-nmu-in-changelog
 httrack source: source-nmu-has-incorrect-version-number
 
 # The bundled HTML pages are the genuine upstream documentation taken from

debian/upstream/metadata

@@ -1,6 +0,0 @@
----
-Repository: https://github.com/xroche/httrack.git
-Repository-Browse: https://github.com/xroche/httrack
-Bug-Database: https://github.com/xroche/httrack/issues
-Bug-Submit: https://github.com/xroche/httrack/issues/new
-Contact: Xavier Roche <roche@httrack.com>

fuzz/Makefile.am

@@ -1,45 +0,0 @@
-# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
-if FUZZERS
-noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
-	fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx
-endif
-
-AM_CPPFLAGS = \
-	@DEFAULT_CFLAGS@ \
-	@THREADS_CFLAGS@ \
-	@V6_FLAG@ \
-	@LFS_FLAG@ \
-	-I$(top_srcdir)/src \
-	-I$(top_srcdir)/src/coucal
-
-# Static-link libhttrack.la: the internal symbols are hidden in the .so.
-AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
-LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
-
-fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
-fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
-fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
-fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
-fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
-fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
-fuzz_url_SOURCES = fuzz-url.c fuzz.h
-fuzz_header_SOURCES = fuzz-header.c fuzz.h
-fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
-
-# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
-EXTRA_DIST = README.md run-fuzzers.sh \
-	corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
-	corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
-	corpus/idna/idna.txt corpus/idna/unicode.txt \
-	corpus/idna/regress-multilabel-leak.txt \
-	corpus/entities/entities.txt \
-	corpus/unescape/percent.txt \
-	corpus/filters/filter.bin corpus/filters/filter-size.bin \
-	corpus/filters/regress-empty-subject-unique.bin \
-	corpus/filters/redos-star-classes.bin \
-	corpus/url/http-url.txt corpus/url/relative-path.txt \
-	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
-	corpus/header/full-response.txt corpus/header/redirect.txt \
-	corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
-	corpus/cachendx/regress-overadvance.bin \
-	corpus/cachendx/regress-truncated-entry.bin

fuzz/README.md

@@ -1,15 +0,0 @@
-# Fuzzing httrack
-
-libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
-
-```sh
-./bootstrap
-mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
-CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
-  LDFLAGS="-fsanitize=address,undefined" \
-  bash /path/to/httrack/configure --enable-fuzzers --disable-shared
-make
-bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60   # 60s per target
-```
-
-Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.

fuzz/corpus/cachendx/new-format.txt

@@ -1,7 +0,0 @@
-8
-CACHE-1.1
-28
-Mon, 01 Jan 2024 00:00:00 GMT
-www.example.com
-/index.html
-123

fuzz/corpus/cachendx/old-format.txt

@@ -1,5 +0,0 @@
-3
-1.0
-www.example.com
-/page
-5

fuzz/corpus/cachendx/regress-overadvance.bin

@@ -1,2 +0,0 @@
-32768
-CACHE-1.1

fuzz/corpus/cachendx/regress-truncated-entry.bin

@@ -1,5 +0,0 @@
-8
-CACHE-1.1
-1
-x
-www.example.com

fuzz/corpus/charset/latin1.txt

@@ -1 +0,0 @@
-
café naďve ¤

fuzz/corpus/charset/sjis.txt

@@ -1 +0,0 @@
-コンピュータ

fuzz/corpus/entities/entities.txt

@@ -1 +0,0 @@
-&<>A☃é&notarealentity;�

fuzz/corpus/filters/regress-empty-subject-unique.bin

@@ -1 +0,0 @@
-**((

fuzz/corpus/header/full-response.txt

@@ -1,10 +0,0 @@
-HTTP/1.1 200 OK
-Content-Type: text/html; charset=utf-8
-Content-Length: 1234
-Content-Encoding: gzip
-Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
-Etag: "abc"
-Location: http://example.com/x
-Set-Cookie: ID=42; path=/; domain=.example.com
-Content-Range: bytes 0-99/100
-Transfer-Encoding: chunked

fuzz/corpus/header/redirect.txt

@@ -1,3 +0,0 @@
-HTTP/1.0 301 Moved
-Location: /elsewhere
-Content-Disposition: attachment; filename="a.pdf"

fuzz/corpus/idna/idna.txt

@@ -1 +0,0 @@
-xn--bcher-kva.example.com

fuzz/corpus/idna/regress-multilabel-leak.txt

@@ -1 +0,0 @@
-büchev.例å­bücheple

fuzz/corpus/idna/unicode.txt

@@ -1 +0,0 @@
-bücher.例子.example

fuzz/corpus/meta/meta-charset.html

@@ -1 +0,0 @@
-<html><head><meta charset="utf-8"></head><body>x</body></html>

fuzz/corpus/meta/meta-content-first.html

@@ -1 +0,0 @@
-<meta content="text/html; charset=gb2312" http-equiv="content-type">

fuzz/corpus/meta/meta-http-equiv.html

@@ -1 +0,0 @@
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">

fuzz/corpus/meta/meta-truncated.html

@@ -1 +0,0 @@
-<meta charset='utf-8

fuzz/corpus/meta/meta-unquoted.html

@@ -1 +0,0 @@
-<meta   charset = utf-8 />

fuzz/corpus/unescape/percent.txt

@@ -1 +0,0 @@
-%41%zz%%20%c3%a9+%2e%2e%2f

fuzz/corpus/url/http-url.txt

@@ -1 +0,0 @@
-http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag

fuzz/corpus/url/regress-file-empty-path.txt

@@ -1 +0,0 @@
-file://

fuzz/corpus/url/regress-long-path-abort.txt

@@ -1 +0,0 @@
-ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../

fuzz/corpus/url/relative-path.txt

@@ -1 +0,0 @@
-ftp://ftp.example.com/pub/../file.txt

fuzz/fuzz-cachendx.c

@@ -1,65 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
-   walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
-   Mirrors the -#C cache-listing scan (htscoremain.c). */
-#include "fuzz.h"
-#include "htscache.h"
-#include "htslib.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  const char *const end = buf + size;
-  char firstline[256];
-  char *a = buf;
-
-  /* header: two length-prefixed fields (version, last-modified) */
-  a += cache_brstr(a, firstline, sizeof(firstline));
-  a += cache_brstr(a, firstline, sizeof(firstline));
-
-  /* body: newline-delimited host/file/position triples; the length-prefixed
-     scan must stay inside the buffer */
-  while (a != NULL && a < end) {
-    char BIGSTK line[HTS_URLMAXSIZE * 2];
-    char linepos[256];
-    int pos;
-
-    a = strchr(a + 1, '\n');
-    if (a == NULL)
-      break;
-    a++;
-    a += cache_binput(a, end, line, HTS_URLMAXSIZE);
-    a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
-    a += cache_binput(a, end, linepos, 200);
-    sscanf(linepos, "%d", &pos);
-    (void) pos;
-  }
-
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-charset.c

@@ -1,84 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
-   UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
-#include "fuzz.h"
-#include "htscharset.h"
-
-static const char *const charsets[] = {
-    "utf-8",    "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
-    "us-ascii", "shift_jis",  "euc-jp",     "iso-2022-jp", "gb2312",
-    "big5",     "euc-kr",     "koi8-r",     "utf-16",      "unknown-charset",
-};
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  const char *charset;
-  char *s;
-
-  if (size == 0)
-    return 0;
-  charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
-  data++, size--;
-  s = fuzz_strdup(data, size);
-
-  {
-    char *utf8 = hts_convertStringToUTF8(s, size, charset);
-    freet(utf8);
-  }
-  {
-    char *enc = hts_convertStringFromUTF8(s, size, charset);
-    freet(enc);
-  }
-  {
-    size_t nChars = 0;
-    hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
-
-    if (ucs != NULL) {
-      char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
-      freet(back);
-      freet(ucs);
-    }
-  }
-  {
-    size_t i = 0;
-
-    while (i < size) {
-      hts_UCS4 uc = 0;
-      const size_t nr = hts_readUTF8(s + i, size - i, &uc);
-      char out[8];
-
-      if (nr == 0)
-        break;
-      hts_writeUTF8(uc, out, sizeof(out));
-      i += nr;
-    }
-  }
-
-  freet(s);
-  return 0;
-}

fuzz/fuzz-entities.c

@@ -1,51 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
-   destination size, so truncation bounds get exercised too. */
-#include "fuzz.h"
-#include "htsencoding.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  static const size_t dsizes[] = {1, 2, 8, 64, 4096};
-  size_t dsize;
-  char *src, *dest;
-
-  if (size == 0)
-    return 0;
-  dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
-  data++, size--;
-  src = fuzz_strdup(data, size);
-  dest = malloct(dsize);
-
-  (void) hts_unescapeEntities(src, dest, dsize);
-  (void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
-
-  freet(dest);
-  freet(src);
-  return 0;
-}

fuzz/fuzz-filters.c

@@ -1,65 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
-   here). Input splits on the first NUL: pattern, then subject string. */
-#include "fuzz.h"
-#include "htsfilters.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  const char *joker = buf;
-  const uint8_t *sep = memchr(data, '\0', size);
-  /* subject in its own allocation so ASan bounds it apart from the pattern */
-  char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
-                          : fuzz_strdup(data + size, 0);
-
-  (void) strjoker(nom, joker, NULL, NULL);
-  {
-    LLint sz = (LLint) size;
-    int size_flag = 0;
-
-    (void) strjoker(nom, joker, &sz, &size_flag);
-  }
-  (void) strjokerfind(nom, joker);
-  {
-    char *filter = malloct(strlen(joker) + 2);
-    char *filters[1];
-    LLint sz = (LLint) size;
-    int size_flag = 0, depth = 0;
-
-    filter[0] = '-';
-    memcpy(filter + 1, joker, strlen(joker) + 1);
-    filters[0] = filter;
-    (void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
-    freet(filter);
-  }
-
-  freet(nom);
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-header.c

@@ -1,74 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
-   status line, treathead on each following header. Both consume raw bytes
-   off the wire and copy fields into fixed htsblk buffers; treathead also
-   mutates its line in place and drives cookie parsing. */
-#include "fuzz.h"
-#include "htslib.h"
-#include "htsbauth.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  htsblk r;
-  t_cookie *cookie = calloct(1, sizeof(*cookie));
-  char *line = malloct(size + 1);
-  char *p = buf;
-  int first = 1;
-
-  memset(&r, 0, sizeof(r));
-  cookie->max_len = (int) sizeof(cookie->data);
-  r.location = malloct(HTS_URLMAXSIZE * 2);
-  r.location[0] = '\0';
-
-  /* feed one header line at a time, as the receive loop does */
-  while (p != NULL && *p != '\0') {
-    char *nl = strchr(p, '\n');
-    size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
-    size_t i, len = 0;
-
-    /* binput drops every '\r' on the wire; mirror it */
-    for (i = 0; i < n; i++)
-      if (p[i] != '\r')
-        line[len++] = p[i];
-    line[len] = '\0';
-    if (first) {
-      treatfirstline(&r, line);
-      first = 0;
-    } else {
-      treathead(cookie, "www.example.com", "/", &r, line);
-    }
-    p = (nl != NULL) ? nl + 1 : NULL;
-  }
-
-  freet(r.location);
-  freet(line);
-  freet(cookie);
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-idna.c

@@ -1,47 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
-#include "fuzz.h"
-#include "htscharset.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *s = fuzz_strdup(data, size);
-
-  {
-    char *idna = hts_convertStringUTF8ToIDNA(s, size);
-    freet(idna);
-  }
-  {
-    char *utf8 = hts_convertStringIDNAToUTF8(s, size);
-    freet(utf8);
-  }
-  (void) hts_isStringIDNA(s, size);
-
-  freet(s);
-  return 0;
-}

fuzz/fuzz-meta.c

@@ -1,40 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
-   <meta> charset declaration. */
-#include "fuzz.h"
-#include "htscharset.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *html = fuzz_strdup(data, size);
-  char *charset = hts_getCharsetFromMeta(html, size);
-
-  freet(charset);
-  freet(html);
-  return 0;
-}

fuzz/fuzz-unescape.c

@@ -1,53 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
-   output buffer size, so the bounded-copy contract is exercised. */
-#include "fuzz.h"
-#include "httrack-library.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  static const size_t bsizes[] = {1, 2, 16, 256, 8192};
-  size_t bsize;
-  char *s, *catbuff;
-
-  if (size == 0)
-    return 0;
-  bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
-  data++, size--;
-  s = fuzz_strdup(data, size);
-  catbuff = malloct(bsize);
-
-  (void) unescape_http(catbuff, bsize, s);
-  (void) unescape_http_unharm(catbuff, bsize, s, 0);
-  (void) unescape_http_unharm(catbuff, bsize, s, 1);
-  unescape_amp(s);
-
-  freet(catbuff);
-  freet(s);
-  return 0;
-}

fuzz/fuzz-url.c

@@ -1,49 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
-   is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
-   in place. */
-#include "fuzz.h"
-#include "htscore.h"
-#include "htslib.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *s = fuzz_strdup(data, size);
-  lien_adrfil *af = calloct(1, sizeof(*af));
-  /* fil_simplifie rewrites in place and may grow an empty path to "./" */
-  char *path = malloct(size + 3);
-
-  (void) ident_url_absolute(s, af);
-  memcpy(path, s, size + 1);
-  fil_simplifie(path);
-
-  freet(path);
-  freet(af);
-  freet(s);
-  return 0;
-}

fuzz/run-fuzzers.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-# Drive every built harness against its seed corpus.
-#   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
-#   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
-# Replay is crash/leak-only and never mutates; the per-unit -timeout guards
-# both modes against pathological slowdowns (e.g. pre-#501 strjoker).
-set -euo pipefail
-
-srcdir=$(cd "$(dirname "$0")" && pwd)
-bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
-mode=${2:-20}
-
-status=0
-for f in "$bld"/fuzz-*; do
-    if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
-    case "$f" in *.o | *.c | *.dSYM) continue ;; esac
-    name=$(basename "$f")
-    corpus="$srcdir/corpus/${name#fuzz-}"
-    if [ "$mode" = "check" ]; then
-        echo "=== $name (replay) ==="
-        [ -d "$corpus" ] || continue
-        if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
-            echo "*** $name FAILED on its corpus" >&2
-            status=1
-        fi
-        continue
-    fi
-    work=$(mktemp -d)
-    args=("$work")
-    [ -d "$corpus" ] && args+=("$corpus")
-    echo "=== $name (${mode}s) ==="
-    if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
-        -artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
-        echo "*** $name FAILED; artifacts:" >&2
-        ls -l "$work" >&2
-        status=1
-    else
-        rm -rf "$work"
-    fi
-done
-exit $status

history.txt

@@ -4,59 +4,7 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
-3.49-12
+3.49-9
-+ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
-+ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
-+ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
-+ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
-+ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
-+ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
-+ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
-+ Fixed: cookies.txt was created world-readable (#511)
-+ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
-+ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
-+ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
-+ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
-+ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
-+ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
-+ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
-
-3.49-11
-+ New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
-+ New: advertise deflate in Accept-Encoding and decode deflate responses (#450)
-+ New: follow <source> and <track> media elements as embedded links (#451)
-+ New: added modern web MIME types to the type/extension table (#448)
-+ Fixed: enforce the -E time limit during a slow transfer instead of only between files (#481)
-+ Fixed: sniff the leading bytes of a download so a misdeclared Content-Type no longer renames a correct URL extension
-+ Fixed: fast transfers could be saved under their temporary .delayed placeholder name (#5, #107)
-+ Fixed: follow a redirect that maps to the same saved file instead of writing a self-pointing stub (#159)
-+ Fixed: several network-facing buffer overflows in the FTP, Java and HTML parsers
-+ Fixed: the htsjava plugin could not be loaded (hidden entry points, stale library name)
-+ Fixed: HTML-escape truncation and a cache-buffer leak in the parser
-+ Changed: modernized the default User-Agent to an honest HTTrack identifier (#449)
-+ Changed: decode the full WHATWG set of HTML named character references (#443)
-+ Changed: refreshed stale HTTP status, proxy-port and TLS-floor constants (#453)
-+ Changed: multiple internal hardening, build, test and CI improvements
-
-3.49-10
-+ New: --cookies-file to preload a Netscape cookies.txt before crawling (#215)
-+ New: --pause to space out file downloads by a random delay (#185)
-+ New: --strip-query to drop selected query keys from the dedup naming (#112)
-+ Changed: split the -%u URL hacks into independent --keep-www-prefix, --keep-double-slashes and --keep-query-order toggles (#271)
-+ Fixed: follow a redirect Location after dropping its #fragment, instead of requesting the fragment and polluting the saved name (#204)
-+ Fixed: escaped brackets inside a *[...] filter character class (#148)
-+ Fixed: honor the server's Content-Range when resuming a partial download, instead of appending overlapping bytes (#198)
-+ Fixed: abort the download as soon as the response type is excluded by -mime:, instead of fetching then discarding the body (#58)
-+ Fixed: keep size-based filter rules neutral until the file size is known (#143)
-+ Fixed: stop the mirror with a clean fatal error on a cache write failure, instead of crashing (#174, #219)
-+ Fixed: stop the 412/416 partial re-get loop on --continue and --update (#206)
-+ Fixed: keep an unrecognized URL tail instead of mangling it to .html (#115)
-+ Fixed: honor --tolerant (-%B) on a broken Content-Length, and fix an out-of-bounds read it exposed (#32, #41)
-+ Fixed: fall back to the next resolved address when a connection fails or stalls, instead of hanging on a dead IPv6 address
-+ Fixed: report why a -%L URL list could not be loaded (#49)
-+ Changed: multiple internal hardening, build and CI improvements
-
-.49-9
 + Fixed: file-type detection from the Content-Type header: trust a declared type over a binary URL extension, honor --assume under the delayed type check, and keep a known extension against a bogus or empty Content-Type (#267, #29, #56)
 + Fixed: an uninitialized-buffer read when the Content-Type is empty (#411)
 + Fixed: restored C++ source-compatibility of the installed headers so reverse dependencies (httraqt) build again (#413)

html/faq.html

@@ -609,9 +609,10 @@ most mirrors do work. We still are working to improve the mirror quality of HTTr
 <br>
 
 </a><a NAME="Q2b">Q: <strong>Some Java applets do not work properly!</strong><br>
-A: <em>Java applets may not work in some cases, because HTTrack does not parse compiled class files to find
+A: <em>Java applets may not work in some cases, for example if HTTrack failed to detect all included classes
-the resources they load. Java applets often need to be online anyway, since remote files are fetched directly
+or files called within the class file. Sometimes, Java applets need to be online, because remote files are
-at runtime.<br>
+directly caught. Finally, the site structure can be incompatible with the class (always try to keep the original site structure
+when you want to get Java classes)<br>
 If there is no way to make some classes work properly, you can exclude them with the filters. 
 They will be available, but only online.
 </em>

html/fcguide.html

@@ -243,7 +243,7 @@ Build options:
 Spider options:
   bN accept cookies in cookies.txt (0=do not accept,* 1=accept) (--cookies[=N])
   u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (--check-type[=N])
-  j *parse scripts (j0 don't parse) (--parse-java[=N])
+  j *parse Java Classes (j0 don't parse) (--parse-java[=N])
   sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) (--robots[=N])
  %h  force HTTP/1.0 requests (reduce update features, only for old servers or proxies) (--http-10)
  %B  tolerant requests (accept bogus responses on some servers, but not standard!) (--tolerant)
@@ -417,7 +417,7 @@ site. Specifically, the defauls are:
   X *purge old files after update (X0 keep delete)
   bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
   u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
-  j *parse scripts (j0 don't parse)
+  j *parse Java Classes (j0 don't parse)
   sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
   C  create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)
   f *log file mode
@@ -497,9 +497,9 @@ about your browser retaining them.
 <p align=justify> This causes different document types to be analyzed
 differently.
 
-<pre><b><i>  j *parse scripts (j0 don't parse) </i></b></pre>
+<pre><b><i>  j *parse Java Classes (j0 don't parse) </i></b></pre>
 
-<p align=justify> This causes scripts to be parsed looking for
+<p align=justify> This causes Java class files to be parsed looking for
 URLs.
 
 <pre><b><i>  sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) </i></b></pre>
@@ -1178,7 +1178,7 @@ accepted and sent back in order to allow access.
 Spider options:
  bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
  u   check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
- j   *parse scripts (j0 don't parse)
+ j   *parse Java Classes (j0 don't parse)
  sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
  %h  force HTTP/1.0 requests (reduce update features, only for old servers or proxies)
  %B  tolerant requests (accept bogus responses on some servers, but not standard!)
@@ -2451,10 +2451,12 @@ work.  We still are working to improve the mirror quality of HTTrack.
 
 <p align=justify><b>Q: Some Java applets do not work properly! </b>
 
-<p align=justify> A: Java applets may not work in some cases, because
+<p align=justify> A: Java applets may not work in some cases, for
-HTTrack does not parse compiled class files to find the resources they
+example if HTTrack failed to detect all included classes or files called
-load.  Java applets often need to be online anyway, since remote files
+within the class file.  Sometimes, Java applets need to be online,
-are fetched directly at runtime.
+because remote files are directly caught.  Finally, the site structure
+can be incompatible with the class (always try to keep the original site
+structure when you want to get Java classes)
 
 <p align=justify> If there is no way to make some classes work properly,
 you can exclude them with the filters.  They will be available, but only

html/filters.html

@@ -247,7 +247,7 @@ See also: The <a href="faq.html#VF1">FAQ</a><br>
         <td>the \ character</td>
       </tr>
       <tr>
-        <td nowrap><tt>*[\[,\]]</tt></td>
+        <td nowrap><tt>*[\[\]]</tt></td>
         <td>the [ or ] character</td>
       </tr>
       <tr>

html/httrack.man.html

@@ -764,9 +764,9 @@ check, * u1 check but /, u2 check always)
 <td width="78%">
 
 
-<p>*parse scripts (j0 don t parse, bitmask: |1 parse
+<p>*parse Java Classes (j0 don t parse, bitmask: |1 parse
-default, |4 don t parse .js |8 don t be aggressive)
+default, |2 don t parse .class |4 don t parse .js |8 don t
-(--parse-java[=N])</p></td></tr>
+be aggressive) (--parse-java[=N])</p></td></tr>
 <tr valign="top" align="left">
 <td width="11%"></td>
 <td width="6%">
@@ -889,8 +889,8 @@ only) (--protocol[=N])</p></td></tr>
 <td width="78%">
 
 
-<p>disable a specific external mime module (-%w
+<p>disable a specific external mime module (-%w htsswf -%w
-httrack-plugin) (--disable-module &lt;param&gt;)</p></td></tr>
+htsjava) (--disable-module &lt;param&gt;)</p></td></tr>
 </table>
 
 <p style="margin-left:11%; margin-top: 1em"><b>Browser
@@ -2175,6 +2175,10 @@ Several scripts generating complex filenames may not find
 them (ex:
 img.src=&rsquo;image&rsquo;+a+Mobj.dst+&rsquo;.gif&rsquo;)</small></p>
 
+<p style="margin-left:11%; margin-top: 1em"><small>- Some
+java classes may not find some files on them (class
+included)</small></p>
+
 <p style="margin-left:11%; margin-top: 1em"><small>-
 Cgi-bin links may not work properly in some cases
 (parameters needed). To avoid them: use filters like

html/options.html

@@ -288,8 +288,8 @@ This option define whether the engine has to abandon a host if a timeout/"too sl
 &P extended parsing, attempt to parse all links (even in unknown tags or Javascript)
 This option activates the extended parsing, that attempt to find links in unknown Html code/javascript
 
-j *parse scripts (j0 don't parse)
+j *parse Java Classes (j0 don't parse)
-This option defines whether the engine has to parse scripts or not to catch included files
+This option define whether the engine has to parse java files or not to catch included files
 
 I *make an index (I0 don't make)
 This option define whether the engine has to generate an index.html on the top directory

html/plug.html

@@ -126,6 +126,7 @@ Note: the <i>Initialization</i>, <i>Main functions</i>, <i>Options handling</i>
 <li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
 <li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
 <li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
+<li>the <tt>htsjava.c</tt> source file (the java class plugin ; overrides 'detect' and 'parse')</li>
 <li>the example given at the end of this document</li>
 </ul>
 

html/step9_opt6.html

@@ -118,8 +118,8 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
   <br>Avoid "never", because the local mirror could be bogus
   </small><br><br>
 <!-- -->
-  <li>Parse scripts</li>
+  <li>Parse java files</li>
-  <br><small>Must the engine parse scripts to seek included filenames?
+  <br><small>Must the engine parse .java files (java classes) to seek included filenames?
   <br>It is checked by default
   </small><br><br>
 <!-- -->

lang/README.md

@@ -1,37 +0,0 @@
-# Translating HTTrack
-
-Interface strings live here, one `.txt` file per language. `English.txt` is the reference: every other file maps each English string to its translation.
-
-## File format
-
-Plain text, entries in consecutive pairs of lines:
-
-```
-<English string>
-<translation>
-```
-
-The first line of a pair is the lookup key and must stay identical to the one in `English.txt`; translate only the second line. Missing entries fall back to the English text at runtime, so a partial translation works.
-
-Preserve any `\r\n`, `\t` and `printf` placeholders (`%s`, `%d`, ...) in the translation.
-
-A few `LANGUAGE_*` entries at the top describe the file itself:
-
-| Key | Meaning |
-| --- | --- |
-| `LANGUAGE_NAME` | Name shown in the language picker, in its own language (`Deutsch`, not `German`) |
-| `LANGUAGE_ISO` | ISO 639 code, with region if needed (`de`, `pt_BR`) |
-| `LANGUAGE_CHARSET` | Encoding the file is saved in (`ISO-8859-1`, `windows-1251`, `UTF-8`, ...) |
-| `LANGUAGE_AUTHOR` | Your name and contact |
-| `LANGUAGE_WINDOWSID` | Windows locale name used by WinHTTrack (`German (Standard)`) |
-
-Save the file in exactly its declared `LANGUAGE_CHARSET`; an editor that rewrites it as UTF-8 will corrupt the non-ASCII bytes.
-
-## Adding or updating a language
-
-1. Copy `English.txt` to `<Language>.txt`, or edit the existing file.
-2. Translate each second line; leave the English keys untouched.
-3. Fill in the `LANGUAGE_*` header for a new file.
-4. Open a pull request, or attach the file to a GitHub issue.
-
-When new strings land in `English.txt` they show up untranslated (as English) until a translator fills them in.

m4/ax_add_fortify_source.m4

@@ -1,119 +0,0 @@
-# ===========================================================================
-#  https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_ADD_FORTIFY_SOURCE
-#
-# DESCRIPTION
-#
-#   Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
-#   redefinition warnings, other cpp warnings or linker. Some distributions
-#   (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
-#   their compilers, leading to unnecessary warnings in the form of
-#
-#     <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
-#     <built-in>: note: this is the location of the previous definition
-#
-#   which is a problem if -Werror is enabled. This macro checks whether
-#   _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
-#   to CPPFLAGS.
-#
-#   Newer mingw-w64 msys2 package comes with a bug in
-#   headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
-#   and would need -lssp or -fstack-protector.  See
-#   https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
-#   link it.
-#
-# LICENSE
-#
-#   Copyright (c) 2017 David Seifert <soap@gentoo.org>
-#   Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved.  This file is offered as-is, without any
-#   warranty.
-
-#serial 10
-
-AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
-    ac_save_cflags=$CFLAGS
-    ac_cwerror_flag=yes
-    AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
-    ax_add_fortify_3_failed=
-    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],
-            [[
-                #ifndef _FORTIFY_SOURCE
-                    return 0;
-                #else
-                    _FORTIFY_SOURCE_already_defined;
-                #endif
-            ]]
-        )],
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([[
-                #define _FORTIFY_SOURCE 3
-                #include <string.h>
-                int main(void) {
-                    char *s = " ";
-                    strcpy(s, "x");
-                    return strlen(s)-1;
-                }
-              ]]
-            )],
-            [
-              AC_MSG_RESULT([yes])
-              CFLAGS=$ac_save_cflags
-              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
-            ], [
-              AC_MSG_RESULT([no])
-              ax_add_fortify_3_failed=1
-            ],
-        ),
-        [
-          AC_MSG_RESULT([no])
-          ax_add_fortify_3_failed=1
-        ])
-    if test -n "$ax_add_fortify_3_failed"
-    then
-    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],
-            [[
-                #ifndef _FORTIFY_SOURCE
-                    return 0;
-                #else
-                    _FORTIFY_SOURCE_already_defined;
-                #endif
-            ]]
-        )],
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([[
-                #define _FORTIFY_SOURCE 2
-                #include <string.h>
-                int main(void) {
-                    char *s = " ";
-                    strcpy(s, "x");
-                    return strlen(s)-1;
-                }
-              ]]
-            )],
-            [
-              AC_MSG_RESULT([yes])
-              CFLAGS=$ac_save_cflags
-              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
-            ], [
-              AC_MSG_RESULT([no])
-              CFLAGS=$ac_save_cflags
-            ],
-        ),
-        [
-          AC_MSG_RESULT([no])
-          CFLAGS=$ac_save_cflags
-        ])
-    fi
-])

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "12 July 2026" "httrack website copier"
+.TH httrack 1 "27 June 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -24,7 +24,6 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-EN, \-\-max\-time[=N]\fR ]
 [ \fB\-AN, \-\-max\-rate[=N]\fR ]
 [ \fB\-%cN, \-\-connection\-per\-second[=N]\fR ]
-[ \fB\-%G, \-\-pause\fR ]
 [ \fB\-GN, \-\-max\-pause[=N]\fR ]
 [ \fB\-cN, \-\-sockets[=N]\fR ]
 [ \fB\-TN, \-\-timeout[=N]\fR ]
@@ -50,8 +49,6 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-%p, \-\-preserve\fR ]
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
-[ \fB\-%K, \-\-cookies\-file\fR ]
-[ \fB\-%Y, \-\-why\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -157,8 +154,6 @@ maximum mirror time in seconds (60=1 minute, 3600=1 hour) (\-\-max\-time[=N])
 maximum transfer rate in bytes/seconds (1000=1KB/s max) (\-\-max\-rate[=N])
 .IP \-%cN
 maximum number of connections/seconds (*%c10) (\-\-connection\-per\-second[=N])
-.IP \-%G
-random pause of MIN[:MAX] seconds between files (e.g. %G5:10) (\-\-pause <param>)
 .IP \-GN
 pause transfer if N bytes reached, and wait until lock file is deleted (\-\-max\-pause[=N])
 .SS Flow control:
@@ -217,14 +212,10 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 .SS Spider options:
 .IP \-bN
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
-.IP \-%K
-load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
-.IP \-%Y
-explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j
-*parse scripts (j0 don't parse, bitmask: |1 parse default, |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
+*parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
 .IP \-sN
 follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules)) (\-\-robots[=N])
 .IP \-%h
@@ -248,7 +239,7 @@ can also be used to force a specific file type: \-\-assume foo.cgi=text/html
 .IP \-@iN
 internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only) (\-\-protocol[=N])
 .IP \-%w
-disable a specific external mime module (\-%w httrack\-plugin) (\-\-disable\-module <param>)
+disable a specific external mime module (\-%w htsswf \-%w htsjava) (\-\-disable\-module <param>)
 .SS Browser ID:
 .IP \-F
 user\-agent field sent in HTTP headers (\-F "user\-agent name") (\-\-user\-agent <param>)
@@ -514,6 +505,8 @@ that would have solved them.
 .SM
 \- Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
 .SM
+\- Some java classes may not find some files on them (class included)
+.SM
 \- Cgi\-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like \-*cgi\-bin*
 .SH BUGS
 Please reports bugs to

src/Makefile.am

@@ -44,7 +44,7 @@ httrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 proxytrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 
-lib_LTLIBRARIES = libhttrack.la
+lib_LTLIBRARIES = libhttrack.la libhtsjava.la
 
 htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h
 proxytrack_SOURCES = proxy/main.c \
@@ -62,7 +62,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htsname.c htsrobots.c htstools.c htswizard.c \
 	htsalias.c htsthread.c htsindex.c htsbauth.c \
 	htsmd5.c htszlib.c htswrap.c htsconcat.c \
-	htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
+	htsmodules.c htscharset.c punycode.c htsencoding.c \
 	md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
 	hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
@@ -70,7 +70,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h  \
 	htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
 	htshelp.h htsindex.h htslib.h htsmd5.h \
-	htsmodules.h htsname.h htsnet.h htssniff.h \
+	htsmodules.h htsname.h htsnet.h \
 	htsopt.h htsrobots.h htsthread.h  \
 	htstools.h htswizard.h htswrap.h htszlib.h  \
 	htsstrings.h htsarrays.h httrack-library.h \
@@ -82,6 +82,14 @@ libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(OPENSSL_LIBS) $(DL_LIBS) $
 libhttrack_la_CFLAGS = $(AM_CFLAGS) -DLIBHTTRACK_EXPORTS -DZLIB_CONST
 libhttrack_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO)
 
+libhtsjava_la_SOURCES = htsjava.c htsjava.h
+libhtsjava_la_LIBADD = $(THREADS_LIBS) $(DL_LIBS) libhttrack.la
+# This thin JNI wrapper reaches libc only through libhttrack, so the direct
+# libc edge is dropped from DT_NEEDED (library-not-linked-against-libc). Force
+# libc back as a dependency; configure gates the flag since only a GNU-style
+# linker accepts it (LIBC_FORCE_LINK is empty on e.g. macOS).
+libhtsjava_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO) $(LIBC_FORCE_LINK)
+
 EXTRA_DIST = httrack.h webhttrack \
 		coucal/murmurhash3.h.diff \
 		coucal/murmurhash3.h.orig \
@@ -111,6 +119,7 @@ EXTRA_DIST = httrack.h webhttrack \
 		coucal/README.md \
 		coucal/sample.c \
 		coucal/tests.c \
+		htsjava.vcproj \
 		httrack.dsp httrack.dsw httrack.vcproj \
 		libhttrack.dsp libhttrack.dsw libhttrack.vcproj \
 		webhttrack.dsp webhttrack.dsw webhttrack.vcproj

src/htsalias.c

@@ -112,12 +112,6 @@ const char *hts_optalias[][4] = {
   {"include-query-string", "-%q", "single", ""},
   {"strip-query", "-%g", "param1",
    "strip [host/pattern=]key1,key2,... from URLs"},
-  {"cookies-file", "-%K", "param1",
-   "load extra cookies from a Netscape cookies.txt"},
-  {"why", "-%Y", "param1",
-   "explain which filter rule accepts or rejects a URL, then exit"},
-  {"pause", "-%G", "param1",
-   "random pause of MIN[:MAX] seconds between files"},
   {"generate-errors", "-o", "single", ""},
   {"do-not-generate-errors", "-o0", "single", ""},
   {"purge-old", "-X", "param", ""},
@@ -285,6 +279,7 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
       char *position;
       int need_param = 1;
 
+      //int return_param=0;
       int pos;
 
       command[0] = param[0] = addcommand[0] = '\0';
@@ -362,6 +357,7 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
               strlcatbuff(return_argv[0], "0", return_argv_size);
             else if (strcmp(param, "on") == 0) {
               // on is the default
+              // strcatbuff(return_argv[0],"1");
             } else
               strlcatbuff(return_argv[0], param, return_argv_size);
           }

src/htsback.c

@@ -43,12 +43,14 @@ Please visit our Website: http://www.httrack.com
 
 #include "htsback.h"
 
+//#ifdef _WIN32
 #include "htsftp.h"
 #if HTS_USEZLIB
 #include "htszlib.h"
 #else
 #error HTS_USEZLIB not defined
 #endif
+//#endif
 
 #ifdef _WIN32
 #ifndef __cplusplus
@@ -68,15 +70,6 @@ static int slot_can_be_cached_on_disk(const lien_back * back);
 static int slot_can_be_cleaned(const lien_back * back);
 static int slot_can_be_finalized(httrackp * opt, const lien_back * back);
 
-/* Which hard quota, if any, is currently aborting the mirror. */
-typedef enum {
-  HTS_MIRROR_LIMIT_NONE = 0,
-  HTS_MIRROR_LIMIT_SIZE,
-  HTS_MIRROR_LIMIT_TIME,
-} hts_mirror_limit;
-
-static hts_mirror_limit back_mirror_limit(httrackp *opt);
-
 struct_back *back_new(httrackp *opt, int back_max) {
   int i;
   struct_back *sback = calloct(1, sizeof(struct_back));
@@ -533,13 +526,12 @@ int back_nsoc_overall(const struct_back * sback) {
 
 /* generate temporary file on lien_back */
 /* Note: utf-8 */
-static int create_back_tmpfile(httrackp *opt, lien_back *const back,
+static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
-                               const char *ext) {
   // do not use tempnam() but a regular filename
   back->tmpfile_buffer[0] = '\0';
   if (back->url_sav != NULL && back->url_sav[0] != '\0') {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.z", 
-             back->url_sav, ext);
+             back->url_sav);
     back->tmpfile = back->tmpfile_buffer;
     if (structcheck(back->tmpfile) != 0) {
       hts_log_print(opt, LOG_WARNING, "can not create directory to %s", 
@@ -547,8 +539,9 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
       return -1;
     }
   } else {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer),
-             StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
+             "%s/tmp%d.z", StringBuff(opt->path_html_utf8),
+             opt->state.tmpnameid++);
     back->tmpfile = back->tmpfile_buffer;
   }
   /* OK */
@@ -556,32 +549,6 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
   return 0;
 }
 
-/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
-   existing file moved the good copy to back->tmpfile before truncating url_sav.
-   commit keeps the new file and drops the backup; else restore it so an aborted
-   transfer leaves the previous copy intact. Skips the zlib .z temp. */
-static void back_finalize_backup(httrackp *opt, lien_back *const back,
-                                 const hts_boolean commit) {
-  if (back->tmpfile == NULL || back->r.compressed)
-    return;
-  if (commit) {
-    (void) UNLINK(back->tmpfile); /* new copy is good; drop the backup */
-  } else {
-    if (back->r.out != NULL) {
-      fclose(back->r.out);
-      back->r.out = NULL;
-    }
-    (void) UNLINK(back->url_sav); /* drop the failed partial */
-    /* On rename failure keep the backup: it still holds the good copy, so
-       losing it would be worse than an orphaned temp. */
-    if (RENAME(back->tmpfile, back->url_sav) != 0)
-      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
-                    "could not restore %s; previous copy kept as %s",
-                    back->url_sav, back->tmpfile);
-  }
-  back->tmpfile = NULL;
-}
-
 // objet (lien) téléchargé ou transféré depuis le cache
 //
 // fermer les paramètres de transfert,
@@ -605,19 +572,15 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
         && back[p].r.size != back[p].r.totalsize && !opt->tolerant) {
       if (back[p].status == STATUS_READY) {
         hts_log_print(opt, LOG_WARNING,
-                      "incomplete transfer (expected " LLintP
+                      "file not stored in cache due to bogus state (broken size, expected "
-                      " bytes, got " LLintP
+                      LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
-                      "): file not cached, will be retried on the next update"
+                      back[p].r.size, back[p].url_adr, back[p].url_fil);
-                      " (use -%%B to cache anyway): %s%s",
-                      back[p].r.totalsize, back[p].r.size, back[p].url_adr,
-                      back[p].url_fil);
       } else {
         hts_log_print(opt, LOG_INFO,
                       "incomplete file not yet stored in cache (expected "
                       LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
                       back[p].r.size, back[p].url_adr, back[p].url_fil);
       }
-      back_finalize_backup(opt, &back[p], HTS_FALSE);
       return -1;
     }
 
@@ -631,12 +594,13 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
 #if HTS_USEZLIB
         if (back[p].r.compressed) {
           if (back[p].r.size > 0) {
+            //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
             // stats
             back[p].compressed_size = back[p].r.size;
             // en mémoire -> passage sur disque
             if (!back[p].r.is_write) {
               // do not use tempnam() but a regular filename
-              if (create_back_tmpfile(opt, &back[p], "z") == 0) {
+              if (create_back_tmpfile(opt, &back[p]) == 0) {
                 assertf(back[p].tmpfile != NULL);
                 /* note: tmpfile is utf-8 */
                 back[p].r.out = FOPEN(back[p].tmpfile, "wb");
@@ -709,9 +673,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
           }
         }
 #endif
-        /* Body fully received: keep the freshly written url_sav, drop the
-           backup of the previous copy. */
-        back_finalize_backup(opt, &back[p], HTS_TRUE);
         /* Write mode to disk */
         if (back[p].r.is_write && back[p].r.adr != NULL) {
           freet(back[p].r.adr);
@@ -918,12 +879,11 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                          back[p].url_fil, NULL);
           } else {
             /* Partial file, but marked as "ok" ? */
-            hts_log_print(
+            hts_log_print(opt, LOG_WARNING,
-                opt, LOG_WARNING,
+                          "file not stored in cache due to bogus state (incomplete type with %s (%d), size "
-                "file with unresolved type not cached (%s (%d), size " LLintP
+                          LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode,
-                "): %s%s",
+                          (LLint) back[p].r.size, back[p].url_adr,
-                back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size,
+                          back[p].url_fil);
-                back[p].url_adr, back[p].url_fil);
           }
         }
 
@@ -940,9 +900,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
       }
     }
   }
-  /* Aborted, error, or not ready: url_sav (if written) is broken; restore the
-     previous copy from the backup. */
-  back_finalize_backup(opt, &back[p], HTS_FALSE);
   return -1;
 }
 
@@ -966,6 +923,7 @@ int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
     /* clear everything but connection: switch, close, and reswitch */
     back_connxfr(src, &tmp);
     back_delete(opt, cache, sback, p);
+    //deletehttp(src);
     back_connxfr(&tmp, src);
     src->req.flush_garbage = 1; /* ignore CRLF garbage */
     return 1;
@@ -1400,49 +1358,7 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
   return 0;
 }
 
-/* Move a still-writing .delayed placeholder to its final name (#483). */
-hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
-                                const char *newname) {
-  hts_boolean renamed;
-
-  if (!back->r.is_write || back->tmpfile != NULL ||
-      !IS_DELAYED_EXT(back->url_sav) || strcmp(back->url_sav, newname) == 0)
-    return HTS_TRUE; /* nothing bound to the placeholder name */
-  if (back->r.out != NULL) {
-    fclose(back->r.out);
-    back->r.out = NULL;
-  }
-  renamed = RENAME(back->url_sav, newname) == 0 ? HTS_TRUE : HTS_FALSE;
-  if (renamed && (back->status == STATUS_READY ||
-                  (back->r.out = FOPEN(newname, "ab")) != NULL)) {
-    filenote(&opt->state.strc, newname, NULL);
-    hts_log_print(opt, LOG_DEBUG, "moved placeholder %s to %s", back->url_sav,
-                  newname);
-    return HTS_TRUE;
-  }
-  /* partial lost: drop only what we own (Windows rename won't overwrite) */
-  hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "unable to move %s to %s",
-                back->url_sav, newname);
-  back->r.statuscode = STATUSCODE_INVALID;
-  strcpybuff(back->r.msg, "Write error on disk");
-  back->r.is_write = 0;
-  (void) UNLINK(renamed ? newname : back->url_sav);
-  return HTS_FALSE;
-}
-
 // effacer entrée
-/* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
-void back_delayed_discard(httrackp *opt, lien_back *back) {
-  if (back->r.out != NULL) {
-    fclose(back->r.out);
-    back->r.out = NULL;
-  }
-  back->r.is_write = 0;
-  if (opt != NULL)
-    url_savename_refname_remove(opt, back->url_adr, back->url_fil);
-  (void) UNLINK(back->url_sav);
-}
-
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p) {
   lien_back *const back = sback->lnk;
@@ -1450,12 +1366,6 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
 
   assertf(p >= 0 && p < back_max);
   if (p >= 0 && p < sback->count) {     // on sait jamais..
-    /* mid-write cancel: drop a .delayed placeholder; real-named partials
-       survive for resume (--continue) */
-    if (back[p].r.is_write && IS_DELAYED_EXT(back[p].url_sav) &&
-        (back[p].status != STATUS_READY || back[p].r.statuscode <= 0)) {
-      back_delayed_discard(opt, &back[p]);
-    }
     // Vérificateur d'intégrité
 #if DEBUG_CHECKINT
     _CHECKINT(&back[p], "Appel back_delete")
@@ -1475,6 +1385,7 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                       back[p].url_adr, back[p].url_fil, back[p].url_sav);
       }
       if (cache != NULL) {
+        //hts_log_print(opt, LOG_TRACE, "finalizing from back_delete");
         back_finalize(opt, cache, sback, p);
       }
     }
@@ -1615,6 +1526,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     if (back[p].r.soc != INVALID_SOCKET) {      /* we never know */
       deletehttp(&back[p].r);
     }
+    //memset(&(back[p].r), 0, sizeof(htsblk)); 
     hts_init_htsblk(&back[p].r);
     back[p].r.location = back[p].location_buffer;
 
@@ -1622,6 +1534,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     strcpybuff(back[p].url_adr, adr);
     strcpybuff(back[p].url_fil, fil);
     strcpybuff(back[p].url_sav, save);
+    //back[p].links_index = links_index;
     // copier referer si besoin
     strcpybuff(back[p].referer_adr, "");
     strcpybuff(back[p].referer_fil, "");
@@ -1689,62 +1602,83 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     }
     // tester cache
     if ((strcmp(adr, "file://"))        /* pas fichier */
-        && ((!test) ||
+        &&((!test) || (cache->type == 1))       /* cache prioritaire, laisser passer en test! */
-            (cache->type == 1)) /* cache prioritaire, laisser passer en test! */
+        &&((strnotempty(save)) || (strcmp(fil, "/robots.txt") == 0))) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
-        &&
+#if HTS_FAST_CACHE
-        ((strnotempty(save)) || (strcmp(fil, "/robots.txt") ==
-                                 0))) { // si en test on ne doit pas utiliser le
-                                        // cache sinon telescopage avec le 302..
       intptr_t hash_pos;
       int hash_pos_return = 0;
-
+#else
+      char *a = NULL;
+#endif
+#if HTS_FAST_CACHE
       if (cache->hashtable) {
+#else
+      if (cache->use) {
+#endif
         char BIGSTK buff[HTS_URLMAXSIZE * 4];
 
+#if HTS_FAST_CACHE
         strcpybuff(buff, adr);
         strcatbuff(buff, fil);
         hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
+#else
+        buff[0] = '\0';
+        strcatbuff(buff, "\n");
+        strcatbuff(buff, adr);
+        strcatbuff(buff, "\n");
+        strcatbuff(buff, fil);
+        strcatbuff(buff, "\n");
+        a = strstr(cache->use, buff);
+#endif
 
+        // Ok, noté en cache->. mais bien présent dans le cache ou sur disque?
+#if HTS_FAST_CACHE
         // negative values when data is not in cache
         if (hash_pos_return < 0) {
-          if (!test) { // not test mode
+#else
-            /* note: no check with IS_DELAYED_EXT() enabled - postcheck by
+        if (a) {
-             * client please! */
+#endif
-            if (save[0] != '\0' && !IS_DELAYED_EXT(save) &&
+          if (!test) {          // non mode test
-                fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <=
+#if HTS_FAST_CACHE==0
-                    0) { // final file missing or empty
+            int pos = -1;
+
+            a += strlen(buff);
+            sscanf(a, "%d", &pos);      // lire position
+#endif
+
+#if HTS_FAST_CACHE==0
+            if (pos < 0) {      // pas de mise en cache data, vérifier existence
+#endif
+              /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
+              if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <= 0) {  // fichier final n'existe pas ou est vide!
                 int found = 0;
 
-              /* It is possible that the file has been moved due to changes in
+                /* It is possible that the file has been moved due to changes in build structure */
-               * build structure */
                 {
                   char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
                   htsblk r;
 
                   previous_save[0] = '\0';
-                r = cache_readex(opt, cache, adr, fil, /*head */ NULL,
+                  r =
+                    cache_readex(opt, cache, adr, fil, /*head */ NULL,
                                  /*bound to back[p] (temporary) */
                                  back[p].location_buffer, previous_save, /*ro */
                                  1);
                   /* Is supposed to be on disk only */
                   if (r.is_write && previous_save[0] != '\0') {
-                  /* Exists, but with another (old) filename: rename (almost)
+                    /* Exists, but with another (old) filename: rename (almost) silently */
-                   * silently */
+                    if (strcmp(previous_save, save) != 0
-                  if (strcmp(previous_save, save) != 0 &&
+                        && fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) {
-                      fexist_utf8(
-                          fconv(catbuff, sizeof(catbuff), previous_save))) {
                       rename(fconv(catbuff, sizeof(catbuff), previous_save),
                              fconv(catbuff2, sizeof(catbuff2), save));
                       if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
                         found = 1;
                         hts_log_print(opt, LOG_DEBUG,
-                                    "File '%s' has been renamed since last "
+                                      "File '%s' has been renamed since last mirror to '%s' ; applying changes",
-                                    "mirror to '%s' ; applying changes",
                                       previous_save, save);
                       } else {
                         hts_log_print(opt, LOG_ERROR,
-                                    "Could not rename '%s' to '%s' ; will have "
+                                      "Could not rename '%s' to '%s' ; will have to retransfer it",
-                                    "to retransfer it",
                                       previous_save, save);
                       }
                     }
@@ -1754,38 +1688,51 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
                 /* Not found ? */
                 if (!found) {
-                // invalidate: gone from disk, force a refetch
+#if HTS_FAST_CACHE
                   hash_pos_return = 0;
-                if (opt->norecatch) {
+#else
-                  if (!fexist_utf8(fconv(
+                  a = NULL;
-                          catbuff, sizeof(catbuff),
+#endif
-                          save))) { // declared but missing: user erased it
+                  // dévalider car non présent sur disque dans structure originale!!!
-                    FILE *fp =
+                  // sinon, le fichier est ok à priori, mais on renverra un if-modified-since pour
-                        FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
+                  // en être sûr
+                  if (opt->norecatch) { // tester norecatch
+                    if (!fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {   // fichier existe pas mais déclaré: on l'a effacé
+                      FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
 
                       if (fp)
                         fclose(fp);
                       hts_log_print(opt, LOG_WARNING,
-                                  "Previous file '%s' not found (erased by "
+                                    "Previous file '%s' not found (erased by user ?), ignoring: %s%s",
-                                  "user ?), ignoring: %s%s",
                                     save, back[p].url_adr, back[p].url_fil);
                     }
                   } else {
                     hts_log_print(opt, LOG_WARNING,
-                                "Previous file '%s' not found (erased by user "
+                                  "Previous file '%s' not found (erased by user ?), recatching: %s%s",
-                                "?), recatching: %s%s",
                                   save, back[p].url_adr, back[p].url_fil);
                   }
                 }
               }                 // fsize() <= 0
+#if HTS_FAST_CACHE==0
+            }
+#endif
           }
         }
         //
       } else {
+#if HTS_FAST_CACHE
         hash_pos_return = 0;
+#else
+        a = NULL;
+#endif
       }
 
-      if (hash_pos_return) { // in cache, with data
+      // Existe pas en cache, ou bien pas de cache présent
+#if HTS_FAST_CACHE
+      if (hash_pos_return) {    // OK existe en cache (et données aussi)!
+#else
+      if (a != NULL) {          // OK existe en cache (et données aussi)!
+#endif
         const int cache_is_prioritary = cache->type == 1
           || opt->state.stop != 0;
         if (cache_is_prioritary) {      // cache prioritaire (pas de test if-modified..)
@@ -1827,6 +1774,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
             }
             back[p].r.notmodified = 1;  // fichier non modifié
             back[p].status = STATUS_READY;      // OK prêt
+            //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);        // not modified
             back_set_finished(sback, p);
 
             // finalize transfer
@@ -1841,6 +1789,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
           } else {              // erreur
             // effacer r
             hts_init_htsblk(&back[p].r);
+            //memset(&(back[p].r), 0, sizeof(htsblk)); 
             back[p].r.location = back[p].location_buffer;
             // et continuer (chercher le fichier)
           }
@@ -1906,8 +1855,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #endif
         }
       }
-      /* Not in cache ; maybe in temporary cache ? Warning: non-movable
+      /* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */
-         "url_sav" */
       else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
         const off_t file_size = fsize_utf8(itemback->url_sav);
 
@@ -2036,6 +1984,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       // ouvrir liaison, envoyer requète
       // ne pas traiter ou recevoir l'en tête immédiatement
       hts_init_htsblk(&back[p].r);
+      // memset(&(back[p].r), 0, sizeof(htsblk));
       back[p].r.location = back[p].location_buffer;
       // fresh connect: address list not yet probed, start at the first
       sback->connect_fallback[p].addr_index = 0;
@@ -2086,15 +2035,20 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #if HTS_USEOPENSSL
       else if (strfield(back[p].url_adr, "https://")) {     // let's rock
         back[p].r.ssl = 1;
+        // back[p].r.ssl_soc = NULL;
         back[p].r.ssl_con = NULL;
       }
 #endif
 
       if (!back_trylive(opt, cache, sback, p)) {
 #if HTS_XGETHOST
-        back[p].status = STATUS_WAIT_DNS; // host name resolution attempt
+#if HDEBUG
-        soc = INVALID_SOCKET;             // not opened yet
+        printf("back_solve..\n");
-        if (host_wait(opt, &back[p])) {   // ready (file, or cached dns)
+#endif
+        back[p].status = STATUS_WAIT_DNS;       // tentative de résolution du nom de host
+        soc = INVALID_SOCKET;   // pas encore ouverte
+        back_solve(opt, &back[p]);      // préparer
+        if (host_wait(opt, &back[p])) { // prêt, par ex fichier ou dispo dans dns
 #if HDEBUG
           printf("ok, dns cache ready..\n");
 #endif
@@ -2147,9 +2101,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
         back[p].rateout = -1;   // pas de gestion (default)
       }
 
-      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où
+      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est
-      // cela est rattrapable (exemple: 301,302 moved xxx -> refresh sur la
+      // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
-      // page!)
+      //if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
 
 #if CNXDEBUG
       printf("Xfopen ok, poll..\n");
@@ -2167,6 +2121,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       if (soc == INVALID_SOCKET) {      // erreur socket
         back[p].status = STATUS_READY;  // FINI
         back_set_finished(sback, p);
+        //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
         back[p].r.soc = INVALID_SOCKET;
       } else {
         if (!back[p].r.is_file)
@@ -2187,6 +2142,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
     // note: si il y a erreur (404,etc) status=2 (terminé/échec) mais
     // le lien est considéré comme traité
+    //if (back[p].soc<0)  // erreur
+    //  return -1;
 
     return 0;
   } else {
@@ -2237,9 +2194,40 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 }
 
 #if HTS_XGETHOST
-// Resolution is synchronous inside the connect path; no pre-resolve step, so
+// attendre que le host (ou celui du proxy) ait été résolu
-// the host is always immediately ready.
+// si c'est un fichier, la résolution est immédiate
-int host_wait(httrackp *opt, lien_back *back) { return 1; }
+// idem pour ftp://
+void back_solve(httrackp * opt, lien_back * back) {
+  assertf(opt != NULL);
+  assertf(back != NULL);
+  if ((!strfield(back->url_adr, "file://"))
+      && !strfield(back->url_adr, "ftp://")
+    ) {
+    const char *a;
+
+    if (!(back->r.req.proxy.active))
+      a = back->url_adr;
+    else
+      a = back->r.req.proxy.name;
+    assertf(a != NULL);
+    a = jump_protocol_const(a);
+    if (check_hostname_dns(a)) {
+      hts_log_print(opt, LOG_DEBUG, "resolved: %s", a);
+    } else {
+      hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
+    }
+    //if (hts_dnstest(opt, a, 1) == 2) { // non encore testé!..
+    //  hts_log_print(opt, LOG_DEBUG, "resolving in background: %s", a);
+    //}
+  }
+}
+
+// détermine si le host a pu être résolu
+int host_wait(httrackp * opt, lien_back * back) {
+  // Always synchronous. No more background DNS resolution
+  // (does not really improve performances)
+  return 1;
+}
 #endif
 
 // élimine les fichiers non html en backing (anticipation)
@@ -2250,7 +2238,6 @@ int host_wait(httrackp *opt, lien_back *back) { return 1; }
 static int slot_can_be_cleaned(const lien_back * back) {
   return (back->status == STATUS_READY) // ready
     /* Check autoclean */
-         && (!back->locked)   // not held by hts_wait_delayed (name pending)
     && (!back->testmode)        // not test mode
     && (strnotempty(back->url_sav))     // filename exists
     && (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
@@ -2279,6 +2266,11 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) {
         (void) back_flush_output(opt, cache, sback, i); // flush output buffers
         usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr,
                     back[i].url_fil);
+        //if (back[i].links_index >= 0) {
+        //  assertf(back[i].links_index < opt->hash->max_lien);
+        //  opt->hash->liens[back[i].links_index]->pass2 = -1;
+        //  // *back[i].pass2_ptr=-1;  // Done!
+        //}
         /* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
         {
           int index = hash_read(opt->hash, back[i].url_sav, NULL, HASH_STRUCT_FILENAME );       // lecture type 0 (sav)
@@ -2426,40 +2418,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
-  /* Time/size limit exceeded past grace: abort in-flight transfers so no wait
-     loop starves (#481, #77). FTP slots stay, their thread owns the socket. */
-  if (!back_checkmirror(opt)) {
-    const hts_mirror_limit limit = back_mirror_limit(opt);
-    const char *const reason =
-        (limit == HTS_MIRROR_LIMIT_SIZE) ? "size limit" : "time limit";
-    const char *const slotmsg = (limit == HTS_MIRROR_LIMIT_SIZE)
-                                    ? "Mirror Size Limit"
-                                    : "Mirror Time Out";
-    int aborted = 0;
-    unsigned int i;
-
-    for (i = 0; i < (unsigned int) back_max; i++) {
-      if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
-        if (back[i].r.soc != INVALID_SOCKET) {
-          deletehttp(&back[i].r);
-        }
-        back[i].r.soc = INVALID_SOCKET;
-        /* drop a .delayed placeholder; real partials survive for resume */
-        if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
-          back_delayed_discard(opt, &back[i]);
-        back[i].r.statuscode = STATUSCODE_TIMEOUT;
-        strcpybuff(back[i].r.msg, slotmsg);
-        back[i].status = STATUS_READY;
-        back_set_finished(sback, i);
-        aborted++;
-      }
-    }
-    if (aborted > 0)
-      hts_log_print(opt, LOG_WARNING, "%s reached, %d transfer(s) aborted",
-                    reason, aborted);
-    return;
-  }
-
   // recevoir tant qu'il y a des données (avec un maximum de max_loop boucles)
   do_wait = 0;
   gestion_timeout = 0;
@@ -2468,6 +2426,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
     busy_state = busy_recv = 0;
 
+#if 0
+    check_rate(stat_timestart, opt->maxrate);   // vérifier taux de transfert
+#endif
     // inscrire les sockets actuelles, et rechercher l'ID la plus élevée
     FD_ZERO(&fds);
     FD_ZERO(&fds_c);
@@ -2478,6 +2439,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
     max_c = 1;
     for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+      // for(i=0;i<back_max;i++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       // en cas de gestion du connect préemptif
@@ -2515,6 +2477,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       if ((back[i].status > 0) && (back[i].status < 100)) {     // en réception http
 
 #if BDEBUG==1
+        //printf("....socket in progress: %d\n",back[i].r.soc);
 #endif
         // non local et non ftp
         if (!back[i].r.is_file) {
@@ -2601,6 +2564,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
     // recevoir les données arrivées
     for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+      // for(i=0;i<back_max;i++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       if (back[i].status > 0) {
@@ -2740,6 +2704,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back[i].rateout_time = back[i].ka_time_start;
             }
             // envoyer header
+            //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
             HTS_STAT.stat_nrequests++;
             if (!back[i].head_request)
               http_sendhead(opt, opt->cookie, 0, back[i].send_too,
@@ -2806,6 +2771,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 #if HTS_XGETHOST
       else if (back[i].status == STATUS_WAIT_DNS) {     // attendre gethostbyname
 #if DEBUGDNS
+        //printf("status 101 for %s\n",back[i].url_adr);
 #endif
 
         if (!gestion_timeout)
@@ -2925,10 +2891,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
           // range size hack old location
 
 #if HTS_DIRECTDISK
-          // Shortcut: store the file directly on disk when possible,
+          // Court-circuit:
-          // sparing memory
+          // Peut-on stocker le fichier directement sur disque?
-          if (back[i].status &&
+          // Ahh que ca serait vachement mieux et que ahh que la mémoire vous dit merci!
-              !back[i].locked) { // name still pending when locked
+          if (back[i].status) {
             if (back[i].r.is_write == 0) {      // mode mémoire
               if (back[i].r.adr == NULL) {      // rien n'a été écrit
                 if (!back[i].testmode) {        // pas mode test
@@ -2946,8 +2912,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                 strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "gz") == 0
                                 && strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "tgz") == 0
                               ) {
-                              if (create_back_tmpfile(opt, &back[i], "z") ==
+                              if (create_back_tmpfile(opt, &back[i]) == 0) {
-                                  0) {
                                 assertf(back[i].tmpfile != NULL);
                                 /* note: tmpfile is utf-8 */
                                 if ((back[i].r.out =
@@ -2960,20 +2925,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
-                              /* Re-fetch over an existing file (#77 follow-up):
-                                 move the good copy aside before truncating it
-                                 so an aborted transfer can restore it. url_sav
-                                 is still written normally (file list intact).
-                               */
-                              back[i].tmpfile = NULL;
-                              if (fexist_utf8(back[i].url_sav)) {
-                                if (create_back_tmpfile(opt, &back[i], "bak") !=
-                                        0 ||
-                                    RENAME(back[i].url_sav, back[i].tmpfile) !=
-                                        0) {
-                                  back[i].tmpfile = NULL;
-                                }
-                              }
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {
@@ -3094,6 +3045,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
           if (back[i].status == 1) {
             if (back[i].is_chunk) {     // attendre prochain chunk
               if (back[i].r.size == back[i].r.totalsize) {      // fin chunk!
+                //printf("chunk end at %d\n",back[i].r.size);
                 back[i].status = STATUS_CHUNK_CR;       /* fetch ending CRLF */
                 if (back[i].chunk_adr != NULL) {
                   freet(back[i].chunk_adr);
@@ -3157,8 +3109,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
             }
 
             if (back[i].r.totalsize >= 0) {     // tester totalsize
+              //if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) {    // tester totalsize
               if (back[i].r.totalsize != back[i].r.size) {      // pas la même!
                 if (!opt->tolerant) {
+                  //#if HTS_CL_IS_FATAL
                   deleteaddr(&back[i].r);
                   if (back[i].r.size < back[i].r.totalsize)
                     back[i].r.statuscode = STATUSCODE_CONNERROR;        // recatch
@@ -3167,13 +3121,14 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           " expected)", (LLint) back[i].r.size,
                           (LLint) back[i].r.totalsize);
                 } else {
+                  //#else
                   // Un warning suffira..
                   hts_log_print(opt, LOG_WARNING,
                                 "Incorrect length (" LLintP "!=" LLintP
-                                " expected) for %s%s",
+                                " expected) for %s%s", (LLint) back[i].r.size,
-                                (LLint) back[i].r.size,
                                 (LLint) back[i].r.totalsize, back[i].url_adr,
                                 back[i].url_fil);
+                  //#endif
                 }
               }
             }
@@ -3300,8 +3255,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                          back[i].r.size);
 #endif
                   /* End */
+                  //if (back[i].status==STATUS_CHUNK_CR) {
                   back[i].status = STATUS_READY;        // fin  
                   back_set_finished(sback, i);
+                  //}
 
                   // finalize transfer if not temporary
                   if (!IS_DELAYED_EXT(back[i].url_sav)) {
@@ -3512,11 +3469,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     continue;
                   }
 
-                  // The server really sent 304 here; the *-hacks below force
-                  // NOT_MODIFIED only after confirming the file is complete.
-                  const hts_boolean server_sent_304 =
-                      (back[i].r.statuscode == HTTP_NOT_MODIFIED);
-
                   /*
                      Solve "false" 416 problems
                    */
@@ -3545,21 +3497,23 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.statuscode == 406) {    // 'Not Acceptable'
                     back[i].r.statuscode = HTTP_OK;
                   }
-                  // On update, keep the good copy on error (mask as 304); skip
+                  // 'do not erase already downloaded file'
-                  // resume paths so a stale-partial 416 still re-fetches.
+                  // on an updated file
-                  if (HTTP_IS_ERROR(back[i].r.statuscode) &&
+                  // with an error : consider a 304 error
-                      back[i].is_update && !back[i].testmode &&
+                  if (!opt->delete_old) {
-                      back[i].range_req_size == 0 && back[i].url_sav[0] &&
+                    if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
-                      fexist_utf8(back[i].url_sav)) {
+                        && !back[i].testmode) {
+                      if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
                         hts_log_print(opt, LOG_DEBUG,
-                                  "Error %d (%s) ignored on update, keeping "
+                                      "Error ignored %d (%s) because of 'no purge' option for %s%s",
-                                  "existing copy: %s%s",
                                       back[i].r.statuscode, back[i].r.msg,
                                       back[i].url_adr, back[i].url_fil);
                         back[i].r.statuscode = HTTP_NOT_MODIFIED;
                         deletehttp(&back[i].r);
                         back[i].r.soc = INVALID_SOCKET;
                       }
+                    }
+                  }
                   // Various hacks to limit re-transfers when updating a mirror
                   // Force update if same size detected
                   if (opt->sizehack) {
@@ -3611,7 +3565,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) {       // 'OK'
                         if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) {  // not HTML
                           if (strnotempty(back[i].url_sav)) {   // target found
-                            off_t size = fsize_utf8(back[i].url_sav);
+                            int size = fsize_utf8(back[i].url_sav);     // target size
 
                             if (size >= 0) {
                               if (back[i].r.totalsize == size) {        // same size!
@@ -3706,21 +3660,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     }
                   }
 
-                  // Out-of-protocol 304 to a Range resume: the file is still
-                  // partial, so drop it and refetch instead of trusting it.
-                  if (server_sent_304 && back[i].range_req_size > 0) {
-                    url_savename_refname_remove(opt, back[i].url_adr,
-                                                back[i].url_fil);
-                    UNLINK(back[i].url_sav);
-                    deletehttp(&back[i].r);
-                    back[i].r.soc = INVALID_SOCKET;
-                    back[i].r.statuscode = STATUSCODE_NON_FATAL;
-                    strcpybuff(back[i].r.msg,
-                               "Bogus 304 on resume, restarting");
-                    back[i].status = STATUS_READY;
-                    back_set_finished(sback, i);
-                  }
-
                   /* sinon, continuer */
                   /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok récupérer body? */
                   // head: terminé
@@ -3786,6 +3725,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                     back[i].url_adr, back[i].url_fil);
 
                       // finalize
+                      //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);        // not modified
                       if (back[i].r.statuscode > 0) {
                         hts_log_print(opt, LOG_TRACE, "finalizing after cache load");
                         back_finalize(opt, cache, sback, i);
@@ -3795,11 +3735,37 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                              back[i].url_adr, back[i].url_fil);
 #endif
 
+                      //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                     } else {    // erreur
                       back[i].status = STATUS_READY;    // terminé
                       back_set_finished(sback, i);
+                      //printf("erreur cache\n");
+
                     }
 
+/********** NO - must complete the body! ********** */
+#if 0
+                  } else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
+                             || (back[i].r.statuscode == 412)
+                             || (back[i].r.statuscode == 416)
+                    ) {         // Ne pas prendre le html, erreurs connues et gérées
+#if HTS_DEBUG_CLOSESOCK
+                    DEBUG_W
+                      ("back_wait(301,302,303,307,412,416..): deletehttp\n");
+#endif
+                    // Couper connexion
+                    /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
+                    back_maydeletehttp(opt, cache, sback, i);
+
+                    back[i].status = STATUS_READY;      // terminé
+                    back_set_finished(sback, i);
+                    // finalize
+                    if (back[i].r.statuscode > 0) {
+                      hts_log_print(opt, LOG_TRACE, "finalizing redirect & 4xx");
+                      back_finalize(opt, cache, sback, i);
+                    }
+#endif
+/********** **************************** ********** */
                   }
                   // MIME type excluded by a -mime: filter: abort, don't fetch
                   // the body (#58)
@@ -3839,7 +3805,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       const hts_boolean range_ok =
                           back[i].r.crange > 0 && resume >= 0 &&
                           resume <= (LLint) sz &&
-                          back[i].r.crange_end == back[i].r.crange - 1 &&
+                          back[i].r.crange_end + 1 == back[i].r.crange &&
                           (back[i].r.totalsize < 0 ||
                            back[i].r.totalsize ==
                                back[i].r.crange_end - resume + 1);
@@ -3908,19 +3874,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           if (fp) {
                             LLint alloc_mem = resume + 1;
 
-                            // Bound the in-memory buffer to a 32-bit size (real
+                            if (back[i].r.totalsize >= 0)
-                            // in-RAM resources are far smaller); a hostile
-                            // Content-Length that would overflow the add or the
-                            // (size_t) cast is dropped and refetched instead.
-                            if (back[i].r.totalsize > INT32_MAX - alloc_mem) {
-                              url_savename_refname_remove(opt, back[i].url_adr,
-                                                          back[i].url_fil);
-                              UNLINK(back[i].url_sav);
-                              alloc_mem = -1;
-                            } else if (back[i].r.totalsize >= 0)
                               alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
-                            if (alloc_mem >= 0 && deleteaddr(&back[i].r) &&
+                            if (deleteaddr(&back[i].r)
-                                (back[i].r.adr =
+                                && (back[i].r.adr =
                                     (char *) malloct((size_t) alloc_mem))) {
                               back[i].r.size = resume;
                               if (back[i].r.totalsize >= 0)
@@ -3982,6 +3939,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           deletehttp(&back[i].r);
                         }
                         back[i].r.soc = INVALID_SOCKET;
+                        //back[i].r.statuscode=206;  ????????
                         back[i].r.statuscode = STATUSCODE_NON_FATAL;
                         if (strnotempty(back[i].r.msg))
                           strcpybuff(back[i].r.msg,
@@ -4002,13 +3960,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                               && (back[i].r.adr = (char *) malloct(2))) {
                             back[i].r.adr[0] = 0;
                           }
-                          /* locked = name pending; the waiter finalizes after
-                             patching url_sav (else: cached as .delayed, #5) */
-                          if (!back[i].locked) {
                           hts_log_print(opt, LOG_TRACE, "finalizing empty");
                           back_finalize(opt, cache, sback, i);
-                          }
                         } else if (!back[i].r.is_chunk) {       // pas de chunk
+                          //if (back[i].r.http11!=2) {    // pas de chunk
                           back[i].is_chunk = 0;
                           back[i].status = 1;   // start body
                         } else {
@@ -4069,11 +4024,14 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].status < 0) {
           if (!back[i].testmode) {      // pas en test
             UNLINK(back[i].url_sav);    // éliminer fichier (endommagé)
+            //printf("&& %s\n",back[i].url_sav);
           }
         }
 #endif
 
         /* funny log for commandline users */
+        //if (!opt->quiet) {  
+        // petite animation
         if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
           if (back[i].status == STATUS_READY) {
             if (back[i].r.statuscode == HTTP_OK)
@@ -4086,6 +4044,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
             fflush(stdout);
           }
         }
+        //}
 
       }                         // status>0
     }                           // for
@@ -4096,6 +4055,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
       act = time_local();       // temps en secondes
       for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+        // for(i=0;i<back_max;i++) {
         unsigned int i = (i_mod + mod_random) % (back_max);
 
         if (back[i].status > 0) {       // réception/connexion/..
@@ -4123,6 +4083,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 continue;
               }
             }
+            //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
             if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) {
               hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr,
                 back[i].url_fil);
@@ -4198,51 +4159,32 @@ int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize)
   return 1;
 }
 
-/* Grace left to the smooth stop before in-flight transfers are aborted. */
-static int back_maxtime_grace(const int maxtime) {
-  return maximum(5, minimum(30, maxtime / 10));
-}
-
-/* Bytes the smooth stop may overrun before in-flight transfers are aborted.
-   No floor (unlike the time grace): a size overrun should abort promptly. */
-static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
-
-/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481).
-   -M measures received volume (HTS_TOTAL_RECV), not saved 200-only stat_bytes
-   which undercounts redirect/error-heavy crawls (#520). */
-static hts_mirror_limit back_mirror_limit(httrackp *opt) {
-  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
-      HTS_STAT.HTS_TOTAL_RECV - opt->maxsite >=
-          back_maxsize_grace(opt->maxsite))
-    return HTS_MIRROR_LIMIT_SIZE;
-  if (opt->maxtime > 0) {
-    const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
-
-    if (elapsed >= opt->maxtime &&
-        elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
-      return HTS_MIRROR_LIMIT_TIME;
-  }
-  return HTS_MIRROR_LIMIT_NONE;
-}
-
 int back_checkmirror(httrackp * opt) {
-  /* request a smooth stop the first time each cap is reached */
+  // Check max size
-  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
+  if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
-      !opt->state.stop) {
+    if (!opt->state.stop) {     /* not yet stopped */
       hts_log_print(opt, LOG_ERROR,
                     "More than " LLintP
                     " bytes have been transferred.. giving up",
                     (LLint) opt->maxsite);
+      /* cancel mirror smoothly */
       hts_request_stop(opt, 0);
     }
-  if (opt->maxtime > 0 && !opt->state.stop &&
+    return 1;                   /* don'k break mirror too sharply for size limits, but stop requested */
-      (time_local() - HTS_STAT.stat_timestart) >= opt->maxtime) {
+    /*return 0;
+     */
+  }
+  // Check max time
+  if ((opt->maxtime > 0)
+      && ((time_local() - HTS_STAT.stat_timestart) >= opt->maxtime)) {
+    if (!opt->state.stop) {     /* not yet stopped */
       hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
                     opt->maxtime);
+      /* cancel mirror smoothly */
       hts_request_stop(opt, 0);
     }
-  /* hard stop once a cap overruns its grace (callers must stop waiting) */
+  }
-  return back_mirror_limit(opt) == HTS_MIRROR_LIMIT_NONE;
+  return 1;                     /* Ok, go on */
 }
 
 // octets transférés + add

src/htsback.h

@@ -113,12 +113,6 @@ void back_set_locked(struct_back * sback, const int p);
 void back_set_unlocked(struct_back * sback, const int p);
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p);
-/* Discard back's on-disk .delayed placeholder and its refname. */
-void back_delayed_discard(httrackp *opt, lien_back *back);
-/* Move back's .delayed placeholder (and open stream) to newname;
-   HTS_FALSE = file lost, slot flagged in error. */
-hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
-                                const char *newname);
 void back_index_unlock(struct_back * sback, const int p);
 int back_clear_entry(lien_back * back);
 int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
@@ -138,11 +132,10 @@ LLint back_transferred(LLint add, struct_back * sback);
 
 // hostback
 #if HTS_XGETHOST
+void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
-/* Enforce -M/-E quotas: smooth-stops when reached; returns 0 once the -M cap
-   or -E deadline overruns its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htsbasenet.h

@@ -55,6 +55,8 @@ Please visit our Website: http://www.httrack.com
 // inline function in Wspiapi.h.
 #include <ws2tcpip.h>
 #include <Wspiapi.h>
+// #include <winsock2.h>
+// #include <tpipv6.h>
 
 #endif
 
@@ -127,8 +129,6 @@ typedef enum HTTPStatusCode {
   HTTP_UNSUPPORTED_MEDIA_TYPE = 415,
   HTTP_REQUESTED_RANGE_NOT_SATISFIABLE = 416,
   HTTP_EXPECTATION_FAILED = 417,
-  HTTP_TOO_MANY_REQUESTS = 429,
-  HTTP_UNAVAILABLE_FOR_LEGAL_REASONS = 451,
   HTTP_INTERNAL_SERVER_ERROR = 500,
   HTTP_NOT_IMPLEMENTED = 501,
   HTTP_BAD_GATEWAY = 502,

src/htsbasiccharsets.sh

@@ -3,12 +3,12 @@
 
 # Change this to download files
 if false; then
-    echo "mget https://www.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/ISO8859/8859-*.TXT" | lftp
-    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/PC/CP*.TXT" | lftp
-    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/WINDOWS/CP*.TXT" | lftp
-    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MICSFT/EBCDIC/CP*.TXT" | lftp
-    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/CP*.TXT" | lftp
-    echo "mget https://www.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
+    echo "mget ftp://ftp.unicode.org/Public/MAPPINGS/VENDORS/MISC/KOI8*.TXT" | lftp
     rm -f CP932.TXT CP936.TXT CP949.TXT CP950.TXT
 fi
 

src/htsbauth.c

@@ -107,6 +107,7 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
 #if DEBUG_COOK
   printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
          cook_name, cook_value, domain, path);
+  //printf(">>>cook: %s<<<\n",cookie->data);
 #endif
   return 0;
 }
@@ -207,6 +208,8 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   char catbuff[CATBUFF_SIZE];
   char buffer[8192];
 
+  //  cookie->data[0]='\0';
+
   // Fusionner d'abord les éventuels cookies IE
 #ifdef _WIN32
   {
@@ -307,25 +310,14 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   return -1;
 }
 
-/* Write cookies.txt; returns 0 on success. The jar holds live session
+// écrire cookies.txt
-   cookies, so keep it owner-only on Unix (Windows inherits folder ACLs). */
+// !=0 : erreur
 int cookie_save(t_cookie * cookie, const char *name) {
   char catbuff[CATBUFF_SIZE];
 
   if (strnotempty(cookie->data)) {
     char BIGSTK line[8192];
-#ifdef _WIN32
     FILE *fp = fopen(fconv(catbuff, sizeof(catbuff), name), "wb");
-#else
-    const int fd = open(fconv(catbuff, sizeof(catbuff), name),
-                        O_WRONLY | O_CREAT | O_TRUNC, HTS_PROTECT_FILE);
-    FILE *fp = fd != -1 ? fdopen(fd, "wb") : NULL;
-
-    if (fd != -1 && fp == NULL)
-      close(fd);    /* fdopen failed: don't leak the descriptor */
-    if (fp != NULL) /* O_CREAT's mode skips pre-existing jars: tighten those */
-      (void) fchmod(fd, HTS_PROTECT_FILE);
-#endif
 
     if (fp) {
       char *a = cookie->data;

src/htscache.h

@@ -66,11 +66,8 @@ htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr,
                   const char *fil, const char *save, char *location);
 htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, const char *save, char *location);
-/* Like cache_read, but also yields entries whose transfer broke; return_save
-   (optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name. */
 htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
-                                   const char *adr, const char *fil,
+                                   const char *adr, const char *fil);
-                                   char *return_save);
 htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr,
                     const char *fil, const char *save, char *location,
                     char *return_save, int readonly);
@@ -78,22 +75,14 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, htsblk * r);
 void cache_init(cache_back * cache, httrackp * opt);
 
-/* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
+int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
-typedef enum {
+                    const char *str2, char *outbuff, int len);
-  CACHE_RECONCILE_PROMOTE,     /* no new cache: promote the old generation */
+int cache_readdata(cache_back * cache, const char *str1, const char *str2,
-  CACHE_RECONCILE_INTERRUPTED, /* aborted run: keep the larger generation */
+                   char **inbuff, int *len);
-  CACHE_RECONCILE_ROLLBACK     /* nothing transferred: restore the old one */
-} hts_cache_reconcile_mode;
-
-/* Reconcile the on-disk cache generations according to mode; a no-op when
-   the involved files are absent. */
-void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
 
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);
-/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
-int cache_binput(char *adr, const char *end, char *s, int max);
 int cache_brint(char *adr, int *i);
 void cache_rint(FILE * fp, int *i);
 void cache_rLLint(FILE * fp, LLint * i);

src/htscache_selftest.c

@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
 #include "htszlib.h"
 
 #include <errno.h>
-#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -78,6 +77,14 @@ static void selftest_open_for_read(cache_back *cache, httrackp *opt) {
 }
 
 static void selftest_close(cache_back *cache) {
+  if (cache->dat != NULL) {
+    fclose(cache->dat);
+    cache->dat = NULL;
+  }
+  if (cache->ndx != NULL) {
+    fclose(cache->ndx);
+    cache->ndx = NULL;
+  }
   if (cache->zipOutput != NULL) {
     zipClose(cache->zipOutput,
              "Created by HTTrack Website Copier (cache self-test)");
@@ -314,7 +321,6 @@ typedef struct {
   size_t budget;  /**< bytes allowed through before writes start failing */
   int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
   int writes;     /**< zwrite call count, to detect re-entry into the stream */
-  int fail_once;  /**< recover (unlimited budget) after the first failure */
 } writefail_inject;
 
 /* zwrite that copies until the budget runs out, then fails with inj->fail_errno
@@ -329,8 +335,6 @@ static uLong selftest_failing_zwrite(voidpf opaque, voidpf stream,
     inj->budget -= (size_t) size;
     return (uLong) fwrite(buf, 1, (size_t) size, (FILE *) stream);
   }
-  if (inj->fail_once)
-    inj->budget = (size_t) -1; /* the backend recovers after this failure */
   errno = inj->fail_errno;
   return 0; /* short write -> the minizip op returns an error */
 }
@@ -369,50 +373,9 @@ static void writefail_store(httrackp *opt, cache_back *cache, const char *fil,
   freet(bodycopy);
 }
 
-/* Store an entry claiming a >2GB body; the degrade path never reads data. */
+/* #174/#219: a failing cache write used to crash via assertf(); it must instead
-static void writefail_store_oversized(httrackp *opt, cache_back *cache,
+   stop the mirror (exit_xh = -1) without crashing. Assert that, plus the cache
-                                      const char *fil, int is_write) {
+   is flagged and a sibling write doesn't re-enter the broken stream. */
-  htsblk r;
-  char locbuf[4];
-
-  hts_init_htsblk(&r);
-  r.statuscode = 200;
-  r.size = (LLint) INT_MAX + 1;
-  strcpybuff(r.msg, "OK");
-  strcpybuff(r.contenttype, "application/octet-stream");
-  locbuf[0] = '\0';
-  r.location = locbuf;
-  r.is_write = (short int) is_write;
-  cache_add(opt, cache, &r, "example.com", fil, "example.com/big.bin", 1, NULL);
-}
-
-/* Read back `entryname`: extra field (cached headers) and body. Returns the
-   body length, or -1 if the entry is absent or unreadable. */
-static int writefail_read_entry(const char *path, const char *entryname,
-                                char *extra, size_t extralen, char *body,
-                                size_t bodylen) {
-  unzFile z = unzOpen(path);
-  int n = -1;
-
-  if (z == NULL)
-    return -1;
-  if (unzLocateFile(z, entryname, 1) == UNZ_OK &&
-      unzOpenCurrentFile(z) == UNZ_OK) {
-    const int elen = unzGetLocalExtrafield(z, extra, (unsigned) (extralen - 1));
-
-    if (elen >= 0) {
-      extra[elen] = '\0';
-      n = unzReadCurrentFile(z, body, (unsigned) bodylen);
-    }
-    unzCloseCurrentFile(z);
-  }
-  unzClose(z);
-  return n;
-}
-
-/* Cache write-failure policy (#174/#219): fatal errno or a failure streak
-   stops the mirror (exit_xh=-1, no crash); isolated/oversized drops the entry.
- */
 int cache_write_failure_selftest(httrackp *opt, const char *dir) {
   int fail = 0;
   char path[HTS_URLMAXSIZE];
@@ -425,8 +388,9 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
   gen_body(body, body_len, 1 /* incompressible */);
   fconcat(path, sizeof(path), dir, "/wfail.zip");
 
-  /* phase 0: fatal errno (ENOSPC) aborts at once; phase 1: persistent EIO
+  /* phase 0: fail on the body write, fatal errno (ENOSPC, the disk-full
-     drops entries until the streak caps out, then aborts. */
+     branch). phase 1: fail on the open, non-fatal errno (EIO, dropped-share
+     branch). Both must abort the mirror. */
   for (phase = 0; phase < 2; phase++) {
     cache_back cache;
     writefail_inject inj;
@@ -435,7 +399,6 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     inj.budget = (phase == 0) ? 4096 : 0;
     inj.fail_errno = (phase == 0) ? ENOSPC : EIO;
     inj.writes = 0;
-    inj.fail_once = 0;
     memset(&cache, 0, sizeof(cache));
     cache.type = 1;
     cache.log = stderr;
@@ -449,25 +412,7 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     }
 
     opt->state.exit_xh = 0; /* clear; the failing write must set it to -1 */
-    if (phase == 0) {
     writefail_store(opt, &cache, "/blob.bin", body, body_len);
-    } else {
-      /* the abort must land exactly on the 8th consecutive failure */
-      int i;
-
-      for (i = 0; i < 7; i++) {
-        char fil[32];
-
-        snprintf(fil, sizeof(fil), "/b%d.bin", i);
-        writefail_store(opt, &cache, fil, body, 16);
-      }
-      if (cache.zipWriteFailed) {
-        fprintf(stderr, "cache-writefail: phase 1: aborted before the "
-                        "8th consecutive failure\n");
-        fail++;
-      }
-      writefail_store(opt, &cache, "/b7.bin", body, 16);
-    }
     if (!cache.zipWriteFailed) {
       fprintf(stderr, "cache-writefail: phase %d: write error not caught\n",
               phase);
@@ -498,136 +443,6 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     }
   }
 
-  /* failures with successes in between reset the streak: never aborts */
-  {
-    cache_back cache;
-    writefail_inject inj;
-    int i;
-
-    inj.budget = (size_t) -1;
-    inj.fail_errno = EIO;
-    inj.writes = 0;
-    inj.fail_once = 0;
-    memset(&cache, 0, sizeof(cache));
-    cache.type = 1;
-    cache.log = stderr;
-    cache.errlog = stderr;
-    cache.hashtable = coucal_new(0);
-    cache.zipOutput = selftest_open_failing_zip(path, &inj);
-    opt->state.exit_xh = 0;
-
-    for (i = 0; i < 10; i++) {
-      char fil[32];
-
-      inj.budget = 0; /* this store fails */
-      snprintf(fil, sizeof(fil), "/s%d.bin", i);
-      writefail_store(opt, &cache, fil, body, 16);
-      inj.budget = (size_t) -1; /* this one succeeds and resets the streak */
-      snprintf(fil, sizeof(fil), "/ok%d.bin", i);
-      writefail_store(opt, &cache, fil, body, 16);
-    }
-    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
-      fprintf(stderr,
-              "cache-writefail: scattered: non-consecutive failures aborted "
-              "the mirror (flagged=%d, exit_xh=%d)\n",
-              (int) cache.zipWriteFailed, opt->state.exit_xh);
-      fail++;
-    }
-    zipClose(cache.zipOutput, NULL);
-    cache.zipOutput = NULL;
-  }
-
-  /* isolated failure: only that entry drops; a later sibling round-trips */
-  {
-    cache_back cache;
-    writefail_inject inj;
-    char extra[8192];
-    char rbody[64];
-    int n;
-
-    inj.budget = 4096;
-    inj.fail_errno = EIO;
-    inj.writes = 0;
-    inj.fail_once = 1;
-    memset(&cache, 0, sizeof(cache));
-    cache.type = 1;
-    cache.log = stderr;
-    cache.errlog = stderr;
-    cache.hashtable = coucal_new(0);
-    cache.zipOutput = selftest_open_failing_zip(path, &inj);
-    opt->state.exit_xh = 0;
-
-    writefail_store(opt, &cache, "/blob.bin", body, body_len);
-    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
-      fprintf(stderr,
-              "cache-writefail: skip: isolated failure aborted the mirror "
-              "(flagged=%d, exit_xh=%d)\n",
-              (int) cache.zipWriteFailed, opt->state.exit_xh);
-      fail++;
-    }
-    writefail_store(opt, &cache, "/blob2.bin", body, 16);
-    zipClose(cache.zipOutput, NULL);
-    cache.zipOutput = NULL;
-    n = writefail_read_entry(path, "http://example.com/blob2.bin", extra,
-                             sizeof(extra), rbody, sizeof(rbody));
-    if (n != 16 || memcmp(rbody, body, 16) != 0) {
-      fprintf(stderr,
-              "cache-writefail: skip: sibling entry lost after a skipped "
-              "entry (%d)\n",
-              n);
-      fail++;
-    }
-  }
-
-  /* >2GB bodies: in-memory drops the entry, on-disk degrades to headers-only */
-  {
-    cache_back cache;
-    writefail_inject inj;
-    char extra[8192];
-    char rbody[64];
-    int n;
-
-    inj.budget = (size_t) -1; /* no injected failure */
-    inj.fail_errno = 0;
-    inj.writes = 0;
-    inj.fail_once = 0;
-    memset(&cache, 0, sizeof(cache));
-    cache.type = 1;
-    cache.log = stderr;
-    cache.errlog = stderr;
-    cache.hashtable = coucal_new(0);
-    cache.zipOutput = selftest_open_failing_zip(path, &inj);
-    opt->state.exit_xh = 0;
-
-    writefail_store_oversized(opt, &cache, "/bigmem.bin", 0 /* in-memory */);
-    writefail_store_oversized(opt, &cache, "/bigdisk.bin", 1 /* on-disk */);
-    zipClose(cache.zipOutput, NULL);
-    cache.zipOutput = NULL;
-
-    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
-      fprintf(stderr,
-              "cache-writefail: oversize: mirror aborted (flagged=%d, "
-              "exit_xh=%d)\n",
-              (int) cache.zipWriteFailed, opt->state.exit_xh);
-      fail++;
-    }
-    if (writefail_read_entry(path, "http://example.com/bigmem.bin", extra,
-                             sizeof(extra), rbody, sizeof(rbody)) >= 0) {
-      fprintf(stderr,
-              "cache-writefail: oversize: in-memory entry was stored\n");
-      fail++;
-    }
-    n = writefail_read_entry(path, "http://example.com/bigdisk.bin", extra,
-                             sizeof(extra), rbody, sizeof(rbody));
-    if (n != 0 || strstr(extra, "X-In-Cache: 0") == NULL) {
-      fprintf(stderr,
-              "cache-writefail: oversize: on-disk entry not stored "
-              "headers-only (%d)\n",
-              n);
-      fail++;
-    }
-  }
-
   freet(body);
   return fail;
 }
@@ -901,583 +716,3 @@ int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
 
   return failures;
 }
-
-/* --- hts_cache_reconcile() policies -------------------------------------- */
-
-/* All reconcile inputs/outputs, wiped between cases. */
-static const char *const reconcile_files[] = {
-    "hts-cache/new.zip", "hts-cache/old.zip",   "hts-cache/new.dat",
-    "hts-cache/old.dat", "hts-cache/new.ndx",   "hts-cache/old.ndx",
-    "hts-cache/new.lst", "hts-cache/old.lst",   "hts-cache/new.txt",
-    "hts-cache/old.txt", "hts-in_progress.lock"};
-
-static char *reconcile_st_path(httrackp *opt, const char *name) {
-  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                 StringBuff(opt->path_log), name);
-}
-
-static void reconcile_wipe(httrackp *opt) {
-  size_t i;
-
-  for (i = 0; i < sizeof(reconcile_files) / sizeof(reconcile_files[0]); i++)
-    remove(reconcile_st_path(opt, reconcile_files[i]));
-}
-
-/* Create a filler file of exactly `size` bytes. */
-static void reconcile_put(httrackp *opt, const char *name, size_t size) {
-  FILE *const fp = fopen(reconcile_st_path(opt, name), "wb");
-  static const char filler[1024] = {'x'};
-
-  assertf(fp != NULL);
-  while (size > 0) {
-    const size_t n = size > sizeof(filler) ? sizeof(filler) : size;
-
-    assertf(fwrite(filler, 1, n, fp) == n);
-    size -= n;
-  }
-  fclose(fp);
-}
-
-/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
-static int reconcile_expect(httrackp *opt, const char *name, off_t size,
-                            const char *what) {
-  const off_t got = fsize(reconcile_st_path(opt, name));
-
-  if (got != size) {
-    fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
-            name, (int) got, (int) size);
-    return 1;
-  }
-  return 0;
-}
-
-int cache_reconcile_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-
-  /* around the interrupted-run thresholds (new < 32768, old > 65536) */
-  static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
-
-  golden_setup(opt, dir);
-#ifdef _WIN32
-  mkdir(reconcile_st_path(opt, "hts-cache"));
-#else
-  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
-#endif
-
-  /* PROMOTE: a zip old generation replaces a missing new one */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "promote-zip");
-  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "promote-zip");
-
-  /* PROMOTE: an existing new.zip is left alone */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", TINY, "promote-zip-noop");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
-
-  /* PROMOTE: the removed pre-3.31 legacy pair is left alone */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/old.dat", SOLID);
-  reconcile_put(opt, "hts-cache/old.ndx", TINY);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", TINY, "promote-dat");
-
-  /* INTERRUPTED: no lock file, no action */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-nolock");
-
-  /* INTERRUPTED: an absent new.zip must NOT promote old.zip (fsize(-1) would
-     spuriously pass "< TINY"); leave the solid old generation for ROLLBACK */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-in_progress.lock", 0);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", -1, "interrupted-nonew");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "interrupted-nonew");
-
-  /* INTERRUPTED: stalled tiny new.zip loses to a solid old.zip (was dead for
-     zip caches: the arm was gated on a legacy new.dat) */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-in_progress.lock", 0);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", SOLID, "interrupted-zip");
-  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "interrupted-zip");
-
-  /* INTERRUPTED: old below the confidence threshold, keep new */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-in_progress.lock", 0);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  reconcile_put(opt, "hts-cache/old.zip", MID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-smallold");
-
-  /* INTERRUPTED: new big enough to trust, keep it */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-in_progress.lock", 0);
-  reconcile_put(opt, "hts-cache/new.zip", MID);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
-
-  /* INTERRUPTED: the removed pre-3.31 legacy pair is left alone */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-in_progress.lock", 0);
-  reconcile_put(opt, "hts-cache/new.dat", TINY);
-  reconcile_put(opt, "hts-cache/new.ndx", TINY);
-  reconcile_put(opt, "hts-cache/old.dat", SOLID);
-  reconcile_put(opt, "hts-cache/old.ndx", MID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.dat", TINY, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.dat", SOLID, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.ndx", MID, "interrupted-dat");
-
-  /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
-     its only good generation here) */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  reconcile_put(opt, "hts-cache/old.zip", SOLID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "rollback-zip");
-  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "rollback-zip");
-
-  /* ROLLBACK: sidecars are restored regardless of format */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.lst", TINY);
-  reconcile_put(opt, "hts-cache/old.lst", MID);
-  reconcile_put(opt, "hts-cache/old.txt", MID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
-  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
-
-  /* ROLLBACK: sidecars restored, the removed pre-3.31 pair left alone */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.dat", TINY);
-  reconcile_put(opt, "hts-cache/old.dat", SOLID);
-  reconcile_put(opt, "hts-cache/old.ndx", MID);
-  reconcile_put(opt, "hts-cache/old.lst", MID);
-  reconcile_put(opt, "hts-cache/old.txt", MID);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", TINY, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", MID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
-
-  /* ROLLBACK: nothing to restore, the new generation stays */
-  reconcile_wipe(opt);
-  reconcile_put(opt, "hts-cache/new.zip", TINY);
-  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.zip", TINY, "rollback-noop");
-
-  reconcile_wipe(opt);
-  return failures;
-}
-
-/* The pre-3.31 .dat/.ndx import was removed: cache_init must refuse the
-   legacy pair, leave the files in place, and not flag an update run. */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-  int variant;
-
-  selftest_tag = "cache-legacy";
-  golden_setup(opt, dir);
-#ifdef _WIN32
-  mkdir(reconcile_st_path(opt, "hts-cache"));
-#else
-  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
-#endif
-
-  /* variant 0: old.dat/old.ndx (--update layout); 1: new.dat/new.ndx (ro) */
-  for (variant = 0; variant < 2; variant++) {
-    cache_back cache;
-    const char *const dat =
-        variant == 0 ? "hts-cache/old.dat" : "hts-cache/new.dat";
-    const char *const ndx =
-        variant == 0 ? "hts-cache/old.ndx" : "hts-cache/new.ndx";
-
-    reconcile_wipe(opt);
-    reconcile_put(opt, dat, 4096);
-    reconcile_put(opt, ndx, 4096);
-    opt->is_update = 0;
-    selftest_open(&cache, opt, /*ro*/ variant == 1);
-    if (cache_readable(&cache)) {
-      printf("cache-legacy: variant %d imported a removed format\n", variant);
-      failures++;
-    }
-    if (coucal_nitems(cache.hashtable) != 0) { /* no phantom index entries */
-      printf("cache-legacy: variant %d imported index entries\n", variant);
-      failures++;
-    }
-    if (opt->is_update) {
-      printf("cache-legacy: variant %d flagged an update\n", variant);
-      failures++;
-    }
-    if (fsize(reconcile_st_path(opt, dat)) != 4096 ||
-        fsize(reconcile_st_path(opt, ndx)) != 4096) {
-      printf("cache-legacy: variant %d touched the legacy files\n", variant);
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  /* a healthy zip must win over stray legacy files, with no refusal */
-  {
-    cache_back cache;
-    const char *const body = "<html>zip wins</html>";
-
-    reconcile_wipe(opt);
-    selftest_open_for_write(&cache, opt);
-    store_entry(opt, &cache, "example.com", "/", "example.com/index.html", 200,
-                "OK", "text/html", "utf-8", "", "", "", "", body, strlen(body));
-    selftest_close(&cache);
-    reconcile_put(opt, "hts-cache/old.ndx", 4096);
-    reconcile_put(opt, "hts-cache/old.dat", 4096);
-    selftest_open_for_read(&cache, opt);
-    if (!cache_readable(&cache)) {
-      printf("cache-legacy: stray legacy files shadowed the zip cache\n");
-      failures++;
-    }
-    failures +=
-        check_entry(opt, &cache, "example.com", "/", 200, "OK", "text/html",
-                    "utf-8", "", "", "", "", body, strlen(body));
-    if (fsize(reconcile_st_path(opt, "hts-cache/old.ndx")) != 4096 ||
-        fsize(reconcile_st_path(opt, "hts-cache/old.dat")) != 4096) {
-      printf("cache-legacy: zip-wins pass touched the legacy files\n");
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  reconcile_wipe(opt);
-  return failures;
-}
-
-/* --- read-side corruption injection --------------------------------------- */
-
-/* canary read back intact after each corruption; victim gets the byte surgery
- */
-#define CORRUPT_ADR "corrupt.example.com"
-static char corrupt_body_a[33 + 1];
-static char corrupt_body_b[44 + 1];
-
-/* Write a fresh two-entry cache: /canary.html then /victim.html. */
-static void corrupt_build(httrackp *opt) {
-  cache_back cache;
-
-  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
-  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
-  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
-  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  selftest_open_for_write(&cache, opt);
-  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
-              strlen(corrupt_body_a));
-  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
-              strlen(corrupt_body_b));
-  selftest_close(&cache);
-}
-
-/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
-   is later overwritten with a forged oversized X-Size (same byte length). */
-static void corrupt_build_etag(httrackp *opt) {
-  cache_back cache;
-
-  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
-  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
-  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
-  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  selftest_open_for_write(&cache, opt);
-  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
-              strlen(corrupt_body_a));
-  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
-              "OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
-              corrupt_body_b, strlen(corrupt_body_b));
-  selftest_close(&cache);
-}
-
-/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
-   body on disk): the shape every non-html file is stored with. */
-static void corrupt_build_disk(httrackp *opt) {
-  cache_back cache;
-  htsblk w;
-  char locw[4];
-  char BIGSTK save[HTS_URLMAXSIZE * 2];
-  char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
-  char *path;
-  FILE *fp;
-
-  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
-  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
-  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
-          CORRUPT_ADR "/victim.bin");
-  selftest_open_for_write(&cache, opt);
-  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
-              strlen(corrupt_body_a));
-  hts_init_htsblk(&w);
-  w.statuscode = 200;
-  w.size = (LLint) sizeof(corrupt_body_b) - 1;
-  strcpybuff(w.msg, "OK");
-  strcpybuff(w.contenttype, "application/octet-stream");
-  strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
-  locw[0] = '\0';
-  w.location = locw;
-  w.is_write = 0;
-  cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
-            0 /* all_in_cache */, NULL);
-  selftest_close(&cache);
-  /* the reader only checks this file exists; it never reads it here */
-  path = fconv(catbuff, sizeof(catbuff), save);
-  (void) structcheck(path);
-  fp = FOPEN(path, "wb");
-  assertf(fp != NULL);
-  fclose(fp);
-}
-
-/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
-static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
-                          const char *rep, size_t nth, size_t total) {
-  LLint fsz = 0;
-  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
-  const size_t n = (size_t) fsz;
-  size_t k, hits = 0, at = 0;
-  FILE *fp;
-
-  assertf(data != NULL);
-  for (k = 0; k + patlen <= n; k++) {
-    if (memcmp(data + k, pat, patlen) == 0) {
-      hits++;
-      if (hits == nth)
-        at = k;
-    }
-  }
-  assertf(hits == total);
-  memcpy(data + at, rep, patlen);
-  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
-  assertf(fp != NULL);
-  assertf(fwrite(data, 1, n, fp) == n);
-  fclose(fp);
-  freet(data);
-}
-
-/* Garbage the first bytes of the victim's deflated data (2nd local header). */
-static void corrupt_victim_body(httrackp *opt) {
-  LLint fsz = 0;
-  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
-  const size_t n = (size_t) fsz;
-  size_t k, hits = 0, off = 0;
-  FILE *fp;
-
-  assertf(data != NULL);
-  for (k = 0; k + 4 <= n; k++) {
-    if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
-      const size_t namelen =
-          (unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
-      const size_t extralen =
-          (unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
-
-      off = k + 30 + namelen + extralen;
-    }
-  }
-  assertf(hits == 2);
-  assertf(off != 0 && off + 4 <= n);
-  memset(data + off, 0xFF, 4);
-  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
-  assertf(fp != NULL);
-  assertf(fwrite(data, 1, n, fp) == n);
-  fclose(fp);
-  freet(data);
-}
-
-/* Read the corrupt /victim.html and, in the SAME read session, the intact
-   /canary.html: the victim must be rejected (wantmsg pins which path) and the
-   canary must still decode byte-exact, proving one bad entry never taints a
-   sibling read. */
-static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
-                                     const char *wantmsg, const char *what) {
-  cache_back cache;
-  htsblk v, c;
-  char BIGSTK lv[HTS_URLMAXSIZE * 2];
-  char BIGSTK lc[HTS_URLMAXSIZE * 2];
-  int fail = 0;
-
-  selftest_open_for_read(&cache, opt);
-  lv[0] = lc[0] = '\0';
-  v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
-  if (v.statuscode != STATUSCODE_INVALID) {
-    fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
-            selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
-    fail++;
-  }
-  if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
-    fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
-            selftest_tag, what, v.msg, wantmsg);
-    fail++;
-  }
-  c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
-  if (c.statuscode != 200 || c.adr == NULL ||
-      c.size != (LLint) strlen(corrupt_body_a) ||
-      memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
-    fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
-            c.statuscode);
-    fail++;
-  }
-  if (v.adr != NULL)
-    freet(v.adr);
-  if (c.adr != NULL)
-    freet(c.adr);
-  selftest_close(&cache);
-  return fail;
-}
-
-static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
-                                 const char *what) {
-  return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
-}
-
-/* Headers-only probe of the disk victim: must parse OK with the size kept. */
-static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
-                                      const char *what) {
-  cache_back cache;
-  htsblk v;
-  char BIGSTK lv[HTS_URLMAXSIZE * 2];
-  int fail = 0;
-
-  selftest_open_for_read(&cache, opt);
-  lv[0] = '\0';
-  v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
-  if (v.statuscode != 200 || v.size != wantsize) {
-    fprintf(stderr,
-            "%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
-            selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
-    fail++;
-  }
-  if (v.adr != NULL)
-    freet(v.adr);
-  selftest_close(&cache);
-  return fail;
-}
-
-/* One zip corruption case: build, patch, then check victim+canary in-session.
- */
-static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
-                            size_t nth, size_t total, const char *wantmsg,
-                            const char *what) {
-  corrupt_build(opt);
-  corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
-  return corrupt_expect_victim(opt, wantmsg, what);
-}
-
-int cache_corruption_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-
-  selftest_tag = "cache-corrupt";
-  golden_setup(opt, dir);
-
-  failures +=
-      corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
-                       "Cache Read Error : Read Data", "oversized X-Size");
-  failures +=
-      corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
-                       "Cache Read Error : Bad Size", "negative X-Size");
-  /* both entries carry the line; the victim's is the second */
-  failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
-                               "Previous cache file not found (empty filename)",
-                               "blanked X-In-Cache");
-  /* smashed local file header: the entry is dropped at index load */
-  failures +=
-      corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
-                       "File Cache Entry Not Found", "smashed local header");
-
-  corrupt_build(opt);
-  corrupt_victim_body(opt);
-  failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
-                                    "garbled deflate stream");
-
-  /* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
-     truncates negative in the (int) cast the malloc uses: a wraparound alloc.
-     cache_add asserts size fits an int, so such a value only reaches the reader
-     from a corrupt/foreign cache; inject it by overwriting the victim's long
-     Etag line with a same-length forged X-Size line (the parser keeps the last
-     X-Size it sees), keeping the zip byte-length and offsets intact. */
-  corrupt_build_etag(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: 2147483648AAAAAAAA", 1, 1);
-  failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
-                                    "X-Size above INT_MAX");
-
-  /* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
-     is how every >2GB non-html file is stored. It must survive a header probe
-     (or every update re-fetches the file); an in-memory read still rejects. */
-  corrupt_build_disk(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: 2147483648AAAAAAAA", 1, 1);
-  failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
-                                         "headers-only X-Size above INT_MAX");
-  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
-                                        "Cache Read Error : Bad Size",
-                                        "in-memory X-Size above INT_MAX");
-
-  /* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
-  corrupt_build_disk(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: 2147483647AAAAAAAA", 1, 1);
-  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
-                                        "Cache Read Error : Bad Size",
-                                        "in-memory X-Size at INT_MAX");
-
-  /* the negative check must stay global, headers-only included */
-  corrupt_build_disk(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: -2147483648AAAAAAA", 1, 1);
-  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
-                                        "Cache Read Error : Bad Size",
-                                        "headers-only negative X-Size");
-
-  return failures;
-}

src/htscache_selftest.h

@@ -52,23 +52,10 @@ int cache_selftests(httrackp *opt, const char *dir);
    committed file, never by the test). Returns the failed-check count. */
 int cache_golden_selftest(httrackp *opt, const char *dir, int regen);
 
-/* Cache write-failure policy (#174/#219): abort on fatal errno or a streak,
+/* #174/#219: assert a failing cache write aborts the mirror cleanly instead of
-   drop just the entry otherwise. Returns the failed-check count. */
+   crashing. Returns the failed-check count. */
 int cache_write_failure_selftest(httrackp *opt, const char *dir);
 
-/* Exercise the hts_cache_reconcile() generation policies on file fixtures
-   under <dir>. Returns the failed-check count. */
-int cache_reconcile_selftest(httrackp *opt, const char *dir);
-
-/* Verify cache_init refuses a pre-3.31 .dat/.ndx cache without touching it.
-   Returns the number of failed checks (0 = pass). */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir);
-
-/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
-   under <dir> and assert every case degrades to STATUSCODE_INVALID without
-   tainting a sibling entry. */
-int cache_corruption_selftest(httrackp *opt, const char *dir);
-
 #endif
 
 #endif

src/htscatchurl.c

@@ -64,7 +64,7 @@ Please visit our Website: http://www.httrack.com
 // catch_url_init(&port,&return_host);
 HTSEXT_API T_SOC catch_url_init_std(int *port_prox, char *adr_prox) {
   T_SOC soc;
-  int try_to_listen_to[] = {8080, 3128, 80, 81, 82, 8081, 3129, 0, -1};
+  int try_to_listen_to[] = { 8080, 3128, 80, 81, 82, 8081, 3129, 31337, 0, -1 };
   int i = 0;
 
   do {
@@ -175,9 +175,7 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
       //
       socinput(soc, line, 1000);
       if (strnotempty(line)) {
-        /* widths bound the caller buffers: method[32], url[HTS_URLMAXSIZE*2],
+        if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
-           protocol[256] */
-        if (sscanf(line, "%31s %2047s %255s", method, url, protocol) == 3) {
           lien_adrfil af;
 
           // méthode en majuscule
@@ -197,6 +195,7 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
             htsblk blkretour;
 
             hts_init_htsblk(&blkretour);
+            //memset(&blkretour, 0, sizeof(htsblk));    // effacer
             blkretour.location = loc;   // si non nul, contiendra l'adresse véritable en cas de moved xx
             // Lire en têtes restants
             sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
@@ -206,8 +205,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
               strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
               strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
             }
-            // CR/LF final de l'en tête inutile car déja placé via la ligne vide
+            // CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
-            // juste au dessus
+            //strcatbuff(data,"\r\n");
             if (blkretour.totalsize > 0) {
               int len = (int) min(blkretour.totalsize, 32000);
               int pos = (int) strlen(data);

src/htscharset.c

@@ -84,7 +84,7 @@ static int hts_equalsAlphanum(const char *a, const char *b) {
 
 /* Copy the memory region [s .. s + size - 1 ] as a \0-terminated string. */
 static char *hts_stringMemCopy(const char *s, size_t size) {
-  char *dest = malloct(size + 1);
+  char *dest = malloc(size + 1);
 
   if (dest != NULL) {
     memcpy(dest, s, size);
@@ -549,6 +549,42 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
 
 #endif
 
+HTS_STATIC char *hts_getCharsetFromContentType(const char *mime) {
+  /* text/html; charset=utf-8 */
+  const char *const charset = "charset";
+  char *pos = strstr(mime, charset);
+
+  if (pos != NULL) {
+    /* Skip spaces */
+    int eq = 0;
+
+    for(pos += strlen(charset);
+        *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\''; pos++) {
+      if (*pos == '=') {
+        eq = 1;
+      }
+    }
+    if (eq == 1) {
+      int len;
+
+      for(len = 0;
+          pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'';
+          pos++) ;
+      if (len != 0) {
+        char *const s = malloc(len + 1);
+        int i;
+
+        for(i = 0; i < len; i++) {
+          s[i] = pos[i];
+        }
+        s[len] = '\0';
+        return s;
+      }
+    }
+  }
+  return NULL;
+}
+
 #ifdef _WIN32
 #define strcasecmp(a,b) stricmp(a,b)
 #define strncasecmp(a,b,n) strnicmp(a,b,n)
@@ -608,106 +644,58 @@ int hts_isCharsetUTF8(const char *charset) {
          || strcasecmp(charset, "utf8") == 0 );
 }
 
-/* Extract X from a "text/html; charset=X" content= attribute value. */
-static char *charset_from_content(const char *s, size_t len) {
-  size_t i;
-
-  for (i = 0; i + 7 < len; i++) {
-    if ((i == 0 || is_space(s[i - 1]) || s[i - 1] == ';') &&
-        strncasecmp(&s[i], "charset", 7) == 0 && is_space_or_equal(s[i + 7])) {
-      size_t j, val;
-
-      for (j = i + 7; j < len && is_space_or_equal_or_quote(s[j]); j++)
-        ;
-      for (val = j; j < len && s[j] != '"' && s[j] != '\'' && s[j] != ';' &&
-                    !is_space(s[j]);
-           j++)
-        ;
-      if (j != val) {
-        return hts_stringMemCopy(&s[val], j - val);
-      }
-    }
-  }
-  return NULL;
-}
-
 char *hts_getCharsetFromMeta(const char *html, size_t size) {
-  size_t i;
+  int i;
 
-  /* HTML5 <meta charset=X> or legacy
+  /* <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8" > */
-     <meta http-equiv="Content-Type" content="text/html; charset=X">,
+  for(i = 0; i < size; i++) {
-     attributes in any order; first resolvable tag wins. */
+    if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0
-  for (i = 0; i + 6 < size; i++) {
+        && is_space(html[i + 5])) {
-    size_t j;
+      /* Skip spaces */
-    const char *equiv = NULL, *content = NULL;
+      for(i += 5; is_space(html[i]); i++) ;
-    size_t equiv_len = 0, content_len = 0;
+      if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0
+          && is_space_or_equal(html[i + 10])) {
+        for(i += 10; is_space_or_equal_or_quote(html[i]); i++) ;
+        if (strncasecmp(&html[i], "CONTENT-TYPE", 12) == 0) {
+          for(i += 12; is_space_or_equal_or_quote(html[i]); i++) ;
+          if (strncasecmp(&html[i], "CONTENT", 7) == 0
+              && is_space_or_equal(html[i + 7])) {
+            for(i += 7; is_space_or_equal_or_quote(html[i]); i++) ;
+            /* Skip content-type */
+            for(;
+                i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'';
+                i++) ;
+            /* Expect charset attribute here */
+            if (html[i] == ';') {
+              for(i++; is_space(html[i]); i++) ;
+              /* Look for charset */
+              if (strncasecmp(&html[i], "charset", 7) == 0
+                  && is_space_or_equal(html[i + 7])) {
+                int len;
 
-    if (html[i] != '<' || strncasecmp(&html[i + 1], "meta", 4) != 0 ||
+                for(i += 7; is_space_or_equal(html[i]) || html[i] == '\'';
-        !is_space(html[i + 5])) {
+                    i++) ;
-      continue;
+                /* Charset */
-    }
+                for(len = 0;
-    /* Attribute scan, strictly bounded by size. */
+                    i + len < size && html[i + len] != '"'
-    for (j = i + 6; j < size && html[j] != '>';) {
+                    && html[i + len] != '\'' && html[i + len] != ' '; len++) ;
-      size_t name, name_len, val = 0, val_len = 0;
+                /* No error ? */
+                if (len != 0 && i < size) {
+                  char *const s = malloc(len + 1);
+                  int j;
 
-      if (is_space(html[j]) || html[j] == '/') {
+                  for(j = 0; j < len; j++) {
-        j++;
+                    s[j] = html[i + j];
-        continue;
                   }
-      for (name = j; j < size && html[j] != '=' && html[j] != '>' &&
+                  s[len] = '\0';
-                     html[j] != '/' && !is_space(html[j]);
-           j++)
-        ;
-      name_len = j - name;
-      for (; j < size && is_space(html[j]); j++)
-        ;
-      if (j < size && html[j] == '=') {
-        for (j++; j < size && is_space(html[j]); j++)
-          ;
-        if (j < size && (html[j] == '"' || html[j] == '\'')) {
-          const char quote = html[j++];
-
-          for (val = j; j < size && html[j] != quote; j++)
-            ;
-          if (j >= size) /* unterminated quote: drop the tag */
-            break;
-          val_len = j++ - val;
-        } else {
-          /* unquoted: ends at whitespace or '>' only (HTML5 prescan); '/'
-             belongs to the value except as the tail of a self-close */
-          for (val = j; j < size && !is_space(html[j]) && html[j] != '>'; j++)
-            ;
-          val_len = j - val;
-          if (val_len != 0 && j < size && html[j] == '>' &&
-              html[val + val_len - 1] == '/') {
-            val_len--;
-          }
-        }
-      }
-      /* first occurrence of each attribute wins, as in browsers */
-      if (val_len != 0) {
-        if (name_len == 7 && strncasecmp(&html[name], "charset", 7) == 0) {
-          return hts_stringMemCopy(&html[val], val_len);
-        } else if (equiv == NULL && name_len == 10 &&
-                   strncasecmp(&html[name], "http-equiv", 10) == 0) {
-          equiv = &html[val];
-          equiv_len = val_len;
-        } else if (content == NULL && name_len == 7 &&
-                   strncasecmp(&html[name], "content", 7) == 0) {
-          content = &html[val];
-          content_len = val_len;
-        }
-      }
-    }
-    if (equiv_len == 12 && strncasecmp(equiv, "content-type", 12) == 0 &&
-        content != NULL) {
-      char *const s = charset_from_content(content, content_len);
-
-      if (s != NULL) {
                   return s;
                 }
               }
-    i = j;
+            }
+          }
+        }
+      }
+    }
   }
   return NULL;
 }
@@ -853,6 +841,71 @@ static unsigned int nlz8(unsigned char x) {
     }                                           \
   } while(0)
 
+/* Sample. */
+#if 0
+
+int main(int argc, char **argv) {
+  int i;
+  int hex = 0;
+  
+#define READ_INT(DEST)                                  \
+  ( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1)      \
+    || (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
+  
+  for(i = 1 ; i < argc ; i++) {
+    unsigned int uc, from, to;
+    
+    if (strcmp(argv[i], "--hex") == 0) {
+      hex = 1;
+    }
+    else if (strcmp(argv[i], "--decimal") == 0) {
+      hex = 0;
+    }
+    else if (strcmp(argv[i], "--decode") == 0) {
+#define RD fgetc_unlocked(stdin)
+#define WR(C) do {                              \
+        if (C != -1) {                          \
+          printf("%04x\n", C);                  \
+        } else if (!feof(stdin)) {              \
+          fprintf(stderr, "read error\n");      \
+          exit(EXIT_FAILURE);                   \
+        }                                       \
+      } while(0)
+      while(!feof(stdin)) {
+        READ_UNICODE(RD, WR);
+      }
+#undef RD
+#undef WR
+    }
+    else if (strcmp(argv[i], "-range") == 0
+             && i + 2 < argc
+             && (++i, 1)
+             && READ_INT(from)
+             && (++i, 1)
+             && READ_INT(to)
+             ) {
+      unsigned int i;
+      for(i = from ; i < to ; i++) {
+#define EM(C) fputc_unlocked(C, stdout)
+        EMIT_UNICODE(i, EM);
+#undef EM
+      }
+    }
+    else if (READ_INT(uc)) {
+#define EM(C) fputc_unlocked(C, stdout)
+      EMIT_UNICODE(uc, EM);
+#undef EM
+    }
+    else {
+      return EXIT_FAILURE;
+    }
+  }
+  
+  return EXIT_SUCCESS;
+}
+
+#endif
+
 /* IDNA helpers. */
 #undef ADD_BYTE
 #undef INCREASE_CAPA
@@ -922,15 +975,15 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
                 /* Reader: can read bytes up to j */
 #define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
 
-                /* Writer: a malformed sequence abandons the encode (NULL) */
+                /* Writer: upon error, return FFFD (replacement character) */
-#define WR(C)                                                                  \
+#define WR(C) do { \
-  do {                                                                         \
   if ((C) != -1) { \
     /* copy character */ \
     assertf(segOutputSize < segSize); \
     segInt[segOutputSize++] = (C); \
-    } else {                                                                   \
+  } \
-      freet(segInt);                                                           \
+  /* In case of error, abort. */ \
+  else { \
     FREE_BUFFER(); \
     return NULL; \
   } \
@@ -1152,45 +1205,17 @@ int hts_isStringUTF8(const char *s, size_t size) {
   const unsigned char *const data = (const unsigned char*) s;
   size_t i;
 
-  /* Strict RFC 3629: reject overlongs, surrogates, > U+10FFFF, 5/6-byte. */
   for(i = 0 ; i < size ; ) {
-    const unsigned char c = data[i];
+    /* Reader: can read bytes up to j */
-    size_t len, k;
+#define RD ( i < size ? data[i++] : -1 )
-    hts_UCS4 uc, min;
 
-    if (c < 0x80) {
+    /* Writer: upon error, return FFFD (replacement character) */
-      i++;
+#define WR(C) if ((C) == -1) { return 0; }
-      continue;
-    } else if (c >= 0xc2 && c <= 0xdf) {
-      len = 2;
-      min = 0x80;
-      uc = c & 0x1f;
-    } else if (c >= 0xe0 && c <= 0xef) {
-      len = 3;
-      min = 0x800;
-      uc = c & 0x0f;
-    } else if (c >= 0xf0 && c <= 0xf4) {
-      len = 4;
-      min = 0x10000;
-      uc = c & 0x07;
-    } else { /* continuation byte, overlong C0/C1, or F5..FF lead */
-      return 0;
-    }
-    if (size - i < len) {
-      return 0;
-    }
-    for (k = 1; k < len; k++) {
-      const unsigned char cc = data[i + k];
 
-      if ((cc & 0xc0) != 0x80) {
+    /* Read Unicode character. */
-        return 0;
+    READ_UNICODE(RD, WR);
-      }
+#undef RD
-      uc = (uc << 6) | (cc & 0x3f);
+#undef WR
-    }
-    if (uc < min || uc > 0x10FFFF || (uc >= 0xD800 && uc <= 0xDFFF)) {
-      return 0;
-    }
-    i += len;
   }
 
   return 1;

src/htscharset.h

@@ -76,8 +76,7 @@ extern char *hts_convertStringIDNAToUTF8(const char *s, size_t size);
 extern int hts_isStringIDNA(const char *s, size_t size);
 
 /**
- * Extract the <meta> charset from the HTML buffer "html" (HTML5 charset= or
+ * Extract the charset from the HTML buffer "html"
- * legacy http-equiv form). Returns a malloc'ed string, or NULL if none.
  **/
 extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 
@@ -87,8 +86,7 @@ extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 extern int hts_isStringAscii(const char *s, size_t size);
 
 /**
- * Is the given string valid UTF-8 ? Strict RFC 3629: overlong forms,
+ * Is the given string an UTF-8 string ?
- * surrogates, codepoints above U+10FFFF and 5/6-byte sequences are rejected.
  **/
 extern int hts_isStringUTF8(const char *s, size_t size);
 

src/htsconfig.h

@@ -54,6 +54,10 @@ Please visit our Website: http://www.httrack.com
 // extension par défaut pour fichiers n'en ayant pas
 #define DEFAULT_EXT ".html"
 #define DEFAULT_EXT_SHORT ".htm"
+// #define DEFAULT_BIN_EXT   ".bin"
+// #define DEFAULT_BIN_EXT_SHORT ".bin"
+// #define DEFAULT_EXT       ".txt"
+// #define DEFAULT_EXT_SHORT ".txt"
 
 // éviter les /nul, /con..
 #define HTS_OVERRIDE_DOS_FOLDERS 1
@@ -76,6 +80,13 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
+// gérer une table de hachage?
+// REMOVED
+// #define HTS_HASH 1
+
+// fast cache (build hash table)
+#define HTS_FAST_CACHE 1
+
 // le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
 // valide)
 #define GT_ENDS_COMMENT 1
@@ -86,10 +97,27 @@ Please visit our Website: http://www.httrack.com
 // always transform a '//' into a sigle '/'
 #define HTS_STRIP_DOUBLE_SLASH 0
 
+// case-sensitive pour les dossiers et fichiers (0/1)
+// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est
+// pas très utile..
+// ..et pas bcp respecté]
+// REMOVED
+// #define HTS_CASSE 0
+
+// Un fichier ayant une taille différente du content-length doit il être annulé?
+// SEE opt.tolerant and opt.http10
+// #define HTS_CL_IS_FATAL 0
+
 // une erreur supprime le fichier sur disque
 // (non fixé pour cause de retry)
 #define HTS_REMOVE_BAD_FILES 0
 
+// en cas de Range: xx- donnant un Content-length: xx
+// alors skipper le fichier, considéré comme transmis
+// #define HTS_SKIP_FULL_RANGE 1
+
+// nombre max de filtres que l'utilisateur peut fixer
+// #define HTS_FILTERSMAX 10000
 #define HTS_FILTERSINC 1000
 
 // connect non bloquant? (poll sur write)

src/htscore.h

@@ -193,7 +193,8 @@ struct cache_back {
   /* */
   int type;
   int ro;                   /**< read-only: no new cache is written */
-
+  FILE *dat, *ndx, *olddat; /**< new data, new index, old data files */
+  char *use;                /**< in-memory list of cached adr+fil keys */
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
   char lastmodified[256];
@@ -215,7 +216,6 @@ struct cache_back {
   int zipEntriesCapa;
   hts_boolean
       zipWriteFailed; /**< a cache write failed; stop touching the stream */
-  int zipWriteFailures; /**< consecutive entry write failures; reset on store */
 };
 
 #ifndef HTS_DEF_FWSTRUCT_hash_struct
@@ -287,12 +287,12 @@ struct filecreate_params {
 
 /* True if a new cache is being written (plain or zip backend). */
 HTS_STATIC int cache_writable(cache_back * cache) {
-  return (cache != NULL && cache->zipOutput != NULL);
+  return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL));
 }
 
 /* True if an old cache is available to read (plain or zip backend). */
 HTS_STATIC int cache_readable(cache_back * cache) {
-  return (cache != NULL && cache->zipInput != NULL);
+  return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL));
 }
 
 #endif
@@ -362,20 +362,6 @@ void usercommand(httrackp * opt, int exe, const char *cmd, const char *file,
 
 void usercommand_exe(const char *cmd, const char *file);
 
-// Finish the makeindex index.html (footer + refresh meta), run usercommand.
-// Updates *makeindex_done/*makeindex_fp in place; adr/fil are the mode strings.
-void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
-                          FILE **makeindex_fp, int makeindex_links,
-                          const char *makeindex_firstlink,
-                          const char *template_footer, const char *adr,
-                          const char *fil);
-
-// Flush ht_buff[0..ht_len] to save on disk (skip if MD5 unchanged); *fp
-// closed+NULLed on write. Precondition: ht_len>0.
-void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
-                          FILE **fp, const char *ht_buff, size_t ht_len,
-                          const char *adr, const char *fil, const char *save);
-
 int filters_init(char ***ptrfilters, int maxfilter, int filterinc);
 
 int fspc(httrackp * opt, FILE * fp, const char *type);
@@ -411,6 +397,10 @@ char *readfile_utf8(const char *fil);
 
 char *readfile_or(const char *fil, const char *defaultdata);
 
+#if 0
+void check_rate(TStamp stat_timestart, int maxrate);
+#endif
+
 /* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
    Not thread-safe; call from the single crawl loop. */
 
@@ -428,19 +418,6 @@ int back_pluggable_sockets(struct_back * sback, httrackp * opt);
 
 int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);
 
-/* One engine-loop tick: refresh the transfer stats and run the loop callback
-   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
-hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr);
-
-/* Wait until a test socket can be plugged, pumping transfers, stats and the
-   loop callback; gives up past the -E deadline. HTS_FALSE = callback abort. */
-hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
-                                      cache_back *cache, int ptr);
-
-/* Randomized inter-file pause target in [min_ms,max_ms] (#185), derived from a
-   timestamp seed so it is stable within one gap and rerolls per launch. */
-int hts_pause_target_ms(TStamp seed, int min_ms, int max_ms);
-
 /* Schedule more links from the heap into free slots. Returns the number queued,
    or <=0 if none could be added (no free slot / paused / stopped). */
 int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
@@ -462,6 +439,11 @@ int ask_continue(httrackp * opt);
 /* Number of decimal digits in n. */
 int nombre_digit(int n);
 
+// Java
+#if 0
+int hts_add_file(char *file, int file_position);
+#endif
+
 // Polling
 #if HTS_POLL
 int check_flot(T_SOC s);
@@ -484,8 +466,4 @@ void voidf(void);
 /* HTML marker comment marking where the top index is spliced. */
 #define HTS_TOPINDEX "TOP_INDEX_HTTRACK"
 
-/* Worst-case byte expansion HT_ADD_HTMLESCAPED* must reserve per escaper. */
-#define HTS_HTMLESCAPE_MAXEXP 5      /* escape_for_html_print: '&'->"&amp;" */
-#define HTS_HTMLESCAPE_FULL_MAXEXP 6 /* _full: high byte->"&#xHH;" */
-
 #endif

src/htscoremain.c

@@ -145,6 +145,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   char *url = NULL;             // URLS séparées par un espace
   int url_sz = 65535;
 
+  //char url[65536];         // URLS séparées par un espace
   // the parametres
   int httrack_logmode = 3;      // ONE log file
 
@@ -289,11 +290,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         if (tmp_argc == 1) {    /* pas -P & co */
           if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */
             if (argv_url < 0)
-              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
+              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
-                            // previous options
+            //if (argv_url>=0) {
             argv_url++;
             if (!argv_firsturl)
               argv_firsturl = x_argv[x_argc - 1];
+            //}
           } else {
             if (strcmp(tmp_argv[0], "-h") == 0) {
               help(argv[0], !opt->quiet);
@@ -322,8 +324,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         } else if (tmp_argc == 2) {
           if ((strcmp(tmp_argv[0], "-%L") == 0)) {      // liste d'URLs
             if (argv_url < 0)
-              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
+              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
-                            // previous options
+            //if (argv_url>=0)
             argv_url++;         /* forcer */
           }
         }
@@ -401,11 +403,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     else
                       inQuote = 1;
                   } else if (!inQuote && !noDbl && argv[na][i] == ',') {
+                    //StringAddchar(path, '\0');
+                    //j = 0;
                     path = &opt->path_log;
                   } else {
                     StringAddchar(*path, argv[na][i]);
+                    //path[j++] = argv[na][i];
                   }
                 }
+                //path[j++] = '\0';
                 if (StringLength(opt->path_log) == 0) {
                   StringCopyS(opt->path_log, opt->path_html);
                 }
@@ -414,6 +420,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if (check_path(&opt->path_html, argv_firsturl)) {
                   opt->dir_topindex = 1;        // rebuilt top index
                 }
+                //printf("-->%s\n%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log));                
               }
               break;
             }                   // switch
@@ -533,14 +540,73 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           }
         }
       } while(lastp != NULL);
+      //fclose(fp);
     }
   }
 
-  // No new cache but an old one? promote it
+  // Existence d'un cache - pas de new mais un old.. renommer
 #if DEBUG_STEPS
   printf("Checking cache\n");
 #endif
-  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  if (!fexist
+      (fconcat
+       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+       StringBuff(opt->path_log), "hts-cache/new.zip"))) {
+    if (fexist
+        (fconcat
+         (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+         StringBuff(opt->path_log), "hts-cache/old.zip"))) {
+      rename(fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+             StringBuff(opt->path_log),
+              "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                            StringBuff(opt->path_log),
+                                            "hts-cache/new.zip"));
+    }
+  } else
+    if ((!fexist
+         (fconcat
+          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+          StringBuff(opt->path_log), "hts-cache/new.dat")))
+        ||
+        (!fexist
+         (fconcat
+          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+          StringBuff(opt->path_log),
+           "hts-cache/new.ndx")))) {
+    if ((fexist
+         (fconcat
+          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+          StringBuff(opt->path_log), "hts-cache/old.dat")))
+        &&
+        (fexist
+         (fconcat
+          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+          StringBuff(opt->path_log),
+           "hts-cache/old.ndx")))) {
+      remove(fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+             StringBuff(opt->path_log),
+              "hts-cache/new.dat"));
+      remove(fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+             StringBuff(opt->path_log),
+              "hts-cache/new.ndx"));
+      //remove(fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
+      rename(fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+             StringBuff(opt->path_log),
+              "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                            StringBuff(opt->path_log),
+                                            "hts-cache/new.dat"));
+      rename(fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+              "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                            StringBuff(opt->path_log),
+                                            "hts-cache/new.ndx"));
+      //rename(fconcat(StringBuff(opt->path_log),"hts-cache/old.lst"),fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
+    }
+  }
 
   /* Interrupted mirror detected */
   if (!opt->quiet) {
@@ -708,6 +774,24 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             HTS_PANIC_PRINTF(s);
             htsmain_free();
             return -1;
+#else
+#if 0
+            char _args[8][256];
+            char *args[8];
+
+            printf("Cheking for updates...\n");
+            strcpybuff(_args[0], argv[0]);
+            strcpybuff(_args[1], "--get");
+            sprintf(_args[2], HTS_UPDATE_WEBSITE, 0, "");
+            strcpybuff(_args[3], "--quickinfo");
+            args[0] = _args[0];
+            args[1] = _args[1];
+            args[2] = _args[2];
+            args[3] = _args[3];
+            args[4] = NULL;
+            if (execvp(args[0], args) == -1) {
+            }
+#endif
 #endif
           }
           //
@@ -725,6 +809,103 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     }
   }
 
+  // Compter urls/jokers
+  /*
+     if (argv_url<=0) { 
+     int na;
+     argv_url=0;
+     for(na=1;na<argc;na++) {
+     if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) {
+     na++;    // sauter nom de proxy
+     } else if (!cmdl_opt(argv[na])) { 
+     argv_url++;   // un de plus       
+     } else if (strcmp(argv[na],"-h")==0) {
+     help(argv[0],!opt->quiet);
+     htsmain_free();
+     return 0;
+     } else {
+     if ((strchr(argv[na],'q')!=NULL))
+     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
+     if ((strchr(argv[na],'i')!=NULL)) {  // doit.log!
+     argv_url=0;
+     na=argc;
+     }
+     }
+     }
+     }  
+   */
+
+  // Ici on ajoute les arguments qui ont été appelés avant au cas où on récupère une session
+  // Exemple: httrack www.truc.fr -L0 puis ^C puis httrack sans URL : ajouter URL précédente
+  /*
+     if (argv_url==0) {
+     //if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) {  // il existe déja un cache précédent.. renommer
+     if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) {    // un cache est présent
+
+     x_argvblk=(char*) calloct(32768,1);
+
+     if (x_argvblk!=NULL) {
+     FILE* fp;
+     int x_argc;
+
+     //strcpybuff(x_argvblk,"httrack ");
+     fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb");
+     if (fp) {
+     linput(fp,x_argvblk+strlen(x_argvblk),8192);
+     fclose(fp); fp=NULL;
+     }
+
+     // calculer arguments selon derniers arguments
+     x_argv[0]=argv[0];
+     x_argc=1;
+     {
+     char* p=x_argvblk;
+     do {
+     x_argv[x_argc++]=p;
+     //p=strstr(p," ");
+     // exemple de chaine: "echo \"test\"" c:\a "\$0"
+     p=next_token(p,1);    // prochain token
+     if (p) {
+     *p=0;    // octet nul (tableau)
+     p++;
+     }            
+     } while(p!=NULL);
+     }
+     // recopier arguments actuels (pointeurs uniquement)
+     {
+     int na;
+     for(na=1;na<argc;na++) {
+     if (strcmp(argv[na],"-O") != 0)    // SAUF le path!
+     x_argv[x_argc++]=argv[na];
+     else
+     na++;
+     }
+     }
+     argc=x_argc;      // nouvel argc
+     argv=x_argv;      // nouvel argv
+     }
+
+     }
+     //}
+     }
+   */
+
+  // Vérifier quiet
+  /*
+     { 
+     int na;    
+     for(na=1;na<argc;na++) {
+     if (!cmdl_opt(argv[na])) { 
+     if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0))
+     na++;    // sauter nom de proxy
+     } else {
+     if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL))
+     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
+     }
+     }
+     }
+   */
+
   /* Engine self-tests: -#test lists them, -#test=NAME [args] runs one. Handled
      here, ahead of the no-URL usage gate below, so they need no dummy URL. */
   {
@@ -831,8 +1012,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   } else {                      // plus de 2 paramètres
     // un fichier log existe?
     if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                       StringBuff(opt->path_log),
+      StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // fichier lock?
-                       "hts-in_progress.lock"))) { // fichier lock?
+      //char s[32];
 
       opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
       if (opt->quiet == 0) {
@@ -863,8 +1044,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           }
         }
       }
-    } else if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+    } else
-                              StringBuff(opt->path_html), "index.html"))) {
+      if (fexist
+          (fconcat
+           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "index.html"))) {
+      //char s[32];
       opt->cache = HTS_CACHE_TEST_UPDATE;
       if (opt->quiet == 0) {
         if ((fexist
@@ -959,6 +1143,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             //
           case 'g':            // récupérer un (ou plusieurs) fichiers isolés
             opt->wizard = HTS_WIZARD_AUTO;
+            //opt->wizard=0;             // pas de wizard
             opt->cache = HTS_CACHE_NONE; // ni de cache
             opt->makeindex = 0; // ni d'index
             httrack_logmode = 1;        // erreurs à l'écran
@@ -1058,6 +1243,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             while(isdigit((unsigned char) *(com + 1)))
               com++;
             break;
+            //
+            //case 'A': opt->urlmode=1; break;
+            //case 'R': opt->urlmode=2; break;
           case 'K':
             opt->urlmode = HTS_URLMODE_ABSOLUTE;
             if (isdigit((unsigned char) *(com + 1))) {
@@ -1069,6 +1257,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               while(isdigit((unsigned char) *(com + 1)))
                 com++;
             }
+            //if (*(com+1)=='0') { opt->urlmode=2; com++; } break;
+            //
           case 'c':
             if (isdigit((unsigned char) *(com + 1))) {
               sscanf(com + 1, "%d", &opt->maxsoc);
@@ -1452,7 +1642,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF
                     ("Option %w needs to be followed by a blank space, and a module name");
-                  printf("Example: -%%w httrack-plugin\n");
+                  printf("Example: -%%w htsswf\n");
                   htsmain_free();
                   return -1;
                 } else {
@@ -1556,6 +1746,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   } else {
                     char *a;
 
+                    //char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs);
                     for(a = argv[na]; *a != '\0'; a++) {
                       if (*a == ';') {  /* next one */
                         StringAddchar(opt->mimedefs, '\n');
@@ -1785,68 +1976,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   StringCat(opt->strip_query, argv[na]);
                 }
                 break;
-              case 'K': // cookies-file: extra Netscape cookies.txt to preload
-                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
-                  HTS_PANIC_PRINTF(
-                      "Option cookies-file needs a blank space and "
-                      "a cookies.txt path");
-                  printf("Example: --cookies-file \"/home/me/cookies.txt\"\n");
-                  htsmain_free();
-                  return -1;
-                } else {
-                  na++;
-                  if (strlen(argv[na]) >= 1024) {
-                    HTS_PANIC_PRINTF("Cookie file path too long");
-                    htsmain_free();
-                    return -1;
-                  }
-                  StringCopy(opt->cookies_file, argv[na]);
-                }
-                break;
-              case 'Y': // why: explain the filter verdict for a URL, no crawl
-                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
-                  HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
-                  printf(
-                      "Example: --why \"http://www.example.com/file.zip\"\n");
-                  htsmain_free();
-                  return -1;
-                } else {
-                  na++;
-                  if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
-                    HTS_PANIC_PRINTF("why URL too long");
-                    htsmain_free();
-                    return -1;
-                  }
-                  StringCopy(opt->why_url, argv[na]);
-                }
-                break;
-              case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
-                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
-                  HTS_PANIC_PRINTF("Option pause needs a blank space and a "
-                                   "delay in seconds (MIN[:MAX])");
-                  printf("Example: --pause 5:10\n");
-                  htsmain_free();
-                  return -1;
-                } else {
-                  double pmin = 0, pmax = 0;
-                  int nf;
-
-                  na++;
-                  nf = sscanf(argv[na], "%lf:%lf", &pmin, &pmax);
-                  if (nf < 2)
-                    pmax = pmin; /* a single value means a fixed delay */
-                  /* positive-form bounds: NaN fails every comparison, so this
-                     rejects it before the undefined (int)(NaN*1000) cast */
-                  if (nf < 1 || !(pmin >= 0 && pmax >= pmin && pmax <= 86400)) {
-                    HTS_PANIC_PRINTF("Invalid --pause range (expected "
-                                     "MIN[:MAX] seconds, 0<=MIN<=MAX<=86400)");
-                    htsmain_free();
-                    return -1;
-                  }
-                  opt->pause_min_ms = (int) (pmin * 1000.0);
-                  opt->pause_max_ms = (int) (pmax * 1000.0);
-                }
-                break;
               case 't':        /* do not change type (ending) of filenames according to the MIME type */
                 opt->no_type_change = 1;
                 if (*(com+1)=='0') { opt->no_type_change = 0; com++; }
@@ -1911,7 +2040,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   HTS_PANIC_PRINTF(s);
                   htsmain_free();
                   return -1;
-              } break;
+                }
+                break;
+
+                //case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break;
               }
             }
             break;
@@ -1953,20 +2085,18 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     char BIGSTK url[HTS_URLMAXSIZE * 2];
                     char linepos[256];
                     int pos;
-                    LLint cacheNdxLen = 0;
+                    char *cacheNdx =
-                    char *cacheNdx = readfile2(
+                      readfile(fconcat
-                        fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                                StringBuff(opt->path_log), "hts-cache/new.ndx"),
+                                "hts-cache/new.ndx"));
-                        &cacheNdxLen);
                     cache_init(&cache, opt);    /* load cache */
                     if (cacheNdx != NULL) {
                       char firstline[256];
                       char *a = cacheNdx;
-                      const char *const end = cacheNdx + cacheNdxLen;
 
                       a += cache_brstr(a, firstline, sizeof(firstline));
                       a += cache_brstr(a, firstline, sizeof(firstline));
-                      while (a != NULL && a < end) {
+                      while(a != NULL) {
                         a = strchr(a + 1, '\n');        /* start of line */
                         if (a) {
                           htsblk r;
@@ -1974,15 +2104,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                           /* */
                           a++;
                           /* read "host/file" */
-                          a += cache_binput(a, end, afs.af.adr, HTS_URLMAXSIZE);
+                          a += binput(a, afs.af.adr, HTS_URLMAXSIZE);
-                          a += cache_binput(a, end, afs.af.fil, HTS_URLMAXSIZE);
+                          a += binput(a, afs.af.fil, HTS_URLMAXSIZE);
                           url[0] = '\0';
                           if (!link_has_authority(afs.af.adr))
                             strcatbuff(url, "http://");
                           strcatbuff(url, afs.af.adr);
                           strcatbuff(url, afs.af.fil);
                           /* read position */
-                          a += cache_binput(a, end, linepos, 200);
+                          a += binput(a, linepos, 200);
                           sscanf(linepos, "%d", &pos);
                           if (!hasFilter
                               || (strjoker(url, filter, NULL, NULL) != NULL)
@@ -2157,7 +2287,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 return 0;
                 break;
               case '~':        /* internal lib test */
-                HTS_PANIC_PRINTF("Option #~ is disabled for security reasons");
+                HTS_PANIC_PRINTF
+                  ("Option #~ is disabled for security reasons");
+                //Disabled because choke on GCC 4.3 (toni from links2linux.de)
+                //{
+                //  char thisIsATestYouShouldSeeAnError[12];
+                //  const char *const bufferOverflowTest = "0123456789012345678901234567890123456789";
+                //  strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest);
+                //  return 0;
+                //}
                 break;
               case 'f':
                 opt->flush = 1;
@@ -2259,6 +2397,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               opt->proxy.active = 1;
               // Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
               a = argv[na] + strlen(argv[na]) - 1;
+              // a=strstr(argv[na],":");  // port
               while((a > argv[na]) && (*a != ':') && (*a != '@'))
                 a--;
               if (*a == ':') {  // un port est présent, <proxy>:port
@@ -2363,14 +2502,116 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
 #endif
 #endif
 
+  //printf("WARNING! This is *only* a beta-release of HTTrack\n");
   io_flush;
 
 #if DEBUG_STEPS
   printf("Cache & log settings\n");
 #endif
 
-  // If both cache generations exist, keep the most complete one
+  // on utilise le cache..
-  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  // en cas de présence des deux versions, garder la version la plus avancée,
+  // cad la version contenant le plus de fichiers  
+  if (opt->cache) {
+    if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // problemes..
+      if (fexist
+          (fconcat
+           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+            "hts-cache/new.dat"))) {
+        if (fexist
+            (fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+              "hts-cache/old.zip"))) {
+          if (fsize
+              (fconcat
+               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                "hts-cache/new.zip")) < 32768) {
+            if (fsize
+                (fconcat
+                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                  "hts-cache/old.zip")) > 65536) {
+              if (fsize
+                  (fconcat
+                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                    "hts-cache/old.zip")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                          StringBuff(opt->
+                                                                     path_log),
+                                                          "hts-cache/new.zip")))
+              {
+                remove(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/new.zip"));
+                rename(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                      StringBuff(opt->path_log),
+                                                      "hts-cache/new.zip"));
+              }
+            }
+          }
+        }
+      } else
+        if (fexist
+            (fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+              "hts-cache/new.dat"))
+            &&
+            fexist(fconcat
+                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                    "hts-cache/new.ndx"))) {
+        if (fexist
+            (fconcat
+             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+              "hts-cache/old.dat"))
+            &&
+            fexist(fconcat
+                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                    "hts-cache/old.ndx"))) {
+          // switcher si new<32Ko et old>65Ko (tailles arbitraires) ?
+          // ce cas est peut être une erreur ou un crash d'un miroir ancien, prendre
+          // alors l'ancien cache
+          if (fsize
+              (fconcat
+               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                "hts-cache/new.dat")) < 32768) {
+            if (fsize
+                (fconcat
+                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                  "hts-cache/old.dat")) > 65536) {
+              if (fsize
+                  (fconcat
+                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                    "hts-cache/old.dat")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                          StringBuff(opt->
+                                                                     path_log),
+                                                          "hts-cache/new.dat")))
+              {
+                remove(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/new.dat"));
+                remove(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/new.ndx"));
+                rename(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                      StringBuff(opt->path_log),
+                                                      "hts-cache/new.dat"));
+                rename(fconcat
+                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                        "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                                      StringBuff(opt->path_log),
+                                                      "hts-cache/new.ndx"));
+                //} else {  // ne rien faire
+                //  remove("hts-cache/old.dat");
+                //  remove("hts-cache/old.ndx");
+              }
+            }
+          }
+        }
+      }
+    }
+  }
   // Débuggage des en têtes
   if (_DEBUG_HEAD) {
     ioinfo =
@@ -2446,6 +2687,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     {
       FILE *fp = NULL;
 
+      //int n=0;
       char t[256];
 
       time_local_rfc822(t);     // faut bien que ca serve quelque part l'heure RFC1945 arf'
@@ -2479,8 +2721,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       }
 
       strcpybuff(n_lock,
-                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+             fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                         StringBuff(opt->path_log), "hts-in_progress.lock"));
+                     "hts-in_progress.lock"));
+      //sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
       /*do {
          if (!n)
          sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
@@ -2556,6 +2799,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           fprintf(fp, LF);
           fclose(fp);
           fp = NULL;
+          //} else if (opt->debug>1) {
+          //  printf("! FileOpen error, \"%s\"\n",strerror(errno));
         }
       }
       // petit message dans le lock

src/htsdefines.h

@@ -69,15 +69,11 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
 typedef struct t_hts_callbackarg t_hts_callbackarg;
 #endif
 
-/* Marks a symbol an external wrapper module exports back to the engine.
+/* Marks a symbol an external wrapper module exports back to the engine
-   Must override -fvisibility=hidden on ELF, or dlopen()ed plugins
+   (dllexport on Windows, nothing elsewhere). */
-   hide their own hts_plug()/hts_unplug() entry points. */
 #ifndef EXTERNAL_FUNCTION
 #ifdef _WIN32
 #define EXTERNAL_FUNCTION __declspec(dllexport)
-#elif ((defined(__GNUC__) && (__GNUC__ >= 4)) ||                               \
-       (defined(HAVE_VISIBILITY) && HAVE_VISIBILITY))
-#define EXTERNAL_FUNCTION __attribute__((visibility("default")))
 #else
 #define EXTERNAL_FUNCTION
 #endif

src/htsdns_selftest.c

@@ -297,26 +297,6 @@ int dns_selftests(httrackp *opt) {
     IPV6_resolver = 0;
   }
 
-  /* "host:port" resolves as the bare host: the cache/connect path strips
-     ":port" before both the backend and the cache key (#181). */
-  mock_reset_calls();
-  {
-    SOCaddr a;
-    char ip[64];
-    const char *err = NULL;
-
-    /* cache miss: the backend is hit with the stripped host, not "host:port" */
-    CHECK(hts_dns_resolve2(opt, "v6only.test:8080", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* the bare host shares that one cache entry (stripped key) */
-    CHECK(hts_dns_resolve2(opt, "v6only.test", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* a port variant of an already-cached host also hits, right address */
-    CHECK(hts_dns_resolve2(opt, "v4only.test:1234", &a, &err) != NULL);
-    SOCaddr_inetntoa(ip, sizeof(ip), a);
-    CHECK(strcmp(ip, "1.2.3.4") == 0);
-  }
-
   /* newhttp_addr() must connect to the addr_index-th address, not always the
      first: this is what back_connect_next relies on to reach the fallback. */
   {

src/htsencoding.c

@@ -30,13 +30,11 @@ Please visit our Website: http://www.httrack.com
 /* Author: Xavier Roche                                         */
 /* ------------------------------------------------------------ */
 
-#include <stdint.h>
-
 #include "htscharset.h"
 #include "htsencoding.h"
 #include "htssafe.h"
 
-/* static int decode_entity(const uint64_t hash, const size_t len);
+/* static int decode_entity(const unsigned int hash, const size_t len);
 */
 #include "htsentities.h"
 
@@ -52,31 +50,30 @@ static int get_hex_value(char c) {
     return -1;
 }
 
-/* 64-bit FNV-1a; must match htsentities.sh, which keys the entity table on it.
+/* Numerical Recipes,
- */
+   see <http://en.wikipedia.org/wiki/Linear_congruential_generator> */
-#define HASH_INIT 0xcbf29ce484222325ULL
+#define HASH_PRIME ( 1664525 )
-#define HASH_PRIME 0x100000001b3ULL
+#define HASH_CONST ( 1013904223 )
-#define HASH_ADD(HASH, C)                                                      \
+#define HASH_ADD(HASH, C) do {                  \
-  do {                                                                         \
-    (HASH) ^= (unsigned char) (C);                                             \
     (HASH) *= HASH_PRIME;                       \
+    (HASH) += HASH_CONST;                       \
+    (HASH) += (C);                              \
   } while(0)
 
 int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t max, const char *charset) {
   size_t i, j, ampStart, ampStartDest;
   int uc;
   int hex;
-  uint64_t hash;
+  unsigned int hash;
 
   assertf(max != 0);
-  for (i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0, uc = -1, hex = 0,
+  for(i = 0, j = 0, ampStart = (size_t) -1, ampStartDest = 0,
-      hash = HASH_INIT;
+        uc = -1, hex = 0, hash = 0 ; src[i] != '\0' ; i++) {
-       src[i] != '\0'; i++) {
     /* start of entity */
     if (src[i] == '&') {
       ampStart = i;
       ampStartDest = j;
-      hash = HASH_INIT;
+      hash = 0;
       uc = -1;
     }
     /* inside a potential entity */
@@ -177,11 +174,14 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
       }
       /* alphanumerical entity */
       else {
-        /* alphanum, capped at the longest name
+        /* alphanum and not too far ('&thetasym;' is the longest) */
-         * '&CounterClockwiseContourIntegral;' (31) */
+        if (i <= ampStart + 10 &&
-        if (i <= ampStart + 31 && ((src[i] >= '0' && src[i] <= '9') ||
+            (
-                                   (src[i] >= 'A' && src[i] <= 'Z') ||
+             (src[i] >= '0' && src[i] <= '9')
-                                   (src[i] >= 'a' && src[i] <= 'z'))) {
+             || (src[i] >= 'A' && src[i] <= 'Z')
+             || (src[i] >= 'a' && src[i] <= 'z')
+             )
+            ) {
           /* compute hash */
           HASH_ADD(hash, (unsigned char) src[i]);
         } else {
@@ -191,8 +191,8 @@ int hts_unescapeEntitiesWithCharset(const char *src, char *dest, const size_t ma
       }
     }
     
-    /* reserve one byte for the trailing NUL written after the loop */
+    /* copy */
-    if (j + 1 >= max) {
+    if (j + 1 > max) {
       /* overflow */
       return -1;
     }
@@ -300,11 +300,6 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
 
           /* Was the character read successfully ? */
           if (nRead == utfBufferSize) {
-            /* the 'continue' below skips the NUL-reserve guard: re-check */
-            if (utfBufferJ + utfBufferSize >= max) {
-              return -1;
-            }
-
             /* Rollback write position to sequence start write position */
             j = utfBufferJ;
 
@@ -319,8 +314,8 @@ int hts_unescapeUrlSpecial(const char *src, char *dest, const size_t max,
       }
     }
 
-    /* reserve one byte for the trailing NUL written after the loop */
+    /* Check for overflow */
-    if (j + 1 >= max) {
+    if (j + 1 > max) {
       return -1;
     }
 

src/htsentities.sh

@@ -1,92 +1,75 @@
 #!/bin/bash
 #
-# Regenerate htsentities.h from the WHATWG named character references.
 
-set -euo pipefail
+src=html40.txt
-
+url=http://www.w3.org/TR/1998/REC-html40-19980424/html40.txt
-src=entities.json
-url=https://html.spec.whatwg.org/entities.json
 dest=htsentities.h
 
-# 64-bit FNV-1a of $1, printed as a C constant. Must match the hash in
+(
-# htsencoding.c. The offset basis is stored as its wrapped (signed) bit pattern;
+    cat <<EOF
-# bash arithmetic is 64-bit two's complement, so the result is bit-exact.
+/*
-fnv1a() {
+  -- ${dest} --
-    local s=$1 i c h=$((0xcbf29ce484222325))
+  FILE GENERATED BY $0, DO NOT MODIFY
-    for ((i = 0; i < ${#s}; i++)); do
-        printf -v c '%d' "'${s:i:1}"
-        h=$(((h ^ (c & 0xff)) * 0x100000001b3))
-    done
-    printf '0x%016xULL' "$h"
-}
 
-if [ ! -f "$src" ]; then
+  We compute the LCG hash
-    curl -fsS "$url" -o "$src"
+  (see <http://en.wikipedia.org/wiki/Linear_congruential_generator>)
-fi
+  for each entity. We should in theory check using strncmp() that we
+  actually have the correct entity, but this is actually statistically
+  not needed.
 
-# Keep ';'-terminated single-codepoint names; the ~93 multi-codepoint refs can't
+  We may want to do better, but we expect the hash function to be uniform, and
-# fit decode_entity's single-codepoint return and are skipped (left verbatim).
+  let the compiler be smart enough to optimize the switch (for example by
-pairs=$(jq -r '
+  checking in log2() intervals)
-    to_entries
-    | map(select((.key | endswith(";")) and (.value.codepoints | length == 1)))
-    | sort_by(.key)
-    | .[] | "\(.key | ltrimstr("&") | rtrimstr(";"))\t\(.value.codepoints[0])"' "$src")
   
-# Skipped multi-codepoint names, kept to prove none aliases an emitted hash.
+  This code has been generated using the evil $0 script.
-skipped=$(jq -r '
+*/
-    to_entries
-    | map(select((.key | endswith(";")) and (.value.codepoints | length > 1)))
-    | .[] | .key | ltrimstr("&") | rtrimstr(";")' "$src")
 
-cases=""
+static int decode_entity(const unsigned int hash, const size_t len) {
-emit_hashes=""
-while IFS=$'\t' read -r name cp; do
-    hash=$(fnv1a "$name")
-    cases+="    /* $name */"$'\n'
-    cases+="  case $hash:"$'\n'
-    cases+="    if (len == ${#name}) {"$'\n'
-    cases+="      return $cp;"$'\n'
-    cases+="    }"$'\n'
-    cases+="    break;"$'\n'
-    emit_hashes+="$hash"$'\n'
-done <<<"$pairs"
-
-skip_hashes=""
-while IFS= read -r name; do
-    [ -n "$name" ] && skip_hashes+="$(fnv1a "$name")"$'\n'
-done <<<"$skipped"
-
-# The switch keys on the hash alone, so the dispatch is correct only while every
-# emitted name hashes uniquely; prove it here, no runtime name compare needed.
-dups=$(printf '%s' "$emit_hashes" | sort | uniq -d || true)
-if [ -n "$dups" ]; then
-    echo "FATAL: two entity names share a hash (duplicate switch case); change the hash:" >&2
-    echo "$dups" >&2
-    exit 1
-fi
-# A skipped name colliding with an emitted hash would mis-decode instead of
-# staying verbatim; forbid that too.
-aliased=$(comm -12 <(printf '%s' "$emit_hashes" | sort -u) <(printf '%s' "$skip_hashes" | sort -u) || true)
-if [ -n "$aliased" ]; then
-    echo "FATAL: a skipped multi-codepoint name aliases an emitted hash:" >&2
-    echo "$aliased" >&2
-    exit 1
-fi
-
-cat >"$dest" <<EOF
-/* GENERATED by $0 from the WHATWG named character references
-   (${url}). DO NOT EDIT.
-   Dispatch keys on a 64-bit FNV-1a hash of the entity name; the generator
-   aborts on any hash collision, so no runtime name compare is needed. */
-
-#include <stdint.h>
-
-static int decode_entity(const uint64_t hash, const size_t len) {
   switch(hash) {
-${cases}  }
+EOF
+    (
+        if test -f ${src}; then
+            cat ${src}
+        else
+            GET "${url}"
+        fi
+    ) |
+        grep -E '^<!ENTITY [a-zA-Z0-9_]' |
+        sed \
+            -e 's/<!ENTITY //' -e "s/[[:space:]][[:space:]]*/ /g" \
+            -e 's/-->$//' \
+            -e 's/\([^ ]*\) CDATA "&#\([^\"]*\);" -- \(.*\)/\1 \2 \3/' |
+        (
+            read -r A
+            while test -n "$A"; do
+                ent="${A%% *}"
+                code=$(echo "$A" | cut -f2 -d' ')
+                # compute hash
+                hash=0
+                i=0
+                a=1664525
+                c=1013904223
+                m="$((1 << 32))"
+                while test "$i" -lt ${#ent}; do
+                    d="$(echo -n "${ent:${i}:1}" | hexdump -v -e '/1 "%d"')"
+                    hash="$((((hash * a) % (m) + d + c) % (m)))"
+                    i=$((i + 1))
+                done
+                echo -e "    /* $A */"
+                echo -e "  case ${hash}u:"
+                echo -e "    if (len == ${#ent} /* && strncmp(ent, \"${ent}\") == 0 */) {"
+                echo -e "      return ${code};"
+                echo -e "    }"
+                echo -e "    break;"
+
+                # next
+                read -r A
+            done
+        )
+    cat <<EOF
+  }
   /* unknown */
   return -1;
 }
 EOF
-
+) >${dest}
-echo "wrote $dest ($(grep -c '^  case ' "$dest") entities)" >&2

src/htsfilters.c

@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
 #include "htsbase.h"
 #include "htslib.h"
 #include <ctype.h>
-#include <stdint.h>
 /* END specific definitions */
 
 // à partir d'un tableau de {"+*.toto","-*.zip","+*.tata"} définit si nom est autorisé
@@ -95,127 +94,17 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
   return verdict;
 }
 
-int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
+// supercomparateur joker (tm)
-                     const char *nom2, LLint *size, int *size_flag,
+// compare a et b (b=avec joker dedans), case insensitive [voir CI]
-                     int *depth) {
+// renvoi l'adresse de la première lettre de la chaine
-  int depth1 = 0, depth2 = 0;
+// (càd *[..]toto.. renvoi adresse de toto dans la chaine)
-  int flag1 = 0, flag2 = 0;
+// accepte les délires du genre www.*.*/ * / * truc*.*
-  LLint sz1 = 0, sz2 = 0;
+// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
-  int jok1, jok2, use1;
+// * = toute lettre
-
+// --?-- : spécifique à HTTrack et aux ?
-  if (size) {
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
-    sz1 = *size;
-    sz2 = *size;
-  }
-  jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
-                     size_flag ? &flag1 : NULL, &depth1);
-  jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
-                     size_flag ? &flag2 : NULL, &depth2);
-  use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
-  if (size)
-    *size = use1 ? sz1 : sz2;
-  if (size_flag)
-    *size_flag = use1 ? flag1 : flag2;
-  if (depth)
-    *depth = use1 ? depth1 : depth2;
-  return use1 ? jok1 : jok2;
-}
-
-/* Real filters/URLs fit HTS_URLMAXSIZE*2; past it a hostile pattern recurses
-   O(len) deep or runs O(n^2*stars). Cap length (depth) and steps (work). */
-#define STRJOKER_MAXLEN (HTS_URLMAXSIZE * 2)
-#define STRJOKER_MAXSTEPS 2000000u
-
-/* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
-   pair keeps star-heavy patterns polynomial instead of exponential (#501). */
-typedef struct strjoker_memo {
-  const char *chaine0, *joker0; /* offsets are relative to these bases */
-  size_t stride;                /* strlen(joker0) + 1 */
-  unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
-  size_t *nsteps; /* shared work counter; NULL: unbounded (oracle) */
-} strjoker_memo;
-
-static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
-                                 const char *joker, LLint *size,
-                                 int *size_flag);
-
-/* A pair that failed once fails forever (*size is only written on the success
-   path), so record NULL results and cut the re-exploration. */
-static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
-                                const char *joker, LLint *size,
-                                int *size_flag) {
-  size_t bit = 0;
-  const char *adr;
-
-  if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
-    return NULL; /* work budget spent: fail the match safely */
-  if (memo->failed) {
-    bit = (size_t) (chaine - memo->chaine0) * memo->stride +
-          (size_t) (joker - memo->joker0);
-    if (memo->failed[bit >> 3] & (unsigned char) (1u << (bit & 7)))
-      return NULL;
-  }
-  adr = strjoker_impl(memo, chaine, joker, size, size_flag);
-  if (adr == NULL && memo->failed)
-    memo->failed[bit >> 3] |= (unsigned char) (1u << (bit & 7));
-  return adr;
-}
-
-/* Match chaine against joker with a shared work budget *nsteps (a strjokerfind
-   sweep passes one counter, bounding the whole scan). */
-static const char *strjoker_bounded(const char *chaine, const char *joker,
-                                    LLint *size, int *size_flag,
-                                    size_t *nsteps) {
-  strjoker_memo memo = {chaine, joker, 0, NULL, nsteps};
-  unsigned char stackbits[2048];
-  hts_boolean onheap = HTS_FALSE;
-  const char *adr;
-
-  if (chaine != NULL && joker != NULL) {
-    const size_t l1 = strlen(chaine), l2 = strlen(joker);
-
-    if (l1 > STRJOKER_MAXLEN || l2 > STRJOKER_MAXLEN)
-      return NULL; /* beyond any real filter/URL: bound depth+work */
-    memo.stride = l2 + 1;
-    if (l1 + 1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
-      const size_t bytes = ((l1 + 1) * memo.stride + 7) / 8;
-
-      if (bytes <= sizeof(stackbits)) {
-        memset(stackbits, 0, bytes);
-        memo.failed = stackbits;
-      } else {
-        memo.failed = (unsigned char *) calloct(bytes, 1);
-        onheap = memo.failed != NULL ? HTS_TRUE : HTS_FALSE;
-      }
-    }
-  }
-  adr = strjoker_rec(&memo, chaine, joker, size, size_flag);
-  if (onheap)
-    freet(memo.failed);
-  return adr;
-}
-
-// wildcard comparator: match chaine against joker (pattern), case-insensitive
-// returns the address of the first matched letter past any leading joker
-// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
-                                LLint *size, int *size_flag) {
-  size_t nsteps = 0;
-
-  return strjoker_bounded(chaine, joker, size, size_flag, &nsteps);
-}
-
-/* Test-only oracle: the same matcher without the failure memo. */
-const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
-                            int *size_flag) {
-  const strjoker_memo memo = {chaine, joker, 0, NULL};
-
-  return strjoker_rec(&memo, chaine, joker, size, size_flag);
-}
-
-static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
-                                 const char *joker, LLint *size,
                           int *size_flag) {
+  //int err=0;
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -304,12 +193,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
           int len = (int) strlen(joker);
 
           while((joker[i] != RIGHT) && (joker[i]) && (i < len)) {
-            // '\' escapes the next char as a literal member, e.g. *[\[\]]
+            if ((joker[i] == '<') || (joker[i] == '>')) {       // *[<10]
-            if (joker[i] == '\\' && joker[i + 1] != '\0') {
-              i++;
-              pass[(int) (unsigned char) joker[i]] = 1;
-              i++;
-            } else if ((joker[i] == '<') || (joker[i] == '>')) { // *[<10]
               int lsize = 0;
               int lverdict;
 
@@ -337,9 +221,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
                 while(isdigit((unsigned char) joker[i]))
                   i++;
               }
-            } else if (joker[i + 1] == '-' && joker[i + 2] != '\0') {
+            } else if (joker[i + 1] == '-') {   // 2 car, ex: *[A-Z]
-              // range *[A-Z]; the '\0' guard rejects a truncated *[a- (else
-              // i+=3 overshoots the NUL)
               if ((int) (unsigned char) joker[i + 2] >
                   (int) (unsigned char) joker[i]) {
                 int j;
@@ -347,9 +229,14 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
                 for(j = (int) (unsigned char) joker[i];
                     j <= (int) (unsigned char) joker[i + 2]; j++)
                   pass[j] = 1;
+
               }
+              // else err=1;
               i += 3;
             } else {            // 1 car, ex: *[ ]
+              if (joker[i + 2] == '\\' && joker[i + 3] != 0) {  // escaped char, such as *[\[] or *[\]]
+                i++;
+              }
               pass[(int) (unsigned char) joker[i]] = 1;
               i++;
             }
@@ -371,6 +258,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       for(i = 0; i < 256; i++)
         pass[i] = 1;            // tout autoriser
       jmp = 1;
+      ////if (joker[2]==LEFT) jmp=3;        // permet de recher *<crochet ouvrant>
     }
 
     {
@@ -389,7 +277,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 
       // tester sans le joker (pas ()+ mais ()*)
       if (!unique) {
-        if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag))) {
+        if ((adr = strjoker(chaine, joker + jmp, size, size_flag))) {
           return adr;
         }
       }
@@ -398,11 +286,10 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       if (!unique)
         max = (int) strlen(chaine);
       else                      /* *(a) only match a (not aaaaa) */
-        max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
+        max = 1;
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
-          if ((adr = strjoker_rec(memo, chaine + i + 1, joker + jmp, size,
+          if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
-                                  size_flag))) {
             return adr;
           }
           i++;
@@ -412,8 +299,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 
       // tester chaîne vide
       if (i != max + 2)         // avant c'est ok
-        if ((adr = strjoker_rec(memo, chaine + max, joker + jmp, size,
+        if ((adr = strjoker(chaine + max, joker + jmp, size, size_flag)))
-                                size_flag)))
           return adr;
 
       return NULL;              // perdu
@@ -435,7 +321,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       // comparaison ok?
       if (ok) {
         // continuer la comparaison.
-        if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag))
+        if (strjoker(chaine + jmp, joker + jmp, size, size_flag))
           return chaine;        // retourner 1e lettre
       }
 
@@ -451,18 +337,12 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 // d'un strcpy sur une variable ayant un nom en lettres et copiant une chaine de chiffres
 // ATTENTION!! Eviter les jokers en début, où gare au temps machine!
 const char *strjokerfind(const char *chaine, const char *joker) {
-  size_t nsteps = 0; /* one budget for the whole scan */
   const char *adr;
 
-  if (chaine != NULL && strlen(chaine) > STRJOKER_MAXLEN)
-    return NULL;
   while(*chaine) {
-    if ((adr = strjoker_bounded(chaine, joker, NULL, NULL,
+    if ((adr = strjoker(chaine, joker, NULL, NULL))) {  // ok trouvé
-                                &nsteps))) { // ok trouvé
       return adr;
     }
-    if (nsteps > STRJOKER_MAXSTEPS) // scan budget spent: no match
-      return NULL;
     chaine++;
   }
   return NULL;

src/htsfilters.h

@@ -41,17 +41,8 @@ Please visit our Website: http://www.httrack.com
 
 int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
                 int *size_flag, int *depth);
-/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
-   scheme); the match latest in the list wins, a "don't know" verdict defers.
-   Returns the merged verdict; the out-params carry the winner's values. */
-int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
-                     const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
-/* strjoker() without the failure memo (exponential worst case); test-only
-   oracle for the memoized matcher. */
-const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
-                            int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);
 #endif
 

src/htsftp.c

@@ -54,9 +54,15 @@ Please visit our Website: http://www.httrack.com
 #endif
 
 // ftp mode passif
+// #if HTS_INET6==0
 #define FTP_PASV 1
+// #else
+// no passive mode for v6
+// #define FTP_PASV 0
+// #endif
 
 #define FTP_DEBUG 0
+//#define FORK_DEBUG 0
 
 #if USE_BEGINTHREAD
 
@@ -122,33 +128,6 @@ void launch_ftp(FTPDownloadStruct * params) {
   return 0; \
   }
 
-/* Bounded split of a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
-void ftp_split_userpass(const char *src, const char *end, char *user,
-                        size_t user_size, char *pass, size_t pass_size) {
-  size_t n = 0;
-
-  assertf(user_size > 0 && pass_size > 0); /* the size-1 math underflows on 0 */
-
-  while (src[n] != '\0' && src[n] != ':') {
-    if (n < user_size - 1)
-      user[n] = src[n];
-    n++;
-  }
-  user[n < user_size ? n : user_size - 1] = '\0';
-  pass[0] = '\0';
-  if (src[n] == ':') { // password follows the colon
-    const size_t base = n + 1;
-    size_t k = 0;
-
-    while (&src[base + k + 1] < end && src[base + k] != '\0') {
-      if (k < pass_size - 1)
-        pass[k] = src[base + k];
-      k++;
-    }
-    pass[k < pass_size ? k : pass_size - 1] = '\0';
-  }
-}
-
 // la véritable fonction une fois lancées les routines thread/fork
 int run_launch_ftp(FTPDownloadStruct * pStruct) {
   lien_back *back = pStruct->pBack;
@@ -194,14 +173,43 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   while(*real_adr == '/')
     real_adr++;                 // sauter /
   if ((adr = jump_identification(real_adr)) != real_adr) {      // user
-    ftp_split_userpass(real_adr, adr, user, sizeof(user), pass, sizeof(pass));
+    int i = -1;
+
+    pass[0] = '\0';
+    do {
+      i++;
+      user[i] = real_adr[i];
+    } while((real_adr[i] != ':') && (real_adr[i]));
+    user[i] = '\0';
+    if (real_adr[i] == ':') {   // pass
+      int j = -1;
+
+      i++;                      // oui on saute aussi le :
+      do {
+        j++;
+        pass[j] = real_adr[i + j];
+      } while(((&real_adr[i + j + 1]) < adr) && (real_adr[i + j]));
+      pass[j] = '\0';
+    }
   }
   // Calculer RETR <nom>
   {
     char *a;
 
+#if 0
+    a = back->url_fil + strlen(back->url_fil) - 1;
+    while((a > back->url_fil) && (*a != '/'))
+      a--;
+    if (*a != '/') {
+      a = NULL;
+    }
+#else
     a = back->url_fil;
+#endif
     if (a != NULL && *a != '\0') {
+#if 0
+      a++;                      // sauter /
+#endif
       ftp_filename = a;
       if (strnotempty(a)) {
         char catbuff[CATBUFF_SIZE];
@@ -225,6 +233,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
       }
     } else {
       strcpybuff(back->r.msg, "Unexpected PORT error");
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
     }
   }
@@ -258,6 +267,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
       snprintf(back->r.msg, sizeof(back->r.msg),
                "Unable to get server's address: %s", error);
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_NON_FATAL;
       _HALT_FTP return 0;
     }
@@ -270,17 +280,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
     if (soc_ctl == INVALID_SOCKET) {
       strcpybuff(back->r.msg, "Unable to create a socket");
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
     }
 
     SOCaddr_initport(server, port);
+    // server.sin_port = htons((unsigned short int) port);
 
     // connexion (bloquante, on est en thread)
     strcpybuff(back->info, "connect");
 
     if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
       strcpybuff(back->r.msg, "Unable to connect to the server");
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
 #ifdef _WIN32
@@ -322,22 +335,64 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               // ok
             } else {
               strcpybuff(back->r.msg, "TYPE I error");
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
+#if 0
+            // --CWD--
+            char *a;
+
+            a = back->url_fil + strlen(back->url_fil) - 1;
+            while((a > back->url_fil) && (*a != '/'))
+              a--;
+            if (*a == '/') {    // ok repéré
+              char BIGSTK target[1024];
+
+              target[0] = '\0';
+              strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
+              if (strnotempty(target) == 0)
+                strcatbuff(target, "/");
+              strcpybuff(back->info, "cwd");
+              snprintf(line, sizeof(line), "CWD %s", target);
+              send_line(soc_ctl, line);
+              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+              _CHECK_HALT_FTP;
+              if (line[0] == '2') {
+                send_line(soc_ctl, "TYPE I");
+                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+                _CHECK_HALT_FTP;
+                if (line[0] == '2') {
+                  // ok..
+                } else {
+                  strcpybuff(back->r.msg, "TYPE I error");
+                  // back->status=STATUS_FTP_READY;    // fini
+                  back->r.statuscode = STATUSCODE_INVALID;
+                }
+              } else {
+                snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
+                // back->status=STATUS_FTP_READY;    // fini
+                back->r.statuscode = STATUSCODE_INVALID;
+              }                 // sinon on est prêts
+            } else {
+              strcpybuff(back->r.msg, "Unexpected ftp error");
+              // back->status=STATUS_FTP_READY;    // fini
+              back->r.statuscode = STATUSCODE_INVALID;
+            }
+#endif
 
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
+            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
-                     linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
+          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
-                   linejmp(line));
+          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
       } else {
-        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
+        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
-                 linejmp(line));
+        // back->status=STATUS_FTP_READY;    // fini
         back->r.statuscode = STATUSCODE_INVALID;
       }
 
@@ -404,8 +459,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
             // -- fin analyse de l'adresse IP et du port --
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
-                     linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
@@ -436,13 +491,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 }
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
+              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
-                       linejmp(line));
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
-                     linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         }
@@ -548,8 +603,9 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   deletesoc(soc_dat);
                   soc_dat = INVALID_SOCKET;
                   //
-                  snprintf(back->r.msg, sizeof(back->r.msg),
+                  snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
-                           "RETR command error: %s", linejmp(line));
+                          linejmp(line));
+                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = STATUSCODE_INVALID;
                 }               // sinon on est prêts
               } else {
@@ -560,20 +616,23 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 soc_dat = INVALID_SOCKET;
                 //
                 strcpybuff(back->r.msg, "Unable to connect");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }                 // sinon on est prêts
             } else {
               strcpybuff(back->r.msg, "Unable to create a socket");
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }                   // sinon on est prêts
           } else {
             snprintf(back->r.msg, sizeof(back->r.msg),
                      "Unable to resolve IP %s: %s", adr_ip, error);
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
+          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
-                   linejmp(line));
+          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }                       // sinon on est prêts
 #else
@@ -594,16 +653,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               //T_SOC soc_dat;
               if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
                 strcpybuff(back->r.msg, "Unable to accept connection");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg),
+              snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
-                       "RETR command error: %s", linejmp(line));
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
+            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
-                     linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -613,6 +673,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 #endif
         } else {
           strcpybuff(back->r.msg, "Unable to listen to a port");
+          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
 #endif
@@ -636,18 +697,21 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             int len = 1;
             int read_len = 1024;
 
+            //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
+
             while((len > 0) && (!stop_ftp(back))) {
               // attendre les données
               len = 1;          // pas d'erreur pour le moment
               switch (wait_socket_receive(soc_dat, timeout)) {
               case -1:
                 strcpybuff(back->r.msg, "FTP read error");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
               case 0:
-                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
+                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
-                         timeout);
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
@@ -669,14 +733,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                          }
                        */
                       strcpybuff(back->r.msg, "Write error");
+                      // back->status=STATUS_FTP_READY;    // fini
                       back->r.statuscode = STATUSCODE_INVALID;
                       len = 0;  // error
                     }
                   } else {
                     strcpybuff(back->r.msg, "Unexpected write error");
+                    // back->status=STATUS_FTP_READY;    // fini
                     back->r.statuscode = STATUSCODE_INVALID;
                   }
                 } else {        // Erreur ou terminé
+                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = 0;
                   if (back->r.totalsize > 0
                       && back->r.size != back->r.totalsize) {
@@ -685,6 +752,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   }
                 }
                 read_len = 1024;
+                //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
               }
             }
             if (back->r.fp) {
@@ -693,6 +761,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
           } else {
             strcpybuff(back->r.msg, "Unable to write file");
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -708,14 +777,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               get_ftp_line(soc_ctl, line, sizeof(line), timeout);
               if (line[0] == '2') {     // OK
                 strcpybuff(back->r.msg, "OK");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = HTTP_OK;
               } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
+                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
-                         linejmp(line));
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
               strcpybuff(back->r.msg, "FTP read error");
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           }
@@ -741,6 +812,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     back->r.statuscode = HTTP_OK;
     strcpybuff(back->r.msg, "OK");
   }
+  // back->status=STATUS_FTP_READY;    // fini
   return 0;
 }
 
@@ -907,11 +979,13 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
       break;
     }
 
+    //HTS_TOTAL_RECV_CHECK(dummy);     // Diminuer au besoin si trop de données reçues
     switch (recv(soc, &b, 1, 0)) {
+      //case 0: break;    // pas encore --> erreur (on attend)!
     case 1:
       HTS_STAT.HTS_TOTAL_RECV += 1;     // compter flux entrant
-      if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
+      if ((b != 10) && (b != 13))
-        data[i++] = b; // truncate hostile over-long reply lines
+        data[i++] = b;
       break;
     default:
       if (ptrline)
@@ -1034,6 +1108,7 @@ int wait_socket_receive(T_SOC soc, int timeout) {
 int stop_ftp(lien_back * back) {
   if (back->stop_ftp) {
     strcpybuff(back->r.msg, "Cancelled by User");
+    // back->status=STATUS_FTP_READY;    // fini
     back->r.statuscode = STATUSCODE_INVALID;
     return 1;
   }

src/htsftp.h

@@ -70,11 +70,6 @@ int back_launch_ftp(FTPDownloadStruct * params);
 int run_launch_ftp(FTPDownloadStruct * params);
 int send_line(T_SOC soc, const char *data);
 int get_ftp_line(T_SOC soc, char *line, size_t line_size, int timeout);
-/* Split a "user[:pass]@" prefix (end = jump_identification result) into
-   bounded, NUL-terminated user/pass buffers, truncating to fit.
-   Both sizes must be nonzero. */
-void ftp_split_userpass(const char *src, const char *end, char *user,
-                        size_t user_size, char *pass, size_t pass_size);
 T_SOC get_datasocket(char *to_send, size_t to_send_size);
 int stop_ftp(lien_back * back);
 char *linejmp(char *line);

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-12"
+#define HTTRACK_VERSION "3.49-9"
-#define HTTRACK_VERSIONID "3.49.12"
+#define HTTRACK_VERSIONID "3.49.9"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 
@@ -56,10 +56,6 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
-// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
-#include <stdint.h>
-#include <inttypes.h>
-
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -115,6 +111,29 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
+#ifndef HTS_LONGLONG
+#ifdef SIZEOF_LONG_LONG
+#if SIZEOF_LONG_LONG == 8
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+#ifndef HTS_LONGLONG
+#ifdef __sun
+#define HTS_LONGLONG 0
+#endif
+#ifdef __osf__
+#define HTS_LONGLONG 0
+#endif
+#ifdef __linux
+#define HTS_LONGLONG 1
+#endif
+#ifdef _WIN32
+#define HTS_LONGLONG 1
+#endif
+#endif
+#endif
+
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -210,10 +229,6 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DEFAULT_FOOTER                                                     \
   "<!-- Mirrored from %s%s by HTTrack Website Copier/" HTTRACK_AFF_VERSION     \
   " " HTTRACK_AFF_AUTHORS ", %s -->"
-/* Honest crawler User-Agent; no fake OS/browser to go stale. */
-#define HTS_DEFAULT_USER_AGENT                                                 \
-  "Mozilla/5.0 (compatible; HTTrack/" HTTRACK_AFF_VERSION                      \
-  "; +https://www.httrack.com/)"
 #define HTTRACK_WEB "http://www.httrack.com"
 #define HTS_UPDATE_WEBSITE                                                     \
   "http://www.httrack.com/"                                                    \
@@ -307,11 +322,59 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
+#ifndef HTS_LONGLONG
-typedef int64_t LLint;
+#ifdef HTS_NO_64_BIT
-typedef int64_t TStamp;
+#define HTS_LONGLONG 0
-/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
+#else
-#define LLintP "%" PRId64
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+/* Wide integer types, chosen per platform.
+   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
+           plain int where 64-bit is unavailable).
+   TStamp: timestamp/duration in the same width (a double in the no-64-bit
+           fallback).
+   LLintP: the printf conversion for an LLint. */
+#if HTS_LONGLONG
+#ifdef LLINT_FORMAT
+typedef LLINT_TYPE LLint;
+
+typedef LLINT_TYPE TStamp;
+
+#define LLintP LLINT_FORMAT
+#else
+
+#ifdef _WIN32
+typedef __int64 LLint;
+
+typedef __int64 TStamp;
+
+#define LLintP "%I64d"
+#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
+       defined(__64BIT__))
+
+typedef long int LLint;
+
+typedef long int TStamp;
+
+#define LLintP "%ld"
+#else
+typedef long long int LLint;
+
+typedef long long int TStamp;
+
+#define LLintP "%lld"
+#endif
+
+#endif /* HTS_LONGLONG */
+
+#else
+typedef int LLint;
+
+#define LLintP "%d"
+typedef double TStamp;
+#endif
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain
@@ -355,10 +418,9 @@ typedef int T_SOC;
 #endif
 
 /* Permission bits for created folders and files (mkdir and chmod).
-   PROTECT_FOLDER/FILE are owner-only. With HTS_ACCESS set (the default) the
+   PROTECT_FOLDER is owner-only. With HTS_ACCESS set (the default) the ACCESS_
-   ACCESS_ modes also grant group/other read; otherwise they stay owner-only. */
+   modes also grant group/other read; otherwise they stay owner-only. */
 #define HTS_PROTECT_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
-#define HTS_PROTECT_FILE (S_IRUSR | S_IWUSR)
 
 #if HTS_ACCESS
 #define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)

src/htshelp.c

@@ -152,6 +152,16 @@ void help_wizard(httrackp * opt) {
 #define str (buffers->str)
 #define argv (buffers->argv)
 
+  //char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
+  //char *mainpath = (char *) malloct(256);
+  //char *projname = (char *) malloct(256);
+  //char *stropt = (char *) malloct(2048);        // options
+  //char *stropt2 = (char *) malloct(2048);       // options longues
+  //char *strwild = (char *) malloct(2048);       // wildcards
+  //char *cmd = (char *) malloct(4096);
+  //char *str = (char *) malloct(256);
+  //char **argv = (char **) malloct(256 * sizeof(char *));
+
   //
   char *a;
 
@@ -217,6 +227,8 @@ void help_wizard(httrackp * opt) {
   strcatbuff(stropt2, str);
   strcatbuff(stropt2, projname);
   strcatbuff(stropt2, "\" ");
+  // Créer si ce n'est fait un index.html 1er niveau
+  make_empty_index(str);
   //
   printf("\n");
   printf("Enter URLs (separated by commas or blank spaces) :");
@@ -337,6 +349,8 @@ void help_wizard(httrackp * opt) {
       }
       hts_main(argc, argv);
     }
+    //} else {
+    //  help("httrack",1);
   }
 
   /* Free buffers */
@@ -441,6 +455,23 @@ void help_catchurl(const char *dest_path) {
     printf("Unable to create a temporary proxy (no remaining port)\n");
 }
 
+// Créer un index.html vide
+void make_empty_index(const char *str) {
+#if 0
+  if (!fexist(fconcat(str, "index.html"))) {
+    FILE *fp = fopen(fconcat(str, "index.html"), "wb");
+
+    if (fp) {
+      fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
+      fprintf(fp,
+              "<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
+              CRLF);
+      fclose(fp);
+    }
+  }
+#endif
+}
+
 // mini-aide  (h: help)
 //           y
 void help(const char *app, int more) {
@@ -490,7 +521,6 @@ void help(const char *app, int more) {
   infomsg("  EN maximum mirror time in seconds (60=1 minute, 3600=1 hour)");
   infomsg("  AN maximum transfer rate in bytes/seconds (1000=1KB/s max)");
   infomsg(" %cN maximum number of connections/seconds (*%c10)");
-  infomsg(" %G  random pause of MIN[:MAX] seconds between files (e.g. %G5:10)");
   infomsg
     ("  GN pause transfer if N bytes reached, and wait until lock file is deleted");
   infomsg("");
@@ -542,12 +572,10 @@ void help(const char *app, int more) {
   infomsg("");
   infomsg("Spider options:");
   infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
-  infomsg(" %K  load extra cookies from a Netscape cookies.txt");
-  infomsg(" %Y  explain which filter rule accepts or rejects a URL, then exit");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
-  infomsg("  j *parse scripts (j0 don't parse, bitmask: |1 parse default, |4 "
+  infomsg
-          "don't parse .js |8 don't be aggressive)");
+    ("  j *parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive)");
   infomsg
     ("  sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules))");
   infomsg
@@ -571,7 +599,8 @@ void help(const char *app, int more) {
     ("     can also be used to force a specific file type: --assume foo.cgi=text/html");
   infomsg
     (" @iN internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only)");
-  infomsg(" %w  disable a specific external mime module (-%w httrack-plugin)");
+  infomsg
+    (" %w  disable a specific external mime module (-%w htsswf -%w htsjava)");
   infomsg("");
   infomsg("Browser ID:");
   infomsg
@@ -777,4 +806,7 @@ void help(const char *app, int more) {
   infomsg("[compiled: " HTS_PLATFORM_NAME "]");
 #endif
   infomsg(NULL);
+
+//  infomsg("  R  *relative links (e.g ../link)\n");
+//  infomsg("  A   absolute links (e.g /www.adr/link)\n");
 }

src/htshelp.h

@@ -45,6 +45,7 @@ typedef struct httrackp httrackp;
 
 void infomsg(const char *msg);
 void help(const char *app, int more);
+void make_empty_index(const char *str);
 void help_wizard(httrackp * opt);
 int help_query(const char *list, int def);
 void help_catchurl(const char *dest_path);

src/htsindex.c

@@ -142,6 +142,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
   char keyword[KEYW_LEN + 32];
   int i = 0;
 
+  //
+  //int WordIndexSize = 1024;
   coucal WordIndexHash = NULL;
   FILE *tmpfp = NULL;
 
@@ -176,6 +178,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
              || (strfield2(mime, "text/css"))
     ) {
     inscript = 1;
+    //} else if (strfield2(mime, "text/vnd.wap.wml")) {   // humm won't work in many cases
+    //  inscript=0;
   } else
     return 0;
 
@@ -271,6 +275,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
 
   // Process indexing for this page
   {
+    //FILE* fp=NULL;
+    //fp=fopen(concat(indexpath,"index.txt"),"ab");
     if (fp_tmpproject) {
       while(!feof(tmpfp)) {
         char line[KEYW_LEN + 32];
@@ -280,6 +286,7 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
           intptr_t e = 0;
 
           if (coucal_read(WordIndexHash, line, &e)) {
+            //if (e) {
             char BIGSTK savelst[HTS_URLMAXSIZE * 2];
 
             e++;                /* 0 means "once" */
@@ -293,9 +300,11 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
             fprintf(fp_tmpproject, "%s %d %s\n", line,
                     (int) (KEYW_SORT_MAXCOUNT - e), savelst);
             hts_primindex_size++;
+            //}
           }
         }
       }
+      //fclose(fp);
     }
   }
 
@@ -321,6 +330,7 @@ void index_finish(const char *indexpath, int mode) {
   off_t size = fpsize(fp_tmpproject);
 
   if (size > 0) {
+    //FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
     if (fp_tmpproject) {
       tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
       if (tab) {
@@ -388,6 +398,8 @@ void index_finish(const char *indexpath, int mode) {
                       if (total_hit) {
                         if (mode == 1)  // TEXT
                           fprintf(fp, "\t=%d\r\n", total_hit);
+                        //else                // HTML
+                        //  fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
                         if ((((total_hit * 1000) / hts_primindex_words) >=
                              KEYW_USELESS1K)
                             || (((total_line * 1000) / index) >=
@@ -404,6 +416,8 @@ void index_finish(const char *indexpath, int mode) {
                           if (mode == 1)        // TEXT
                             fprintf(fp, "\t(%d)\r\n",
                                     ((total_hit * 1000) / hts_primindex_words));
+                          //else                // HTML
+                          //  fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
                         }
                       }
                       if (mode == 1)    // TEXT

src/htsjava.c

@@ -0,0 +1,516 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: Java classes parser                                    */
+/* Author: Yann Philippot                                       */
+/* ------------------------------------------------------------ */
+
+/* Version: Oct/2000 */
+/* Fixed: problems with class structure (10/2000) */
+
+// htsjava.c - Parseur de classes java
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#if ( defined(_WIN32) ||defined(HAVE_SYS_TYPES_H) )
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Standard httrack module includes */
+#include "httrack-library.h"
+#include "htsopt.h"
+#include "htsdefines.h"
+
+/* Module structures */
+#include "htsmodules.h"
+
+/* We link to libhttrack, we can use its functions */
+#include "httrack-library.h"
+
+/* This file */
+#include "htsjava.h"
+
+static int reverse_endian(void) {
+  int endian = 1;
+
+  return (*((char *) &endian) == 1);
+}
+
+/* big/little endian swap */
+#define hts_swap16(A) ( (((A) & 0xFF)<<8) | (((A) & 0xFF00)>>8) )
+#define hts_swap32(A) ( (( (hts_swap16(A)) & 0xFFFF)<<16) | (( (hts_swap16(A>>16)) & 0xFFFF)) )
+
+/* Static definitions */
+static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp, RESP_STRUCT,
+                             int *);
+static unsigned short int readshort(FILE * fp);
+static int tris(httrackp * opt, char *);
+static char *printname(char[1024], char[1024]);
+
+// ** HTS_xx sinon pas pris par VC++
+#define HTS_CLASS  7
+#define HTS_FIELDREF  9
+#define HTS_METHODREF  10
+#define HTS_STRING  8
+#define HTS_INTEGER  3
+#define HTS_FLOAT  4
+#define HTS_LONG  5
+#define HTS_DOUBLE  6
+#define HTS_INTERFACE  11
+#define HTS_NAMEANDTYPE  12
+#define HTS_ASCIZ  1
+#define HTS_UNICODE  2
+
+#define JAVADEBUG 0
+
+static const char *libName = "htsjava";
+
+#ifdef _WIN32
+#define strcasecmp(a,b) stricmp(a,b)
+#define strncasecmp(a,b,n) strnicmp(a,b,n)
+#endif
+
+static int detect_mime(htsmoduleStruct * str) {
+  const char *savename = str->filename;
+
+  if (savename) {
+    int len = (int) strlen(savename);
+
+    if (len > 6 && strcasecmp(savename + len - 6, ".class") == 0) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+static int hts_detect_java(t_hts_callbackarg * carg, httrackp * opt,
+                           htsmoduleStruct * str) {
+  /* Call parent functions if multiple callbacks are chained. */
+  if (CALLBACKARG_PREV_FUN(carg, detect) != NULL) {
+    if (CALLBACKARG_PREV_FUN(carg, detect)
+        (CALLBACKARG_PREV_CARG(carg), opt, str)) {
+      return 1;                 /* Found before us, let them have the priority */
+    }
+  }
+
+  /* Check MIME */
+  if (detect_mime(str)) {
+    str->wrapper_name = libName;        /* Our ID */
+    return 1;                   /* Known format, we take it */
+  }
+
+  return 0;                     /* Unknown format */
+}
+
+static off_t fsize(const char *s) {
+  STRUCT_STAT st;
+
+  if (STAT(s, &st) == 0 && S_ISREG(st.st_mode)) {
+    return st.st_size;
+  } else {
+    return -1;
+  }
+}
+
+static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
+                          htsmoduleStruct * str) {
+  /* The wrapper_name memebr has changed: not for us anymore */
+  if (str->wrapper_name == NULL || strcmp(str->wrapper_name, libName) != 0) {
+    /* Call parent functions if multiple callbacks are chained. */
+    if (CALLBACKARG_PREV_FUN(carg, parse) != NULL) {
+      return CALLBACKARG_PREV_FUN(carg, parse) (CALLBACKARG_PREV_CARG(carg),
+                                                opt, str);
+    }
+    strcpy(str->err_msg,
+           "unexpected error: bad wrapper_name and no previous wrapper");
+    return 0;                   /* Unexpected error */
+  } else {
+    if (detect_mime(str)) {
+
+      /* (Legacy code) */
+      char catbuff[CATBUFF_SIZE];
+      FILE *fpout;
+      JAVA_HEADER header;
+      RESP_STRUCT *tab;
+      const char *file = str->filename;
+
+      str->relativeToHtmlLink = 1;
+
+#if JAVADEBUG
+      printf("fopen\n");
+#endif
+      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
+        //fprintf(stderr, "Cannot open input file.\n");
+        sprintf(str->err_msg, "Unable to open file %s", file);
+        return 0;               // une erreur..
+      }
+#if JAVADEBUG
+      printf("fread\n");
+#endif
+      //if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) {   // pas complet..
+      if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
+        fclose(fpout);
+        sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
+                (LLint) fsize(file));
+        return 0;
+      }
+#if JAVADEBUG
+      printf("header\n");
+#endif
+      // tester en tête
+      if (reverse_endian()) {
+        header.magic = hts_swap32(header.magic);
+        header.count = hts_swap16(header.count);
+      }
+      if (header.magic != 0xCAFEBABE) {
+        sprintf(str->err_msg, "non java file");
+        if (fpout) {
+          fclose(fpout);
+          fpout = NULL;
+        }
+        return 0;
+      }
+
+      tab = (RESP_STRUCT *) calloc(header.count, sizeof(RESP_STRUCT));
+      if (!tab) {
+        sprintf(str->err_msg, "Unable to alloc %d bytes",
+                (int) sizeof(RESP_STRUCT));
+        if (fpout) {
+          fclose(fpout);
+          fpout = NULL;
+        }
+        return 0;               // erreur..
+      }
+#if JAVADEBUG
+      printf("calchead\n");
+#endif
+      {
+        int i;
+
+        for(i = 1; i < header.count; i++) {
+          int err = 0;          // ++    
+
+          tab[i] = readtable(str, fpout, tab[i], &err);
+          if (!err) {
+            if ((tab[i].type == HTS_LONG) || (tab[i].type == HTS_DOUBLE))
+              i++;              //2 element si double ou float
+          } else {              // ++ une erreur est survenue!
+            if (strnotempty(str->err_msg) == 0)
+              strcpy(str->err_msg, "Internal readtable error");
+            free(tab);
+            if (fpout) {
+              fclose(fpout);
+              fpout = NULL;
+            }
+            return 0;
+          }
+        }
+
+      }
+
+#if JAVADEBUG
+      printf("addfiles\n");
+#endif
+      {
+        //unsigned int acess;
+        unsigned int Class;
+        unsigned int SClass;
+        int i;
+
+        //acess = readshort(fpout);
+        Class = readshort(fpout);
+        SClass = readshort(fpout);
+
+        for(i = 1; i < header.count; i++) {
+
+          if (tab[i].type == HTS_CLASS) {
+
+            if ((tab[i].index1 < header.count) && (tab[i].index1 >= 0)) {
+
+              if ((tab[i].index1 != SClass) && (tab[i].index1 != Class)
+                  && (tab[tab[i].index1].name[0] != '[')) {
+
+                if (!strstr(tab[tab[i].index1].name, "java/")) {
+                  char BIGSTK tempo[1024];
+
+                  tempo[0] = '\0';
+
+                  snprintf(tempo, sizeof(tempo), "%s.class", tab[tab[i].index1].name);
+#if JAVADEBUG
+                  printf("add %s\n", tempo);
+#endif
+                  if (tab[tab[i].index1].file_position >= 0)
+                    str->addLink(str, tempo);   /* tab[tab[i].index1].file_position */
+                }
+
+              }
+            } else {
+              i = header.count; // exit 
+            }
+          }
+
+        }
+      }
+
+#if JAVADEBUG
+      printf("end\n");
+#endif
+      free(tab);
+      if (fpout) {
+        fclose(fpout);
+        fpout = NULL;
+      }
+      return 1;
+
+    } else {
+      strcpy(str->err_msg, "bad MIME type");
+    }
+  }
+  return 0;                     /* Error */
+}
+
+/*
+module entry point 
+*/
+EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
+EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
+  /* Plug callback functions */
+  CHAIN_FUNCTION(opt, detect, hts_detect_java, NULL);
+  CHAIN_FUNCTION(opt, parse, hts_parse_java, NULL);
+
+  return 1;                     /* success */
+}
+
+// error: !=0 si erreur fatale
+static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
+                             RESP_STRUCT trans, int *error) {
+  char rname[1024];
+  unsigned short int length;
+  int j;
+
+  *error = 0;                   // pas d'erreur
+  trans.file_position = -1;
+  trans.type = (int) (unsigned char) fgetc(fp);
+  switch (trans.type) {
+  case HTS_CLASS:
+    strcpy(trans.name, "Class");
+    trans.index1 = readshort(fp);
+    break;
+
+  case HTS_FIELDREF:
+    strcpy(trans.name, "Field Reference");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+
+  case HTS_METHODREF:
+    strcpy(trans.name, "Method Reference");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+
+  case HTS_INTERFACE:
+    strcpy(trans.name, "Interface Method Reference");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+  case HTS_NAMEANDTYPE:
+    strcpy(trans.name, "Name and Type");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+
+  case HTS_STRING:             // CONSTANT_String
+    strcpy(trans.name, "String");
+    trans.index1 = readshort(fp);
+    break;
+
+  case HTS_INTEGER:
+    strcpy(trans.name, "Integer");
+    for(j = 0; j < 4; j++)
+      fgetc(fp);
+    break;
+
+  case HTS_FLOAT:
+    strcpy(trans.name, "Float");
+    for(j = 0; j < 4; j++)
+      fgetc(fp);
+    break;
+
+  case HTS_LONG:
+    strcpy(trans.name, "Long");
+    for(j = 0; j < 8; j++)
+      fgetc(fp);
+    break;
+  case HTS_DOUBLE:
+    strcpy(trans.name, "Double");
+    for(j = 0; j < 8; j++)
+      fgetc(fp);
+    break;
+
+  case HTS_ASCIZ:
+  case HTS_UNICODE:
+
+    if (trans.type == HTS_ASCIZ)
+      strcpy(trans.name, "HTS_ASCIZ");
+    else
+      strcpy(trans.name, "HTS_UNICODE");
+
+    {
+      char BIGSTK buffer[1024];
+      char *p;
+
+      p = &buffer[0];
+
+      //fflush(fp); 
+      trans.file_position = ftell(fp);
+      length = readshort(fp);
+      if (length < HTS_URLMAXSIZE) {
+        // while ((length > 0) && (length<500)) {
+        while(length > 0) {
+          *p++ = fgetc(fp);
+
+          length--;
+        }
+        *p = '\0';
+
+        //#if JDEBUG
+        //      if(tris(buffer)==1) printf("%s\n ",buffer);
+        //      if(tris(buffer)==2)  printf("%s\n ",printname(buffer));
+        //#endif
+        if (tris(str->opt, buffer) == 1)
+          str->addLink(str, buffer);    /* trans.file_position */
+        else if (tris(str->opt, buffer) == 2)
+          str->addLink(str, printname(rname, buffer));
+
+        strcpy(trans.name, buffer);
+      } else {                  // gros pb
+        while((length > 0) && (!feof(fp))) {
+          fgetc(fp);
+          length--;
+        }
+        if (!feof(fp)) {
+          trans.type = -1;
+        } else {
+          sprintf(str->err_msg, "Internal structure error (ASCII)");
+          *error = 1;
+        }
+        return (trans);
+      }
+    }
+    break;
+  default:
+    // printf("Type inconnue\n");
+    // on arrête tout 
+    sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
+    *error = 1;
+    return (trans);
+    break;
+  }
+  return (trans);
+}
+
+static unsigned short int readshort(FILE * fp) {
+  unsigned short int valint;
+
+  if (fread(&valint, sizeof(valint), 1, fp) == 1) {
+    if (reverse_endian())
+      return hts_swap16(valint);
+    else
+      return valint;
+  } else {
+    return 0;
+  }
+
+}
+
+static int tris(httrackp * opt, char *buffer) {
+  char catbuff[CATBUFF_SIZE];
+
+  //
+  // Java
+  if ((buffer[0] == '[') && buffer[1] == 'L' && (!strstr(buffer, "java/")))
+    return 2;
+  if (strstr(buffer, ".gif") || strstr(buffer, ".jpg")
+      || strstr(buffer, ".jpeg") || strstr(buffer, ".au"))
+    return 1;
+  // Ajouts R.X: test type
+  // Autres fichiers
+  {
+    char type[256];
+
+    if (get_httptype_sized(opt, type, sizeof(type), buffer,
+                           0)) // recognized type
+      return 1;
+    // ajout RX 05/2001
+    else if (is_dyntype(get_ext(catbuff, sizeof(catbuff), buffer)))      // asp,cgi...
+      return 1;
+  }
+  return 0;
+}
+
+static char *printname(char rname[1024], char name[1024]) {
+  char *p;
+  char *p1;
+  size_t j;
+
+  rname[0] = '\0';
+  //
+
+  p = &name[0];
+
+  if (*p != '[') {
+    return rname;  // ""
+  }
+  p += 2;
+  //rname=(char*)calloct(strlen(name)+8,sizeof(char));
+  p1 = rname;
+  for(j = 0; name[j] != '\0'; j++, p++) {
+    if (*p == '/')
+      *p1 = '.';
+    if (*p == ';') {
+      *p1 = '\0';
+      strcat(rname, ".class");
+      return rname;
+    } else
+      *p1 = *p;
+    p1++;
+  }
+  p1 -= 3;
+  *p1 = '\0';
+  return rname;
+
+}

src/htsjava.h

@@ -25,27 +25,42 @@ would dishonor our work and waste the many hours we have spent on it.
 Please visit our Website: http://www.httrack.com
 */
 
-/* Shared helpers for the libFuzzer harnesses. */
+/* ------------------------------------------------------------ */
-#ifndef FUZZ_H
+/* File: Java classes parser .h                                 */
-#define FUZZ_H
+/* Author: Yann Philippot                                       */
+/* ------------------------------------------------------------ */
 
-#define HTS_INTERNAL_BYTECODE
+#ifndef HTSJAVA_DEFH
+#define HTSJAVA_DEFH
 
-#include <stddef.h>
+#ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER
-#include <stdint.h>
+#define HTS_DEF_FWSTRUCT_JAVA_HEADER
-#include <string.h>
+typedef struct JAVA_HEADER JAVA_HEADER;
+#endif
+struct JAVA_HEADER {
+  unsigned long int magic;
+  unsigned short int minor;
+  unsigned short int major;
+  unsigned short int count;
+};
 
-#include "htsbase.h"
+#ifndef HTS_DEF_FWSTRUCT_RESP_STRUCT
+#define HTS_DEF_FWSTRUCT_RESP_STRUCT
+typedef struct RESP_STRUCT RESP_STRUCT;
+#endif
+struct RESP_STRUCT {
+  int file_position;
+  //
+  unsigned int index1;
+  unsigned int type;
+  char name[1024];
+};
 
-/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
+/* Library internal definictions */
-static char *fuzz_strdup(const uint8_t *data, size_t size) {
+#ifdef HTS_INTERNAL_BYTECODE
-  char *s = malloct(size + 1);
 
-  memcpy(s, data, size);
+EXTERNAL_FUNCTION int hts_plug_java(httrackp * opt, const char *argv);
-  s[size] = '\0';
+
-  return s;
+#endif
-}
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
 
 #endif

src/htsjava.vcproj

@@ -0,0 +1,373 @@
+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="9,00"
+	Name="htsjava"
+	ProjectGUID="{2F353408-2C6C-4D31-9AC0-08BDC064DF64}"
+	RootNamespace="htsjava"
+	Keyword="Win32Proj"
+	TargetFrameworkVersion="131072"
+	>
+	<Platforms>
+		<Platform
+			Name="Win32"
+		/>
+		<Platform
+			Name="x64"
+		/>
+	</Platforms>
+	<ToolFiles>
+	</ToolFiles>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="C:\temp\htsjava\Debug"
+			IntermediateDirectory="C:\temp\htsjava\Debug"
+			ConfigurationType="2"
+			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
+				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="1"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="4"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				OutputFile="C:\temp\httrack\htsjava.dll"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				ProgramDatabaseFile="$(OutDir)/htsjava.pdb"
+				SubSystem="2"
+				RandomizedBaseAddress="1"
+				DataExecutionPrevention="0"
+				ImportLibrary="$(OutDir)/htsjava.lib"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Debug|x64"
+			OutputDirectory="$(SolutionDir)$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="2"
+			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
+				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="1"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				OutputFile="C:\temp\httrack64\htsjava.dll"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				ProgramDatabaseFile="$(OutDir)/htsjava.pdb"
+				SubSystem="2"
+				RandomizedBaseAddress="1"
+				DataExecutionPrevention="0"
+				ImportLibrary="$(OutDir)/htsjava.lib"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|Win32"
+			OutputDirectory="C:\temp\htsjava\Release"
+			IntermediateDirectory="C:\temp\htsjava\Release"
+			ConfigurationType="2"
+			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
+				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
+				RuntimeLibrary="0"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="0"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				OutputFile="O:\HTTrack\httrack\htsjava.dll"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				ProgramDatabaseFile="O:\HTTrack\httrack\htsjava.pdb"
+				SubSystem="2"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				RandomizedBaseAddress="1"
+				DataExecutionPrevention="0"
+				ImportLibrary="$(OutDir)/htsjava.lib"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+				CommandLine="&quot;C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\signtool.exe&quot; sign /f &quot;Z:\Mes Documents\certificats\httrack-com\httrack-signing-certificate.pfx&quot; /t http://timestamp.verisign.com/scripts/timstamp.dll $(TargetPath)&#x0D;&#x0A;"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|x64"
+			OutputDirectory="$(SolutionDir)$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="2"
+			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
+				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
+				RuntimeLibrary="0"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="0"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				OutputFile="O:\HTTrack\httrack\x64\htsjava.dll"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				ProgramDatabaseFile="O:\HTTrack\httrack\x64\htsjava.pdb"
+				SubSystem="2"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				RandomizedBaseAddress="1"
+				DataExecutionPrevention="0"
+				ImportLibrary="$(OutDir)/htsjava.lib"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+				CommandLine="&quot;C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\signtool.exe&quot; sign /f &quot;Z:\Mes Documents\certificats\httrack-com\httrack-signing-certificate.pfx&quot; /t http://timestamp.verisign.com/scripts/timstamp.dll $(TargetPath)&#x0D;&#x0A;"
+			/>
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="Source Files"
+			Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
+			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+			>
+			<File
+				RelativePath="htsjava.c"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Header Files"
+			Filter="h;hpp;hxx;hm;inl;inc;xsd"
+			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
+			>
+			<File
+				RelativePath="htsjava.h"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Resource Files"
+			Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
+			UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
+			>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>

src/htslib.c

@@ -563,62 +563,58 @@ const char *hts_mime[][2] = {
   {"", ""}
 };
 
-/* Modern web formats (post-2010), kept in their own table: appending to the
-   legacy hts_mime[] above makes clang-format reflow its whole initializer.
-   Scanned after hts_mime[], so it never shadows a legacy mapping. */
-static const char *hts_mime_modern[][2] = {
-    {"image/webp", "webp"},
-    {"image/avif", "avif"},
-    {"image/heic", "heic"},
-    {"font/woff", "woff"},
-    {"font/woff2", "woff2"},
-    {"font/ttf", "ttf"},
-    {"font/otf", "otf"},
-    {"application/json", "json"},
-    {"application/ld+json", "jsonld"},
-    {"application/manifest+json", "webmanifest"},
-    {"application/wasm", "wasm"},
-    {"text/javascript", "js"},
-    {"text/javascript", "mjs"},
-    {"text/markdown", "md"},
-    {"video/mp4", "mp4"},
-    {"video/webm", "webm"},
-    {"video/ogg", "ogv"},
-    {"video/mp2t", "ts"},
-    {"audio/mp4", "m4a"},
-    {"audio/aac", "aac"},
-    {"audio/ogg", "oga"},
-    {"audio/opus", "opus"},
-    {"audio/flac", "flac"},
-    {"audio/webm", "weba"},
-    {"application/x-7z-compressed", "7z"},
-    {"application/x-rar-compressed", "rar"},
-    {"application/zstd", "zst"},
-    {"", ""}};
-
 // Reserved (RFC2396)
 #define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
-#define CHAR_RESERVED(c)                                                       \
+#define CHAR_RESERVED(c)  ( CIS(c,';') \
-  (CIS(c, ';') || CIS(c, '/') || CIS(c, '?') || CIS(c, ':') || CIS(c, '@') ||  \
+                         || CIS(c,'/') \
-   CIS(c, '&') || CIS(c, '=') || CIS(c, '+') || CIS(c, '$') || CIS(c, ','))
+                         || CIS(c,'?') \
+                         || CIS(c,':') \
+                         || CIS(c,'@') \
+                         || CIS(c,'&') \
+                         || CIS(c,'=') \
+                         || CIS(c,'+') \
+                         || CIS(c,'$') \
+                         || CIS(c,',') )
+//#define CHAR_RESERVED(c)  ( strchr(";/?:@&=+$,",(unsigned char)(c)) != 0 )
 // Delimiters (RFC2396)
-#define CHAR_DELIM(c)                                                          \
+#define CHAR_DELIM(c)     ( CIS(c,'<') \
-  (CIS(c, '<') || CIS(c, '>') || CIS(c, '#') || CIS(c, '%') || CIS(c, '\"'))
+                         || CIS(c,'>') \
+                         || CIS(c,'#') \
+                         || CIS(c,'%') \
+                         || CIS(c,'\"') )
+//#define CHAR_DELIM(c)     ( strchr("<>#%\"",(unsigned char)(c)) != 0 )
 // Unwise (RFC2396)
-#define CHAR_UNWISE(c)                                                         \
+#define CHAR_UNWISE(c)    ( CIS(c,'{') \
-  (CIS(c, '{') || CIS(c, '}') || CIS(c, '|') || CIS(c, '\\') || CIS(c, '^') || \
+                         || CIS(c,'}') \
-   CIS(c, '[') || CIS(c, ']') || CIS(c, '`'))
+                         || CIS(c,'|') \
+                         || CIS(c,'\\') \
+                         || CIS(c,'^') \
+                         || CIS(c,'[') \
+                         || CIS(c,']') \
+                         || CIS(c,'`') )
+//#define CHAR_UNWISE(c)    ( strchr("{}|\\^[]`",(unsigned char)(c)) != 0 )
 // Special (escape chars) (RFC2396 + >127 )
 #define CHAR_LOW(c)       ( ((unsigned char)(c) <= 31) )
 #define CHAR_HIG(c)       ( ((unsigned char)(c) >= 127) )
 #define CHAR_SPECIAL(c)   ( CHAR_LOW(c) || CHAR_HIG(c) )
 // We try to avoid them and encode them instead
-#define CHAR_XXAVOID(c)                                                        \
+#define CHAR_XXAVOID(c)   ( CIS(c,' ') \
-  (CIS(c, ' ') || CIS(c, '*') || CIS(c, '\'') || CIS(c, '\"') ||               \
+                         || CIS(c,'*') \
-   CIS(c, '&') || CIS(c, '!'))
+                         || CIS(c,'\'') \
-#define CHAR_MARK(c)                                                           \
+                         || CIS(c,'\"') \
-  (CIS(c, '-') || CIS(c, '_') || CIS(c, '.') || CIS(c, '!') || CIS(c, '~') ||  \
+                         || CIS(c,'&') \
-   CIS(c, '*') || CIS(c, '\'') || CIS(c, '(') || CIS(c, ')'))
+                         || CIS(c,'!') )
+//#define CHAR_XXAVOID(c)   ( strchr(" *'\"!",(unsigned char)(c)) != 0 )
+#define CHAR_MARK(c)      ( CIS(c,'-') \
+                         || CIS(c,'_') \
+                         || CIS(c,'.') \
+                         || CIS(c,'!') \
+                         || CIS(c,'~') \
+                         || CIS(c,'*') \
+                         || CIS(c,'\'') \
+                         || CIS(c,'(') \
+                         || CIS(c,')') )
+//#define CHAR_MARK(c)      ( strchr("-_.!~*'()",(unsigned char)(c)) != 0 )
 
 // conversion éventuelle / vers antislash
 #ifdef _WIN32
@@ -813,11 +809,14 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
 // waitconnect: attendre le connect()
 // note: dans retour, on met les params du proxy
 T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
-                  const char *xsend, const char *adr, const char *fil,
+                  const char *xsend, const char *adr, const char *fil, htsblk * retour) {
-                  htsblk *retour) {
+  //htsblk retour;
+  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
   T_SOC soc = INVALID_SOCKET;
   char BIGSTK tempo_fil[HTS_URLMAXSIZE * 2];
 
+  //char *p,*q;
+
   // retour prédéfini: erreur
   if (retour) {
     retour->adr = NULL;
@@ -902,6 +901,7 @@ T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
         strcpybuff(retour->msg, "Unable to open local file");
       else {
         // Note: On passe par un FILE* (plus propre)
+        //soc=open(fil,O_RDONLY,0);    // en lecture seule!
         retour->fp = FOPEN(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), 
           unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil)), "rb");      // ouvrir
         if (retour->fp == NULL)
@@ -978,10 +978,17 @@ T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
 
         } while(strnotempty(rcvd));
 
+        //rcvsize=-1;    // forCER CHARGEMENT INCONNU
+
+        //if (retour)
+        //  retour->totalsize=rcvsize;
+
       } else {                  // si GET, on recevra l'en tête APRES
+        //rcvsize=-1;    // on ne connait pas la taille de l'en-tête
         if (retour)
           retour->totalsize = -1;
       }
+
     }
 
   }
@@ -1059,10 +1066,10 @@ static int append_cookie_header(buff_struct *bstr, t_cookie *cookie,
   return cook;
 }
 
-/* Build the request Cookie line for domain/path into dst (always
+/* Self-test entry for append_cookie_header(): build the request Cookie line
-   NUL-terminated). Returns the number of cookies emitted. */
+   into dst (always NUL-terminated). Returns the number of cookies emitted. */
-int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
+int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
-                       char *dst, size_t dst_size) {
+                                const char *path, char *dst, size_t dst_size) {
   buff_struct bstr = {dst, dst_size, 0};
 
   assertf(dst != NULL && dst_size > 0);
@@ -1078,12 +1085,19 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
   char BIGSTK buffer_head_request[16384];
   buff_struct bstr = { buffer_head_request, sizeof(buffer_head_request), 0 };
 
+  //int use_11=0;     // HTTP 1.1 utilisé
   int direct_url = 0;           // ne pas analyser l'url (exemple: ftp://)
   const char *search_tag = NULL;
 
   // Initialize buffer
   buffer_head_request[0] = '\0';
 
+  // header Date
+  //strcatbuff(buff,"Date: ");
+  //time_gmt_rfc822(buff);    // obtenir l'heure au format rfc822
+  //sendc("\n");
+  //strcatbuff(buff,buff);
+
   // possibilité non documentée: >post: et >postfile:
   // si présence d'un tag >post: alors executer un POST
   // exemple: http://www.example.com/test.cgi?foo>post:posteddata=10&foo=5
@@ -1102,8 +1116,7 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
           char BIGSTK protocol[256], url[HTS_URLMAXSIZE * 2], method[256];
 
           linput(fp, line, 1000);
-          /* widths bound method[256], url[HTS_URLMAXSIZE*2], protocol[256] */
+          if (sscanf(line, "%s %s %s", method, url, protocol) == 3) {
-          if (sscanf(line, "%255s %2047s %255s", method, url, protocol) == 3) {
             size_t ret;
             // selon que l'on a ou pas un proxy
             if (retour->req.proxy.active) {
@@ -1178,8 +1191,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
 
     // protocole
     if (!retour->req.http11) {  // forcer HTTP/1.0
+      //use_11=0;
       print_buffer(&bstr, " HTTP/1.0\x0d\x0a");
     } else {                    // Requète 1.1
+      //use_11=1;
       print_buffer(&bstr, " HTTP/1.1\x0d\x0a");
     }
 
@@ -1278,12 +1293,16 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
 
       // Compression accepted ?
       if (retour->req.http11) {
-        hts_boolean compressible = HTS_FALSE;
 #if HTS_USEZLIB
-        compressible = (!retour->req.range_used && !retour->req.nocompression);
+        if ((!retour->req.range_used)
+            && (!retour->req.nocompression))
+          print_buffer(&bstr, "Accept-Encoding: " "gzip" /* gzip if the preffered encoding */
+                     ", " "identity;q=0.9" H_CRLF);
+        else
+          print_buffer(&bstr, "Accept-Encoding: identity" H_CRLF);       /* no compression */
+#else
+        print_buffer(&bstr, "Accept-Encoding: identity" H_CRLF); /* no compression */
 #endif
-        print_buffer(&bstr, "Accept-Encoding: %s" H_CRLF,
-                     hts_acceptencoding(compressible));
       }
 
       /* Authentification */
@@ -1315,7 +1334,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
           print_buffer(&bstr, "Authorization: Basic %s"H_CRLF, autorisation);
         }
       }
+
     }
+    //strcatbuff(buff,"Accept-Charset: iso-8859-1,*,utf-8\n");
 
     // Custom header(s)
     if (strnotempty(retour->req.headers)) {
@@ -1469,11 +1490,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a += strlen("filename=");
         while(is_space(*a))
           a++;
+        //a=strchr(a,'"');
         if (a) {
           char *c = NULL;
 
+          //a++;      /* jump " */
           while((c = strchr(a, '/')))   /* skip all / (see RFC2616) */
             a = c + 1;
+          //b=strchr(a+1,'"');
           b = a + strlen(a) - 1;
           while(is_space(*b))
             b--;
@@ -1491,6 +1515,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
     while(is_realspace(*(rcvd + p)))
       p++;                      // sauter espaces
     if ((int) strlen(rcvd + p) < 64) {  // pas trop long?
+      //struct tm* tm_time=convert_time_rfc822(rcvd+p);
       strcpybuff(retour->lastmodified, rcvd + p);
     }
   } else if ((p = strfield(rcvd, "Date:")) != 0) {
@@ -1498,6 +1523,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       while(is_realspace(*(rcvd + p)))
         p++;                    // sauter espaces
       if ((int) strlen(rcvd + p) < 64) {        // pas trop long?
+        //struct tm* tm_time=convert_time_rfc822(rcvd+p);
         strcpybuff(retour->lastmodified, rcvd + p);
       }
     }
@@ -1510,11 +1536,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       else                      // erreur.. ignorer
         retour->etag[0] = '\0';
     }
-  } else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
+  }
+  // else if ((p=strfield(rcvd,"Transfer-Encoding: chunked"))!=0) {  // chunk!
+  else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) {   // chunk!
     while(is_realspace(*(rcvd + p)))
       p++;                      // sauter espaces
     if (strfield(rcvd + p, "chunked")) {
       retour->is_chunk = 1;     // chunked
+      //retour->http11=2;     // chunked
 #if HDEBUG
       printf("ok, Transfer-Encoding: détecté\n");
 #endif
@@ -1562,7 +1591,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       }
       // An empty/whitespace Content-Type value yields no token: keep the
       // sentinel default rather than reading an uninitialized tempo.
-      if (sscanf(rcvd + p, "%1099s", tempo) == 1) { // tempo[1100], server data
+      if (sscanf(rcvd + p, "%s", tempo) == 1) {
         if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
           strcpybuff(retour->contenttype, tempo);
         else
@@ -1586,7 +1615,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a = strchr(rcvd + p, '/');
         if (a != NULL) {
           a++;
-          if (sscanf(a, LLintP, &retour->crange) == 1 && retour->crange >= 0) {
+          if (sscanf(a, LLintP, &retour->crange) == 1) {
             retour->crange_start = 0;
             retour->crange_end = retour->crange - 1;
           } else {
@@ -1594,12 +1623,6 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           }
         }
       }
-      // A valid Content-Range has no negative field; reject hostile values so
-      // the crange +/- 1 arithmetic downstream cannot sign-overflow (UB).
-      if (retour->crange_start < 0 || retour->crange_end < 0 ||
-          retour->crange < 0) {
-        retour->crange_start = retour->crange_end = retour->crange = 0;
-      }
     }
   } else if ((p = strfield(rcvd, "Connection:")) != 0) {
     char *a = rcvd + p;
@@ -1663,11 +1686,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         if (a)
           *a = '\0';
       }
-      // Bound to tempo[1100]; no token => leave empty, don't read uninit tempo.
+      sscanf(a, "%s", tempo);
-      if (sscanf(a, "%1099s", tempo) == 1 && strlen(tempo) < 64)
+      if (strlen(tempo) < 64)   // pas trop long!!
         strcpybuff(retour->contentencoding, tempo);
       else
-        retour->contentencoding[0] = '\0';
+        retour->contentencoding[0] = '\0';      // erreur
 #if HTS_USEZLIB
       /* Check known encodings */
       if (retour->contentencoding[0]) {
@@ -1697,8 +1720,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           retour->location[0] = '\0';
       }
     }
-  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) &&
+  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) && (cookie)) {  // ohh un cookie
-             (cookie)) {        // ohh un cookie
     char *a = rcvd + p;         // pointeur
     char domain[256];           // domaine cookie (.netscape.com)
     char path[256];             // chemin (/)
@@ -1708,9 +1730,6 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
 #if DEBUG_COOK
     printf("set-cookie detected\n");
 #endif
-    /* Over-long host overflows the fail-safe domain[] copy (abort); drop it. */
-    if (adr && strlen(jump_identification_const(adr)) >= sizeof(domain))
-      return;
     while(*a) {
       char *token_st, *token_end;
       char *value_st, *value_end;
@@ -1746,7 +1765,9 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           value_st = a;
           while((*a != ';') && (*a))
             a++;                // prochain ;
+          //while( ((*a!='"') || (*(a-1)=='\\')) && (*a)) a++;    // prochain " (et pas \")
           value_end = a;
+          //if (*a==';') {  // finit par un ;
           // vérifier débordements
           if ((((int) (token_end - token_st)) < 200)
               && (((int) (value_end - value_st)) < 8000)
@@ -1897,10 +1918,6 @@ HTSEXT_API const char *infostatuscode_const(int statuscode) {
     return "Requested Range Not Satisfiable";
   case 417:
     return "Expectation Failed";
-  case 429:
-    return "Too Many Requests";
-  case 451:
-    return "Unavailable For Legal Reasons";
   case 500:
     return "Internal Server Error";
   case 501:
@@ -2007,15 +2024,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
 
   if (bufl > 0) {
     if (!r->is_write) {         // stocker en mémoire
-      // In-memory content must fit a 32-bit index (allocs below add 1, reads
-      // use int offsets): reject a hostile Content-Length or endless stream.
-      const LLint inmem_want =
-          (r->totalsize >= 0) ? r->totalsize : (r->size + bufl);
-      if (inmem_want >= INT32_MAX) {
-        r->statuscode = STATUSCODE_INVALID;
-        strcpybuff(r->msg, "In-memory content too large");
-        return READ_ERROR;
-      }
       if (r->totalsize >= 0) {  // totalsize déterminé ET ALLOUE
         if (r->adr == NULL) {
           r->adr = (char *) malloct((size_t) r->totalsize + 1);
@@ -2097,6 +2105,8 @@ LLint http_xfread1(htsblk * r, int bufl) {
             nl = READ_ERROR;
           }
         }
+        //if ((nl < 0) || ((r->totalsize>0) && (r->size >= r->totalsize)))
+        //  nl=-1;  // break
 
         // libérer bloc tempo
         freet(buff);
@@ -2184,6 +2194,9 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
   T_SOC soc;
   htsblk retour;
 
+  //int rcvsize=-1;
+  //char* rcv=NULL;    // adresse de retour
+  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
   TStamp tl;
   int timeout = 30;             // timeout pour un check (arbitraire) // **
 
@@ -2192,8 +2205,11 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
 
   loc[0] = '\0';
   hts_init_htsblk(&retour);
+  //memset(&retour, 0, sizeof(htsblk));    // effacer
   retour.location = loc;        // si non nul, contiendra l'adresse véritable en cas de moved xx
 
+  //soc=http_fopen(adr,fil,&retour,NULL);  // ouvrir, + header
+
   // on ouvre en head, et on traite l'en tête
   soc = http_xfopen(opt, 1, 0, 1, NULL, adr, fil, &retour);     // ouvrir HEAD, + envoi header
 
@@ -2515,10 +2531,6 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
   // effacer adrfil->adr et adrfil->fil
   adrfil->adr[0] = adrfil->fil[0] = '\0';
 
-  // Reject an over-long URL: a root-relative path is copied whole into fil[].
-  if (strlen(url) >= HTS_URLMAXSIZE * 2)
-    return -1;
-
 #if HDEBUG
   printf("protocol: %s\n", url);
 #endif
@@ -2573,16 +2585,12 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
 
     // chemin www... trop long!!
     if ((((int) (q - p))) > HTS_URLMAXSIZE) {
+      //strcpybuff(retour.msg,"Path too long");
       return -1;                // erreur
     }
     // recopier adrfil->adresse www..
     strncatbuff(adrfil->adr, p, ((int) (q - p)));
     // *( adrfil->adr+( ((int) q) - ((int) p) ) )=0;  // faut arrêter la fumette!
-
-    // fil[] holds the path plus a possible leading '/' the top strlen() misses.
-    if (strlen(q) >= sizeof(adrfil->fil) - (q[0] != '/' ? 1 : 0))
-      return -1;
-
     // recopier chemin /pub/..
     if (q[0] != '/')            // page par défaut (/)
       strcatbuff(adrfil->fil, "/");
@@ -2599,7 +2607,7 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
     if (*p == '/' || *p == '\\') {      /* adrfil->file:///.. */
       strcatbuff(adrfil->fil, p);       // fichier local ; adrfil->adr="#"
     } else {
-      if (*p == '\0' || p[1] != ':') {  /* empty path: not a DOS drive letter */
+      if (p[1] != ':') {
         strcatbuff(adrfil->fil, "//");  /* adrfil->file://server/foo */
         strcatbuff(adrfil->fil, p);
       } else {
@@ -2763,6 +2771,7 @@ void deletesoc_r(htsblk * r) {
 #if HTS_USEOPENSSL
   if (r->ssl_con) {
     SSL_shutdown(r->ssl_con);
+    // SSL_CTX_set_quiet_shutdown(r->ssl_con->ctx, 1);
     SSL_free(r->ssl_con);
     r->ssl_con = NULL;
   }
@@ -3062,10 +3071,11 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
 #define ToLLint(a) ((LLint)(a))
 #define ToLLintKiB (ToLLint(1024))
 #define ToLLintMiB (ToLLintKiB*ToLLintKiB)
+#ifdef HTS_LONGLONG
 #define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
 #define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
-#define ToLLintPiB                                                             \
+#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
-  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#endif
 HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
   if (n < ToLLintKiB) {
     sprintf(strc->buff1, "%d", (int) (LLint) n);
@@ -3074,7 +3084,9 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
             (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
     strcpybuff(strc->buff2, "KiB");
-  } else if (n < ToLLintGiB) {
+  }
+#ifdef HTS_LONGLONG
+  else if (n < ToLLintGiB) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
             (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
     strcpybuff(strc->buff2, "MiB");
@@ -3091,6 +3103,13 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
             (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
     strcpybuff(strc->buff2, "PiB");
   }
+#else
+  else {
+    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
+            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
+    strcpybuff(strc->buff2, "MiB");
+  }
+#endif
   strc->buffadr[0] = strc->buff1;
   strc->buffadr[1] = strc->buff2;
   return strc->buffadr;
@@ -3141,6 +3160,7 @@ int finput(T_SOC fd, char *s, int max) {
   int j = 0;
 
   do {
+    //c=fgetc(fp);
     if (read((int) fd, &c, 1) <= 0) {
       c = 0;
     }
@@ -4078,31 +4098,23 @@ DECLARE_APPEND_ESCAPE_VERSION(escape_uri)
 
 #undef DECLARE_APPEND_ESCAPE_VERSION
 
-// In-place escaping: copy dest aside, then escape that copy back into dest.
+// Same as above, but in-place
-typedef size_t (*escape_fn_t)(const char *src, char *dest, size_t size);
-
-static size_t inplace_escape(char *const dest, const size_t size,
-                             escape_fn_t escape) {
-  char buffer[256];
-  const size_t len = strnlen(dest, size);
-  const int in_buffer = len + 1 < sizeof(buffer);
-  char *src = in_buffer ? buffer : malloct(len + 1);
-  size_t ret;
-  assertf(src != NULL);
-  assertf(len < size);
-  memcpy(src, dest, len + 1);
-  ret = escape(src, dest, size);
-  if (!in_buffer) {
-    freet(src);
-  }
-  return ret;
-}
-
-// Thin exported wrappers binding inplace_escape() to each escaper (ABI).
 #undef DECLARE_INPLACE_ESCAPE_VERSION
 #define DECLARE_INPLACE_ESCAPE_VERSION(NAME) \
 HTSEXT_API size_t inplace_ ##NAME(char *const dest, const size_t size) { \
-    return inplace_escape(dest, size, NAME);                                   \
+  char buffer[256]; \
+  const size_t len = strnlen(dest, size); \
+  const int in_buffer = len + 1 < sizeof(buffer); \
+  char *src = in_buffer ? buffer : malloct(len + 1); \
+  size_t ret; \
+  assertf(src != NULL); \
+  assertf(len < size); \
+  memcpy(src, dest, len + 1); \
+  ret = NAME(src, dest, size); \
+  if (!in_buffer) { \
+    freet(src); \
+  } \
+  return ret; \
 }
 
 DECLARE_INPLACE_ESCAPE_VERSION(escape_in_url)
@@ -4296,20 +4308,6 @@ void guess_httptype(httrackp * opt, char *s, const char *fil) {
   (void) get_httptype_sized(opt, s, HTS_MIMETYPE_SIZE, fil, 1);
 }
 
-// first match in a NUL-terminated {mime,ext} table. key selects the lookup
-// column (0=mime, 1=ext); returns the other column, or NULL if no row matches
-// (a "*" partner means the row carries no value).
-static const char *hts_mime_lookup(const char *(*table)[2], int key,
-                                   const char *needle) {
-  int j;
-
-  for (j = 0; strnotempty(table[j][1]); j++) {
-    if (strfield2(table[j][key], needle) && table[j][!key][0] != '*')
-      return table[j][!key];
-  }
-  return NULL;
-}
-
 // write the mime type for fil into s (capacity ssize)
 // flag: 1 to always return a type (the "application/..." / octet-stream
 // fallback) returns 1 if a type was written to s, 0 otherwise
@@ -4333,16 +4331,18 @@ HTSEXT_API hts_boolean get_httptype_sized(httrackp *opt, char *s, size_t ssize,
     while ((a > fil) && (*a != '.') && (*a != '/'))
       a--;
     if (a >= fil && *a == '.' && strlen(a) < 32) {
-      const char *mime;
+      int j = 0;
 
       a++;
-      mime = hts_mime_lookup(hts_mime, 1, a);
+      while(strnotempty(hts_mime[j][1])) {
-      if (mime == NULL)
+        if (strfield2(hts_mime[j][1], a)) {
-        mime = hts_mime_lookup(hts_mime_modern, 1, a);
+          if (hts_mime[j][0][0] != '*') { // a match exists
-      if (mime != NULL) {
+            strlcpybuff(s, hts_mime[j][0], ssize);
-        strlcpybuff(s, mime, ssize);
             return 1;
           }
+        }
+        j++;
+      }
 
       if (flag) {
         snprintf(s, ssize, "application/%s", a);
@@ -4365,11 +4365,6 @@ HTSEXT_API void get_httptype(httrackp *opt, char *s, const char *fil,
   (void) get_httptype_sized(opt, s, HTS_MIMETYPE_SIZE, fil, flag);
 }
 
-/* Advertised Accept-Encoding; gzip and deflate both decode via hts_zunpack */
-const char *hts_acceptencoding(hts_boolean compressible) {
-  return compressible ? "gzip, deflate, identity;q=0.9" : "identity";
-}
-
 // get type of fil (php)
 // s: buffer (text/html) or NULL
 // return: 1 if known by user
@@ -4481,17 +4476,19 @@ int get_userhttptype(httrackp * opt, char *s, const char *fil) {
 // returns 1 if an extension was found (and written to s), 0 otherwise
 int give_mimext(char *s, size_t ssize, const char *st) {
   int ok = 0;
-  const char *ext;
+  int j = 0;
 
   st = hts_effective_mime(st); /* no declared type: derive an html ext */
   s[0] = '\0';
-  ext = hts_mime_lookup(hts_mime, 0, st);
+  while((!ok) && (strnotempty(hts_mime[j][1]))) {
-  if (ext == NULL)
+    if (strfield2(hts_mime[j][0], st)) {
-    ext = hts_mime_lookup(hts_mime_modern, 0, st);
+      if (hts_mime[j][1][0] != '*') { // a match exists
-  if (ext != NULL) {
+        strlcpybuff(s, hts_mime[j][1], ssize);
-    strlcpybuff(s, ext, ssize);
         ok = 1;
       }
+    }
+    j++;
+  }
   // wrap "x" mimetypes, such as:
   // application/x-mp3
   // or
@@ -4815,6 +4812,7 @@ LLint check_downloadable_bytes(int rate) {
 
     time_now = mtime_local();
     elapsed_useconds = time_now - HTS_STAT.istat_timestart[id_timer];
+    // NO totally stupid - elapsed_useconds+=1000;      // for the next second, too
     bytes_transferred_during_period =
       (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[id_timer]);
 
@@ -4827,6 +4825,32 @@ LLint check_downloadable_bytes(int rate) {
     return TAILLE_BUFFER;
 }
 
+//
+// 0 : OK
+// 1 : slow down
+#if 0
+int HTS_TOTAL_RECV_CHECK(int var) {
+  if (HTS_STAT.HTS_TOTAL_RECV_STATE)
+    return 1;
+  /*
+     {
+     if (HTS_STAT.HTS_TOTAL_RECV_STATE==3) { 
+     var = min(var,32); 
+     Sleep(250); 
+     } else if (HTS_STAT.HTS_TOTAL_RECV_STATE==2) { 
+     var = min(var,256); 
+     Sleep(100); 
+     } else { 
+     var/=2; 
+     if (var<=0) var=1; 
+     Sleep(50); 
+     } 
+     }
+   */
+  return 0;
+}
+#endif
+
 // Lecture dans buff de size octets au maximum en utilisant la socket r (structure htsblk)
 // returns: 
 // >0 : data received
@@ -4835,6 +4859,7 @@ LLint check_downloadable_bytes(int rate) {
 int hts_read(htsblk * r, char *buff, int size) {
   int retour;
 
+  //  return read(soc,buff,size);
   if (r->is_file) {
 #if HTS_WIDE_DEBUG
     DEBUG_W("read(%p, %d, %d)\n" _(void *)buff _(int) size _(int) r->fp);
@@ -4852,6 +4877,7 @@ int hts_read(htsblk * r, char *buff, int size) {
     if (r->soc == INVALID_SOCKET)
       printf("!!WIDE_DEBUG ERROR, soc==INVALID hts_read\n");
 #endif
+    //HTS_TOTAL_RECV_CHECK(size);         // Diminuer au besoin si trop de données reçues
 #if HTS_USEOPENSSL
     if (r->ssl) {
       retour = SSL_read(r->ssl_con, buff, size);
@@ -5286,13 +5312,18 @@ SOCaddr* hts_dns_resolve(httrackp * opt, const char *_iadr, SOCaddr *const addr)
 
 // --- Tracage des mallocs() ---
 #ifdef HTS_TRACE_MALLOC
+//#define htsLocker(A, N) htsLocker(A, N)
 #define htsLocker(A, N) do {} while(0)
 static mlink trmalloc = { NULL, 0, 0, NULL };
 
 static int trmalloc_id = 0;
 static htsmutex *mallocMutex = NULL;
-
+static void hts_meminit(void) {
-static void hts_meminit(void) {}
+  //if (mallocMutex == NULL) {
+  //  mallocMutex = calloc(sizeof(*mallocMutex), 1);
+  //  htsLocker(mallocMutex, -999);
+  //}
+}
 void *hts_malloc(size_t len) {
   void *adr;
 
@@ -5376,6 +5407,7 @@ void hts_free(void *adr) {
               htsboundary);
       lnk->next = lnk->next->next;
       free((void *) blk_free);
+      //blk_free->id=-1;
       free((char *) adr - bsize);
       htsLocker(mallocMutex, 0);
       return;
@@ -5444,6 +5476,7 @@ mlink *hts_find(char *adr) {
 
     if (depl < 0)
       depl = -depl;
+    //assertf(depl < 512000);   /* near the stack frame.. doesn't look like malloc but stack variable */
     return NULL;
   }
 }
@@ -5500,6 +5533,7 @@ int ftp_available(void) {
 #else
 int ftp_available(void) {
   return 1;                     // ok!
+  //return 0;   // SOUS UNIX, PROBLEMESs
 }
 #endif
 
@@ -5713,19 +5747,13 @@ HTSEXT_API int hts_init(void) {
       assertf("OpenSSL version seems vulnerable to heartbleed bug (CVE-2014-0160)" == NULL);
     }
 
+    // OpenSSL_add_all_algorithms();
     openssl_ctx = SSL_CTX_new(method);
     if (!openssl_ctx) {
       fprintf(stderr, "fatal: unable to initialize TLS: SSL_CTX_new()\n");
       abortLog("unable to initialize TLS: SSL_CTX_new()");
       assertf("unable to initialize TLS" == NULL);
     }
-    /* Pin a TLS floor (no SSLv3/TLS1.0/1.1); no cert verify, by design. */
-#if OPENSSL_VERSION_NUMBER >= 0x10100000L
-    SSL_CTX_set_min_proto_version(openssl_ctx, TLS1_2_VERSION);
-#else
-    SSL_CTX_set_options(openssl_ctx, SSL_OP_NO_SSLv2 | SSL_OP_NO_SSLv3 |
-                                         SSL_OP_NO_TLSv1 | SSL_OP_NO_TLSv1_1);
-#endif
   }
 #endif
 
@@ -5926,7 +5954,15 @@ int multipleStringMatch(const char *s, const char *match) {
 }
 
 HTSEXT_API httrackp *hts_create_opt(void) {
-  static const char *defaultModules[] = {"httrack-plugin", NULL};
+#if ( defined(_WIN32) || defined(__ANDROID__) )
+  static const char *defaultModules[] = {
+    "htsswf", "htsjava", "httrack-plugin", NULL
+  };
+#else
+  static const char *defaultModules[] = {
+    "libhtsswf.so.1", "libhtsjava.so.2", "httrack-plugin", NULL
+  };
+#endif
   httrackp *opt = malloc(sizeof(httrackp));
 
   /* default options */
@@ -5969,7 +6005,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->shell = HTS_FALSE;
   opt->proxy.active = 0;        // pas de proxy
   opt->user_agent_send = HTS_TRUE;
-  StringCopy(opt->user_agent, HTS_DEFAULT_USER_AGENT);
+  StringCopy(opt->user_agent,
+             "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)");
   StringCopy(opt->referer, "");
   StringCopy(opt->from, "");
   opt->savename_83 = HTS_SAVENAME_83_LONG; // long names by default
@@ -6008,10 +6045,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->no_query_dedup = HTS_FALSE;
   StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
   StringCopy(opt->strip_query, "");
-  StringCopy(opt->cookies_file, "");
-  StringCopy(opt->why_url, "");
-  opt->pause_min_ms = 0;
-  opt->pause_max_ms = 0;
   opt->ftp_proxy = HTS_TRUE;
   opt->convert_utf8 = HTS_TRUE;
   StringCopy(opt->filelist, "");
@@ -6025,6 +6058,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->log = stdout;
   opt->errlog = stderr;
   opt->flush = HTS_TRUE;
+  // opt->aff_progress=0;
   opt->keyboard = HTS_FALSE;
   //
   StringCopy(opt->path_html, "");
@@ -6034,8 +6068,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   //
   opt->maxlink = 100000;        // 100,000 liens max par défaut
   opt->maxfilter = 200;         // 200 filtres max par défaut
-  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON
+  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT --
-                                // PARAMETRABLE POUR L'INSTANT --
+  //opt->maxcache_anticipate=256;  // maximum de liens à anticiper
   opt->maxtime = -1;            // temps max en secondes
   opt->maxrate = 100000;        // taux maxi
   opt->maxconn = 5.0;           // nombre connexions/s
@@ -6156,8 +6190,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
     StringFree(opt->footer);
     StringFree(opt->mod_blacklist);
     StringFree(opt->strip_query);
-    StringFree(opt->cookies_file);
-    StringFree(opt->why_url);
 
     StringFree(opt->path_html);
     StringFree(opt->path_html_utf8);
@@ -6189,9 +6221,11 @@ const hts_stat_struct* hts_get_stats(httrackp * opt) {
 }
 
 // defaut wrappers
-static void __cdecl htsdefault_init(t_hts_callbackarg *carg) {}
+static void __cdecl htsdefault_init(t_hts_callbackarg * carg) {
-
+}
-static void __cdecl htsdefault_uninit(t_hts_callbackarg *carg) {}
+static void __cdecl htsdefault_uninit(t_hts_callbackarg * carg) {
+  // hts_freevar();
+}
 static int __cdecl htsdefault_start(t_hts_callbackarg * carg, httrackp * opt) {
   return 1;
 }