review: trim comments to the one-line default

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -55,9 +55,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -101,8 +99,7 @@ jobs:
           sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
             -exec mv {} {}.hidden \;
           ! command -v python3
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+          make check
 
       - name: Print the test log on failure
         if: failure()
@@ -135,52 +132,13 @@ jobs:
       - name: Build
         run: make -j"$(sysctl -n hw.ncpu)"
 
-      - name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
-        # 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
-        # instantly like Linux; alias them onto lo0 so they don't stall to timeout.
-        run: |
-          set -euo pipefail
-          sudo ifconfig lo0 alias 127.0.0.2 up
-          sudo ifconfig lo0 alias 127.0.0.3 up
-
       - name: Test
-        run: |
-          jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
-  # Runtime smoke of the WebHTTrack launcher on macOS: it carries a Darwin-only
-  # browser path (open -W) and nothing else exercises htsserver. Install into a
-  # temp prefix, then check webhttrack brings up htsserver and serves the UI.
-  webhttrack-macos:
-    name: webhttrack smoke (macOS arm64)
-    runs-on: macos-14
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          brew install autoconf automake libtool autoconf-archive
-
-      - name: Build and install into a temp prefix
-        run: |
-          set -euo pipefail
-          ssl="$(brew --prefix openssl@3)"
-          autoreconf -fi
-          ./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib" \
-            --prefix="$RUNNER_TEMP/inst"
-          make -j"$(sysctl -n hw.ncpu)"
-          make install
-
-      - name: Smoke-test webhttrack
-        run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
-
   # Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
   # -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
   # and bounds math can truncate or wrap in ways 64-bit never reveals (the axis
@@ -213,9 +171,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -270,9 +226,7 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()
@@ -318,52 +272,12 @@ jobs:
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
           # skipped here (uninstrumented libz floods MSan with false positives).
           tests="$(cd tests && ls 01_engine-*.test | tr '\n' ' ')"
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs" TESTS="$tests"
+          make check TESTS="$tests"
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
-  # libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
-  # parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
-  # (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
-  # catastrophic backtracking -- and pins the regression seeds for the bugs the
-  # fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
-  fuzz:
-    name: fuzz (libFuzzer corpus replay, clang)
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential clang autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev
-
-      - name: Configure (fuzzers, static)
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure CC=clang \
-            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
-            LDFLAGS="-fsanitize=address,undefined" \
-            --enable-fuzzers --disable-shared
-
-      - name: Build
-        run: make -j"$(nproc)"
-
-      - name: Replay corpora
-        env:
-          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
-          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
-        run: bash fuzz/run-fuzzers.sh fuzz check
-
   # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
   # the configuration users on minimal systems build, and one libssl is not even
   # installed here so configure cannot silently re-enable it. The matrix above
@@ -394,9 +308,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
-        run: |
-          jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+        run: make check
 
       - name: Print the test log on failure
         if: failure()

.github/workflows/codeql.yml

@@ -1,63 +0,0 @@
-# CodeQL static analysis (C). The security-extended suite covers the classes
-# this codebase actually fights: overflows, tainted-size allocs, format bugs.
-name: CodeQL
-
-on:
-  push:
-    branches: [master]
-  pull_request:
-  schedule:
-    # Weekly re-scan of master so new/updated queries land without a push.
-    - cron: "17 4 * * 1"
-
-permissions:
-  contents: read
-
-concurrency:
-  group: codeql-${{ github.ref }}
-  cancel-in-progress: true
-
-jobs:
-  analyze:
-    name: analyze (c-cpp)
-    runs-on: ubuntu-24.04
-    permissions:
-      contents: read
-      # Upload findings to the repo's code-scanning dashboard.
-      security-events: write
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev
-
-      - uses: github/codeql-action/init@v4
-        with:
-          languages: c-cpp
-          build-mode: manual
-          queries: security-extended
-          # fopen's umask-controlled 0666 is intended for mirror/cache/log
-          # output; the one credential file (cookies.txt) is kept 0600 on Unix.
-          config: |
-            query-filters:
-              - exclude:
-                  id: cpp/world-writable-file-creation
-
-      # Manual build: CodeQL traces the compiler, so build exactly what ships.
-      - name: Build
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure
-          make -j"$(nproc)"
-
-      - uses: github/codeql-action/analyze@v4
-        with:
-          category: "/language:c-cpp"

.gitignore

@@ -34,9 +34,8 @@ Makefile
 *.so.*
 *.a
 
-# make dist output; dist/ holds httrack-gh-release staging artifacts.
+# make dist output.
 /httrack-*.tar.gz
-/dist/
 
 # Editor / autotools backup files.
 *~

AGENTS.md

@@ -6,19 +6,8 @@ the operational checklist: toolchain, invariants, and how to ship a change.
 ## Build & test
 - Fresh clone first: `git submodule update --init src/coucal`
 - `./bootstrap` (regenerates `configure` via `autoreconf`; needs autoconf,
-  automake, libtool), then `bash configure && make -j"$(nproc)" && make check
-  -j"$(nproc)"`. Always pass `-j` to `make check`: the suite runs under
-  automake's parallel harness and each crawl test binds its own ephemeral-port
-  server, so `-j` never contends and a multi-minute serial run drops to
-  seconds. A new `.test` added to `$(TESTS)` is scheduled onto a free worker
-  automatically; only a test slower than the current longest raises the floor.
-  On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
-  spend much of their wall time asleep (server trickles, httrack self-pacing),
-  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
-  platform, macOS included: the test server raises its listen backlog
-  (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
-  `-c16` bigcrawl the way Python's default backlog of 5 did.
-  Or run `sh build.sh` to do bootstrap + configure + make in one shot.
+  automake, libtool), then `bash configure && make && make check`. Or run
+  `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

Makefile.am

@@ -1,5 +1,5 @@
 
-SUBDIRS = src man m4 libtest templates lang html tests fuzz
+SUBDIRS = src man m4 libtest templates lang html tests
 
 ACLOCAL_AMFLAGS = -I m4
 

README

@@ -40,6 +40,7 @@ These are the principals limits of HTTrack for that moment. Note that we did not
 that would have solved them.
 
 - Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
+- Some java classes may not find some files on them (class included)
 - Cgi-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like -*cgi-bin*
 
 

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,11 +29,11 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
-# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
-# so it stays soname .so.3; bump revision.
+# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
+# installed headers (no struct layout or exported signature changed vs
+# 3.49.10), so it stays soname .so.3; bump revision.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:4:0"
+VERSION_INFO="3:3:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 
@@ -64,6 +64,16 @@ AC_SUBST(LT_CV_OBJDIR,$lt_cv_objdir)
 # Export version info
 AC_SUBST(VERSION_INFO)
 
+# Versioned plugin name for dlopen() in hts_create_opt(); soname major is
+# libtool's current - age, so this tracks VERSION_INFO bumps automatically.
+HTS_SONAME_MAJOR=$((${VERSION_INFO%%:*} - ${VERSION_INFO##*:}))
+case "$host_os" in
+darwin*) HTS_LIBHTSJAVA_NAME="libhtsjava.$HTS_SONAME_MAJOR.dylib" ;;
+*) HTS_LIBHTSJAVA_NAME="libhtsjava.so.$HTS_SONAME_MAJOR" ;;
+esac
+AC_DEFINE_UNQUOTED([HTS_LIBHTSJAVA_NAME], ["$HTS_LIBHTSJAVA_NAME"],
+  [Versioned libhtsjava runtime name, derived from VERSION_INFO])
+
 ### Default CFLAGS
 DEFAULT_CFLAGS="-Wall -Wformat -Wformat-security \
 -Wmultichar -Wwrite-strings -Wcast-qual -Wcast-align \
@@ -92,8 +102,7 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
 AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
 AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
 AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
-AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
-	[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
+AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
 AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
 AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
 AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -101,16 +110,9 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
 AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
 AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])
 
-# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
-# sanitizer builds, whose interceptors want the unfortified calls.
-case "$CFLAGS" in
-*-fsanitize=*) ;;
-*) AX_ADD_FORTIFY_SOURCE ;;
-esac
-
 # Force libc back into DT_NEEDED for libraries that reach it only through
-# libhttrack (the libtest callbacks), but only with a GNU-style linker; Apple
-# ld rejects these flags and links libSystem unconditionally.
+# libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
+# linker; Apple ld rejects these flags and links libSystem unconditionally.
 AX_CHECK_LINK_FLAG([-Wl,--push-state,--no-as-needed,-lc,--pop-state],
 	[LIBC_FORCE_LINK="-Wl,--push-state,--no-as-needed,-lc,--pop-state"])
 AC_SUBST([LIBC_FORCE_LINK])
@@ -308,37 +310,6 @@ AC_ARG_ENABLE([online-unit-tests],
 	])
 AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)
 
-## libFuzzer harnesses (fuzz/); requires clang
-AC_MSG_CHECKING(whether to build fuzzers)
-AC_ARG_ENABLE([fuzzers],
-	[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
-	[
-	case "${enableval}" in
-	no|yes)
-		fuzzers=$enableval
-		AC_MSG_RESULT($enableval)
-		;;
-	*)
-		AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
-		;;
-	esac
-	],
-	[
-	fuzzers=no
-	AC_MSG_RESULT(no)
-	])
-if test x"$fuzzers" = x"yes"; then
-	# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
-	AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
-		[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
-		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
-	# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
-	if test x"$enable_shared" != x"no"; then
-		AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
-	fi
-fi
-AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
-
 # Final output
 AC_CONFIG_FILES([
 Makefile
@@ -350,6 +321,5 @@ lang/Makefile
 html/Makefile
 libtest/Makefile
 tests/Makefile
-fuzz/Makefile
 ])
 AC_OUTPUT

debian/changelog

@@ -1,13 +1,3 @@
-httrack (3.49.12-1) unstable; urgency=medium
-
-  * New upstream release: security and crawl-correctness fixes (remote stack
-    overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
-    world-readable cookies.txt, filter-pattern denial of service) plus a new
-    --why filter diagnostic; full list in history.txt.
-  * Build with _FORTIFY_SOURCE and -fstack-protector-strong.
-
- -- Xavier Roche <xavier@debian.org>  Fri, 10 Jul 2026 21:11:09 +0200
-
 httrack (3.49.11-1) unstable; urgency=medium
 
   * New upstream release: crawl correctness and security fixes (network-facing

debian/copyright

@@ -7,8 +7,9 @@ Files: *
 Copyright: 1998-2026 Xavier Roche and other contributors
 License: GPL-3+
 Comment:
- htsbasenet.h links against the system OpenSSL library (originally by Eric
- Young); no OpenSSL/SSLeay code is bundled here.
+ The engine includes contributions from Yann Philippot (src/htsjava.c,
+ src/htsjava.h). htsbasenet.h links against the system OpenSSL library
+ (originally by Eric Young); no OpenSSL/SSLeay code is bundled here.
 
 Files: src/minizip/*
 Copyright: 1998-2010 Gilles Vollant

debian/libhttrack-dev.files

@@ -1,4 +1,5 @@
 usr/include/httrack
 usr/lib/*/libhttrack.{so}
+usr/lib/*/libhtsjava.{so}
 usr/share/httrack/libtest
 usr/lib/*/httrack/libtest

debian/libhttrack3.files

@@ -1,2 +1,3 @@
 usr/lib/*/libhttrack.so.3*
+usr/lib/*/libhtsjava.so.3*
 usr/share/httrack/templates

fuzz/Makefile.am

@@ -1,45 +0,0 @@
-# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
-if FUZZERS
-noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
-	fuzz-unescape fuzz-filters fuzz-url fuzz-header fuzz-cachendx
-endif
-
-AM_CPPFLAGS = \
-	@DEFAULT_CFLAGS@ \
-	@THREADS_CFLAGS@ \
-	@V6_FLAG@ \
-	@LFS_FLAG@ \
-	-I$(top_srcdir)/src \
-	-I$(top_srcdir)/src/coucal
-
-# Static-link libhttrack.la: the internal symbols are hidden in the .so.
-AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
-LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
-
-fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
-fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
-fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
-fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
-fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
-fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
-fuzz_url_SOURCES = fuzz-url.c fuzz.h
-fuzz_header_SOURCES = fuzz-header.c fuzz.h
-fuzz_cachendx_SOURCES = fuzz-cachendx.c fuzz.h
-
-# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
-EXTRA_DIST = README.md run-fuzzers.sh \
-	corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
-	corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
-	corpus/idna/idna.txt corpus/idna/unicode.txt \
-	corpus/idna/regress-multilabel-leak.txt \
-	corpus/entities/entities.txt \
-	corpus/unescape/percent.txt \
-	corpus/filters/filter.bin corpus/filters/filter-size.bin \
-	corpus/filters/regress-empty-subject-unique.bin \
-	corpus/filters/redos-star-classes.bin \
-	corpus/url/http-url.txt corpus/url/relative-path.txt \
-	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
-	corpus/header/full-response.txt corpus/header/redirect.txt \
-	corpus/cachendx/new-format.txt corpus/cachendx/old-format.txt \
-	corpus/cachendx/regress-overadvance.bin \
-	corpus/cachendx/regress-truncated-entry.bin

fuzz/README.md

@@ -1,15 +0,0 @@
-# Fuzzing httrack
-
-libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
-
-```sh
-./bootstrap
-mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
-CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
-  LDFLAGS="-fsanitize=address,undefined" \
-  bash /path/to/httrack/configure --enable-fuzzers --disable-shared
-make
-bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60   # 60s per target
-```
-
-Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.

fuzz/corpus/cachendx/new-format.txt

@@ -1,7 +0,0 @@
-8
-CACHE-1.1
-28
-Mon, 01 Jan 2024 00:00:00 GMT
-www.example.com
-/index.html
-123

fuzz/corpus/cachendx/old-format.txt

@@ -1,5 +0,0 @@
-3
-1.0
-www.example.com
-/page
-5

fuzz/corpus/cachendx/regress-overadvance.bin

@@ -1,2 +0,0 @@
-32768
-CACHE-1.1

fuzz/corpus/cachendx/regress-truncated-entry.bin

@@ -1,5 +0,0 @@
-8
-CACHE-1.1
-1
-x
-www.example.com

fuzz/corpus/charset/latin1.txt

@@ -1 +0,0 @@
-
café naďve ¤

fuzz/corpus/charset/sjis.txt

@@ -1 +0,0 @@
-コンピュータ

fuzz/corpus/entities/entities.txt

@@ -1 +0,0 @@
-&<>A☃é&notarealentity;�

fuzz/corpus/filters/regress-empty-subject-unique.bin

@@ -1 +0,0 @@
-**((

fuzz/corpus/header/full-response.txt

@@ -1,10 +0,0 @@
-HTTP/1.1 200 OK
-Content-Type: text/html; charset=utf-8
-Content-Length: 1234
-Content-Encoding: gzip
-Last-Modified: Mon, 01 Jan 2024 00:00:00 GMT
-Etag: "abc"
-Location: http://example.com/x
-Set-Cookie: ID=42; path=/; domain=.example.com
-Content-Range: bytes 0-99/100
-Transfer-Encoding: chunked

fuzz/corpus/header/redirect.txt

@@ -1,3 +0,0 @@
-HTTP/1.0 301 Moved
-Location: /elsewhere
-Content-Disposition: attachment; filename="a.pdf"

fuzz/corpus/idna/idna.txt

@@ -1 +0,0 @@
-xn--bcher-kva.example.com

fuzz/corpus/idna/regress-multilabel-leak.txt

@@ -1 +0,0 @@
-büchev.例å­bücheple

fuzz/corpus/idna/unicode.txt

@@ -1 +0,0 @@
-bücher.例子.example

fuzz/corpus/meta/meta-charset.html

@@ -1 +0,0 @@
-<html><head><meta charset="utf-8"></head><body>x</body></html>

fuzz/corpus/meta/meta-content-first.html

@@ -1 +0,0 @@
-<meta content="text/html; charset=gb2312" http-equiv="content-type">

fuzz/corpus/meta/meta-http-equiv.html

@@ -1 +0,0 @@
-<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">

fuzz/corpus/meta/meta-truncated.html

@@ -1 +0,0 @@
-<meta charset='utf-8

fuzz/corpus/meta/meta-unquoted.html

@@ -1 +0,0 @@
-<meta   charset = utf-8 />

fuzz/corpus/unescape/percent.txt

@@ -1 +0,0 @@
-%41%zz%%20%c3%a9+%2e%2e%2f

fuzz/corpus/url/http-url.txt

@@ -1 +0,0 @@
-http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag

fuzz/corpus/url/regress-file-empty-path.txt

@@ -1 +0,0 @@
-file://

fuzz/corpus/url/regress-long-path-abort.txt

@@ -1 +0,0 @@
-ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../

fuzz/corpus/url/relative-path.txt

@@ -1 +0,0 @@
-ftp://ftp.example.com/pub/../file.txt

fuzz/fuzz-cachendx.c

@@ -1,65 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the cache-index (.ndx) parser: a corrupt or truncated index must not
-   walk the length-prefixed cache_brstr/cache_binput scan past the buffer.
-   Mirrors the -#C cache-listing scan (htscoremain.c). */
-#include "fuzz.h"
-#include "htscache.h"
-#include "htslib.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  const char *const end = buf + size;
-  char firstline[256];
-  char *a = buf;
-
-  /* header: two length-prefixed fields (version, last-modified) */
-  a += cache_brstr(a, firstline, sizeof(firstline));
-  a += cache_brstr(a, firstline, sizeof(firstline));
-
-  /* body: newline-delimited host/file/position triples; the length-prefixed
-     scan must stay inside the buffer */
-  while (a != NULL && a < end) {
-    char BIGSTK line[HTS_URLMAXSIZE * 2];
-    char linepos[256];
-    int pos;
-
-    a = strchr(a + 1, '\n');
-    if (a == NULL)
-      break;
-    a++;
-    a += cache_binput(a, end, line, HTS_URLMAXSIZE);
-    a += cache_binput(a, end, line + strlen(line), HTS_URLMAXSIZE);
-    a += cache_binput(a, end, linepos, 200);
-    sscanf(linepos, "%d", &pos);
-    (void) pos;
-  }
-
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-charset.c

@@ -1,84 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
-   UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
-#include "fuzz.h"
-#include "htscharset.h"
-
-static const char *const charsets[] = {
-    "utf-8",    "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
-    "us-ascii", "shift_jis",  "euc-jp",     "iso-2022-jp", "gb2312",
-    "big5",     "euc-kr",     "koi8-r",     "utf-16",      "unknown-charset",
-};
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  const char *charset;
-  char *s;
-
-  if (size == 0)
-    return 0;
-  charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
-  data++, size--;
-  s = fuzz_strdup(data, size);
-
-  {
-    char *utf8 = hts_convertStringToUTF8(s, size, charset);
-    freet(utf8);
-  }
-  {
-    char *enc = hts_convertStringFromUTF8(s, size, charset);
-    freet(enc);
-  }
-  {
-    size_t nChars = 0;
-    hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
-
-    if (ucs != NULL) {
-      char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
-      freet(back);
-      freet(ucs);
-    }
-  }
-  {
-    size_t i = 0;
-
-    while (i < size) {
-      hts_UCS4 uc = 0;
-      const size_t nr = hts_readUTF8(s + i, size - i, &uc);
-      char out[8];
-
-      if (nr == 0)
-        break;
-      hts_writeUTF8(uc, out, sizeof(out));
-      i += nr;
-    }
-  }
-
-  freet(s);
-  return 0;
-}

fuzz/fuzz-entities.c

@@ -1,51 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
-   destination size, so truncation bounds get exercised too. */
-#include "fuzz.h"
-#include "htsencoding.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  static const size_t dsizes[] = {1, 2, 8, 64, 4096};
-  size_t dsize;
-  char *src, *dest;
-
-  if (size == 0)
-    return 0;
-  dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
-  data++, size--;
-  src = fuzz_strdup(data, size);
-  dest = malloct(dsize);
-
-  (void) hts_unescapeEntities(src, dest, dsize);
-  (void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
-
-  freet(dest);
-  freet(src);
-  return 0;
-}

fuzz/fuzz-filters.c

@@ -1,65 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
-   here). Input splits on the first NUL: pattern, then subject string. */
-#include "fuzz.h"
-#include "htsfilters.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  const char *joker = buf;
-  const uint8_t *sep = memchr(data, '\0', size);
-  /* subject in its own allocation so ASan bounds it apart from the pattern */
-  char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
-                          : fuzz_strdup(data + size, 0);
-
-  (void) strjoker(nom, joker, NULL, NULL);
-  {
-    LLint sz = (LLint) size;
-    int size_flag = 0;
-
-    (void) strjoker(nom, joker, &sz, &size_flag);
-  }
-  (void) strjokerfind(nom, joker);
-  {
-    char *filter = malloct(strlen(joker) + 2);
-    char *filters[1];
-    LLint sz = (LLint) size;
-    int size_flag = 0, depth = 0;
-
-    filter[0] = '-';
-    memcpy(filter + 1, joker, strlen(joker) + 1);
-    filters[0] = filter;
-    (void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
-    freet(filter);
-  }
-
-  freet(nom);
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-header.c

@@ -1,74 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 2026 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the HTTP response-header parser (htslib.c): treatfirstline on the
-   status line, treathead on each following header. Both consume raw bytes
-   off the wire and copy fields into fixed htsblk buffers; treathead also
-   mutates its line in place and drives cookie parsing. */
-#include "fuzz.h"
-#include "htslib.h"
-#include "htsbauth.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *buf = fuzz_strdup(data, size);
-  htsblk r;
-  t_cookie *cookie = calloct(1, sizeof(*cookie));
-  char *line = malloct(size + 1);
-  char *p = buf;
-  int first = 1;
-
-  memset(&r, 0, sizeof(r));
-  cookie->max_len = (int) sizeof(cookie->data);
-  r.location = malloct(HTS_URLMAXSIZE * 2);
-  r.location[0] = '\0';
-
-  /* feed one header line at a time, as the receive loop does */
-  while (p != NULL && *p != '\0') {
-    char *nl = strchr(p, '\n');
-    size_t n = (nl != NULL) ? (size_t) (nl - p) : strlen(p);
-    size_t i, len = 0;
-
-    /* binput drops every '\r' on the wire; mirror it */
-    for (i = 0; i < n; i++)
-      if (p[i] != '\r')
-        line[len++] = p[i];
-    line[len] = '\0';
-    if (first) {
-      treatfirstline(&r, line);
-      first = 0;
-    } else {
-      treathead(cookie, "www.example.com", "/", &r, line);
-    }
-    p = (nl != NULL) ? nl + 1 : NULL;
-  }
-
-  freet(r.location);
-  freet(line);
-  freet(cookie);
-  freet(buf);
-  return 0;
-}

fuzz/fuzz-idna.c

@@ -1,47 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
-#include "fuzz.h"
-#include "htscharset.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *s = fuzz_strdup(data, size);
-
-  {
-    char *idna = hts_convertStringUTF8ToIDNA(s, size);
-    freet(idna);
-  }
-  {
-    char *utf8 = hts_convertStringIDNAToUTF8(s, size);
-    freet(utf8);
-  }
-  (void) hts_isStringIDNA(s, size);
-
-  freet(s);
-  return 0;
-}

fuzz/fuzz-meta.c

@@ -1,40 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
-   <meta> charset declaration. */
-#include "fuzz.h"
-#include "htscharset.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *html = fuzz_strdup(data, size);
-  char *charset = hts_getCharsetFromMeta(html, size);
-
-  freet(charset);
-  freet(html);
-  return 0;
-}

fuzz/fuzz-unescape.c

@@ -1,53 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
-   output buffer size, so the bounded-copy contract is exercised. */
-#include "fuzz.h"
-#include "httrack-library.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  static const size_t bsizes[] = {1, 2, 16, 256, 8192};
-  size_t bsize;
-  char *s, *catbuff;
-
-  if (size == 0)
-    return 0;
-  bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
-  data++, size--;
-  s = fuzz_strdup(data, size);
-  catbuff = malloct(bsize);
-
-  (void) unescape_http(catbuff, bsize, s);
-  (void) unescape_http_unharm(catbuff, bsize, s, 0);
-  (void) unescape_http_unharm(catbuff, bsize, s, 1);
-  unescape_amp(s);
-
-  freet(catbuff);
-  freet(s);
-  return 0;
-}

fuzz/fuzz-url.c

@@ -1,49 +0,0 @@
-/* ------------------------------------------------------------ */
-/*
-HTTrack Website Copier, Offline Browser for Windows and Unix
-Copyright (C) 1998 Xavier Roche and other contributors
-
-SPDX-License-Identifier: GPL-3.0-or-later
-
-This program is free software: you can redistribute it and/or modify
-it under the terms of the GNU General Public License as published by
-the Free Software Foundation, either version 3 of the License, or
-(at your option) any later version.
-
-This program is distributed in the hope that it will be useful,
-but WITHOUT ANY WARRANTY; without even the implied warranty of
-MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-GNU General Public License for more details.
-
-You should have received a copy of the GNU General Public License
-along with this program. If not, see <http://www.gnu.org/licenses/>.
-
-Ethical use: we kindly ask that you NOT use this software to harvest email
-addresses or to collect any other private information about people. Doing so
-would dishonor our work and waste the many hours we have spent on it.
-
-Please visit our Website: http://www.httrack.com
-*/
-
-/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
-   is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
-   in place. */
-#include "fuzz.h"
-#include "htscore.h"
-#include "htslib.h"
-
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
-  char *s = fuzz_strdup(data, size);
-  lien_adrfil *af = calloct(1, sizeof(*af));
-  /* fil_simplifie rewrites in place and may grow an empty path to "./" */
-  char *path = malloct(size + 3);
-
-  (void) ident_url_absolute(s, af);
-  memcpy(path, s, size + 1);
-  fil_simplifie(path);
-
-  freet(path);
-  freet(af);
-  freet(s);
-  return 0;
-}

fuzz/run-fuzzers.sh

@@ -1,41 +0,0 @@
-#!/bin/bash
-# Drive every built harness against its seed corpus.
-#   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
-#   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
-# Replay is crash/leak-only and never mutates; the per-unit -timeout guards
-# both modes against pathological slowdowns (e.g. pre-#501 strjoker).
-set -euo pipefail
-
-srcdir=$(cd "$(dirname "$0")" && pwd)
-bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
-mode=${2:-20}
-
-status=0
-for f in "$bld"/fuzz-*; do
-    if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
-    case "$f" in *.o | *.c | *.dSYM) continue ;; esac
-    name=$(basename "$f")
-    corpus="$srcdir/corpus/${name#fuzz-}"
-    if [ "$mode" = "check" ]; then
-        echo "=== $name (replay) ==="
-        [ -d "$corpus" ] || continue
-        if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
-            echo "*** $name FAILED on its corpus" >&2
-            status=1
-        fi
-        continue
-    fi
-    work=$(mktemp -d)
-    args=("$work")
-    [ -d "$corpus" ] && args+=("$corpus")
-    echo "=== $name (${mode}s) ==="
-    if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
-        -artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
-        echo "*** $name FAILED; artifacts:" >&2
-        ls -l "$work" >&2
-        status=1
-    else
-        rm -rf "$work"
-    fi
-done
-exit $status

history.txt

@@ -4,23 +4,6 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
-3.49-12
-+ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
-+ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
-+ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
-+ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
-+ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
-+ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
-+ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
-+ Fixed: cookies.txt was created world-readable (#511)
-+ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
-+ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
-+ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
-+ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
-+ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
-+ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
-+ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
-
 3.49-11
 + New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
 + New: advertise deflate in Accept-Encoding and decode deflate responses (#450)

html/faq.html

@@ -609,9 +609,10 @@ most mirrors do work. We still are working to improve the mirror quality of HTTr
 <br>
 
 </a><a NAME="Q2b">Q: <strong>Some Java applets do not work properly!</strong><br>
-A: <em>Java applets may not work in some cases, because HTTrack does not parse compiled class files to find
-the resources they load. Java applets often need to be online anyway, since remote files are fetched directly
-at runtime.<br>
+A: <em>Java applets may not work in some cases, for example if HTTrack failed to detect all included classes
+or files called within the class file. Sometimes, Java applets need to be online, because remote files are
+directly caught. Finally, the site structure can be incompatible with the class (always try to keep the original site structure
+when you want to get Java classes)<br>
 If there is no way to make some classes work properly, you can exclude them with the filters. 
 They will be available, but only online.
 </em>

html/fcguide.html

@@ -243,7 +243,7 @@ Build options:
 Spider options:
   bN accept cookies in cookies.txt (0=do not accept,* 1=accept) (--cookies[=N])
   u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (--check-type[=N])
-  j *parse scripts (j0 don't parse) (--parse-java[=N])
+  j *parse Java Classes (j0 don't parse) (--parse-java[=N])
   sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) (--robots[=N])
  %h  force HTTP/1.0 requests (reduce update features, only for old servers or proxies) (--http-10)
  %B  tolerant requests (accept bogus responses on some servers, but not standard!) (--tolerant)
@@ -417,7 +417,7 @@ site. Specifically, the defauls are:
   X *purge old files after update (X0 keep delete)
   bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
   u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
-  j *parse scripts (j0 don't parse)
+  j *parse Java Classes (j0 don't parse)
   sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
   C  create/use a cache for updates and retries (C0 no cache,C1 cache is prioritary,* C2 test update before)
   f *log file mode
@@ -497,9 +497,9 @@ about your browser retaining them.
 <p align=justify> This causes different document types to be analyzed
 differently.
 
-<pre><b><i>  j *parse scripts (j0 don't parse) </i></b></pre>
+<pre><b><i>  j *parse Java Classes (j0 don't parse) </i></b></pre>
 
-<p align=justify> This causes scripts to be parsed looking for
+<p align=justify> This causes Java class files to be parsed looking for
 URLs.
 
 <pre><b><i>  sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always) </i></b></pre>
@@ -1178,7 +1178,7 @@ accepted and sent back in order to allow access.
 Spider options:
  bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
  u   check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
- j   *parse scripts (j0 don't parse)
+ j   *parse Java Classes (j0 don't parse)
  sN  follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always)
  %h  force HTTP/1.0 requests (reduce update features, only for old servers or proxies)
  %B  tolerant requests (accept bogus responses on some servers, but not standard!)
@@ -2451,10 +2451,12 @@ work.  We still are working to improve the mirror quality of HTTrack.
 
 <p align=justify><b>Q: Some Java applets do not work properly! </b>
 
-<p align=justify> A: Java applets may not work in some cases, because
-HTTrack does not parse compiled class files to find the resources they
-load.  Java applets often need to be online anyway, since remote files
-are fetched directly at runtime.
+<p align=justify> A: Java applets may not work in some cases, for
+example if HTTrack failed to detect all included classes or files called
+within the class file.  Sometimes, Java applets need to be online,
+because remote files are directly caught.  Finally, the site structure
+can be incompatible with the class (always try to keep the original site
+structure when you want to get Java classes)
 
 <p align=justify> If there is no way to make some classes work properly,
 you can exclude them with the filters.  They will be available, but only

html/httrack.man.html

@@ -764,9 +764,9 @@ check, * u1 check but /, u2 check always)
 <td width="78%">
 
 
-<p>*parse scripts (j0 don t parse, bitmask: |1 parse
-default, |4 don t parse .js |8 don t be aggressive)
-(--parse-java[=N])</p></td></tr>
+<p>*parse Java Classes (j0 don t parse, bitmask: |1 parse
+default, |2 don t parse .class |4 don t parse .js |8 don t
+be aggressive) (--parse-java[=N])</p></td></tr>
 <tr valign="top" align="left">
 <td width="11%"></td>
 <td width="6%">
@@ -889,8 +889,8 @@ only) (--protocol[=N])</p></td></tr>
 <td width="78%">
 
 
-<p>disable a specific external mime module (-%w
-httrack-plugin) (--disable-module &lt;param&gt;)</p></td></tr>
+<p>disable a specific external mime module (-%w htsswf -%w
+htsjava) (--disable-module &lt;param&gt;)</p></td></tr>
 </table>
 
 <p style="margin-left:11%; margin-top: 1em"><b>Browser
@@ -2175,6 +2175,10 @@ Several scripts generating complex filenames may not find
 them (ex:
 img.src=&rsquo;image&rsquo;+a+Mobj.dst+&rsquo;.gif&rsquo;)</small></p>
 
+<p style="margin-left:11%; margin-top: 1em"><small>- Some
+java classes may not find some files on them (class
+included)</small></p>
+
 <p style="margin-left:11%; margin-top: 1em"><small>-
 Cgi-bin links may not work properly in some cases
 (parameters needed). To avoid them: use filters like

html/options.html

@@ -288,8 +288,8 @@ This option define whether the engine has to abandon a host if a timeout/"too sl
 &P extended parsing, attempt to parse all links (even in unknown tags or Javascript)
 This option activates the extended parsing, that attempt to find links in unknown Html code/javascript
 
-j *parse scripts (j0 don't parse)
-This option defines whether the engine has to parse scripts or not to catch included files
+j *parse Java Classes (j0 don't parse)
+This option define whether the engine has to parse java files or not to catch included files
 
 I *make an index (I0 don't make)
 This option define whether the engine has to generate an index.html on the top directory

html/plug.html

@@ -103,219 +103,220 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
 
 <br>
 
-You can write external functions to be plugged in the httrack library very easily. 
-We'll see there some examples.
+You can write external functions to be plugged in the httrack library very easily. 
+We'll see there some examples.
 
 <br><br>
-
-The <tt>httrack</tt> commandline tool allows (since the 3.30 release) to plug external functions to various callbacks defined in httrack.
-The 3.41 release introduces a cleaned up verion of callbacks, with two major changes:
-<ul>
-<li>Cleaned up function prototypes, with two arguments always passed (the caller carg structure, and the httrackp* object), convenient to pass an user-defined pointer (see <tt>CALLBACKARG_USERDEF(carg)</tt>)</li>
-<li>The httrackp* option structure can be directly accessed to plug callbacks (no need to give the callback name and function name in the commandline!)</li>
-<li>The callback plug is made through the CHAIN_FUNCTION() helper, allowing to chain multiple callbacks of the same type (the callbacks MUST preserve the chain by calling ancestors)</li>
-</ul>
-
-<br>
-References: 
-<ul>
-<li>the <tt>httrack-library.h</tt> prototype file
-<br />
+
+The <tt>httrack</tt> commandline tool allows (since the 3.30 release) to plug external functions to various callbacks defined in httrack.
+The 3.41 release introduces a cleaned up verion of callbacks, with two major changes:
+<ul>
+<li>Cleaned up function prototypes, with two arguments always passed (the caller carg structure, and the httrackp* object), convenient to pass an user-defined pointer (see <tt>CALLBACKARG_USERDEF(carg)</tt>)</li>
+<li>The httrackp* option structure can be directly accessed to plug callbacks (no need to give the callback name and function name in the commandline!)</li>
+<li>The callback plug is made through the CHAIN_FUNCTION() helper, allowing to chain multiple callbacks of the same type (the callbacks MUST preserve the chain by calling ancestors)</li>
+</ul>
+
+<br>
+References: 
+<ul>
+<li>the <tt>httrack-library.h</tt> prototype file
+<br />
 Note: the <i>Initialization</i>, <i>Main functions</i>, <i>Options handling</i> and <i>Wrapper functions</i> sections are generally the only ones to be considered.
 </li>
-<li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
-<li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
-<li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
-<li>the example given at the end of this document</li>
-</ul>
-
-<br />
-Below the list of functions to be defined in the module (plugin).<br />
-<br />
-
-<table width="100%">
-<tr><td><b><i>module function name</i></b></td><td><b>function description</b></td><td><b>function signature</b></td></tr>
-<tr><td background="img/fade.gif"><i>hts_plug</i></td><td background="img/fade.gif">
-The module entry point. The opt structure can be used to plug callbacks, using the CHAIN_FUNCTION() macro helper. The argv optional argument is the one passed in the commandline as --wrapper parameter.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)<br />
-
-<br />
-Wrappers can be plugged inside hts_plug() using:<br />
-<tt>
-CHAIN_FUNCTION(opt, &lt;callback name&gt;, &lt;our callback function name&gt;, &lt;our callback function optional custom pointer argument&gt;);
-</tt>
-<br />
-
-<br />
-Example:
-<br />
-<tt>
-CHAIN_FUNCTION(opt, check_html, process, userdef);
-</tt>
-<br />
-
-</td><td background="img/fade.gif"><tt>extern int hts_plug(httrackp *opt, const char* argv);</tt></td></tr>
-
-<!-- -->
-
-<tr><td background="img/fade.gif"><i>hts_unplug</i></td><td background="img/fade.gif">
-The module exit point. To free allocated resources without using global variables, use the uninit callback (see below)</td><td background="img/fade.gif"><tt>extern int hts_unplug(httrackp *opt);</tt></td></tr>
-
-</table>
-
-
-<br />
-Note that all callbacks (except init and uninit) take as first two argument:
-<ul>
-<li>the t_hts_callbackarg structure<br />
-this structure holds the callback chain (parent callbacks defined before the current callback) pointers, and the user-defined pointer ; see <tt>CALLBACKARG_USERDEF(carg)</tt>)
-</li>
-<li>the httrackp structure<br />
-this structure, holding all current httrack options and mirror state, can be read or mofidied
-</li>
-</ul>
-
-<br />
-Below the list of callbacks, and associated external wrappers.
-
-<table width="100%">
-<tr><td><b><i>callback name</i></b></td><td><b>callback description</b></td><td><b>callback function signature</b></td></tr>
-
-<tr><td background="img/fade.gif"><i>init</i></td><td background="img/fade.gif">Note: the use the "start" callback is advised. Called during initialization.<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>uninit</i></td><td background="img/fade.gif">Note: the use os the "end" callback is advised.<br />Called during un-initialization<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>start</i></td><td background="img/fade.gif">Called when the mirror starts. The <tt>opt</tt> structure passed lists all options defined for this mirror. You may modify the <tt>opt</tt> structure to fit your needs.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>end</i></td><td background="img/fade.gif">Called when the mirror ends<br>return value: 1 upon success, 0 upon error (the mirror will then be considered aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>chopt</i></td><td background="img/fade.gif">Called when options are to be changed. The <tt>opt</tt> structure passed lists all options, updated to take account of recent changes<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>preprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is to be parsed (original, not yet modified document). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>postprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is parsed and transformed (links rewritten). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_html</i></td><td background="img/fade.gif">Called when a document (which may not be an html document) is to be parsed. The <tt>html</tt> address points to the document data, of lenth <tt>len</tt>. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the parsing can be processed, 0 if the file must be skipped without being parsed</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question. The <tt>question</tt> string contains the question for the (human) user<br>return value: the string answer ("" for default reply)</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query2</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>query3</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>loop</i></td><td background="img/fade.gif">Called periodically (informational, to display statistics)<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back, int back_max, int back_index, int lien_tot, int lien_ntot, int stat_time, hts_stat_struct* stats);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_link</i></td><td background="img/fade.gif">Called when a link has to be tested. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, int status);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>check_mime</i></td><td background="img/fade.gif">Called when a link download has begun, and needs to be tested against its MIME type. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested, and the <tt>mime</tt> string contains the link type being processed. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, const char* mime, int status);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>pause</i></td><td background="img/fade.gif">Called when the engine must pause. When the <tt>lockfile</tt> passed is deleted, the function can return<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* lockfile);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>filesave</i></td><td background="img/fade.gif">Called when a file is to be saved on disk<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* file);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>filesave2</i></td><td background="img/fade.gif">Called when a file is to be saved or checked on disk<br>The hostname, filename and local filename are given. Two additional flags tells if the local file is new (is_new), if the local file is to be modified (is_modified), and if the file was not updated remotely (not_updated).<br>(!is_new && !is_modified): the file is up-to-date, and will not be modified<br>(is_new && is_modified): a new file will be written (or an updated file is being written)<br>(!is_new && is_modified): a file is being updated (append)<br>(is_new && !is_modified): an empty file will be written ("do not recatch locally erased files")<br>not_updated: the file was not re-downloaded because it was up-to-date (no data transferred again)<br><br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* hostname, const char* filename, const char* localfile, int is_new, int is_modified, int not_updated);</tt></td></tr>
-
-<tr><td background="img/fade.gif"><i>linkdetected</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>linkdetected2</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link, const const char* tag_start);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>xfrstatus</i></td><td background="img/fade.gif">Called when a file has been processed (downloaded, updated, or error)<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>savename</i></td><td background="img/fade.gif">Called when a local filename has to be processed. The <tt>adr_complete</tt> and <tt>fil_complete</tt> are the address and URI of the file being saved ; the <tt>referer_adr</tt> and <tt>referer_fil</tt> are the address and URI of the referer link. The <tt>save</tt> string contains the local filename being used. You may modifiy the <tt>save</tt> string to fit your needs, up to 1024 bytes (note: filename collisions, if any, will be handled by the engine by renaming the file into file-2.ext, file-3.ext ..).<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>sendhead</i></td><td background="img/fade.gif">Called when HTTP headers are to be sent to the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>outgoing</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>receivehead</i></td><td background="img/fade.gif">Called when HTTP headers are recevived from the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>incoming</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);</tt></td></tr>
-
-<tr><td background="img/fade.gif"><i>detect</i></td><td background="img/fade.gif">Called when an unknown document is to be parsed. The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the type is known and can be parsed, 0 if the document type is unknown</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
-<tr><td background="img/fade.gif"><i>parse</i></td><td background="img/fade.gif">The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the document was successfully parsed, 0 if an error occurred</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
-
-
-</table>
+<li>the <tt>htsdefines.h</tt> prototype file, which describes callback function prototypes</li>
+<li>the <tt>htsopt.h</tt> prototype file, which describes the full httrackp* structure</li>
+<li>the <tt>callbacks-example*.c</tt> files given in the httrack archive</li>
+<li>the <tt>htsjava.c</tt> source file (the java class plugin ; overrides 'detect' and 'parse')</li>
+<li>the example given at the end of this document</li>
+</ul>
+
+<br />
+Below the list of functions to be defined in the module (plugin).<br />
+<br />
+
+<table width="100%">
+<tr><td><b><i>module function name</i></b></td><td><b>function description</b></td><td><b>function signature</b></td></tr>
+<tr><td background="img/fade.gif"><i>hts_plug</i></td><td background="img/fade.gif">
+The module entry point. The opt structure can be used to plug callbacks, using the CHAIN_FUNCTION() macro helper. The argv optional argument is the one passed in the commandline as --wrapper parameter.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)<br />
+
+<br />
+Wrappers can be plugged inside hts_plug() using:<br />
+<tt>
+CHAIN_FUNCTION(opt, &lt;callback name&gt;, &lt;our callback function name&gt;, &lt;our callback function optional custom pointer argument&gt;);
+</tt>
+<br />
+
+<br />
+Example:
+<br />
+<tt>
+CHAIN_FUNCTION(opt, check_html, process, userdef);
+</tt>
+<br />
+
+</td><td background="img/fade.gif"><tt>extern int hts_plug(httrackp *opt, const char* argv);</tt></td></tr>
+
+<!-- -->
+
+<tr><td background="img/fade.gif"><i>hts_unplug</i></td><td background="img/fade.gif">
+The module exit point. To free allocated resources without using global variables, use the uninit callback (see below)</td><td background="img/fade.gif"><tt>extern int hts_unplug(httrackp *opt);</tt></td></tr>
+
+</table>
+
+
+<br />
+Note that all callbacks (except init and uninit) take as first two argument:
+<ul>
+<li>the t_hts_callbackarg structure<br />
+this structure holds the callback chain (parent callbacks defined before the current callback) pointers, and the user-defined pointer ; see <tt>CALLBACKARG_USERDEF(carg)</tt>)
+</li>
+<li>the httrackp structure<br />
+this structure, holding all current httrack options and mirror state, can be read or mofidied
+</li>
+</ul>
+
+<br />
+Below the list of callbacks, and associated external wrappers.
+
+<table width="100%">
+<tr><td><b><i>callback name</i></b></td><td><b>callback description</b></td><td><b>callback function signature</b></td></tr>
+
+<tr><td background="img/fade.gif"><i>init</i></td><td background="img/fade.gif">Note: the use the "start" callback is advised. Called during initialization.<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>uninit</i></td><td background="img/fade.gif">Note: the use os the "end" callback is advised.<br />Called during un-initialization<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>start</i></td><td background="img/fade.gif">Called when the mirror starts. The <tt>opt</tt> structure passed lists all options defined for this mirror. You may modify the <tt>opt</tt> structure to fit your needs.<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>end</i></td><td background="img/fade.gif">Called when the mirror ends<br>return value: 1 upon success, 0 upon error (the mirror will then be considered aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>chopt</i></td><td background="img/fade.gif">Called when options are to be changed. The <tt>opt</tt> structure passed lists all options, updated to take account of recent changes<br>return value: 1 upon success, 0 upon error (the mirror will then be aborted)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>preprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is to be parsed (original, not yet modified document). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>postprocess</i></td><td background="img/fade.gif">Called when a document (which is an html document) is parsed and transformed (links rewritten). The <tt>html</tt> address points to the document data address (char**), and the <tt>length</tt> address points to the lenth of this document. Both pointer values (address and size) can be modified to change the document. It is up to the callback function to reallocate the given pointer (using the hts_realloc()/hts_free() library functions), which will be free()'ed by the engine. Hence, return of static buffers is strictly forbidden, and the use of hts_strdup() in such cases is advised. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the new pointers can be applied (default value)</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char** html, int* len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_html</i></td><td background="img/fade.gif">Called when a document (which may not be an html document) is to be parsed. The <tt>html</tt> address points to the document data, of lenth <tt>len</tt>. The <tt>url_address</tt> and <tt>url_file</tt> are the address and URI of the file being processed<br>return value: 1 if the parsing can be processed, 0 if the file must be skipped without being parsed</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* html, int len, const char* url_address, const char* url_file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question. The <tt>question</tt> string contains the question for the (human) user<br>return value: the string answer ("" for default reply)</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query2</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>query3</i></td><td background="img/fade.gif">Called when the wizard needs to ask a question</td><td background="img/fade.gif"><tt>const char* mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* question);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>loop</i></td><td background="img/fade.gif">Called periodically (informational, to display statistics)<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back, int back_max, int back_index, int lien_tot, int lien_ntot, int stat_time, hts_stat_struct* stats);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_link</i></td><td background="img/fade.gif">Called when a link has to be tested. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, int status);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>check_mime</i></td><td background="img/fade.gif">Called when a link download has begun, and needs to be tested against its MIME type. The <tt>adr</tt> and <tt>fil</tt> are the address and URI of the link being tested, and the <tt>mime</tt> string contains the link type being processed. The passed <tt>status</tt> value has the following meaning: 0 if the link is to be accepted by default, 1 if the link is to be refused by default, and -1 if no decision has yet been taken by the engine<br>return value: same meaning as the passed <tt>status</tt> value ; you may generally return -1 to let the engine take the decision by itself</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr, const char* fil, const char* mime, int status);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>pause</i></td><td background="img/fade.gif">Called when the engine must pause. When the <tt>lockfile</tt> passed is deleted, the function can return<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* lockfile);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>filesave</i></td><td background="img/fade.gif">Called when a file is to be saved on disk<br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* file);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>filesave2</i></td><td background="img/fade.gif">Called when a file is to be saved or checked on disk<br>The hostname, filename and local filename are given. Two additional flags tells if the local file is new (is_new), if the local file is to be modified (is_modified), and if the file was not updated remotely (not_updated).<br>(!is_new && !is_modified): the file is up-to-date, and will not be modified<br>(is_new && is_modified): a new file will be written (or an updated file is being written)<br>(!is_new && is_modified): a file is being updated (append)<br>(is_new && !is_modified): an empty file will be written ("do not recatch locally erased files")<br>not_updated: the file was not re-downloaded because it was up-to-date (no data transferred again)<br><br>return value: none</td><td background="img/fade.gif"><tt>void  mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* hostname, const char* filename, const char* localfile, int is_new, int is_modified, int not_updated);</tt></td></tr>
+
+<tr><td background="img/fade.gif"><i>linkdetected</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>linkdetected2</i></td><td background="img/fade.gif">Called when a link has been detected<br>return value: 1 if the link can be analyzed, 0 if the link must not even be considered</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* link, const const char* tag_start);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>xfrstatus</i></td><td background="img/fade.gif">Called when a file has been processed (downloaded, updated, or error)<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, lien_back* back);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>savename</i></td><td background="img/fade.gif">Called when a local filename has to be processed. The <tt>adr_complete</tt> and <tt>fil_complete</tt> are the address and URI of the file being saved ; the <tt>referer_adr</tt> and <tt>referer_fil</tt> are the address and URI of the referer link. The <tt>save</tt> string contains the local filename being used. You may modifiy the <tt>save</tt> string to fit your needs, up to 1024 bytes (note: filename collisions, if any, will be handled by the engine by renaming the file into file-2.ext, file-3.ext ..).<br>return value: must return 1</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, const char* adr_complete, const char* fil_complete, const char* referer_adr, const char* referer_fil, char* save);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>sendhead</i></td><td background="img/fade.gif">Called when HTTP headers are to be sent to the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>outgoing</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* outgoing);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>receivehead</i></td><td background="img/fade.gif">Called when HTTP headers are recevived from the remote server. The <tt>buff</tt> buffer contains text headers, <tt>adr</tt> and <tt>fil</tt> the URL, and <tt>referer_adr</tt> and <tt>referer_fil</tt> the referer URL. The <tt>incoming</tt> structure contains all information related to the current slot.<br>return value: 1 if the mirror can continue, 0 if the mirror must be aborted</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, char* buff, const char* adr, const char* fil, const char* referer_adr, const char* referer_fil, htsblk* incoming);</tt></td></tr>
+
+<tr><td background="img/fade.gif"><i>detect</i></td><td background="img/fade.gif">Called when an unknown document is to be parsed. The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the type is known and can be parsed, 0 if the document type is unknown</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
+<tr><td background="img/fade.gif"><i>parse</i></td><td background="img/fade.gif">The <tt>str</tt> structure contains all information related to the document.<br>return value: 1 if the document was successfully parsed, 0 if an error occurred</td><td background="img/fade.gif"><tt>int   mycallback(t_hts_callbackarg *carg, httrackp* opt, htsmoduleStruct* str);</tt></td></tr>
+
+
+</table>
 
 <br><br>
-Note: the optional libhttrack-plugin module (libhttrack-plugin.dll or libhttrack-plugin.so), if found in the library environment, is loaded automatically, and its <tt>hts_plug()</tt> function being called.<br />
-
-
-<br />
-An example is generally more efficient than anything else, so let's write our first module, aimed to stupidely print all parsed html files:
-<table width="100%" border="2">
-<tr><td>
-<pre>
-/* system includes */
-#include &lt;stdio.h&gt;
-#include &lt;stdlib.h&gt;
-#include &lt;string.h&gt;
-
-/* standard httrack module includes */
-#include "httrack-library.h"
-#include "htsopt.h"
-#include "htsdefines.h"
-
-/* local function called as "check_html" callback */
-static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt, 
-                        /*other parameters are callback-specific*/
-                        char* html, int len, const char* url_address, const char* url_file) {
-  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
-
-  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
-  if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
-    if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
-                                                html, len, url_address, url_file)) {
-        return 0;  /* abort */
-      }
-  }
-
-  printf("file %s%s content: %s\n", url_address, url_file, html);
-  return 1;  /* success */
-}
-
-/* local function called as "end" callback */
-static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
-  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
-
-  /* processing */
-  fprintf(stderr, "That's all, folks!\n");
-
-  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
-  if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
-    /* status is ok on our side, return other callabck's status */
-    return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
-  }
-
-  return 1;  /* success */
-}
-
-/*
-module entry point
-the function name and prototype MUST match this prototype
-*/
-EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
-  /* optional argument passed in the commandline we won't be using here */
-  const char *arg = strchr(argv, ',');
-  if (arg != NULL)
-    arg++;
-
-  /* plug callback functions */
-  CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
-  CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
-
-  return 1;  /* success */
-}
-
-/*
-module exit point
-the function name and prototype MUST match this prototype
-*/
-EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
-  fprintf(stder, "Module unplugged");
-
-  return 1;  /* success */
-}
-</pre>
-</td></tr></table>
-
-<br />
-Compile this file ; for example:
-<br />
-<tt>
-gcc -O -g3 -shared -o mylibrary.so myexample.c
-</tt>
-<br />
-and plug the module using the commandline ; for example:
-<br />
-<tt>
-httrack --wrapper mylibrary http://www.example.com
-</tt>
-<br />
-or, if some parameters are desired:
-<br />
-<tt>
-httrack --wrapper mylibrary,myparameter-string http://www.example.com
-</tt>
-<br />
-(the "myparameter-string" string will be available in the 'arg' parameter passed to the hts_plug entry point)
-<br />
-
+Note: the optional libhttrack-plugin module (libhttrack-plugin.dll or libhttrack-plugin.so), if found in the library environment, is loaded automatically, and its <tt>hts_plug()</tt> function being called.<br />
+
 
+<br />
+An example is generally more efficient than anything else, so let's write our first module, aimed to stupidely print all parsed html files:
+<table width="100%" border="2">
+<tr><td>
+<pre>
+/* system includes */
+#include &lt;stdio.h&gt;
+#include &lt;stdlib.h&gt;
+#include &lt;string.h&gt;
+
+/* standard httrack module includes */
+#include "httrack-library.h"
+#include "htsopt.h"
+#include "htsdefines.h"
+
+/* local function called as "check_html" callback */
+static int process_file(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt, 
+                        /*other parameters are callback-specific*/
+                        char* html, int len, const char* url_address, const char* url_file) {
+  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
+
+  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
+  if (CALLBACKARG_PREV_FUN(carg, check_html) != NULL) {
+    if (!CALLBACKARG_PREV_FUN(carg, check_html)(CALLBACKARG_PREV_CARG(carg), opt,
+                                                html, len, url_address, url_file)) {
+        return 0;  /* abort */
+      }
+  }
+
+  printf("file %s%s content: %s\n", url_address, url_file, html);
+  return 1;  /* success */
+}
+
+/* local function called as "end" callback */
+static int end_of_mirror(t_hts_callbackarg /*the carg structure, holding various information*/*carg, /*the option settings*/httrackp *opt) {
+  void *ourDummyArg = (void*) CALLBACKARG_USERDEF(carg);    /*optional user-defined arg*/
+
+  /* processing */
+  fprintf(stderr, "That's all, folks!\n");
+
+  /* call parent functions if multiple callbacks are chained. you can skip this part, if you don't want previous callbacks to be called. */
+  if (CALLBACKARG_PREV_FUN(carg, end) != NULL) {
+    /* status is ok on our side, return other callabck's status */
+    return CALLBACKARG_PREV_FUN(carg, end)(CALLBACKARG_PREV_CARG(carg), opt);
+  }
+
+  return 1;  /* success */
+}
+
+/*
+module entry point
+the function name and prototype MUST match this prototype
+*/
+EXTERNAL_FUNCTION int hts_plug(httrackp *opt, const char* argv) {
+  /* optional argument passed in the commandline we won't be using here */
+  const char *arg = strchr(argv, ',');
+  if (arg != NULL)
+    arg++;
+
+  /* plug callback functions */
+  CHAIN_FUNCTION(opt, check_html, process_file, /*optional user-defined arg*/NULL);
+  CHAIN_FUNCTION(opt, end, end_of_mirror, /*optional user-defined arg*/NULL);
+
+  return 1;  /* success */
+}
+
+/*
+module exit point
+the function name and prototype MUST match this prototype
+*/
+EXTERNAL_FUNCTION int hts_unplug(httrackp *opt) {
+  fprintf(stder, "Module unplugged");
+
+  return 1;  /* success */
+}
+</pre>
+</td></tr></table>
+
+<br />
+Compile this file ; for example:
+<br />
+<tt>
+gcc -O -g3 -shared -o mylibrary.so myexample.c
+</tt>
+<br />
+and plug the module using the commandline ; for example:
+<br />
+<tt>
+httrack --wrapper mylibrary http://www.example.com
+</tt>
+<br />
+or, if some parameters are desired:
+<br />
+<tt>
+httrack --wrapper mylibrary,myparameter-string http://www.example.com
+</tt>
+<br />
+(the "myparameter-string" string will be available in the 'arg' parameter passed to the hts_plug entry point)
+<br />
+
+
 <br><br>
 
 <!-- ==================== Start epilogue ==================== -->

html/step9_opt6.html

@@ -118,8 +118,8 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
   <br>Avoid "never", because the local mirror could be bogus
   </small><br><br>
 <!-- -->
-  <li>Parse scripts</li>
-  <br><small>Must the engine parse scripts to seek included filenames?
+  <li>Parse java files</li>
+  <br><small>Must the engine parse .java files (java classes) to seek included filenames?
   <br>It is checked by default
   </small><br><br>
 <!-- -->

m4/ax_add_fortify_source.m4

@@ -1,119 +0,0 @@
-# ===========================================================================
-#  https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
-# ===========================================================================
-#
-# SYNOPSIS
-#
-#   AX_ADD_FORTIFY_SOURCE
-#
-# DESCRIPTION
-#
-#   Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
-#   redefinition warnings, other cpp warnings or linker. Some distributions
-#   (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
-#   their compilers, leading to unnecessary warnings in the form of
-#
-#     <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
-#     <built-in>: note: this is the location of the previous definition
-#
-#   which is a problem if -Werror is enabled. This macro checks whether
-#   _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
-#   to CPPFLAGS.
-#
-#   Newer mingw-w64 msys2 package comes with a bug in
-#   headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
-#   and would need -lssp or -fstack-protector.  See
-#   https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
-#   link it.
-#
-# LICENSE
-#
-#   Copyright (c) 2017 David Seifert <soap@gentoo.org>
-#   Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
-#
-#   Copying and distribution of this file, with or without modification, are
-#   permitted in any medium without royalty provided the copyright notice
-#   and this notice are preserved.  This file is offered as-is, without any
-#   warranty.
-
-#serial 10
-
-AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
-    ac_save_cflags=$CFLAGS
-    ac_cwerror_flag=yes
-    AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
-    ax_add_fortify_3_failed=
-    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],
-            [[
-                #ifndef _FORTIFY_SOURCE
-                    return 0;
-                #else
-                    _FORTIFY_SOURCE_already_defined;
-                #endif
-            ]]
-        )],
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([[
-                #define _FORTIFY_SOURCE 3
-                #include <string.h>
-                int main(void) {
-                    char *s = " ";
-                    strcpy(s, "x");
-                    return strlen(s)-1;
-                }
-              ]]
-            )],
-            [
-              AC_MSG_RESULT([yes])
-              CFLAGS=$ac_save_cflags
-              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
-            ], [
-              AC_MSG_RESULT([no])
-              ax_add_fortify_3_failed=1
-            ],
-        ),
-        [
-          AC_MSG_RESULT([no])
-          ax_add_fortify_3_failed=1
-        ])
-    if test -n "$ax_add_fortify_3_failed"
-    then
-    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
-    AC_LINK_IFELSE([
-        AC_LANG_PROGRAM([],
-            [[
-                #ifndef _FORTIFY_SOURCE
-                    return 0;
-                #else
-                    _FORTIFY_SOURCE_already_defined;
-                #endif
-            ]]
-        )],
-        AC_LINK_IFELSE([
-            AC_LANG_SOURCE([[
-                #define _FORTIFY_SOURCE 2
-                #include <string.h>
-                int main(void) {
-                    char *s = " ";
-                    strcpy(s, "x");
-                    return strlen(s)-1;
-                }
-              ]]
-            )],
-            [
-              AC_MSG_RESULT([yes])
-              CFLAGS=$ac_save_cflags
-              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
-            ], [
-              AC_MSG_RESULT([no])
-              CFLAGS=$ac_save_cflags
-            ],
-        ),
-        [
-          AC_MSG_RESULT([no])
-          CFLAGS=$ac_save_cflags
-        ])
-    fi
-])

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "12 July 2026" "httrack website copier"
+.TH httrack 1 "27 June 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -51,7 +51,6 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
 [ \fB\-%K, \-\-cookies\-file\fR ]
-[ \fB\-%Y, \-\-why\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -219,12 +218,10 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
 .IP \-%K
 load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
-.IP \-%Y
-explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j
-*parse scripts (j0 don't parse, bitmask: |1 parse default, |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
+*parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive) (\-\-parse\-java[=N])
 .IP \-sN
 follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules)) (\-\-robots[=N])
 .IP \-%h
@@ -248,7 +245,7 @@ can also be used to force a specific file type: \-\-assume foo.cgi=text/html
 .IP \-@iN
 internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only) (\-\-protocol[=N])
 .IP \-%w
-disable a specific external mime module (\-%w httrack\-plugin) (\-\-disable\-module <param>)
+disable a specific external mime module (\-%w htsswf \-%w htsjava) (\-\-disable\-module <param>)
 .SS Browser ID:
 .IP \-F
 user\-agent field sent in HTTP headers (\-F "user\-agent name") (\-\-user\-agent <param>)
@@ -514,6 +511,8 @@ that would have solved them.
 .SM
 \- Several scripts generating complex filenames may not find them (ex: img.src='image'+a+Mobj.dst+'.gif')
 .SM
+\- Some java classes may not find some files on them (class included)
+.SM
 \- Cgi\-bin links may not work properly in some cases (parameters needed). To avoid them: use filters like \-*cgi\-bin*
 .SH BUGS
 Please reports bugs to

src/Makefile.am

@@ -44,7 +44,7 @@ httrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 proxytrack_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 
-lib_LTLIBRARIES = libhttrack.la
+lib_LTLIBRARIES = libhttrack.la libhtsjava.la
 
 htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h
 proxytrack_SOURCES = proxy/main.c \
@@ -82,6 +82,14 @@ libhttrack_la_LIBADD = $(THREADS_LIBS) $(ZLIB_LIBS) $(OPENSSL_LIBS) $(DL_LIBS) $
 libhttrack_la_CFLAGS = $(AM_CFLAGS) -DLIBHTTRACK_EXPORTS -DZLIB_CONST
 libhttrack_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO)
 
+libhtsjava_la_SOURCES = htsjava.c htsjava.h
+libhtsjava_la_LIBADD = $(THREADS_LIBS) $(DL_LIBS) libhttrack.la
+# This thin JNI wrapper reaches libc only through libhttrack, so the direct
+# libc edge is dropped from DT_NEEDED (library-not-linked-against-libc). Force
+# libc back as a dependency; configure gates the flag since only a GNU-style
+# linker accepts it (LIBC_FORCE_LINK is empty on e.g. macOS).
+libhtsjava_la_LDFLAGS = $(AM_LDFLAGS) -version-info $(VERSION_INFO) $(LIBC_FORCE_LINK)
+
 EXTRA_DIST = httrack.h webhttrack \
 		coucal/murmurhash3.h.diff \
 		coucal/murmurhash3.h.orig \
@@ -111,6 +119,7 @@ EXTRA_DIST = httrack.h webhttrack \
 		coucal/README.md \
 		coucal/sample.c \
 		coucal/tests.c \
+		htsjava.vcproj \
 		httrack.dsp httrack.dsw httrack.vcproj \
 		libhttrack.dsp libhttrack.dsw libhttrack.vcproj \
 		webhttrack.dsp webhttrack.dsw webhttrack.vcproj

src/htsalias.c

@@ -114,8 +114,6 @@ const char *hts_optalias[][4] = {
    "strip [host/pattern=]key1,key2,... from URLs"},
   {"cookies-file", "-%K", "param1",
    "load extra cookies from a Netscape cookies.txt"},
-  {"why", "-%Y", "param1",
-   "explain which filter rule accepts or rejects a URL, then exit"},
   {"pause", "-%G", "param1",
    "random pause of MIN[:MAX] seconds between files"},
   {"generate-errors", "-o", "single", ""},
@@ -285,6 +283,7 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
       char *position;
       int need_param = 1;
 
+      //int return_param=0;
       int pos;
 
       command[0] = param[0] = addcommand[0] = '\0';
@@ -362,6 +361,7 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
               strlcatbuff(return_argv[0], "0", return_argv_size);
             else if (strcmp(param, "on") == 0) {
               // on is the default
+              // strcatbuff(return_argv[0],"1");
             } else
               strlcatbuff(return_argv[0], param, return_argv_size);
           }

src/htsback.c

@@ -43,12 +43,14 @@ Please visit our Website: http://www.httrack.com
 
 #include "htsback.h"
 
+//#ifdef _WIN32
 #include "htsftp.h"
 #if HTS_USEZLIB
 #include "htszlib.h"
 #else
 #error HTS_USEZLIB not defined
 #endif
+//#endif
 
 #ifdef _WIN32
 #ifndef __cplusplus
@@ -68,15 +70,6 @@ static int slot_can_be_cached_on_disk(const lien_back * back);
 static int slot_can_be_cleaned(const lien_back * back);
 static int slot_can_be_finalized(httrackp * opt, const lien_back * back);
 
-/* Which hard quota, if any, is currently aborting the mirror. */
-typedef enum {
-  HTS_MIRROR_LIMIT_NONE = 0,
-  HTS_MIRROR_LIMIT_SIZE,
-  HTS_MIRROR_LIMIT_TIME,
-} hts_mirror_limit;
-
-static hts_mirror_limit back_mirror_limit(httrackp *opt);
-
 struct_back *back_new(httrackp *opt, int back_max) {
   int i;
   struct_back *sback = calloct(1, sizeof(struct_back));
@@ -533,22 +526,22 @@ int back_nsoc_overall(const struct_back * sback) {
 
 /* generate temporary file on lien_back */
 /* Note: utf-8 */
-static int create_back_tmpfile(httrackp *opt, lien_back *const back,
-                               const char *ext) {
+static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
   // do not use tempnam() but a regular filename
   back->tmpfile_buffer[0] = '\0';
   if (back->url_sav != NULL && back->url_sav[0] != '\0') {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
-             back->url_sav, ext);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.z", 
+             back->url_sav);
     back->tmpfile = back->tmpfile_buffer;
     if (structcheck(back->tmpfile) != 0) {
-      hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
+      hts_log_print(opt, LOG_WARNING, "can not create directory to %s", 
                     back->tmpfile);
       return -1;
     }
   } else {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
-             StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer),
+             "%s/tmp%d.z", StringBuff(opt->path_html_utf8),
+             opt->state.tmpnameid++);
     back->tmpfile = back->tmpfile_buffer;
   }
   /* OK */
@@ -556,32 +549,6 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
   return 0;
 }
 
-/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
-   existing file moved the good copy to back->tmpfile before truncating url_sav.
-   commit keeps the new file and drops the backup; else restore it so an aborted
-   transfer leaves the previous copy intact. Skips the zlib .z temp. */
-static void back_finalize_backup(httrackp *opt, lien_back *const back,
-                                 const hts_boolean commit) {
-  if (back->tmpfile == NULL || back->r.compressed)
-    return;
-  if (commit) {
-    (void) UNLINK(back->tmpfile); /* new copy is good; drop the backup */
-  } else {
-    if (back->r.out != NULL) {
-      fclose(back->r.out);
-      back->r.out = NULL;
-    }
-    (void) UNLINK(back->url_sav); /* drop the failed partial */
-    /* On rename failure keep the backup: it still holds the good copy, so
-       losing it would be worse than an orphaned temp. */
-    if (RENAME(back->tmpfile, back->url_sav) != 0)
-      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
-                    "could not restore %s; previous copy kept as %s",
-                    back->url_sav, back->tmpfile);
-  }
-  back->tmpfile = NULL;
-}
-
 // objet (lien) téléchargé ou transféré depuis le cache
 //
 // fermer les paramètres de transfert,
@@ -617,7 +584,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                       LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
                       back[p].r.size, back[p].url_adr, back[p].url_fil);
       }
-      back_finalize_backup(opt, &back[p], HTS_FALSE);
       return -1;
     }
 
@@ -631,12 +597,13 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
 #if HTS_USEZLIB
         if (back[p].r.compressed) {
           if (back[p].r.size > 0) {
+            //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
             // stats
             back[p].compressed_size = back[p].r.size;
             // en mémoire -> passage sur disque
             if (!back[p].r.is_write) {
               // do not use tempnam() but a regular filename
-              if (create_back_tmpfile(opt, &back[p], "z") == 0) {
+              if (create_back_tmpfile(opt, &back[p]) == 0) {
                 assertf(back[p].tmpfile != NULL);
                 /* note: tmpfile is utf-8 */
                 back[p].r.out = FOPEN(back[p].tmpfile, "wb");
@@ -709,9 +676,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
           }
         }
 #endif
-        /* Body fully received: keep the freshly written url_sav, drop the
-           backup of the previous copy. */
-        back_finalize_backup(opt, &back[p], HTS_TRUE);
         /* Write mode to disk */
         if (back[p].r.is_write && back[p].r.adr != NULL) {
           freet(back[p].r.adr);
@@ -940,9 +904,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
       }
     }
   }
-  /* Aborted, error, or not ready: url_sav (if written) is broken; restore the
-     previous copy from the backup. */
-  back_finalize_backup(opt, &back[p], HTS_FALSE);
   return -1;
 }
 
@@ -966,6 +927,7 @@ int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
     /* clear everything but connection: switch, close, and reswitch */
     back_connxfr(src, &tmp);
     back_delete(opt, cache, sback, p);
+    //deletehttp(src);
     back_connxfr(&tmp, src);
     src->req.flush_garbage = 1; /* ignore CRLF garbage */
     return 1;
@@ -1400,39 +1362,9 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
   return 0;
 }
 
-/* Move a still-writing .delayed placeholder to its final name (#483). */
-hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
-                                const char *newname) {
-  hts_boolean renamed;
-
-  if (!back->r.is_write || back->tmpfile != NULL ||
-      !IS_DELAYED_EXT(back->url_sav) || strcmp(back->url_sav, newname) == 0)
-    return HTS_TRUE; /* nothing bound to the placeholder name */
-  if (back->r.out != NULL) {
-    fclose(back->r.out);
-    back->r.out = NULL;
-  }
-  renamed = RENAME(back->url_sav, newname) == 0 ? HTS_TRUE : HTS_FALSE;
-  if (renamed && (back->status == STATUS_READY ||
-                  (back->r.out = FOPEN(newname, "ab")) != NULL)) {
-    filenote(&opt->state.strc, newname, NULL);
-    hts_log_print(opt, LOG_DEBUG, "moved placeholder %s to %s", back->url_sav,
-                  newname);
-    return HTS_TRUE;
-  }
-  /* partial lost: drop only what we own (Windows rename won't overwrite) */
-  hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "unable to move %s to %s",
-                back->url_sav, newname);
-  back->r.statuscode = STATUSCODE_INVALID;
-  strcpybuff(back->r.msg, "Write error on disk");
-  back->r.is_write = 0;
-  (void) UNLINK(renamed ? newname : back->url_sav);
-  return HTS_FALSE;
-}
-
 // effacer entrée
 /* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
-void back_delayed_discard(httrackp *opt, lien_back *back) {
+static void back_delayed_discard(httrackp *opt, lien_back *back) {
   if (back->r.out != NULL) {
     fclose(back->r.out);
     back->r.out = NULL;
@@ -1475,6 +1407,7 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                       back[p].url_adr, back[p].url_fil, back[p].url_sav);
       }
       if (cache != NULL) {
+        //hts_log_print(opt, LOG_TRACE, "finalizing from back_delete");
         back_finalize(opt, cache, sback, p);
       }
     }
@@ -1615,6 +1548,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     if (back[p].r.soc != INVALID_SOCKET) {      /* we never know */
       deletehttp(&back[p].r);
     }
+    //memset(&(back[p].r), 0, sizeof(htsblk)); 
     hts_init_htsblk(&back[p].r);
     back[p].r.location = back[p].location_buffer;
 
@@ -1622,6 +1556,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
     strcpybuff(back[p].url_adr, adr);
     strcpybuff(back[p].url_fil, fil);
     strcpybuff(back[p].url_sav, save);
+    //back[p].links_index = links_index;
     // copier referer si besoin
     strcpybuff(back[p].referer_adr, "");
     strcpybuff(back[p].referer_fil, "");
@@ -1688,104 +1623,138 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       }
     }
     // tester cache
-    if ((strcmp(adr, "file://")) /* pas fichier */
-        && ((!test) ||
-            (cache->type == 1)) /* cache prioritaire, laisser passer en test! */
-        &&
-        ((strnotempty(save)) || (strcmp(fil, "/robots.txt") ==
-                                 0))) { // si en test on ne doit pas utiliser le
-                                        // cache sinon telescopage avec le 302..
+    if ((strcmp(adr, "file://"))        /* pas fichier */
+        &&((!test) || (cache->type == 1))       /* cache prioritaire, laisser passer en test! */
+        &&((strnotempty(save)) || (strcmp(fil, "/robots.txt") == 0))) { // si en test on ne doit pas utiliser le cache sinon telescopage avec le 302..
+#if HTS_FAST_CACHE
       intptr_t hash_pos;
       int hash_pos_return = 0;
-
+#else
+      char *a = NULL;
+#endif
+#if HTS_FAST_CACHE
       if (cache->hashtable) {
+#else
+      if (cache->use) {
+#endif
         char BIGSTK buff[HTS_URLMAXSIZE * 4];
 
+#if HTS_FAST_CACHE
         strcpybuff(buff, adr);
         strcatbuff(buff, fil);
         hash_pos_return = coucal_read(cache->hashtable, buff, &hash_pos);
+#else
+        buff[0] = '\0';
+        strcatbuff(buff, "\n");
+        strcatbuff(buff, adr);
+        strcatbuff(buff, "\n");
+        strcatbuff(buff, fil);
+        strcatbuff(buff, "\n");
+        a = strstr(cache->use, buff);
+#endif
 
+        // Ok, noté en cache->. mais bien présent dans le cache ou sur disque?
+#if HTS_FAST_CACHE
         // negative values when data is not in cache
         if (hash_pos_return < 0) {
-          if (!test) { // not test mode
-            /* note: no check with IS_DELAYED_EXT() enabled - postcheck by
-             * client please! */
-            if (save[0] != '\0' && !IS_DELAYED_EXT(save) &&
-                fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <=
-                    0) { // final file missing or empty
-              int found = 0;
+#else
+        if (a) {
+#endif
+          if (!test) {          // non mode test
+#if HTS_FAST_CACHE==0
+            int pos = -1;
 
-              /* It is possible that the file has been moved due to changes in
-               * build structure */
-              {
-                char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
-                htsblk r;
+            a += strlen(buff);
+            sscanf(a, "%d", &pos);      // lire position
+#endif
 
-                previous_save[0] = '\0';
-                r = cache_readex(opt, cache, adr, fil, /*head */ NULL,
+#if HTS_FAST_CACHE==0
+            if (pos < 0) {      // pas de mise en cache data, vérifier existence
+#endif
+              /* note: no check with IS_DELAYED_EXT() enabled - postcheck by client please! */
+              if (save[0] != '\0' && !IS_DELAYED_EXT(save) && fsize_utf8(fconv(catbuff, sizeof(catbuff), save)) <= 0) {  // fichier final n'existe pas ou est vide!
+                int found = 0;
+
+                /* It is possible that the file has been moved due to changes in build structure */
+                {
+                  char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
+                  htsblk r;
+
+                  previous_save[0] = '\0';
+                  r =
+                    cache_readex(opt, cache, adr, fil, /*head */ NULL,
                                  /*bound to back[p] (temporary) */
                                  back[p].location_buffer, previous_save, /*ro */
                                  1);
-                /* Is supposed to be on disk only */
-                if (r.is_write && previous_save[0] != '\0') {
-                  /* Exists, but with another (old) filename: rename (almost)
-                   * silently */
-                  if (strcmp(previous_save, save) != 0 &&
-                      fexist_utf8(
-                          fconv(catbuff, sizeof(catbuff), previous_save))) {
-                    rename(fconv(catbuff, sizeof(catbuff), previous_save),
-                           fconv(catbuff2, sizeof(catbuff2), save));
-                    if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
-                      found = 1;
-                      hts_log_print(opt, LOG_DEBUG,
-                                    "File '%s' has been renamed since last "
-                                    "mirror to '%s' ; applying changes",
-                                    previous_save, save);
-                    } else {
-                      hts_log_print(opt, LOG_ERROR,
-                                    "Could not rename '%s' to '%s' ; will have "
-                                    "to retransfer it",
-                                    previous_save, save);
+                  /* Is supposed to be on disk only */
+                  if (r.is_write && previous_save[0] != '\0') {
+                    /* Exists, but with another (old) filename: rename (almost) silently */
+                    if (strcmp(previous_save, save) != 0
+                        && fexist_utf8(fconv(catbuff, sizeof(catbuff), previous_save))) {
+                      rename(fconv(catbuff, sizeof(catbuff), previous_save),
+                             fconv(catbuff2, sizeof(catbuff2), save));
+                      if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {
+                        found = 1;
+                        hts_log_print(opt, LOG_DEBUG,
+                                      "File '%s' has been renamed since last mirror to '%s' ; applying changes",
+                                      previous_save, save);
+                      } else {
+                        hts_log_print(opt, LOG_ERROR,
+                                      "Could not rename '%s' to '%s' ; will have to retransfer it",
+                                      previous_save, save);
+                      }
                     }
                   }
+                  back[p].location_buffer[0] = '\0';
                 }
-                back[p].location_buffer[0] = '\0';
-              }
 
-              /* Not found ? */
-              if (!found) {
-                // invalidate: gone from disk, force a refetch
-                hash_pos_return = 0;
-                if (opt->norecatch) {
-                  if (!fexist_utf8(fconv(
-                          catbuff, sizeof(catbuff),
-                          save))) { // declared but missing: user erased it
-                    FILE *fp =
-                        FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
+                /* Not found ? */
+                if (!found) {
+#if HTS_FAST_CACHE
+                  hash_pos_return = 0;
+#else
+                  a = NULL;
+#endif
+                  // dévalider car non présent sur disque dans structure originale!!!
+                  // sinon, le fichier est ok à priori, mais on renverra un if-modified-since pour
+                  // en être sûr
+                  if (opt->norecatch) { // tester norecatch
+                    if (!fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {   // fichier existe pas mais déclaré: on l'a effacé
+                      FILE *fp = FOPEN(fconv(catbuff, sizeof(catbuff), save), "wb");
 
-                    if (fp)
-                      fclose(fp);
+                      if (fp)
+                        fclose(fp);
+                      hts_log_print(opt, LOG_WARNING,
+                                    "Previous file '%s' not found (erased by user ?), ignoring: %s%s",
+                                    save, back[p].url_adr, back[p].url_fil);
+                    }
+                  } else {
                     hts_log_print(opt, LOG_WARNING,
-                                  "Previous file '%s' not found (erased by "
-                                  "user ?), ignoring: %s%s",
+                                  "Previous file '%s' not found (erased by user ?), recatching: %s%s",
                                   save, back[p].url_adr, back[p].url_fil);
                   }
-                } else {
-                  hts_log_print(opt, LOG_WARNING,
-                                "Previous file '%s' not found (erased by user "
-                                "?), recatching: %s%s",
-                                save, back[p].url_adr, back[p].url_fil);
                 }
-              }
-            } // fsize() <= 0
+              }                 // fsize() <= 0
+#if HTS_FAST_CACHE==0
+            }
+#endif
           }
         }
         //
       } else {
+#if HTS_FAST_CACHE
         hash_pos_return = 0;
+#else
+        a = NULL;
+#endif
       }
 
-      if (hash_pos_return) { // in cache, with data
+      // Existe pas en cache, ou bien pas de cache présent
+#if HTS_FAST_CACHE
+      if (hash_pos_return) {    // OK existe en cache (et données aussi)!
+#else
+      if (a != NULL) {          // OK existe en cache (et données aussi)!
+#endif
         const int cache_is_prioritary = cache->type == 1
           || opt->state.stop != 0;
         if (cache_is_prioritary) {      // cache prioritaire (pas de test if-modified..)
@@ -1826,7 +1795,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
                             back[p].url_adr, back[p].url_fil);
             }
             back[p].r.notmodified = 1;  // fichier non modifié
-            back[p].status = STATUS_READY; // OK prêt
+            back[p].status = STATUS_READY;      // OK prêt
+            //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);        // not modified
             back_set_finished(sback, p);
 
             // finalize transfer
@@ -1841,6 +1811,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
           } else {              // erreur
             // effacer r
             hts_init_htsblk(&back[p].r);
+            //memset(&(back[p].r), 0, sizeof(htsblk)); 
             back[p].r.location = back[p].location_buffer;
             // et continuer (chercher le fichier)
           }
@@ -1906,8 +1877,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #endif
         }
       }
-      /* Not in cache ; maybe in temporary cache ? Warning: non-movable
-         "url_sav" */
+      /* Not in cache ; maybe in temporary cache ? Warning: non-movable "url_sav" */
       else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
         const off_t file_size = fsize_utf8(itemback->url_sav);
 
@@ -2036,6 +2006,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       // ouvrir liaison, envoyer requète
       // ne pas traiter ou recevoir l'en tête immédiatement
       hts_init_htsblk(&back[p].r);
+      // memset(&(back[p].r), 0, sizeof(htsblk));
       back[p].r.location = back[p].location_buffer;
       // fresh connect: address list not yet probed, start at the first
       sback->connect_fallback[p].addr_index = 0;
@@ -2086,15 +2057,20 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #if HTS_USEOPENSSL
       else if (strfield(back[p].url_adr, "https://")) {     // let's rock
         back[p].r.ssl = 1;
+        // back[p].r.ssl_soc = NULL;
         back[p].r.ssl_con = NULL;
       }
 #endif
 
       if (!back_trylive(opt, cache, sback, p)) {
 #if HTS_XGETHOST
-        back[p].status = STATUS_WAIT_DNS; // host name resolution attempt
-        soc = INVALID_SOCKET;             // not opened yet
-        if (host_wait(opt, &back[p])) {   // ready (file, or cached dns)
+#if HDEBUG
+        printf("back_solve..\n");
+#endif
+        back[p].status = STATUS_WAIT_DNS;       // tentative de résolution du nom de host
+        soc = INVALID_SOCKET;   // pas encore ouverte
+        back_solve(opt, &back[p]);      // préparer
+        if (host_wait(opt, &back[p])) { // prêt, par ex fichier ou dispo dans dns
 #if HDEBUG
           printf("ok, dns cache ready..\n");
 #endif
@@ -2147,9 +2123,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
         back[p].rateout = -1;   // pas de gestion (default)
       }
 
-      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où
-      // cela est rattrapable (exemple: 301,302 moved xxx -> refresh sur la
-      // page!)
+      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est
+      // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
+      //if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
 
 #if CNXDEBUG
       printf("Xfopen ok, poll..\n");
@@ -2167,6 +2143,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
       if (soc == INVALID_SOCKET) {      // erreur socket
         back[p].status = STATUS_READY;  // FINI
         back_set_finished(sback, p);
+        //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
         back[p].r.soc = INVALID_SOCKET;
       } else {
         if (!back[p].r.is_file)
@@ -2187,6 +2164,8 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
     // note: si il y a erreur (404,etc) status=2 (terminé/échec) mais
     // le lien est considéré comme traité
+    //if (back[p].soc<0)  // erreur
+    //  return -1;
 
     return 0;
   } else {
@@ -2237,9 +2216,40 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 }
 
 #if HTS_XGETHOST
-// Resolution is synchronous inside the connect path; no pre-resolve step, so
-// the host is always immediately ready.
-int host_wait(httrackp *opt, lien_back *back) { return 1; }
+// attendre que le host (ou celui du proxy) ait été résolu
+// si c'est un fichier, la résolution est immédiate
+// idem pour ftp://
+void back_solve(httrackp * opt, lien_back * back) {
+  assertf(opt != NULL);
+  assertf(back != NULL);
+  if ((!strfield(back->url_adr, "file://"))
+      && !strfield(back->url_adr, "ftp://")
+    ) {
+    const char *a;
+
+    if (!(back->r.req.proxy.active))
+      a = back->url_adr;
+    else
+      a = back->r.req.proxy.name;
+    assertf(a != NULL);
+    a = jump_protocol_const(a);
+    if (check_hostname_dns(a)) {
+      hts_log_print(opt, LOG_DEBUG, "resolved: %s", a);
+    } else {
+      hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
+    }
+    //if (hts_dnstest(opt, a, 1) == 2) { // non encore testé!..
+    //  hts_log_print(opt, LOG_DEBUG, "resolving in background: %s", a);
+    //}
+  }
+}
+
+// détermine si le host a pu être résolu
+int host_wait(httrackp * opt, lien_back * back) {
+  // Always synchronous. No more background DNS resolution
+  // (does not really improve performances)
+  return 1;
+}
 #endif
 
 // élimine les fichiers non html en backing (anticipation)
@@ -2279,6 +2289,11 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) {
         (void) back_flush_output(opt, cache, sback, i); // flush output buffers
         usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr,
                     back[i].url_fil);
+        //if (back[i].links_index >= 0) {
+        //  assertf(back[i].links_index < opt->hash->max_lien);
+        //  opt->hash->liens[back[i].links_index]->pass2 = -1;
+        //  // *back[i].pass2_ptr=-1;  // Done!
+        //}
         /* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
         {
           int index = hash_read(opt->hash, back[i].url_sav, NULL, HASH_STRUCT_FILENAME );       // lecture type 0 (sav)
@@ -2426,15 +2441,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
-  /* Time/size limit exceeded past grace: abort in-flight transfers so no wait
-     loop starves (#481, #77). FTP slots stay, their thread owns the socket. */
+  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
+     starves (#481). FTP slots stay, their thread owns the socket. */
   if (!back_checkmirror(opt)) {
-    const hts_mirror_limit limit = back_mirror_limit(opt);
-    const char *const reason =
-        (limit == HTS_MIRROR_LIMIT_SIZE) ? "size limit" : "time limit";
-    const char *const slotmsg = (limit == HTS_MIRROR_LIMIT_SIZE)
-                                    ? "Mirror Size Limit"
-                                    : "Mirror Time Out";
     int aborted = 0;
     unsigned int i;
 
@@ -2448,15 +2457,15 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
           back_delayed_discard(opt, &back[i]);
         back[i].r.statuscode = STATUSCODE_TIMEOUT;
-        strcpybuff(back[i].r.msg, slotmsg);
+        strcpybuff(back[i].r.msg, "Mirror Time Out");
         back[i].status = STATUS_READY;
         back_set_finished(sback, i);
         aborted++;
       }
     }
     if (aborted > 0)
-      hts_log_print(opt, LOG_WARNING, "%s reached, %d transfer(s) aborted",
-                    reason, aborted);
+      hts_log_print(opt, LOG_WARNING,
+                    "time limit reached, %d transfer(s) aborted", aborted);
     return;
   }
 
@@ -2468,6 +2477,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
     busy_state = busy_recv = 0;
 
+#if 0
+    check_rate(stat_timestart, opt->maxrate);   // vérifier taux de transfert
+#endif
     // inscrire les sockets actuelles, et rechercher l'ID la plus élevée
     FD_ZERO(&fds);
     FD_ZERO(&fds_c);
@@ -2477,7 +2489,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
     nfds = INVALID_SOCKET;
 
     max_c = 1;
-    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+      // for(i=0;i<back_max;i++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       // en cas de gestion du connect préemptif
@@ -2514,7 +2527,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         // poll pour la lecture sur les sockets
       if ((back[i].status > 0) && (back[i].status < 100)) {     // en réception http
 
-#if BDEBUG == 1
+#if BDEBUG==1
+        //printf("....socket in progress: %d\n",back[i].r.soc);
 #endif
         // non local et non ftp
         if (!back[i].r.is_file) {
@@ -2600,7 +2614,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       busy_recv = 0;
 
     // recevoir les données arrivées
-    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+      // for(i=0;i<back_max;i++) {
       unsigned int i = (i_mod + mod_random) % (back_max);
 
       if (back[i].status > 0) {
@@ -2740,6 +2755,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back[i].rateout_time = back[i].ka_time_start;
             }
             // envoyer header
+            //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
             HTS_STAT.stat_nrequests++;
             if (!back[i].head_request)
               http_sendhead(opt, opt->cookie, 0, back[i].send_too,
@@ -2806,6 +2822,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 #if HTS_XGETHOST
       else if (back[i].status == STATUS_WAIT_DNS) {     // attendre gethostbyname
 #if DEBUGDNS
+        //printf("status 101 for %s\n",back[i].url_adr);
 #endif
 
         if (!gestion_timeout)
@@ -2946,8 +2963,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                 strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "gz") == 0
                                 && strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "tgz") == 0
                               ) {
-                              if (create_back_tmpfile(opt, &back[i], "z") ==
-                                  0) {
+                              if (create_back_tmpfile(opt, &back[i]) == 0) {
                                 assertf(back[i].tmpfile != NULL);
                                 /* note: tmpfile is utf-8 */
                                 if ((back[i].r.out =
@@ -2960,20 +2976,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
-                              /* Re-fetch over an existing file (#77 follow-up):
-                                 move the good copy aside before truncating it
-                                 so an aborted transfer can restore it. url_sav
-                                 is still written normally (file list intact).
-                               */
-                              back[i].tmpfile = NULL;
-                              if (fexist_utf8(back[i].url_sav)) {
-                                if (create_back_tmpfile(opt, &back[i], "bak") !=
-                                        0 ||
-                                    RENAME(back[i].url_sav, back[i].tmpfile) !=
-                                        0) {
-                                  back[i].tmpfile = NULL;
-                                }
-                              }
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {
@@ -3093,7 +3095,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
           // Si réception chunk, tester si on est pas à la fin!
           if (back[i].status == 1) {
             if (back[i].is_chunk) {     // attendre prochain chunk
-              if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
+              if (back[i].r.size == back[i].r.totalsize) {      // fin chunk!
+                //printf("chunk end at %d\n",back[i].r.size);
                 back[i].status = STATUS_CHUNK_CR;       /* fetch ending CRLF */
                 if (back[i].chunk_adr != NULL) {
                   freet(back[i].chunk_adr);
@@ -3156,9 +3159,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
               back_finalize(opt, cache, sback, i);
             }
 
-            if (back[i].r.totalsize >= 0) { // tester totalsize
+            if (back[i].r.totalsize >= 0) {     // tester totalsize
+              //if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) {    // tester totalsize
               if (back[i].r.totalsize != back[i].r.size) {      // pas la même!
                 if (!opt->tolerant) {
+                  //#if HTS_CL_IS_FATAL
                   deleteaddr(&back[i].r);
                   if (back[i].r.size < back[i].r.totalsize)
                     back[i].r.statuscode = STATUSCODE_CONNERROR;        // recatch
@@ -3167,13 +3172,14 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           " expected)", (LLint) back[i].r.size,
                           (LLint) back[i].r.totalsize);
                 } else {
+                  //#else
                   // Un warning suffira..
                   hts_log_print(opt, LOG_WARNING,
                                 "Incorrect length (" LLintP "!=" LLintP
-                                " expected) for %s%s",
-                                (LLint) back[i].r.size,
+                                " expected) for %s%s", (LLint) back[i].r.size,
                                 (LLint) back[i].r.totalsize, back[i].url_adr,
                                 back[i].url_fil);
+                  //#endif
                 }
               }
             }
@@ -3300,8 +3306,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                          back[i].r.size);
 #endif
                   /* End */
-                  back[i].status = STATUS_READY; // fin
+                  //if (back[i].status==STATUS_CHUNK_CR) {
+                  back[i].status = STATUS_READY;        // fin  
                   back_set_finished(sback, i);
+                  //}
 
                   // finalize transfer if not temporary
                   if (!IS_DELAYED_EXT(back[i].url_sav)) {
@@ -3512,11 +3520,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     continue;
                   }
 
-                  // The server really sent 304 here; the *-hacks below force
-                  // NOT_MODIFIED only after confirming the file is complete.
-                  const hts_boolean server_sent_304 =
-                      (back[i].r.statuscode == HTTP_NOT_MODIFIED);
-
                   /*
                      Solve "false" 416 problems
                    */
@@ -3545,20 +3548,22 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.statuscode == 406) {    // 'Not Acceptable'
                     back[i].r.statuscode = HTTP_OK;
                   }
-                  // On update, keep the good copy on error (mask as 304); skip
-                  // resume paths so a stale-partial 416 still re-fetches.
-                  if (HTTP_IS_ERROR(back[i].r.statuscode) &&
-                      back[i].is_update && !back[i].testmode &&
-                      back[i].range_req_size == 0 && back[i].url_sav[0] &&
-                      fexist_utf8(back[i].url_sav)) {
-                    hts_log_print(opt, LOG_DEBUG,
-                                  "Error %d (%s) ignored on update, keeping "
-                                  "existing copy: %s%s",
-                                  back[i].r.statuscode, back[i].r.msg,
-                                  back[i].url_adr, back[i].url_fil);
-                    back[i].r.statuscode = HTTP_NOT_MODIFIED;
-                    deletehttp(&back[i].r);
-                    back[i].r.soc = INVALID_SOCKET;
+                  // 'do not erase already downloaded file'
+                  // on an updated file
+                  // with an error : consider a 304 error
+                  if (!opt->delete_old) {
+                    if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
+                        && !back[i].testmode) {
+                      if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
+                        hts_log_print(opt, LOG_DEBUG,
+                                      "Error ignored %d (%s) because of 'no purge' option for %s%s",
+                                      back[i].r.statuscode, back[i].r.msg,
+                                      back[i].url_adr, back[i].url_fil);
+                        back[i].r.statuscode = HTTP_NOT_MODIFIED;
+                        deletehttp(&back[i].r);
+                        back[i].r.soc = INVALID_SOCKET;
+                      }
+                    }
                   }
                   // Various hacks to limit re-transfers when updating a mirror
                   // Force update if same size detected
@@ -3611,7 +3616,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) {       // 'OK'
                         if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) {  // not HTML
                           if (strnotempty(back[i].url_sav)) {   // target found
-                            off_t size = fsize_utf8(back[i].url_sav);
+                            int size = fsize_utf8(back[i].url_sav);     // target size
 
                             if (size >= 0) {
                               if (back[i].r.totalsize == size) {        // same size!
@@ -3706,21 +3711,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     }
                   }
 
-                  // Out-of-protocol 304 to a Range resume: the file is still
-                  // partial, so drop it and refetch instead of trusting it.
-                  if (server_sent_304 && back[i].range_req_size > 0) {
-                    url_savename_refname_remove(opt, back[i].url_adr,
-                                                back[i].url_fil);
-                    UNLINK(back[i].url_sav);
-                    deletehttp(&back[i].r);
-                    back[i].r.soc = INVALID_SOCKET;
-                    back[i].r.statuscode = STATUSCODE_NON_FATAL;
-                    strcpybuff(back[i].r.msg,
-                               "Bogus 304 on resume, restarting");
-                    back[i].status = STATUS_READY;
-                    back_set_finished(sback, i);
-                  }
-
                   /* sinon, continuer */
                   /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok récupérer body? */
                   // head: terminé
@@ -3786,6 +3776,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                     back[i].url_adr, back[i].url_fil);
 
                       // finalize
+                      //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);        // not modified
                       if (back[i].r.statuscode > 0) {
                         hts_log_print(opt, LOG_TRACE, "finalizing after cache load");
                         back_finalize(opt, cache, sback, i);
@@ -3795,11 +3786,37 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                              back[i].url_adr, back[i].url_fil);
 #endif
 
+                      //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                     } else {    // erreur
                       back[i].status = STATUS_READY;    // terminé
                       back_set_finished(sback, i);
+                      //printf("erreur cache\n");
+
                     }
 
+/********** NO - must complete the body! ********** */
+#if 0
+                  } else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
+                             || (back[i].r.statuscode == 412)
+                             || (back[i].r.statuscode == 416)
+                    ) {         // Ne pas prendre le html, erreurs connues et gérées
+#if HTS_DEBUG_CLOSESOCK
+                    DEBUG_W
+                      ("back_wait(301,302,303,307,412,416..): deletehttp\n");
+#endif
+                    // Couper connexion
+                    /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
+                    back_maydeletehttp(opt, cache, sback, i);
+
+                    back[i].status = STATUS_READY;      // terminé
+                    back_set_finished(sback, i);
+                    // finalize
+                    if (back[i].r.statuscode > 0) {
+                      hts_log_print(opt, LOG_TRACE, "finalizing redirect & 4xx");
+                      back_finalize(opt, cache, sback, i);
+                    }
+#endif
+/********** **************************** ********** */
                   }
                   // MIME type excluded by a -mime: filter: abort, don't fetch
                   // the body (#58)
@@ -3839,7 +3856,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       const hts_boolean range_ok =
                           back[i].r.crange > 0 && resume >= 0 &&
                           resume <= (LLint) sz &&
-                          back[i].r.crange_end == back[i].r.crange - 1 &&
+                          back[i].r.crange_end + 1 == back[i].r.crange &&
                           (back[i].r.totalsize < 0 ||
                            back[i].r.totalsize ==
                                back[i].r.crange_end - resume + 1);
@@ -3908,20 +3925,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           if (fp) {
                             LLint alloc_mem = resume + 1;
 
-                            // Bound the in-memory buffer to a 32-bit size (real
-                            // in-RAM resources are far smaller); a hostile
-                            // Content-Length that would overflow the add or the
-                            // (size_t) cast is dropped and refetched instead.
-                            if (back[i].r.totalsize > INT32_MAX - alloc_mem) {
-                              url_savename_refname_remove(opt, back[i].url_adr,
-                                                          back[i].url_fil);
-                              UNLINK(back[i].url_sav);
-                              alloc_mem = -1;
-                            } else if (back[i].r.totalsize >= 0)
+                            if (back[i].r.totalsize >= 0)
                               alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
-                            if (alloc_mem >= 0 && deleteaddr(&back[i].r) &&
-                                (back[i].r.adr =
-                                     (char *) malloct((size_t) alloc_mem))) {
+                            if (deleteaddr(&back[i].r)
+                                && (back[i].r.adr =
+                                    (char *) malloct((size_t) alloc_mem))) {
                               back[i].r.size = resume;
                               if (back[i].r.totalsize >= 0)
                                 back[i].r.totalsize += resume; // -> full size
@@ -3982,6 +3990,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           deletehttp(&back[i].r);
                         }
                         back[i].r.soc = INVALID_SOCKET;
+                        //back[i].r.statuscode=206;  ????????
                         back[i].r.statuscode = STATUSCODE_NON_FATAL;
                         if (strnotempty(back[i].r.msg))
                           strcpybuff(back[i].r.msg,
@@ -4008,7 +4017,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                             hts_log_print(opt, LOG_TRACE, "finalizing empty");
                             back_finalize(opt, cache, sback, i);
                           }
-                        } else if (!back[i].r.is_chunk) { // pas de chunk
+                        } else if (!back[i].r.is_chunk) {       // pas de chunk
+                          //if (back[i].r.http11!=2) {    // pas de chunk
                           back[i].is_chunk = 0;
                           back[i].status = 1;   // start body
                         } else {
@@ -4069,11 +4079,14 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].status < 0) {
           if (!back[i].testmode) {      // pas en test
             UNLINK(back[i].url_sav);    // éliminer fichier (endommagé)
+            //printf("&& %s\n",back[i].url_sav);
           }
         }
 #endif
 
         /* funny log for commandline users */
+        //if (!opt->quiet) {  
+        // petite animation
         if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
           if (back[i].status == STATUS_READY) {
             if (back[i].r.statuscode == HTTP_OK)
@@ -4086,16 +4099,18 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
             fflush(stdout);
           }
         }
+        //}
 
       }                         // status>0
-    } // for
+    }                           // for
 
     // vérifier timeouts
     if (gestion_timeout) {
       TStamp act;
 
       act = time_local();       // temps en secondes
-      for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+      for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
+        // for(i=0;i<back_max;i++) {
         unsigned int i = (i_mod + mod_random) % (back_max);
 
         if (back[i].status > 0) {       // réception/connexion/..
@@ -4123,6 +4138,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 continue;
               }
             }
+            //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
             if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) {
               hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr,
                 back[i].url_fil);
@@ -4203,46 +4219,38 @@ static int back_maxtime_grace(const int maxtime) {
   return maximum(5, minimum(30, maxtime / 10));
 }
 
-/* Bytes the smooth stop may overrun before in-flight transfers are aborted.
-   No floor (unlike the time grace): a size overrun should abort promptly. */
-static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
-
-/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481).
-   -M measures received volume (HTS_TOTAL_RECV), not saved 200-only stat_bytes
-   which undercounts redirect/error-heavy crawls (#520). */
-static hts_mirror_limit back_mirror_limit(httrackp *opt) {
-  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
-      HTS_STAT.HTS_TOTAL_RECV - opt->maxsite >=
-          back_maxsize_grace(opt->maxsite))
-    return HTS_MIRROR_LIMIT_SIZE;
+int back_checkmirror(httrackp * opt) {
+  // Check max size
+  if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
+    if (!opt->state.stop) {     /* not yet stopped */
+      hts_log_print(opt, LOG_ERROR,
+                    "More than " LLintP
+                    " bytes have been transferred.. giving up",
+                    (LLint) opt->maxsite);
+      /* cancel mirror smoothly */
+      hts_request_stop(opt, 0);
+    }
+    return 1;                   /* don'k break mirror too sharply for size limits, but stop requested */
+    /*return 0;
+     */
+  }
+  // Check max time
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
 
-    if (elapsed >= opt->maxtime &&
-        elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
-      return HTS_MIRROR_LIMIT_TIME;
+    if (elapsed >= opt->maxtime) {
+      if (!opt->state.stop) { /* not yet stopped */
+        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                      opt->maxtime);
+        /* cancel mirror smoothly */
+        hts_request_stop(opt, 0);
+      }
+      /* smooth stop starved past the grace period: stop waiting (#481) */
+      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+        return 0;
+    }
   }
-  return HTS_MIRROR_LIMIT_NONE;
-}
-
-int back_checkmirror(httrackp *opt) {
-  /* request a smooth stop the first time each cap is reached */
-  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
-      !opt->state.stop) {
-    hts_log_print(opt, LOG_ERROR,
-                  "More than " LLintP
-                  " bytes have been transferred.. giving up",
-                  (LLint) opt->maxsite);
-    hts_request_stop(opt, 0);
-  }
-  if (opt->maxtime > 0 && !opt->state.stop &&
-      (time_local() - HTS_STAT.stat_timestart) >= opt->maxtime) {
-    hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-                  opt->maxtime);
-    hts_request_stop(opt, 0);
-  }
-  /* hard stop once a cap overruns its grace (callers must stop waiting) */
-  return back_mirror_limit(opt) == HTS_MIRROR_LIMIT_NONE;
+  return 1;                     /* Ok, go on */
 }
 
 // octets transférés + add

src/htsback.h

@@ -113,12 +113,6 @@ void back_set_locked(struct_back * sback, const int p);
 void back_set_unlocked(struct_back * sback, const int p);
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p);
-/* Discard back's on-disk .delayed placeholder and its refname. */
-void back_delayed_discard(httrackp *opt, lien_back *back);
-/* Move back's .delayed placeholder (and open stream) to newname;
-   HTS_FALSE = file lost, slot flagged in error. */
-hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
-                                const char *newname);
 void back_index_unlock(struct_back * sback, const int p);
 int back_clear_entry(lien_back * back);
 int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
@@ -138,11 +132,12 @@ LLint back_transferred(LLint add, struct_back * sback);
 
 // hostback
 #if HTS_XGETHOST
+void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
-/* Enforce -M/-E quotas: smooth-stops when reached; returns 0 once the -M cap
-   or -E deadline overruns its grace period (callers must stop waiting). */
+/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
+   the -E deadline overran its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htsbasenet.h

@@ -55,6 +55,8 @@ Please visit our Website: http://www.httrack.com
 // inline function in Wspiapi.h.
 #include <ws2tcpip.h>
 #include <Wspiapi.h>
+// #include <winsock2.h>
+// #include <tpipv6.h>
 
 #endif
 

src/htsbauth.c

@@ -107,6 +107,7 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
 #if DEBUG_COOK
   printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
          cook_name, cook_value, domain, path);
+  //printf(">>>cook: %s<<<\n",cookie->data);
 #endif
   return 0;
 }
@@ -207,6 +208,8 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   char catbuff[CATBUFF_SIZE];
   char buffer[8192];
 
+  //  cookie->data[0]='\0';
+
   // Fusionner d'abord les éventuels cookies IE
 #ifdef _WIN32
   {
@@ -307,25 +310,14 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
   return -1;
 }
 
-/* Write cookies.txt; returns 0 on success. The jar holds live session
-   cookies, so keep it owner-only on Unix (Windows inherits folder ACLs). */
+// écrire cookies.txt
+// !=0 : erreur
 int cookie_save(t_cookie * cookie, const char *name) {
   char catbuff[CATBUFF_SIZE];
 
   if (strnotempty(cookie->data)) {
     char BIGSTK line[8192];
-#ifdef _WIN32
     FILE *fp = fopen(fconv(catbuff, sizeof(catbuff), name), "wb");
-#else
-    const int fd = open(fconv(catbuff, sizeof(catbuff), name),
-                        O_WRONLY | O_CREAT | O_TRUNC, HTS_PROTECT_FILE);
-    FILE *fp = fd != -1 ? fdopen(fd, "wb") : NULL;
-
-    if (fd != -1 && fp == NULL)
-      close(fd);    /* fdopen failed: don't leak the descriptor */
-    if (fp != NULL) /* O_CREAT's mode skips pre-existing jars: tighten those */
-      (void) fchmod(fd, HTS_PROTECT_FILE);
-#endif
 
     if (fp) {
       char *a = cookie->data;

src/htscache.h

@@ -89,11 +89,14 @@ typedef enum {
    the involved files are absent. */
 void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
 
+int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
+                    const char *str2, char *outbuff, int len);
+int cache_readdata(cache_back * cache, const char *str1, const char *str2,
+                   char **inbuff, int *len);
+
 void cache_rstr(FILE *fp, char *s, size_t s_size);
 char *cache_rstr_addr(FILE * fp);
 int cache_brstr(char *adr, char *s, size_t s_size);
-/* binput over a NUL-terminated buffer, bounded: no read starts at/past end. */
-int cache_binput(char *adr, const char *end, char *s, int max);
 int cache_brint(char *adr, int *i);
 void cache_rint(FILE * fp, int *i);
 void cache_rLLint(FILE * fp, LLint * i);

src/htscache_selftest.c

@@ -78,6 +78,14 @@ static void selftest_open_for_read(cache_back *cache, httrackp *opt) {
 }
 
 static void selftest_close(cache_back *cache) {
+  if (cache->dat != NULL) {
+    fclose(cache->dat);
+    cache->dat = NULL;
+  }
+  if (cache->ndx != NULL) {
+    fclose(cache->ndx);
+    cache->ndx = NULL;
+  }
   if (cache->zipOutput != NULL) {
     zipClose(cache->zipOutput,
              "Created by HTTrack Website Copier (cache self-test)");
@@ -981,15 +989,26 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
 
-  /* PROMOTE: the removed pre-3.31 legacy pair is left alone */
+  /* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
+     zip cache existed) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", TINY);
   hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "promote-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
+
+  /* PROMOTE: a half-written legacy new pair is replaced by the old pair */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
 
   /* INTERRUPTED: no lock file, no action */
   reconcile_wipe(opt);
@@ -1039,7 +1058,7 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures +=
       reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
 
-  /* INTERRUPTED: the removed pre-3.31 legacy pair is left alone */
+  /* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-in_progress.lock", 0);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
@@ -1048,13 +1067,9 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
   failures +=
-      reconcile_expect(opt, "hts-cache/new.dat", TINY, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
   failures +=
-      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.dat", SOLID, "interrupted-dat");
-  failures +=
-      reconcile_expect(opt, "hts-cache/old.ndx", MID, "interrupted-dat");
+      reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
 
   /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
      its only good generation here) */
@@ -1074,18 +1089,17 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
 
-  /* ROLLBACK: sidecars restored, the removed pre-3.31 pair left alone */
+  /* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
   reconcile_wipe(opt);
   reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
   reconcile_put(opt, "hts-cache/old.dat", SOLID);
   reconcile_put(opt, "hts-cache/old.ndx", MID);
   reconcile_put(opt, "hts-cache/old.lst", MID);
   reconcile_put(opt, "hts-cache/old.txt", MID);
   hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
-  failures += reconcile_expect(opt, "hts-cache/new.dat", TINY, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/new.ndx", -1, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.dat", SOLID, "rollback-dat");
-  failures += reconcile_expect(opt, "hts-cache/old.ndx", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
   failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
 
@@ -1098,386 +1112,3 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
   reconcile_wipe(opt);
   return failures;
 }
-
-/* The pre-3.31 .dat/.ndx import was removed: cache_init must refuse the
-   legacy pair, leave the files in place, and not flag an update run. */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-  int variant;
-
-  selftest_tag = "cache-legacy";
-  golden_setup(opt, dir);
-#ifdef _WIN32
-  mkdir(reconcile_st_path(opt, "hts-cache"));
-#else
-  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
-#endif
-
-  /* variant 0: old.dat/old.ndx (--update layout); 1: new.dat/new.ndx (ro) */
-  for (variant = 0; variant < 2; variant++) {
-    cache_back cache;
-    const char *const dat =
-        variant == 0 ? "hts-cache/old.dat" : "hts-cache/new.dat";
-    const char *const ndx =
-        variant == 0 ? "hts-cache/old.ndx" : "hts-cache/new.ndx";
-
-    reconcile_wipe(opt);
-    reconcile_put(opt, dat, 4096);
-    reconcile_put(opt, ndx, 4096);
-    opt->is_update = 0;
-    selftest_open(&cache, opt, /*ro*/ variant == 1);
-    if (cache_readable(&cache)) {
-      printf("cache-legacy: variant %d imported a removed format\n", variant);
-      failures++;
-    }
-    if (coucal_nitems(cache.hashtable) != 0) { /* no phantom index entries */
-      printf("cache-legacy: variant %d imported index entries\n", variant);
-      failures++;
-    }
-    if (opt->is_update) {
-      printf("cache-legacy: variant %d flagged an update\n", variant);
-      failures++;
-    }
-    if (fsize(reconcile_st_path(opt, dat)) != 4096 ||
-        fsize(reconcile_st_path(opt, ndx)) != 4096) {
-      printf("cache-legacy: variant %d touched the legacy files\n", variant);
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  /* a healthy zip must win over stray legacy files, with no refusal */
-  {
-    cache_back cache;
-    const char *const body = "<html>zip wins</html>";
-
-    reconcile_wipe(opt);
-    selftest_open_for_write(&cache, opt);
-    store_entry(opt, &cache, "example.com", "/", "example.com/index.html", 200,
-                "OK", "text/html", "utf-8", "", "", "", "", body, strlen(body));
-    selftest_close(&cache);
-    reconcile_put(opt, "hts-cache/old.ndx", 4096);
-    reconcile_put(opt, "hts-cache/old.dat", 4096);
-    selftest_open_for_read(&cache, opt);
-    if (!cache_readable(&cache)) {
-      printf("cache-legacy: stray legacy files shadowed the zip cache\n");
-      failures++;
-    }
-    failures +=
-        check_entry(opt, &cache, "example.com", "/", 200, "OK", "text/html",
-                    "utf-8", "", "", "", "", body, strlen(body));
-    if (fsize(reconcile_st_path(opt, "hts-cache/old.ndx")) != 4096 ||
-        fsize(reconcile_st_path(opt, "hts-cache/old.dat")) != 4096) {
-      printf("cache-legacy: zip-wins pass touched the legacy files\n");
-      failures++;
-    }
-    if (cache.lst != NULL) {
-      fclose(cache.lst);
-      cache.lst = NULL;
-    }
-    if (cache.txt != NULL) {
-      fclose(cache.txt);
-      cache.txt = NULL;
-    }
-    selftest_close(&cache);
-  }
-
-  reconcile_wipe(opt);
-  return failures;
-}
-
-/* --- read-side corruption injection --------------------------------------- */
-
-/* canary read back intact after each corruption; victim gets the byte surgery
- */
-#define CORRUPT_ADR "corrupt.example.com"
-static char corrupt_body_a[33 + 1];
-static char corrupt_body_b[44 + 1];
-
-/* Write a fresh two-entry cache: /canary.html then /victim.html. */
-static void corrupt_build(httrackp *opt) {
-  cache_back cache;
-
-  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
-  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
-  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
-  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  selftest_open_for_write(&cache, opt);
-  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
-              strlen(corrupt_body_a));
-  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
-              strlen(corrupt_body_b));
-  selftest_close(&cache);
-}
-
-/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
-   is later overwritten with a forged oversized X-Size (same byte length). */
-static void corrupt_build_etag(httrackp *opt) {
-  cache_back cache;
-
-  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
-  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
-  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
-  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  selftest_open_for_write(&cache, opt);
-  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
-              strlen(corrupt_body_a));
-  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
-              "OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
-              corrupt_body_b, strlen(corrupt_body_b));
-  selftest_close(&cache);
-}
-
-/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
-   body on disk): the shape every non-html file is stored with. */
-static void corrupt_build_disk(httrackp *opt) {
-  cache_back cache;
-  htsblk w;
-  char locw[4];
-  char BIGSTK save[HTS_URLMAXSIZE * 2];
-  char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
-  char *path;
-  FILE *fp;
-
-  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
-  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
-  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
-          CORRUPT_ADR "/victim.bin");
-  selftest_open_for_write(&cache, opt);
-  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
-              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
-              strlen(corrupt_body_a));
-  hts_init_htsblk(&w);
-  w.statuscode = 200;
-  w.size = (LLint) sizeof(corrupt_body_b) - 1;
-  strcpybuff(w.msg, "OK");
-  strcpybuff(w.contenttype, "application/octet-stream");
-  strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
-  locw[0] = '\0';
-  w.location = locw;
-  w.is_write = 0;
-  cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
-            0 /* all_in_cache */, NULL);
-  selftest_close(&cache);
-  /* the reader only checks this file exists; it never reads it here */
-  path = fconv(catbuff, sizeof(catbuff), save);
-  (void) structcheck(path);
-  fp = FOPEN(path, "wb");
-  assertf(fp != NULL);
-  fclose(fp);
-}
-
-/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
-static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
-                          const char *rep, size_t nth, size_t total) {
-  LLint fsz = 0;
-  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
-  const size_t n = (size_t) fsz;
-  size_t k, hits = 0, at = 0;
-  FILE *fp;
-
-  assertf(data != NULL);
-  for (k = 0; k + patlen <= n; k++) {
-    if (memcmp(data + k, pat, patlen) == 0) {
-      hits++;
-      if (hits == nth)
-        at = k;
-    }
-  }
-  assertf(hits == total);
-  memcpy(data + at, rep, patlen);
-  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
-  assertf(fp != NULL);
-  assertf(fwrite(data, 1, n, fp) == n);
-  fclose(fp);
-  freet(data);
-}
-
-/* Garbage the first bytes of the victim's deflated data (2nd local header). */
-static void corrupt_victim_body(httrackp *opt) {
-  LLint fsz = 0;
-  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
-  const size_t n = (size_t) fsz;
-  size_t k, hits = 0, off = 0;
-  FILE *fp;
-
-  assertf(data != NULL);
-  for (k = 0; k + 4 <= n; k++) {
-    if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
-      const size_t namelen =
-          (unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
-      const size_t extralen =
-          (unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
-
-      off = k + 30 + namelen + extralen;
-    }
-  }
-  assertf(hits == 2);
-  assertf(off != 0 && off + 4 <= n);
-  memset(data + off, 0xFF, 4);
-  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
-  assertf(fp != NULL);
-  assertf(fwrite(data, 1, n, fp) == n);
-  fclose(fp);
-  freet(data);
-}
-
-/* Read the corrupt /victim.html and, in the SAME read session, the intact
-   /canary.html: the victim must be rejected (wantmsg pins which path) and the
-   canary must still decode byte-exact, proving one bad entry never taints a
-   sibling read. */
-static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
-                                     const char *wantmsg, const char *what) {
-  cache_back cache;
-  htsblk v, c;
-  char BIGSTK lv[HTS_URLMAXSIZE * 2];
-  char BIGSTK lc[HTS_URLMAXSIZE * 2];
-  int fail = 0;
-
-  selftest_open_for_read(&cache, opt);
-  lv[0] = lc[0] = '\0';
-  v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
-  if (v.statuscode != STATUSCODE_INVALID) {
-    fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
-            selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
-    fail++;
-  }
-  if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
-    fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
-            selftest_tag, what, v.msg, wantmsg);
-    fail++;
-  }
-  c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
-  if (c.statuscode != 200 || c.adr == NULL ||
-      c.size != (LLint) strlen(corrupt_body_a) ||
-      memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
-    fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
-            c.statuscode);
-    fail++;
-  }
-  if (v.adr != NULL)
-    freet(v.adr);
-  if (c.adr != NULL)
-    freet(c.adr);
-  selftest_close(&cache);
-  return fail;
-}
-
-static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
-                                 const char *what) {
-  return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
-}
-
-/* Headers-only probe of the disk victim: must parse OK with the size kept. */
-static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
-                                      const char *what) {
-  cache_back cache;
-  htsblk v;
-  char BIGSTK lv[HTS_URLMAXSIZE * 2];
-  int fail = 0;
-
-  selftest_open_for_read(&cache, opt);
-  lv[0] = '\0';
-  v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
-  if (v.statuscode != 200 || v.size != wantsize) {
-    fprintf(stderr,
-            "%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
-            selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
-    fail++;
-  }
-  if (v.adr != NULL)
-    freet(v.adr);
-  selftest_close(&cache);
-  return fail;
-}
-
-/* One zip corruption case: build, patch, then check victim+canary in-session.
- */
-static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
-                            size_t nth, size_t total, const char *wantmsg,
-                            const char *what) {
-  corrupt_build(opt);
-  corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
-  return corrupt_expect_victim(opt, wantmsg, what);
-}
-
-int cache_corruption_selftest(httrackp *opt, const char *dir) {
-  int failures = 0;
-
-  selftest_tag = "cache-corrupt";
-  golden_setup(opt, dir);
-
-  failures +=
-      corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
-                       "Cache Read Error : Read Data", "oversized X-Size");
-  failures +=
-      corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
-                       "Cache Read Error : Bad Size", "negative X-Size");
-  /* both entries carry the line; the victim's is the second */
-  failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
-                               "Previous cache file not found (empty filename)",
-                               "blanked X-In-Cache");
-  /* smashed local file header: the entry is dropped at index load */
-  failures +=
-      corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
-                       "File Cache Entry Not Found", "smashed local header");
-
-  corrupt_build(opt);
-  corrupt_victim_body(opt);
-  failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
-                                    "garbled deflate stream");
-
-  /* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
-     truncates negative in the (int) cast the malloc uses: a wraparound alloc.
-     cache_add asserts size fits an int, so such a value only reaches the reader
-     from a corrupt/foreign cache; inject it by overwriting the victim's long
-     Etag line with a same-length forged X-Size line (the parser keeps the last
-     X-Size it sees), keeping the zip byte-length and offsets intact. */
-  corrupt_build_etag(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: 2147483648AAAAAAAA", 1, 1);
-  failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
-                                    "X-Size above INT_MAX");
-
-  /* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
-     is how every >2GB non-html file is stored. It must survive a header probe
-     (or every update re-fetches the file); an in-memory read still rejects. */
-  corrupt_build_disk(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: 2147483648AAAAAAAA", 1, 1);
-  failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
-                                         "headers-only X-Size above INT_MAX");
-  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
-                                        "Cache Read Error : Bad Size",
-                                        "in-memory X-Size above INT_MAX");
-
-  /* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
-  corrupt_build_disk(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: 2147483647AAAAAAAA", 1, 1);
-  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
-                                        "Cache Read Error : Bad Size",
-                                        "in-memory X-Size at INT_MAX");
-
-  /* the negative check must stay global, headers-only included */
-  corrupt_build_disk(opt);
-  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
-                "X-Size: -2147483648AAAAAAA", 1, 1);
-  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
-                                        "Cache Read Error : Bad Size",
-                                        "headers-only negative X-Size");
-
-  return failures;
-}

src/htscache_selftest.h

@@ -60,15 +60,6 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir);
    under <dir>. Returns the failed-check count. */
 int cache_reconcile_selftest(httrackp *opt, const char *dir);
 
-/* Verify cache_init refuses a pre-3.31 .dat/.ndx cache without touching it.
-   Returns the number of failed checks (0 = pass). */
-int cache_legacy_refused_selftest(httrackp *opt, const char *dir);
-
-/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
-   under <dir> and assert every case degrades to STATUSCODE_INVALID without
-   tainting a sibling entry. */
-int cache_corruption_selftest(httrackp *opt, const char *dir);
-
 #endif
 
 #endif

src/htscatchurl.c

@@ -197,6 +197,7 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
             htsblk blkretour;
 
             hts_init_htsblk(&blkretour);
+            //memset(&blkretour, 0, sizeof(htsblk));    // effacer
             blkretour.location = loc;   // si non nul, contiendra l'adresse véritable en cas de moved xx
             // Lire en têtes restants
             sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
@@ -206,8 +207,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
               strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
               strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
             }
-            // CR/LF final de l'en tête inutile car déja placé via la ligne vide
-            // juste au dessus
+            // CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
+            //strcatbuff(data,"\r\n");
             if (blkretour.totalsize > 0) {
               int len = (int) min(blkretour.totalsize, 32000);
               int pos = (int) strlen(data);

src/htscharset.c

@@ -84,7 +84,7 @@ static int hts_equalsAlphanum(const char *a, const char *b) {
 
 /* Copy the memory region [s .. s + size - 1 ] as a \0-terminated string. */
 static char *hts_stringMemCopy(const char *s, size_t size) {
-  char *dest = malloct(size + 1);
+  char *dest = malloc(size + 1);
 
   if (dest != NULL) {
     memcpy(dest, s, size);
@@ -549,6 +549,42 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
 
 #endif
 
+HTS_STATIC char *hts_getCharsetFromContentType(const char *mime) {
+  /* text/html; charset=utf-8 */
+  const char *const charset = "charset";
+  char *pos = strstr(mime, charset);
+
+  if (pos != NULL) {
+    /* Skip spaces */
+    int eq = 0;
+
+    for(pos += strlen(charset);
+        *pos == ' ' || *pos == '=' || *pos == '"' || *pos == '\''; pos++) {
+      if (*pos == '=') {
+        eq = 1;
+      }
+    }
+    if (eq == 1) {
+      int len;
+
+      for(len = 0;
+          pos[len] == ' ' || pos[len] == ';' || pos[len] == '"' || *pos == '\'';
+          pos++) ;
+      if (len != 0) {
+        char *const s = malloc(len + 1);
+        int i;
+
+        for(i = 0; i < len; i++) {
+          s[i] = pos[i];
+        }
+        s[len] = '\0';
+        return s;
+      }
+    }
+  }
+  return NULL;
+}
+
 #ifdef _WIN32
 #define strcasecmp(a,b) stricmp(a,b)
 #define strncasecmp(a,b,n) strnicmp(a,b,n)
@@ -608,106 +644,58 @@ int hts_isCharsetUTF8(const char *charset) {
          || strcasecmp(charset, "utf8") == 0 );
 }
 
-/* Extract X from a "text/html; charset=X" content= attribute value. */
-static char *charset_from_content(const char *s, size_t len) {
-  size_t i;
-
-  for (i = 0; i + 7 < len; i++) {
-    if ((i == 0 || is_space(s[i - 1]) || s[i - 1] == ';') &&
-        strncasecmp(&s[i], "charset", 7) == 0 && is_space_or_equal(s[i + 7])) {
-      size_t j, val;
-
-      for (j = i + 7; j < len && is_space_or_equal_or_quote(s[j]); j++)
-        ;
-      for (val = j; j < len && s[j] != '"' && s[j] != '\'' && s[j] != ';' &&
-                    !is_space(s[j]);
-           j++)
-        ;
-      if (j != val) {
-        return hts_stringMemCopy(&s[val], j - val);
-      }
-    }
-  }
-  return NULL;
-}
-
 char *hts_getCharsetFromMeta(const char *html, size_t size) {
-  size_t i;
+  int i;
 
-  /* HTML5 <meta charset=X> or legacy
-     <meta http-equiv="Content-Type" content="text/html; charset=X">,
-     attributes in any order; first resolvable tag wins. */
-  for (i = 0; i + 6 < size; i++) {
-    size_t j;
-    const char *equiv = NULL, *content = NULL;
-    size_t equiv_len = 0, content_len = 0;
+  /* <META HTTP-EQUIV="CONTENT-TYPE" CONTENT="text/html; charset=utf-8" > */
+  for(i = 0; i < size; i++) {
+    if (html[i] == '<' && strncasecmp(&html[i + 1], "meta", 4) == 0
+        && is_space(html[i + 5])) {
+      /* Skip spaces */
+      for(i += 5; is_space(html[i]); i++) ;
+      if (strncasecmp(&html[i], "HTTP-EQUIV", 10) == 0
+          && is_space_or_equal(html[i + 10])) {
+        for(i += 10; is_space_or_equal_or_quote(html[i]); i++) ;
+        if (strncasecmp(&html[i], "CONTENT-TYPE", 12) == 0) {
+          for(i += 12; is_space_or_equal_or_quote(html[i]); i++) ;
+          if (strncasecmp(&html[i], "CONTENT", 7) == 0
+              && is_space_or_equal(html[i + 7])) {
+            for(i += 7; is_space_or_equal_or_quote(html[i]); i++) ;
+            /* Skip content-type */
+            for(;
+                i < size && html[i] != ';' && html[i] != '"' && html[i] != '\'';
+                i++) ;
+            /* Expect charset attribute here */
+            if (html[i] == ';') {
+              for(i++; is_space(html[i]); i++) ;
+              /* Look for charset */
+              if (strncasecmp(&html[i], "charset", 7) == 0
+                  && is_space_or_equal(html[i + 7])) {
+                int len;
 
-    if (html[i] != '<' || strncasecmp(&html[i + 1], "meta", 4) != 0 ||
-        !is_space(html[i + 5])) {
-      continue;
-    }
-    /* Attribute scan, strictly bounded by size. */
-    for (j = i + 6; j < size && html[j] != '>';) {
-      size_t name, name_len, val = 0, val_len = 0;
+                for(i += 7; is_space_or_equal(html[i]) || html[i] == '\'';
+                    i++) ;
+                /* Charset */
+                for(len = 0;
+                    i + len < size && html[i + len] != '"'
+                    && html[i + len] != '\'' && html[i + len] != ' '; len++) ;
+                /* No error ? */
+                if (len != 0 && i < size) {
+                  char *const s = malloc(len + 1);
+                  int j;
 
-      if (is_space(html[j]) || html[j] == '/') {
-        j++;
-        continue;
-      }
-      for (name = j; j < size && html[j] != '=' && html[j] != '>' &&
-                     html[j] != '/' && !is_space(html[j]);
-           j++)
-        ;
-      name_len = j - name;
-      for (; j < size && is_space(html[j]); j++)
-        ;
-      if (j < size && html[j] == '=') {
-        for (j++; j < size && is_space(html[j]); j++)
-          ;
-        if (j < size && (html[j] == '"' || html[j] == '\'')) {
-          const char quote = html[j++];
-
-          for (val = j; j < size && html[j] != quote; j++)
-            ;
-          if (j >= size) /* unterminated quote: drop the tag */
-            break;
-          val_len = j++ - val;
-        } else {
-          /* unquoted: ends at whitespace or '>' only (HTML5 prescan); '/'
-             belongs to the value except as the tail of a self-close */
-          for (val = j; j < size && !is_space(html[j]) && html[j] != '>'; j++)
-            ;
-          val_len = j - val;
-          if (val_len != 0 && j < size && html[j] == '>' &&
-              html[val + val_len - 1] == '/') {
-            val_len--;
+                  for(j = 0; j < len; j++) {
+                    s[j] = html[i + j];
+                  }
+                  s[len] = '\0';
+                  return s;
+                }
+              }
+            }
           }
         }
       }
-      /* first occurrence of each attribute wins, as in browsers */
-      if (val_len != 0) {
-        if (name_len == 7 && strncasecmp(&html[name], "charset", 7) == 0) {
-          return hts_stringMemCopy(&html[val], val_len);
-        } else if (equiv == NULL && name_len == 10 &&
-                   strncasecmp(&html[name], "http-equiv", 10) == 0) {
-          equiv = &html[val];
-          equiv_len = val_len;
-        } else if (content == NULL && name_len == 7 &&
-                   strncasecmp(&html[name], "content", 7) == 0) {
-          content = &html[val];
-          content_len = val_len;
-        }
-      }
     }
-    if (equiv_len == 12 && strncasecmp(equiv, "content-type", 12) == 0 &&
-        content != NULL) {
-      char *const s = charset_from_content(content, content_len);
-
-      if (s != NULL) {
-        return s;
-      }
-    }
-    i = j;
   }
   return NULL;
 }
@@ -853,6 +841,71 @@ static unsigned int nlz8(unsigned char x) {
     }                                           \
   } while(0)
 
+/* Sample. */
+#if 0
+
+int main(int argc, char **argv) {
+  int i;
+  int hex = 0;
+  
+#define READ_INT(DEST)                                  \
+  ( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1)      \
+    || (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
+  
+  for(i = 1 ; i < argc ; i++) {
+    unsigned int uc, from, to;
+    
+    if (strcmp(argv[i], "--hex") == 0) {
+      hex = 1;
+    }
+    else if (strcmp(argv[i], "--decimal") == 0) {
+      hex = 0;
+    }
+    else if (strcmp(argv[i], "--decode") == 0) {
+#define RD fgetc_unlocked(stdin)
+#define WR(C) do {                              \
+        if (C != -1) {                          \
+          printf("%04x\n", C);                  \
+        } else if (!feof(stdin)) {              \
+          fprintf(stderr, "read error\n");      \
+          exit(EXIT_FAILURE);                   \
+        }                                       \
+      } while(0)
+      while(!feof(stdin)) {
+        READ_UNICODE(RD, WR);
+      }
+#undef RD
+#undef WR
+    }
+    else if (strcmp(argv[i], "-range") == 0
+             && i + 2 < argc
+             && (++i, 1)
+             && READ_INT(from)
+             && (++i, 1)
+             && READ_INT(to)
+             ) {
+      unsigned int i;
+      for(i = from ; i < to ; i++) {
+#define EM(C) fputc_unlocked(C, stdout)
+        EMIT_UNICODE(i, EM);
+#undef EM
+      }
+    }
+    else if (READ_INT(uc)) {
+#define EM(C) fputc_unlocked(C, stdout)
+      EMIT_UNICODE(uc, EM);
+#undef EM
+    }
+    else {
+      return EXIT_FAILURE;
+    }
+  }
+  
+  return EXIT_SUCCESS;
+}
+
+#endif
+
 /* IDNA helpers. */
 #undef ADD_BYTE
 #undef INCREASE_CAPA
@@ -922,19 +975,19 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
                 /* Reader: can read bytes up to j */
 #define RD ( utfSeq < j ? segData[utfSeq++] : -1 )
 
-                /* Writer: a malformed sequence abandons the encode (NULL) */
-#define WR(C)                                                                  \
-  do {                                                                         \
-    if ((C) != -1) {                                                           \
-      /* copy character */                                                     \
-      assertf(segOutputSize < segSize);                                        \
-      segInt[segOutputSize++] = (C);                                           \
-    } else {                                                                   \
-      freet(segInt);                                                           \
-      FREE_BUFFER();                                                           \
-      return NULL;                                                             \
-    }                                                                          \
-  } while (0)
+                /* Writer: upon error, return FFFD (replacement character) */
+#define WR(C) do { \
+  if ((C) != -1) { \
+    /* copy character */ \
+    assertf(segOutputSize < segSize); \
+    segInt[segOutputSize++] = (C); \
+  } \
+  /* In case of error, abort. */ \
+  else { \
+    FREE_BUFFER(); \
+    return NULL; \
+  } \
+} while(0)
 
                 /* Read/Write Unicode character. */
                 READ_UNICODE(RD, WR);
@@ -1152,45 +1205,17 @@ int hts_isStringUTF8(const char *s, size_t size) {
   const unsigned char *const data = (const unsigned char*) s;
   size_t i;
 
-  /* Strict RFC 3629: reject overlongs, surrogates, > U+10FFFF, 5/6-byte. */
   for(i = 0 ; i < size ; ) {
-    const unsigned char c = data[i];
-    size_t len, k;
-    hts_UCS4 uc, min;
+    /* Reader: can read bytes up to j */
+#define RD ( i < size ? data[i++] : -1 )
 
-    if (c < 0x80) {
-      i++;
-      continue;
-    } else if (c >= 0xc2 && c <= 0xdf) {
-      len = 2;
-      min = 0x80;
-      uc = c & 0x1f;
-    } else if (c >= 0xe0 && c <= 0xef) {
-      len = 3;
-      min = 0x800;
-      uc = c & 0x0f;
-    } else if (c >= 0xf0 && c <= 0xf4) {
-      len = 4;
-      min = 0x10000;
-      uc = c & 0x07;
-    } else { /* continuation byte, overlong C0/C1, or F5..FF lead */
-      return 0;
-    }
-    if (size - i < len) {
-      return 0;
-    }
-    for (k = 1; k < len; k++) {
-      const unsigned char cc = data[i + k];
+    /* Writer: upon error, return FFFD (replacement character) */
+#define WR(C) if ((C) == -1) { return 0; }
 
-      if ((cc & 0xc0) != 0x80) {
-        return 0;
-      }
-      uc = (uc << 6) | (cc & 0x3f);
-    }
-    if (uc < min || uc > 0x10FFFF || (uc >= 0xD800 && uc <= 0xDFFF)) {
-      return 0;
-    }
-    i += len;
+    /* Read Unicode character. */
+    READ_UNICODE(RD, WR);
+#undef RD
+#undef WR
   }
 
   return 1;

src/htscharset.h

@@ -76,8 +76,7 @@ extern char *hts_convertStringIDNAToUTF8(const char *s, size_t size);
 extern int hts_isStringIDNA(const char *s, size_t size);
 
 /**
- * Extract the <meta> charset from the HTML buffer "html" (HTML5 charset= or
- * legacy http-equiv form). Returns a malloc'ed string, or NULL if none.
+ * Extract the charset from the HTML buffer "html"
  **/
 extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 
@@ -87,8 +86,7 @@ extern char *hts_getCharsetFromMeta(const char *html, size_t size);
 extern int hts_isStringAscii(const char *s, size_t size);
 
 /**
- * Is the given string valid UTF-8 ? Strict RFC 3629: overlong forms,
- * surrogates, codepoints above U+10FFFF and 5/6-byte sequences are rejected.
+ * Is the given string an UTF-8 string ?
  **/
 extern int hts_isStringUTF8(const char *s, size_t size);
 

src/htsconfig.h

@@ -54,6 +54,10 @@ Please visit our Website: http://www.httrack.com
 // extension par défaut pour fichiers n'en ayant pas
 #define DEFAULT_EXT ".html"
 #define DEFAULT_EXT_SHORT ".htm"
+// #define DEFAULT_BIN_EXT   ".bin"
+// #define DEFAULT_BIN_EXT_SHORT ".bin"
+// #define DEFAULT_EXT       ".txt"
+// #define DEFAULT_EXT_SHORT ".txt"
 
 // éviter les /nul, /con..
 #define HTS_OVERRIDE_DOS_FOLDERS 1
@@ -76,6 +80,13 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1
 
+// gérer une table de hachage?
+// REMOVED
+// #define HTS_HASH 1
+
+// fast cache (build hash table)
+#define HTS_FAST_CACHE 1
+
 // le > peut être considéré comme un tag de fermeture de commentaire (<!-- > est
 // valide)
 #define GT_ENDS_COMMENT 1
@@ -86,10 +97,27 @@ Please visit our Website: http://www.httrack.com
 // always transform a '//' into a sigle '/'
 #define HTS_STRIP_DOUBLE_SLASH 0
 
+// case-sensitive pour les dossiers et fichiers (0/1)
+// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est
+// pas très utile..
+// ..et pas bcp respecté]
+// REMOVED
+// #define HTS_CASSE 0
+
+// Un fichier ayant une taille différente du content-length doit il être annulé?
+// SEE opt.tolerant and opt.http10
+// #define HTS_CL_IS_FATAL 0
+
 // une erreur supprime le fichier sur disque
 // (non fixé pour cause de retry)
 #define HTS_REMOVE_BAD_FILES 0
 
+// en cas de Range: xx- donnant un Content-length: xx
+// alors skipper le fichier, considéré comme transmis
+// #define HTS_SKIP_FULL_RANGE 1
+
+// nombre max de filtres que l'utilisateur peut fixer
+// #define HTS_FILTERSMAX 10000
 #define HTS_FILTERSINC 1000
 
 // connect non bloquant? (poll sur write)

src/htscore.c

@@ -119,77 +119,46 @@ hts_log_print(opt, LOG_INFO, "engine: end"); \
 RUN_CALLBACK0(opt, end); \
 }
 
-#define XH_extuninit                                                           \
-  do {                                                                         \
-    HTMLCHECK_UNINIT                                                           \
-    hts_record_free(opt);                                                      \
-    if (filters && filters[0]) {                                               \
-      freet(filters[0]);                                                       \
-      filters[0] = NULL;                                                       \
-    }                                                                          \
-    if (filters) {                                                             \
-      freet(filters);                                                          \
-      filters = NULL;                                                          \
-    }                                                                          \
-    back_delete_all(opt, &cache, sback);                                       \
-    back_free(&sback);                                                         \
-    checkrobots_free(&robots);                                                 \
-    if (cache.zipOutput) {                                                     \
-      zipClose(cache.zipOutput,                                                \
-               "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \
-      cache.zipOutput = NULL;                                                  \
-    }                                                                          \
-    if (cache.zipInput) {                                                      \
-      unzClose(cache.zipInput);                                                \
-      cache.zipInput = NULL;                                                   \
-    }                                                                          \
-    if (cache.lst) {                                                           \
-      fclose(cache.lst);                                                       \
-      cache.lst = opt->state.strc.lst = NULL;                                  \
-    }                                                                          \
-    if (cache.txt) {                                                           \
-      fclose(cache.txt);                                                       \
-      cache.txt = NULL;                                                        \
-    }                                                                          \
-    if (opt->log != NULL)                                                      \
-      fflush(opt->log);                                                        \
-    if (makestat_fp) {                                                         \
-      fclose(makestat_fp);                                                     \
-      makestat_fp = NULL;                                                      \
-    }                                                                          \
-    if (maketrack_fp) {                                                        \
-      fclose(maketrack_fp);                                                    \
-      maketrack_fp = NULL;                                                     \
-    }                                                                          \
-    if (opt->accept_cookie)                                                    \
-      cookie_save(opt->cookie,                                                 \
-                  fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),           \
-                          StringBuff(opt->path_log), "cookies.txt"));          \
-    if (makeindex_fp) {                                                        \
-      fclose(makeindex_fp);                                                    \
-      makeindex_fp = NULL;                                                     \
-    }                                                                          \
-    if (cache_hashtable) {                                                     \
-      coucal_delete(&cache_hashtable);                                         \
-    }                                                                          \
-    if (cache_tests) {                                                         \
-      coucal_delete(&cache_tests);                                             \
-    }                                                                          \
-    if (template_header) {                                                     \
-      freet(template_header);                                                  \
-      template_header = NULL;                                                  \
-    }                                                                          \
-    if (template_body) {                                                       \
-      freet(template_body);                                                    \
-      template_body = NULL;                                                    \
-    }                                                                          \
-    if (template_footer) {                                                     \
-      freet(template_footer);                                                  \
-      template_footer = NULL;                                                  \
-    }                                                                          \
-    hash_free(&hash);                                                          \
-    clearCallbacks(&opt->state.callbacks);                                     \
-  } while (0)
+#define XH_extuninit do { \
+  HTMLCHECK_UNINIT \
+  hts_record_free(opt); \
+  if (filters && filters[0]) { \
+  freet(filters[0]); filters[0]=NULL; \
+  } \
+  if (filters) { \
+  freet(filters); filters=NULL; \
+  } \
+  back_delete_all(opt,&cache,sback); \
+  back_free(&sback); \
+  checkrobots_free(&robots);\
+  if (cache.use) { freet(cache.use); cache.use=NULL; } \
+  if (cache.dat) { fclose(cache.dat); cache.dat=NULL; }  \
+  if (cache.ndx) { fclose(cache.ndx); cache.ndx=NULL; } \
+  if (cache.zipOutput) { \
+    zipClose(cache.zipOutput, "Created by HTTrack Website Copier/"HTTRACK_VERSION); \
+    cache.zipOutput = NULL; \
+  } \
+  if (cache.zipInput) { \
+    unzClose(cache.zipInput); \
+    cache.zipInput = NULL; \
+  } \
+  if (cache.olddat) { fclose(cache.olddat); cache.olddat=NULL; } \
+  if (cache.lst) { fclose(cache.lst); cache.lst=opt->state.strc.lst=NULL; } \
+  if (cache.txt) { fclose(cache.txt); cache.txt=NULL; } \
+  if (opt->log != NULL) fflush(opt->log); \
+  if (makestat_fp) { fclose(makestat_fp); makestat_fp=NULL; } \
+  if (maketrack_fp){ fclose(maketrack_fp); maketrack_fp=NULL; } \
+  if (opt->accept_cookie) cookie_save(opt->cookie,fconcat(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),StringBuff(opt->path_log),"cookies.txt")); \
+  if (makeindex_fp) { fclose(makeindex_fp); makeindex_fp=NULL; } \
+  if (cache_hashtable) { coucal_delete(&cache_hashtable); } \
+  if (cache_tests)     { coucal_delete(&cache_tests); } \
+  if (template_header) { freet(template_header); template_header=NULL; } \
+  if (template_body)   { freet(template_body); template_body=NULL; } \
+  if (template_footer) { freet(template_footer); template_footer=NULL; } \
+  hash_free(&hash); \
+  clearCallbacks(&opt->state.callbacks); \
+  /*structcheck_init(-1);*/ \
+} while(0)
 #define XH_uninit do { XH_extuninit; if (r.adr) { freet(r.adr); r.adr=NULL; } } while(0)
 
 struct lien_buffers {
@@ -472,11 +441,32 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
   *makeindex_done = 1;
 }
 
-/* Flush the parsed HTML output buffer to disk. */
+/* Flush the parsed HTML output buffer to disk, skipping the rewrite when the
+ * on-disk MD5 is unchanged. */
 void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
                           FILE **fp, const char *ht_buff, size_t ht_len,
                           const char *adr, const char *fil, const char *save) {
-  {
+  char digest[32 + 2];
+  off_t fsize_old =
+      fsize(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), save));
+  int ok = 0;
+
+  digest[0] = '\0';
+  domd5mem(ht_buff, ht_len, digest, 1);
+  if (fsize_old == (off_t) ht_len) {
+    int mlen = 0;
+    char *mbuff;
+
+    cache_readdata(cache, "//[HTML-MD5]//", save, &mbuff, &mlen);
+    if (mlen)
+      mbuff[mlen] = '\0';
+    if ((mlen == 32) && (strcmp(((mbuff != NULL) ? mbuff : ""), digest) == 0)) {
+      ok = 1;
+      hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s", save);
+    }
+    freet(mbuff);
+  }
+  if (!ok) {
     file_notify(opt, adr, fil, save, 1, 1, r->notmodified);
     *fp = filecreate(&opt->state.strc, save);
     if (*fp) {
@@ -508,7 +498,13 @@ void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
       if (fcheck)
         hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
     }
+  } else {
+    file_notify(opt, adr, fil, save, 0, 0, r->notmodified);
+    filenote(&opt->state.strc, save, NULL);
   }
+  if (cache->ndx)
+    cache_writedata(cache->ndx, cache->dat, "//[HTML-MD5]//", save, digest,
+                    (int) strlen(digest));
 }
 
 /* does it look like XML ? (SVG et al.) */
@@ -527,6 +523,7 @@ int httpmirror(char *url1, httrackp * opt) {
   hash_struct *const hashptr = &hash;
   t_cookie BIGSTK cookie;       // gestion des cookies
 
+  //char* tab_alloc=NULL;
   int ptr;                      // pointeur actuel sur les liens
 
   //
@@ -537,12 +534,15 @@ int httpmirror(char *url1, httrackp * opt) {
   // pour les stats, nombre de fichiers & octets écrits
   LLint stat_fragment = 0;      // pour la fragmentation
 
+  //TStamp istat_timestart;   // départ pour calcul instantanné
+  //
   TStamp last_info_shell = 0;
   int info_shell = 0;
 
   // filtres
   char **filters = NULL;
 
+  //int filter_max=0;
   int filptr = 0;
 
   //
@@ -582,6 +582,7 @@ int httpmirror(char *url1, httrackp * opt) {
   // noter heure actuelle de départ en secondes
   memset(&HTS_STAT, 0, sizeof(HTS_STAT));
   HTS_STAT.stat_timestart = time_local();
+  //istat_timestart=stat_timestart;
   HTS_STAT.istat_timestart[0] = HTS_STAT.istat_timestart[1] = mtime_local();
   /* reset stats */
   HTS_STAT.HTS_TOTAL_RECV = 0;
@@ -615,6 +616,9 @@ int httpmirror(char *url1, httrackp * opt) {
   // initialiser usercommand
   usercommand(opt, opt->sys_com_exec, StringBuff(opt->sys_com), "", "", "");
 
+  // initialiser structcheck
+  // structcheck_init(1);
+
   // initialiser verif_backblue
   verif_backblue(opt, NULL);
   verif_external(opt, 0, 0);
@@ -634,6 +638,9 @@ int httpmirror(char *url1, httrackp * opt) {
                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_bin),
                  "templates/index-footer.html"), HTS_INDEX_FOOTER);
 
+  // initialiser mimedefs
+  //get_userhttptype(opt,1,StringBuff(opt->mimedefs),NULL);
+
   // Initialiser indexation
   if (opt->kindex)
     index_init(StringBuff(opt->path_html));
@@ -677,6 +684,7 @@ int httpmirror(char *url1, httrackp * opt) {
   opt->filters.filters = &filters;
   //
   opt->filters.filptr = &filptr;
+  //opt->filters.filter_max=&filter_max;
 
   // hash table
   opt->hash = &hash;
@@ -723,7 +731,8 @@ int httpmirror(char *url1, httrackp * opt) {
       else if (*a == '-')
         joker = 1;
 
-      if (joker) { // joker ou filters
+      if (joker) {              // joker ou filters
+        //char* p;
         char BIGSTK tempo[HTS_URLMAXSIZE * 2];
         int type;
         int plus = 0;
@@ -778,11 +787,12 @@ int httpmirror(char *url1, httrackp * opt) {
               XH_extuninit;
               return 0;
             }
+            //opt->filters.filters=filters;
           }
 
         }
 
-      } else { // adresse normale
+      } else {                  // adresse normale
         char BIGSTK url[HTS_URLMAXSIZE * 2];
 
         // prochaine adresse
@@ -803,52 +813,6 @@ int httpmirror(char *url1, httrackp * opt) {
       }
     }                           // while
 
-    /* --why: print which filter rule decides for this URL, then stop */
-    if (StringNotEmpty(opt->why_url)) {
-      char BIGSTK url[HTS_URLMAXSIZE * 2];
-      lien_adrfil af;
-
-      if (strstr(StringBuff(opt->why_url), ":/") == NULL)
-        snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
-      else
-        strcpybuff(url, StringBuff(opt->why_url));
-      if (ident_url_absolute(url, &af) < 0) {
-        printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
-      } else {
-        char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
-        int jok, jokDepth = 0;
-
-        /* the two forms the wizard tests */
-        strcpybuff(l, jump_identification_const(af.adr));
-        if (*af.fil != '/')
-          strcatbuff(l, "/");
-        strcatbuff(l, af.fil);
-        if (!link_has_authority(af.adr))
-          strcpybuff(lfull, "http://");
-        else
-          lfull[0] = '\0';
-        strcatbuff(lfull, af.adr);
-        if (*af.fil != '/')
-          strcatbuff(lfull, "/");
-        strcatbuff(lfull, af.fil);
-        jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
-                               NULL, &jokDepth);
-        if (jok > 0)
-          printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
-                 filters[jokDepth]);
-        else if (jok < 0)
-          printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
-                 filters[jokDepth]);
-        else
-          printf("%s: no filter rule matches; the wizard decides "
-                 "(same-site links are followed by default)" LF,
-                 url);
-      }
-      freet(primary);
-      XH_extuninit;
-      return 1;
-    }
-
     /* load URL file list */
     /* OPTIMIZED for fast load */
     if (StringNotEmpty(opt->filelist)) {
@@ -903,6 +867,7 @@ int httpmirror(char *url1, httrackp * opt) {
             htsbuff_cat(&primarybuff, "\n");
           }
         }
+        // fclose(fp);
         hts_log_print(opt, LOG_NOTICE, "%d links added from %s", n,
                       StringBuff(opt->filelist));
 
@@ -957,6 +922,7 @@ int httpmirror(char *url1, httrackp * opt) {
 #endif
 
   // backing
+  //soc_max=opt->maxsoc;
   if (opt->maxsoc > 0) {
 #if BDEBUG==2
     _CLRSCR;
@@ -1270,10 +1236,15 @@ int httpmirror(char *url1, httrackp * opt) {
       // tester r.adr
       if (!error) {
         // erreur, pas de fichier chargé:
-        if ((!r.adr) && (r.is_write == 0) && (r.statuscode != 301) &&
-            (r.statuscode != 302) && (r.statuscode != 303) &&
-            (r.statuscode != 307) && (r.statuscode != 412) &&
-            (r.statuscode != 416)) {
+        if ((!r.adr) && (r.is_write == 0)
+            && (r.statuscode != 301)
+            && (r.statuscode != 302)
+            && (r.statuscode != 303)
+            && (r.statuscode != 307)
+            && (r.statuscode != 412)
+            && (r.statuscode != 416)
+          ) {
+          // error=1;
 
           // peut être que le fichier était trop gros?
           if ((istoobig
@@ -1355,8 +1326,30 @@ int httpmirror(char *url1, httrackp * opt) {
                                          ||may_be_hypertext_mime(opt, r.contenttype, urlfil()))   /* Is real media, .. */
           ) {
 
-          /* Convert charset to UTF-8 - NOT! (what about links ? remote server
-           * side will have troubles with converted names) */
+          /* Convert charset to UTF-8 - NOT! (what about links ? remote server side will have troubles with converted names) */
+          //if (r.adr != NULL && r.size != 0 && opt->convert_utf8) {
+          //  char *charset;
+          //  char *pos;
+          //  if (r.charset[0] != '\0') {
+          //    charset = strdup(r.charset);
+          //  } else {
+          //    charset = hts_getCharsetFromMeta(r.adr, r.size);
+          //  }
+          //  if (charset != NULL) {
+          //    char *const utf8 = hts_convertStringToUTF8(r.adr, r.size, charset);
+          //    /* Use new buffer */
+          //    if (utf8 != NULL) {
+          //      freet(r.adr);
+          //      r.size = strlen(utf8);
+          //      r.adr = utf8;
+          //      /* New UTF-8 charset */
+          //      r.charset[0] = '\0';
+          //      strcpy(r.charset, "utf-8");
+          //    }
+          //    /* Free charset */
+          //    free(charset);
+          //  }
+          //}
 
           /* Check bogus chars */
           if ((r.adr) && (r.size)) {
@@ -1497,6 +1490,50 @@ int httpmirror(char *url1, httrackp * opt) {
 
         }
       }
+      // MOVED IN back_finalize()
+      //
+      // -------------------- 
+      // REAL MEDIA HACK
+      // Check if we have to load locally the file
+      // --------------------
+      //if (!error) {
+      //  if (r.statuscode == HTTP_OK) {    // OK (ou 304 en backing)
+      //    if (r.adr==NULL) {    // Written file
+      //      if (may_be_hypertext_mime(r.contenttype, urlfil())) {   // to parse!
+      //        LLint sz;
+      //        sz=fsize_utf8(savename());
+      //        if (sz>0) {   // ok, exists!
+      //          if (sz < 8192) {   // ok, small file --> to parse!
+      //            FILE* fp=FOPEN(savename(),"rb");
+      //            if (fp) {
+      //              r.adr=malloct(sz + 1);
+      //              if (r.adr) {
+      //                if (fread(r.adr,1,sz,fp) == sz) {
+      //                  r.size=sz;
+      //                                                r.adr[sz] = '\0';
+      //                                                r.is_write = 0;
+      //                } else {
+      //                  freet(r.adr);
+      //                  r.size=0;
+      //                  r.adr = NULL;
+      //                  r.statuscode=STATUSCODE_INVALID;
+      //                  strcpybuff(r.msg, ".RAM read error");
+      //                }
+      //                fclose(fp);
+      //                fp=NULL;
+      //                // remove (temporary) file!
+      //                remove(savename());
+      //              }
+      //              if (fp)
+      //                fclose(fp);
+      //            }
+      //          }
+      //        }
+      //      }
+      //    }
+      //  }
+      //}
+      // EN OF REAL MEDIA HACK
 
       // ---stockage en cache---
       // stocker dans le cache?
@@ -1617,6 +1654,24 @@ int httpmirror(char *url1, httrackp * opt) {
       }
 #endif
 
+      /* info: updated */
+      /*
+         if (ptr>0) {
+         // "mis à jour"
+         if ((!r.notmodified) && (opt->is_update) && (!store_errpage)) {    // page modifiée
+         if (strnotempty(savename())) {
+         HTS_STAT.stat_updated_files++;
+         //if ((opt->debug>0) && (opt->log!=NULL)) {
+         hts_log_print(opt, LOG_INFO, "File updated: %s%s",urladr(),urlfil());
+         }
+         } else {
+         if (!store_errpage) {
+         hts_log_print(opt, LOG_INFO, "File recorded: %s%s",urladr(),urlfil());
+         }
+         }
+         }
+       */
+
       // ------------------------------------------------------
       // traitement (parsing)
       // ------------------------------------------------------
@@ -1662,8 +1717,13 @@ int httpmirror(char *url1, httrackp * opt) {
                 free(charset);
             }
             /* Could not detect charset: could it be UTF-8 ? */
-            /* No, we can not do that: browsers do not do it
+            /* No, we can not do that: browsers do not do it 
                (and it would break links). */
+            //if (page_charset[0] == '\0') {
+            //  if (is_unicode_utf8(r.adr, r.size)) {
+            //    strcpy(page_charset, "utf-8");
+            //  }
+            //}
             /* Could not detect charset */
             if (page_charset[0] == '\0') {
               hts_log_print(opt, LOG_INFO,
@@ -1741,6 +1801,10 @@ int httpmirror(char *url1, httrackp * opt) {
               XH_uninit;
               return -1;
             }
+
+            // I'll have to segment this part
+// #include "htsparse.c"
+
           }
         }
         // Fin parsing HTML
@@ -1836,6 +1900,7 @@ int httpmirror(char *url1, httrackp * opt) {
              HTS_STAT.stat_files++;
              HTS_STAT.stat_bytes+=r.size;
            */
+          //printf("ok......\n");
         } else {
           // Si on doit sauver une page HTML sans la scanner, cela signifie que le niveau de
           // récursion nous en empêche
@@ -1904,11 +1969,10 @@ int httpmirror(char *url1, httrackp * opt) {
             }
           }
 
-          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
-          // An empty body must not overwrite the file when the transfer failed
-          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
-          // copy to 0 (#77 follow-up).
-          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
+          //printf("extern=%s\n",r.contenttype);
+
+          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!          
+          if (r.adr != NULL || r.size == 0) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {
@@ -1929,6 +1993,7 @@ int httpmirror(char *url1, httrackp * opt) {
                */
             }
           }
+
         }
 
         /* Parsing of other media types (java, ram..) */
@@ -2027,9 +2092,16 @@ int httpmirror(char *url1, httrackp * opt) {
           // prochain pass2
           while((ptr < opt->lien_tot) ? (!heap(ptr)->pass2) : 0)
             ptr++;
+
+          //printf("first link==%d\n");
+
         }
       }
     }
+    // copy abort state if necessary from outside
+    //if (!exit_xh && opt->state.exit_xh) {
+    //  exit_xh=opt->state.exit_xh;
+    //}
     // a-t-on dépassé le quota?
     if (!back_checkmirror(opt)) {
       ptr = opt->lien_tot;
@@ -2186,11 +2258,13 @@ int httpmirror(char *url1, httrackp * opt) {
     char BIGSTK htstime[256];
     char BIGSTK infoupdated[256];
 
+    // int n=(int) (stat_loaded/(time_local()-HTS_STAT.stat_timestart));
     LLint n =
       (LLint) (HTS_STAT.HTS_TOTAL_RECV /
                (max(1, time_local() - HTS_STAT.stat_timestart)));
 
     sec2str(htstime, time_local() - HTS_STAT.stat_timestart);
+    //sprintf(finalInfo + strlen(finalInfo),LF"HTS-mirror complete in %s : %d links scanned, %d files written (%d bytes overall) [%d bytes received at %d bytes/sec]"LF,htstime,lien_tot-1,HTS_STAT.stat_files,stat_bytes,stat_loaded,n);
     infoupdated[0] = '\0';
     if (opt->is_update) {
       if (HTS_STAT.stat_updated_files > 0) {
@@ -2273,6 +2347,12 @@ int httpmirror(char *url1, httrackp * opt) {
 
 */
 int engine_stats(void) {
+#if 0
+  static FILE *debug_fp = NULL; /* ok */
+
+  if (!debug_fp)
+    debug_fp = fopen("esstat.txt", "wb");
+#endif
   HTS_STAT.stat_nsocket = HTS_STAT.stat_errors = HTS_STAT.nbk = 0;
   HTS_STAT.nb = 0;
   if (HTS_STAT.HTS_TOTAL_RECV > 2048) {
@@ -2283,6 +2363,10 @@ int engine_stats(void) {
       if ((cdif - HTS_STAT.istat_timestart[i]) >= 2000) {
         TStamp dif;
 
+#if 0
+        fprintf(debug_fp, "set timer %d\n", i);
+        fflush(debug_fp);
+#endif
         dif = cdif - HTS_STAT.istat_timestart[i];
         if ((TStamp) (dif / 1000) > 0) {
           LLint byt = (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[i]);
@@ -2301,6 +2385,10 @@ int engine_stats(void) {
     // timer #0 resync timer #1 when reaching 1 second limit
     if (HTS_STAT.istat_reference01 != HTS_STAT.istat_timestart[0]) {
       if ((cdif - HTS_STAT.istat_timestart[0]) >= 1000) {
+#if 0
+        fprintf(debug_fp, "resync timer 1\n");
+        fflush(debug_fp);
+#endif
         HTS_STAT.istat_bytes[1] = HTS_STAT.HTS_TOTAL_RECV;
         HTS_STAT.istat_timestart[1] = cdif;
         HTS_STAT.istat_reference01 = HTS_STAT.istat_timestart[0];
@@ -2321,6 +2409,7 @@ void host_ban(httrackp * opt, int ptr,
   lien_back *const back = sback->lnk;
   const int back_max = sback->count;
 
+  //int l;
   int i;
 
   if (host[0] == '!')
@@ -2377,7 +2466,9 @@ void host_ban(httrackp * opt, int ptr,
   }
 
   // effacer liens
-  for (i = 0; i < opt->lien_tot; i++) {
+  //l=strlen(host);
+  for(i = 0; i < opt->lien_tot; i++) {
+    //if (heap(i)->adr_len==l) {    // même taille de chaîne
     // Calcul de taille sécurisée
     if (heap(i)) {
       if (heap(i)->adr) {
@@ -2416,6 +2507,7 @@ void host_ban(httrackp * opt, int ptr,
                     "WARNING! HostCancel detected memory leaks [null at %d]",
                     i);
     }
+    //}
   }
 }
 
@@ -2832,6 +2924,7 @@ FILE *fileappend(filenote_strc * strc, const char *s) {
   // noter lst
   filenote(strc, s, NULL);
 
+  // if (*s=='/') strcpybuff(fname,s+1); else strcpybuff(fname,s);    // pas de / (root!!) // ** SIIIIIII!!! à cause de -O <path>
   strcpybuff(fname, s);
 
 #if HTS_DOSNAME
@@ -2875,6 +2968,7 @@ int filecreateempty(filenote_strc * strc, const char *filename) {
 int filenote(filenote_strc * strc, const char *s, filecreate_params * params) {
   // gestion du fichier liste liste
   if (params) {
+    //filecreate_params* p = (filecreate_params*) params;
     strcpybuff(strc->path, params->path);
     strc->lst = params->lst;
     return 0;
@@ -2954,8 +3048,9 @@ void usercommand_exe(const char *cmd, const char *file) {
   }
 }
 
-static void postprocess_file(httrackp *opt, const char *save, const char *adr,
+static void postprocess_file(httrackp * opt, const char *save, const char *adr,
                              const char *fil) {
+  //int first = 0;
   /* MIME-html archive to build */
   if (opt != NULL && opt->mimehtml) {
     if (adr != NULL && strcmp(adr, "primary") == 0) {
@@ -2983,6 +3078,7 @@ static void postprocess_file(httrackp *opt, const char *save, const char *adr,
       }
 
       if (!opt->state.mimehtml_created) {
+        //first = 1;
         opt->state.mimefp =
           fopen(fconcat
                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
@@ -3126,6 +3222,28 @@ int fspc(httrackp * opt, FILE * fp, const char *type) {
   return 0;
 }
 
+// vérifier taux de transfert
+#if 0
+void check_rate(TStamp stat_timestart, int maxrate) {
+  // vérifier taux de transfert (pas trop grand?)
+  /*
+     if (maxrate>0) {
+     int r = (int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-stat_timestart));    // taux actuel de transfert
+     HTS_STAT.HTS_TOTAL_RECV_STATE=0;
+     if (r>maxrate) {    // taux>taux autorisé
+     int taux = (int) (((TStamp) (r - maxrate) * 100) / (TStamp) maxrate);
+     if (taux<15)
+     HTS_STAT.HTS_TOTAL_RECV_STATE=1;   // ralentir un peu (<15% dépassement)
+     else if (taux<50)
+     HTS_STAT.HTS_TOTAL_RECV_STATE=2;   // beaucoup (<50% dépassement)
+     else
+     HTS_STAT.HTS_TOTAL_RECV_STATE=3;   // énormément (>50% dépassement)
+     }
+     }
+   */
+}
+#endif
+
 // ---
 // sous routines liées au moteur et au backing
 
@@ -3135,8 +3253,21 @@ int backlinks_done(const struct_back * sback,
                    int ptr) {
   int n = 0;
 
+#if 0
+  int i;
+
+  //Links done and stored in cache
+  for(i = ptr + 1; i < lien_tot; i++) {
+    if (heap(i)) {
+      if (heap(i)->pass2 == -1) {
+        n++;
+      }
+    }
+  }
+#else
   // finalized in background
   n += HTS_STAT.stat_background;
+#endif
   n += back_done_incache(sback);
   return n;
 }
@@ -3273,7 +3404,8 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
     /* on a déja parcouru */
     if (p < cache->ptr_ant)
       p = cache->ptr_ant;
-    while (p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
+    while(p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
+      //while((p<lien_tot) && (n>0) && (p < ptr+opt->maxcache_anticipate)) {
       int ok = 1;
 
       // on ne met pas le fichier en backing si il doit être traité après ou s'il a déja été traité
@@ -3285,6 +3417,12 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
           ok = 0;
       }
 
+      // Why in hell did I do that ?
+      //if (ok && heap(p)->sav != NULL && heap(p)->sav[0] != '\0' 
+      //  && hash_read(opt->hash,heap(p)->sav,NULL,HASH_STRUCT_FILENAME ) >= 0)     // lookup in liens_record
+      //{
+      //  ok = 0;
+      //}
       if (heap(p)->sav == NULL || heap(p)->sav[0] == '\0'
           || hash_read(opt->hash, heap(p)->sav, NULL, HASH_STRUCT_FILENAME ) < 0) {
         ok = 0;
@@ -3313,7 +3451,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
         }
       }
       p++;
-    } // while
+    }                           // while
     /* sauver position dernière anticipation */
     cache->ptr_ant = p;
     cache->ptr_last = ptr;
@@ -3575,7 +3713,20 @@ HTSEXT_API hts_boolean hts_has_stopped(httrackp *opt) {
   return ended;
 }
 
+// régler en cours de route les paramètres réglables..
+// -1 : erreur
+//HTSEXT_API int hts_setopt(httrackp* set_opt) {
+//  if (set_opt) {
+//    httrackp* engine_opt=hts_declareoptbuffer(NULL);
+//    if (engine_opt) {
+//      //_hts_setopt=opt;
+//      copy_htsopt(set_opt,engine_opt);
+//    }
+//  }
+//  return 0;
+//}
 // ajout d'URL
+// -1 : erreur
 HTSEXT_API hts_boolean hts_addurl(httrackp *opt, char **url) {
   if (url)
     opt->state._hts_addurl = url;
@@ -3836,7 +3987,10 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
               heap_top()->link_import = 0; // pas mode import
 
               // écrire autres paramètres de la structure-lien
+              //if (meme_adresse)                                 
               heap_top()->premier = heap(ptr)->premier;
+              //else    // sinon l'objet père est le précédent lui même
+              //  heap_top()->premier=ptr;
 
               heap_top()->precedent = ptr;
               // noter la priorité
@@ -3847,6 +4001,9 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
               heap_top()->pass2 = max(pass_fix, numero_passe);
               heap_top()->retry = opt->retry;
 
+              //strcpybuff(heap_top()->adr,adr);
+              //strcpybuff(heap_top()->fil,fil);
+              //strcpybuff(heap_top()->sav,save); 
               hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s",
                             heap_top()->adr, heap_top()->fil,
                             heap_top()->sav);
@@ -3875,6 +4032,7 @@ void voidf(void) {
   a = "|                      HTTrack Website Copier   |" CRLF;
   a = "|Code:         Windows Interface Xavier Roche   |" CRLF;
   a = "|                    HTS/HTTrack Xavier Roche   |" CRLF;
+  a = "|                .class Parser Yann Philippot   |" CRLF;
   a = "|                                               |" CRLF;
   a = "|Tested on:                 Windows95,98,NT,2K  |" CRLF;
   a = "|                           Linux PC            |" CRLF;

src/htscore.h

@@ -193,7 +193,8 @@ struct cache_back {
   /* */
   int type;
   int ro;                   /**< read-only: no new cache is written */
-
+  FILE *dat, *ndx, *olddat; /**< new data, new index, old data files */
+  char *use;                /**< in-memory list of cached adr+fil keys */
   FILE *lst;                /**< file list, used for purge */
   FILE *txt;                /**< human-readable file list (info) */
   char lastmodified[256];
@@ -287,12 +288,12 @@ struct filecreate_params {
 
 /* True if a new cache is being written (plain or zip backend). */
 HTS_STATIC int cache_writable(cache_back * cache) {
-  return (cache != NULL && cache->zipOutput != NULL);
+  return (cache != NULL && (cache->dat != NULL || cache->zipOutput != NULL));
 }
 
 /* True if an old cache is available to read (plain or zip backend). */
 HTS_STATIC int cache_readable(cache_back * cache) {
-  return (cache != NULL && cache->zipInput != NULL);
+  return (cache != NULL && (cache->olddat != NULL || cache->zipInput != NULL));
 }
 
 #endif
@@ -411,6 +412,10 @@ char *readfile_utf8(const char *fil);
 
 char *readfile_or(const char *fil, const char *defaultdata);
 
+#if 0
+void check_rate(TStamp stat_timestart, int maxrate);
+#endif
+
 /* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
    Not thread-safe; call from the single crawl loop. */
 
@@ -462,6 +467,11 @@ int ask_continue(httrackp * opt);
 /* Number of decimal digits in n. */
 int nombre_digit(int n);
 
+// Java
+#if 0
+int hts_add_file(char *file, int file_position);
+#endif
+
 // Polling
 #if HTS_POLL
 int check_flot(T_SOC s);

src/htscoremain.c

@@ -145,6 +145,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   char *url = NULL;             // URLS séparées par un espace
   int url_sz = 65535;
 
+  //char url[65536];         // URLS séparées par un espace
   // the parametres
   int httrack_logmode = 3;      // ONE log file
 
@@ -289,11 +290,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         if (tmp_argc == 1) {    /* pas -P & co */
           if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */
             if (argv_url < 0)
-              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
-                            // previous options
+              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
+            //if (argv_url>=0) {
             argv_url++;
             if (!argv_firsturl)
               argv_firsturl = x_argv[x_argc - 1];
+            //}
           } else {
             if (strcmp(tmp_argv[0], "-h") == 0) {
               help(argv[0], !opt->quiet);
@@ -322,8 +324,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         } else if (tmp_argc == 2) {
           if ((strcmp(tmp_argv[0], "-%L") == 0)) {      // liste d'URLs
             if (argv_url < 0)
-              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
-                            // previous options
+              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
+            //if (argv_url>=0)
             argv_url++;         /* forcer */
           }
         }
@@ -401,11 +403,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     else
                       inQuote = 1;
                   } else if (!inQuote && !noDbl && argv[na][i] == ',') {
+                    //StringAddchar(path, '\0');
+                    //j = 0;
                     path = &opt->path_log;
                   } else {
                     StringAddchar(*path, argv[na][i]);
+                    //path[j++] = argv[na][i];
                   }
                 }
+                //path[j++] = '\0';
                 if (StringLength(opt->path_log) == 0) {
                   StringCopyS(opt->path_log, opt->path_html);
                 }
@@ -414,6 +420,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if (check_path(&opt->path_html, argv_firsturl)) {
                   opt->dir_topindex = 1;        // rebuilt top index
                 }
+                //printf("-->%s\n%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log));                
               }
               break;
             }                   // switch
@@ -532,7 +539,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             insert_after++;
           }
         }
-      } while (lastp != NULL);
+      } while(lastp != NULL);
+      //fclose(fp);
     }
   }
 
@@ -708,6 +716,24 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             HTS_PANIC_PRINTF(s);
             htsmain_free();
             return -1;
+#else
+#if 0
+            char _args[8][256];
+            char *args[8];
+
+            printf("Cheking for updates...\n");
+            strcpybuff(_args[0], argv[0]);
+            strcpybuff(_args[1], "--get");
+            sprintf(_args[2], HTS_UPDATE_WEBSITE, 0, "");
+            strcpybuff(_args[3], "--quickinfo");
+            args[0] = _args[0];
+            args[1] = _args[1];
+            args[2] = _args[2];
+            args[3] = _args[3];
+            args[4] = NULL;
+            if (execvp(args[0], args) == -1) {
+            }
+#endif
 #endif
           }
           //
@@ -725,6 +751,103 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     }
   }
 
+  // Compter urls/jokers
+  /*
+     if (argv_url<=0) { 
+     int na;
+     argv_url=0;
+     for(na=1;na<argc;na++) {
+     if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) {
+     na++;    // sauter nom de proxy
+     } else if (!cmdl_opt(argv[na])) { 
+     argv_url++;   // un de plus       
+     } else if (strcmp(argv[na],"-h")==0) {
+     help(argv[0],!opt->quiet);
+     htsmain_free();
+     return 0;
+     } else {
+     if ((strchr(argv[na],'q')!=NULL))
+     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
+     if ((strchr(argv[na],'i')!=NULL)) {  // doit.log!
+     argv_url=0;
+     na=argc;
+     }
+     }
+     }
+     }  
+   */
+
+  // Ici on ajoute les arguments qui ont été appelés avant au cas où on récupère une session
+  // Exemple: httrack www.truc.fr -L0 puis ^C puis httrack sans URL : ajouter URL précédente
+  /*
+     if (argv_url==0) {
+     //if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) {  // il existe déja un cache précédent.. renommer
+     if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) {    // un cache est présent
+
+     x_argvblk=(char*) calloct(32768,1);
+
+     if (x_argvblk!=NULL) {
+     FILE* fp;
+     int x_argc;
+
+     //strcpybuff(x_argvblk,"httrack ");
+     fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb");
+     if (fp) {
+     linput(fp,x_argvblk+strlen(x_argvblk),8192);
+     fclose(fp); fp=NULL;
+     }
+
+     // calculer arguments selon derniers arguments
+     x_argv[0]=argv[0];
+     x_argc=1;
+     {
+     char* p=x_argvblk;
+     do {
+     x_argv[x_argc++]=p;
+     //p=strstr(p," ");
+     // exemple de chaine: "echo \"test\"" c:\a "\$0"
+     p=next_token(p,1);    // prochain token
+     if (p) {
+     *p=0;    // octet nul (tableau)
+     p++;
+     }            
+     } while(p!=NULL);
+     }
+     // recopier arguments actuels (pointeurs uniquement)
+     {
+     int na;
+     for(na=1;na<argc;na++) {
+     if (strcmp(argv[na],"-O") != 0)    // SAUF le path!
+     x_argv[x_argc++]=argv[na];
+     else
+     na++;
+     }
+     }
+     argc=x_argc;      // nouvel argc
+     argv=x_argv;      // nouvel argv
+     }
+
+     }
+     //}
+     }
+   */
+
+  // Vérifier quiet
+  /*
+     { 
+     int na;    
+     for(na=1;na<argc;na++) {
+     if (!cmdl_opt(argv[na])) { 
+     if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0))
+     na++;    // sauter nom de proxy
+     } else {
+     if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL))
+     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
+     }
+     }
+     }
+   */
+
   /* Engine self-tests: -#test lists them, -#test=NAME [args] runs one. Handled
      here, ahead of the no-URL usage gate below, so they need no dummy URL. */
   {
@@ -831,8 +954,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   } else {                      // plus de 2 paramètres
     // un fichier log existe?
     if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                       StringBuff(opt->path_log),
-                       "hts-in_progress.lock"))) { // fichier lock?
+      StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // fichier lock?
+      //char s[32];
 
       opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
       if (opt->quiet == 0) {
@@ -863,8 +986,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           }
         }
       }
-    } else if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                              StringBuff(opt->path_html), "index.html"))) {
+    } else
+      if (fexist
+          (fconcat
+           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "index.html"))) {
+      //char s[32];
       opt->cache = HTS_CACHE_TEST_UPDATE;
       if (opt->quiet == 0) {
         if ((fexist
@@ -959,6 +1085,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             //
           case 'g':            // récupérer un (ou plusieurs) fichiers isolés
             opt->wizard = HTS_WIZARD_AUTO;
+            //opt->wizard=0;             // pas de wizard
             opt->cache = HTS_CACHE_NONE; // ni de cache
             opt->makeindex = 0; // ni d'index
             httrack_logmode = 1;        // erreurs à l'écran
@@ -1058,6 +1185,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             while(isdigit((unsigned char) *(com + 1)))
               com++;
             break;
+            //
+            //case 'A': opt->urlmode=1; break;
+            //case 'R': opt->urlmode=2; break;
           case 'K':
             opt->urlmode = HTS_URLMODE_ABSOLUTE;
             if (isdigit((unsigned char) *(com + 1))) {
@@ -1069,6 +1199,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               while(isdigit((unsigned char) *(com + 1)))
                 com++;
             }
+            //if (*(com+1)=='0') { opt->urlmode=2; com++; } break;
+            //
           case 'c':
             if (isdigit((unsigned char) *(com + 1))) {
               sscanf(com + 1, "%d", &opt->maxsoc);
@@ -1452,7 +1584,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF
                     ("Option %w needs to be followed by a blank space, and a module name");
-                  printf("Example: -%%w httrack-plugin\n");
+                  printf("Example: -%%w htsswf\n");
                   htsmain_free();
                   return -1;
                 } else {
@@ -1556,6 +1688,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   } else {
                     char *a;
 
+                    //char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs);
                     for(a = argv[na]; *a != '\0'; a++) {
                       if (*a == ';') {  /* next one */
                         StringAddchar(opt->mimedefs, '\n');
@@ -1803,23 +1936,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   StringCopy(opt->cookies_file, argv[na]);
                 }
                 break;
-              case 'Y': // why: explain the filter verdict for a URL, no crawl
-                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
-                  HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
-                  printf(
-                      "Example: --why \"http://www.example.com/file.zip\"\n");
-                  htsmain_free();
-                  return -1;
-                } else {
-                  na++;
-                  if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
-                    HTS_PANIC_PRINTF("why URL too long");
-                    htsmain_free();
-                    return -1;
-                  }
-                  StringCopy(opt->why_url, argv[na]);
-                }
-                break;
               case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
                 if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                   HTS_PANIC_PRINTF("Option pause needs a blank space and a "
@@ -1911,7 +2027,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                   HTS_PANIC_PRINTF(s);
                   htsmain_free();
                   return -1;
-              } break;
+                }
+                break;
+
+                //case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break;
               }
             }
             break;
@@ -1953,20 +2072,18 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     char BIGSTK url[HTS_URLMAXSIZE * 2];
                     char linepos[256];
                     int pos;
-                    LLint cacheNdxLen = 0;
-                    char *cacheNdx = readfile2(
-                        fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                StringBuff(opt->path_log), "hts-cache/new.ndx"),
-                        &cacheNdxLen);
+                    char *cacheNdx =
+                      readfile(fconcat
+                               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                                "hts-cache/new.ndx"));
                     cache_init(&cache, opt);    /* load cache */
                     if (cacheNdx != NULL) {
                       char firstline[256];
                       char *a = cacheNdx;
-                      const char *const end = cacheNdx + cacheNdxLen;
 
                       a += cache_brstr(a, firstline, sizeof(firstline));
                       a += cache_brstr(a, firstline, sizeof(firstline));
-                      while (a != NULL && a < end) {
+                      while(a != NULL) {
                         a = strchr(a + 1, '\n');        /* start of line */
                         if (a) {
                           htsblk r;
@@ -1974,15 +2091,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                           /* */
                           a++;
                           /* read "host/file" */
-                          a += cache_binput(a, end, afs.af.adr, HTS_URLMAXSIZE);
-                          a += cache_binput(a, end, afs.af.fil, HTS_URLMAXSIZE);
+                          a += binput(a, afs.af.adr, HTS_URLMAXSIZE);
+                          a += binput(a, afs.af.fil, HTS_URLMAXSIZE);
                           url[0] = '\0';
                           if (!link_has_authority(afs.af.adr))
                             strcatbuff(url, "http://");
                           strcatbuff(url, afs.af.adr);
                           strcatbuff(url, afs.af.fil);
                           /* read position */
-                          a += cache_binput(a, end, linepos, 200);
+                          a += binput(a, linepos, 200);
                           sscanf(linepos, "%d", &pos);
                           if (!hasFilter
                               || (strjoker(url, filter, NULL, NULL) != NULL)
@@ -2157,7 +2274,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 return 0;
                 break;
               case '~':        /* internal lib test */
-                HTS_PANIC_PRINTF("Option #~ is disabled for security reasons");
+                HTS_PANIC_PRINTF
+                  ("Option #~ is disabled for security reasons");
+                //Disabled because choke on GCC 4.3 (toni from links2linux.de)
+                //{
+                //  char thisIsATestYouShouldSeeAnError[12];
+                //  const char *const bufferOverflowTest = "0123456789012345678901234567890123456789";
+                //  strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest);
+                //  return 0;
+                //}
                 break;
               case 'f':
                 opt->flush = 1;
@@ -2259,6 +2384,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               opt->proxy.active = 1;
               // Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
               a = argv[na] + strlen(argv[na]) - 1;
+              // a=strstr(argv[na],":");  // port
               while((a > argv[na]) && (*a != ':') && (*a != '@'))
                 a--;
               if (*a == ':') {  // un port est présent, <proxy>:port
@@ -2363,6 +2489,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
 #endif
 #endif
 
+  //printf("WARNING! This is *only* a beta-release of HTTrack\n");
   io_flush;
 
 #if DEBUG_STEPS
@@ -2446,6 +2573,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     {
       FILE *fp = NULL;
 
+      //int n=0;
       char t[256];
 
       time_local_rfc822(t);     // faut bien que ca serve quelque part l'heure RFC1945 arf'
@@ -2479,8 +2607,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       }
 
       strcpybuff(n_lock,
-                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                         StringBuff(opt->path_log), "hts-in_progress.lock"));
+             fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
+                     "hts-in_progress.lock"));
+      //sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
       /*do {
          if (!n)
          sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
@@ -2556,6 +2685,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           fprintf(fp, LF);
           fclose(fp);
           fp = NULL;
+          //} else if (opt->debug>1) {
+          //  printf("! FileOpen error, \"%s\"\n",strerror(errno));
         }
       }
       // petit message dans le lock

src/htsdefines.h

@@ -70,7 +70,7 @@ typedef struct t_hts_callbackarg t_hts_callbackarg;
 #endif
 
 /* Marks a symbol an external wrapper module exports back to the engine.
-   Must override -fvisibility=hidden on ELF, or dlopen()ed plugins
+   Must override -fvisibility=hidden on ELF, or dlopen()ed plugins (htsjava)
    hide their own hts_plug()/hts_unplug() entry points. */
 #ifndef EXTERNAL_FUNCTION
 #ifdef _WIN32

src/htsdns_selftest.c

@@ -297,26 +297,6 @@ int dns_selftests(httrackp *opt) {
     IPV6_resolver = 0;
   }
 
-  /* "host:port" resolves as the bare host: the cache/connect path strips
-     ":port" before both the backend and the cache key (#181). */
-  mock_reset_calls();
-  {
-    SOCaddr a;
-    char ip[64];
-    const char *err = NULL;
-
-    /* cache miss: the backend is hit with the stripped host, not "host:port" */
-    CHECK(hts_dns_resolve2(opt, "v6only.test:8080", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* the bare host shares that one cache entry (stripped key) */
-    CHECK(hts_dns_resolve2(opt, "v6only.test", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* a port variant of an already-cached host also hits, right address */
-    CHECK(hts_dns_resolve2(opt, "v4only.test:1234", &a, &err) != NULL);
-    SOCaddr_inetntoa(ip, sizeof(ip), a);
-    CHECK(strcmp(ip, "1.2.3.4") == 0);
-  }
-
   /* newhttp_addr() must connect to the addr_index-th address, not always the
      first: this is what back_connect_next relies on to reach the fallback. */
   {

src/htsfilters.c

@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
 #include "htsbase.h"
 #include "htslib.h"
 #include <ctype.h>
-#include <stdint.h>
 /* END specific definitions */
 
 // à partir d'un tableau de {"+*.toto","-*.zip","+*.tata"} définit si nom est autorisé
@@ -95,127 +94,17 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
   return verdict;
 }
 
-int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
-                     const char *nom2, LLint *size, int *size_flag,
-                     int *depth) {
-  int depth1 = 0, depth2 = 0;
-  int flag1 = 0, flag2 = 0;
-  LLint sz1 = 0, sz2 = 0;
-  int jok1, jok2, use1;
-
-  if (size) {
-    sz1 = *size;
-    sz2 = *size;
-  }
-  jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
-                     size_flag ? &flag1 : NULL, &depth1);
-  jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
-                     size_flag ? &flag2 : NULL, &depth2);
-  use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
-  if (size)
-    *size = use1 ? sz1 : sz2;
-  if (size_flag)
-    *size_flag = use1 ? flag1 : flag2;
-  if (depth)
-    *depth = use1 ? depth1 : depth2;
-  return use1 ? jok1 : jok2;
-}
-
-/* Real filters/URLs fit HTS_URLMAXSIZE*2; past it a hostile pattern recurses
-   O(len) deep or runs O(n^2*stars). Cap length (depth) and steps (work). */
-#define STRJOKER_MAXLEN (HTS_URLMAXSIZE * 2)
-#define STRJOKER_MAXSTEPS 2000000u
-
-/* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
-   pair keeps star-heavy patterns polynomial instead of exponential (#501). */
-typedef struct strjoker_memo {
-  const char *chaine0, *joker0; /* offsets are relative to these bases */
-  size_t stride;                /* strlen(joker0) + 1 */
-  unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
-  size_t *nsteps; /* shared work counter; NULL: unbounded (oracle) */
-} strjoker_memo;
-
-static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
-                                 const char *joker, LLint *size,
-                                 int *size_flag);
-
-/* A pair that failed once fails forever (*size is only written on the success
-   path), so record NULL results and cut the re-exploration. */
-static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
-                                const char *joker, LLint *size,
-                                int *size_flag) {
-  size_t bit = 0;
-  const char *adr;
-
-  if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
-    return NULL; /* work budget spent: fail the match safely */
-  if (memo->failed) {
-    bit = (size_t) (chaine - memo->chaine0) * memo->stride +
-          (size_t) (joker - memo->joker0);
-    if (memo->failed[bit >> 3] & (unsigned char) (1u << (bit & 7)))
-      return NULL;
-  }
-  adr = strjoker_impl(memo, chaine, joker, size, size_flag);
-  if (adr == NULL && memo->failed)
-    memo->failed[bit >> 3] |= (unsigned char) (1u << (bit & 7));
-  return adr;
-}
-
-/* Match chaine against joker with a shared work budget *nsteps (a strjokerfind
-   sweep passes one counter, bounding the whole scan). */
-static const char *strjoker_bounded(const char *chaine, const char *joker,
-                                    LLint *size, int *size_flag,
-                                    size_t *nsteps) {
-  strjoker_memo memo = {chaine, joker, 0, NULL, nsteps};
-  unsigned char stackbits[2048];
-  hts_boolean onheap = HTS_FALSE;
-  const char *adr;
-
-  if (chaine != NULL && joker != NULL) {
-    const size_t l1 = strlen(chaine), l2 = strlen(joker);
-
-    if (l1 > STRJOKER_MAXLEN || l2 > STRJOKER_MAXLEN)
-      return NULL; /* beyond any real filter/URL: bound depth+work */
-    memo.stride = l2 + 1;
-    if (l1 + 1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
-      const size_t bytes = ((l1 + 1) * memo.stride + 7) / 8;
-
-      if (bytes <= sizeof(stackbits)) {
-        memset(stackbits, 0, bytes);
-        memo.failed = stackbits;
-      } else {
-        memo.failed = (unsigned char *) calloct(bytes, 1);
-        onheap = memo.failed != NULL ? HTS_TRUE : HTS_FALSE;
-      }
-    }
-  }
-  adr = strjoker_rec(&memo, chaine, joker, size, size_flag);
-  if (onheap)
-    freet(memo.failed);
-  return adr;
-}
-
-// wildcard comparator: match chaine against joker (pattern), case-insensitive
-// returns the address of the first matched letter past any leading joker
-// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
-                                LLint *size, int *size_flag) {
-  size_t nsteps = 0;
-
-  return strjoker_bounded(chaine, joker, size, size_flag, &nsteps);
-}
-
-/* Test-only oracle: the same matcher without the failure memo. */
-const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
-                            int *size_flag) {
-  const strjoker_memo memo = {chaine, joker, 0, NULL};
-
-  return strjoker_rec(&memo, chaine, joker, size, size_flag);
-}
-
-static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
-                                 const char *joker, LLint *size,
-                                 int *size_flag) {
+// supercomparateur joker (tm)
+// compare a et b (b=avec joker dedans), case insensitive [voir CI]
+// renvoi l'adresse de la première lettre de la chaine
+// (càd *[..]toto.. renvoi adresse de toto dans la chaine)
+// accepte les délires du genre www.*.*/ * / * truc*.*
+// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
+// * = toute lettre
+// --?-- : spécifique à HTTrack et aux ?
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
+                          int *size_flag) {
+  //int err=0;
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -347,7 +236,9 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
                 for(j = (int) (unsigned char) joker[i];
                     j <= (int) (unsigned char) joker[i + 2]; j++)
                   pass[j] = 1;
+
               }
+              // else err=1;
               i += 3;
             } else { // 1 car, ex: *[ ]
               pass[(int) (unsigned char) joker[i]] = 1;
@@ -371,6 +262,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       for(i = 0; i < 256; i++)
         pass[i] = 1;            // tout autoriser
       jmp = 1;
+      ////if (joker[2]==LEFT) jmp=3;        // permet de recher *<crochet ouvrant>
     }
 
     {
@@ -389,7 +281,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 
       // tester sans le joker (pas ()+ mais ()*)
       if (!unique) {
-        if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag))) {
+        if ((adr = strjoker(chaine, joker + jmp, size, size_flag))) {
           return adr;
         }
       }
@@ -398,11 +290,10 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       if (!unique)
         max = (int) strlen(chaine);
       else                      /* *(a) only match a (not aaaaa) */
-        max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
+        max = 1;
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
-          if ((adr = strjoker_rec(memo, chaine + i + 1, joker + jmp, size,
-                                  size_flag))) {
+          if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
             return adr;
           }
           i++;
@@ -412,8 +303,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 
       // tester chaîne vide
       if (i != max + 2)         // avant c'est ok
-        if ((adr = strjoker_rec(memo, chaine + max, joker + jmp, size,
-                                size_flag)))
+        if ((adr = strjoker(chaine + max, joker + jmp, size, size_flag)))
           return adr;
 
       return NULL;              // perdu
@@ -435,7 +325,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       // comparaison ok?
       if (ok) {
         // continuer la comparaison.
-        if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag))
+        if (strjoker(chaine + jmp, joker + jmp, size, size_flag))
           return chaine;        // retourner 1e lettre
       }
 
@@ -451,18 +341,12 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 // d'un strcpy sur une variable ayant un nom en lettres et copiant une chaine de chiffres
 // ATTENTION!! Eviter les jokers en début, où gare au temps machine!
 const char *strjokerfind(const char *chaine, const char *joker) {
-  size_t nsteps = 0; /* one budget for the whole scan */
   const char *adr;
 
-  if (chaine != NULL && strlen(chaine) > STRJOKER_MAXLEN)
-    return NULL;
   while(*chaine) {
-    if ((adr = strjoker_bounded(chaine, joker, NULL, NULL,
-                                &nsteps))) { // ok trouvé
+    if ((adr = strjoker(chaine, joker, NULL, NULL))) {  // ok trouvé
       return adr;
     }
-    if (nsteps > STRJOKER_MAXSTEPS) // scan budget spent: no match
-      return NULL;
     chaine++;
   }
   return NULL;

src/htsfilters.h

@@ -41,17 +41,8 @@ Please visit our Website: http://www.httrack.com
 
 int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
                 int *size_flag, int *depth);
-/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
-   scheme); the match latest in the list wins, a "don't know" verdict defers.
-   Returns the merged verdict; the out-params carry the winner's values. */
-int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
-                     const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
-/* strjoker() without the failure memo (exponential worst case); test-only
-   oracle for the memoized matcher. */
-const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
-                            int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);
 #endif
 

src/htsftp.c

@@ -54,9 +54,15 @@ Please visit our Website: http://www.httrack.com
 #endif
 
 // ftp mode passif
+// #if HTS_INET6==0
 #define FTP_PASV 1
+// #else
+// no passive mode for v6
+// #define FTP_PASV 0
+// #endif
 
 #define FTP_DEBUG 0
+//#define FORK_DEBUG 0
 
 #if USE_BEGINTHREAD
 
@@ -200,8 +206,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
   {
     char *a;
 
+#if 0
+    a = back->url_fil + strlen(back->url_fil) - 1;
+    while((a > back->url_fil) && (*a != '/'))
+      a--;
+    if (*a != '/') {
+      a = NULL;
+    }
+#else
     a = back->url_fil;
+#endif
     if (a != NULL && *a != '\0') {
+#if 0
+      a++;                      // sauter /
+#endif
       ftp_filename = a;
       if (strnotempty(a)) {
         char catbuff[CATBUFF_SIZE];
@@ -225,6 +243,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
       }
     } else {
       strcpybuff(back->r.msg, "Unexpected PORT error");
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
     }
   }
@@ -258,6 +277,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
       snprintf(back->r.msg, sizeof(back->r.msg),
                "Unable to get server's address: %s", error);
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_NON_FATAL;
       _HALT_FTP return 0;
     }
@@ -270,17 +290,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
     if (soc_ctl == INVALID_SOCKET) {
       strcpybuff(back->r.msg, "Unable to create a socket");
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
     }
 
     SOCaddr_initport(server, port);
+    // server.sin_port = htons((unsigned short int) port);
 
     // connexion (bloquante, on est en thread)
     strcpybuff(back->info, "connect");
 
     if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
       strcpybuff(back->r.msg, "Unable to connect to the server");
+      // back->status=STATUS_FTP_READY;    // fini
       back->r.statuscode = STATUSCODE_INVALID;
       _HALT_FTP return 0;
 #ifdef _WIN32
@@ -322,22 +345,64 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               // ok
             } else {
               strcpybuff(back->r.msg, "TYPE I error");
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
+#if 0
+            // --CWD--
+            char *a;
+
+            a = back->url_fil + strlen(back->url_fil) - 1;
+            while((a > back->url_fil) && (*a != '/'))
+              a--;
+            if (*a == '/') {    // ok repéré
+              char BIGSTK target[1024];
+
+              target[0] = '\0';
+              strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
+              if (strnotempty(target) == 0)
+                strcatbuff(target, "/");
+              strcpybuff(back->info, "cwd");
+              snprintf(line, sizeof(line), "CWD %s", target);
+              send_line(soc_ctl, line);
+              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+              _CHECK_HALT_FTP;
+              if (line[0] == '2') {
+                send_line(soc_ctl, "TYPE I");
+                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
+                _CHECK_HALT_FTP;
+                if (line[0] == '2') {
+                  // ok..
+                } else {
+                  strcpybuff(back->r.msg, "TYPE I error");
+                  // back->status=STATUS_FTP_READY;    // fini
+                  back->r.statuscode = STATUSCODE_INVALID;
+                }
+              } else {
+                snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
+                // back->status=STATUS_FTP_READY;    // fini
+                back->r.statuscode = STATUSCODE_INVALID;
+              }                 // sinon on est prêts
+            } else {
+              strcpybuff(back->r.msg, "Unexpected ftp error");
+              // back->status=STATUS_FTP_READY;    // fini
+              back->r.statuscode = STATUSCODE_INVALID;
+            }
+#endif
 
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
-                     linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
-                   linejmp(line));
+          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
+          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
       } else {
-        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
-                 linejmp(line));
+        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
+        // back->status=STATUS_FTP_READY;    // fini
         back->r.statuscode = STATUSCODE_INVALID;
       }
 
@@ -404,8 +469,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
             // -- fin analyse de l'adresse IP et du port --
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
-                     linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
@@ -436,13 +501,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 }
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
-                       linejmp(line));
+              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
-                     linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         }
@@ -548,8 +613,9 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   deletesoc(soc_dat);
                   soc_dat = INVALID_SOCKET;
                   //
-                  snprintf(back->r.msg, sizeof(back->r.msg),
-                           "RETR command error: %s", linejmp(line));
+                  snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
+                          linejmp(line));
+                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = STATUSCODE_INVALID;
                 }               // sinon on est prêts
               } else {
@@ -560,20 +626,23 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                 soc_dat = INVALID_SOCKET;
                 //
                 strcpybuff(back->r.msg, "Unable to connect");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }                 // sinon on est prêts
             } else {
               strcpybuff(back->r.msg, "Unable to create a socket");
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }                   // sinon on est prêts
           } else {
             snprintf(back->r.msg, sizeof(back->r.msg),
                      "Unable to resolve IP %s: %s", adr_ip, error);
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }                     // sinon on est prêts
         } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
-                   linejmp(line));
+          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
+          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }                       // sinon on est prêts
 #else
@@ -594,16 +663,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               //T_SOC soc_dat;
               if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
                 strcpybuff(back->r.msg, "Unable to accept connection");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
-              snprintf(back->r.msg, sizeof(back->r.msg),
-                       "RETR command error: %s", linejmp(line));
+              snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
-                     linejmp(line));
+            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -613,6 +683,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 #endif
         } else {
           strcpybuff(back->r.msg, "Unable to listen to a port");
+          // back->status=STATUS_FTP_READY;    // fini
           back->r.statuscode = STATUSCODE_INVALID;
         }
 #endif
@@ -636,18 +707,21 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             int len = 1;
             int read_len = 1024;
 
+            //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
+
             while((len > 0) && (!stop_ftp(back))) {
               // attendre les données
               len = 1;          // pas d'erreur pour le moment
               switch (wait_socket_receive(soc_dat, timeout)) {
               case -1:
                 strcpybuff(back->r.msg, "FTP read error");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
               case 0:
-                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
-                         timeout);
+                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
                 len = 0;        // fin
                 break;
@@ -669,14 +743,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                          }
                        */
                       strcpybuff(back->r.msg, "Write error");
+                      // back->status=STATUS_FTP_READY;    // fini
                       back->r.statuscode = STATUSCODE_INVALID;
                       len = 0;  // error
                     }
                   } else {
                     strcpybuff(back->r.msg, "Unexpected write error");
+                    // back->status=STATUS_FTP_READY;    // fini
                     back->r.statuscode = STATUSCODE_INVALID;
                   }
-                } else { // Erreur ou terminé
+                } else {        // Erreur ou terminé
+                  // back->status=STATUS_FTP_READY;    // fini
                   back->r.statuscode = 0;
                   if (back->r.totalsize > 0
                       && back->r.size != back->r.totalsize) {
@@ -685,6 +762,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                   }
                 }
                 read_len = 1024;
+                //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
               }
             }
             if (back->r.fp) {
@@ -693,6 +771,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
             }
           } else {
             strcpybuff(back->r.msg, "Unable to write file");
+            // back->status=STATUS_FTP_READY;    // fini
             back->r.statuscode = STATUSCODE_INVALID;
           }
 #ifdef _WIN32
@@ -708,14 +787,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
               get_ftp_line(soc_ctl, line, sizeof(line), timeout);
               if (line[0] == '2') {     // OK
                 strcpybuff(back->r.msg, "OK");
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = HTTP_OK;
               } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
-                         linejmp(line));
+                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
+                // back->status=STATUS_FTP_READY;    // fini
                 back->r.statuscode = STATUSCODE_INVALID;
               }
             } else {
               strcpybuff(back->r.msg, "FTP read error");
+              // back->status=STATUS_FTP_READY;    // fini
               back->r.statuscode = STATUSCODE_INVALID;
             }
           }
@@ -741,6 +822,7 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     back->r.statuscode = HTTP_OK;
     strcpybuff(back->r.msg, "OK");
   }
+  // back->status=STATUS_FTP_READY;    // fini
   return 0;
 }
 
@@ -907,7 +989,9 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
       break;
     }
 
+    //HTS_TOTAL_RECV_CHECK(dummy);     // Diminuer au besoin si trop de données reçues
     switch (recv(soc, &b, 1, 0)) {
+      //case 0: break;    // pas encore --> erreur (on attend)!
     case 1:
       HTS_STAT.HTS_TOTAL_RECV += 1;     // compter flux entrant
       if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
@@ -1034,6 +1118,7 @@ int wait_socket_receive(T_SOC soc, int timeout) {
 int stop_ftp(lien_back * back) {
   if (back->stop_ftp) {
     strcpybuff(back->r.msg, "Cancelled by User");
+    // back->status=STATUS_FTP_READY;    // fini
     back->r.statuscode = STATUSCODE_INVALID;
     return 1;
   }

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-12"
-#define HTTRACK_VERSIONID "3.49.12"
+#define HTTRACK_VERSION "3.49-11"
+#define HTTRACK_VERSIONID "3.49.11"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 
@@ -56,10 +56,6 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
-// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
-#include <stdint.h>
-#include <inttypes.h>
-
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -115,6 +111,29 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
+#ifndef HTS_LONGLONG
+#ifdef SIZEOF_LONG_LONG
+#if SIZEOF_LONG_LONG == 8
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+#ifndef HTS_LONGLONG
+#ifdef __sun
+#define HTS_LONGLONG 0
+#endif
+#ifdef __osf__
+#define HTS_LONGLONG 0
+#endif
+#ifdef __linux
+#define HTS_LONGLONG 1
+#endif
+#ifdef _WIN32
+#define HTS_LONGLONG 1
+#endif
+#endif
+#endif
+
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -307,11 +326,59 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
-typedef int64_t LLint;
-typedef int64_t TStamp;
-/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
-#define LLintP "%" PRId64
+#ifndef HTS_LONGLONG
+#ifdef HTS_NO_64_BIT
+#define HTS_LONGLONG 0
+#else
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+/* Wide integer types, chosen per platform.
+   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
+           plain int where 64-bit is unavailable).
+   TStamp: timestamp/duration in the same width (a double in the no-64-bit
+           fallback).
+   LLintP: the printf conversion for an LLint. */
+#if HTS_LONGLONG
+#ifdef LLINT_FORMAT
+typedef LLINT_TYPE LLint;
+
+typedef LLINT_TYPE TStamp;
+
+#define LLintP LLINT_FORMAT
+#else
+
+#ifdef _WIN32
+typedef __int64 LLint;
+
+typedef __int64 TStamp;
+
+#define LLintP "%I64d"
+#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
+       defined(__64BIT__))
+
+typedef long int LLint;
+
+typedef long int TStamp;
+
+#define LLintP "%ld"
+#else
+typedef long long int LLint;
+
+typedef long long int TStamp;
+
+#define LLintP "%lld"
+#endif
+
+#endif /* HTS_LONGLONG */
+
+#else
+typedef int LLint;
+
+#define LLintP "%d"
+typedef double TStamp;
+#endif
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain
@@ -355,10 +422,9 @@ typedef int T_SOC;
 #endif
 
 /* Permission bits for created folders and files (mkdir and chmod).
-   PROTECT_FOLDER/FILE are owner-only. With HTS_ACCESS set (the default) the
-   ACCESS_ modes also grant group/other read; otherwise they stay owner-only. */
+   PROTECT_FOLDER is owner-only. With HTS_ACCESS set (the default) the ACCESS_
+   modes also grant group/other read; otherwise they stay owner-only. */
 #define HTS_PROTECT_FOLDER (S_IRUSR | S_IWUSR | S_IXUSR)
-#define HTS_PROTECT_FILE (S_IRUSR | S_IWUSR)
 
 #if HTS_ACCESS
 #define HTS_ACCESS_FILE (S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH)

src/htshelp.c

@@ -152,6 +152,16 @@ void help_wizard(httrackp * opt) {
 #define str (buffers->str)
 #define argv (buffers->argv)
 
+  //char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
+  //char *mainpath = (char *) malloct(256);
+  //char *projname = (char *) malloct(256);
+  //char *stropt = (char *) malloct(2048);        // options
+  //char *stropt2 = (char *) malloct(2048);       // options longues
+  //char *strwild = (char *) malloct(2048);       // wildcards
+  //char *cmd = (char *) malloct(4096);
+  //char *str = (char *) malloct(256);
+  //char **argv = (char **) malloct(256 * sizeof(char *));
+
   //
   char *a;
 
@@ -217,6 +227,8 @@ void help_wizard(httrackp * opt) {
   strcatbuff(stropt2, str);
   strcatbuff(stropt2, projname);
   strcatbuff(stropt2, "\" ");
+  // Créer si ce n'est fait un index.html 1er niveau
+  make_empty_index(str);
   //
   printf("\n");
   printf("Enter URLs (separated by commas or blank spaces) :");
@@ -337,6 +349,8 @@ void help_wizard(httrackp * opt) {
       }
       hts_main(argc, argv);
     }
+    //} else {
+    //  help("httrack",1);
   }
 
   /* Free buffers */
@@ -441,6 +455,23 @@ void help_catchurl(const char *dest_path) {
     printf("Unable to create a temporary proxy (no remaining port)\n");
 }
 
+// Créer un index.html vide
+void make_empty_index(const char *str) {
+#if 0
+  if (!fexist(fconcat(str, "index.html"))) {
+    FILE *fp = fopen(fconcat(str, "index.html"), "wb");
+
+    if (fp) {
+      fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
+      fprintf(fp,
+              "<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
+              CRLF);
+      fclose(fp);
+    }
+  }
+#endif
+}
+
 // mini-aide  (h: help)
 //           y
 void help(const char *app, int more) {
@@ -543,11 +574,10 @@ void help(const char *app, int more) {
   infomsg("Spider options:");
   infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
   infomsg(" %K  load extra cookies from a Netscape cookies.txt");
-  infomsg(" %Y  explain which filter rule accepts or rejects a URL, then exit");
   infomsg
     ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
-  infomsg("  j *parse scripts (j0 don't parse, bitmask: |1 parse default, |4 "
-          "don't parse .js |8 don't be aggressive)");
+  infomsg
+    ("  j *parse Java Classes (j0 don't parse, bitmask: |1 parse default, |2 don't parse .class |4 don't parse .js |8 don't be aggressive)");
   infomsg
     ("  sN follow robots.txt and meta robots tags (0=never,1=sometimes,* 2=always, 3=always (even strict rules))");
   infomsg
@@ -571,7 +601,8 @@ void help(const char *app, int more) {
     ("     can also be used to force a specific file type: --assume foo.cgi=text/html");
   infomsg
     (" @iN internet protocol (0=both ipv6+ipv4, 4=ipv4 only, 6=ipv6 only)");
-  infomsg(" %w  disable a specific external mime module (-%w httrack-plugin)");
+  infomsg
+    (" %w  disable a specific external mime module (-%w htsswf -%w htsjava)");
   infomsg("");
   infomsg("Browser ID:");
   infomsg
@@ -777,4 +808,7 @@ void help(const char *app, int more) {
   infomsg("[compiled: " HTS_PLATFORM_NAME "]");
 #endif
   infomsg(NULL);
+
+//  infomsg("  R  *relative links (e.g ../link)\n");
+//  infomsg("  A   absolute links (e.g /www.adr/link)\n");
 }

src/htshelp.h

@@ -45,6 +45,7 @@ typedef struct httrackp httrackp;
 
 void infomsg(const char *msg);
 void help(const char *app, int more);
+void make_empty_index(const char *str);
 void help_wizard(httrackp * opt);
 int help_query(const char *list, int def);
 void help_catchurl(const char *dest_path);

src/htsindex.c

@@ -142,6 +142,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
   char keyword[KEYW_LEN + 32];
   int i = 0;
 
+  //
+  //int WordIndexSize = 1024;
   coucal WordIndexHash = NULL;
   FILE *tmpfp = NULL;
 
@@ -176,6 +178,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
              || (strfield2(mime, "text/css"))
     ) {
     inscript = 1;
+    //} else if (strfield2(mime, "text/vnd.wap.wml")) {   // humm won't work in many cases
+    //  inscript=0;
   } else
     return 0;
 
@@ -271,6 +275,8 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
 
   // Process indexing for this page
   {
+    //FILE* fp=NULL;
+    //fp=fopen(concat(indexpath,"index.txt"),"ab");
     if (fp_tmpproject) {
       while(!feof(tmpfp)) {
         char line[KEYW_LEN + 32];
@@ -280,6 +286,7 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
           intptr_t e = 0;
 
           if (coucal_read(WordIndexHash, line, &e)) {
+            //if (e) {
             char BIGSTK savelst[HTS_URLMAXSIZE * 2];
 
             e++;                /* 0 means "once" */
@@ -293,9 +300,11 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
             fprintf(fp_tmpproject, "%s %d %s\n", line,
                     (int) (KEYW_SORT_MAXCOUNT - e), savelst);
             hts_primindex_size++;
+            //}
           }
         }
       }
+      //fclose(fp);
     }
   }
 
@@ -321,6 +330,7 @@ void index_finish(const char *indexpath, int mode) {
   off_t size = fpsize(fp_tmpproject);
 
   if (size > 0) {
+    //FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
     if (fp_tmpproject) {
       tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
       if (tab) {
@@ -388,6 +398,8 @@ void index_finish(const char *indexpath, int mode) {
                       if (total_hit) {
                         if (mode == 1)  // TEXT
                           fprintf(fp, "\t=%d\r\n", total_hit);
+                        //else                // HTML
+                        //  fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
                         if ((((total_hit * 1000) / hts_primindex_words) >=
                              KEYW_USELESS1K)
                             || (((total_line * 1000) / index) >=
@@ -404,6 +416,8 @@ void index_finish(const char *indexpath, int mode) {
                           if (mode == 1)        // TEXT
                             fprintf(fp, "\t(%d)\r\n",
                                     ((total_hit * 1000) / hts_primindex_words));
+                          //else                // HTML
+                          //  fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
                         }
                       }
                       if (mode == 1)    // TEXT

src/htsjava.c

@@ -0,0 +1,528 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: Java classes parser                                    */
+/* Author: Yann Philippot                                       */
+/* ------------------------------------------------------------ */
+
+/* Version: Oct/2000 */
+/* Fixed: problems with class structure (10/2000) */
+
+// htsjava.c - Parseur de classes java
+
+#ifdef HAVE_CONFIG_H
+#include "config.h"
+#endif
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#if ( defined(_WIN32) ||defined(HAVE_SYS_TYPES_H) )
+#include <sys/types.h>
+#endif
+#ifdef HAVE_UNISTD_H
+#include <unistd.h>
+#endif
+
+/* Standard httrack module includes */
+#include "httrack-library.h"
+#include "htsopt.h"
+#include "htsdefines.h"
+
+/* Module structures */
+#include "htsmodules.h"
+
+/* We link to libhttrack, we can use its functions */
+#include "httrack-library.h"
+
+/* This file */
+#include "htsjava.h"
+
+/* calloct/freet wrappers */
+#include "htssafe.h"
+
+static int reverse_endian(void) {
+  int endian = 1;
+
+  return (*((char *) &endian) == 1);
+}
+
+/* big/little endian swap */
+#define hts_swap16(A) ( (((A) & 0xFF)<<8) | (((A) & 0xFF00)>>8) )
+#define hts_swap32(A) ( (( (hts_swap16(A)) & 0xFFFF)<<16) | (( (hts_swap16(A>>16)) & 0xFFFF)) )
+
+/* Static definitions */
+static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp, RESP_STRUCT,
+                             int *);
+static unsigned short int readshort(FILE * fp);
+static int tris(httrackp * opt, char *);
+static char *printname(char[1024], char[1024]);
+
+// ** HTS_xx sinon pas pris par VC++
+#define HTS_CLASS  7
+#define HTS_FIELDREF  9
+#define HTS_METHODREF  10
+#define HTS_STRING  8
+#define HTS_INTEGER  3
+#define HTS_FLOAT  4
+#define HTS_LONG  5
+#define HTS_DOUBLE  6
+#define HTS_INTERFACE  11
+#define HTS_NAMEANDTYPE  12
+#define HTS_ASCIZ  1
+#define HTS_UNICODE  2
+
+#define JAVADEBUG 0
+
+static const char *libName = "htsjava";
+
+#ifdef _WIN32
+#define strcasecmp(a,b) stricmp(a,b)
+#define strncasecmp(a,b,n) strnicmp(a,b,n)
+#endif
+
+static int detect_mime(htsmoduleStruct * str) {
+  const char *savename = str->filename;
+
+  if (savename) {
+    int len = (int) strlen(savename);
+
+    if (len > 6 && strcasecmp(savename + len - 6, ".class") == 0) {
+      return 1;
+    }
+  }
+  return 0;
+}
+
+static int hts_detect_java(t_hts_callbackarg * carg, httrackp * opt,
+                           htsmoduleStruct * str) {
+  /* Call parent functions if multiple callbacks are chained. */
+  if (CALLBACKARG_PREV_FUN(carg, detect) != NULL) {
+    if (CALLBACKARG_PREV_FUN(carg, detect)
+        (CALLBACKARG_PREV_CARG(carg), opt, str)) {
+      return 1;                 /* Found before us, let them have the priority */
+    }
+  }
+
+  /* Check MIME */
+  if (detect_mime(str)) {
+    str->wrapper_name = libName;        /* Our ID */
+    return 1;                   /* Known format, we take it */
+  }
+
+  return 0;                     /* Unknown format */
+}
+
+static off_t fsize(const char *s) {
+  STRUCT_STAT st;
+
+  if (STAT(s, &st) == 0 && S_ISREG(st.st_mode)) {
+    return st.st_size;
+  } else {
+    return -1;
+  }
+}
+
+static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
+                          htsmoduleStruct * str) {
+  /* The wrapper_name memebr has changed: not for us anymore */
+  if (str->wrapper_name == NULL || strcmp(str->wrapper_name, libName) != 0) {
+    /* Call parent functions if multiple callbacks are chained. */
+    if (CALLBACKARG_PREV_FUN(carg, parse) != NULL) {
+      return CALLBACKARG_PREV_FUN(carg, parse) (CALLBACKARG_PREV_CARG(carg),
+                                                opt, str);
+    }
+    strcpy(str->err_msg,
+           "unexpected error: bad wrapper_name and no previous wrapper");
+    return 0;                   /* Unexpected error */
+  } else {
+    if (detect_mime(str)) {
+
+      /* (Legacy code) */
+      char catbuff[CATBUFF_SIZE];
+      FILE *fpout;
+      JAVA_HEADER header;
+      RESP_STRUCT *tab;
+      const char *file = str->filename;
+
+      str->relativeToHtmlLink = 1;
+
+#if JAVADEBUG
+      printf("fopen\n");
+#endif
+      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
+        //fprintf(stderr, "Cannot open input file.\n");
+        sprintf(str->err_msg, "Unable to open file %s", file);
+        return 0;               // une erreur..
+      }
+#if JAVADEBUG
+      printf("fread\n");
+#endif
+      //if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) {   // pas complet..
+      if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
+        fclose(fpout);
+        sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
+                (LLint) fsize(file));
+        return 0;
+      }
+#if JAVADEBUG
+      printf("header\n");
+#endif
+      // tester en tête
+      if (reverse_endian()) {
+        header.magic = hts_swap32(header.magic);
+        header.count = hts_swap16(header.count);
+      }
+      if (header.magic != 0xCAFEBABE) {
+        sprintf(str->err_msg, "non java file");
+        if (fpout) {
+          fclose(fpout);
+          fpout = NULL;
+        }
+        return 0;
+      }
+
+      /* A constant-pool entry is >= 1 byte on disk; reject a count exceeding
+         the file size (hostile .class ~68 MB alloc DoS). */
+      if (!hts_count_fits(header.count, (LLint) fsize(file))) {
+        fclose(fpout);
+        sprintf(str->err_msg,
+                "Invalid constant pool count %u (file len " LLintP ")",
+                (unsigned) header.count, (LLint) fsize(file));
+        return 0;
+      }
+      tab = (RESP_STRUCT *) calloct(header.count, sizeof(RESP_STRUCT));
+      if (!tab) {
+        sprintf(str->err_msg, "Unable to alloc %d bytes",
+                (int) sizeof(RESP_STRUCT));
+        if (fpout) {
+          fclose(fpout);
+          fpout = NULL;
+        }
+        return 0;               // erreur..
+      }
+#if JAVADEBUG
+      printf("calchead\n");
+#endif
+      {
+        int i;
+
+        for(i = 1; i < header.count; i++) {
+          int err = 0;          // ++    
+
+          tab[i] = readtable(str, fpout, tab[i], &err);
+          if (!err) {
+            if ((tab[i].type == HTS_LONG) || (tab[i].type == HTS_DOUBLE))
+              i++;              //2 element si double ou float
+          } else {              // ++ une erreur est survenue!
+            if (strnotempty(str->err_msg) == 0)
+              strcpy(str->err_msg, "Internal readtable error");
+            freet(tab);
+            if (fpout) {
+              fclose(fpout);
+              fpout = NULL;
+            }
+            return 0;
+          }
+        }
+
+      }
+
+#if JAVADEBUG
+      printf("addfiles\n");
+#endif
+      {
+        //unsigned int acess;
+        unsigned int Class;
+        unsigned int SClass;
+        int i;
+
+        //acess = readshort(fpout);
+        Class = readshort(fpout);
+        SClass = readshort(fpout);
+
+        for(i = 1; i < header.count; i++) {
+
+          if (tab[i].type == HTS_CLASS) {
+
+            if ((tab[i].index1 < header.count) && (tab[i].index1 >= 0)) {
+
+              if ((tab[i].index1 != SClass) && (tab[i].index1 != Class)
+                  && (tab[tab[i].index1].name[0] != '[')) {
+
+                if (!strstr(tab[tab[i].index1].name, "java/")) {
+                  char BIGSTK tempo[1024];
+
+                  tempo[0] = '\0';
+
+                  snprintf(tempo, sizeof(tempo), "%s.class", tab[tab[i].index1].name);
+#if JAVADEBUG
+                  printf("add %s\n", tempo);
+#endif
+                  if (tab[tab[i].index1].file_position >= 0)
+                    str->addLink(str, tempo);   /* tab[tab[i].index1].file_position */
+                }
+
+              }
+            } else {
+              i = header.count; // exit 
+            }
+          }
+
+        }
+      }
+
+#if JAVADEBUG
+      printf("end\n");
+#endif
+      freet(tab);
+      if (fpout) {
+        fclose(fpout);
+        fpout = NULL;
+      }
+      return 1;
+
+    } else {
+      strcpy(str->err_msg, "bad MIME type");
+    }
+  }
+  return 0;                     /* Error */
+}
+
+/*
+module entry point 
+*/
+EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv);
+EXTERNAL_FUNCTION int hts_plug(httrackp * opt, const char *argv) {
+  /* Plug callback functions */
+  CHAIN_FUNCTION(opt, detect, hts_detect_java, NULL);
+  CHAIN_FUNCTION(opt, parse, hts_parse_java, NULL);
+
+  return 1;                     /* success */
+}
+
+// error: !=0 si erreur fatale
+static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
+                             RESP_STRUCT trans, int *error) {
+  char rname[1024];
+  unsigned short int length;
+  int j;
+
+  *error = 0;                   // pas d'erreur
+  trans.file_position = -1;
+  trans.type = (int) (unsigned char) fgetc(fp);
+  switch (trans.type) {
+  case HTS_CLASS:
+    strcpy(trans.name, "Class");
+    trans.index1 = readshort(fp);
+    break;
+
+  case HTS_FIELDREF:
+    strcpy(trans.name, "Field Reference");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+
+  case HTS_METHODREF:
+    strcpy(trans.name, "Method Reference");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+
+  case HTS_INTERFACE:
+    strcpy(trans.name, "Interface Method Reference");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+  case HTS_NAMEANDTYPE:
+    strcpy(trans.name, "Name and Type");
+    trans.index1 = readshort(fp);
+    readshort(fp);
+    break;
+
+  case HTS_STRING:             // CONSTANT_String
+    strcpy(trans.name, "String");
+    trans.index1 = readshort(fp);
+    break;
+
+  case HTS_INTEGER:
+    strcpy(trans.name, "Integer");
+    for(j = 0; j < 4; j++)
+      fgetc(fp);
+    break;
+
+  case HTS_FLOAT:
+    strcpy(trans.name, "Float");
+    for(j = 0; j < 4; j++)
+      fgetc(fp);
+    break;
+
+  case HTS_LONG:
+    strcpy(trans.name, "Long");
+    for(j = 0; j < 8; j++)
+      fgetc(fp);
+    break;
+  case HTS_DOUBLE:
+    strcpy(trans.name, "Double");
+    for(j = 0; j < 8; j++)
+      fgetc(fp);
+    break;
+
+  case HTS_ASCIZ:
+  case HTS_UNICODE:
+
+    if (trans.type == HTS_ASCIZ)
+      strcpy(trans.name, "HTS_ASCIZ");
+    else
+      strcpy(trans.name, "HTS_UNICODE");
+
+    {
+      char BIGSTK buffer[1024];
+      char *p;
+
+      p = &buffer[0];
+
+      //fflush(fp); 
+      trans.file_position = ftell(fp);
+      length = readshort(fp);
+      if (length < HTS_URLMAXSIZE) {
+        // while ((length > 0) && (length<500)) {
+        while(length > 0) {
+          *p++ = fgetc(fp);
+
+          length--;
+        }
+        *p = '\0';
+
+        //#if JDEBUG
+        //      if(tris(buffer)==1) printf("%s\n ",buffer);
+        //      if(tris(buffer)==2)  printf("%s\n ",printname(buffer));
+        //#endif
+        if (tris(str->opt, buffer) == 1)
+          str->addLink(str, buffer);    /* trans.file_position */
+        else if (tris(str->opt, buffer) == 2)
+          str->addLink(str, printname(rname, buffer));
+
+        strcpy(trans.name, buffer);
+      } else {                  // gros pb
+        while((length > 0) && (!feof(fp))) {
+          fgetc(fp);
+          length--;
+        }
+        if (!feof(fp)) {
+          trans.type = -1;
+        } else {
+          sprintf(str->err_msg, "Internal structure error (ASCII)");
+          *error = 1;
+        }
+        return (trans);
+      }
+    }
+    break;
+  default:
+    // printf("Type inconnue\n");
+    // on arrête tout 
+    sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
+    *error = 1;
+    return (trans);
+    break;
+  }
+  return (trans);
+}
+
+static unsigned short int readshort(FILE * fp) {
+  unsigned short int valint;
+
+  if (fread(&valint, sizeof(valint), 1, fp) == 1) {
+    if (reverse_endian())
+      return hts_swap16(valint);
+    else
+      return valint;
+  } else {
+    return 0;
+  }
+
+}
+
+static int tris(httrackp * opt, char *buffer) {
+  char catbuff[CATBUFF_SIZE];
+
+  //
+  // Java
+  if ((buffer[0] == '[') && buffer[1] == 'L' && (!strstr(buffer, "java/")))
+    return 2;
+  if (strstr(buffer, ".gif") || strstr(buffer, ".jpg")
+      || strstr(buffer, ".jpeg") || strstr(buffer, ".au"))
+    return 1;
+  // Ajouts R.X: test type
+  // Autres fichiers
+  {
+    char type[256];
+
+    if (get_httptype_sized(opt, type, sizeof(type), buffer,
+                           0)) // recognized type
+      return 1;
+    // ajout RX 05/2001
+    else if (is_dyntype(get_ext(catbuff, sizeof(catbuff), buffer)))      // asp,cgi...
+      return 1;
+  }
+  return 0;
+}
+
+static char *printname(char rname[1024], char name[1024]) {
+  char *p;
+  char *p1;
+  size_t j;
+
+  rname[0] = '\0';
+  //
+
+  p = &name[0];
+
+  if (*p != '[') {
+    return rname;  // ""
+  }
+  p += 2;
+  //rname=(char*)calloct(strlen(name)+8,sizeof(char));
+  p1 = rname;
+  for(j = 0; name[j] != '\0'; j++, p++) {
+    if (*p == '/')
+      *p1 = '.';
+    if (*p == ';') {
+      *p1 = '\0';
+      strcat(rname, ".class");
+      return rname;
+    } else
+      *p1 = *p;
+    p1++;
+  }
+  p1 -= 3;
+  *p1 = '\0';
+  return rname;
+
+}

src/htsjava.h

@@ -25,27 +25,46 @@ would dishonor our work and waste the many hours we have spent on it.
 Please visit our Website: http://www.httrack.com
 */
 
-/* Shared helpers for the libFuzzer harnesses. */
-#ifndef FUZZ_H
-#define FUZZ_H
+/* ------------------------------------------------------------ */
+/* File: Java classes parser .h                                 */
+/* Author: Yann Philippot                                       */
+/* ------------------------------------------------------------ */
 
-#define HTS_INTERNAL_BYTECODE
+#ifndef HTSJAVA_DEFH
+#define HTSJAVA_DEFH
 
-#include <stddef.h>
 #include <stdint.h>
-#include <string.h>
 
-#include "htsbase.h"
+#ifndef HTS_DEF_FWSTRUCT_JAVA_HEADER
+#define HTS_DEF_FWSTRUCT_JAVA_HEADER
+typedef struct JAVA_HEADER JAVA_HEADER;
+#endif
+/* 10-byte on-disk .class header image, fread() directly: fields need exact
+   widths (LP64's 8-byte 'unsigned long' magic never matched 0xCAFEBABE). */
+struct JAVA_HEADER {
+  uint32_t magic;
+  uint16_t minor;
+  uint16_t major;
+  uint16_t count;
+};
 
-/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
-static char *fuzz_strdup(const uint8_t *data, size_t size) {
-  char *s = malloct(size + 1);
+#ifndef HTS_DEF_FWSTRUCT_RESP_STRUCT
+#define HTS_DEF_FWSTRUCT_RESP_STRUCT
+typedef struct RESP_STRUCT RESP_STRUCT;
+#endif
+struct RESP_STRUCT {
+  int file_position;
+  //
+  unsigned int index1;
+  unsigned int type;
+  char name[1024];
+};
 
-  memcpy(s, data, size);
-  s[size] = '\0';
-  return s;
-}
+/* Library internal definictions */
+#ifdef HTS_INTERNAL_BYTECODE
 
-int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
+EXTERNAL_FUNCTION int hts_plug_java(httrackp * opt, const char *argv);
+
+#endif
 
 #endif

src/htsjava.vcproj

@@ -0,0 +1,373 @@
+<?xml version="1.0" encoding="Windows-1252"?>
+<VisualStudioProject
+	ProjectType="Visual C++"
+	Version="9,00"
+	Name="htsjava"
+	ProjectGUID="{2F353408-2C6C-4D31-9AC0-08BDC064DF64}"
+	RootNamespace="htsjava"
+	Keyword="Win32Proj"
+	TargetFrameworkVersion="131072"
+	>
+	<Platforms>
+		<Platform
+			Name="Win32"
+		/>
+		<Platform
+			Name="x64"
+		/>
+	</Platforms>
+	<ToolFiles>
+	</ToolFiles>
+	<Configurations>
+		<Configuration
+			Name="Debug|Win32"
+			OutputDirectory="C:\temp\htsjava\Debug"
+			IntermediateDirectory="C:\temp\htsjava\Debug"
+			ConfigurationType="2"
+			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
+				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="1"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="4"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				OutputFile="C:\temp\httrack\htsjava.dll"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				ProgramDatabaseFile="$(OutDir)/htsjava.pdb"
+				SubSystem="2"
+				RandomizedBaseAddress="1"
+				DataExecutionPrevention="0"
+				ImportLibrary="$(OutDir)/htsjava.lib"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Debug|x64"
+			OutputDirectory="$(SolutionDir)$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="2"
+			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				Optimization="0"
+				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
+				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
+				MinimalRebuild="true"
+				BasicRuntimeChecks="3"
+				RuntimeLibrary="1"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="3"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				OutputFile="C:\temp\httrack64\htsjava.dll"
+				LinkIncremental="2"
+				GenerateDebugInformation="true"
+				ProgramDatabaseFile="$(OutDir)/htsjava.pdb"
+				SubSystem="2"
+				RandomizedBaseAddress="1"
+				DataExecutionPrevention="0"
+				ImportLibrary="$(OutDir)/htsjava.lib"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|Win32"
+			OutputDirectory="C:\temp\htsjava\Release"
+			IntermediateDirectory="C:\temp\htsjava\Release"
+			ConfigurationType="2"
+			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
+				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
+				RuntimeLibrary="0"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="0"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				OutputFile="O:\HTTrack\httrack\htsjava.dll"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				ProgramDatabaseFile="O:\HTTrack\httrack\htsjava.pdb"
+				SubSystem="2"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				RandomizedBaseAddress="1"
+				DataExecutionPrevention="0"
+				ImportLibrary="$(OutDir)/htsjava.lib"
+				TargetMachine="1"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+				CommandLine="&quot;C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\signtool.exe&quot; sign /f &quot;Z:\Mes Documents\certificats\httrack-com\httrack-signing-certificate.pfx&quot; /t http://timestamp.verisign.com/scripts/timstamp.dll $(TargetPath)&#x0D;&#x0A;"
+			/>
+		</Configuration>
+		<Configuration
+			Name="Release|x64"
+			OutputDirectory="$(SolutionDir)$(PlatformName)\$(ConfigurationName)"
+			IntermediateDirectory="$(PlatformName)\$(ConfigurationName)"
+			ConfigurationType="2"
+			InheritedPropertySheets="$(VCInstallDir)VCProjectDefaults\UpgradeFromVC71.vsprops"
+			CharacterSet="2"
+			>
+			<Tool
+				Name="VCPreBuildEventTool"
+			/>
+			<Tool
+				Name="VCCustomBuildTool"
+			/>
+			<Tool
+				Name="VCXMLDataGeneratorTool"
+			/>
+			<Tool
+				Name="VCWebServiceProxyGeneratorTool"
+			/>
+			<Tool
+				Name="VCMIDLTool"
+				TargetEnvironment="3"
+			/>
+			<Tool
+				Name="VCCLCompilerTool"
+				AdditionalIncludeDirectories="C:\Dev\openssl-1.0.1j\include"
+				PreprocessorDefinitions="_CRT_SECURE_NO_WARNINGS;_CRT_NONSTDC_NO_DEPRECATE"
+				RuntimeLibrary="0"
+				UsePrecompiledHeader="0"
+				WarningLevel="3"
+				Detect64BitPortabilityProblems="true"
+				DebugInformationFormat="0"
+			/>
+			<Tool
+				Name="VCManagedResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCResourceCompilerTool"
+			/>
+			<Tool
+				Name="VCPreLinkEventTool"
+			/>
+			<Tool
+				Name="VCLinkerTool"
+				OutputFile="O:\HTTrack\httrack\x64\htsjava.dll"
+				LinkIncremental="1"
+				GenerateDebugInformation="true"
+				ProgramDatabaseFile="O:\HTTrack\httrack\x64\htsjava.pdb"
+				SubSystem="2"
+				OptimizeReferences="2"
+				EnableCOMDATFolding="2"
+				RandomizedBaseAddress="1"
+				DataExecutionPrevention="0"
+				ImportLibrary="$(OutDir)/htsjava.lib"
+				TargetMachine="17"
+			/>
+			<Tool
+				Name="VCALinkTool"
+			/>
+			<Tool
+				Name="VCManifestTool"
+			/>
+			<Tool
+				Name="VCXDCMakeTool"
+			/>
+			<Tool
+				Name="VCBscMakeTool"
+			/>
+			<Tool
+				Name="VCFxCopTool"
+			/>
+			<Tool
+				Name="VCAppVerifierTool"
+			/>
+			<Tool
+				Name="VCPostBuildEventTool"
+				CommandLine="&quot;C:\Program Files\Microsoft SDKs\Windows\v6.0A\Bin\signtool.exe&quot; sign /f &quot;Z:\Mes Documents\certificats\httrack-com\httrack-signing-certificate.pfx&quot; /t http://timestamp.verisign.com/scripts/timstamp.dll $(TargetPath)&#x0D;&#x0A;"
+			/>
+		</Configuration>
+	</Configurations>
+	<References>
+	</References>
+	<Files>
+		<Filter
+			Name="Source Files"
+			Filter="cpp;c;cxx;def;odl;idl;hpj;bat;asm;asmx"
+			UniqueIdentifier="{4FC737F1-C7A5-4376-A066-2A32D752A2FF}"
+			>
+			<File
+				RelativePath="htsjava.c"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Header Files"
+			Filter="h;hpp;hxx;hm;inl;inc;xsd"
+			UniqueIdentifier="{93995380-89BD-4b04-88EB-625FBE52EBFB}"
+			>
+			<File
+				RelativePath="htsjava.h"
+				>
+			</File>
+		</Filter>
+		<Filter
+			Name="Resource Files"
+			Filter="rc;ico;cur;bmp;dlg;rc2;rct;bin;rgs;gif;jpg;jpeg;jpe;resx"
+			UniqueIdentifier="{67DA6AB6-F800-4c08-8B7A-83BB121AAD01}"
+			>
+		</Filter>
+	</Files>
+	<Globals>
+	</Globals>
+</VisualStudioProject>

src/htslib.c

@@ -598,27 +598,56 @@ static const char *hts_mime_modern[][2] = {
 
 // Reserved (RFC2396)
 #define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
-#define CHAR_RESERVED(c)                                                       \
-  (CIS(c, ';') || CIS(c, '/') || CIS(c, '?') || CIS(c, ':') || CIS(c, '@') ||  \
-   CIS(c, '&') || CIS(c, '=') || CIS(c, '+') || CIS(c, '$') || CIS(c, ','))
+#define CHAR_RESERVED(c)  ( CIS(c,';') \
+                         || CIS(c,'/') \
+                         || CIS(c,'?') \
+                         || CIS(c,':') \
+                         || CIS(c,'@') \
+                         || CIS(c,'&') \
+                         || CIS(c,'=') \
+                         || CIS(c,'+') \
+                         || CIS(c,'$') \
+                         || CIS(c,',') )
+//#define CHAR_RESERVED(c)  ( strchr(";/?:@&=+$,",(unsigned char)(c)) != 0 )
 // Delimiters (RFC2396)
-#define CHAR_DELIM(c)                                                          \
-  (CIS(c, '<') || CIS(c, '>') || CIS(c, '#') || CIS(c, '%') || CIS(c, '\"'))
+#define CHAR_DELIM(c)     ( CIS(c,'<') \
+                         || CIS(c,'>') \
+                         || CIS(c,'#') \
+                         || CIS(c,'%') \
+                         || CIS(c,'\"') )
+//#define CHAR_DELIM(c)     ( strchr("<>#%\"",(unsigned char)(c)) != 0 )
 // Unwise (RFC2396)
-#define CHAR_UNWISE(c)                                                         \
-  (CIS(c, '{') || CIS(c, '}') || CIS(c, '|') || CIS(c, '\\') || CIS(c, '^') || \
-   CIS(c, '[') || CIS(c, ']') || CIS(c, '`'))
+#define CHAR_UNWISE(c)    ( CIS(c,'{') \
+                         || CIS(c,'}') \
+                         || CIS(c,'|') \
+                         || CIS(c,'\\') \
+                         || CIS(c,'^') \
+                         || CIS(c,'[') \
+                         || CIS(c,']') \
+                         || CIS(c,'`') )
+//#define CHAR_UNWISE(c)    ( strchr("{}|\\^[]`",(unsigned char)(c)) != 0 )
 // Special (escape chars) (RFC2396 + >127 )
 #define CHAR_LOW(c)       ( ((unsigned char)(c) <= 31) )
 #define CHAR_HIG(c)       ( ((unsigned char)(c) >= 127) )
 #define CHAR_SPECIAL(c)   ( CHAR_LOW(c) || CHAR_HIG(c) )
 // We try to avoid them and encode them instead
-#define CHAR_XXAVOID(c)                                                        \
-  (CIS(c, ' ') || CIS(c, '*') || CIS(c, '\'') || CIS(c, '\"') ||               \
-   CIS(c, '&') || CIS(c, '!'))
-#define CHAR_MARK(c)                                                           \
-  (CIS(c, '-') || CIS(c, '_') || CIS(c, '.') || CIS(c, '!') || CIS(c, '~') ||  \
-   CIS(c, '*') || CIS(c, '\'') || CIS(c, '(') || CIS(c, ')'))
+#define CHAR_XXAVOID(c)   ( CIS(c,' ') \
+                         || CIS(c,'*') \
+                         || CIS(c,'\'') \
+                         || CIS(c,'\"') \
+                         || CIS(c,'&') \
+                         || CIS(c,'!') )
+//#define CHAR_XXAVOID(c)   ( strchr(" *'\"!",(unsigned char)(c)) != 0 )
+#define CHAR_MARK(c)      ( CIS(c,'-') \
+                         || CIS(c,'_') \
+                         || CIS(c,'.') \
+                         || CIS(c,'!') \
+                         || CIS(c,'~') \
+                         || CIS(c,'*') \
+                         || CIS(c,'\'') \
+                         || CIS(c,'(') \
+                         || CIS(c,')') )
+//#define CHAR_MARK(c)      ( strchr("-_.!~*'()",(unsigned char)(c)) != 0 )
 
 // conversion éventuelle / vers antislash
 #ifdef _WIN32
@@ -812,12 +841,15 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
 // treat: traiter header?
 // waitconnect: attendre le connect()
 // note: dans retour, on met les params du proxy
-T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
-                  const char *xsend, const char *adr, const char *fil,
-                  htsblk *retour) {
+T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
+                  const char *xsend, const char *adr, const char *fil, htsblk * retour) {
+  //htsblk retour;
+  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
   T_SOC soc = INVALID_SOCKET;
   char BIGSTK tempo_fil[HTS_URLMAXSIZE * 2];
 
+  //char *p,*q;
+
   // retour prédéfini: erreur
   if (retour) {
     retour->adr = NULL;
@@ -902,6 +934,7 @@ T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
         strcpybuff(retour->msg, "Unable to open local file");
       else {
         // Note: On passe par un FILE* (plus propre)
+        //soc=open(fil,O_RDONLY,0);    // en lecture seule!
         retour->fp = FOPEN(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), 
           unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil)), "rb");      // ouvrir
         if (retour->fp == NULL)
@@ -978,10 +1011,17 @@ T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
 
         } while(strnotempty(rcvd));
 
-      } else { // si GET, on recevra l'en tête APRES
+        //rcvsize=-1;    // forCER CHARGEMENT INCONNU
+
+        //if (retour)
+        //  retour->totalsize=rcvsize;
+
+      } else {                  // si GET, on recevra l'en tête APRES
+        //rcvsize=-1;    // on ne connait pas la taille de l'en-tête
         if (retour)
           retour->totalsize = -1;
       }
+
     }
 
   }
@@ -1059,10 +1099,10 @@ static int append_cookie_header(buff_struct *bstr, t_cookie *cookie,
   return cook;
 }
 
-/* Build the request Cookie line for domain/path into dst (always
-   NUL-terminated). Returns the number of cookies emitted. */
-int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
-                       char *dst, size_t dst_size) {
+/* Self-test entry for append_cookie_header(): build the request Cookie line
+   into dst (always NUL-terminated). Returns the number of cookies emitted. */
+int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
+                                const char *path, char *dst, size_t dst_size) {
   buff_struct bstr = {dst, dst_size, 0};
 
   assertf(dst != NULL && dst_size > 0);
@@ -1078,12 +1118,19 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
   char BIGSTK buffer_head_request[16384];
   buff_struct bstr = { buffer_head_request, sizeof(buffer_head_request), 0 };
 
+  //int use_11=0;     // HTTP 1.1 utilisé
   int direct_url = 0;           // ne pas analyser l'url (exemple: ftp://)
   const char *search_tag = NULL;
 
   // Initialize buffer
   buffer_head_request[0] = '\0';
 
+  // header Date
+  //strcatbuff(buff,"Date: ");
+  //time_gmt_rfc822(buff);    // obtenir l'heure au format rfc822
+  //sendc("\n");
+  //strcatbuff(buff,buff);
+
   // possibilité non documentée: >post: et >postfile:
   // si présence d'un tag >post: alors executer un POST
   // exemple: http://www.example.com/test.cgi?foo>post:posteddata=10&foo=5
@@ -1177,9 +1224,11 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
     }
 
     // protocole
-    if (!retour->req.http11) { // forcer HTTP/1.0
+    if (!retour->req.http11) {  // forcer HTTP/1.0
+      //use_11=0;
       print_buffer(&bstr, " HTTP/1.0\x0d\x0a");
-    } else { // Requète 1.1
+    } else {                    // Requète 1.1
+      //use_11=1;
       print_buffer(&bstr, " HTTP/1.1\x0d\x0a");
     }
 
@@ -1315,7 +1364,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
           print_buffer(&bstr, "Authorization: Basic %s"H_CRLF, autorisation);
         }
       }
+
     }
+    //strcatbuff(buff,"Accept-Charset: iso-8859-1,*,utf-8\n");
 
     // Custom header(s)
     if (strnotempty(retour->req.headers)) {
@@ -1469,11 +1520,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a += strlen("filename=");
         while(is_space(*a))
           a++;
+        //a=strchr(a,'"');
         if (a) {
           char *c = NULL;
 
+          //a++;      /* jump " */
           while((c = strchr(a, '/')))   /* skip all / (see RFC2616) */
             a = c + 1;
+          //b=strchr(a+1,'"');
           b = a + strlen(a) - 1;
           while(is_space(*b))
             b--;
@@ -1490,14 +1544,16 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
   } else if ((p = strfield(rcvd, "Last-Modified:")) != 0) {
     while(is_realspace(*(rcvd + p)))
       p++;                      // sauter espaces
-    if ((int) strlen(rcvd + p) < 64) { // pas trop long?
+    if ((int) strlen(rcvd + p) < 64) {  // pas trop long?
+      //struct tm* tm_time=convert_time_rfc822(rcvd+p);
       strcpybuff(retour->lastmodified, rcvd + p);
     }
   } else if ((p = strfield(rcvd, "Date:")) != 0) {
     if (strnotempty(retour->lastmodified) == 0) {       /* pas encore de last-modified */
       while(is_realspace(*(rcvd + p)))
         p++;                    // sauter espaces
-      if ((int) strlen(rcvd + p) < 64) { // pas trop long?
+      if ((int) strlen(rcvd + p) < 64) {        // pas trop long?
+        //struct tm* tm_time=convert_time_rfc822(rcvd+p);
         strcpybuff(retour->lastmodified, rcvd + p);
       }
     }
@@ -1510,11 +1566,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       else                      // erreur.. ignorer
         retour->etag[0] = '\0';
     }
-  } else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
+  }
+  // else if ((p=strfield(rcvd,"Transfer-Encoding: chunked"))!=0) {  // chunk!
+  else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) {   // chunk!
     while(is_realspace(*(rcvd + p)))
       p++;                      // sauter espaces
     if (strfield(rcvd + p, "chunked")) {
-      retour->is_chunk = 1; // chunked
+      retour->is_chunk = 1;     // chunked
+      //retour->http11=2;     // chunked
 #if HDEBUG
       printf("ok, Transfer-Encoding: détecté\n");
 #endif
@@ -1562,7 +1621,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       }
       // An empty/whitespace Content-Type value yields no token: keep the
       // sentinel default rather than reading an uninitialized tempo.
-      if (sscanf(rcvd + p, "%1099s", tempo) == 1) { // tempo[1100], server data
+      if (sscanf(rcvd + p, "%s", tempo) == 1) {
         if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
           strcpybuff(retour->contenttype, tempo);
         else
@@ -1586,7 +1645,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a = strchr(rcvd + p, '/');
         if (a != NULL) {
           a++;
-          if (sscanf(a, LLintP, &retour->crange) == 1 && retour->crange >= 0) {
+          if (sscanf(a, LLintP, &retour->crange) == 1) {
             retour->crange_start = 0;
             retour->crange_end = retour->crange - 1;
           } else {
@@ -1594,12 +1653,6 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           }
         }
       }
-      // A valid Content-Range has no negative field; reject hostile values so
-      // the crange +/- 1 arithmetic downstream cannot sign-overflow (UB).
-      if (retour->crange_start < 0 || retour->crange_end < 0 ||
-          retour->crange < 0) {
-        retour->crange_start = retour->crange_end = retour->crange = 0;
-      }
     }
   } else if ((p = strfield(rcvd, "Connection:")) != 0) {
     char *a = rcvd + p;
@@ -1663,11 +1716,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         if (a)
           *a = '\0';
       }
-      // Bound to tempo[1100]; no token => leave empty, don't read uninit tempo.
-      if (sscanf(a, "%1099s", tempo) == 1 && strlen(tempo) < 64)
+      sscanf(a, "%s", tempo);
+      if (strlen(tempo) < 64)   // pas trop long!!
         strcpybuff(retour->contentencoding, tempo);
       else
-        retour->contentencoding[0] = '\0';
+        retour->contentencoding[0] = '\0';      // erreur
 #if HTS_USEZLIB
       /* Check known encodings */
       if (retour->contentencoding[0]) {
@@ -1697,8 +1750,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           retour->location[0] = '\0';
       }
     }
-  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) &&
-             (cookie)) {        // ohh un cookie
+  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) && (cookie)) {  // ohh un cookie
     char *a = rcvd + p;         // pointeur
     char domain[256];           // domaine cookie (.netscape.com)
     char path[256];             // chemin (/)
@@ -1708,9 +1760,6 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
 #if DEBUG_COOK
     printf("set-cookie detected\n");
 #endif
-    /* Over-long host overflows the fail-safe domain[] copy (abort); drop it. */
-    if (adr && strlen(jump_identification_const(adr)) >= sizeof(domain))
-      return;
     while(*a) {
       char *token_st, *token_end;
       char *value_st, *value_end;
@@ -1745,8 +1794,10 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
             a++;                // sauter espaces
           value_st = a;
           while((*a != ';') && (*a))
-            a++; // prochain ;
+            a++;                // prochain ;
+          //while( ((*a!='"') || (*(a-1)=='\\')) && (*a)) a++;    // prochain " (et pas \")
           value_end = a;
+          //if (*a==';') {  // finit par un ;
           // vérifier débordements
           if ((((int) (token_end - token_st)) < 200)
               && (((int) (value_end - value_st)) < 8000)
@@ -2007,15 +2058,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
 
   if (bufl > 0) {
     if (!r->is_write) {         // stocker en mémoire
-      // In-memory content must fit a 32-bit index (allocs below add 1, reads
-      // use int offsets): reject a hostile Content-Length or endless stream.
-      const LLint inmem_want =
-          (r->totalsize >= 0) ? r->totalsize : (r->size + bufl);
-      if (inmem_want >= INT32_MAX) {
-        r->statuscode = STATUSCODE_INVALID;
-        strcpybuff(r->msg, "In-memory content too large");
-        return READ_ERROR;
-      }
       if (r->totalsize >= 0) {  // totalsize déterminé ET ALLOUE
         if (r->adr == NULL) {
           r->adr = (char *) malloct((size_t) r->totalsize + 1);
@@ -2097,6 +2139,8 @@ LLint http_xfread1(htsblk * r, int bufl) {
             nl = READ_ERROR;
           }
         }
+        //if ((nl < 0) || ((r->totalsize>0) && (r->size >= r->totalsize)))
+        //  nl=-1;  // break
 
         // libérer bloc tempo
         freet(buff);
@@ -2184,6 +2228,9 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
   T_SOC soc;
   htsblk retour;
 
+  //int rcvsize=-1;
+  //char* rcv=NULL;    // adresse de retour
+  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
   TStamp tl;
   int timeout = 30;             // timeout pour un check (arbitraire) // **
 
@@ -2192,8 +2239,11 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
 
   loc[0] = '\0';
   hts_init_htsblk(&retour);
+  //memset(&retour, 0, sizeof(htsblk));    // effacer
   retour.location = loc;        // si non nul, contiendra l'adresse véritable en cas de moved xx
 
+  //soc=http_fopen(adr,fil,&retour,NULL);  // ouvrir, + header
+
   // on ouvre en head, et on traite l'en tête
   soc = http_xfopen(opt, 1, 0, 1, NULL, adr, fil, &retour);     // ouvrir HEAD, + envoi header
 
@@ -2515,10 +2565,6 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
   // effacer adrfil->adr et adrfil->fil
   adrfil->adr[0] = adrfil->fil[0] = '\0';
 
-  // Reject an over-long URL: a root-relative path is copied whole into fil[].
-  if (strlen(url) >= HTS_URLMAXSIZE * 2)
-    return -1;
-
 #if HDEBUG
   printf("protocol: %s\n", url);
 #endif
@@ -2573,16 +2619,12 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
 
     // chemin www... trop long!!
     if ((((int) (q - p))) > HTS_URLMAXSIZE) {
+      //strcpybuff(retour.msg,"Path too long");
       return -1;                // erreur
     }
     // recopier adrfil->adresse www..
     strncatbuff(adrfil->adr, p, ((int) (q - p)));
     // *( adrfil->adr+( ((int) q) - ((int) p) ) )=0;  // faut arrêter la fumette!
-
-    // fil[] holds the path plus a possible leading '/' the top strlen() misses.
-    if (strlen(q) >= sizeof(adrfil->fil) - (q[0] != '/' ? 1 : 0))
-      return -1;
-
     // recopier chemin /pub/..
     if (q[0] != '/')            // page par défaut (/)
       strcatbuff(adrfil->fil, "/");
@@ -2599,7 +2641,7 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
     if (*p == '/' || *p == '\\') {      /* adrfil->file:///.. */
       strcatbuff(adrfil->fil, p);       // fichier local ; adrfil->adr="#"
     } else {
-      if (*p == '\0' || p[1] != ':') {  /* empty path: not a DOS drive letter */
+      if (p[1] != ':') {
         strcatbuff(adrfil->fil, "//");  /* adrfil->file://server/foo */
         strcatbuff(adrfil->fil, p);
       } else {
@@ -2763,6 +2805,7 @@ void deletesoc_r(htsblk * r) {
 #if HTS_USEOPENSSL
   if (r->ssl_con) {
     SSL_shutdown(r->ssl_con);
+    // SSL_CTX_set_quiet_shutdown(r->ssl_con->ctx, 1);
     SSL_free(r->ssl_con);
     r->ssl_con = NULL;
   }
@@ -3061,11 +3104,12 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
 /* See http://physics.nist.gov/cuu/Units/binary.html */
 #define ToLLint(a) ((LLint)(a))
 #define ToLLintKiB (ToLLint(1024))
-#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
-#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#define ToLLintPiB                                                             \
-  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
+#ifdef HTS_LONGLONG
+#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#endif
 HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
   if (n < ToLLintKiB) {
     sprintf(strc->buff1, "%d", (int) (LLint) n);
@@ -3074,7 +3118,9 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
             (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
     strcpybuff(strc->buff2, "KiB");
-  } else if (n < ToLLintGiB) {
+  }
+#ifdef HTS_LONGLONG
+  else if (n < ToLLintGiB) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
             (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
     strcpybuff(strc->buff2, "MiB");
@@ -3091,6 +3137,13 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
             (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
     strcpybuff(strc->buff2, "PiB");
   }
+#else
+  else {
+    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
+            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
+    strcpybuff(strc->buff2, "MiB");
+  }
+#endif
   strc->buffadr[0] = strc->buff1;
   strc->buffadr[1] = strc->buff2;
   return strc->buffadr;
@@ -3141,6 +3194,7 @@ int finput(T_SOC fd, char *s, int max) {
   int j = 0;
 
   do {
+    //c=fgetc(fp);
     if (read((int) fd, &c, 1) <= 0) {
       c = 0;
     }
@@ -3156,7 +3210,7 @@ int finput(T_SOC fd, char *s, int max) {
         break;
       }
     }
-  } while ((c != 0) && (j < max - 1));
+  } while((c != 0) && (j < max - 1));
   s[j] = '\0';
   return j;
 }
@@ -4815,6 +4869,7 @@ LLint check_downloadable_bytes(int rate) {
 
     time_now = mtime_local();
     elapsed_useconds = time_now - HTS_STAT.istat_timestart[id_timer];
+    // NO totally stupid - elapsed_useconds+=1000;      // for the next second, too
     bytes_transferred_during_period =
       (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[id_timer]);
 
@@ -4827,6 +4882,32 @@ LLint check_downloadable_bytes(int rate) {
     return TAILLE_BUFFER;
 }
 
+//
+// 0 : OK
+// 1 : slow down
+#if 0
+int HTS_TOTAL_RECV_CHECK(int var) {
+  if (HTS_STAT.HTS_TOTAL_RECV_STATE)
+    return 1;
+  /*
+     {
+     if (HTS_STAT.HTS_TOTAL_RECV_STATE==3) { 
+     var = min(var,32); 
+     Sleep(250); 
+     } else if (HTS_STAT.HTS_TOTAL_RECV_STATE==2) { 
+     var = min(var,256); 
+     Sleep(100); 
+     } else { 
+     var/=2; 
+     if (var<=0) var=1; 
+     Sleep(50); 
+     } 
+     }
+   */
+  return 0;
+}
+#endif
+
 // Lecture dans buff de size octets au maximum en utilisant la socket r (structure htsblk)
 // returns: 
 // >0 : data received
@@ -4835,6 +4916,7 @@ LLint check_downloadable_bytes(int rate) {
 int hts_read(htsblk * r, char *buff, int size) {
   int retour;
 
+  //  return read(soc,buff,size);
   if (r->is_file) {
 #if HTS_WIDE_DEBUG
     DEBUG_W("read(%p, %d, %d)\n" _(void *)buff _(int) size _(int) r->fp);
@@ -4852,6 +4934,7 @@ int hts_read(htsblk * r, char *buff, int size) {
     if (r->soc == INVALID_SOCKET)
       printf("!!WIDE_DEBUG ERROR, soc==INVALID hts_read\n");
 #endif
+    //HTS_TOTAL_RECV_CHECK(size);         // Diminuer au besoin si trop de données reçues
 #if HTS_USEOPENSSL
     if (r->ssl) {
       retour = SSL_read(r->ssl_con, buff, size);
@@ -5286,13 +5369,18 @@ SOCaddr* hts_dns_resolve(httrackp * opt, const char *_iadr, SOCaddr *const addr)
 
 // --- Tracage des mallocs() ---
 #ifdef HTS_TRACE_MALLOC
+//#define htsLocker(A, N) htsLocker(A, N)
 #define htsLocker(A, N) do {} while(0)
 static mlink trmalloc = { NULL, 0, 0, NULL };
 
 static int trmalloc_id = 0;
 static htsmutex *mallocMutex = NULL;
-
-static void hts_meminit(void) {}
+static void hts_meminit(void) {
+  //if (mallocMutex == NULL) {
+  //  mallocMutex = calloc(sizeof(*mallocMutex), 1);
+  //  htsLocker(mallocMutex, -999);
+  //}
+}
 void *hts_malloc(size_t len) {
   void *adr;
 
@@ -5376,6 +5464,7 @@ void hts_free(void *adr) {
               htsboundary);
       lnk->next = lnk->next->next;
       free((void *) blk_free);
+      //blk_free->id=-1;
       free((char *) adr - bsize);
       htsLocker(mallocMutex, 0);
       return;
@@ -5444,6 +5533,7 @@ mlink *hts_find(char *adr) {
 
     if (depl < 0)
       depl = -depl;
+    //assertf(depl < 512000);   /* near the stack frame.. doesn't look like malloc but stack variable */
     return NULL;
   }
 }
@@ -5499,7 +5589,8 @@ int ftp_available(void) {
 }
 #else
 int ftp_available(void) {
-  return 1; // ok!
+  return 1;                     // ok!
+  //return 0;   // SOUS UNIX, PROBLEMESs
 }
 #endif
 
@@ -5713,6 +5804,7 @@ HTSEXT_API int hts_init(void) {
       assertf("OpenSSL version seems vulnerable to heartbleed bug (CVE-2014-0160)" == NULL);
     }
 
+    // OpenSSL_add_all_algorithms();
     openssl_ctx = SSL_CTX_new(method);
     if (!openssl_ctx) {
       fprintf(stderr, "fatal: unable to initialize TLS: SSL_CTX_new()\n");
@@ -5926,7 +6018,17 @@ int multipleStringMatch(const char *s, const char *match) {
 }
 
 HTSEXT_API httrackp *hts_create_opt(void) {
-  static const char *defaultModules[] = {"httrack-plugin", NULL};
+#if ( defined(_WIN32) || defined(__ANDROID__) )
+  static const char *defaultModules[] = {
+    "htsswf", "htsjava", "httrack-plugin", NULL
+  };
+#else
+#ifndef HTS_LIBHTSJAVA_NAME
+#define HTS_LIBHTSJAVA_NAME "libhtsjava.so" /* non-autoconf fallback */
+#endif
+  static const char *defaultModules[] = {"libhtsswf.so.1", HTS_LIBHTSJAVA_NAME,
+                                         "httrack-plugin", NULL};
+#endif
   httrackp *opt = malloc(sizeof(httrackp));
 
   /* default options */
@@ -6009,7 +6111,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
   StringCopy(opt->strip_query, "");
   StringCopy(opt->cookies_file, "");
-  StringCopy(opt->why_url, "");
   opt->pause_min_ms = 0;
   opt->pause_max_ms = 0;
   opt->ftp_proxy = HTS_TRUE;
@@ -6025,6 +6126,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->log = stdout;
   opt->errlog = stderr;
   opt->flush = HTS_TRUE;
+  // opt->aff_progress=0;
   opt->keyboard = HTS_FALSE;
   //
   StringCopy(opt->path_html, "");
@@ -6034,8 +6136,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   //
   opt->maxlink = 100000;        // 100,000 liens max par défaut
   opt->maxfilter = 200;         // 200 filtres max par défaut
-  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON
-                                // PARAMETRABLE POUR L'INSTANT --
+  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT --
+  //opt->maxcache_anticipate=256;  // maximum de liens à anticiper
   opt->maxtime = -1;            // temps max en secondes
   opt->maxrate = 100000;        // taux maxi
   opt->maxconn = 5.0;           // nombre connexions/s
@@ -6157,7 +6259,6 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
     StringFree(opt->mod_blacklist);
     StringFree(opt->strip_query);
     StringFree(opt->cookies_file);
-    StringFree(opt->why_url);
 
     StringFree(opt->path_html);
     StringFree(opt->path_html_utf8);
@@ -6189,9 +6290,11 @@ const hts_stat_struct* hts_get_stats(httrackp * opt) {
 }
 
 // defaut wrappers
-static void __cdecl htsdefault_init(t_hts_callbackarg *carg) {}
-
-static void __cdecl htsdefault_uninit(t_hts_callbackarg *carg) {}
+static void __cdecl htsdefault_init(t_hts_callbackarg * carg) {
+}
+static void __cdecl htsdefault_uninit(t_hts_callbackarg * carg) {
+  // hts_freevar();
+}
 static int __cdecl htsdefault_start(t_hts_callbackarg * carg, httrackp * opt) {
   return 1;
 }

src/htslib.h

@@ -171,6 +171,7 @@ HTSEXT_API const char* hts_version(void);
 // fonctions unix/winsock
 int hts_read(htsblk * r, char *buff, int size);
 
+//int HTS_TOTAL_RECV_CHECK(int var);
 LLint check_downloadable_bytes(int rate);
 
 HTSEXT_API int hts_uninit_module(void);
@@ -184,11 +185,12 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
                   const char *referer_adr, const char *referer_fil,
                   htsblk * retour);
 /* Build the request "Cookie:" header line for stored cookies matching
-   domain/path into dst (NUL-terminated), wrapping the logic http_sendhead()
-   uses. Returns cookies emitted. */
-int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
-                       char *dst, size_t dst_size);
+   domain/path into dst (NUL-terminated). Exposed for the -#Q self-test;
+   wraps the same logic http_sendhead() uses. Returns cookies emitted. */
+int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
+                                const char *path, char *dst, size_t dst_size);
 
+//int newhttp(char* iadr,char* err=NULL);
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
               int waitconnect);
 /* Like newhttp(), but connect to the addr_index-th resolved address of the host

src/htsmodules.c

@@ -55,6 +55,12 @@ extern int fspc(httrackp * opt, FILE * fp, const char *type);
 
 /* >>> Put all modules variables here */
 
+#if 0
+t_gzopen gzopen = NULL;
+t_gzread gzread = NULL;
+t_gzclose gzclose = NULL;
+#endif
+
 int V6_is_available = HTS_INET6;
 
 static char WHAT_is_available[64] = "";
@@ -106,6 +112,18 @@ int hts_parse_externals(htsmoduleStruct * str) {
   return -1;
 }
 
+//static void addCallback(htscallbacks* chain, void* moduleHandle, htscallbacksfncptr exitFnc) {
+//  while(chain->next != NULL) {
+//    chain = chain->next;
+//  }
+//  chain->next = calloct(1, sizeof(htscallbacks));
+//  assertf(chain->next != NULL);
+//  chain = chain->next;
+//  memset(chain, 0, sizeof(*chain));
+//  chain->exitFnc = exitFnc;
+//  chain->moduleHandle = moduleHandle;
+//}
+
 void clearCallbacks(htscallbacks * chain_);
 void clearCallbacks(htscallbacks * chain_) {
   htscallbacks *chain;

src/htsname.c

@@ -1332,7 +1332,8 @@ int url_savename(lien_adrfilsave *const afs,
 
     while((a > afs->save) && (*a != '.') && (*a != '/'))
       a--;
-    if (*a != '.') {                         // agh pas de point
+    if (*a != '.') {            // agh pas de point
+      //strcatbuff(save,".none");                 // a éviter
       strcatbuff(afs->save, ".html");        // préférable!
       hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",
                     adr_complete, fil_complete, afs->save);
@@ -1470,6 +1471,17 @@ int url_savename(lien_adrfilsave *const afs,
   fil_simplifie(afs->save);
 
   /* convert name to UTF-8 ? Note: already done while parsing. */
+  //if (charset != NULL && charset[0] != '\0') {
+  //  char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
+
+  //  if (s != NULL) {
+  //    hts_log_print(opt, LOG_DEBUG,
+  //                  "engine: save-name: charset conversion from '%s' to '%s' using charset '%s'",
+  //                  save, s, charset);
+  //    strcpybuff(save, s);
+  //    free(s);
+  //  }
+  //}
 
   /* callback */
   RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr,
@@ -1617,6 +1629,10 @@ int url_savename(lien_adrfilsave *const afs,
         int sameAdr = (strfield2(heap(i)->adr, normadr) != 0);
         int sameFil;
 
+        // NO - URL hack is only for stripping // and www.
+        //if (opt->urlhack != 0)
+        //  sameFil = ( strfield2(heap(i)->fil, normfil) != 0);
+        //else
         sameFil = (strcmp(heap(i)->fil, normfil) == 0);
         if (sameAdr && sameFil) {       // ok c'est le même lien, adresse déja définie
           /* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */
@@ -1682,13 +1698,17 @@ int url_savename(lien_adrfilsave *const afs,
 
           strcpybuff(afs->save, tempo);
 
+          //printf("switched: %s\n",save);
+
         }                       // if
       }
 #if DEBUG_SAVENAME
       printf("\nEnd search, %s\n", fil_complete);
 #endif
-    } while (!nom_ok);
+    } while(!nom_ok);
+
   }
+  //printf("'%s' %s %s\n",save,adr,fil);
 
   return 0;
 }

src/htsopt.h

@@ -105,6 +105,7 @@ typedef struct htsfilters htsfilters;
 struct htsfilters {
   char ***filters; /**< pointer to the +/-pattern filter array */
   int *filptr;     /**< pointer to the current filter count */
+  // int*    filter_max;
 };
 
 /* User callbacks chain */
@@ -439,6 +440,7 @@ struct httrackp {
   float maxconn;          /**< max connections per second */
   int waittime;           /**< scheduled start time (wall-clock seconds) */
   hts_cachemode cache;    /**< cache generation mode */
+  // int aff_progress;     // progress bar
   hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
   t_proxy proxy;     /**< proxy configuration */
   hts_savename_83
@@ -490,6 +492,7 @@ struct httrackp {
   hts_verbosedisplay verbosedisplay; /**< animated text progress display */
   String footer; /**< footer/info line injected into pages */
   int maxcache;  /**< in-memory cache backing limit (bytes) */
+  // int maxcache_anticipate; // maximum links to anticipate (upper bound)
   hts_boolean ftp_proxy;    /**< use the HTTP proxy for FTP too */
   String filelist;          /**< file listing URLs to include */
   String urllist;           /**< file listing filters to include */
@@ -536,8 +539,6 @@ struct httrackp {
                                  (--cookies-file) */
   int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
   int pause_max_ms; /**< inter-file pause upper bound, ms */
-  String why_url;   /**< URL to diagnose (--why): print the deciding filter rule
-                         and exit without crawling */
 };
 
 /* Running statistics for a mirror. */
@@ -548,6 +549,8 @@ typedef struct hts_stat_struct hts_stat_struct;
 struct hts_stat_struct {
   LLint HTS_TOTAL_RECV; /**< total bytes received from the network */
   LLint stat_bytes;     /**< total bytes written to disk */
+  // int HTS_TOTAL_RECV_STATE;  // status: 0 ok 1: slow down a little 2: slow
+  // down 3: a lot
   TStamp stat_timestart; /**< mirror start time */
   //
   LLint total_packed;    /**< compressed bytes received (on the wire) */
@@ -679,6 +682,7 @@ struct lien_url {
   int depth;        /**< remaining allowed depth; >0 strong, 0 weak */
   int pass2;        /**< second-pass marker; -1 means handled in background */
   char link_import; /**< imported after a move; skip the usual up/down rules */
+  // int moved;          // pointer to moved
   int retry;    /**< remaining retries */
   int testmode; /**< test only: send just a HEAD */
 };
@@ -723,6 +727,9 @@ struct lien_back {
   LLint chunk_size; /**< size of the chunk being loaded */
   LLint chunk_blocksize; /**< data size declared by the chunk */
   LLint compressed_size; /**< compressed size (stats only) */
+  //
+  // int links_index;        // to access liens[links_index]
+  //
   char info[256]; /**< status text, e.g. for FTP */
   int stop_ftp;   /**< stop flag for FTP */
   int finalized;  /**< finalized (memory optimization) */

src/htsparse.c

@@ -192,48 +192,6 @@ static void hts_automate_lookup(const script_automate *aut) {
   }
 }
 
-/* Attribute name owning the quoted value at 'quote' inside a tag, spanning
-   [name, *nend); NULL when the quote is not an attribute value. */
-static const char *dirty_attr_name(const char *quote, const char *tag_start,
-                                   const char **nend) {
-  const char *a = quote - 1;
-  while (a > tag_start && is_taborspace(*a))
-    a--;
-  if (a == tag_start || *a != '=')
-    return NULL;
-  a--;
-  while (a > tag_start && is_taborspace(*a))
-    a--;
-  *nend = a + 1;
-  while (a > tag_start && *a != '=' && *a != '\"' && *a != '\'' &&
-         !is_realspace(*a))
-    a--;
-  a++;
-  // a name starting right after '<' is the tag name, not an attribute
-  return a < *nend && a > tag_start + 1 ? a : NULL;
-}
-
-/* Accept the in-tag quoted value at 'quote' for dirty parsing? Resolves the
-   owning attribute itself (intag_startattr is unreliable mid-tag) and rejects
-   no-detect/xmlns names. */
-static hts_boolean dirty_attr_detectable(const char *quote,
-                                         const char *tag_start) {
-  const char *nend;
-  const char *name = dirty_attr_name(quote, tag_start, &nend);
-  int i;
-  if (name == NULL)
-    return HTS_FALSE;
-  for (i = 0; strnotempty(hts_nodetect[i]); i++) {
-    const int l = strfield(name, hts_nodetect[i]);
-    if (l && name + l == nend)
-      return HTS_FALSE;
-  }
-  i = strfield(name, "xmlns");
-  if (i && (name + i == nend || name[i] == ':'))
-    return HTS_FALSE;
-  return HTS_TRUE;
-}
-
 /* Advance the cursor by 'steps' bytes, feeding each to the automaton. */
 static void hts_automate_increment(const script_automate *aut, int steps) {
   while (steps > 0) {
@@ -395,13 +353,16 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
       // terminaison (" ou ') du "<body onLoad=.."
       int inscriptgen = 0;      // on est dans un code générant, ex après obj.write("..
 
+      //int inscript_check_comments=0, inscript_in_comments=0;    // javascript comments
       char scriptgen_q = '\0';  // caractère faisant office de guillemet (' ou ")
 
+      //int no_esc_utf=0;      // ne pas echapper chars > 127
       int nofollow = 0;         // ne pas scanner
 
       //
       int parseall_lastc = '\0';        // dernier caractère parsé pour parseall
 
+      //int parseall_incomment=0;   // dans un /* */ (exemple: a = /* URL */ "img.gif";)
       //
       const char *intag_start = html;
       const char *intag_name = NULL;
@@ -514,6 +475,12 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           lastsaved = html;
         }
       }
+      // Detect UTF8 format
+      //if (is_unicode_utf8(r->adr, (unsigned int) r->size) == 1) {
+      //  no_esc_utf=1;
+      //} else {
+      //  no_esc_utf=0;
+      //}
 
       // Hack to prevent any problems with ram files of other files
       *(r->adr + r->size) = '\0';
@@ -678,6 +645,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           ) {
           intag = 1;
           intag_ctype = 0;
+          //parseall_incomment=0;
+          //inquote=0;  // effacer quote
           intag_start = html;
           for(intag_name = html + 1; is_realspace(*intag_name); intag_name++) ;
           intag_start_valid = 1;
@@ -695,6 +664,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               if (len > 0) {
                 if (strfield(token, "content-type")) {
                   intag_ctype = 1;
+                  //NOPE-we do not convert the whole page actually
+                  //intag_start[1] = 'X';
                 } else if (strfield(token, "refresh")) {
                   intag_ctype = 2;
                 }
@@ -755,6 +726,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           html_inline_safe(urlfil(), safe_fil, sizeof(safe_fil)), gmttime,
                           HTTRACK_VERSIONID, /* EOF */ NULL);
                   strcatbuff(tempo, eol);
+                  //fwrite(tempo,1,strlen(tempo),fp);
                   HT_ADD(tempo);
                 }
                 // Emit charset ?
@@ -782,8 +754,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           ) {
           if (inscript_tag) {
             inscript_tag = inscript = 0;
-            // reset the automaton on exit or its state leaks into plain HTML
-            inscript_state_pos = INSCRIPT_START;
             intag = 0;
             incomment = 0;
             intag_start_valid = 0;
@@ -795,8 +765,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             intag = 0;          //inquote=0;
 
             // entrée dans du javascript?
-            // on parse ICI car il se peut qu'on ait eu a parser les src=..
-            // dedans
+            // on parse ICI car il se peut qu'on ait eu a parser les src=.. dedans
+            //if (!inscript) {  // sinon on est dans un obj.write("..
             if ((intag_start_valid) && (check_tag(intag_start, "script")
                                         || check_tag(intag_start, "style")
                 )
@@ -842,9 +812,12 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             }
 #endif
           }
-        } else if (intag || inscript ||
-                   in_media) { // nous sommes dans un tag/commentaire, tester si
-                               // on recoit un tag
+          //}
+        }
+        //else if (*adr==34) {
+        //  inquote=(inquote?0:1);
+        //}
+        else if (intag || inscript || in_media) {       // nous sommes dans un tag/commentaire, tester si on recoit un tag
           int p_type = 0;
           int p_nocatch = 0;
           int p_searchMETAURL = 0;      // chercher ..URL=<url>
@@ -864,7 +837,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 if (*html == inscript_tag_lastc) {
                   /* sortir */
                   inscript_tag = inscript = 0;
-                  inscript_state_pos = INSCRIPT_START;
                   incomment = 0;
                   if (opt->parsedebug) {
                     HT_ADD("<@@ /inscript @@>");
@@ -1083,6 +1055,27 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       if (p) {
                         if (intag_ctype == 1) {
                           p = 0;
+#if 0
+                          //if ((pos=rech_tageq(html, "content"))) {
+                          char temp[256];
+                          char *token = NULL;
+                          int len = rech_endtoken(html + pos, &token);
+
+                          if (len > 0 && len < sizeof(temp) - 2) {
+                            char *chpos;
+
+                            temp[0] = '\0';
+                            strncat(temp, token, len);
+                            if ((chpos = strstr(temp, "charset"))
+                                && (chpos = strchr(chpos, '='))
+                              ) {
+                              chpos++;
+                              while(is_space(*chpos))
+                                chpod++;
+                              //chpos
+                            }
+                          }
+#endif
                         }
                         // <META HTTP-EQUIV="Refresh" CONTENT="3;URL=http://www.example.com">
                         else if (intag_ctype == 2) {
@@ -1267,6 +1260,21 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   }
                 } else if (inscript) {
 
+#if 0
+                  /* Check // javascript comments */
+                  if (*html == 10 || *html == 13) {
+                    inscript_check_comments = 1;
+                    inscript_in_comments = 0;
+                  } else if (inscript_check_comments) {
+                    if (!is_realspace(*html)) {
+                      inscript_check_comments = 0;
+                      if (html[0] == '/' && html[1] == '/') {
+                        inscript_in_comments = 1;
+                      }
+                    }
+                  }
+#endif
+
                   /* Parse */
                   assertf(inscript_name != NULL);
                   if (*html == '/'
@@ -1279,17 +1287,18 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       && inscript_locked == 0) {
                     const char *a = html;
 
+                    //while(is_realspace(*(--a)));
                     while(is_realspace(*a))
                       a--;
                     a--;
                     if (*a == '<') {    // sûr que c'est un tag?
                       inscript = 0;
-                      inscript_state_pos = INSCRIPT_START;
                       if (opt->parsedebug) {
                         HT_ADD("<@@ /inscript @@>");
                       }
                     }
-                  } else if (inscript_state_pos == INSCRIPT_START) {
+                  } else if (inscript_state_pos ==
+                             INSCRIPT_START /*!inscript_in_comments */ ) {
                     /*
                        Script Analyzing - different types supported:
                        foo="url"
@@ -1478,10 +1487,12 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               }
 
             } else {            // ptr == 0
+              //p=rech_tageq(adr,"primary");    // lien primaire, yeah
               p = 0;            // No stupid tag anymore, raw link
               valid_p = 1;      // Valid even if p==0
               while((html[p] == '\r') || (html[p] == '\n'))
                 p++;
+              //can_avoid_quotes=1;
               ending_p = '\r';
             }
 
@@ -1494,11 +1505,25 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           // risque: générer de faux fichiers parazites
           // fix: ne parse plus dans les commentaires
           // ------------------------------------------------------------
-          if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 &&
-              (ptr > 0) && (!in_media)) { // option parsing "brut"
+          if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 && (ptr > 0) && (!in_media) /* && (!inscript_in_comments) */ ) {  // option parsing "brut"
+            //int incomment_justquit=0;
             if (!is_realspace(*html)) {
               int noparse = 0;
 
+              // Gestion des /* */
+#if 0
+              if (inscript) {
+                if (parseall_incomment) {
+                  if ((*html == '/') && (*(html - 1) == '*'))
+                    parseall_incomment = 0;
+                  incomment_justquit = 1;       // ne pas noter dernier caractère
+                } else {
+                  if ((*html == '/') && (*(html + 1) == '*'))
+                    parseall_incomment = 1;
+                }
+              } else
+                parseall_incomment = 0;
+#endif
               /* ensure automate state  0 (not in comments, quotes..) */
               if (inscript
                   && (inscript_state_pos != INSCRIPT_INQUOTE
@@ -1512,6 +1537,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
               // recherche d'URLs
               if (!noparse) {
+                //if ((!parseall_incomment) && (!noparse)) {
                 if (!p) {       // non déja trouvé
                   if (html != r->adr) {  // >1 caractère
                     // scanner les chaines
@@ -1535,16 +1561,15 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             && (count > 0)) {
                           char c;
 
+                          //char* aend;
+                          //
+                          //aend=a;     // sauver début
                           a++;
                           while(is_taborspace(*a))
                             a++;
                           c = *a;
-                          // in-tag, an attribute value ends at its quote: no
-                          // delimiter required after it (mid-tag attrs, #201)
-                          if (strchr("),;>/+\r\n", c) ||
-                              (intag && !inscript && intag_start_valid &&
-                               dirty_attr_detectable(html, intag_start))) {
-                            // '/' covers a value followed by a JS comment
+                          if (strchr("),;>/+\r\n", c)) {        // exemple: ..img.gif";
+                            // le / est pour funct("img.gif" /* URL */);
                             char BIGSTK tempo[HTS_URLMAXSIZE * 2];
                             char type[256];
                             int url_ok = 0;     // url valide?
@@ -1589,8 +1614,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                               if (!invalid_url) {
                                 // Un plus à la fin? Alors ne pas prendre sauf si extension ("/toto.html#"+tag)
                                 if (c != '+') { // PAS de plus à la fin
-                                  // "Comparisons of scheme names MUST be
-                                  // case-insensitive" (RFC2616)
+#if 0
+                                  char *a;
+#endif
+                                  // "Comparisons of scheme names MUST be case-insensitive" (RFC2616)                                  
                                   if ((strfield(tempo, "http:"))
                                       || (strfield(tempo, "ftp:"))
 #if HTS_USEOPENSSL
@@ -1603,6 +1630,15 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                     if (inscript)       // sinon si pas javascript, méfiance (répertoire style base?)
                                       url_ok = 1;
                                   }
+#if 0
+                                  else if ((a = strchr(tempo, '/'))) {  // un slash: ok..
+                                    if (inscript) {     // sinon si pas javascript, méfiance (style "text/css")
+                                      if (strchr(a + 1, '/'))   // un seul / : abandon (STYLE type='text/css')
+                                        if (!strchr(tempo, ' '))        // avoid spaces (too dangerous for comments)
+                                          url_ok = 1;
+                                    }
+                                  }
+#endif
                                 }
                                 // Prendre si extension reconnue
                                 if (!url_ok) {
@@ -1681,9 +1717,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       }
                     }
                   }
-                } // p == 0
+                }               // p == 0               
 
-              } // not in comment
+              }                 // not in comment
 
               // plus dans un commentaire
               if (inscript_state_pos == INSCRIPT_START
@@ -1692,15 +1728,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               }
 
             }                   // if realspace
-          } // if parseall
+          }                     // if parseall
 
           // ------------------------------------------------------------
           // p!=0 : on a repéré un éventuel lien
           // ------------------------------------------------------------
           //
-          if ((p > 0) || (valid_p)) { // on a repéré un lien
+          if ((p > 0) || (valid_p)) {   // on a repéré un lien
+            //int lien_valide=0;
             const char *eadr = NULL;  /* fin de l'URL */
 
+            //char* quote_adr=NULL;     /* adresse du ? dans l'adresse */
             int ok = 1;
             char quote = '\0';
             int quoteinscript = 0;
@@ -1723,6 +1761,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               lastsaved = html;  // dernier écrit+1
             }
             // sauter espaces
+            // adr+=p;
             hts_automate_increment(&saut, p);
             while((is_space(*html)
                    || (inscriptgen && html[0] == '\\' && is_space(html[1])
@@ -1751,7 +1790,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             // sauter éventuel \" ou \' javascript
             if (inscript) {     // on est dans un obj.write("..
               if (*html == '\\') {
-                if ((*(html + 1) == '\'') || (*(html + 1) == '"')) { // \" ou \'
+                if ((*(html + 1) == '\'') || (*(html + 1) == '"')) {      // \" ou \'
+                  // html+=2;    // sauter
                   hts_automate_increment(&saut, 2);
                 }
               }
@@ -1826,6 +1866,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   ok = -1;      // ne pas traiter ce lien
 
                 if (ok > 0) {
+                  //if (*eadr!=' ') {  
                   if (is_space(*eadr)) {        // guillemets,CR, etc
                     if ((*eadr == quote && (!quoteinscript || *(eadr - 1) == '\\'))     // end quote
                         || (noquote && (*eadr == '\"' || *eadr == '\''))        // end at any quote
@@ -1871,6 +1912,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       break;
                     }
                   }
+                  //}
                 }
                 eadr++;
               } while(ok == 1);
@@ -1899,10 +1941,14 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               char BIGSTK lien[HTS_URLMAXSIZE * 2];
               int meme_adresse = 0;     // 0 par défaut pour primary
 
+              //char *copie_de_adr=html;
+              //char* p;
+
               // construire lien (découpage)
               if (eadr - html - 1 < HTS_URLMAXSIZE) {        // pas trop long?
                 strncpy(lien, html, eadr - html - 1);
                 lien[eadr - html - 1] = '\0';
+                //printf("link: %s\n",lien);          
                 // supprimer les espaces
                 while((lien[strlen(lien) - 1] == ' ') && (strnotempty(lien)))
                   lien[strlen(lien) - 1] = '\0';
@@ -2007,7 +2053,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                 /* Unescape/escape %20 and other &nbsp; */
                 {
-                  // NULL when UTF-8 conversion is off (-%T0)
+                  // Note: always true (iso-8859-1 as default)
                   const char *const charset = str->page_charset_;
                   const int hasCharset = charset != NULL 
                     && *charset != '\0';
@@ -2030,12 +2076,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                   // Force to encode non-printable chars (should never happend)
                   escape_remove_control(lien);
-
-                  // charset conversion for the URI filename (not the query
-                  // string), unless the bytes already are valid UTF-8:
-                  // converting those would double-encode them (#180)
-                  if (hasCharset && !hts_isCharsetUTF8(charset) &&
-                      !hts_isStringUTF8(lien, strlen(lien))) {
+                  
+                  // charset conversion for the URI filename, 
+                  // and not already UTF-8
+                  // (note: not for the query string!)
+                  if (hasCharset && !hts_isCharsetUTF8(charset)) {
                     char *const s = hts_convertStringToUTF8(lien, strlen(lien), charset);
                     if (s != NULL) {
                       hts_log_print(opt, LOG_DEBUG,
@@ -2147,6 +2192,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     a += 2;
                   else
                     a = lien;
+                  // while((*a) && (*a!='/') && (*a!=':')) a++;
                   a = jump_toport(a);
                   if (a) {      // port
                     int port = 0;
@@ -2154,6 +2200,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     char *b = a + 1;
 
 #if HTS_USEOPENSSL
+                    // FIXME
+                    //if (strfield(adr, "https:")) {
+                    //}
 #endif
                     while(isdigit((unsigned char) *b)) {
                       port *= 10;
@@ -2244,6 +2293,13 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             *(a + 1) = '\0';
                           }
                         }
+                        //char BIGSTK tempo[HTS_URLMAXSIZE*2];
+                        //strcpybuff(tempo,"http://");
+                        //strcatbuff(tempo,urladr());    // host
+                        //if (*lien!='/')
+                        //  strcatbuff(tempo,"/");
+                        //strcatbuff(tempo,lien);
+                        //strcpybuff(lien,tempo);
                       }
                     }
 
@@ -2272,17 +2328,22 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       }
                       // stocker base ou codebase?
                       switch (p_type) {
-                      case 2: {
-                        strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
-                      } break; // base
-                      case -2: {
-                        strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
-                      } break; // base
+                      case 2:{
+                          //if (*lien!='/') strcatbuff(base,"/");
+                          strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
+                        }
+                        break;  // base
+                      case -2:{
+                          //if (*lien!='/') strcatbuff(codebase,"/");
+                          strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
+                        }
+                        break;  // base
                       }
 
                       hts_log_print(opt, LOG_DEBUG,
                                     "code/codebase link %s base %s", lien,
                                     base);
+                      //printf("base code: %s - %s\n",lien,base);
                     }
 
                   } else {
@@ -2395,6 +2456,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 // Tester si un lien doit être accepté ou refusé (wizard)
                 // forbidden_url=1 : lien refusé
                 // forbidden_url=0 : lien accepté
+                //if ((ptr>0) && (p_type!=2) && (p_type!=-2)) {    // tester autorisations?
                 if ((p_type != 2) && (p_type != -2)) {  // tester autorisations?
                   if (!p_nocatch) {
                     if (afs.af.adr[0] != '\0') {
@@ -2444,7 +2506,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
                       /* Calc */
                       last_adr[0] = '\0';
-                      strcpybuff(last_adr, afs.af.adr); // ancienne adresse
+                      //char last_fil[HTS_URLMAXSIZE*2]="";
+                      strcpybuff(last_adr, afs.af.adr);        // ancienne adresse
+                      //strcpybuff(last_fil,fil);    // ancien chemin
                       r_sv =
                         url_savename(&afs, &former, heap(ptr)->adr, heap(ptr)->fil, opt,
                                      sback, cache, hash, ptr,
@@ -2475,6 +2539,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             }
                           }
                         }
+                        //import_done=1;    // c'est un import!
                         meme_adresse = 0;       // on a changé
                       }
                     } else {
@@ -2672,8 +2737,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                               if (lienrelatif(tempo, sizeof(tempo), save,
                                               relativesavename()) == 0) {
                                 /* Never escape high-chars (we don't know the encoding!!) */
-                                inplace_escape_uri_utf(
-                                    tempo, sizeof(tempo)); // escape with %xx
+                                inplace_escape_uri_utf(tempo, sizeof(tempo));  // escape with %xx
+                                //if (!no_esc_utf)
+                                //  escape_uri(tempo);     // escape with %xx
+                                //else
+                                //  escape_uri_utf(tempo);     // escape with %xx
                                 HT_ADD_HTMLESCAPED(tempo);      // page externe
                                 if (add_url) {
                                   HT_ADD("?link=");     // page externe
@@ -2910,6 +2978,16 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           // unquoted url() (CSS/JS): keep parens escaped
                           if (ending_p == ')')
                             escape_url_parens(tempo, sizeof(tempo));
+
+                          //if (!no_esc_utf)
+                          //  escape_uri(tempo);     // escape with %xx
+                          //else {
+                          //  /* No escaping at all - remaining upper chars will be escaped below */
+                          //  /* FIXME - Should be done in all local cases */
+                          //  //x_escape_html(tempo);
+                          //  //escape_uri_utf(tempo);     // FIXME - escape with %xx
+                          //  //escape_uri(tempo);     // escape with %xx
+                          //}
                         }
                         hts_log_print(opt, LOG_DEBUG,
                                       "relative link at %s build with %s and %s: %s",
@@ -2967,9 +3045,22 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                               HT_ADD(tempo4);   // refresh code="
                             }
                           }
+                          //lastsaved=adr;    // dernier écrit+1
                         }
 
                         if ((opt->getmode & HTS_GETMODE_HTML) && (ptr > 0)) {
+                          // convert to local codepage - NOT, already converted into %NN, and passed to the remote server so we do not have anything to do
+                          //if (str->page_charset_ != NULL && *str->page_charset_ != '\0') {
+                          //  char *const local_save = hts_convertStringFromUTF8(tempo, strlen(tempo), str->page_charset_);
+                          //  if (local_save != NULL) {
+                          //    strcpybuff(tempo, local_save);
+                          //    free(local_save);
+                          //  } else {
+                          //    if ((opt->debug>1) && (opt->log!=NULL)) {
+                          //      fprintf(opt->log, "Warning: could not build local charset representation of '%s' in '%s'"LF, tempo, str->page_charset_);
+                          //    }
+                          //  }
+                          //}
 
                           // écrire le lien modifié, relatif
                           // Note: escape all chars, even >127 (no UTF)
@@ -2993,6 +3084,16 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                       }
                     } // sinon le lien sera écrit normalement
 
+#if 0
+                    if (fexist(save)) { // le fichier existe..
+                      adr[0] = '\0';
+                      //if ((opt->debug>0) && (opt->log!=NULL)) {
+                      hts_log_print(opt, LOG_WARNING,
+                                    "Link has already been written on disk, cancelled: %s",
+                                    save);
+                    }
+#endif
+
                     /* Security check */
                     if (strlen(afs.save) >= HTS_URLMAXSIZE) {
                       afs.af.adr[0] = '\0';
@@ -3034,7 +3135,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                         /*
                            if (strnotempty(save)) {
                            if (ishtml(opt,save) == 1) {
+                           // descore_prio = 2;
                            } else {
+                           // descore_prio = 1;
                            }
                            }
                          */
@@ -3070,6 +3173,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           // >>>> CREER LE LIEN <<<<
                           //
                           // enregistrer lien à charger
+                          //heap_top()->adr[0]=heap_top()->fil[0]=heap_top()->sav[0]='\0';
+                          // même adresse: l'objet père est l'objet père de l'actuel
 
                           // DEBUT ROBOTS.TXT AJOUT
                           if (!just_test_it) {
@@ -3148,6 +3253,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             heap_top()->premier = heap(ptr)->premier;
                           else  // sinon l'objet père est le précédent lui même
                             heap_top()->premier = heap_top_index();
+                          // heap_top()->premier=ptr;
 
                           heap_top()->precedent = ptr;
                           // noter la priorité
@@ -3159,6 +3265,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                           heap_top()->pass2 = pass_fix;
                           heap_top()->retry = opt->retry;
 
+                          //strcpybuff(heap_top()->adr,adr);
+                          //strcpybuff(heap_top()->fil,fil);
+                          //strcpybuff(heap_top()->sav,save); 
                           if (!just_test_it) {
                             hts_log_print(opt, LOG_DEBUG,
                                           "OK, NOTE: %s%s -> %s",
@@ -3193,6 +3302,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             if (eadr > html) {
               hts_automate_increment(&saut, (int) (eadr - 1 - html));
             }
+            // adr=eadr-1;  // ** sauter
 
             /* srcset candidate loop: skip the descriptor and comma, then
                re-enter the capture for the next URL. Backward goto, not a loop:
@@ -3222,9 +3332,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                inscript_state_pos = INSCRIPT_START;
                } */
 
- } // if (p)
+          }                     // if (p) 
 
-        } // si '<' ou '>'
+        }                       // si '<' ou '>'
 
         // plus loin
         html++;                  // automate will be checked next loop
@@ -3294,7 +3404,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             *stre->exit_xh_ = 1;        // exit requested
             XH_uninit;
             return -1;
+            //adr = r->adr + r->size;  // exit
           } else if (opt->state._hts_cancel == 1) {
+            // adr = r->adr + r->size;  // exit
             nofollow = 1;       // moins violent
             opt->state._hts_cancel = 0;
           }
@@ -3352,6 +3464,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
         fp = NULL;
       }
     }
+    // sauver fichier
+    //structcheck(savename());
+    //filesave(opt,r->adr,r->size,savename());
 
   }                             // analyse OK
 
@@ -3390,11 +3505,17 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
   // DEBUT rattrapage des 301,302,307..
   // ------------------------------------------------------------
   if (!error) {
+    ////////{
+    // on a chargé un fichier en plus
+    // if (!error) stat_loaded+=r.size;
 
     // ------------------------------------------------------------
     // Rattrapage des 301,302,307 (moved) et 412,416 - les 304 le sont dans le backing 
     // ------------------------------------------------------------
     if (HTTP_IS_REDIRECT(r->statuscode)) {
+      //if (r->adr!=NULL) {   // adr==null si fichier direct. [catch: davename normalement si cgi]
+      //int i=0;
+      // char* p;
 
       hts_log_print(opt, LOG_WARNING, "%s for %s%s", r->msg, urladr(), urlfil());
 
@@ -3421,6 +3542,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
           // A same-file alias redirect must be followed, not stubbed (#159).
           const hts_boolean same_savefile = hts_redirect_same_savefile(
               opt, urladr(), urlfil(), moved->adr, moved->fil);
+          //if (ident_url_absolute(mov_url,moved->adr,moved->fil)!=-1) {    // ok URL reconnue
           // c'est (en gros) la même URL..
           // si c'est un problème de casse dans le host c'est que le serveur est buggé
           // ("RFC says.." : host name IS case insensitive)
@@ -3460,6 +3582,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
             } /* sinon traité normalement */
           }
 
+          //if ((strfield2(moved->adr,urladr())!=0) && (strfield2(moved->fil,urlfil())!=0)) {  // identique à casse près
           if (get_it == 1) {
             // court-circuiter le reste du traitement
             // et reculer pour mieux sauter
@@ -3470,6 +3593,9 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
             error = 1;
             hts_invalidate_link(opt, ptr);  // invalidate hashtable entry
             // noter NOUVEAU lien
+            //xxc xxc
+            //  set_prio_to=0+1;  // protection if the moved URL is an html page!!
+            //xxc xxc
             {
               // calculer lien et éventuellement modifier addresse/fichier
               if (url_savename(&savedmoved, NULL,
@@ -3507,6 +3633,8 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
               }
             }
 
+            //printf("-> %s %s %s\n",liens[lien_tot-1]->adr,liens[lien_tot-1]->fil,liens[lien_tot-1]->sav);
+
             // note métaphysique: il se peut qu'il y ait un index.html et un INDEX.HTML
             // sous DOS ca marche pas très bien... mais comme je suis génial url_savename()
             // est à même de régler ce problème
@@ -3628,6 +3756,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
 
       // cas où l'on peut reessayer
       switch (r->statuscode) {
+        //case -1: can_retry=1; break;
       case STATUSCODE_TIMEOUT:
         if (opt->hostcontrol) { // timeout et retry épuisés
           if ((opt->hostcontrol & HTS_HOSTCONTROL_BAN_TIMEOUT) &&
@@ -3759,7 +3888,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
     // FIN rattrapage des 301,302,307..
     // ------------------------------------------------------------
 
-  } // if !error
+  }                             // if !error
 
   /* Apply changes */
   ENGINE_SAVE_CONTEXT();
@@ -3826,6 +3955,9 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
           }
         }
       }
+      // On désalloue le buffer d'enregistrement des chemins créée, au cas où pendant la pause
+      // l'utilisateur ferait un rm -r après avoir effectué un tar
+      // structcheck_init(1);
       {
         FILE *fp =
           fopen(fconcat
@@ -4206,8 +4338,33 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
                     "link #%d is ready, no more on the stack, skipping: %s%s..",
                     ptr, urladr(), urlfil());
 
-      return 2; // goto jump_if_done;
+      // prochain lien
+      // ptr++;
+
+      return 2;                 // goto jump_if_done;
+
     }
+#if 0
+    /* FIXME - finalized HAS NO MORE THIS MEANING */
+    /* link put in cache by the backing system for memory spare - reclaim */
+    else if (back[b].finalized) {
+      assertf(back[b].r.adr == NULL);
+      /* read file in cache */
+      back[b].r =
+        cache_read_ro(opt, cache, back[b].url_adr, back[b].url_fil,
+                      back[b].url_sav, back[b].location_buffer);
+      /* ensure correct location buffer set */
+      back[b].r.location = back[b].location_buffer;
+      if (back[b].r.statuscode == STATUSCODE_INVALID) {
+        hts_log_print(opt, LOG_ERROR,
+                      "Unexpected error: %s%s not found anymore in cache",
+                      back[b].url_adr, back[b].url_fil);
+      } else {
+        hts_log_print(opt, LOG_DEBUG, "reclaim file %s%s (%d)", back[b].url_adr,
+                      back[b].url_fil, back[b].r.statuscode);
+      }
+    }
+#endif
 
     if (!opt->verbosedisplay) {
       if (!opt->quiet) {
@@ -4231,6 +4388,7 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
       }
       fflush(stdout);
     }
+    //}
 
     // ------------------------------------------------------------
     // Vérificateur d'intégrité
@@ -4311,8 +4469,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
       *forbidden_url == 0 && IS_DELAYED_EXT(afs->save) && !opt->state.stop) {
     int loops;
     int continue_loop;
-    char BIGSTK
-        cookie_before[16384]; /* #15: this URL's Cookie header, pre-request */
 
     hts_log_print(opt, LOG_DEBUG, "Waiting for type to be known: %s%s", afs->af.adr,
                   afs->af.fil);
@@ -4321,12 +4477,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
     for(loops = 0, continue_loop = 1;
         IS_DELAYED_EXT(afs->save) && continue_loop && loops < 7; loops++) {
       continue_loop = 0;
-      /* #15: snapshot the Cookie header THIS url would send, so only its own
-         self-redirect Set-Cookie trips the retry, not a concurrent slot's. */
-      cookie_before[0] = '\0';
-      if (opt->accept_cookie && opt->cookie != NULL)
-        http_cookie_header(opt->cookie, jump_identification_const(afs->af.adr),
-                           afs->af.fil, cookie_before, sizeof(cookie_before));
 
       /* Wait for an available slot */
       if (!hts_wait_available_socket(sback, opt, cache, ptr))
@@ -4406,6 +4556,9 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
         if (back[b].r.statuscode == STATUSCODE_INVALID && back[b].r.adr == NULL) {
           lien_back delayed_back;
 
+          //char BIGSTK delayed_ctype[128];
+          // delayed_ctype[0] = '\0';
+          // strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1);    // copier content-type
           back_copy_static(&back[b], &delayed_back);
 
           /* Delete entry */
@@ -4413,12 +4566,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           back_maydelete(opt, cache, sback, b);    // cancel
           b = -1;
 
-          /* the cancel may leave the now-unreferenced placeholder on disk
-           * (#483) */
-          if (fexist_utf8(delayed_back.url_sav)) {
-            back_delayed_discard(opt, &delayed_back);
-          }
-
           /* Recompute filename with MIME type */
           afs->save[0] = '\0';
           url_savename(afs, former, heap(ptr)->adr,
@@ -4510,6 +4657,9 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
         if (b >= 0) {
           lien_back delayed_back;
 
+          //char BIGSTK delayed_ctype[128];
+          //delayed_ctype[0] = '\0';
+          //strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1);    // copier content-type
           back_copy_static(&back[b], &delayed_back);
 
           /* Error */
@@ -4524,17 +4674,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           /* Moved! */
           else if (HTTP_IS_REDIRECT(back[b].r.statuscode)) {
             char BIGSTK mov_url[HTS_URLMAXSIZE * 2];
-            char BIGSTK cookie_after[16384];
-            hts_boolean cookies_changed;
-
-            /* #15: cookie-wall signal: this URL's own Cookie header changed
-               across its self-redirect (a Set-Cookie it just set). */
-            cookie_after[0] = '\0';
-            if (opt->accept_cookie && opt->cookie != NULL)
-              http_cookie_header(
-                  opt->cookie, jump_identification_const(afs->af.adr),
-                  afs->af.fil, cookie_after, sizeof(cookie_after));
-            cookies_changed = strcmp(cookie_before, cookie_after) != 0;
 
             mov_url[0] = '\0';
             strcpybuff(mov_url, back[b].r.location);    // copier URL
@@ -4604,24 +4743,11 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
                   url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, 
                                opt, sback, cache, hash, ptr, numero_passe,
                                &delayed_back);
-                } else if (cookies_changed) {
-                  // #15: cookie-wall self-redirect; evict the cached
-                  // fast-header so the re-issue refetches with the cookie.
-                  if (cache->cached_tests != NULL)
-                    coucal_remove(cache->cached_tests,
-                                  concat(OPT_GET_BUFF(opt),
-                                         OPT_GET_BUFF_SIZE(opt), afs->af.adr,
-                                         afs->af.fil));
-                  afs->save[0] = '\0';
-                  url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, opt,
-                               sback, cache, hash, ptr, numero_passe,
-                               &delayed_back);
-                  continue_loop = 1;
                 } else {
                   hts_log_print(opt, LOG_WARNING,
                                 "Unable to test %s%s (loop to same filename)",
                                 afs->af.adr, afs->af.fil);
-                } // loop to same location
+                }               // loop to same location
               }                 // ident_url_relatif()
             }                   // location
           }                     // redirect
@@ -4656,9 +4782,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
 
           /* Still have a back reference */
           if (b >= 0) {
-            /* move a still-writing placeholder before the url_sav patch
-               blinds every cleanup to it (#483) */
-            back_delayed_rename(opt, &back[b], afs->save);
             /* patch url_sav BEFORE finalize: it records/caches under this name
              */
             strcpybuff(back[b].url_sav, afs->save);
@@ -4685,8 +4808,13 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
       /* 'no error page' selected or file discarded by size rules! */
       if (!opt->errpage || (in_error == STATUSCODE_TOO_BIG)) {
         /* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */
-        /* Do not post-exclude the link here (*forbidden_url): the cache system
-           would never process it again, and it would be refetched endlessly */
+#if 0
+        /* No (3.43) - don't do that. We must not post-exclude an authorized link, because this will prevent the cache
+           system from processing it, leading to refetch it endlessly. Just accept it, and handle the error as
+           usual during parsing.
+         */
+        *forbidden_url = 1;     /* Forbidden! */
+#endif
         if (in_error == STATUSCODE_TOO_BIG) {
           hts_log_print(opt, LOG_INFO,
                         "link not taken because of its size (%d bytes) at %s%s",

src/htsparse.h

@@ -66,6 +66,7 @@ struct htsmoduleStructExtended {
   char ***filters_;
   robots_wizard *robots_;
   hash_struct *hash_;
+  //int *lien_max_;
 
   /* Base & codebase */
   char *base;

src/htssafe.h

@@ -456,6 +456,13 @@ static HTS_INLINE HTS_UNUSED const char *htsbuff_str(const htsbuff *b) {
   return b->buf;
 }
 
+/** True if 'count' records of >= 1 byte each fit in 'available' bytes; guards
+    an attacker-controlled count driving a large allocation. */
+static HTS_INLINE HTS_UNUSED hts_boolean hts_count_fits(size_t count,
+                                                        LLint available) {
+  return (available >= 0 && (LLint) count <= available) ? HTS_TRUE : HTS_FALSE;
+}
+
 /* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
    added bounds checking. freet() also NULLs the freed pointer and tolerates
    NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the

src/htsselftest.c

@@ -46,7 +46,6 @@ Please visit our Website: http://www.httrack.com
 #include "htsdefines.h"
 #include "htslib.h"
 #include "htsparse.h"
-#include "htscache.h"
 #include "htscache_selftest.h"
 #include "htsdns_selftest.h"
 #include "htscharset.h"
@@ -580,16 +579,15 @@ static int st_filter(httrackp *opt, int argc, char **argv) {
   int matched;
 
   (void) opt;
-  if (argc < 1) {
+  if (argc < 2) {
     fprintf(stderr, "filter: needs a filter pattern and a string\n");
     return 1;
   }
-  /* exact-size heap copies so a sanitizer traps an over-read; a missing
-     subject means "" (not reachable as a CLI arg) */
-  str = strdupt(argc >= 2 ? argv[1] : "");
+  /* exact-size heap copies so a sanitizer traps any over-read of the pattern */
+  str = strdupt(argv[1]);
   pat = strdupt(argv[0]);
   matched = strjoker(str, pat, NULL, NULL) != NULL;
-  printf("%s does %s %s\n", str, matched ? "match" : "NOT match", argv[0]);
+  printf("%s does %s %s\n", argv[1], matched ? "match" : "NOT match", argv[0]);
   freet(str);
   freet(pat);
   return 0;
@@ -621,159 +619,6 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* Mime-type filter verdict via fa_strjoker(type=1): only mime: rules apply. */
-static int st_filtermime(httrackp *opt, int argc, char **argv) {
-  int verdict;
-
-  (void) opt;
-  if (argc < 2) {
-    fprintf(stderr, "filtermime: needs <mime> <filter> [filter...]\n");
-    return 1;
-  }
-  verdict = fa_strjoker(1, &argv[1], argc - 1, argv[0], NULL, NULL, NULL);
-  printf("verdict=%s\n", verdict > 0   ? "allowed"
-                         : verdict < 0 ? "forbidden"
-                                       : "unknown");
-  return 0;
-}
-
-/* SplitMix64: deterministic, platform-independent case generator. */
-static uint64_t st_mix64(uint64_t *state) {
-  uint64_t z = (*state += UINT64_C(0x9E3779B97F4A7C15));
-
-  z = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4B5B9);
-  z = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB);
-  return z ^ (z >> 31);
-}
-
-/* Differential test: memoized strjoker vs the no-memo oracle on seeded random
-   pattern/subject/size cases must agree on result, *size and *size_flag. */
-static int st_filtermemo(httrackp *opt, int argc, char **argv) {
-  static const char *const pieces[] = {
-      "a",       "b",        "A",    "c",      ".",       "/",
-      "?",       "*",        "*[a]", "*[ab]",  "*[a-c]",  "*[A-Z]",
-      "*[<5]",   "*[>5]",    "*(a)", "*(a,b)", "*[file]", "*[path]",
-      "*[name]", "*[param]", "*[]",  "*[\\a]", "*[a-"};
-  static const char subject_chars[] = "abAc./?";
-  uint64_t rng = UINT64_C(0x501);
-  int iters = 20000, matched = 0, unmatched = 0;
-  int it;
-
-  (void) opt;
-  if (argc >= 1)
-    sscanf(argv[0], "%d", &iters);
-  for (it = 0; it < iters; it++) {
-    char pat[64], str[16];
-    size_t pl = 0;
-    const int npieces = (int) (st_mix64(&rng) % 5);
-    const int slen = (int) (st_mix64(&rng) % 13);
-    const int szsel = (int) (st_mix64(&rng) % 4); /* none/unknown/3/20 */
-    LLint sz1, sz2, off1, off2;
-    int i, flag1 = 0, flag2 = 0;
-    char *hpat, *hstr;
-    const char *adr;
-
-    for (i = 0; i < npieces; i++) {
-      const char *p =
-          pieces[st_mix64(&rng) % (sizeof(pieces) / sizeof(pieces[0]))];
-      const size_t len = strlen(p);
-
-      if (len >= sizeof(pat) - pl)
-        break;
-      memcpy(pat + pl, p, len);
-      pl += len;
-    }
-    pat[pl] = '\0';
-    for (i = 0; i < slen; i++)
-      str[i] = subject_chars[st_mix64(&rng) % (sizeof(subject_chars) - 1)];
-    str[slen] = '\0';
-    sz1 = sz2 = (szsel <= 1) ? -1 : (szsel == 2) ? 3 : 20;
-    /* exact-size heap copies per run so a sanitizer traps any over-read */
-    hpat = strdupt(pat);
-    hstr = strdupt(str);
-    adr = strjoker(hstr, hpat, szsel ? &sz1 : NULL, szsel ? &flag1 : NULL);
-    off1 = adr != NULL ? (LLint) (adr - hstr) : -1;
-    freet(hpat);
-    freet(hstr);
-    hpat = strdupt(pat);
-    hstr = strdupt(str);
-    adr =
-        strjoker_nomemo(hstr, hpat, szsel ? &sz2 : NULL, szsel ? &flag2 : NULL);
-    off2 = adr != NULL ? (LLint) (adr - hstr) : -1;
-    freet(hpat);
-    freet(hstr);
-    if (off1 != off2 || sz1 != sz2 || flag1 != flag2) {
-      printf("filtermemo MISMATCH pat=[%s] str=[%s] szsel=%d: "
-             "off %d/%d size %d/%d flag %d/%d\n",
-             pat, str, szsel, (int) off1, (int) off2, (int) sz1, (int) sz2,
-             flag1, flag2);
-      return 1;
-    }
-    if (off1 >= 0)
-      matched++;
-    else
-      unmatched++;
-  }
-  /* both polarities must actually occur or the test proves nothing */
-  assertf(matched > 0 && unmatched > 0);
-  printf("filtermemo: %d cases OK\n", iters);
-  return 0;
-}
-
-/* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
-static int st_filterdual(httrackp *opt, int argc, char **argv) {
-  int depth = -1, verdict;
-
-  (void) opt;
-  if (argc < 3) {
-    fprintf(stderr,
-            "filterdual: needs <string1> <string2> <filter> [filter...]\n");
-    return 1;
-  }
-  verdict = fa_strjoker_dual(0, &argv[2], argc - 2, argv[0], argv[1], NULL,
-                             NULL, &depth);
-  printf("verdict=%s rule=%d\n",
-         verdict > 0   ? "allowed"
-         : verdict < 0 ? "forbidden"
-                       : "unknown",
-         depth);
-  return 0;
-}
-
-/* Length/work caps stop a hostile pattern stack-overflowing or hanging the
-   process (OSS-Fuzz 5060751291908096 / 5745936014573568). */
-static int st_filterbounds(httrackp *opt, int argc, char **argv) {
-  const size_t big = 100000; /* well past the length cap */
-  const size_t stars = 900;  /* star-heavy pattern, under the cap */
-  char *subj = malloct(big + 1);
-  char *pat = malloct(2 * stars + 2);
-  size_t i;
-
-  (void) opt;
-  (void) argc;
-  (void) argv;
-  memset(subj, 'a', big);
-  subj[big] = '\0';
-  /* '*' matches anything, but an over-length subject is refused by the cap */
-  assertf(strjoker(subj, "*", NULL, NULL) == NULL);
-  assertf(strjokerfind(subj, "*") == NULL);
-  /* within-cap star-heavy dead-end: the step budget keeps it bounded (a hang
-     would time the test out) */
-  for (i = 0; i < stars; i++) {
-    pat[2 * i] = '*';
-    pat[2 * i + 1] = 'a';
-  }
-  pat[2 * stars] = 'b'; /* never matches an all-'a' subject */
-  pat[2 * stars + 1] = '\0';
-  subj[stars] = '\0';
-  assertf(strjoker(subj, pat, NULL, NULL) == NULL);
-  assertf(strjokerfind(subj, pat) == NULL);
-  freet(pat);
-  freet(subj);
-  printf("filterbounds: OK\n");
-  return 0;
-}
-
 static int st_simplify(httrackp *opt, int argc, char **argv) {
   (void) opt;
   if (argc < 1) {
@@ -822,30 +667,6 @@ static int st_charset(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-static int st_metacharset(httrackp *opt, int argc, char **argv) {
-  char *s;
-
-  (void) opt;
-  if (argc < 1) {
-    fprintf(stderr, "metacharset: needs an html string\n");
-    return 1;
-  }
-  s = hts_getCharsetFromMeta(argv[0], strlen(argv[0]));
-  printf("%s\n", s != NULL ? s : "(none)");
-  freet(s);
-  return 0;
-}
-
-static int st_isutf8(httrackp *opt, int argc, char **argv) {
-  (void) opt;
-  if (argc < 1) {
-    fprintf(stderr, "isutf8: needs a string\n");
-    return 1;
-  }
-  printf("%d\n", hts_isStringUTF8(argv[0], strlen(argv[0])) ? 1 : 0);
-  return 0;
-}
-
 static int st_idna_encode(httrackp *opt, int argc, char **argv) {
   char *s;
 
@@ -1273,57 +1094,6 @@ static int st_resolve(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* Split a URL into (adr, fil), or print "error" if rejected. A second arg pads
-   the URL with that many 'a's to reach lengths a CLI arg can't. */
-static int st_identurl(httrackp *opt, int argc, char **argv) {
-  lien_adrfil af;
-  char *url;
-  size_t len, pad = 0;
-
-  (void) opt;
-  if (argc < 1) {
-    fprintf(stderr, "identurl: needs a URL\n");
-    return 1;
-  }
-  if (argc >= 2)
-    pad = (size_t) atoi(argv[1]);
-  len = strlen(argv[0]);
-  url = malloct(len + pad + 1);
-  memcpy(url, argv[0], len);
-  memset(url + len, 'a', pad);
-  url[len + pad] = '\0';
-  if (ident_url_absolute(url, &af) >= 0)
-    printf("adr=%s fil=%s\n", af.adr, af.fil);
-  else
-    printf("error\n");
-  freet(url);
-  return 0;
-}
-
-/* Regression for the one-byte fil[] overflow: a 2047-byte hostless "?"-URL used
-   to abort in strncat_safe_ when the missing leading '/' pushed fil to 2048. */
-static int st_identabs(httrackp *opt, int argc, char **argv) {
-  lien_adrfil af;
-  const size_t len =
-      sizeof(af.fil) - 1; /* 2047: max URL the top guard admits */
-  char *url = malloct(len + 1);
-
-  (void) opt;
-  (void) argc;
-  (void) argv;
-  url[0] = '?';
-  memset(url + 1, 'a', len - 1);
-  url[len] = '\0';
-  assertf(ident_url_absolute(url, &af) == -1);
-  freet(url);
-  /* valid URLs still parse, so the guard is not over-rejecting */
-  assertf(ident_url_absolute("http://www.example.com/a/b/c.html?x=1", &af) ==
-          0);
-  assertf(ident_url_absolute("www.foo.com?bar=1", &af) == 0);
-  printf("identabs self-test OK\n");
-  return 0;
-}
-
 /* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
    no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
    registering an already-crawled link (dedup/collision paths). */
@@ -1345,102 +1115,6 @@ static int st_header(httrackp *opt, int argc, char **argv) {
     treathead(NULL, "www.example.com", "/", &r, line);
   }
   printf("contenttype=%s cdispo=%s\n", r.contenttype, r.cdispo);
-  printf("contentencoding=%s\n", r.contentencoding);
-  return 0;
-}
-
-/* An over-long header value must not overflow treathead's tempo[1100]. */
-static int st_headerlong(httrackp *opt, int argc, char **argv) {
-  htsblk r;
-  char BIGSTK line[HTS_URLMAXSIZE * 2];
-  const char *const name = argc >= 1 ? argv[0] : "Content-Type:";
-  const int pad = 1500; /* > tempo[1100] */
-  size_t n;
-
-  (void) opt;
-  memset(&r, 0, sizeof(r));
-  n = (size_t) snprintf(line, sizeof(line), "%s ", name);
-  memset(line + n, 'a', pad);
-  line[n + pad] = '\0';
-  treathead(NULL, "www.example.com", "/", &r, line);
-  printf("contenttype_len=%d contentencoding_len=%d\n",
-         (int) strlen(r.contenttype), (int) strlen(r.contentencoding));
-  return 0;
-}
-
-/* http_xfread1 must refuse an in-memory buffer whose size would exceed a 32-bit
-   index (hostile Content-Length or endless stream) rather than allocate it.
-   The guard returns before any socket read, so no real connection is needed. */
-static int st_xfread_limit(httrackp *opt, int argc, char **argv) {
-  htsblk r;
-
-  (void) opt;
-  (void) argc;
-  (void) argv;
-
-  // Content-Length just over 2 GiB.
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = (LLint) INT32_MAX + 1;
-  printf("bylen: refused=%d adr=%s msg=%s\n",
-         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
-         r.msg);
-  if (r.adr != NULL)
-    freet(r.adr);
-
-  // Unknown length, buffer already at the limit: the next read would exceed it.
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = -1;
-  r.size = (LLint) INT32_MAX;
-  printf("bygrow: refused=%d adr=%s msg=%s\n",
-         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
-         r.msg);
-  if (r.adr != NULL)
-    freet(r.adr);
-
-  // Exactly at the 2 GiB index (size + bufl == INT32_MAX): must also be
-  // refused, since the reallocs below add 1 (a `> INT32_MAX` check would let
-  // this through and overflow the int realloc size).
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = -1;
-  r.size = (LLint) INT32_MAX - 8192;
-  http_xfread1(&r, 8192);
-  printf("boundary: msg=%s\n", r.msg);
-
-  // A legitimate small size must NOT be refused by the guard (the read then
-  // fails on the invalid socket, but the size-too-large msg must not be set).
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = 1000;
-  http_xfread1(&r, 8192);
-  printf("accept: msg=%s\n", r.msg);
-  if (r.adr != NULL)
-    freet(r.adr);
-  return 0;
-}
-
-/* Parse a Content-Range header and print the sanitized triple. A hostile value
-   (negative or INT64 extreme) must clamp to 0 without signed-overflow UB. */
-static int st_crange(httrackp *opt, int argc, char **argv) {
-  int i;
-
-  (void) opt;
-  if (argc < 1) {
-    fprintf(stderr, "crange: needs at least one raw Content-Range line\n");
-    return 1;
-  }
-  for (i = 0; i < argc; i++) {
-    htsblk r;
-    char BIGSTK line[HTS_URLMAXSIZE * 2];
-
-    memset(&r, 0, sizeof(r));
-    strcpybuff(line, argv[i]);
-    treathead(NULL, "www.example.com", "/", &r, line);
-    printf("crange_start=" LLintP " crange_end=" LLintP " crange=" LLintP "\n",
-           (LLint) r.crange_start, (LLint) r.crange_end, (LLint) r.crange);
-  }
   return 0;
 }
 
@@ -1648,78 +1322,6 @@ static int st_cache(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
-/* A corrupt cache index (.ndx) must not walk the length-prefixed scan past
-   the buffer. Checks the two primitives the loader is built from. */
-static int st_cacheindex(httrackp *opt, int argc, char **argv) {
-  int fail = 0;
-
-  (void) opt;
-  (void) argc;
-  (void) argv;
-
-  /* A length prefix that overstates the bytes present must bound the advance
-     to the buffer, not trust the declared length. */
-  {
-    static const char src[] = "32768\nCACHE-1.1";
-    const size_t len = sizeof(src) - 1;
-    char *buf = malloct(len + 1);
-    char s[256];
-    int off;
-
-    memcpy(buf, src, len + 1);
-    off = cache_brstr(buf, s, sizeof(s));
-    if (off > (int) len) {
-      printf("cacheindex: over-advance off=%d len=%d\n", off, (int) len);
-      fail = 1;
-    }
-    if (strcmp(s, "CACHE-1.1") != 0) {
-      printf("cacheindex: value=%s\n", s);
-      fail = 1;
-    }
-    freet(buf);
-  }
-
-  /* cache_binput reads a field while in bounds, but refuses one starting at
-     or past end-of-buffer. */
-  {
-    char buf[8] = "ab\ncd";
-    const char *const end = buf + 5;
-    char s[16];
-
-    if (cache_binput(buf, end, s, sizeof(s)) != 3 || strcmp(s, "ab") != 0)
-      fail = 1; /* normal read: "ab" then the '\n', 3 bytes consumed */
-    if (cache_binput(end, end, s, sizeof(s)) != 0 || s[0] != '\0')
-      fail = 1;
-  }
-
-  /* Drive the full loader scan over a truncated index: a declared length that
-     overshoots plus a half-written entry. ASan aborts here on the pre-fix
-     scan; the cursor must never leave the buffer. */
-  {
-    static const char src[] = "9\nCACHE-1.1\n99\nwww.example.com\n/a";
-    const size_t len = sizeof(src) - 1;
-    char *buf = malloct(len + 1);
-    const char *const end = buf + len;
-    char line[256];
-    char *a = buf;
-
-    memcpy(buf, src, len + 1);
-    a += cache_brstr(a, line, sizeof(line));
-    a += cache_brstr(a, line, sizeof(line));
-    while (a != NULL && a < end) {
-      a = strchr(a + 1, '\n');
-      if (a == NULL)
-        break;
-      a++;
-      a += cache_binput(a, end, line, sizeof(line));
-    }
-    freet(buf);
-  }
-
-  printf("cacheindex: %s\n", fail ? "FAIL" : "OK");
-  return fail;
-}
-
 static int st_cache_golden(httrackp *opt, int argc, char **argv) {
   int regen, err;
 
@@ -1745,30 +1347,6 @@ static int st_cache_writefail(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
-static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
-  int err;
-
-  if (argc < 1) {
-    fprintf(stderr, "cache-corrupt: needs a directory\n");
-    return 1;
-  }
-  err = cache_corruption_selftest(opt, argv[0]);
-  printf("cache-corrupt: %s\n", err ? "FAIL" : "OK");
-  return err;
-}
-
-static int st_cache_legacy(httrackp *opt, int argc, char **argv) {
-  int err;
-
-  if (argc < 1) {
-    fprintf(stderr, "cache-legacy: needs a directory\n");
-    return 1;
-  }
-  err = cache_legacy_refused_selftest(opt, argv[0]);
-  printf("cache-legacy: %s\n", err ? "FAIL" : "OK");
-  return err;
-}
-
 static int st_reconcile(httrackp *opt, int argc, char **argv) {
   int err;
 
@@ -1813,54 +1391,13 @@ static int st_cookies(httrackp *opt, int argc, char **argv) {
     return 1;
   }
 
-  http_cookie_header(&cookie, dom, "/", hdr, sizeof(hdr));
+  http_cookie_header_selftest(&cookie, dom, "/", hdr, sizeof(hdr));
   if (strcmp(hdr, expected) != 0)
     err = 1;
-
-  /* A hostile over-long request host must not overflow domain[256] in
-     treathead's default-domain copy (that would abort the mirror). */
-  {
-    static t_cookie ck2;
-    htsblk r;
-    char host[600];
-
-    memset(&r, 0, sizeof(r));
-    memset(host, 'a', sizeof(host) - 1);
-    host[sizeof(host) - 1] = '\0';
-    ck2.max_len = (int) sizeof(ck2.data);
-    ck2.data[0] = '\0';
-    treathead(&ck2, host, "/", &r, "Set-Cookie: SID=1; path=/");
-    if (strnotempty(ck2.data)) // oversize-host cookie was not dropped
-      err = 1;
-    /* control: a normal host still yields a cookie through treathead */
-    treathead(&ck2, dom, "/", &r, "Set-Cookie: SID=1; path=/");
-    if (strstr(ck2.data, "SID") == NULL) // guard wrongly dropped a valid cookie
-      err = 1;
-  }
   if (strstr(hdr, "$Version") != NULL || strstr(hdr, "$Path") != NULL)
     err = 1;
   if (strstr(hdr, "junk") != NULL) // wrong-domain cookie leaked
     err = 1;
-#ifndef _WIN32
-  /* the jar holds live session cookies: cookie_save must keep it 0600 */
-  {
-    const char *jar = "st-cookies-jar.txt";
-    struct stat st;
-
-    (void) UNLINK(jar);
-    assertf(cookie_save(&cookie, jar) == 0);
-    assertf(stat(jar, &st) == 0);
-    assertf((st.st_mode & 07777) == HTS_PROTECT_FILE);
-    assertf(st.st_size > 0); /* mode-only checks would pass an empty jar */
-    /* a pre-existing world-readable jar must be tightened, not kept */
-    assertf(chmod(jar, 0644) == 0);
-    assertf(cookie_save(&cookie, jar) == 0);
-    assertf(stat(jar, &st) == 0);
-    assertf((st.st_mode & 07777) == HTS_PROTECT_FILE);
-    assertf(st.st_size > 0);
-    (void) UNLINK(jar);
-  }
-#endif
   printf("cookie-header: %s\n", err ? "FAIL" : "OK");
   if (err)
     printf("  got: %s\n", hdr);
@@ -2302,19 +1839,6 @@ static int ae_write_collision(const char *path, const unsigned char *src,
   return ok ? 0 : 1;
 }
 
-/* Write src[0..len) to path as-is; 0 on success. */
-static int ae_write_raw(const char *path, const unsigned char *src,
-                        size_t len) {
-  FILE *const f = FOPEN(path, "wb");
-  int ok;
-
-  if (f == NULL)
-    return 1;
-  ok = fwrite(src, 1, len, f) == len;
-  fclose(f);
-  return ok ? 0 : 1;
-}
-
 /* Compare path's bytes to expect[0..len); 0 if equal. Streams (large files). */
 static int ae_check_decoded(const char *path, const unsigned char *expect,
                             size_t len) {
@@ -2382,46 +1906,6 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
     assertf(ae_write_collision(inpath, body, 64) == 0);
     assertf(hts_zunpack(inpath, outpath) == 64);
     assertf(ae_check_decoded(outpath, body, 64) == 0);
-    /* Identity fallback (#47): a plain body mislabeled as compressed is kept
-       verbatim, small and multi-chunk (> one 8 KiB fread). */
-    assertf(ae_write_raw(inpath, small, slen) == 0);
-    assertf(hts_zunpack(inpath, outpath) == (int) slen);
-    assertf(ae_check_decoded(outpath, small, slen) == 0);
-    {
-      const size_t ilen = 16 * 1024;
-      unsigned char *idbody = malloct(ilen);
-
-      assertf(idbody != NULL);
-      for (i = 0; i < ilen; i++)
-        idbody[i] = small[i % slen];
-      assertf(ae_write_raw(inpath, idbody, ilen) == 0);
-      assertf(hts_zunpack(inpath, outpath) == (int) ilen);
-      assertf(ae_check_decoded(outpath, idbody, ilen) == 0);
-      freet(idbody);
-    }
-    /* Truncated gzip (CRC+ISIZE cut), zlib (ADLER32 cut) and raw deflate
-       must all still fail, not fall back to a verbatim copy. */
-    {
-      static const struct {
-        int wb;
-        size_t cut;
-      } tr[] = {{16 + MAX_WBITS, 8}, {MAX_WBITS, 4}, {-MAX_WBITS, 5}};
-
-      for (i = 0; i < sizeof(tr) / sizeof(tr[0]); i++) {
-        unsigned char z[512];
-        size_t zlen;
-        FILE *f;
-
-        assertf(ae_write_packed(inpath, tr[i].wb, small, slen) == 0);
-        f = FOPEN(inpath, "rb");
-        assertf(f != NULL);
-        zlen = fread(z, 1, sizeof(z), f);
-        fclose(f);
-        assertf(zlen > tr[i].cut && zlen < sizeof(z));
-        assertf(ae_write_raw(inpath, z, zlen - tr[i].cut) == 0);
-        assertf(hts_zunpack(inpath, outpath) < 0);
-      }
-    }
     freet(body);
   }
 #else
@@ -2580,6 +2064,21 @@ static int st_ftpuser(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* hts_count_fits caps the .class constant-pool entry count to the file size,
+   rejecting the ~68 MB-per-file calloc DoS. */
+static int st_java(httrackp *opt, int argc, char **argv) {
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  assertf(hts_count_fits(10, 1000) == HTS_TRUE);
+  assertf(hts_count_fits(0, 10) == HTS_TRUE);
+  assertf(hts_count_fits(65535, 10) == HTS_FALSE);
+  assertf(hts_count_fits(1, 0) == HTS_FALSE);
+  assertf(hts_count_fits(1, -1) == HTS_FALSE);
+  printf("java constant-pool cap self-test OK\n");
+  return 0;
+}
+
 /* ------------------------------------------------------------ */
 /* Registry: name -> handler, with a usage hint and a one-line description. */
 /* ------------------------------------------------------------ */
@@ -2595,14 +2094,6 @@ static const struct selftest_entry {
     {"filtersize", "<size> <string> <filter>...",
      "size-aware filter verdict (negative size = unknown/scan time)",
      st_filtersize},
-    {"filtermime", "<mime> <filter>...",
-     "mime-type filter verdict (fa_strjoker type=1)", st_filtermime},
-    {"filtermemo", "[iterations]",
-     "memoized vs unmemoized matcher differential", st_filtermemo},
-    {"filterdual", "<string1> <string2> <filter>...",
-     "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
-    {"filterbounds", "", "matcher length/work caps reject hostile patterns",
-     st_filterbounds},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
@@ -2613,9 +2104,6 @@ static const struct selftest_entry {
     {"mime", "<filename>", "MIME type for a filename", st_mime},
     {"charset", "<charset> <string>",
      "convert a string to UTF-8 from a charset", st_charset},
-    {"metacharset", "<html>", "extract the <meta> charset from an HTML page",
-     st_metacharset},
-    {"isutf8", "<string>", "is the string valid UTF-8 (1/0)", st_isutf8},
     {"idna-encode", "<host>", "encode a hostname to IDNA/punycode",
      st_idna_encode},
     {"idna-decode", "<host>", "decode an IDNA/punycode hostname",
@@ -2632,35 +2120,19 @@ static const struct selftest_entry {
      st_relative},
     {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
      st_resolve},
-    {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
-    {"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
-     st_identabs},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
-    {"headerlong", "[header-name:]",
-     "over-long header value must not overflow the parse scratch",
-     st_headerlong},
-    {"crange", "<raw-content-range-line> ...",
-     "Content-Range parse integer safety", st_crange},
-    {"xfread-limit", "", "in-memory receive buffer size bound",
-     st_xfread_limit},
     {"savename", "<fil> <content-type> [key=value ...]",
      "local save-name for a URL", st_savename},
     {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
      st_sniff},
     {"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
-    {"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
-     st_cacheindex},
     {"cache-golden", "<dir> [regen]", "frozen cache-format read self-test",
      st_cache_golden},
     {"cache-writefail", "<dir>", "cache write-failure handling self-test",
      st_cache_writefail},
     {"reconcile", "<dir>", "cache generation reconcile policy self-test",
      st_reconcile},
-    {"cache-legacy", "<dir>", "pre-3.31 legacy cache refusal self-test",
-     st_cache_legacy},
-    {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
-     st_cache_corrupt},
     {"dns", "", "DNS resolver/cache self-test", st_dns},
     {"cookies", "", "cookie request-header self-test", st_cookies},
     {"useragent", "", "default User-Agent self-test", st_useragent},
@@ -2680,6 +2152,7 @@ static const struct selftest_entry {
      st_ftpline},
 #endif
     {"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
+    {"java", "", "java .class constant-pool count cap self-test", st_java},
 };
 
 static void list_selftests(void) {

src/htsserver.c

@@ -265,6 +265,7 @@ T_SOC smallserver_init(int *port, char *adr) {
       SOCaddr_initport(server, *port);
       if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
         if (listen(soc, 10) >= 0) {
+          // SOCaddr_inetntoa(adr, 128, server2);
           strcpy(adr, h_loc);
         } else {
 #ifdef _WIN32
@@ -438,6 +439,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
         meth = 10;
       } else {
 #ifdef _DEBUG
+        // assert(FALSE);
 #endif
       }
       if (meth) {
@@ -1358,6 +1360,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
 
             StringCat(headers, error_hdr);
             StringCat(output, error);
+            //assert(file == NULL);
           }
         }
       } else {
@@ -1385,10 +1388,12 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                   != StringLength(output)))
         ) {
 #ifdef _DEBUG
+        //assert(FALSE);
 #endif
       }
     } else {
 #ifdef _DEBUG
+      // assert(FALSE);
 #endif
     }
 
@@ -1535,7 +1540,8 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
             } while(strnotempty(test));
           }
 
-          if (!strnotempty(test)) { // éviter doublons
+          if (!strnotempty(test)) {     // éviter doublons
+            // conv_printf(key,key);
             const size_t len = strlen(intkey);
             char *const buff = (char *) malloc(len + 1);
 
@@ -1626,6 +1632,8 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
                     intkey = "";
                 } else {
                   if (loops > 0) {
+                    //err_msg += intkey;
+                    //err_msg += " ";
                   }
                 }
               }
@@ -1732,7 +1740,13 @@ static void LANG_DELETE(void) {
 }
 
 // sélection de la langue
-static void LANG_INIT(const char *path) { LANG_T(path, 0); }
+static void LANG_INIT(const char *path) {
+  //CWinApp* pApp = AfxGetApp();
+  //if (pApp) {
+  /* pApp->GetProfileInt("Language","IntId",0); */
+  LANG_T(path, 0 /*pApp->GetProfileInt("Language","IntId",0) */ );
+  //}
+}
 
 static int LANG_T(const char *path, int l) {
   if (l >= 0) {

src/htstools.c

@@ -68,6 +68,16 @@ struct find_handle_struct {
   char path[2048];
 };
 #endif
+//#ifndef HTS_DEF_FWSTRUCT_topindex_chain
+//#define HTS_DEF_FWSTRUCT_topindex_chain
+//typedef struct topindex_chain topindex_chain;
+//#endif
+//struct topindex_chain {
+//  int level;                    /* sort level */
+//  char *category;               /* category */
+//  char name[2048];              /* path */
+//  struct topindex_chain *next;  /* next element */
+//};
 
 /* Tools */
 
@@ -283,6 +293,7 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
   char BIGSTK newcurr_fil[HTS_URLMAXSIZE * 2], newlink[HTS_URLMAXSIZE * 2];
   char *curr;
 
+  //int n=0;
   char *a;
   int slash = 0;
 
@@ -327,6 +338,7 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
     if (*curr == '/')
       curr++;
     l = link;
+    //c=curr;
     // couper ce qui est commun
     while((streql(*link, *curr)) && (*link != 0)) {
       link++;
@@ -338,6 +350,8 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
       link--;
       curr--;
     }
+    //if (*link=='/') link++;
+    //if (*curr=='/') curr++;
   }
 
   // calculer la profondeur du répertoire courant et remonter
@@ -348,6 +362,7 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
   while(*a)
     if (*(a++) == '/')
       strlcatbuff(s, "../", ssize);
+  //if (strlen(s)==0) strcatbuff(s,"/");
 
   if (slash)
     strlcatbuff(s, "/", ssize); // keep it absolute!

src/htsweb.c

@@ -639,7 +639,8 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b
           for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) {        // no lien
             int i = (back_index + _i) % back_max;       // commencer par le "premier" (l'actuel)
 
-            if (back[i].status >= 0) { // signifie "lien actif"
+            if (back[i].status >= 0) {  // signifie "lien actif"
+              // int ok=0;  // OPTI
               ok = 0;
               switch (j) {
               case 0:          // prioritaire