Add --why to explain which filter rule accepts or rejects a URL (#505 )

The top user-confusion class is filters silently not doing what their author expects. --why <url>, given the rest of the crawl command line, prints the deciding rule ("rejected by rule #2 (-*.zip)") or reports that no rule matches, then exits without crawling. It reuses the real option parsing and filter seeding end to end, and evaluates through the new fa_strjoker_dual(), which also replaces the two hand-rolled copies of the dual-form verdict merge in htswizard.c. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
Enable _FORTIFY_SOURCE and -fstack-protector-strong (#504 )
2026-07-08 01:46:21 +03:00 · 2026-07-07 20:09:25 +02:00 · 2026-07-07 20:04:09 +02:00 · 2026-07-07 20:03:28 +02:00 · 2026-07-07 18:24:53 +02:00 · 2026-07-07 18:14:16 +02:00
75 changed files with 1466 additions and 1529 deletions
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -278,6 +278,45 @@ jobs:
        if: failure()
        run: cat tests/test-suite.log 2>/dev/null || true

+  # libFuzzer smoke: build the fuzz/ harnesses over the pure hostile-input
+  # parsers and replay each seed corpus under ASan+UBSan+LeakSanitizer. Replay
+  # (not open-ended mutation) keeps CI deterministic -- it can't hit strjoker's
+  # catastrophic backtracking -- and pins the regression seeds for the bugs the
+  # fuzzers found. Deep discovery is a maintainer / OSS-Fuzz activity.
+  fuzz:
+    name: fuzz (libFuzzer corpus replay, clang)
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential clang autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - name: Configure (fuzzers, static)
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure CC=clang \
+            CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1 -fno-omit-frame-pointer" \
+            LDFLAGS="-fsanitize=address,undefined" \
+            --enable-fuzzers --disable-shared
+
+      - name: Build
+        run: make -j"$(nproc)"
+
+      - name: Replay corpora
+        env:
+          ASAN_OPTIONS: detect_leaks=1:abort_on_error=1:halt_on_error=1
+          UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
+        run: bash fuzz/run-fuzzers.sh fuzz check
+
  # Optional-dependency build: compile and test with HTTPS/OpenSSL disabled --
  # the configuration users on minimal systems build, and one libssl is not even
  # installed here so configure cannot silently re-enable it. The matrix above
--- a/.github/workflows/codeql.yml
+++ b/.github/workflows/codeql.yml
@@ -0,0 +1,57 @@
+# CodeQL static analysis (C). The security-extended suite covers the classes
+# this codebase actually fights: overflows, tainted-size allocs, format bugs.
+name: CodeQL
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+  schedule:
+    # Weekly re-scan of master so new/updated queries land without a push.
+    - cron: "17 4 * * 1"
+
+permissions:
+  contents: read
+
+concurrency:
+  group: codeql-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  analyze:
+    name: analyze (c-cpp)
+    runs-on: ubuntu-24.04
+    permissions:
+      contents: read
+      # Upload findings to the repo's code-scanning dashboard.
+      security-events: write
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          sudo apt-get update
+          sudo apt-get install -y --no-install-recommends \
+            build-essential autoconf automake libtool autoconf-archive \
+            zlib1g-dev libssl-dev
+
+      - uses: github/codeql-action/init@v4
+        with:
+          languages: c-cpp
+          build-mode: manual
+          queries: security-extended
+
+      # Manual build: CodeQL traces the compiler, so build exactly what ships.
+      - name: Build
+        run: |
+          set -euo pipefail
+          autoreconf -fi
+          ./configure
+          make -j"$(nproc)"
+
+      - uses: github/codeql-action/analyze@v4
+        with:
+          category: "/language:c-cpp"
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,5 +1,5 @@

-SUBDIRS = src man m4 libtest templates lang html tests
+SUBDIRS = src man m4 libtest templates lang html tests fuzz

 ACLOCAL_AMFLAGS = -I m4

--- a/configure.ac
+++ b/configure.ac
@@ -102,7 +102,8 @@ AX_CHECK_COMPILE_FLAG([-Wignored-qualifiers], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -
 AX_CHECK_COMPILE_FLAG([-Werror=attribute-warning], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=attribute-warning"])
 AX_CHECK_COMPILE_FLAG([-Werror=user-defined-warnings], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -Werror=user-defined-warnings"])
 AX_CHECK_COMPILE_FLAG([-fstrict-aliasing -Wstrict-aliasing], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstrict-aliasing -Wstrict-aliasing"])
-AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])
+AX_CHECK_COMPILE_FLAG([-fstack-protector-strong], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector-strong"],
+	[AX_CHECK_COMPILE_FLAG([-fstack-protector], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-protector"])])
 AX_CHECK_COMPILE_FLAG([-fstack-clash-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fstack-clash-protection"])
 AX_CHECK_COMPILE_FLAG([-fcf-protection], [DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fcf-protection"])
 AX_CHECK_LINK_FLAG([-Wl,--discard-all], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,--discard-all"])
@@ -110,6 +111,13 @@ AX_CHECK_LINK_FLAG([-Wl,--no-undefined], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,
 AX_CHECK_LINK_FLAG([-Wl,-z,relro,-z,now], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,relro,-z,now"])
 AX_CHECK_LINK_FLAG([-Wl,-z,noexecstack], [DEFAULT_LDFLAGS="$DEFAULT_LDFLAGS -Wl,-z,noexecstack"])

+# Fortify libc calls (=3, else =2) unless the toolchain predefines it; skip
+# sanitizer builds, whose interceptors want the unfortified calls.
+case "$CFLAGS" in
+*-fsanitize=*) ;;
+*) AX_ADD_FORTIFY_SOURCE ;;
+esac
+
 # Force libc back into DT_NEEDED for libraries that reach it only through
 # libhttrack (libhtsjava, the libtest callbacks), but only with a GNU-style
 # linker; Apple ld rejects these flags and links libSystem unconditionally.
@@ -310,6 +318,37 @@ AC_ARG_ENABLE([online-unit-tests],
 	])
 AC_SUBST(ONLINE_UNIT_TESTS,$online_unit_tests)

+## libFuzzer harnesses (fuzz/); requires clang
+AC_MSG_CHECKING(whether to build fuzzers)
+AC_ARG_ENABLE([fuzzers],
+	[AS_HELP_STRING([--enable-fuzzers],[Build libFuzzer harnesses in fuzz/ (requires clang) @<:@default=no@:>@])],
+	[
+	case "${enableval}" in
+	no|yes)
+		fuzzers=$enableval
+		AC_MSG_RESULT($enableval)
+		;;
+	*)
+		AC_MSG_ERROR(bad value for fuzzers, expected yes/no)
+		;;
+	esac
+	],
+	[
+	fuzzers=no
+	AC_MSG_RESULT(no)
+	])
+if test x"$fuzzers" = x"yes"; then
+	# Instrument the whole build for coverage; harnesses link -fsanitize=fuzzer.
+	AX_CHECK_COMPILE_FLAG([-fsanitize=fuzzer-no-link],
+		[DEFAULT_CFLAGS="$DEFAULT_CFLAGS -fsanitize=fuzzer-no-link"],
+		[AC_MSG_ERROR([--enable-fuzzers requires libFuzzer support (clang)])])
+	# clang's static sanitizer runtimes clash with -Wl,--no-undefined on the .so.
+	if test x"$enable_shared" != x"no"; then
+		AC_MSG_ERROR([--enable-fuzzers requires --disable-shared])
+	fi
+fi
+AM_CONDITIONAL([FUZZERS], [test x"$fuzzers" = x"yes"])
+
 # Final output
 AC_CONFIG_FILES([
 Makefile
@@ -321,5 +360,6 @@ lang/Makefile
 html/Makefile
 libtest/Makefile
 tests/Makefile
+fuzz/Makefile
 ])
 AC_OUTPUT
--- a/fuzz/Makefile.am
+++ b/fuzz/Makefile.am
@@ -0,0 +1,38 @@
+# libFuzzer harnesses; built only with --enable-fuzzers (requires clang).
+if FUZZERS
+noinst_PROGRAMS = fuzz-charset fuzz-meta fuzz-idna fuzz-entities \
+	fuzz-unescape fuzz-filters fuzz-url
+endif
+
+AM_CPPFLAGS = \
+	@DEFAULT_CFLAGS@ \
+	@THREADS_CFLAGS@ \
+	@V6_FLAG@ \
+	@LFS_FLAG@ \
+	-I$(top_srcdir)/src \
+	-I$(top_srcdir)/src/coucal
+
+# Static-link libhttrack.la: the internal symbols are hidden in the .so.
+AM_LDFLAGS = @DEFAULT_LDFLAGS@ -fsanitize=fuzzer -static-libtool-libs
+LDADD = $(top_builddir)/src/libhttrack.la $(THREADS_LIBS)
+
+fuzz_charset_SOURCES = fuzz-charset.c fuzz.h
+fuzz_meta_SOURCES = fuzz-meta.c fuzz.h
+fuzz_idna_SOURCES = fuzz-idna.c fuzz.h
+fuzz_entities_SOURCES = fuzz-entities.c fuzz.h
+fuzz_unescape_SOURCES = fuzz-unescape.c fuzz.h
+fuzz_filters_SOURCES = fuzz-filters.c fuzz.h
+fuzz_url_SOURCES = fuzz-url.c fuzz.h
+
+# List corpus files explicitly: automake does not expand EXTRA_DIST globs.
+EXTRA_DIST = README.md run-fuzzers.sh \
+	corpus/charset/utf8.txt corpus/charset/latin1.txt corpus/charset/sjis.txt \
+	corpus/meta/meta-charset.html corpus/meta/meta-http-equiv.html \
+	corpus/idna/idna.txt corpus/idna/unicode.txt \
+	corpus/idna/regress-multilabel-leak.txt \
+	corpus/entities/entities.txt \
+	corpus/unescape/percent.txt \
+	corpus/filters/filter.bin corpus/filters/filter-size.bin \
+	corpus/filters/regress-empty-subject-unique.bin \
+	corpus/url/http-url.txt corpus/url/relative-path.txt \
+	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt
--- a/fuzz/README.md
+++ b/fuzz/README.md
@@ -0,0 +1,15 @@
+# Fuzzing httrack
+
+libFuzzer harnesses for the pure hostile-input parsers (charset/UTF-8/IDNA codecs, entity and percent decoders, wildcard filters, URL splitter). Off by default; needs clang.
+
+```sh
+./bootstrap
+mkdir /var/tmp/bld-fuzz && cd /var/tmp/bld-fuzz
+CC=clang CFLAGS="-fsanitize=address,undefined -fno-sanitize-recover=all -g -O1" \
+  LDFLAGS="-fsanitize=address,undefined" \
+  bash /path/to/httrack/configure --enable-fuzzers --disable-shared
+make
+bash /path/to/httrack/fuzz/run-fuzzers.sh fuzz 60   # 60s per target
+```
+
+Run one target by hand: `fuzz/fuzz-url -max_total_time=300 corpusdir fuzz/corpus/url`. Seed corpora live in `corpus/<target>/`; a crash reproducer is replayed with `fuzz/fuzz-url crash-file`.
--- a/fuzz/corpus/charset/latin1.txt
+++ b/fuzz/corpus/charset/latin1.txt
@@ -0,0 +1 @@
+café naďve ¤
--- a/fuzz/corpus/charset/sjis.txt
+++ b/fuzz/corpus/charset/sjis.txt
@@ -0,0 +1 @@
+コンピュータ
--- a/fuzz/corpus/charset/utf8.txt
+++ b/fuzz/corpus/charset/utf8.txt
--- a/fuzz/corpus/entities/entities.txt
+++ b/fuzz/corpus/entities/entities.txt
@@ -0,0 +1 @@
+&amp;&lt;&gt;&#65;&#x2603;&eacute;&notarealentity;&#xFFFFFFFF;
--- a/fuzz/corpus/filters/filter-size.bin
+++ b/fuzz/corpus/filters/filter-size.bin
--- a/fuzz/corpus/filters/filter.bin
+++ b/fuzz/corpus/filters/filter.bin
--- a/fuzz/corpus/filters/regress-empty-subject-unique.bin
+++ b/fuzz/corpus/filters/regress-empty-subject-unique.bin
@@ -0,0 +1 @@
+**((
--- a/fuzz/corpus/idna/idna.txt
+++ b/fuzz/corpus/idna/idna.txt
@@ -0,0 +1 @@
+xn--bcher-kva.example.com
--- a/fuzz/corpus/idna/regress-multilabel-leak.txt
+++ b/fuzz/corpus/idna/regress-multilabel-leak.txt
@@ -0,0 +1 @@
+bÃ¼chev.ä¾‹åbÃ¼cheple
--- a/fuzz/corpus/idna/unicode.txt
+++ b/fuzz/corpus/idna/unicode.txt
@@ -0,0 +1 @@
+bücher.例子.example
--- a/fuzz/corpus/meta/meta-charset.html
+++ b/fuzz/corpus/meta/meta-charset.html
@@ -0,0 +1 @@
+<html><head><meta charset="utf-8"></head><body>x</body></html>
--- a/fuzz/corpus/meta/meta-http-equiv.html
+++ b/fuzz/corpus/meta/meta-http-equiv.html
@@ -0,0 +1 @@
+<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">
--- a/fuzz/corpus/unescape/percent.txt
+++ b/fuzz/corpus/unescape/percent.txt
@@ -0,0 +1 @@
+%41%zz%%20%c3%a9+%2e%2e%2f
--- a/fuzz/corpus/url/http-url.txt
+++ b/fuzz/corpus/url/http-url.txt
@@ -0,0 +1 @@
+http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag
--- a/fuzz/corpus/url/regress-file-empty-path.txt
+++ b/fuzz/corpus/url/regress-file-empty-path.txt
@@ -0,0 +1 @@
+file://
--- a/fuzz/corpus/url/regress-long-path-abort.txt
+++ b/fuzz/corpus/url/regress-long-path-abort.txt
@@ -0,0 +1 @@
+ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../
--- a/fuzz/corpus/url/relative-path.txt
+++ b/fuzz/corpus/url/relative-path.txt
@@ -0,0 +1 @@
+ftp://ftp.example.com/pub/../file.txt
--- a/fuzz/fuzz-charset.c
+++ b/fuzz/fuzz-charset.c
@@ -0,0 +1,84 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the charset codecs: hts_convertStringToUTF8/FromUTF8 and the
+   UTF-8/UCS4 primitives (htscharset.c). First input byte picks the charset. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+static const char *const charsets[] = {
+    "utf-8",    "iso-8859-1", "iso-8859-2", "iso-8859-15", "windows-1252",
+    "us-ascii", "shift_jis",  "euc-jp",     "iso-2022-jp", "gb2312",
+    "big5",     "euc-kr",     "koi8-r",     "utf-16",      "unknown-charset",
+};
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  const char *charset;
+  char *s;
+
+  if (size == 0)
+    return 0;
+  charset = charsets[data[0] % (sizeof(charsets) / sizeof(charsets[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+
+  {
+    char *utf8 = hts_convertStringToUTF8(s, size, charset);
+    freet(utf8);
+  }
+  {
+    char *enc = hts_convertStringFromUTF8(s, size, charset);
+    freet(enc);
+  }
+  {
+    size_t nChars = 0;
+    hts_UCS4 *ucs = hts_convertUTF8StringToUCS4(s, size, &nChars);
+
+    if (ucs != NULL) {
+      char *back = hts_convertUCS4StringToUTF8(ucs, nChars);
+      freet(back);
+      freet(ucs);
+    }
+  }
+  {
+    size_t i = 0;
+
+    while (i < size) {
+      hts_UCS4 uc = 0;
+      const size_t nr = hts_readUTF8(s + i, size - i, &uc);
+      char out[8];
+
+      if (nr == 0)
+        break;
+      hts_writeUTF8(uc, out, sizeof(out));
+      i += nr;
+    }
+  }
+
+  freet(s);
+  return 0;
+}
--- a/fuzz/fuzz-entities.c
+++ b/fuzz/fuzz-entities.c
@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the HTML entity decoder (htsencoding.c). First input byte picks the
+   destination size, so truncation bounds get exercised too. */
+#include "fuzz.h"
+#include "htsencoding.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t dsizes[] = {1, 2, 8, 64, 4096};
+  size_t dsize;
+  char *src, *dest;
+
+  if (size == 0)
+    return 0;
+  dsize = dsizes[data[0] % (sizeof(dsizes) / sizeof(dsizes[0]))];
+  data++, size--;
+  src = fuzz_strdup(data, size);
+  dest = malloct(dsize);
+
+  (void) hts_unescapeEntities(src, dest, dsize);
+  (void) hts_unescapeEntitiesWithCharset(src, dest, dsize, "iso-8859-1");
+
+  freet(dest);
+  freet(src);
+  return 0;
+}
--- a/fuzz/fuzz-filters.c
+++ b/fuzz/fuzz-filters.c
@@ -0,0 +1,65 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the wildcard filter matcher (htsfilters.c; #148 bracket-range OOB was
+   here). Input splits on the first NUL: pattern, then subject string. */
+#include "fuzz.h"
+#include "htsfilters.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *buf = fuzz_strdup(data, size);
+  const char *joker = buf;
+  const uint8_t *sep = memchr(data, '\0', size);
+  /* subject in its own allocation so ASan bounds it apart from the pattern */
+  char *nom = sep != NULL ? fuzz_strdup(sep + 1, (data + size) - (sep + 1))
+                          : fuzz_strdup(data + size, 0);
+
+  (void) strjoker(nom, joker, NULL, NULL);
+  {
+    LLint sz = (LLint) size;
+    int size_flag = 0;
+
+    (void) strjoker(nom, joker, &sz, &size_flag);
+  }
+  (void) strjokerfind(nom, joker);
+  {
+    char *filter = malloct(strlen(joker) + 2);
+    char *filters[1];
+    LLint sz = (LLint) size;
+    int size_flag = 0, depth = 0;
+
+    filter[0] = '-';
+    memcpy(filter + 1, joker, strlen(joker) + 1);
+    filters[0] = filter;
+    (void) fa_strjoker(0, filters, 1, nom, &sz, &size_flag, &depth);
+    freet(filter);
+  }
+
+  freet(nom);
+  freet(buf);
+  return 0;
+}
--- a/fuzz/fuzz-idna.c
+++ b/fuzz/fuzz-idna.c
@@ -0,0 +1,47 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the IDNA/punycode codec (htscharset.c, CVE-prone lineage). */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+
+  {
+    char *idna = hts_convertStringUTF8ToIDNA(s, size);
+    freet(idna);
+  }
+  {
+    char *utf8 = hts_convertStringIDNAToUTF8(s, size);
+    freet(utf8);
+  }
+  (void) hts_isStringIDNA(s, size);
+
+  freet(s);
+  return 0;
+}
--- a/fuzz/fuzz-meta.c
+++ b/fuzz/fuzz-meta.c
@@ -0,0 +1,40 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz hts_getCharsetFromMeta (htscharset.c): scans raw attacker HTML for a
+   <meta> charset declaration. */
+#include "fuzz.h"
+#include "htscharset.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *html = fuzz_strdup(data, size);
+  char *charset = hts_getCharsetFromMeta(html, size);
+
+  freet(charset);
+  freet(html);
+  return 0;
+}
--- a/fuzz/fuzz-unescape.c
+++ b/fuzz/fuzz-unescape.c
@@ -0,0 +1,53 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL percent-decoders (htslib.c). First input byte picks the
+   output buffer size, so the bounded-copy contract is exercised. */
+#include "fuzz.h"
+#include "httrack-library.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  static const size_t bsizes[] = {1, 2, 16, 256, 8192};
+  size_t bsize;
+  char *s, *catbuff;
+
+  if (size == 0)
+    return 0;
+  bsize = bsizes[data[0] % (sizeof(bsizes) / sizeof(bsizes[0]))];
+  data++, size--;
+  s = fuzz_strdup(data, size);
+  catbuff = malloct(bsize);
+
+  (void) unescape_http(catbuff, bsize, s);
+  (void) unescape_http_unharm(catbuff, bsize, s, 0);
+  (void) unescape_http_unharm(catbuff, bsize, s, 1);
+  unescape_amp(s);
+
+  freet(catbuff);
+  freet(s);
+  return 0;
+}
--- a/fuzz/fuzz-url.c
+++ b/fuzz/fuzz-url.c
@@ -0,0 +1,49 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Fuzz the URL splitter and path normalizer (htslib.c): ident_url_absolute
+   is the first parser to touch a raw URL; fil_simplifie collapses ./ and ../
+   in place. */
+#include "fuzz.h"
+#include "htscore.h"
+#include "htslib.h"
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {
+  char *s = fuzz_strdup(data, size);
+  lien_adrfil *af = calloct(1, sizeof(*af));
+  /* fil_simplifie rewrites in place and may grow an empty path to "./" */
+  char *path = malloct(size + 3);
+
+  (void) ident_url_absolute(s, af);
+  memcpy(path, s, size + 1);
+  fil_simplifie(path);
+
+  freet(path);
+  freet(af);
+  freet(s);
+  return 0;
+}
--- a/fuzz/fuzz.h
+++ b/fuzz/fuzz.h
@@ -0,0 +1,51 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* Shared helpers for the libFuzzer harnesses. */
+#ifndef FUZZ_H
+#define FUZZ_H
+
+#define HTS_INTERNAL_BYTECODE
+
+#include <stddef.h>
+#include <stdint.h>
+#include <string.h>
+
+#include "htsbase.h"
+
+/* Heap NUL-terminated copy of the fuzzer input, so ASan bounds every read. */
+static char *fuzz_strdup(const uint8_t *data, size_t size) {
+  char *s = malloct(size + 1);
+
+  memcpy(s, data, size);
+  s[size] = '\0';
+  return s;
+}
+
+int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size);
+
+#endif
--- a/fuzz/run-fuzzers.sh
+++ b/fuzz/run-fuzzers.sh
@@ -0,0 +1,41 @@
+#!/bin/bash
+# Drive every built harness against its seed corpus.
+#   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
+#   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
+# Replay is crash/leak-only and never mutates, so it can't hit strjoker's
+# catastrophic backtracking; the timed mode can, hence the per-unit -timeout.
+set -euo pipefail
+
+srcdir=$(cd "$(dirname "$0")" && pwd)
+bld=${1:?usage: run-fuzzers.sh <build-fuzz-dir> [check|seconds]}
+mode=${2:-20}
+
+status=0
+for f in "$bld"/fuzz-*; do
+    if [ ! -f "$f" ] || [ ! -r "$f" ]; then continue; fi
+    case "$f" in *.o | *.c | *.dSYM) continue ;; esac
+    name=$(basename "$f")
+    corpus="$srcdir/corpus/${name#fuzz-}"
+    if [ "$mode" = "check" ]; then
+        echo "=== $name (replay) ==="
+        [ -d "$corpus" ] || continue
+        if ! "$f" -runs=0 -timeout=25 -rss_limit_mb=2048 "$corpus"; then
+            echo "*** $name FAILED on its corpus" >&2
+            status=1
+        fi
+        continue
+    fi
+    work=$(mktemp -d)
+    args=("$work")
+    [ -d "$corpus" ] && args+=("$corpus")
+    echo "=== $name (${mode}s) ==="
+    if ! "$f" -max_total_time="$mode" -timeout=25 -rss_limit_mb=2048 \
+        -artifact_prefix="$work/" -print_final_stats=1 "${args[@]}"; then
+        echo "*** $name FAILED; artifacts:" >&2
+        ls -l "$work" >&2
+        status=1
+    else
+        rm -rf "$work"
+    fi
+done
+exit $status
--- a/m4/ax_add_fortify_source.m4
+++ b/m4/ax_add_fortify_source.m4
@@ -0,0 +1,119 @@
+# ===========================================================================
+#  https://www.gnu.org/software/autoconf-archive/ax_add_fortify_source.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_ADD_FORTIFY_SOURCE
+#
+# DESCRIPTION
+#
+#   Check whether -D_FORTIFY_SOURCE=2 can be added to CPPFLAGS without macro
+#   redefinition warnings, other cpp warnings or linker. Some distributions
+#   (such as Ubuntu or Gentoo Linux) enable _FORTIFY_SOURCE globally in
+#   their compilers, leading to unnecessary warnings in the form of
+#
+#     <command-line>:0:0: error: "_FORTIFY_SOURCE" redefined [-Werror]
+#     <built-in>: note: this is the location of the previous definition
+#
+#   which is a problem if -Werror is enabled. This macro checks whether
+#   _FORTIFY_SOURCE is already defined, and if not, adds -D_FORTIFY_SOURCE=2
+#   to CPPFLAGS.
+#
+#   Newer mingw-w64 msys2 package comes with a bug in
+#   headers-git-7.0.0.5546.d200317d-1. It broke -D_FORTIFY_SOURCE support,
+#   and would need -lssp or -fstack-protector.  See
+#   https://github.com/msys2/MINGW-packages/issues/5803. Try to actually
+#   link it.
+#
+# LICENSE
+#
+#   Copyright (c) 2017 David Seifert <soap@gentoo.org>
+#   Copyright (c) 2019, 2023 Reini Urban <rurban@cpan.org>
+#
+#   Copying and distribution of this file, with or without modification, are
+#   permitted in any medium without royalty provided the copyright notice
+#   and this notice are preserved.  This file is offered as-is, without any
+#   warranty.
+
+#serial 10
+
+AC_DEFUN([AX_ADD_FORTIFY_SOURCE],[
+    ac_save_cflags=$CFLAGS
+    ac_cwerror_flag=yes
+    AX_CHECK_COMPILE_FLAG([-Werror],[CFLAGS="$CFLAGS -Werror"])
+    ax_add_fortify_3_failed=
+    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=3 to CPPFLAGS])
+    AC_LINK_IFELSE([
+        AC_LANG_PROGRAM([],
+            [[
+                #ifndef _FORTIFY_SOURCE
+                    return 0;
+                #else
+                    _FORTIFY_SOURCE_already_defined;
+                #endif
+            ]]
+        )],
+        AC_LINK_IFELSE([
+            AC_LANG_SOURCE([[
+                #define _FORTIFY_SOURCE 3
+                #include <string.h>
+                int main(void) {
+                    char *s = " ";
+                    strcpy(s, "x");
+                    return strlen(s)-1;
+                }
+              ]]
+            )],
+            [
+              AC_MSG_RESULT([yes])
+              CFLAGS=$ac_save_cflags
+              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=3"
+            ], [
+              AC_MSG_RESULT([no])
+              ax_add_fortify_3_failed=1
+            ],
+        ),
+        [
+          AC_MSG_RESULT([no])
+          ax_add_fortify_3_failed=1
+        ])
+    if test -n "$ax_add_fortify_3_failed"
+    then
+    AC_MSG_CHECKING([whether to add -D_FORTIFY_SOURCE=2 to CPPFLAGS])
+    AC_LINK_IFELSE([
+        AC_LANG_PROGRAM([],
+            [[
+                #ifndef _FORTIFY_SOURCE
+                    return 0;
+                #else
+                    _FORTIFY_SOURCE_already_defined;
+                #endif
+            ]]
+        )],
+        AC_LINK_IFELSE([
+            AC_LANG_SOURCE([[
+                #define _FORTIFY_SOURCE 2
+                #include <string.h>
+                int main(void) {
+                    char *s = " ";
+                    strcpy(s, "x");
+                    return strlen(s)-1;
+                }
+              ]]
+            )],
+            [
+              AC_MSG_RESULT([yes])
+              CFLAGS=$ac_save_cflags
+              CPPFLAGS="$CPPFLAGS -D_FORTIFY_SOURCE=2"
+            ], [
+              AC_MSG_RESULT([no])
+              CFLAGS=$ac_save_cflags
+            ],
+        ),
+        [
+          AC_MSG_RESULT([no])
+          CFLAGS=$ac_save_cflags
+        ])
+    fi
+])
--- a/man/httrack.1
+++ b/man/httrack.1
@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "27 June 2026" "httrack website copier"
+.TH httrack 1 "07 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -51,6 +51,7 @@ httrack \- offline browser : copy websites to a local directory
 [ \fB\-%T, \-\-utf8\-conversion\fR ]
 [ \fB\-bN, \-\-cookies[=N]\fR ]
 [ \fB\-%K, \-\-cookies\-file\fR ]
+[ \fB\-%Y, \-\-why\fR ]
 [ \fB\-u, \-\-check\-type[=N]\fR ]
 [ \fB\-j, \-\-parse\-java[=N]\fR ]
 [ \fB\-sN, \-\-robots[=N]\fR ]
@@ -218,6 +219,8 @@ links conversion to UTF\-8 (\-\-utf8\-conversion)
 accept cookies in cookies.txt (0=do not accept,* 1=accept) (\-\-cookies[=N])
 .IP \-%K
 load extra cookies from a Netscape cookies.txt (\-\-cookies\-file <param>)
+.IP \-%Y
+explain which filter rule accepts or rejects a URL, then exit (\-\-why <param>)
 .IP \-u
 check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always) (\-\-check\-type[=N])
 .IP \-j
--- a/src/htsalias.c
+++ b/src/htsalias.c
@@ -114,6 +114,8 @@ const char *hts_optalias[][4] = {
   "strip [host/pattern=]key1,key2,... from URLs"},
  {"cookies-file", "-%K", "param1",
   "load extra cookies from a Netscape cookies.txt"},
+  {"why", "-%Y", "param1",
+   "explain which filter rule accepts or rejects a URL, then exit"},
  {"pause", "-%G", "param1",
   "random pause of MIN[:MAX] seconds between files"},
  {"generate-errors", "-o", "single", ""},
@@ -283,7 +285,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
      char *position;
      int need_param = 1;

-      //int return_param=0;
      int pos;

      command[0] = param[0] = addcommand[0] = '\0';
@@ -361,7 +362,6 @@ int optalias_check(int argc, const char *const *argv, int n_arg,
              strlcatbuff(return_argv[0], "0", return_argv_size);
            else if (strcmp(param, "on") == 0) {
              // on is the default
-              // strcatbuff(return_argv[0],"1");
            } else
              strlcatbuff(return_argv[0], param, return_argv_size);
          }
--- a/src/htsback.c
+++ b/src/htsback.c
@@ -43,14 +43,12 @@ Please visit our Website: http://www.httrack.com

 #include "htsback.h"

-//#ifdef _WIN32
 #include "htsftp.h"
 #if HTS_USEZLIB
 #include "htszlib.h"
 #else
 #error HTS_USEZLIB not defined
 #endif
-//#endif

 #ifdef _WIN32
 #ifndef __cplusplus
@@ -597,7 +595,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
 #if HTS_USEZLIB
        if (back[p].r.compressed) {
          if (back[p].r.size > 0) {
-            //if ( (back[p].r.adr) && (back[p].r.size>0) ) {
            // stats
            back[p].compressed_size = back[p].r.size;
            // en mémoire -> passage sur disque
@@ -927,7 +924,6 @@ int back_letlive(httrackp * opt, cache_back * cache, struct_back * sback,
    /* clear everything but connection: switch, close, and reswitch */
    back_connxfr(src, &tmp);
    back_delete(opt, cache, sback, p);
-    //deletehttp(src);
    back_connxfr(&tmp, src);
    src->req.flush_garbage = 1; /* ignore CRLF garbage */
    return 1;
@@ -1437,7 +1433,6 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                      back[p].url_adr, back[p].url_fil, back[p].url_sav);
      }
      if (cache != NULL) {
-        //hts_log_print(opt, LOG_TRACE, "finalizing from back_delete");
        back_finalize(opt, cache, sback, p);
      }
    }
@@ -1578,7 +1573,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
    if (back[p].r.soc != INVALID_SOCKET) {      /* we never know */
      deletehttp(&back[p].r);
    }
-    //memset(&(back[p].r), 0, sizeof(htsblk)); 
    hts_init_htsblk(&back[p].r);
    back[p].r.location = back[p].location_buffer;

@@ -1586,7 +1580,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
    strcpybuff(back[p].url_adr, adr);
    strcpybuff(back[p].url_fil, fil);
    strcpybuff(back[p].url_sav, save);
-    //back[p].links_index = links_index;
    // copier referer si besoin
    strcpybuff(back[p].referer_adr, "");
    strcpybuff(back[p].referer_fil, "");
@@ -1825,8 +1818,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
                            back[p].url_adr, back[p].url_fil);
            }
            back[p].r.notmodified = 1;  // fichier non modifié
-            back[p].status = STATUS_READY;      // OK prêt
-            //file_notify(back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0, back[p].r.notmodified);        // not modified
+            back[p].status = STATUS_READY; // OK prêt
            back_set_finished(sback, p);

            // finalize transfer
@@ -1841,7 +1833,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
          } else {              // erreur
            // effacer r
            hts_init_htsblk(&back[p].r);
-            //memset(&(back[p].r), 0, sizeof(htsblk)); 
            back[p].r.location = back[p].location_buffer;
            // et continuer (chercher le fichier)
          }
@@ -2036,7 +2027,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
      // ouvrir liaison, envoyer requète
      // ne pas traiter ou recevoir l'en tête immédiatement
      hts_init_htsblk(&back[p].r);
-      // memset(&(back[p].r), 0, sizeof(htsblk));
      back[p].r.location = back[p].location_buffer;
      // fresh connect: address list not yet probed, start at the first
      sback->connect_fallback[p].addr_index = 0;
@@ -2087,7 +2077,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 #if HTS_USEOPENSSL
      else if (strfield(back[p].url_adr, "https://")) {     // let's rock
        back[p].r.ssl = 1;
-        // back[p].r.ssl_soc = NULL;
        back[p].r.ssl_con = NULL;
      }
 #endif
@@ -2153,9 +2142,9 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
        back[p].rateout = -1;   // pas de gestion (default)
      }

-      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où cela est
-      // rattrapable (exemple: 301,302 moved xxx -> refresh sur la page!)
-      //if ((back[p].statuscode!=HTTP_OK) || (soc<0)) { // ERREUR HTTP/autre
+      // Note: on charge les code-page erreurs (erreur 404, etc) dans le cas où
+      // cela est rattrapable (exemple: 301,302 moved xxx -> refresh sur la
+      // page!)

 #if CNXDEBUG
      printf("Xfopen ok, poll..\n");
@@ -2173,7 +2162,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
      if (soc == INVALID_SOCKET) {      // erreur socket
        back[p].status = STATUS_READY;  // FINI
        back_set_finished(sback, p);
-        //if (back[p].soc!=INVALID_SOCKET) deletehttp(back[p].soc);
        back[p].r.soc = INVALID_SOCKET;
      } else {
        if (!back[p].r.is_file)
@@ -2194,8 +2182,6 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char

    // note: si il y a erreur (404,etc) status=2 (terminé/échec) mais
    // le lien est considéré comme traité
-    //if (back[p].soc<0)  // erreur
-    //  return -1;

    return 0;
  } else {
@@ -2268,9 +2254,6 @@ void back_solve(httrackp * opt, lien_back * back) {
    } else {
      hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
    }
-    //if (hts_dnstest(opt, a, 1) == 2) { // non encore testé!..
-    //  hts_log_print(opt, LOG_DEBUG, "resolving in background: %s", a);
-    //}
  }
 }

@@ -2319,11 +2302,6 @@ void back_clean(httrackp * opt, cache_back * cache, struct_back * sback) {
        (void) back_flush_output(opt, cache, sback, i); // flush output buffers
        usercommand(opt, 0, NULL, back[i].url_sav, back[i].url_adr,
                    back[i].url_fil);
-        //if (back[i].links_index >= 0) {
-        //  assertf(back[i].links_index < opt->hash->max_lien);
-        //  opt->hash->liens[back[i].links_index]->pass2 = -1;
-        //  // *back[i].pass2_ptr=-1;  // Done!
-        //}
        /* MANDATORY if we don't want back_fill() to endlessly put the same file on download! */
        {
          int index = hash_read(opt->hash, back[i].url_sav, NULL, HASH_STRUCT_FILENAME );       // lecture type 0 (sav)
@@ -2507,9 +2485,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,

    busy_state = busy_recv = 0;

-#if 0
-    check_rate(stat_timestart, opt->maxrate);   // vérifier taux de transfert
-#endif
    // inscrire les sockets actuelles, et rechercher l'ID la plus élevée
    FD_ZERO(&fds);
    FD_ZERO(&fds_c);
@@ -2519,8 +2494,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
    nfds = INVALID_SOCKET;

    max_c = 1;
-    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-      // for(i=0;i<back_max;i++) {
+    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
      unsigned int i = (i_mod + mod_random) % (back_max);

      // en cas de gestion du connect préemptif
@@ -2557,8 +2531,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
        // poll pour la lecture sur les sockets
      if ((back[i].status > 0) && (back[i].status < 100)) {     // en réception http

-#if BDEBUG==1
-        //printf("....socket in progress: %d\n",back[i].r.soc);
+#if BDEBUG == 1
 #endif
        // non local et non ftp
        if (!back[i].r.is_file) {
@@ -2644,8 +2617,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
      busy_recv = 0;

    // recevoir les données arrivées
-    for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-      // for(i=0;i<back_max;i++) {
+    for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
      unsigned int i = (i_mod + mod_random) % (back_max);

      if (back[i].status > 0) {
@@ -2785,7 +2757,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
              back[i].rateout_time = back[i].ka_time_start;
            }
            // envoyer header
-            //if (strcmp(back[i].url_sav,BACK_ADD_TEST)!=0)    // vrai get
            HTS_STAT.stat_nrequests++;
            if (!back[i].head_request)
              http_sendhead(opt, opt->cookie, 0, back[i].send_too,
@@ -2852,7 +2823,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 #if HTS_XGETHOST
      else if (back[i].status == STATUS_WAIT_DNS) {     // attendre gethostbyname
 #if DEBUGDNS
-        //printf("status 101 for %s\n",back[i].url_adr);
 #endif

        if (!gestion_timeout)
@@ -3125,8 +3095,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
          // Si réception chunk, tester si on est pas à la fin!
          if (back[i].status == 1) {
            if (back[i].is_chunk) {     // attendre prochain chunk
-              if (back[i].r.size == back[i].r.totalsize) {      // fin chunk!
-                //printf("chunk end at %d\n",back[i].r.size);
+              if (back[i].r.size == back[i].r.totalsize) { // fin chunk!
                back[i].status = STATUS_CHUNK_CR;       /* fetch ending CRLF */
                if (back[i].chunk_adr != NULL) {
                  freet(back[i].chunk_adr);
@@ -3189,11 +3158,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
              back_finalize(opt, cache, sback, i);
            }

-            if (back[i].r.totalsize >= 0) {     // tester totalsize
-              //if ((back[i].r.totalsize>=0) && (back[i].status==STATUS_WAIT_HEADERS)) {    // tester totalsize
+            if (back[i].r.totalsize >= 0) { // tester totalsize
              if (back[i].r.totalsize != back[i].r.size) {      // pas la même!
                if (!opt->tolerant) {
-                  //#if HTS_CL_IS_FATAL
                  deleteaddr(&back[i].r);
                  if (back[i].r.size < back[i].r.totalsize)
                    back[i].r.statuscode = STATUSCODE_CONNERROR;        // recatch
@@ -3202,14 +3169,13 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                          " expected)", (LLint) back[i].r.size,
                          (LLint) back[i].r.totalsize);
                } else {
-                  //#else
                  // Un warning suffira..
                  hts_log_print(opt, LOG_WARNING,
                                "Incorrect length (" LLintP "!=" LLintP
-                                " expected) for %s%s", (LLint) back[i].r.size,
+                                " expected) for %s%s",
+                                (LLint) back[i].r.size,
                                (LLint) back[i].r.totalsize, back[i].url_adr,
                                back[i].url_fil);
-                  //#endif
                }
              }
            }
@@ -3336,10 +3302,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                         back[i].r.size);
 #endif
                  /* End */
-                  //if (back[i].status==STATUS_CHUNK_CR) {
-                  back[i].status = STATUS_READY;        // fin  
+                  back[i].status = STATUS_READY; // fin
                  back_set_finished(sback, i);
-                  //}

                  // finalize transfer if not temporary
                  if (!IS_DELAYED_EXT(back[i].url_sav)) {
@@ -3806,7 +3770,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                    back[i].url_adr, back[i].url_fil);

                      // finalize
-                      //file_notify(back[i].url_adr, back[i].url_fil, back[i].url_sav, 0, 0, back[i].r.notmodified);        // not modified
                      if (back[i].r.statuscode > 0) {
                        hts_log_print(opt, LOG_TRACE, "finalizing after cache load");
                        back_finalize(opt, cache, sback, i);
@@ -3816,37 +3779,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                             back[i].url_adr, back[i].url_fil);
 #endif

-                      //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                    } else {    // erreur
                      back[i].status = STATUS_READY;    // terminé
                      back_set_finished(sback, i);
-                      //printf("erreur cache\n");
-
                    }

-/********** NO - must complete the body! ********** */
-#if 0
-                  } else if (HTTP_IS_REDIRECT(back[i].r.statuscode)
-                             || (back[i].r.statuscode == 412)
-                             || (back[i].r.statuscode == 416)
-                    ) {         // Ne pas prendre le html, erreurs connues et gérées
-#if HTS_DEBUG_CLOSESOCK
-                    DEBUG_W
-                      ("back_wait(301,302,303,307,412,416..): deletehttp\n");
-#endif
-                    // Couper connexion
-                    /*KA deletehttp(&back[i].r); back[i].r.soc=INVALID_SOCKET; */
-                    back_maydeletehttp(opt, cache, sback, i);
-
-                    back[i].status = STATUS_READY;      // terminé
-                    back_set_finished(sback, i);
-                    // finalize
-                    if (back[i].r.statuscode > 0) {
-                      hts_log_print(opt, LOG_TRACE, "finalizing redirect & 4xx");
-                      back_finalize(opt, cache, sback, i);
-                    }
-#endif
-/********** **************************** ********** */
                  }
                  // MIME type excluded by a -mime: filter: abort, don't fetch
                  // the body (#58)
@@ -4020,7 +3957,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                          deletehttp(&back[i].r);
                        }
                        back[i].r.soc = INVALID_SOCKET;
-                        //back[i].r.statuscode=206;  ????????
                        back[i].r.statuscode = STATUSCODE_NON_FATAL;
                        if (strnotempty(back[i].r.msg))
                          strcpybuff(back[i].r.msg,
@@ -4047,8 +3983,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                            hts_log_print(opt, LOG_TRACE, "finalizing empty");
                            back_finalize(opt, cache, sback, i);
                          }
-                        } else if (!back[i].r.is_chunk) {       // pas de chunk
-                          //if (back[i].r.http11!=2) {    // pas de chunk
+                        } else if (!back[i].r.is_chunk) { // pas de chunk
                          back[i].is_chunk = 0;
                          back[i].status = 1;   // start body
                        } else {
@@ -4109,14 +4044,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
        if (back[i].status < 0) {
          if (!back[i].testmode) {      // pas en test
            UNLINK(back[i].url_sav);    // éliminer fichier (endommagé)
-            //printf("&& %s\n",back[i].url_sav);
          }
        }
 #endif

        /* funny log for commandline users */
-        //if (!opt->quiet) {  
-        // petite animation
        if (opt->verbosedisplay == HTS_VERBOSE_SIMPLE) {
          if (back[i].status == STATUS_READY) {
            if (back[i].r.statuscode == HTTP_OK)
@@ -4129,18 +4061,16 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
            fflush(stdout);
          }
        }
-        //}

      }                         // status>0
-    }                           // for
+    } // for

    // vérifier timeouts
    if (gestion_timeout) {
      TStamp act;

      act = time_local();       // temps en secondes
-      for(i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
-        // for(i=0;i<back_max;i++) {
+      for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
        unsigned int i = (i_mod + mod_random) % (back_max);

        if (back[i].status > 0) {       // réception/connexion/..
@@ -4168,7 +4098,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                continue;
              }
            }
-            //printf("time check %d\n",((int) (act-back[i].timeout_refresh))-back[i].timeout);
            if (((int) (act - back[i].timeout_refresh)) >= back[i].timeout) {
              hts_log_print(opt, LOG_DEBUG, "connection timed out for %s%s", back[i].url_adr,
                back[i].url_fil);
--- a/src/htsbasenet.h
+++ b/src/htsbasenet.h
@@ -55,8 +55,6 @@ Please visit our Website: http://www.httrack.com
 // inline function in Wspiapi.h.
 #include <ws2tcpip.h>
 #include <Wspiapi.h>
-// #include <winsock2.h>
-// #include <tpipv6.h>

 #endif

--- a/src/htsbauth.c
+++ b/src/htsbauth.c
@@ -107,7 +107,6 @@ int cookie_add(t_cookie * cookie, const char *cook_name, const char *cook_value,
 #if DEBUG_COOK
  printf("add_new cookie: name=\"%s\" value=\"%s\" domain=\"%s\" path=\"%s\"\n",
         cook_name, cook_value, domain, path);
-  //printf(">>>cook: %s<<<\n",cookie->data);
 #endif
  return 0;
 }
@@ -208,8 +207,6 @@ int cookie_load(t_cookie * cookie, const char *fpath, const char *name) {
  char catbuff[CATBUFF_SIZE];
  char buffer[8192];

-  //  cookie->data[0]='\0';
-
  // Fusionner d'abord les éventuels cookies IE
 #ifdef _WIN32
  {
--- a/src/htscache.c
+++ b/src/htscache.c
@@ -265,8 +265,6 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
  char BIGSTK headers[8192];
  int headersSize = 0;

-  //int entryBodySize = 0;
-  //int entryFilenameSize = 0;
  zip_fileinfo fi;
  const char *url_save_suffix = url_save;
  int zErr;
@@ -359,8 +357,6 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
  ZIP_FIELD_STRING(headers, headersSize, "X-Fil", url_fil);     // Original URI filename
  ZIP_FIELD_STRING(headers, headersSize, "X-Save", url_save_suffix);    // Original save filename

-  //entryFilenameSize = (int) ( strlen(url_adr) + strlen(url_fil));
-
  /* Filename */
  if (!link_has_authority(url_adr)) {
    strcpybuff(filename, "http://");
@@ -725,7 +721,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
        char BIGSTK headerBuff[8192 + 2];
        int readSizeHeader;

-        //int totalHeader = 0;
        int dataincache = 0;

        /* For BIG comments */
@@ -776,13 +771,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                                   HTS_URLMAXSIZE * 2);
              ZIP_READFIELD_STRING(line, value, "Content-Disposition", r.cdispo,
                                   sizeof(r.cdispo));
-              //ZIP_READFIELD_STRING(line, value, "X-Addr", ..);            // Original address
-              //ZIP_READFIELD_STRING(line, value, "X-Fil", ..);            // Original URI filename
              ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_,
                                   sizeof(previous_save_));
            }
-          } while(offset < readSizeHeader && !lineEof);
-          //totalHeader = offset;
+          } while (offset < readSizeHeader && !lineEof);

          /* Previous entry */
          if (previous_save_[0] != '\0') {
@@ -964,7 +956,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                  r.statuscode = STATUSCODE_INVALID;
                  strcpybuff(r.msg,
                             "Cache Write Error : Unable to Create File");
-                  //printf("%s\n",save);
                }
              }

@@ -1042,7 +1033,6 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                    strcpybuff(r.msg, "Cache Read Error : Read Data");
                  } else
                    *(r.adr + r.size) = '\0';
-                  //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                } else {        // erreur
                  r.statuscode = STATUSCODE_INVALID;
                  strcpybuff(r.msg, "Cache Memory Error");
@@ -1221,17 +1211,14 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
        // sécurité
        r.adr = NULL;
        r.out = NULL;
-        ////r.location=NULL;  non, fixée lors des 301 ou 302
        r.fp = NULL;

        if ((r.statuscode >= 0) && (r.statuscode <= 999)
            && (r.notmodified >= 0) && (r.notmodified <= 9)) {  // petite vérif intégrité
-          if ((save) && (!header_only)) {       /* ne pas lire uniquement header */
-            //int to_file=0;
+          if ((save) && (!header_only)) { /* ne pas lire uniquement header */

            r.adr = NULL;
            r.soc = INVALID_SOCKET;
-            // // r.location=NULL;

 #if HTS_DIRECTDISK
            // Court-circuit:
@@ -1240,12 +1227,11 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
              int ok = 0;

              r.is_write = 1;   // écrire
-              if (fexist_utf8(fconv(catbuff, sizeof(catbuff), save))) {  // un fichier existe déja
-                //if (fsize_utf8(fconv(save))==r.size) {  // même taille -- NON tant pis (taille mal declaree)
+              if (fexist_utf8(fconv(catbuff, sizeof(catbuff),
+                                    save))) { // un fichier existe déja
                ok = 1;         // plus rien à faire
                filenote(&opt->state.strc, save, NULL); // noter comme connu
                file_notify(opt, adr, fil, save, 0, 0, 0);
-                //}
              }

              if ((pos < 0) && (!ok)) { // Pas de donnée en cache et fichier introuvable : erreur!
@@ -1296,7 +1282,6 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
                  r.statuscode = STATUSCODE_INVALID;
                  strcpybuff(r.msg,
                             "Cache Write Error : Unable to Create File");
-                  //printf("%s\n",save);
                }
              }

@@ -1344,14 +1329,13 @@ static htsblk cache_readex_old(httrackp * opt, cache_back * cache,
                    strcpybuff(r.msg, "Cache Read Error : Read Data");
                  } else
                    *(r.adr + r.size) = '\0';
-                  //printf(">%s status %d\n",back[p].r.contenttype,back[p].r.statuscode);
                } else {        // erreur
                  r.statuscode = STATUSCODE_INVALID;
                  strcpybuff(r.msg, "Cache Memory Error");
                }
              }
            }
-          }                     // si save==null, ne rien charger (juste en tête)
+          } // si save==null, ne rien charger (juste en tête)
        } else {
 #if DEBUGCA
          printf("Cache Read Error : Bad Data");
@@ -1574,25 +1558,10 @@ void cache_init(cache_back * cache, httrackp * opt) {
      mkdir(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache"),
            HTS_PROTECT_FOLDER);
 #endif
-      if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.zip")))) {       // il existe déja un cache précédent.. renommer
-        /* Previous cache from the previous cache version */
-#if 0
-        /* No.. reuse with old httrack releases! */
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.dat")))
-          remove(fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.dat"));
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.ndx")))
-          remove(fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.ndx"));
-#endif
+      if ((fexist(fconcat(
+              OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+              StringBuff(opt->path_log),
+              "hts-cache/new.zip")))) { // a previous cache exists.. rename it
        /* Previous cache version */
        if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) {     // il existe déja un cache précédent.. renommer
          rename(fconcat
@@ -1633,7 +1602,13 @@ void cache_init(cache_back * cache, httrackp * opt) {
        } else {
          hts_log_print(opt, LOG_DEBUG, "Cache: successfully renamed");
        }
-      } else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.dat"))) && (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/new.ndx")))) {        // il existe déja un cache précédent.. renommer
+      } else if ((fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                 StringBuff(opt->path_log),
+                                 "hts-cache/new.dat"))) &&
+                 (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                 StringBuff(opt->path_log),
+                                 "hts-cache/new.ndx")))) { // a previous cache
+                                                           // exists.. rename it
 #if DEBUGCA
        printf("work with former cache\n");
 #endif
@@ -1662,7 +1637,7 @@ void cache_init(cache_back * cache, httrackp * opt) {
                "hts-cache/new.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
                                              StringBuff(opt->path_log),
                                              "hts-cache/old.ndx"));
-      } else {                  // un des deux (ou les deux) fichiers cache absents: effacer l'autre éventuel
+      } else { // one or both cache files missing: remove the remaining one
 #if DEBUGCA
        printf("new cache\n");
 #endif
@@ -2031,12 +2006,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                   "hts-cache/new.lst"), "wb");
          strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
          opt->state.strc.lst = cache->lst;
-          //{
-          //filecreate_params tmp;
-          //strcpybuff(tmp.path,StringBuff(opt->path_html));    // chemin
-          //tmp.lst=cache->lst;                 // fichier lst
-          //filenote("",&tmp);        // initialiser filecreate
-          //}

          // supprimer old.txt
          if (fexist
@@ -2125,12 +2094,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                   "hts-cache/new.lst"), "wb");
          strcpybuff(opt->state.strc.path, StringBuff(opt->path_html));
          opt->state.strc.lst = cache->lst;
-          //{
-          //  filecreate_params tmp;
-          //  strcpybuff(tmp.path,StringBuff(opt->path_html));    // chemin
-          //  tmp.lst=cache->lst;                 // fichier lst
-          //  filenote("",&tmp);        // initialiser filecreate
-          //}

          // supprimer old.txt
          if (fexist
@@ -2162,8 +2125,6 @@ void cache_init(cache_back * cache, httrackp * opt) {
                    "statuscode\tstatus ('servermsg')\tMIME\tEtag|Date\tURL\tlocalfile\t(from URL)"
                    LF);
          }
-          // test
-          // cache_writedata(cache->ndx,cache->dat,"//[TEST]//","test1","TEST PIPO",9);
        }                       // cache->ndx!=NULL
      }                         //cache->zipOutput != NULL

--- a/src/htscatchurl.c
+++ b/src/htscatchurl.c
@@ -197,7 +197,6 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
            htsblk blkretour;

            hts_init_htsblk(&blkretour);
-            //memset(&blkretour, 0, sizeof(htsblk));    // effacer
            blkretour.location = loc;   // si non nul, contiendra l'adresse véritable en cas de moved xx
            // Lire en têtes restants
            sprintf(data, "%s %s %s\r\n", method, af.fil, protocol);
@@ -207,8 +206,8 @@ HTSEXT_API hts_boolean catch_url(T_SOC soc, char *url, char *method,
              strlcatbuff(data, line, CATCH_URL_DATA_SIZE);
              strlcatbuff(data, "\r\n", CATCH_URL_DATA_SIZE);
            }
-            // CR/LF final de l'en tête inutile car déja placé via la ligne vide juste au dessus
-            //strcatbuff(data,"\r\n");
+            // CR/LF final de l'en tête inutile car déja placé via la ligne vide
+            // juste au dessus
            if (blkretour.totalsize > 0) {
              int len = (int) min(blkretour.totalsize, 32000);
              int pos = (int) strlen(data);
--- a/src/htscharset.c
+++ b/src/htscharset.c
@@ -841,71 +841,6 @@ static unsigned int nlz8(unsigned char x) {
    }                                           \
  } while(0)

-/* Sample. */
-#if 0
-
-int main(int argc, char **argv) {
-  int i;
-  int hex = 0;
-  
-#define READ_INT(DEST)                                  \
-  ( ( !hex && sscanf(argv[i], "%d", &(DEST)) == 1)      \
-    || (hex && sscanf(argv[i], "%x", &(DEST)) == 1 ) )
-  
-  for(i = 1 ; i < argc ; i++) {
-    unsigned int uc, from, to;
-    
-    if (strcmp(argv[i], "--hex") == 0) {
-      hex = 1;
-    }
-    else if (strcmp(argv[i], "--decimal") == 0) {
-      hex = 0;
-    }
-    else if (strcmp(argv[i], "--decode") == 0) {
-#define RD fgetc_unlocked(stdin)
-#define WR(C) do {                              \
-        if (C != -1) {                          \
-          printf("%04x\n", C);                  \
-        } else if (!feof(stdin)) {              \
-          fprintf(stderr, "read error\n");      \
-          exit(EXIT_FAILURE);                   \
-        }                                       \
-      } while(0)
-      while(!feof(stdin)) {
-        READ_UNICODE(RD, WR);
-      }
-#undef RD
-#undef WR
-    }
-    else if (strcmp(argv[i], "-range") == 0
-             && i + 2 < argc
-             && (++i, 1)
-             && READ_INT(from)
-             && (++i, 1)
-             && READ_INT(to)
-             ) {
-      unsigned int i;
-      for(i = from ; i < to ; i++) {
-#define EM(C) fputc_unlocked(C, stdout)
-        EMIT_UNICODE(i, EM);
-#undef EM
-      }
-    }
-    else if (READ_INT(uc)) {
-#define EM(C) fputc_unlocked(C, stdout)
-      EMIT_UNICODE(uc, EM);
-#undef EM
-    }
-    else {
-      return EXIT_FAILURE;
-    }
-  }
-  
-  return EXIT_SUCCESS;
-}
-
-#endif
-
 /* IDNA helpers. */
 #undef ADD_BYTE
 #undef INCREASE_CAPA
@@ -975,19 +910,19 @@ char *hts_convertStringUTF8ToIDNA(const char *s, size_t size) {
                /* Reader: can read bytes up to j */
 #define RD ( utfSeq < j ? segData[utfSeq++] : -1 )

-                /* Writer: upon error, return FFFD (replacement character) */
-#define WR(C) do { \
-  if ((C) != -1) { \
-    /* copy character */ \
-    assertf(segOutputSize < segSize); \
-    segInt[segOutputSize++] = (C); \
-  } \
-  /* In case of error, abort. */ \
-  else { \
-    FREE_BUFFER(); \
-    return NULL; \
-  } \
-} while(0)
+                /* Writer: a malformed sequence abandons the encode (NULL) */
+#define WR(C)                                                                  \
+  do {                                                                         \
+    if ((C) != -1) {                                                           \
+      /* copy character */                                                     \
+      assertf(segOutputSize < segSize);                                        \
+      segInt[segOutputSize++] = (C);                                           \
+    } else {                                                                   \
+      freet(segInt);                                                           \
+      FREE_BUFFER();                                                           \
+      return NULL;                                                             \
+    }                                                                          \
+  } while (0)

                /* Read/Write Unicode character. */
                READ_UNICODE(RD, WR);
@@ -1209,7 +1144,7 @@ int hts_isStringUTF8(const char *s, size_t size) {
    /* Reader: can read bytes up to j */
 #define RD ( i < size ? data[i++] : -1 )

-    /* Writer: upon error, return FFFD (replacement character) */
+    /* Writer: a malformed sequence means the string is not UTF-8 (return 0) */
 #define WR(C) if ((C) == -1) { return 0; }

    /* Read Unicode character. */
--- a/src/htsconfig.h
+++ b/src/htsconfig.h
@@ -54,10 +54,6 @@ Please visit our Website: http://www.httrack.com
 // extension par défaut pour fichiers n'en ayant pas
 #define DEFAULT_EXT ".html"
 #define DEFAULT_EXT_SHORT ".htm"
-// #define DEFAULT_BIN_EXT   ".bin"
-// #define DEFAULT_BIN_EXT_SHORT ".bin"
-// #define DEFAULT_EXT       ".txt"
-// #define DEFAULT_EXT_SHORT ".txt"

 // éviter les /nul, /con..
 #define HTS_OVERRIDE_DOS_FOLDERS 1
@@ -80,10 +76,6 @@ Please visit our Website: http://www.httrack.com
 // always direct-to-disk (0/1)
 #define HTS_DIRECTDISK_ALWAYS 1

-// gérer une table de hachage?
-// REMOVED
-// #define HTS_HASH 1
-
 // fast cache (build hash table)
 #define HTS_FAST_CACHE 1

@@ -97,27 +89,10 @@ Please visit our Website: http://www.httrack.com
 // always transform a '//' into a sigle '/'
 #define HTS_STRIP_DOUBLE_SLASH 0

-// case-sensitive pour les dossiers et fichiers (0/1)
-// [normalement 1, mais pose des problèmes (url malformée par exemple) et n'est
-// pas très utile..
-// ..et pas bcp respecté]
-// REMOVED
-// #define HTS_CASSE 0
-
-// Un fichier ayant une taille différente du content-length doit il être annulé?
-// SEE opt.tolerant and opt.http10
-// #define HTS_CL_IS_FATAL 0
-
 // une erreur supprime le fichier sur disque
 // (non fixé pour cause de retry)
 #define HTS_REMOVE_BAD_FILES 0

-// en cas de Range: xx- donnant un Content-length: xx
-// alors skipper le fichier, considéré comme transmis
-// #define HTS_SKIP_FULL_RANGE 1
-
-// nombre max de filtres que l'utilisateur peut fixer
-// #define HTS_FILTERSMAX 10000
 #define HTS_FILTERSINC 1000

 // connect non bloquant? (poll sur write)
--- a/src/htscore.c
+++ b/src/htscore.c
@@ -119,46 +119,93 @@ hts_log_print(opt, LOG_INFO, "engine: end"); \
 RUN_CALLBACK0(opt, end); \
 }

-#define XH_extuninit do { \
-  HTMLCHECK_UNINIT \
-  hts_record_free(opt); \
-  if (filters && filters[0]) { \
-  freet(filters[0]); filters[0]=NULL; \
-  } \
-  if (filters) { \
-  freet(filters); filters=NULL; \
-  } \
-  back_delete_all(opt,&cache,sback); \
-  back_free(&sback); \
-  checkrobots_free(&robots);\
-  if (cache.use) { freet(cache.use); cache.use=NULL; } \
-  if (cache.dat) { fclose(cache.dat); cache.dat=NULL; }  \
-  if (cache.ndx) { fclose(cache.ndx); cache.ndx=NULL; } \
-  if (cache.zipOutput) { \
-    zipClose(cache.zipOutput, "Created by HTTrack Website Copier/"HTTRACK_VERSION); \
-    cache.zipOutput = NULL; \
-  } \
-  if (cache.zipInput) { \
-    unzClose(cache.zipInput); \
-    cache.zipInput = NULL; \
-  } \
-  if (cache.olddat) { fclose(cache.olddat); cache.olddat=NULL; } \
-  if (cache.lst) { fclose(cache.lst); cache.lst=opt->state.strc.lst=NULL; } \
-  if (cache.txt) { fclose(cache.txt); cache.txt=NULL; } \
-  if (opt->log != NULL) fflush(opt->log); \
-  if (makestat_fp) { fclose(makestat_fp); makestat_fp=NULL; } \
-  if (maketrack_fp){ fclose(maketrack_fp); maketrack_fp=NULL; } \
-  if (opt->accept_cookie) cookie_save(opt->cookie,fconcat(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),StringBuff(opt->path_log),"cookies.txt")); \
-  if (makeindex_fp) { fclose(makeindex_fp); makeindex_fp=NULL; } \
-  if (cache_hashtable) { coucal_delete(&cache_hashtable); } \
-  if (cache_tests)     { coucal_delete(&cache_tests); } \
-  if (template_header) { freet(template_header); template_header=NULL; } \
-  if (template_body)   { freet(template_body); template_body=NULL; } \
-  if (template_footer) { freet(template_footer); template_footer=NULL; } \
-  hash_free(&hash); \
-  clearCallbacks(&opt->state.callbacks); \
-  /*structcheck_init(-1);*/ \
-} while(0)
+#define XH_extuninit                                                           \
+  do {                                                                         \
+    HTMLCHECK_UNINIT                                                           \
+    hts_record_free(opt);                                                      \
+    if (filters && filters[0]) {                                               \
+      freet(filters[0]);                                                       \
+      filters[0] = NULL;                                                       \
+    }                                                                          \
+    if (filters) {                                                             \
+      freet(filters);                                                          \
+      filters = NULL;                                                          \
+    }                                                                          \
+    back_delete_all(opt, &cache, sback);                                       \
+    back_free(&sback);                                                         \
+    checkrobots_free(&robots);                                                 \
+    if (cache.use) {                                                           \
+      freet(cache.use);                                                        \
+      cache.use = NULL;                                                        \
+    }                                                                          \
+    if (cache.dat) {                                                           \
+      fclose(cache.dat);                                                       \
+      cache.dat = NULL;                                                        \
+    }                                                                          \
+    if (cache.ndx) {                                                           \
+      fclose(cache.ndx);                                                       \
+      cache.ndx = NULL;                                                        \
+    }                                                                          \
+    if (cache.zipOutput) {                                                     \
+      zipClose(cache.zipOutput,                                                \
+               "Created by HTTrack Website Copier/" HTTRACK_VERSION);          \
+      cache.zipOutput = NULL;                                                  \
+    }                                                                          \
+    if (cache.zipInput) {                                                      \
+      unzClose(cache.zipInput);                                                \
+      cache.zipInput = NULL;                                                   \
+    }                                                                          \
+    if (cache.olddat) {                                                        \
+      fclose(cache.olddat);                                                    \
+      cache.olddat = NULL;                                                     \
+    }                                                                          \
+    if (cache.lst) {                                                           \
+      fclose(cache.lst);                                                       \
+      cache.lst = opt->state.strc.lst = NULL;                                  \
+    }                                                                          \
+    if (cache.txt) {                                                           \
+      fclose(cache.txt);                                                       \
+      cache.txt = NULL;                                                        \
+    }                                                                          \
+    if (opt->log != NULL)                                                      \
+      fflush(opt->log);                                                        \
+    if (makestat_fp) {                                                         \
+      fclose(makestat_fp);                                                     \
+      makestat_fp = NULL;                                                      \
+    }                                                                          \
+    if (maketrack_fp) {                                                        \
+      fclose(maketrack_fp);                                                    \
+      maketrack_fp = NULL;                                                     \
+    }                                                                          \
+    if (opt->accept_cookie)                                                    \
+      cookie_save(opt->cookie,                                                 \
+                  fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),           \
+                          StringBuff(opt->path_log), "cookies.txt"));          \
+    if (makeindex_fp) {                                                        \
+      fclose(makeindex_fp);                                                    \
+      makeindex_fp = NULL;                                                     \
+    }                                                                          \
+    if (cache_hashtable) {                                                     \
+      coucal_delete(&cache_hashtable);                                         \
+    }                                                                          \
+    if (cache_tests) {                                                         \
+      coucal_delete(&cache_tests);                                             \
+    }                                                                          \
+    if (template_header) {                                                     \
+      freet(template_header);                                                  \
+      template_header = NULL;                                                  \
+    }                                                                          \
+    if (template_body) {                                                       \
+      freet(template_body);                                                    \
+      template_body = NULL;                                                    \
+    }                                                                          \
+    if (template_footer) {                                                     \
+      freet(template_footer);                                                  \
+      template_footer = NULL;                                                  \
+    }                                                                          \
+    hash_free(&hash);                                                          \
+    clearCallbacks(&opt->state.callbacks);                                     \
+  } while (0)
 #define XH_uninit do { XH_extuninit; if (r.adr) { freet(r.adr); r.adr=NULL; } } while(0)

 struct lien_buffers {
@@ -523,7 +570,6 @@ int httpmirror(char *url1, httrackp * opt) {
  hash_struct *const hashptr = &hash;
  t_cookie BIGSTK cookie;       // gestion des cookies

-  //char* tab_alloc=NULL;
  int ptr;                      // pointeur actuel sur les liens

  //
@@ -534,15 +580,12 @@ int httpmirror(char *url1, httrackp * opt) {
  // pour les stats, nombre de fichiers & octets écrits
  LLint stat_fragment = 0;      // pour la fragmentation

-  //TStamp istat_timestart;   // départ pour calcul instantanné
-  //
  TStamp last_info_shell = 0;
  int info_shell = 0;

  // filtres
  char **filters = NULL;

-  //int filter_max=0;
  int filptr = 0;

  //
@@ -582,7 +625,6 @@ int httpmirror(char *url1, httrackp * opt) {
  // noter heure actuelle de départ en secondes
  memset(&HTS_STAT, 0, sizeof(HTS_STAT));
  HTS_STAT.stat_timestart = time_local();
-  //istat_timestart=stat_timestart;
  HTS_STAT.istat_timestart[0] = HTS_STAT.istat_timestart[1] = mtime_local();
  /* reset stats */
  HTS_STAT.HTS_TOTAL_RECV = 0;
@@ -616,9 +658,6 @@ int httpmirror(char *url1, httrackp * opt) {
  // initialiser usercommand
  usercommand(opt, opt->sys_com_exec, StringBuff(opt->sys_com), "", "", "");

-  // initialiser structcheck
-  // structcheck_init(1);
-
  // initialiser verif_backblue
  verif_backblue(opt, NULL);
  verif_external(opt, 0, 0);
@@ -638,9 +677,6 @@ int httpmirror(char *url1, httrackp * opt) {
                (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_bin),
                 "templates/index-footer.html"), HTS_INDEX_FOOTER);

-  // initialiser mimedefs
-  //get_userhttptype(opt,1,StringBuff(opt->mimedefs),NULL);
-
  // Initialiser indexation
  if (opt->kindex)
    index_init(StringBuff(opt->path_html));
@@ -684,7 +720,6 @@ int httpmirror(char *url1, httrackp * opt) {
  opt->filters.filters = &filters;
  //
  opt->filters.filptr = &filptr;
-  //opt->filters.filter_max=&filter_max;

  // hash table
  opt->hash = &hash;
@@ -731,8 +766,7 @@ int httpmirror(char *url1, httrackp * opt) {
      else if (*a == '-')
        joker = 1;

-      if (joker) {              // joker ou filters
-        //char* p;
+      if (joker) { // joker ou filters
        char BIGSTK tempo[HTS_URLMAXSIZE * 2];
        int type;
        int plus = 0;
@@ -787,12 +821,11 @@ int httpmirror(char *url1, httrackp * opt) {
              XH_extuninit;
              return 0;
            }
-            //opt->filters.filters=filters;
          }

        }

-      } else {                  // adresse normale
+      } else { // adresse normale
        char BIGSTK url[HTS_URLMAXSIZE * 2];

        // prochaine adresse
@@ -813,6 +846,52 @@ int httpmirror(char *url1, httrackp * opt) {
      }
    }                           // while

+    /* --why: print which filter rule decides for this URL, then stop */
+    if (StringNotEmpty(opt->why_url)) {
+      char BIGSTK url[HTS_URLMAXSIZE * 2];
+      lien_adrfil af;
+
+      if (strstr(StringBuff(opt->why_url), ":/") == NULL)
+        snprintf(url, sizeof(url), "http://%s", StringBuff(opt->why_url));
+      else
+        strcpybuff(url, StringBuff(opt->why_url));
+      if (ident_url_absolute(url, &af) < 0) {
+        printf("--why: unable to parse URL %s" LF, StringBuff(opt->why_url));
+      } else {
+        char BIGSTK l[HTS_URLMAXSIZE * 2], lfull[HTS_URLMAXSIZE * 2];
+        int jok, jokDepth = 0;
+
+        /* the two forms the wizard tests */
+        strcpybuff(l, jump_identification_const(af.adr));
+        if (*af.fil != '/')
+          strcatbuff(l, "/");
+        strcatbuff(l, af.fil);
+        if (!link_has_authority(af.adr))
+          strcpybuff(lfull, "http://");
+        else
+          lfull[0] = '\0';
+        strcatbuff(lfull, af.adr);
+        if (*af.fil != '/')
+          strcatbuff(lfull, "/");
+        strcatbuff(lfull, af.fil);
+        jok = fa_strjoker_dual(/*url */ 0, filters, filptr, lfull, l, NULL,
+                               NULL, &jokDepth);
+        if (jok > 0)
+          printf("%s: accepted by rule #%d (%s)" LF, url, jokDepth + 1,
+                 filters[jokDepth]);
+        else if (jok < 0)
+          printf("%s: rejected by rule #%d (%s)" LF, url, jokDepth + 1,
+                 filters[jokDepth]);
+        else
+          printf("%s: no filter rule matches; the wizard decides "
+                 "(same-site links are followed by default)" LF,
+                 url);
+      }
+      freet(primary);
+      XH_extuninit;
+      return 1;
+    }
+
    /* load URL file list */
    /* OPTIMIZED for fast load */
    if (StringNotEmpty(opt->filelist)) {
@@ -867,7 +946,6 @@ int httpmirror(char *url1, httrackp * opt) {
            htsbuff_cat(&primarybuff, "\n");
          }
        }
-        // fclose(fp);
        hts_log_print(opt, LOG_NOTICE, "%d links added from %s", n,
                      StringBuff(opt->filelist));

@@ -922,7 +1000,6 @@ int httpmirror(char *url1, httrackp * opt) {
 #endif

  // backing
-  //soc_max=opt->maxsoc;
  if (opt->maxsoc > 0) {
 #if BDEBUG==2
    _CLRSCR;
@@ -1236,15 +1313,10 @@ int httpmirror(char *url1, httrackp * opt) {
      // tester r.adr
      if (!error) {
        // erreur, pas de fichier chargé:
-        if ((!r.adr) && (r.is_write == 0)
-            && (r.statuscode != 301)
-            && (r.statuscode != 302)
-            && (r.statuscode != 303)
-            && (r.statuscode != 307)
-            && (r.statuscode != 412)
-            && (r.statuscode != 416)
-          ) {
-          // error=1;
+        if ((!r.adr) && (r.is_write == 0) && (r.statuscode != 301) &&
+            (r.statuscode != 302) && (r.statuscode != 303) &&
+            (r.statuscode != 307) && (r.statuscode != 412) &&
+            (r.statuscode != 416)) {

          // peut être que le fichier était trop gros?
          if ((istoobig
@@ -1326,30 +1398,8 @@ int httpmirror(char *url1, httrackp * opt) {
                                         ||may_be_hypertext_mime(opt, r.contenttype, urlfil()))   /* Is real media, .. */
          ) {

-          /* Convert charset to UTF-8 - NOT! (what about links ? remote server side will have troubles with converted names) */
-          //if (r.adr != NULL && r.size != 0 && opt->convert_utf8) {
-          //  char *charset;
-          //  char *pos;
-          //  if (r.charset[0] != '\0') {
-          //    charset = strdup(r.charset);
-          //  } else {
-          //    charset = hts_getCharsetFromMeta(r.adr, r.size);
-          //  }
-          //  if (charset != NULL) {
-          //    char *const utf8 = hts_convertStringToUTF8(r.adr, r.size, charset);
-          //    /* Use new buffer */
-          //    if (utf8 != NULL) {
-          //      freet(r.adr);
-          //      r.size = strlen(utf8);
-          //      r.adr = utf8;
-          //      /* New UTF-8 charset */
-          //      r.charset[0] = '\0';
-          //      strcpy(r.charset, "utf-8");
-          //    }
-          //    /* Free charset */
-          //    free(charset);
-          //  }
-          //}
+          /* Convert charset to UTF-8 - NOT! (what about links ? remote server
+           * side will have troubles with converted names) */

          /* Check bogus chars */
          if ((r.adr) && (r.size)) {
@@ -1490,50 +1540,6 @@ int httpmirror(char *url1, httrackp * opt) {

        }
      }
-      // MOVED IN back_finalize()
-      //
-      // -------------------- 
-      // REAL MEDIA HACK
-      // Check if we have to load locally the file
-      // --------------------
-      //if (!error) {
-      //  if (r.statuscode == HTTP_OK) {    // OK (ou 304 en backing)
-      //    if (r.adr==NULL) {    // Written file
-      //      if (may_be_hypertext_mime(r.contenttype, urlfil())) {   // to parse!
-      //        LLint sz;
-      //        sz=fsize_utf8(savename());
-      //        if (sz>0) {   // ok, exists!
-      //          if (sz < 8192) {   // ok, small file --> to parse!
-      //            FILE* fp=FOPEN(savename(),"rb");
-      //            if (fp) {
-      //              r.adr=malloct(sz + 1);
-      //              if (r.adr) {
-      //                if (fread(r.adr,1,sz,fp) == sz) {
-      //                  r.size=sz;
-      //                                                r.adr[sz] = '\0';
-      //                                                r.is_write = 0;
-      //                } else {
-      //                  freet(r.adr);
-      //                  r.size=0;
-      //                  r.adr = NULL;
-      //                  r.statuscode=STATUSCODE_INVALID;
-      //                  strcpybuff(r.msg, ".RAM read error");
-      //                }
-      //                fclose(fp);
-      //                fp=NULL;
-      //                // remove (temporary) file!
-      //                remove(savename());
-      //              }
-      //              if (fp)
-      //                fclose(fp);
-      //            }
-      //          }
-      //        }
-      //      }
-      //    }
-      //  }
-      //}
-      // EN OF REAL MEDIA HACK

      // ---stockage en cache---
      // stocker dans le cache?
@@ -1654,24 +1660,6 @@ int httpmirror(char *url1, httrackp * opt) {
      }
 #endif

-      /* info: updated */
-      /*
-         if (ptr>0) {
-         // "mis à jour"
-         if ((!r.notmodified) && (opt->is_update) && (!store_errpage)) {    // page modifiée
-         if (strnotempty(savename())) {
-         HTS_STAT.stat_updated_files++;
-         //if ((opt->debug>0) && (opt->log!=NULL)) {
-         hts_log_print(opt, LOG_INFO, "File updated: %s%s",urladr(),urlfil());
-         }
-         } else {
-         if (!store_errpage) {
-         hts_log_print(opt, LOG_INFO, "File recorded: %s%s",urladr(),urlfil());
-         }
-         }
-         }
-       */
-
      // ------------------------------------------------------
      // traitement (parsing)
      // ------------------------------------------------------
@@ -1717,13 +1705,8 @@ int httpmirror(char *url1, httrackp * opt) {
                free(charset);
            }
            /* Could not detect charset: could it be UTF-8 ? */
-            /* No, we can not do that: browsers do not do it 
+            /* No, we can not do that: browsers do not do it
               (and it would break links). */
-            //if (page_charset[0] == '\0') {
-            //  if (is_unicode_utf8(r.adr, r.size)) {
-            //    strcpy(page_charset, "utf-8");
-            //  }
-            //}
            /* Could not detect charset */
            if (page_charset[0] == '\0') {
              hts_log_print(opt, LOG_INFO,
@@ -1801,10 +1784,6 @@ int httpmirror(char *url1, httrackp * opt) {
              XH_uninit;
              return -1;
            }
-
-            // I'll have to segment this part
-// #include "htsparse.c"
-
          }
        }
        // Fin parsing HTML
@@ -1900,7 +1879,6 @@ int httpmirror(char *url1, httrackp * opt) {
             HTS_STAT.stat_files++;
             HTS_STAT.stat_bytes+=r.size;
           */
-          //printf("ok......\n");
        } else {
          // Si on doit sauver une page HTML sans la scanner, cela signifie que le niveau de
          // récursion nous en empêche
@@ -1969,8 +1947,6 @@ int httpmirror(char *url1, httrackp * opt) {
            }
          }

-          //printf("extern=%s\n",r.contenttype);
-
          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!          
          if (r.adr != NULL || r.size == 0) {
            file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
@@ -2092,16 +2068,9 @@ int httpmirror(char *url1, httrackp * opt) {
          // prochain pass2
          while((ptr < opt->lien_tot) ? (!heap(ptr)->pass2) : 0)
            ptr++;
-
-          //printf("first link==%d\n");
-
        }
      }
    }
-    // copy abort state if necessary from outside
-    //if (!exit_xh && opt->state.exit_xh) {
-    //  exit_xh=opt->state.exit_xh;
-    //}
    // a-t-on dépassé le quota?
    if (!back_checkmirror(opt)) {
      ptr = opt->lien_tot;
@@ -2258,13 +2227,11 @@ int httpmirror(char *url1, httrackp * opt) {
    char BIGSTK htstime[256];
    char BIGSTK infoupdated[256];

-    // int n=(int) (stat_loaded/(time_local()-HTS_STAT.stat_timestart));
    LLint n =
      (LLint) (HTS_STAT.HTS_TOTAL_RECV /
               (max(1, time_local() - HTS_STAT.stat_timestart)));

    sec2str(htstime, time_local() - HTS_STAT.stat_timestart);
-    //sprintf(finalInfo + strlen(finalInfo),LF"HTS-mirror complete in %s : %d links scanned, %d files written (%d bytes overall) [%d bytes received at %d bytes/sec]"LF,htstime,lien_tot-1,HTS_STAT.stat_files,stat_bytes,stat_loaded,n);
    infoupdated[0] = '\0';
    if (opt->is_update) {
      if (HTS_STAT.stat_updated_files > 0) {
@@ -2347,12 +2314,6 @@ int httpmirror(char *url1, httrackp * opt) {

 */
 int engine_stats(void) {
-#if 0
-  static FILE *debug_fp = NULL; /* ok */
-
-  if (!debug_fp)
-    debug_fp = fopen("esstat.txt", "wb");
-#endif
  HTS_STAT.stat_nsocket = HTS_STAT.stat_errors = HTS_STAT.nbk = 0;
  HTS_STAT.nb = 0;
  if (HTS_STAT.HTS_TOTAL_RECV > 2048) {
@@ -2363,10 +2324,6 @@ int engine_stats(void) {
      if ((cdif - HTS_STAT.istat_timestart[i]) >= 2000) {
        TStamp dif;

-#if 0
-        fprintf(debug_fp, "set timer %d\n", i);
-        fflush(debug_fp);
-#endif
        dif = cdif - HTS_STAT.istat_timestart[i];
        if ((TStamp) (dif / 1000) > 0) {
          LLint byt = (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[i]);
@@ -2385,10 +2342,6 @@ int engine_stats(void) {
    // timer #0 resync timer #1 when reaching 1 second limit
    if (HTS_STAT.istat_reference01 != HTS_STAT.istat_timestart[0]) {
      if ((cdif - HTS_STAT.istat_timestart[0]) >= 1000) {
-#if 0
-        fprintf(debug_fp, "resync timer 1\n");
-        fflush(debug_fp);
-#endif
        HTS_STAT.istat_bytes[1] = HTS_STAT.HTS_TOTAL_RECV;
        HTS_STAT.istat_timestart[1] = cdif;
        HTS_STAT.istat_reference01 = HTS_STAT.istat_timestart[0];
@@ -2409,7 +2362,6 @@ void host_ban(httrackp * opt, int ptr,
  lien_back *const back = sback->lnk;
  const int back_max = sback->count;

-  //int l;
  int i;

  if (host[0] == '!')
@@ -2466,9 +2418,7 @@ void host_ban(httrackp * opt, int ptr,
  }

  // effacer liens
-  //l=strlen(host);
-  for(i = 0; i < opt->lien_tot; i++) {
-    //if (heap(i)->adr_len==l) {    // même taille de chaîne
+  for (i = 0; i < opt->lien_tot; i++) {
    // Calcul de taille sécurisée
    if (heap(i)) {
      if (heap(i)->adr) {
@@ -2507,7 +2457,6 @@ void host_ban(httrackp * opt, int ptr,
                    "WARNING! HostCancel detected memory leaks [null at %d]",
                    i);
    }
-    //}
  }
 }

@@ -2924,7 +2873,6 @@ FILE *fileappend(filenote_strc * strc, const char *s) {
  // noter lst
  filenote(strc, s, NULL);

-  // if (*s=='/') strcpybuff(fname,s+1); else strcpybuff(fname,s);    // pas de / (root!!) // ** SIIIIIII!!! à cause de -O <path>
  strcpybuff(fname, s);

 #if HTS_DOSNAME
@@ -2968,7 +2916,6 @@ int filecreateempty(filenote_strc * strc, const char *filename) {
 int filenote(filenote_strc * strc, const char *s, filecreate_params * params) {
  // gestion du fichier liste liste
  if (params) {
-    //filecreate_params* p = (filecreate_params*) params;
    strcpybuff(strc->path, params->path);
    strc->lst = params->lst;
    return 0;
@@ -3048,9 +2995,8 @@ void usercommand_exe(const char *cmd, const char *file) {
  }
 }

-static void postprocess_file(httrackp * opt, const char *save, const char *adr,
+static void postprocess_file(httrackp *opt, const char *save, const char *adr,
                             const char *fil) {
-  //int first = 0;
  /* MIME-html archive to build */
  if (opt != NULL && opt->mimehtml) {
    if (adr != NULL && strcmp(adr, "primary") == 0) {
@@ -3078,7 +3024,6 @@ static void postprocess_file(httrackp * opt, const char *save, const char *adr,
      }

      if (!opt->state.mimehtml_created) {
-        //first = 1;
        opt->state.mimefp =
          fopen(fconcat
                (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
@@ -3222,28 +3167,6 @@ int fspc(httrackp * opt, FILE * fp, const char *type) {
  return 0;
 }

-// vérifier taux de transfert
-#if 0
-void check_rate(TStamp stat_timestart, int maxrate) {
-  // vérifier taux de transfert (pas trop grand?)
-  /*
-     if (maxrate>0) {
-     int r = (int) (HTS_STAT.HTS_TOTAL_RECV/(time_local()-stat_timestart));    // taux actuel de transfert
-     HTS_STAT.HTS_TOTAL_RECV_STATE=0;
-     if (r>maxrate) {    // taux>taux autorisé
-     int taux = (int) (((TStamp) (r - maxrate) * 100) / (TStamp) maxrate);
-     if (taux<15)
-     HTS_STAT.HTS_TOTAL_RECV_STATE=1;   // ralentir un peu (<15% dépassement)
-     else if (taux<50)
-     HTS_STAT.HTS_TOTAL_RECV_STATE=2;   // beaucoup (<50% dépassement)
-     else
-     HTS_STAT.HTS_TOTAL_RECV_STATE=3;   // énormément (>50% dépassement)
-     }
-     }
-   */
-}
-#endif
-
 // ---
 // sous routines liées au moteur et au backing

@@ -3253,21 +3176,8 @@ int backlinks_done(const struct_back * sback,
                   int ptr) {
  int n = 0;

-#if 0
-  int i;
-
-  //Links done and stored in cache
-  for(i = ptr + 1; i < lien_tot; i++) {
-    if (heap(i)) {
-      if (heap(i)->pass2 == -1) {
-        n++;
-      }
-    }
-  }
-#else
  // finalized in background
  n += HTS_STAT.stat_background;
-#endif
  n += back_done_incache(sback);
  return n;
 }
@@ -3404,8 +3314,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
    /* on a déja parcouru */
    if (p < cache->ptr_ant)
      p = cache->ptr_ant;
-    while(p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
-      //while((p<lien_tot) && (n>0) && (p < ptr+opt->maxcache_anticipate)) {
+    while (p < opt->lien_tot && n > 0 && back_checkmirror(opt)) {
      int ok = 1;

      // on ne met pas le fichier en backing si il doit être traité après ou s'il a déja été traité
@@ -3417,12 +3326,6 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
          ok = 0;
      }

-      // Why in hell did I do that ?
-      //if (ok && heap(p)->sav != NULL && heap(p)->sav[0] != '\0' 
-      //  && hash_read(opt->hash,heap(p)->sav,NULL,HASH_STRUCT_FILENAME ) >= 0)     // lookup in liens_record
-      //{
-      //  ok = 0;
-      //}
      if (heap(p)->sav == NULL || heap(p)->sav[0] == '\0'
          || hash_read(opt->hash, heap(p)->sav, NULL, HASH_STRUCT_FILENAME ) < 0) {
        ok = 0;
@@ -3451,7 +3354,7 @@ int back_fill(struct_back * sback, httrackp * opt, cache_back * cache,
        }
      }
      p++;
-    }                           // while
+    } // while
    /* sauver position dernière anticipation */
    cache->ptr_ant = p;
    cache->ptr_last = ptr;
@@ -3713,20 +3616,7 @@ HTSEXT_API hts_boolean hts_has_stopped(httrackp *opt) {
  return ended;
 }

-// régler en cours de route les paramètres réglables..
-// -1 : erreur
-//HTSEXT_API int hts_setopt(httrackp* set_opt) {
-//  if (set_opt) {
-//    httrackp* engine_opt=hts_declareoptbuffer(NULL);
-//    if (engine_opt) {
-//      //_hts_setopt=opt;
-//      copy_htsopt(set_opt,engine_opt);
-//    }
-//  }
-//  return 0;
-//}
 // ajout d'URL
-// -1 : erreur
 HTSEXT_API hts_boolean hts_addurl(httrackp *opt, char **url) {
  if (url)
    opt->state._hts_addurl = url;
@@ -3987,10 +3877,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
              heap_top()->link_import = 0; // pas mode import

              // écrire autres paramètres de la structure-lien
-              //if (meme_adresse)                                 
              heap_top()->premier = heap(ptr)->premier;
-              //else    // sinon l'objet père est le précédent lui même
-              //  heap_top()->premier=ptr;

              heap_top()->precedent = ptr;
              // noter la priorité
@@ -4001,9 +3888,6 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
              heap_top()->pass2 = max(pass_fix, numero_passe);
              heap_top()->retry = opt->retry;

-              //strcpybuff(heap_top()->adr,adr);
-              //strcpybuff(heap_top()->fil,fil);
-              //strcpybuff(heap_top()->sav,save); 
              hts_log_print(opt, LOG_DEBUG, "(module): OK, NOTE: %s%s -> %s",
                            heap_top()->adr, heap_top()->fil,
                            heap_top()->sav);
--- a/src/htscore.h
+++ b/src/htscore.h
@@ -412,10 +412,6 @@ char *readfile_utf8(const char *fil);

 char *readfile_or(const char *fil, const char *defaultdata);

-#if 0
-void check_rate(TStamp stat_timestart, int maxrate);
-#endif
-
 /* Backing (download-slot) scheduler. Operate on the back[] ring (struct_back).
   Not thread-safe; call from the single crawl loop. */

@@ -467,11 +463,6 @@ int ask_continue(httrackp * opt);
 /* Number of decimal digits in n. */
 int nombre_digit(int n);

-// Java
-#if 0
-int hts_add_file(char *file, int file_position);
-#endif
-
 // Polling
 #if HTS_POLL
 int check_flot(T_SOC s);
--- a/src/htscoremain.c
+++ b/src/htscoremain.c
@@ -145,7 +145,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
  char *url = NULL;             // URLS séparées par un espace
  int url_sz = 65535;

-  //char url[65536];         // URLS séparées par un espace
  // the parametres
  int httrack_logmode = 3;      // ONE log file

@@ -290,12 +289,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
        if (tmp_argc == 1) {    /* pas -P & co */
          if (!cmdl_opt(tmp_argv[0])) { /* pas -c0 & co */
            if (argv_url < 0)
-              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
-            //if (argv_url>=0) {
+              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
+                            // previous options
            argv_url++;
            if (!argv_firsturl)
              argv_firsturl = x_argv[x_argc - 1];
-            //}
          } else {
            if (strcmp(tmp_argv[0], "-h") == 0) {
              help(argv[0], !opt->quiet);
@@ -324,8 +322,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
        } else if (tmp_argc == 2) {
          if ((strcmp(tmp_argv[0], "-%L") == 0)) {      // liste d'URLs
            if (argv_url < 0)
-              argv_url = 0;     // -1==force -> 1=one url already detected, wipe all previous options
-            //if (argv_url>=0)
+              argv_url = 0; // -1==force -> 1=one url already detected, wipe all
+                            // previous options
            argv_url++;         /* forcer */
          }
        }
@@ -403,15 +401,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                    else
                      inQuote = 1;
                  } else if (!inQuote && !noDbl && argv[na][i] == ',') {
-                    //StringAddchar(path, '\0');
-                    //j = 0;
                    path = &opt->path_log;
                  } else {
                    StringAddchar(*path, argv[na][i]);
-                    //path[j++] = argv[na][i];
                  }
                }
-                //path[j++] = '\0';
                if (StringLength(opt->path_log) == 0) {
                  StringCopyS(opt->path_log, opt->path_html);
                }
@@ -420,7 +414,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                if (check_path(&opt->path_html, argv_firsturl)) {
                  opt->dir_topindex = 1;        // rebuilt top index
                }
-                //printf("-->%s\n%s\n",StringBuff(opt->path_html),StringBuff(opt->path_log));                
              }
              break;
            }                   // switch
@@ -539,8 +532,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
            insert_after++;
          }
        }
-      } while(lastp != NULL);
-      //fclose(fp);
+      } while (lastp != NULL);
    }
  }

@@ -716,24 +708,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
            HTS_PANIC_PRINTF(s);
            htsmain_free();
            return -1;
-#else
-#if 0
-            char _args[8][256];
-            char *args[8];
-
-            printf("Cheking for updates...\n");
-            strcpybuff(_args[0], argv[0]);
-            strcpybuff(_args[1], "--get");
-            sprintf(_args[2], HTS_UPDATE_WEBSITE, 0, "");
-            strcpybuff(_args[3], "--quickinfo");
-            args[0] = _args[0];
-            args[1] = _args[1];
-            args[2] = _args[2];
-            args[3] = _args[3];
-            args[4] = NULL;
-            if (execvp(args[0], args) == -1) {
-            }
-#endif
 #endif
          }
          //
@@ -751,103 +725,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
    }
  }

-  // Compter urls/jokers
-  /*
-     if (argv_url<=0) { 
-     int na;
-     argv_url=0;
-     for(na=1;na<argc;na++) {
-     if ( (strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0) ) {
-     na++;    // sauter nom de proxy
-     } else if (!cmdl_opt(argv[na])) { 
-     argv_url++;   // un de plus       
-     } else if (strcmp(argv[na],"-h")==0) {
-     help(argv[0],!opt->quiet);
-     htsmain_free();
-     return 0;
-     } else {
-     if ((strchr(argv[na],'q')!=NULL))
-     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
-     if ((strchr(argv[na],'i')!=NULL)) {  // doit.log!
-     argv_url=0;
-     na=argc;
-     }
-     }
-     }
-     }  
-   */
-
-  // Ici on ajoute les arguments qui ont été appelés avant au cas où on récupère une session
-  // Exemple: httrack www.truc.fr -L0 puis ^C puis httrack sans URL : ajouter URL précédente
-  /*
-     if (argv_url==0) {
-     //if ((fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.dat"))) && (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/new.ndx")))) {  // il existe déja un cache précédent.. renommer
-     if (fexist(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"))) {    // un cache est présent
-
-     x_argvblk=(char*) calloct(32768,1);
-
-     if (x_argvblk!=NULL) {
-     FILE* fp;
-     int x_argc;
-
-     //strcpybuff(x_argvblk,"httrack ");
-     fp=fopen(fconcat(StringBuff(opt->path_log),"hts-cache/doit.log"),"rb");
-     if (fp) {
-     linput(fp,x_argvblk+strlen(x_argvblk),8192);
-     fclose(fp); fp=NULL;
-     }
-
-     // calculer arguments selon derniers arguments
-     x_argv[0]=argv[0];
-     x_argc=1;
-     {
-     char* p=x_argvblk;
-     do {
-     x_argv[x_argc++]=p;
-     //p=strstr(p," ");
-     // exemple de chaine: "echo \"test\"" c:\a "\$0"
-     p=next_token(p,1);    // prochain token
-     if (p) {
-     *p=0;    // octet nul (tableau)
-     p++;
-     }            
-     } while(p!=NULL);
-     }
-     // recopier arguments actuels (pointeurs uniquement)
-     {
-     int na;
-     for(na=1;na<argc;na++) {
-     if (strcmp(argv[na],"-O") != 0)    // SAUF le path!
-     x_argv[x_argc++]=argv[na];
-     else
-     na++;
-     }
-     }
-     argc=x_argc;      // nouvel argc
-     argv=x_argv;      // nouvel argv
-     }
-
-     }
-     //}
-     }
-   */
-
-  // Vérifier quiet
-  /*
-     { 
-     int na;    
-     for(na=1;na<argc;na++) {
-     if (!cmdl_opt(argv[na])) { 
-     if ((strcmp(argv[na],"-P")==0) || (strcmp(argv[na],"-N")==0) || (strcmp(argv[na],"-F")==0) || (strcmp(argv[na],"-O")==0) || (strcmp(argv[na],"-V")==0))
-     na++;    // sauter nom de proxy
-     } else {
-     if ((strchr(argv[na],'q')!=NULL) || (strchr(argv[na],'i')!=NULL))
-     opt->quiet=1;    // ne pas poser de questions! (nohup par exemple)
-     }
-     }
-     }
-   */
-
  /* Engine self-tests: -#test lists them, -#test=NAME [args] runs one. Handled
     here, ahead of the no-URL usage gate below, so they need no dummy URL. */
  {
@@ -954,8 +831,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
  } else {                      // plus de 2 paramètres
    // un fichier log existe?
    if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-      StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // fichier lock?
-      //char s[32];
+                       StringBuff(opt->path_log),
+                       "hts-in_progress.lock"))) { // fichier lock?

      opt->cache = HTS_CACHE_PRIORITY; // cache prioritaire
      if (opt->quiet == 0) {
@@ -986,11 +863,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
          }
        }
      }
-    } else
-      if (fexist
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "index.html"))) {
-      //char s[32];
+    } else if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                              StringBuff(opt->path_html), "index.html"))) {
      opt->cache = HTS_CACHE_TEST_UPDATE;
      if (opt->quiet == 0) {
        if ((fexist
@@ -1085,7 +959,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
            //
          case 'g':            // récupérer un (ou plusieurs) fichiers isolés
            opt->wizard = HTS_WIZARD_AUTO;
-            //opt->wizard=0;             // pas de wizard
            opt->cache = HTS_CACHE_NONE; // ni de cache
            opt->makeindex = 0; // ni d'index
            httrack_logmode = 1;        // erreurs à l'écran
@@ -1185,9 +1058,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
            while(isdigit((unsigned char) *(com + 1)))
              com++;
            break;
-            //
-            //case 'A': opt->urlmode=1; break;
-            //case 'R': opt->urlmode=2; break;
          case 'K':
            opt->urlmode = HTS_URLMODE_ABSOLUTE;
            if (isdigit((unsigned char) *(com + 1))) {
@@ -1199,8 +1069,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
              while(isdigit((unsigned char) *(com + 1)))
                com++;
            }
-            //if (*(com+1)=='0') { opt->urlmode=2; com++; } break;
-            //
          case 'c':
            if (isdigit((unsigned char) *(com + 1))) {
              sscanf(com + 1, "%d", &opt->maxsoc);
@@ -1688,7 +1556,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                  } else {
                    char *a;

-                    //char* b = StringBuff(opt->mimedefs) + StringLength(opt->mimedefs);
                    for(a = argv[na]; *a != '\0'; a++) {
                      if (*a == ';') {  /* next one */
                        StringAddchar(opt->mimedefs, '\n');
@@ -1936,6 +1803,23 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                  StringCopy(opt->cookies_file, argv[na]);
                }
                break;
+              case 'Y': // why: explain the filter verdict for a URL, no crawl
+                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
+                  HTS_PANIC_PRINTF("Option why needs a blank space and a URL");
+                  printf(
+                      "Example: --why \"http://www.example.com/file.zip\"\n");
+                  htsmain_free();
+                  return -1;
+                } else {
+                  na++;
+                  if (strlen(argv[na]) >= HTS_URLMAXSIZE) {
+                    HTS_PANIC_PRINTF("why URL too long");
+                    htsmain_free();
+                    return -1;
+                  }
+                  StringCopy(opt->why_url, argv[na]);
+                }
+                break;
              case 'G': // pause: randomized inter-file delay MIN[:MAX] seconds
                if ((na + 1 >= argc) || (argv[na + 1][0] == '-')) {
                  HTS_PANIC_PRINTF("Option pause needs a blank space and a "
@@ -2027,10 +1911,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                  HTS_PANIC_PRINTF(s);
                  htsmain_free();
                  return -1;
-                }
-                break;
-
-                //case 's': opt->sslengine=1; if (isdigit((unsigned char)*(com+1))) { sscanf(com+1,"%d",&opt->sslengine); while(isdigit((unsigned char)*(com+1))) com++; } break;
+              } break;
              }
            }
            break;
@@ -2274,15 +2155,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                return 0;
                break;
              case '~':        /* internal lib test */
-                HTS_PANIC_PRINTF
-                  ("Option #~ is disabled for security reasons");
-                //Disabled because choke on GCC 4.3 (toni from links2linux.de)
-                //{
-                //  char thisIsATestYouShouldSeeAnError[12];
-                //  const char *const bufferOverflowTest = "0123456789012345678901234567890123456789";
-                //  strcpybuff(thisIsATestYouShouldSeeAnError, bufferOverflowTest);
-                //  return 0;
-                //}
+                HTS_PANIC_PRINTF("Option #~ is disabled for security reasons");
                break;
              case 'f':
                opt->flush = 1;
@@ -2384,7 +2257,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
              opt->proxy.active = 1;
              // Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
              a = argv[na] + strlen(argv[na]) - 1;
-              // a=strstr(argv[na],":");  // port
              while((a > argv[na]) && (*a != ':') && (*a != '@'))
                a--;
              if (*a == ':') {  // un port est présent, <proxy>:port
@@ -2489,7 +2361,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
 #endif
 #endif

-  //printf("WARNING! This is *only* a beta-release of HTTrack\n");
  io_flush;

 #if DEBUG_STEPS
@@ -2573,7 +2444,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
    {
      FILE *fp = NULL;

-      //int n=0;
      char t[256];

      time_local_rfc822(t);     // faut bien que ca serve quelque part l'heure RFC1945 arf'
@@ -2607,9 +2477,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
      }

      strcpybuff(n_lock,
-             fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                     "hts-in_progress.lock"));
-      //sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
+                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                         StringBuff(opt->path_log), "hts-in_progress.lock"));
      /*do {
         if (!n)
         sprintf(n_lock,fconcat(OPT_GET_BUFF(opt), StringBuff(opt->path_log),"hts-in_progress.lock"),n);
@@ -2685,8 +2554,6 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
          fprintf(fp, LF);
          fclose(fp);
          fp = NULL;
-          //} else if (opt->debug>1) {
-          //  printf("! FileOpen error, \"%s\"\n",strerror(errno));
        }
      }
      // petit message dans le lock
--- a/src/htsfilters.c
+++ b/src/htsfilters.c
@@ -94,6 +94,32 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
  return verdict;
 }

+int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
+                     const char *nom2, LLint *size, int *size_flag,
+                     int *depth) {
+  int depth1 = 0, depth2 = 0;
+  int flag1 = 0, flag2 = 0;
+  LLint sz1 = 0, sz2 = 0;
+  int jok1, jok2, use1;
+
+  if (size) {
+    sz1 = *size;
+    sz2 = *size;
+  }
+  jok1 = fa_strjoker(type, filters, nfil, nom1, size ? &sz1 : NULL,
+                     size_flag ? &flag1 : NULL, &depth1);
+  jok2 = fa_strjoker(type, filters, nfil, nom2, size ? &sz2 : NULL,
+                     size_flag ? &flag2 : NULL, &depth2);
+  use1 = jok2 == 0 || (jok1 != 0 && depth1 >= depth2);
+  if (size)
+    *size = use1 ? sz1 : sz2;
+  if (size_flag)
+    *size_flag = use1 ? flag1 : flag2;
+  if (depth)
+    *depth = use1 ? depth1 : depth2;
+  return use1 ? jok1 : jok2;
+}
+
 // supercomparateur joker (tm)
 // compare a et b (b=avec joker dedans), case insensitive [voir CI]
 // renvoi l'adresse de la première lettre de la chaine
@@ -102,9 +128,8 @@ int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * siz
 // cet algo est 'un peu' récursif mais ne consomme pas trop de tm
 // * = toute lettre
 // --?-- : spécifique à HTTrack et aux ?
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
-                          int *size_flag) {
-  //int err=0;
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
+                                LLint *size, int *size_flag) {
  if (strnotempty(joker) == 0) {        // fin de chaine joker
    if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
      return chaine;
@@ -236,9 +261,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
                for(j = (int) (unsigned char) joker[i];
                    j <= (int) (unsigned char) joker[i + 2]; j++)
                  pass[j] = 1;
-
              }
-              // else err=1;
              i += 3;
            } else { // 1 car, ex: *[ ]
              pass[(int) (unsigned char) joker[i]] = 1;
@@ -262,7 +285,6 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
      for(i = 0; i < 256; i++)
        pass[i] = 1;            // tout autoriser
      jmp = 1;
-      ////if (joker[2]==LEFT) jmp=3;        // permet de recher *<crochet ouvrant>
    }

    {
@@ -290,7 +312,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
      if (!unique)
        max = (int) strlen(chaine);
      else                      /* *(a) only match a (not aaaaa) */
-        max = 1;
+        max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
      while(i < (int) max) {
        if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
          if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
--- a/src/htsfilters.h
+++ b/src/htsfilters.h
@@ -41,6 +41,11 @@ Please visit our Website: http://www.httrack.com

 int fa_strjoker(int type, char **filters, int nfil, const char *nom, LLint * size,
                int *size_flag, int *depth);
+/* fa_strjoker() on both URL forms the engine builds (nom1 full, nom2 without
+   scheme); the match latest in the list wins, a "don't know" verdict defers.
+   Returns the merged verdict; the out-params carry the winner's values. */
+int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
+                     const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                          int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);
--- a/src/htsftp.c
+++ b/src/htsftp.c
@@ -54,15 +54,9 @@ Please visit our Website: http://www.httrack.com
 #endif

 // ftp mode passif
-// #if HTS_INET6==0
 #define FTP_PASV 1
-// #else
-// no passive mode for v6
-// #define FTP_PASV 0
-// #endif

 #define FTP_DEBUG 0
-//#define FORK_DEBUG 0

 #if USE_BEGINTHREAD

@@ -206,20 +200,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
  {
    char *a;

-#if 0
-    a = back->url_fil + strlen(back->url_fil) - 1;
-    while((a > back->url_fil) && (*a != '/'))
-      a--;
-    if (*a != '/') {
-      a = NULL;
-    }
-#else
    a = back->url_fil;
-#endif
    if (a != NULL && *a != '\0') {
-#if 0
-      a++;                      // sauter /
-#endif
      ftp_filename = a;
      if (strnotempty(a)) {
        char catbuff[CATBUFF_SIZE];
@@ -243,7 +225,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
      }
    } else {
      strcpybuff(back->r.msg, "Unexpected PORT error");
-      // back->status=STATUS_FTP_READY;    // fini
      back->r.statuscode = STATUSCODE_INVALID;
    }
  }
@@ -277,7 +258,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
    if (hts_dns_resolve2(opt, _adr, &server, &error) == NULL) {
      snprintf(back->r.msg, sizeof(back->r.msg),
               "Unable to get server's address: %s", error);
-      // back->status=STATUS_FTP_READY;    // fini
      back->r.statuscode = STATUSCODE_NON_FATAL;
      _HALT_FTP return 0;
    }
@@ -290,20 +270,17 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
    soc_ctl = (T_SOC) socket(SOCaddr_sinfamily(server), SOCK_STREAM, 0);
    if (soc_ctl == INVALID_SOCKET) {
      strcpybuff(back->r.msg, "Unable to create a socket");
-      // back->status=STATUS_FTP_READY;    // fini
      back->r.statuscode = STATUSCODE_INVALID;
      _HALT_FTP return 0;
    }

    SOCaddr_initport(server, port);
-    // server.sin_port = htons((unsigned short int) port);

    // connexion (bloquante, on est en thread)
    strcpybuff(back->info, "connect");

    if (connect(soc_ctl, &SOCaddr_sockaddr(server), SOCaddr_size(server)) != 0) {
      strcpybuff(back->r.msg, "Unable to connect to the server");
-      // back->status=STATUS_FTP_READY;    // fini
      back->r.statuscode = STATUSCODE_INVALID;
      _HALT_FTP return 0;
 #ifdef _WIN32
@@ -345,64 +322,22 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
              // ok
            } else {
              strcpybuff(back->r.msg, "TYPE I error");
-              // back->status=STATUS_FTP_READY;    // fini
              back->r.statuscode = STATUSCODE_INVALID;
            }
-#if 0
-            // --CWD--
-            char *a;
-
-            a = back->url_fil + strlen(back->url_fil) - 1;
-            while((a > back->url_fil) && (*a != '/'))
-              a--;
-            if (*a == '/') {    // ok repéré
-              char BIGSTK target[1024];
-
-              target[0] = '\0';
-              strncatbuff(target, back->url_fil, (int) (a - back->url_fil));
-              if (strnotempty(target) == 0)
-                strcatbuff(target, "/");
-              strcpybuff(back->info, "cwd");
-              snprintf(line, sizeof(line), "CWD %s", target);
-              send_line(soc_ctl, line);
-              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-              _CHECK_HALT_FTP;
-              if (line[0] == '2') {
-                send_line(soc_ctl, "TYPE I");
-                get_ftp_line(soc_ctl, line, sizeof(line), timeout);
-                _CHECK_HALT_FTP;
-                if (line[0] == '2') {
-                  // ok..
-                } else {
-                  strcpybuff(back->r.msg, "TYPE I error");
-                  // back->status=STATUS_FTP_READY;    // fini
-                  back->r.statuscode = STATUSCODE_INVALID;
-                }
-              } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "CWD error: %s", linejmp(line));
-                // back->status=STATUS_FTP_READY;    // fini
-                back->r.statuscode = STATUSCODE_INVALID;
-              }                 // sinon on est prêts
-            } else {
-              strcpybuff(back->r.msg, "Unexpected ftp error");
-              // back->status=STATUS_FTP_READY;    // fini
-              back->r.statuscode = STATUSCODE_INVALID;
-            }
-#endif

          } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "Bad password: %s",
+                     linejmp(line));
            back->r.statuscode = STATUSCODE_INVALID;
          }
        } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s", linejmp(line));
-          // back->status=STATUS_FTP_READY;    // fini
+          snprintf(back->r.msg, sizeof(back->r.msg), "Bad user name: %s",
+                   linejmp(line));
          back->r.statuscode = STATUSCODE_INVALID;
        }
      } else {
-        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s", linejmp(line));
-        // back->status=STATUS_FTP_READY;    // fini
+        snprintf(back->r.msg, sizeof(back->r.msg), "Connection refused: %s",
+                 linejmp(line));
        back->r.statuscode = STATUSCODE_INVALID;
      }

@@ -469,8 +404,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
            }
            // -- fin analyse de l'adresse IP et du port --
          } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
+                     linejmp(line));
            back->r.statuscode = STATUSCODE_INVALID;
          }                     // sinon on est prêts
        } else {
@@ -501,13 +436,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                }
              }
            } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s", linejmp(line));
-              // back->status=STATUS_FTP_READY;    // fini
+              snprintf(back->r.msg, sizeof(back->r.msg), "EPSV incorrect: %s",
+                       linejmp(line));
              back->r.statuscode = STATUSCODE_INVALID;
            }
          } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "PASV/EPSV error: %s",
+                     linejmp(line));
            back->r.statuscode = STATUSCODE_INVALID;
          }                     // sinon on est prêts
        }
@@ -613,9 +548,8 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                  deletesoc(soc_dat);
                  soc_dat = INVALID_SOCKET;
                  //
-                  snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s",
-                          linejmp(line));
-                  // back->status=STATUS_FTP_READY;    // fini
+                  snprintf(back->r.msg, sizeof(back->r.msg),
+                           "RETR command error: %s", linejmp(line));
                  back->r.statuscode = STATUSCODE_INVALID;
                }               // sinon on est prêts
              } else {
@@ -626,23 +560,20 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                soc_dat = INVALID_SOCKET;
                //
                strcpybuff(back->r.msg, "Unable to connect");
-                // back->status=STATUS_FTP_READY;    // fini
                back->r.statuscode = STATUSCODE_INVALID;
              }                 // sinon on est prêts
            } else {
              strcpybuff(back->r.msg, "Unable to create a socket");
-              // back->status=STATUS_FTP_READY;    // fini
              back->r.statuscode = STATUSCODE_INVALID;
            }                   // sinon on est prêts
          } else {
            snprintf(back->r.msg, sizeof(back->r.msg),
                     "Unable to resolve IP %s: %s", adr_ip, error);
-            // back->status=STATUS_FTP_READY;    // fini
            back->r.statuscode = STATUSCODE_INVALID;
          }                     // sinon on est prêts
        } else {
-          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s", linejmp(line));
-          // back->status=STATUS_FTP_READY;    // fini
+          snprintf(back->r.msg, sizeof(back->r.msg), "PASV incorrect: %s",
+                   linejmp(line));
          back->r.statuscode = STATUSCODE_INVALID;
        }                       // sinon on est prêts
 #else
@@ -663,17 +594,16 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
              //T_SOC soc_dat;
              if ((soc_dat = accept(soc_servdat, NULL, NULL)) == INVALID_SOCKET) {
                strcpybuff(back->r.msg, "Unable to accept connection");
-                // back->status=STATUS_FTP_READY;    // fini
                back->r.statuscode = STATUSCODE_INVALID;
              }
            } else {
-              snprintf(back->r.msg, sizeof(back->r.msg), "RETR command error: %s", linejmp(line));
-              // back->status=STATUS_FTP_READY;    // fini
+              snprintf(back->r.msg, sizeof(back->r.msg),
+                       "RETR command error: %s", linejmp(line));
              back->r.statuscode = STATUSCODE_INVALID;
            }
          } else {
-            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s", linejmp(line));
-            // back->status=STATUS_FTP_READY;    // fini
+            snprintf(back->r.msg, sizeof(back->r.msg), "PORT command error: %s",
+                     linejmp(line));
            back->r.statuscode = STATUSCODE_INVALID;
          }
 #ifdef _WIN32
@@ -683,7 +613,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
 #endif
        } else {
          strcpybuff(back->r.msg, "Unable to listen to a port");
-          // back->status=STATUS_FTP_READY;    // fini
          back->r.statuscode = STATUSCODE_INVALID;
        }
 #endif
@@ -707,21 +636,18 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
            int len = 1;
            int read_len = 1024;

-            //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
-
            while((len > 0) && (!stop_ftp(back))) {
              // attendre les données
              len = 1;          // pas d'erreur pour le moment
              switch (wait_socket_receive(soc_dat, timeout)) {
              case -1:
                strcpybuff(back->r.msg, "FTP read error");
-                // back->status=STATUS_FTP_READY;    // fini
                back->r.statuscode = STATUSCODE_INVALID;
                len = 0;        // fin
                break;
              case 0:
-                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)", timeout);
-                // back->status=STATUS_FTP_READY;    // fini
+                snprintf(back->r.msg, sizeof(back->r.msg), "Time out (%d)",
+                         timeout);
                back->r.statuscode = STATUSCODE_INVALID;
                len = 0;        // fin
                break;
@@ -743,17 +669,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                         }
                       */
                      strcpybuff(back->r.msg, "Write error");
-                      // back->status=STATUS_FTP_READY;    // fini
                      back->r.statuscode = STATUSCODE_INVALID;
                      len = 0;  // error
                    }
                  } else {
                    strcpybuff(back->r.msg, "Unexpected write error");
-                    // back->status=STATUS_FTP_READY;    // fini
                    back->r.statuscode = STATUSCODE_INVALID;
                  }
-                } else {        // Erreur ou terminé
-                  // back->status=STATUS_FTP_READY;    // fini
+                } else { // Erreur ou terminé
                  back->r.statuscode = 0;
                  if (back->r.totalsize > 0
                      && back->r.size != back->r.totalsize) {
@@ -762,7 +685,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                  }
                }
                read_len = 1024;
-                //HTS_TOTAL_RECV_CHECK(read_len);         // Diminuer au besoin si trop de données reçues
              }
            }
            if (back->r.fp) {
@@ -771,7 +693,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
            }
          } else {
            strcpybuff(back->r.msg, "Unable to write file");
-            // back->status=STATUS_FTP_READY;    // fini
            back->r.statuscode = STATUSCODE_INVALID;
          }
 #ifdef _WIN32
@@ -787,16 +708,14 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
              get_ftp_line(soc_ctl, line, sizeof(line), timeout);
              if (line[0] == '2') {     // OK
                strcpybuff(back->r.msg, "OK");
-                // back->status=STATUS_FTP_READY;    // fini
                back->r.statuscode = HTTP_OK;
              } else {
-                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s", linejmp(line));
-                // back->status=STATUS_FTP_READY;    // fini
+                snprintf(back->r.msg, sizeof(back->r.msg), "RETR incorrect: %s",
+                         linejmp(line));
                back->r.statuscode = STATUSCODE_INVALID;
              }
            } else {
              strcpybuff(back->r.msg, "FTP read error");
-              // back->status=STATUS_FTP_READY;    // fini
              back->r.statuscode = STATUSCODE_INVALID;
            }
          }
@@ -822,7 +741,6 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
    back->r.statuscode = HTTP_OK;
    strcpybuff(back->r.msg, "OK");
  }
-  // back->status=STATUS_FTP_READY;    // fini
  return 0;
 }

@@ -989,9 +907,7 @@ int get_ftp_line(T_SOC soc, char *ptrline, size_t line_size, int timeout) {
      break;
    }

-    //HTS_TOTAL_RECV_CHECK(dummy);     // Diminuer au besoin si trop de données reçues
    switch (recv(soc, &b, 1, 0)) {
-      //case 0: break;    // pas encore --> erreur (on attend)!
    case 1:
      HTS_STAT.HTS_TOTAL_RECV += 1;     // compter flux entrant
      if ((b != 10) && (b != 13) && (i < (int) sizeof(data) - 1))
@@ -1118,7 +1034,6 @@ int wait_socket_receive(T_SOC soc, int timeout) {
 int stop_ftp(lien_back * back) {
  if (back->stop_ftp) {
    strcpybuff(back->r.msg, "Cancelled by User");
-    // back->status=STATUS_FTP_READY;    // fini
    back->r.statuscode = STATUSCODE_INVALID;
    return 1;
  }
--- a/src/htshelp.c
+++ b/src/htshelp.c
@@ -152,16 +152,6 @@ void help_wizard(httrackp * opt) {
 #define str (buffers->str)
 #define argv (buffers->argv)

-  //char *urls = (char *) malloct(HTS_URLMAXSIZE * 2);
-  //char *mainpath = (char *) malloct(256);
-  //char *projname = (char *) malloct(256);
-  //char *stropt = (char *) malloct(2048);        // options
-  //char *stropt2 = (char *) malloct(2048);       // options longues
-  //char *strwild = (char *) malloct(2048);       // wildcards
-  //char *cmd = (char *) malloct(4096);
-  //char *str = (char *) malloct(256);
-  //char **argv = (char **) malloct(256 * sizeof(char *));
-
  //
  char *a;

@@ -227,8 +217,6 @@ void help_wizard(httrackp * opt) {
  strcatbuff(stropt2, str);
  strcatbuff(stropt2, projname);
  strcatbuff(stropt2, "\" ");
-  // Créer si ce n'est fait un index.html 1er niveau
-  make_empty_index(str);
  //
  printf("\n");
  printf("Enter URLs (separated by commas or blank spaces) :");
@@ -349,8 +337,6 @@ void help_wizard(httrackp * opt) {
      }
      hts_main(argc, argv);
    }
-    //} else {
-    //  help("httrack",1);
  }

  /* Free buffers */
@@ -455,23 +441,6 @@ void help_catchurl(const char *dest_path) {
    printf("Unable to create a temporary proxy (no remaining port)\n");
 }

-// Créer un index.html vide
-void make_empty_index(const char *str) {
-#if 0
-  if (!fexist(fconcat(str, "index.html"))) {
-    FILE *fp = fopen(fconcat(str, "index.html"), "wb");
-
-    if (fp) {
-      fprintf(fp, "<!-- " HTS_TOPINDEX " -->" CRLF);
-      fprintf(fp,
-              "<HTML><BODY>Index is empty!<BR>(File used to index all HTTrack projects)</BODY></HTML>"
-              CRLF);
-      fclose(fp);
-    }
-  }
-#endif
-}
-
 // mini-aide  (h: help)
 //           y
 void help(const char *app, int more) {
@@ -574,6 +543,7 @@ void help(const char *app, int more) {
  infomsg("Spider options:");
  infomsg("  bN accept cookies in cookies.txt (0=do not accept,* 1=accept)");
  infomsg(" %K  load extra cookies from a Netscape cookies.txt");
+  infomsg(" %Y  explain which filter rule accepts or rejects a URL, then exit");
  infomsg
    ("  u  check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)");
  infomsg
@@ -808,7 +778,4 @@ void help(const char *app, int more) {
  infomsg("[compiled: " HTS_PLATFORM_NAME "]");
 #endif
  infomsg(NULL);
-
-//  infomsg("  R  *relative links (e.g ../link)\n");
-//  infomsg("  A   absolute links (e.g /www.adr/link)\n");
 }
--- a/src/htshelp.h
+++ b/src/htshelp.h
@@ -45,7 +45,6 @@ typedef struct httrackp httrackp;

 void infomsg(const char *msg);
 void help(const char *app, int more);
-void make_empty_index(const char *str);
 void help_wizard(httrackp * opt);
 int help_query(const char *list, int def);
 void help_catchurl(const char *dest_path);
--- a/src/htsindex.c
+++ b/src/htsindex.c
@@ -142,8 +142,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
  char keyword[KEYW_LEN + 32];
  int i = 0;

-  //
-  //int WordIndexSize = 1024;
  coucal WordIndexHash = NULL;
  FILE *tmpfp = NULL;

@@ -178,8 +176,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
             || (strfield2(mime, "text/css"))
    ) {
    inscript = 1;
-    //} else if (strfield2(mime, "text/vnd.wap.wml")) {   // humm won't work in many cases
-    //  inscript=0;
  } else
    return 0;

@@ -275,8 +271,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,

  // Process indexing for this page
  {
-    //FILE* fp=NULL;
-    //fp=fopen(concat(indexpath,"index.txt"),"ab");
    if (fp_tmpproject) {
      while(!feof(tmpfp)) {
        char line[KEYW_LEN + 32];
@@ -286,7 +280,6 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
          intptr_t e = 0;

          if (coucal_read(WordIndexHash, line, &e)) {
-            //if (e) {
            char BIGSTK savelst[HTS_URLMAXSIZE * 2];

            e++;                /* 0 means "once" */
@@ -300,11 +293,9 @@ int index_keyword(const char *html_data, LLint size, const char *mime,
            fprintf(fp_tmpproject, "%s %d %s\n", line,
                    (int) (KEYW_SORT_MAXCOUNT - e), savelst);
            hts_primindex_size++;
-            //}
          }
        }
      }
-      //fclose(fp);
    }
  }

@@ -330,7 +321,6 @@ void index_finish(const char *indexpath, int mode) {
  off_t size = fpsize(fp_tmpproject);

  if (size > 0) {
-    //FILE* fp=fopen(concat(indexpath,"index.txt"),"rb");
    if (fp_tmpproject) {
      tab = (char **) malloct(sizeof(char *) * (hts_primindex_size + 2));
      if (tab) {
@@ -398,8 +388,6 @@ void index_finish(const char *indexpath, int mode) {
                      if (total_hit) {
                        if (mode == 1)  // TEXT
                          fprintf(fp, "\t=%d\r\n", total_hit);
-                        //else                // HTML
-                        //  fprintf(fp,"<br>(%d total hits)\r\n",total_hit);
                        if ((((total_hit * 1000) / hts_primindex_words) >=
                             KEYW_USELESS1K)
                            || (((total_line * 1000) / index) >=
@@ -416,8 +404,6 @@ void index_finish(const char *indexpath, int mode) {
                          if (mode == 1)        // TEXT
                            fprintf(fp, "\t(%d)\r\n",
                                    ((total_hit * 1000) / hts_primindex_words));
-                          //else                // HTML
-                          //  fprintf(fp,"(%d)\r\n",((total_hit*1000)/hts_primindex_words));
                        }
                      }
                      if (mode == 1)    // TEXT
--- a/src/htsjava.c
+++ b/src/htsjava.c
@@ -175,15 +175,14 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
 #if JAVADEBUG
      printf("fopen\n");
 #endif
-      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) == NULL) {
-        //fprintf(stderr, "Cannot open input file.\n");
+      if ((fpout = FOPEN(fconv(catbuff, sizeof(catbuff), file), "r+b")) ==
+          NULL) {
        sprintf(str->err_msg, "Unable to open file %s", file);
        return 0;               // une erreur..
      }
 #if JAVADEBUG
      printf("fread\n");
 #endif
-      //if (fread(&header,1,sizeof(JAVA_HEADER),fpout) != sizeof(JAVA_HEADER)) {   // pas complet..
      if (fread(&header, 1, 10, fpout) != 10) { // pas complet..
        fclose(fpout);
        sprintf(str->err_msg, "File header too small (file len = " LLintP ")",
@@ -257,12 +256,10 @@ static int hts_parse_java(t_hts_callbackarg * carg, httrackp * opt,
      printf("addfiles\n");
 #endif
      {
-        //unsigned int acess;
        unsigned int Class;
        unsigned int SClass;
        int i;

-        //acess = readshort(fpout);
        Class = readshort(fpout);
        SClass = readshort(fpout);

@@ -407,11 +404,9 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,

      p = &buffer[0];

-      //fflush(fp); 
      trans.file_position = ftell(fp);
      length = readshort(fp);
      if (length < HTS_URLMAXSIZE) {
-        // while ((length > 0) && (length<500)) {
        while(length > 0) {
          *p++ = fgetc(fp);

@@ -419,17 +414,13 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
        }
        *p = '\0';

-        //#if JDEBUG
-        //      if(tris(buffer)==1) printf("%s\n ",buffer);
-        //      if(tris(buffer)==2)  printf("%s\n ",printname(buffer));
-        //#endif
        if (tris(str->opt, buffer) == 1)
          str->addLink(str, buffer);    /* trans.file_position */
        else if (tris(str->opt, buffer) == 2)
          str->addLink(str, printname(rname, buffer));

        strcpy(trans.name, buffer);
-      } else {                  // gros pb
+      } else { // gros pb
        while((length > 0) && (!feof(fp))) {
          fgetc(fp);
          length--;
@@ -445,7 +436,6 @@ static RESP_STRUCT readtable(htsmoduleStruct * str, FILE * fp,
    }
    break;
  default:
-    // printf("Type inconnue\n");
    // on arrête tout 
    sprintf(str->err_msg, "Internal structure unknown (type %d)", trans.type);
    *error = 1;
@@ -508,7 +498,6 @@ static char *printname(char rname[1024], char name[1024]) {
    return rname;  // ""
  }
  p += 2;
-  //rname=(char*)calloct(strlen(name)+8,sizeof(char));
  p1 = rname;
  for(j = 0; name[j] != '\0'; j++, p++) {
    if (*p == '/')
--- a/src/htslib.c
+++ b/src/htslib.c
@@ -598,56 +598,27 @@ static const char *hts_mime_modern[][2] = {

 // Reserved (RFC2396)
 #define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
-#define CHAR_RESERVED(c)  ( CIS(c,';') \
-                         || CIS(c,'/') \
-                         || CIS(c,'?') \
-                         || CIS(c,':') \
-                         || CIS(c,'@') \
-                         || CIS(c,'&') \
-                         || CIS(c,'=') \
-                         || CIS(c,'+') \
-                         || CIS(c,'$') \
-                         || CIS(c,',') )
-//#define CHAR_RESERVED(c)  ( strchr(";/?:@&=+$,",(unsigned char)(c)) != 0 )
+#define CHAR_RESERVED(c)                                                       \
+  (CIS(c, ';') || CIS(c, '/') || CIS(c, '?') || CIS(c, ':') || CIS(c, '@') ||  \
+   CIS(c, '&') || CIS(c, '=') || CIS(c, '+') || CIS(c, '$') || CIS(c, ','))
 // Delimiters (RFC2396)
-#define CHAR_DELIM(c)     ( CIS(c,'<') \
-                         || CIS(c,'>') \
-                         || CIS(c,'#') \
-                         || CIS(c,'%') \
-                         || CIS(c,'\"') )
-//#define CHAR_DELIM(c)     ( strchr("<>#%\"",(unsigned char)(c)) != 0 )
+#define CHAR_DELIM(c)                                                          \
+  (CIS(c, '<') || CIS(c, '>') || CIS(c, '#') || CIS(c, '%') || CIS(c, '\"'))
 // Unwise (RFC2396)
-#define CHAR_UNWISE(c)    ( CIS(c,'{') \
-                         || CIS(c,'}') \
-                         || CIS(c,'|') \
-                         || CIS(c,'\\') \
-                         || CIS(c,'^') \
-                         || CIS(c,'[') \
-                         || CIS(c,']') \
-                         || CIS(c,'`') )
-//#define CHAR_UNWISE(c)    ( strchr("{}|\\^[]`",(unsigned char)(c)) != 0 )
+#define CHAR_UNWISE(c)                                                         \
+  (CIS(c, '{') || CIS(c, '}') || CIS(c, '|') || CIS(c, '\\') || CIS(c, '^') || \
+   CIS(c, '[') || CIS(c, ']') || CIS(c, '`'))
 // Special (escape chars) (RFC2396 + >127 )
 #define CHAR_LOW(c)       ( ((unsigned char)(c) <= 31) )
 #define CHAR_HIG(c)       ( ((unsigned char)(c) >= 127) )
 #define CHAR_SPECIAL(c)   ( CHAR_LOW(c) || CHAR_HIG(c) )
 // We try to avoid them and encode them instead
-#define CHAR_XXAVOID(c)   ( CIS(c,' ') \
-                         || CIS(c,'*') \
-                         || CIS(c,'\'') \
-                         || CIS(c,'\"') \
-                         || CIS(c,'&') \
-                         || CIS(c,'!') )
-//#define CHAR_XXAVOID(c)   ( strchr(" *'\"!",(unsigned char)(c)) != 0 )
-#define CHAR_MARK(c)      ( CIS(c,'-') \
-                         || CIS(c,'_') \
-                         || CIS(c,'.') \
-                         || CIS(c,'!') \
-                         || CIS(c,'~') \
-                         || CIS(c,'*') \
-                         || CIS(c,'\'') \
-                         || CIS(c,'(') \
-                         || CIS(c,')') )
-//#define CHAR_MARK(c)      ( strchr("-_.!~*'()",(unsigned char)(c)) != 0 )
+#define CHAR_XXAVOID(c)                                                        \
+  (CIS(c, ' ') || CIS(c, '*') || CIS(c, '\'') || CIS(c, '\"') ||               \
+   CIS(c, '&') || CIS(c, '!'))
+#define CHAR_MARK(c)                                                           \
+  (CIS(c, '-') || CIS(c, '_') || CIS(c, '.') || CIS(c, '!') || CIS(c, '~') ||  \
+   CIS(c, '*') || CIS(c, '\'') || CIS(c, '(') || CIS(c, ')'))

 // conversion éventuelle / vers antislash
 #ifdef _WIN32
@@ -841,15 +812,12 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
 // treat: traiter header?
 // waitconnect: attendre le connect()
 // note: dans retour, on met les params du proxy
-T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
-                  const char *xsend, const char *adr, const char *fil, htsblk * retour) {
-  //htsblk retour;
-  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
+T_SOC http_xfopen(httrackp *opt, int mode, int treat, int waitconnect,
+                  const char *xsend, const char *adr, const char *fil,
+                  htsblk *retour) {
  T_SOC soc = INVALID_SOCKET;
  char BIGSTK tempo_fil[HTS_URLMAXSIZE * 2];

-  //char *p,*q;
-
  // retour prédéfini: erreur
  if (retour) {
    retour->adr = NULL;
@@ -934,7 +902,6 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,
        strcpybuff(retour->msg, "Unable to open local file");
      else {
        // Note: On passe par un FILE* (plus propre)
-        //soc=open(fil,O_RDONLY,0);    // en lecture seule!
        retour->fp = FOPEN(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), 
          unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), fil)), "rb");      // ouvrir
        if (retour->fp == NULL)
@@ -1011,17 +978,10 @@ T_SOC http_xfopen(httrackp * opt, int mode, int treat, int waitconnect,

        } while(strnotempty(rcvd));

-        //rcvsize=-1;    // forCER CHARGEMENT INCONNU
-
-        //if (retour)
-        //  retour->totalsize=rcvsize;
-
-      } else {                  // si GET, on recevra l'en tête APRES
-        //rcvsize=-1;    // on ne connait pas la taille de l'en-tête
+      } else { // si GET, on recevra l'en tête APRES
        if (retour)
          retour->totalsize = -1;
      }
-
    }

  }
@@ -1118,19 +1078,12 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
  char BIGSTK buffer_head_request[16384];
  buff_struct bstr = { buffer_head_request, sizeof(buffer_head_request), 0 };

-  //int use_11=0;     // HTTP 1.1 utilisé
  int direct_url = 0;           // ne pas analyser l'url (exemple: ftp://)
  const char *search_tag = NULL;

  // Initialize buffer
  buffer_head_request[0] = '\0';

-  // header Date
-  //strcatbuff(buff,"Date: ");
-  //time_gmt_rfc822(buff);    // obtenir l'heure au format rfc822
-  //sendc("\n");
-  //strcatbuff(buff,buff);
-
  // possibilité non documentée: >post: et >postfile:
  // si présence d'un tag >post: alors executer un POST
  // exemple: http://www.example.com/test.cgi?foo>post:posteddata=10&foo=5
@@ -1224,11 +1177,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
    }

    // protocole
-    if (!retour->req.http11) {  // forcer HTTP/1.0
-      //use_11=0;
+    if (!retour->req.http11) { // forcer HTTP/1.0
      print_buffer(&bstr, " HTTP/1.0\x0d\x0a");
-    } else {                    // Requète 1.1
-      //use_11=1;
+    } else { // Requète 1.1
      print_buffer(&bstr, " HTTP/1.1\x0d\x0a");
    }

@@ -1364,9 +1315,7 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
          print_buffer(&bstr, "Authorization: Basic %s"H_CRLF, autorisation);
        }
      }
-
    }
-    //strcatbuff(buff,"Accept-Charset: iso-8859-1,*,utf-8\n");

    // Custom header(s)
    if (strnotempty(retour->req.headers)) {
@@ -1520,14 +1469,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
        a += strlen("filename=");
        while(is_space(*a))
          a++;
-        //a=strchr(a,'"');
        if (a) {
          char *c = NULL;

-          //a++;      /* jump " */
          while((c = strchr(a, '/')))   /* skip all / (see RFC2616) */
            a = c + 1;
-          //b=strchr(a+1,'"');
          b = a + strlen(a) - 1;
          while(is_space(*b))
            b--;
@@ -1544,16 +1490,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
  } else if ((p = strfield(rcvd, "Last-Modified:")) != 0) {
    while(is_realspace(*(rcvd + p)))
      p++;                      // sauter espaces
-    if ((int) strlen(rcvd + p) < 64) {  // pas trop long?
-      //struct tm* tm_time=convert_time_rfc822(rcvd+p);
+    if ((int) strlen(rcvd + p) < 64) { // pas trop long?
      strcpybuff(retour->lastmodified, rcvd + p);
    }
  } else if ((p = strfield(rcvd, "Date:")) != 0) {
    if (strnotempty(retour->lastmodified) == 0) {       /* pas encore de last-modified */
      while(is_realspace(*(rcvd + p)))
        p++;                    // sauter espaces
-      if ((int) strlen(rcvd + p) < 64) {        // pas trop long?
-        //struct tm* tm_time=convert_time_rfc822(rcvd+p);
+      if ((int) strlen(rcvd + p) < 64) { // pas trop long?
        strcpybuff(retour->lastmodified, rcvd + p);
      }
    }
@@ -1566,14 +1510,11 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
      else                      // erreur.. ignorer
        retour->etag[0] = '\0';
    }
-  }
-  // else if ((p=strfield(rcvd,"Transfer-Encoding: chunked"))!=0) {  // chunk!
-  else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) {   // chunk!
+  } else if ((p = strfield(rcvd, "Transfer-Encoding:")) != 0) { // chunk!
    while(is_realspace(*(rcvd + p)))
      p++;                      // sauter espaces
    if (strfield(rcvd + p, "chunked")) {
-      retour->is_chunk = 1;     // chunked
-      //retour->http11=2;     // chunked
+      retour->is_chunk = 1; // chunked
 #if HDEBUG
      printf("ok, Transfer-Encoding: détecté\n");
 #endif
@@ -1750,7 +1691,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
          retour->location[0] = '\0';
      }
    }
-  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) && (cookie)) {  // ohh un cookie
+  } else if (((p = strfield(rcvd, "Set-Cookie:")) != 0) &&
+             (cookie)) {        // ohh un cookie
    char *a = rcvd + p;         // pointeur
    char domain[256];           // domaine cookie (.netscape.com)
    char path[256];             // chemin (/)
@@ -1794,10 +1736,8 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
            a++;                // sauter espaces
          value_st = a;
          while((*a != ';') && (*a))
-            a++;                // prochain ;
-          //while( ((*a!='"') || (*(a-1)=='\\')) && (*a)) a++;    // prochain " (et pas \")
+            a++; // prochain ;
          value_end = a;
-          //if (*a==';') {  // finit par un ;
          // vérifier débordements
          if ((((int) (token_end - token_st)) < 200)
              && (((int) (value_end - value_st)) < 8000)
@@ -2139,8 +2079,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
            nl = READ_ERROR;
          }
        }
-        //if ((nl < 0) || ((r->totalsize>0) && (r->size >= r->totalsize)))
-        //  nl=-1;  // break

        // libérer bloc tempo
        freet(buff);
@@ -2228,9 +2166,6 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {
  T_SOC soc;
  htsblk retour;

-  //int rcvsize=-1;
-  //char* rcv=NULL;    // adresse de retour
-  //int bufl=TAILLE_BUFFER;    // 8Ko de buffer
  TStamp tl;
  int timeout = 30;             // timeout pour un check (arbitraire) // **

@@ -2239,11 +2174,8 @@ htsblk http_test(httrackp * opt, const char *adr, const char *fil, char *loc) {

  loc[0] = '\0';
  hts_init_htsblk(&retour);
-  //memset(&retour, 0, sizeof(htsblk));    // effacer
  retour.location = loc;        // si non nul, contiendra l'adresse véritable en cas de moved xx

-  //soc=http_fopen(adr,fil,&retour,NULL);  // ouvrir, + header
-
  // on ouvre en head, et on traite l'en tête
  soc = http_xfopen(opt, 1, 0, 1, NULL, adr, fil, &retour);     // ouvrir HEAD, + envoi header

@@ -2565,6 +2497,10 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
  // effacer adrfil->adr et adrfil->fil
  adrfil->adr[0] = adrfil->fil[0] = '\0';

+  // Reject an over-long URL: a root-relative path is copied whole into fil[].
+  if (strlen(url) >= HTS_URLMAXSIZE * 2)
+    return -1;
+
 #if HDEBUG
  printf("protocol: %s\n", url);
 #endif
@@ -2619,7 +2555,6 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {

    // chemin www... trop long!!
    if ((((int) (q - p))) > HTS_URLMAXSIZE) {
-      //strcpybuff(retour.msg,"Path too long");
      return -1;                // erreur
    }
    // recopier adrfil->adresse www..
@@ -2641,7 +2576,7 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
    if (*p == '/' || *p == '\\') {      /* adrfil->file:///.. */
      strcatbuff(adrfil->fil, p);       // fichier local ; adrfil->adr="#"
    } else {
-      if (p[1] != ':') {
+      if (*p == '\0' || p[1] != ':') {  /* empty path: not a DOS drive letter */
        strcatbuff(adrfil->fil, "//");  /* adrfil->file://server/foo */
        strcatbuff(adrfil->fil, p);
      } else {
@@ -2805,7 +2740,6 @@ void deletesoc_r(htsblk * r) {
 #if HTS_USEOPENSSL
  if (r->ssl_con) {
    SSL_shutdown(r->ssl_con);
-    // SSL_CTX_set_quiet_shutdown(r->ssl_con->ctx, 1);
    SSL_free(r->ssl_con);
    r->ssl_con = NULL;
  }
@@ -3194,7 +3128,6 @@ int finput(T_SOC fd, char *s, int max) {
  int j = 0;

  do {
-    //c=fgetc(fp);
    if (read((int) fd, &c, 1) <= 0) {
      c = 0;
    }
@@ -3210,7 +3143,7 @@ int finput(T_SOC fd, char *s, int max) {
        break;
      }
    }
-  } while((c != 0) && (j < max - 1));
+  } while ((c != 0) && (j < max - 1));
  s[j] = '\0';
  return j;
 }
@@ -4869,7 +4802,6 @@ LLint check_downloadable_bytes(int rate) {

    time_now = mtime_local();
    elapsed_useconds = time_now - HTS_STAT.istat_timestart[id_timer];
-    // NO totally stupid - elapsed_useconds+=1000;      // for the next second, too
    bytes_transferred_during_period =
      (HTS_STAT.HTS_TOTAL_RECV - HTS_STAT.istat_bytes[id_timer]);

@@ -4882,32 +4814,6 @@ LLint check_downloadable_bytes(int rate) {
    return TAILLE_BUFFER;
 }

-//
-// 0 : OK
-// 1 : slow down
-#if 0
-int HTS_TOTAL_RECV_CHECK(int var) {
-  if (HTS_STAT.HTS_TOTAL_RECV_STATE)
-    return 1;
-  /*
-     {
-     if (HTS_STAT.HTS_TOTAL_RECV_STATE==3) { 
-     var = min(var,32); 
-     Sleep(250); 
-     } else if (HTS_STAT.HTS_TOTAL_RECV_STATE==2) { 
-     var = min(var,256); 
-     Sleep(100); 
-     } else { 
-     var/=2; 
-     if (var<=0) var=1; 
-     Sleep(50); 
-     } 
-     }
-   */
-  return 0;
-}
-#endif
-
 // Lecture dans buff de size octets au maximum en utilisant la socket r (structure htsblk)
 // returns: 
 // >0 : data received
@@ -4916,7 +4822,6 @@ int HTS_TOTAL_RECV_CHECK(int var) {
 int hts_read(htsblk * r, char *buff, int size) {
  int retour;

-  //  return read(soc,buff,size);
  if (r->is_file) {
 #if HTS_WIDE_DEBUG
    DEBUG_W("read(%p, %d, %d)\n" _(void *)buff _(int) size _(int) r->fp);
@@ -4934,7 +4839,6 @@ int hts_read(htsblk * r, char *buff, int size) {
    if (r->soc == INVALID_SOCKET)
      printf("!!WIDE_DEBUG ERROR, soc==INVALID hts_read\n");
 #endif
-    //HTS_TOTAL_RECV_CHECK(size);         // Diminuer au besoin si trop de données reçues
 #if HTS_USEOPENSSL
    if (r->ssl) {
      retour = SSL_read(r->ssl_con, buff, size);
@@ -5369,18 +5273,13 @@ SOCaddr* hts_dns_resolve(httrackp * opt, const char *_iadr, SOCaddr *const addr)

 // --- Tracage des mallocs() ---
 #ifdef HTS_TRACE_MALLOC
-//#define htsLocker(A, N) htsLocker(A, N)
 #define htsLocker(A, N) do {} while(0)
 static mlink trmalloc = { NULL, 0, 0, NULL };

 static int trmalloc_id = 0;
 static htsmutex *mallocMutex = NULL;
-static void hts_meminit(void) {
-  //if (mallocMutex == NULL) {
-  //  mallocMutex = calloc(sizeof(*mallocMutex), 1);
-  //  htsLocker(mallocMutex, -999);
-  //}
-}
+
+static void hts_meminit(void) {}
 void *hts_malloc(size_t len) {
  void *adr;

@@ -5464,7 +5363,6 @@ void hts_free(void *adr) {
              htsboundary);
      lnk->next = lnk->next->next;
      free((void *) blk_free);
-      //blk_free->id=-1;
      free((char *) adr - bsize);
      htsLocker(mallocMutex, 0);
      return;
@@ -5533,7 +5431,6 @@ mlink *hts_find(char *adr) {

    if (depl < 0)
      depl = -depl;
-    //assertf(depl < 512000);   /* near the stack frame.. doesn't look like malloc but stack variable */
    return NULL;
  }
 }
@@ -5589,8 +5486,7 @@ int ftp_available(void) {
 }
 #else
 int ftp_available(void) {
-  return 1;                     // ok!
-  //return 0;   // SOUS UNIX, PROBLEMESs
+  return 1; // ok!
 }
 #endif

@@ -5804,7 +5700,6 @@ HTSEXT_API int hts_init(void) {
      assertf("OpenSSL version seems vulnerable to heartbleed bug (CVE-2014-0160)" == NULL);
    }

-    // OpenSSL_add_all_algorithms();
    openssl_ctx = SSL_CTX_new(method);
    if (!openssl_ctx) {
      fprintf(stderr, "fatal: unable to initialize TLS: SSL_CTX_new()\n");
@@ -6111,6 +6006,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
  StringCopy(opt->footer, HTS_DEFAULT_FOOTER);
  StringCopy(opt->strip_query, "");
  StringCopy(opt->cookies_file, "");
+  StringCopy(opt->why_url, "");
  opt->pause_min_ms = 0;
  opt->pause_max_ms = 0;
  opt->ftp_proxy = HTS_TRUE;
@@ -6126,7 +6022,6 @@ HTSEXT_API httrackp *hts_create_opt(void) {
  opt->log = stdout;
  opt->errlog = stderr;
  opt->flush = HTS_TRUE;
-  // opt->aff_progress=0;
  opt->keyboard = HTS_FALSE;
  //
  StringCopy(opt->path_html, "");
@@ -6136,8 +6031,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
  //
  opt->maxlink = 100000;        // 100,000 liens max par défaut
  opt->maxfilter = 200;         // 200 filtres max par défaut
-  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON PARAMETRABLE POUR L'INSTANT --
-  //opt->maxcache_anticipate=256;  // maximum de liens à anticiper
+  opt->maxcache = 1048576 * 32; // a peu près 32Mo en cache max -- OPTION NON
+                                // PARAMETRABLE POUR L'INSTANT --
  opt->maxtime = -1;            // temps max en secondes
  opt->maxrate = 100000;        // taux maxi
  opt->maxconn = 5.0;           // nombre connexions/s
@@ -6259,6 +6154,7 @@ HTSEXT_API void hts_free_opt(httrackp * opt) {
    StringFree(opt->mod_blacklist);
    StringFree(opt->strip_query);
    StringFree(opt->cookies_file);
+    StringFree(opt->why_url);

    StringFree(opt->path_html);
    StringFree(opt->path_html_utf8);
@@ -6290,11 +6186,9 @@ const hts_stat_struct* hts_get_stats(httrackp * opt) {
 }

 // defaut wrappers
-static void __cdecl htsdefault_init(t_hts_callbackarg * carg) {
-}
-static void __cdecl htsdefault_uninit(t_hts_callbackarg * carg) {
-  // hts_freevar();
-}
+static void __cdecl htsdefault_init(t_hts_callbackarg *carg) {}
+
+static void __cdecl htsdefault_uninit(t_hts_callbackarg *carg) {}
 static int __cdecl htsdefault_start(t_hts_callbackarg * carg, httrackp * opt) {
  return 1;
 }
--- a/src/htslib.h
+++ b/src/htslib.h
@@ -171,7 +171,6 @@ HTSEXT_API const char* hts_version(void);
 // fonctions unix/winsock
 int hts_read(htsblk * r, char *buff, int size);

-//int HTS_TOTAL_RECV_CHECK(int var);
 LLint check_downloadable_bytes(int rate);

 HTSEXT_API int hts_uninit_module(void);
@@ -190,7 +189,6 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
 int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
                                const char *path, char *dst, size_t dst_size);

-//int newhttp(char* iadr,char* err=NULL);
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
              int waitconnect);
 /* Like newhttp(), but connect to the addr_index-th resolved address of the host
--- a/src/htsmodules.c
+++ b/src/htsmodules.c
@@ -55,12 +55,6 @@ extern int fspc(httrackp * opt, FILE * fp, const char *type);

 /* >>> Put all modules variables here */

-#if 0
-t_gzopen gzopen = NULL;
-t_gzread gzread = NULL;
-t_gzclose gzclose = NULL;
-#endif
-
 int V6_is_available = HTS_INET6;

 static char WHAT_is_available[64] = "";
@@ -112,18 +106,6 @@ int hts_parse_externals(htsmoduleStruct * str) {
  return -1;
 }

-//static void addCallback(htscallbacks* chain, void* moduleHandle, htscallbacksfncptr exitFnc) {
-//  while(chain->next != NULL) {
-//    chain = chain->next;
-//  }
-//  chain->next = calloct(1, sizeof(htscallbacks));
-//  assertf(chain->next != NULL);
-//  chain = chain->next;
-//  memset(chain, 0, sizeof(*chain));
-//  chain->exitFnc = exitFnc;
-//  chain->moduleHandle = moduleHandle;
-//}
-
 void clearCallbacks(htscallbacks * chain_);
 void clearCallbacks(htscallbacks * chain_) {
  htscallbacks *chain;
--- a/src/htsname.c
+++ b/src/htsname.c
@@ -1332,8 +1332,7 @@ int url_savename(lien_adrfilsave *const afs,

    while((a > afs->save) && (*a != '.') && (*a != '/'))
      a--;
-    if (*a != '.') {            // agh pas de point
-      //strcatbuff(save,".none");                 // a éviter
+    if (*a != '.') {                         // agh pas de point
      strcatbuff(afs->save, ".html");        // préférable!
      hts_log_print(opt, LOG_DEBUG, "Default HTML type set for %s%s => %s",
                    adr_complete, fil_complete, afs->save);
@@ -1471,17 +1470,6 @@ int url_savename(lien_adrfilsave *const afs,
  fil_simplifie(afs->save);

  /* convert name to UTF-8 ? Note: already done while parsing. */
-  //if (charset != NULL && charset[0] != '\0') {
-  //  char *const s = hts_convertStringToUTF8(save, (int) strlen(save), charset);
-
-  //  if (s != NULL) {
-  //    hts_log_print(opt, LOG_DEBUG,
-  //                  "engine: save-name: charset conversion from '%s' to '%s' using charset '%s'",
-  //                  save, s, charset);
-  //    strcpybuff(save, s);
-  //    free(s);
-  //  }
-  //}

  /* callback */
  RUN_CALLBACK5(opt, savename, adr_complete, fil_complete, referer_adr,
@@ -1629,10 +1617,6 @@ int url_savename(lien_adrfilsave *const afs,
        int sameAdr = (strfield2(heap(i)->adr, normadr) != 0);
        int sameFil;

-        // NO - URL hack is only for stripping // and www.
-        //if (opt->urlhack != 0)
-        //  sameFil = ( strfield2(heap(i)->fil, normfil) != 0);
-        //else
        sameFil = (strcmp(heap(i)->fil, normfil) == 0);
        if (sameAdr && sameFil) {       // ok c'est le même lien, adresse déja définie
          /* Take the existing name not to screw up with cAsE sEnSiTiViTy of Linux/Unix */
@@ -1698,17 +1682,13 @@ int url_savename(lien_adrfilsave *const afs,

          strcpybuff(afs->save, tempo);

-          //printf("switched: %s\n",save);
-
        }                       // if
      }
 #if DEBUG_SAVENAME
      printf("\nEnd search, %s\n", fil_complete);
 #endif
-    } while(!nom_ok);
-
+    } while (!nom_ok);
  }
-  //printf("'%s' %s %s\n",save,adr,fil);

  return 0;
 }
--- a/src/htsopt.h
+++ b/src/htsopt.h
@@ -105,7 +105,6 @@ typedef struct htsfilters htsfilters;
 struct htsfilters {
  char ***filters; /**< pointer to the +/-pattern filter array */
  int *filptr;     /**< pointer to the current filter count */
-  // int*    filter_max;
 };

 /* User callbacks chain */
@@ -440,7 +439,6 @@ struct httrackp {
  float maxconn;          /**< max connections per second */
  int waittime;           /**< scheduled start time (wall-clock seconds) */
  hts_cachemode cache;    /**< cache generation mode */
-  // int aff_progress;     // progress bar
  hts_boolean shell; /**< driven by a shell over stdin/stdout pipes */
  t_proxy proxy;     /**< proxy configuration */
  hts_savename_83
@@ -492,7 +490,6 @@ struct httrackp {
  hts_verbosedisplay verbosedisplay; /**< animated text progress display */
  String footer; /**< footer/info line injected into pages */
  int maxcache;  /**< in-memory cache backing limit (bytes) */
-  // int maxcache_anticipate; // maximum links to anticipate (upper bound)
  hts_boolean ftp_proxy;    /**< use the HTTP proxy for FTP too */
  String filelist;          /**< file listing URLs to include */
  String urllist;           /**< file listing filters to include */
@@ -539,6 +536,8 @@ struct httrackp {
                                 (--cookies-file) */
  int pause_min_ms; /**< inter-file pause lower bound, ms (0=off, #185) */
  int pause_max_ms; /**< inter-file pause upper bound, ms */
+  String why_url;   /**< URL to diagnose (--why): print the deciding filter rule
+                         and exit without crawling */
 };

 /* Running statistics for a mirror. */
@@ -549,8 +548,6 @@ typedef struct hts_stat_struct hts_stat_struct;
 struct hts_stat_struct {
  LLint HTS_TOTAL_RECV; /**< total bytes received from the network */
  LLint stat_bytes;     /**< total bytes written to disk */
-  // int HTS_TOTAL_RECV_STATE;  // status: 0 ok 1: slow down a little 2: slow
-  // down 3: a lot
  TStamp stat_timestart; /**< mirror start time */
  //
  LLint total_packed;    /**< compressed bytes received (on the wire) */
@@ -682,7 +679,6 @@ struct lien_url {
  int depth;        /**< remaining allowed depth; >0 strong, 0 weak */
  int pass2;        /**< second-pass marker; -1 means handled in background */
  char link_import; /**< imported after a move; skip the usual up/down rules */
-  // int moved;          // pointer to moved
  int retry;    /**< remaining retries */
  int testmode; /**< test only: send just a HEAD */
 };
@@ -727,9 +723,6 @@ struct lien_back {
  LLint chunk_size; /**< size of the chunk being loaded */
  LLint chunk_blocksize; /**< data size declared by the chunk */
  LLint compressed_size; /**< compressed size (stats only) */
-  //
-  // int links_index;        // to access liens[links_index]
-  //
  char info[256]; /**< status text, e.g. for FTP */
  int stop_ftp;   /**< stop flag for FTP */
  int finalized;  /**< finalized (memory optimization) */
--- a/src/htsparse.c
+++ b/src/htsparse.c
@@ -192,6 +192,48 @@ static void hts_automate_lookup(const script_automate *aut) {
  }
 }

+/* Attribute name owning the quoted value at 'quote' inside a tag, spanning
+   [name, *nend); NULL when the quote is not an attribute value. */
+static const char *dirty_attr_name(const char *quote, const char *tag_start,
+                                   const char **nend) {
+  const char *a = quote - 1;
+  while (a > tag_start && is_taborspace(*a))
+    a--;
+  if (a == tag_start || *a != '=')
+    return NULL;
+  a--;
+  while (a > tag_start && is_taborspace(*a))
+    a--;
+  *nend = a + 1;
+  while (a > tag_start && *a != '=' && *a != '\"' && *a != '\'' &&
+         !is_realspace(*a))
+    a--;
+  a++;
+  // a name starting right after '<' is the tag name, not an attribute
+  return a < *nend && a > tag_start + 1 ? a : NULL;
+}
+
+/* Accept the in-tag quoted value at 'quote' for dirty parsing? Resolves the
+   owning attribute itself (intag_startattr is unreliable mid-tag) and rejects
+   no-detect/xmlns names. */
+static hts_boolean dirty_attr_detectable(const char *quote,
+                                         const char *tag_start) {
+  const char *nend;
+  const char *name = dirty_attr_name(quote, tag_start, &nend);
+  int i;
+  if (name == NULL)
+    return HTS_FALSE;
+  for (i = 0; strnotempty(hts_nodetect[i]); i++) {
+    const int l = strfield(name, hts_nodetect[i]);
+    if (l && name + l == nend)
+      return HTS_FALSE;
+  }
+  i = strfield(name, "xmlns");
+  if (i && (name + i == nend || name[i] == ':'))
+    return HTS_FALSE;
+  return HTS_TRUE;
+}
+
 /* Advance the cursor by 'steps' bytes, feeding each to the automaton. */
 static void hts_automate_increment(const script_automate *aut, int steps) {
  while (steps > 0) {
@@ -353,16 +395,13 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
      // terminaison (" ou ') du "<body onLoad=.."
      int inscriptgen = 0;      // on est dans un code générant, ex après obj.write("..

-      //int inscript_check_comments=0, inscript_in_comments=0;    // javascript comments
      char scriptgen_q = '\0';  // caractère faisant office de guillemet (' ou ")

-      //int no_esc_utf=0;      // ne pas echapper chars > 127
      int nofollow = 0;         // ne pas scanner

      //
      int parseall_lastc = '\0';        // dernier caractère parsé pour parseall

-      //int parseall_incomment=0;   // dans un /* */ (exemple: a = /* URL */ "img.gif";)
      //
      const char *intag_start = html;
      const char *intag_name = NULL;
@@ -475,12 +514,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
          lastsaved = html;
        }
      }
-      // Detect UTF8 format
-      //if (is_unicode_utf8(r->adr, (unsigned int) r->size) == 1) {
-      //  no_esc_utf=1;
-      //} else {
-      //  no_esc_utf=0;
-      //}

      // Hack to prevent any problems with ram files of other files
      *(r->adr + r->size) = '\0';
@@ -645,8 +678,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
          ) {
          intag = 1;
          intag_ctype = 0;
-          //parseall_incomment=0;
-          //inquote=0;  // effacer quote
          intag_start = html;
          for(intag_name = html + 1; is_realspace(*intag_name); intag_name++) ;
          intag_start_valid = 1;
@@ -664,8 +695,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
              if (len > 0) {
                if (strfield(token, "content-type")) {
                  intag_ctype = 1;
-                  //NOPE-we do not convert the whole page actually
-                  //intag_start[1] = 'X';
                } else if (strfield(token, "refresh")) {
                  intag_ctype = 2;
                }
@@ -726,7 +755,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                          html_inline_safe(urlfil(), safe_fil, sizeof(safe_fil)), gmttime,
                          HTTRACK_VERSIONID, /* EOF */ NULL);
                  strcatbuff(tempo, eol);
-                  //fwrite(tempo,1,strlen(tempo),fp);
                  HT_ADD(tempo);
                }
                // Emit charset ?
@@ -754,6 +782,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
          ) {
          if (inscript_tag) {
            inscript_tag = inscript = 0;
+            // reset the automaton on exit or its state leaks into plain HTML
+            inscript_state_pos = INSCRIPT_START;
            intag = 0;
            incomment = 0;
            intag_start_valid = 0;
@@ -765,8 +795,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
            intag = 0;          //inquote=0;

            // entrée dans du javascript?
-            // on parse ICI car il se peut qu'on ait eu a parser les src=.. dedans
-            //if (!inscript) {  // sinon on est dans un obj.write("..
+            // on parse ICI car il se peut qu'on ait eu a parser les src=..
+            // dedans
            if ((intag_start_valid) && (check_tag(intag_start, "script")
                                        || check_tag(intag_start, "style")
                )
@@ -812,12 +842,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
            }
 #endif
          }
-          //}
-        }
-        //else if (*adr==34) {
-        //  inquote=(inquote?0:1);
-        //}
-        else if (intag || inscript || in_media) {       // nous sommes dans un tag/commentaire, tester si on recoit un tag
+        } else if (intag || inscript ||
+                   in_media) { // nous sommes dans un tag/commentaire, tester si
+                               // on recoit un tag
          int p_type = 0;
          int p_nocatch = 0;
          int p_searchMETAURL = 0;      // chercher ..URL=<url>
@@ -837,6 +864,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                if (*html == inscript_tag_lastc) {
                  /* sortir */
                  inscript_tag = inscript = 0;
+                  inscript_state_pos = INSCRIPT_START;
                  incomment = 0;
                  if (opt->parsedebug) {
                    HT_ADD("<@@ /inscript @@>");
@@ -1055,27 +1083,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                      if (p) {
                        if (intag_ctype == 1) {
                          p = 0;
-#if 0
-                          //if ((pos=rech_tageq(html, "content"))) {
-                          char temp[256];
-                          char *token = NULL;
-                          int len = rech_endtoken(html + pos, &token);
-
-                          if (len > 0 && len < sizeof(temp) - 2) {
-                            char *chpos;
-
-                            temp[0] = '\0';
-                            strncat(temp, token, len);
-                            if ((chpos = strstr(temp, "charset"))
-                                && (chpos = strchr(chpos, '='))
-                              ) {
-                              chpos++;
-                              while(is_space(*chpos))
-                                chpod++;
-                              //chpos
-                            }
-                          }
-#endif
                        }
                        // <META HTTP-EQUIV="Refresh" CONTENT="3;URL=http://www.example.com">
                        else if (intag_ctype == 2) {
@@ -1260,21 +1267,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                  }
                } else if (inscript) {

-#if 0
-                  /* Check // javascript comments */
-                  if (*html == 10 || *html == 13) {
-                    inscript_check_comments = 1;
-                    inscript_in_comments = 0;
-                  } else if (inscript_check_comments) {
-                    if (!is_realspace(*html)) {
-                      inscript_check_comments = 0;
-                      if (html[0] == '/' && html[1] == '/') {
-                        inscript_in_comments = 1;
-                      }
-                    }
-                  }
-#endif
-
                  /* Parse */
                  assertf(inscript_name != NULL);
                  if (*html == '/'
@@ -1287,18 +1279,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                      && inscript_locked == 0) {
                    const char *a = html;

-                    //while(is_realspace(*(--a)));
                    while(is_realspace(*a))
                      a--;
                    a--;
                    if (*a == '<') {    // sûr que c'est un tag?
                      inscript = 0;
+                      inscript_state_pos = INSCRIPT_START;
                      if (opt->parsedebug) {
                        HT_ADD("<@@ /inscript @@>");
                      }
                    }
-                  } else if (inscript_state_pos ==
-                             INSCRIPT_START /*!inscript_in_comments */ ) {
+                  } else if (inscript_state_pos == INSCRIPT_START) {
                    /*
                       Script Analyzing - different types supported:
                       foo="url"
@@ -1487,12 +1478,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
              }

            } else {            // ptr == 0
-              //p=rech_tageq(adr,"primary");    // lien primaire, yeah
              p = 0;            // No stupid tag anymore, raw link
              valid_p = 1;      // Valid even if p==0
              while((html[p] == '\r') || (html[p] == '\n'))
                p++;
-              //can_avoid_quotes=1;
              ending_p = '\r';
            }

@@ -1505,25 +1494,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
          // risque: générer de faux fichiers parazites
          // fix: ne parse plus dans les commentaires
          // ------------------------------------------------------------
-          if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 && (ptr > 0) && (!in_media) /* && (!inscript_in_comments) */ ) {  // option parsing "brut"
-            //int incomment_justquit=0;
+          if (opt->parseall && (opt->parsejava & HTSPARSE_NO_AGGRESSIVE) == 0 &&
+              (ptr > 0) && (!in_media)) { // option parsing "brut"
            if (!is_realspace(*html)) {
              int noparse = 0;

-              // Gestion des /* */
-#if 0
-              if (inscript) {
-                if (parseall_incomment) {
-                  if ((*html == '/') && (*(html - 1) == '*'))
-                    parseall_incomment = 0;
-                  incomment_justquit = 1;       // ne pas noter dernier caractère
-                } else {
-                  if ((*html == '/') && (*(html + 1) == '*'))
-                    parseall_incomment = 1;
-                }
-              } else
-                parseall_incomment = 0;
-#endif
              /* ensure automate state  0 (not in comments, quotes..) */
              if (inscript
                  && (inscript_state_pos != INSCRIPT_INQUOTE
@@ -1537,7 +1512,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {

              // recherche d'URLs
              if (!noparse) {
-                //if ((!parseall_incomment) && (!noparse)) {
                if (!p) {       // non déja trouvé
                  if (html != r->adr) {  // >1 caractère
                    // scanner les chaines
@@ -1561,15 +1535,16 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                            && (count > 0)) {
                          char c;

-                          //char* aend;
-                          //
-                          //aend=a;     // sauver début
                          a++;
                          while(is_taborspace(*a))
                            a++;
                          c = *a;
-                          if (strchr("),;>/+\r\n", c)) {        // exemple: ..img.gif";
-                            // le / est pour funct("img.gif" /* URL */);
+                          // in-tag, an attribute value ends at its quote: no
+                          // delimiter required after it (mid-tag attrs, #201)
+                          if (strchr("),;>/+\r\n", c) ||
+                              (intag && !inscript && intag_start_valid &&
+                               dirty_attr_detectable(html, intag_start))) {
+                            // '/' covers a value followed by a JS comment
                            char BIGSTK tempo[HTS_URLMAXSIZE * 2];
                            char type[256];
                            int url_ok = 0;     // url valide?
@@ -1614,10 +1589,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                              if (!invalid_url) {
                                // Un plus à la fin? Alors ne pas prendre sauf si extension ("/toto.html#"+tag)
                                if (c != '+') { // PAS de plus à la fin
-#if 0
-                                  char *a;
-#endif
-                                  // "Comparisons of scheme names MUST be case-insensitive" (RFC2616)                                  
+                                  // "Comparisons of scheme names MUST be
+                                  // case-insensitive" (RFC2616)
                                  if ((strfield(tempo, "http:"))
                                      || (strfield(tempo, "ftp:"))
 #if HTS_USEOPENSSL
@@ -1630,15 +1603,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                    if (inscript)       // sinon si pas javascript, méfiance (répertoire style base?)
                                      url_ok = 1;
                                  }
-#if 0
-                                  else if ((a = strchr(tempo, '/'))) {  // un slash: ok..
-                                    if (inscript) {     // sinon si pas javascript, méfiance (style "text/css")
-                                      if (strchr(a + 1, '/'))   // un seul / : abandon (STYLE type='text/css')
-                                        if (!strchr(tempo, ' '))        // avoid spaces (too dangerous for comments)
-                                          url_ok = 1;
-                                    }
-                                  }
-#endif
                                }
                                // Prendre si extension reconnue
                                if (!url_ok) {
@@ -1717,9 +1681,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                      }
                    }
                  }
-                }               // p == 0               
+                } // p == 0

-              }                 // not in comment
+              } // not in comment

              // plus dans un commentaire
              if (inscript_state_pos == INSCRIPT_START
@@ -1728,17 +1692,15 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
              }

            }                   // if realspace
-          }                     // if parseall
+          } // if parseall

          // ------------------------------------------------------------
          // p!=0 : on a repéré un éventuel lien
          // ------------------------------------------------------------
          //
-          if ((p > 0) || (valid_p)) {   // on a repéré un lien
-            //int lien_valide=0;
+          if ((p > 0) || (valid_p)) { // on a repéré un lien
            const char *eadr = NULL;  /* fin de l'URL */

-            //char* quote_adr=NULL;     /* adresse du ? dans l'adresse */
            int ok = 1;
            char quote = '\0';
            int quoteinscript = 0;
@@ -1761,7 +1723,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
              lastsaved = html;  // dernier écrit+1
            }
            // sauter espaces
-            // adr+=p;
            hts_automate_increment(&saut, p);
            while((is_space(*html)
                   || (inscriptgen && html[0] == '\\' && is_space(html[1])
@@ -1790,8 +1751,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
            // sauter éventuel \" ou \' javascript
            if (inscript) {     // on est dans un obj.write("..
              if (*html == '\\') {
-                if ((*(html + 1) == '\'') || (*(html + 1) == '"')) {      // \" ou \'
-                  // html+=2;    // sauter
+                if ((*(html + 1) == '\'') || (*(html + 1) == '"')) { // \" ou \'
                  hts_automate_increment(&saut, 2);
                }
              }
@@ -1866,7 +1826,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                  ok = -1;      // ne pas traiter ce lien

                if (ok > 0) {
-                  //if (*eadr!=' ') {  
                  if (is_space(*eadr)) {        // guillemets,CR, etc
                    if ((*eadr == quote && (!quoteinscript || *(eadr - 1) == '\\'))     // end quote
                        || (noquote && (*eadr == '\"' || *eadr == '\''))        // end at any quote
@@ -1912,7 +1871,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                      break;
                    }
                  }
-                  //}
                }
                eadr++;
              } while(ok == 1);
@@ -1941,14 +1899,10 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
              char BIGSTK lien[HTS_URLMAXSIZE * 2];
              int meme_adresse = 0;     // 0 par défaut pour primary

-              //char *copie_de_adr=html;
-              //char* p;
-
              // construire lien (découpage)
              if (eadr - html - 1 < HTS_URLMAXSIZE) {        // pas trop long?
                strncpy(lien, html, eadr - html - 1);
                lien[eadr - html - 1] = '\0';
-                //printf("link: %s\n",lien);          
                // supprimer les espaces
                while((lien[strlen(lien) - 1] == ' ') && (strnotempty(lien)))
                  lien[strlen(lien) - 1] = '\0';
@@ -2192,7 +2146,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                    a += 2;
                  else
                    a = lien;
-                  // while((*a) && (*a!='/') && (*a!=':')) a++;
                  a = jump_toport(a);
                  if (a) {      // port
                    int port = 0;
@@ -2200,9 +2153,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                    char *b = a + 1;

 #if HTS_USEOPENSSL
-                    // FIXME
-                    //if (strfield(adr, "https:")) {
-                    //}
 #endif
                    while(isdigit((unsigned char) *b)) {
                      port *= 10;
@@ -2293,13 +2243,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                            *(a + 1) = '\0';
                          }
                        }
-                        //char BIGSTK tempo[HTS_URLMAXSIZE*2];
-                        //strcpybuff(tempo,"http://");
-                        //strcatbuff(tempo,urladr());    // host
-                        //if (*lien!='/')
-                        //  strcatbuff(tempo,"/");
-                        //strcatbuff(tempo,lien);
-                        //strcpybuff(lien,tempo);
                      }
                    }

@@ -2328,22 +2271,17 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                      }
                      // stocker base ou codebase?
                      switch (p_type) {
-                      case 2:{
-                          //if (*lien!='/') strcatbuff(base,"/");
-                          strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
-                        }
-                        break;  // base
-                      case -2:{
-                          //if (*lien!='/') strcatbuff(codebase,"/");
-                          strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
-                        }
-                        break;  // base
+                      case 2: {
+                        strlcpybuff(base, lien, HTS_URLMAXSIZE * 2);
+                      } break; // base
+                      case -2: {
+                        strlcpybuff(codebase, lien, HTS_URLMAXSIZE * 2);
+                      } break; // base
                      }

                      hts_log_print(opt, LOG_DEBUG,
                                    "code/codebase link %s base %s", lien,
                                    base);
-                      //printf("base code: %s - %s\n",lien,base);
                    }

                  } else {
@@ -2456,7 +2394,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                // Tester si un lien doit être accepté ou refusé (wizard)
                // forbidden_url=1 : lien refusé
                // forbidden_url=0 : lien accepté
-                //if ((ptr>0) && (p_type!=2) && (p_type!=-2)) {    // tester autorisations?
                if ((p_type != 2) && (p_type != -2)) {  // tester autorisations?
                  if (!p_nocatch) {
                    if (afs.af.adr[0] != '\0') {
@@ -2506,9 +2443,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {

                      /* Calc */
                      last_adr[0] = '\0';
-                      //char last_fil[HTS_URLMAXSIZE*2]="";
-                      strcpybuff(last_adr, afs.af.adr);        // ancienne adresse
-                      //strcpybuff(last_fil,fil);    // ancien chemin
+                      strcpybuff(last_adr, afs.af.adr); // ancienne adresse
                      r_sv =
                        url_savename(&afs, &former, heap(ptr)->adr, heap(ptr)->fil, opt,
                                     sback, cache, hash, ptr,
@@ -2539,7 +2474,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                            }
                          }
                        }
-                        //import_done=1;    // c'est un import!
                        meme_adresse = 0;       // on a changé
                      }
                    } else {
@@ -2737,11 +2671,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                              if (lienrelatif(tempo, sizeof(tempo), save,
                                              relativesavename()) == 0) {
                                /* Never escape high-chars (we don't know the encoding!!) */
-                                inplace_escape_uri_utf(tempo, sizeof(tempo));  // escape with %xx
-                                //if (!no_esc_utf)
-                                //  escape_uri(tempo);     // escape with %xx
-                                //else
-                                //  escape_uri_utf(tempo);     // escape with %xx
+                                inplace_escape_uri_utf(
+                                    tempo, sizeof(tempo)); // escape with %xx
                                HT_ADD_HTMLESCAPED(tempo);      // page externe
                                if (add_url) {
                                  HT_ADD("?link=");     // page externe
@@ -2978,16 +2909,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                          // unquoted url() (CSS/JS): keep parens escaped
                          if (ending_p == ')')
                            escape_url_parens(tempo, sizeof(tempo));
-
-                          //if (!no_esc_utf)
-                          //  escape_uri(tempo);     // escape with %xx
-                          //else {
-                          //  /* No escaping at all - remaining upper chars will be escaped below */
-                          //  /* FIXME - Should be done in all local cases */
-                          //  //x_escape_html(tempo);
-                          //  //escape_uri_utf(tempo);     // FIXME - escape with %xx
-                          //  //escape_uri(tempo);     // escape with %xx
-                          //}
                        }
                        hts_log_print(opt, LOG_DEBUG,
                                      "relative link at %s build with %s and %s: %s",
@@ -3045,22 +2966,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                              HT_ADD(tempo4);   // refresh code="
                            }
                          }
-                          //lastsaved=adr;    // dernier écrit+1
                        }

                        if ((opt->getmode & HTS_GETMODE_HTML) && (ptr > 0)) {
-                          // convert to local codepage - NOT, already converted into %NN, and passed to the remote server so we do not have anything to do
-                          //if (str->page_charset_ != NULL && *str->page_charset_ != '\0') {
-                          //  char *const local_save = hts_convertStringFromUTF8(tempo, strlen(tempo), str->page_charset_);
-                          //  if (local_save != NULL) {
-                          //    strcpybuff(tempo, local_save);
-                          //    free(local_save);
-                          //  } else {
-                          //    if ((opt->debug>1) && (opt->log!=NULL)) {
-                          //      fprintf(opt->log, "Warning: could not build local charset representation of '%s' in '%s'"LF, tempo, str->page_charset_);
-                          //    }
-                          //  }
-                          //}

                          // écrire le lien modifié, relatif
                          // Note: escape all chars, even >127 (no UTF)
@@ -3084,16 +2992,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                      }
                    } // sinon le lien sera écrit normalement

-#if 0
-                    if (fexist(save)) { // le fichier existe..
-                      adr[0] = '\0';
-                      //if ((opt->debug>0) && (opt->log!=NULL)) {
-                      hts_log_print(opt, LOG_WARNING,
-                                    "Link has already been written on disk, cancelled: %s",
-                                    save);
-                    }
-#endif
-
                    /* Security check */
                    if (strlen(afs.save) >= HTS_URLMAXSIZE) {
                      afs.af.adr[0] = '\0';
@@ -3135,9 +3033,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                        /*
                           if (strnotempty(save)) {
                           if (ishtml(opt,save) == 1) {
-                           // descore_prio = 2;
                           } else {
-                           // descore_prio = 1;
                           }
                           }
                         */
@@ -3173,8 +3069,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                          // >>>> CREER LE LIEN <<<<
                          //
                          // enregistrer lien à charger
-                          //heap_top()->adr[0]=heap_top()->fil[0]=heap_top()->sav[0]='\0';
-                          // même adresse: l'objet père est l'objet père de l'actuel

                          // DEBUT ROBOTS.TXT AJOUT
                          if (!just_test_it) {
@@ -3253,7 +3147,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                            heap_top()->premier = heap(ptr)->premier;
                          else  // sinon l'objet père est le précédent lui même
                            heap_top()->premier = heap_top_index();
-                          // heap_top()->premier=ptr;

                          heap_top()->precedent = ptr;
                          // noter la priorité
@@ -3265,9 +3158,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                          heap_top()->pass2 = pass_fix;
                          heap_top()->retry = opt->retry;

-                          //strcpybuff(heap_top()->adr,adr);
-                          //strcpybuff(heap_top()->fil,fil);
-                          //strcpybuff(heap_top()->sav,save); 
                          if (!just_test_it) {
                            hts_log_print(opt, LOG_DEBUG,
                                          "OK, NOTE: %s%s -> %s",
@@ -3302,7 +3192,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
            if (eadr > html) {
              hts_automate_increment(&saut, (int) (eadr - 1 - html));
            }
-            // adr=eadr-1;  // ** sauter

            /* srcset candidate loop: skip the descriptor and comma, then
               re-enter the capture for the next URL. Backward goto, not a loop:
@@ -3332,9 +3221,9 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               inscript_state_pos = INSCRIPT_START;
               } */

-          }                     // if (p) 
+ } // if (p)

-        }                       // si '<' ou '>'
+        } // si '<' ou '>'

        // plus loin
        html++;                  // automate will be checked next loop
@@ -3404,9 +3293,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
            *stre->exit_xh_ = 1;        // exit requested
            XH_uninit;
            return -1;
-            //adr = r->adr + r->size;  // exit
          } else if (opt->state._hts_cancel == 1) {
-            // adr = r->adr + r->size;  // exit
            nofollow = 1;       // moins violent
            opt->state._hts_cancel = 0;
          }
@@ -3464,9 +3351,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
        fp = NULL;
      }
    }
-    // sauver fichier
-    //structcheck(savename());
-    //filesave(opt,r->adr,r->size,savename());

  }                             // analyse OK

@@ -3505,17 +3389,11 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
  // DEBUT rattrapage des 301,302,307..
  // ------------------------------------------------------------
  if (!error) {
-    ////////{
-    // on a chargé un fichier en plus
-    // if (!error) stat_loaded+=r.size;

    // ------------------------------------------------------------
    // Rattrapage des 301,302,307 (moved) et 412,416 - les 304 le sont dans le backing 
    // ------------------------------------------------------------
    if (HTTP_IS_REDIRECT(r->statuscode)) {
-      //if (r->adr!=NULL) {   // adr==null si fichier direct. [catch: davename normalement si cgi]
-      //int i=0;
-      // char* p;

      hts_log_print(opt, LOG_WARNING, "%s for %s%s", r->msg, urladr(), urlfil());

@@ -3542,7 +3420,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
          // A same-file alias redirect must be followed, not stubbed (#159).
          const hts_boolean same_savefile = hts_redirect_same_savefile(
              opt, urladr(), urlfil(), moved->adr, moved->fil);
-          //if (ident_url_absolute(mov_url,moved->adr,moved->fil)!=-1) {    // ok URL reconnue
          // c'est (en gros) la même URL..
          // si c'est un problème de casse dans le host c'est que le serveur est buggé
          // ("RFC says.." : host name IS case insensitive)
@@ -3582,7 +3459,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
            } /* sinon traité normalement */
          }

-          //if ((strfield2(moved->adr,urladr())!=0) && (strfield2(moved->fil,urlfil())!=0)) {  // identique à casse près
          if (get_it == 1) {
            // court-circuiter le reste du traitement
            // et reculer pour mieux sauter
@@ -3593,9 +3469,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
            error = 1;
            hts_invalidate_link(opt, ptr);  // invalidate hashtable entry
            // noter NOUVEAU lien
-            //xxc xxc
-            //  set_prio_to=0+1;  // protection if the moved URL is an html page!!
-            //xxc xxc
            {
              // calculer lien et éventuellement modifier addresse/fichier
              if (url_savename(&savedmoved, NULL,
@@ -3633,8 +3506,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
              }
            }

-            //printf("-> %s %s %s\n",liens[lien_tot-1]->adr,liens[lien_tot-1]->fil,liens[lien_tot-1]->sav);
-
            // note métaphysique: il se peut qu'il y ait un index.html et un INDEX.HTML
            // sous DOS ca marche pas très bien... mais comme je suis génial url_savename()
            // est à même de régler ce problème
@@ -3756,7 +3627,6 @@ int hts_mirror_check_moved(htsmoduleStruct * str,

      // cas où l'on peut reessayer
      switch (r->statuscode) {
-        //case -1: can_retry=1; break;
      case STATUSCODE_TIMEOUT:
        if (opt->hostcontrol) { // timeout et retry épuisés
          if ((opt->hostcontrol & HTS_HOSTCONTROL_BAN_TIMEOUT) &&
@@ -3888,7 +3758,7 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
    // FIN rattrapage des 301,302,307..
    // ------------------------------------------------------------

-  }                             // if !error
+  } // if !error

  /* Apply changes */
  ENGINE_SAVE_CONTEXT();
@@ -3955,9 +3825,6 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
          }
        }
      }
-      // On désalloue le buffer d'enregistrement des chemins créée, au cas où pendant la pause
-      // l'utilisateur ferait un rm -r après avoir effectué un tar
-      // structcheck_init(1);
      {
        FILE *fp =
          fopen(fconcat
@@ -4338,33 +4205,8 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
                    "link #%d is ready, no more on the stack, skipping: %s%s..",
                    ptr, urladr(), urlfil());

-      // prochain lien
-      // ptr++;
-
-      return 2;                 // goto jump_if_done;
-
+      return 2; // goto jump_if_done;
    }
-#if 0
-    /* FIXME - finalized HAS NO MORE THIS MEANING */
-    /* link put in cache by the backing system for memory spare - reclaim */
-    else if (back[b].finalized) {
-      assertf(back[b].r.adr == NULL);
-      /* read file in cache */
-      back[b].r =
-        cache_read_ro(opt, cache, back[b].url_adr, back[b].url_fil,
-                      back[b].url_sav, back[b].location_buffer);
-      /* ensure correct location buffer set */
-      back[b].r.location = back[b].location_buffer;
-      if (back[b].r.statuscode == STATUSCODE_INVALID) {
-        hts_log_print(opt, LOG_ERROR,
-                      "Unexpected error: %s%s not found anymore in cache",
-                      back[b].url_adr, back[b].url_fil);
-      } else {
-        hts_log_print(opt, LOG_DEBUG, "reclaim file %s%s (%d)", back[b].url_adr,
-                      back[b].url_fil, back[b].r.statuscode);
-      }
-    }
-#endif

    if (!opt->verbosedisplay) {
      if (!opt->quiet) {
@@ -4388,7 +4230,6 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
      }
      fflush(stdout);
    }
-    //}

    // ------------------------------------------------------------
    // Vérificateur d'intégrité
@@ -4556,9 +4397,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
        if (back[b].r.statuscode == STATUSCODE_INVALID && back[b].r.adr == NULL) {
          lien_back delayed_back;

-          //char BIGSTK delayed_ctype[128];
-          // delayed_ctype[0] = '\0';
-          // strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1);    // copier content-type
          back_copy_static(&back[b], &delayed_back);

          /* Delete entry */
@@ -4663,9 +4501,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
        if (b >= 0) {
          lien_back delayed_back;

-          //char BIGSTK delayed_ctype[128];
-          //delayed_ctype[0] = '\0';
-          //strncatbuff(delayed_ctype, back[b].r.contenttype, sizeof(delayed_ctype) - 1);    // copier content-type
          back_copy_static(&back[b], &delayed_back);

          /* Error */
@@ -4817,13 +4652,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
      /* 'no error page' selected or file discarded by size rules! */
      if (!opt->errpage || (in_error == STATUSCODE_TOO_BIG)) {
        /* Note: the cache 'cached_tests' system will remember this error, and we'll only issue ONE request */
-#if 0
-        /* No (3.43) - don't do that. We must not post-exclude an authorized link, because this will prevent the cache
-           system from processing it, leading to refetch it endlessly. Just accept it, and handle the error as
-           usual during parsing.
-         */
-        *forbidden_url = 1;     /* Forbidden! */
-#endif
+        /* Do not post-exclude the link here (*forbidden_url): the cache system
+           would never process it again, and it would be refetched endlessly */
        if (in_error == STATUSCODE_TOO_BIG) {
          hts_log_print(opt, LOG_INFO,
                        "link not taken because of its size (%d bytes) at %s%s",
--- a/src/htsparse.h
+++ b/src/htsparse.h
@@ -66,7 +66,6 @@ struct htsmoduleStructExtended {
  char ***filters_;
  robots_wizard *robots_;
  hash_struct *hash_;
-  //int *lien_max_;

  /* Base & codebase */
  char *base;
--- a/src/htsselftest.c
+++ b/src/htsselftest.c
@@ -579,15 +579,16 @@ static int st_filter(httrackp *opt, int argc, char **argv) {
  int matched;

  (void) opt;
-  if (argc < 2) {
+  if (argc < 1) {
    fprintf(stderr, "filter: needs a filter pattern and a string\n");
    return 1;
  }
-  /* exact-size heap copies so a sanitizer traps any over-read of the pattern */
-  str = strdupt(argv[1]);
+  /* exact-size heap copies so a sanitizer traps an over-read; a missing
+     subject means "" (not reachable as a CLI arg) */
+  str = strdupt(argc >= 2 ? argv[1] : "");
  pat = strdupt(argv[0]);
  matched = strjoker(str, pat, NULL, NULL) != NULL;
-  printf("%s does %s %s\n", argv[1], matched ? "match" : "NOT match", argv[0]);
+  printf("%s does %s %s\n", str, matched ? "match" : "NOT match", argv[0]);
  freet(str);
  freet(pat);
  return 0;
@@ -619,6 +620,26 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
  return 0;
 }

+/* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
+static int st_filterdual(httrackp *opt, int argc, char **argv) {
+  int depth = -1, verdict;
+
+  (void) opt;
+  if (argc < 3) {
+    fprintf(stderr,
+            "filterdual: needs <string1> <string2> <filter> [filter...]\n");
+    return 1;
+  }
+  verdict = fa_strjoker_dual(0, &argv[2], argc - 2, argv[0], argv[1], NULL,
+                             NULL, &depth);
+  printf("verdict=%s rule=%d\n",
+         verdict > 0   ? "allowed"
+         : verdict < 0 ? "forbidden"
+                       : "unknown",
+         depth);
+  return 0;
+}
+
 static int st_simplify(httrackp *opt, int argc, char **argv) {
  (void) opt;
  if (argc < 1) {
@@ -1094,6 +1115,33 @@ static int st_resolve(httrackp *opt, int argc, char **argv) {
  return 0;
 }

+/* Split a URL into (adr, fil), or print "error" if rejected. A second arg pads
+   the URL with that many 'a's to reach lengths a CLI arg can't. */
+static int st_identurl(httrackp *opt, int argc, char **argv) {
+  lien_adrfil af;
+  char *url;
+  size_t len, pad = 0;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "identurl: needs a URL\n");
+    return 1;
+  }
+  if (argc >= 2)
+    pad = (size_t) atoi(argv[1]);
+  len = strlen(argv[0]);
+  url = malloct(len + pad + 1);
+  memcpy(url, argv[0], len);
+  memset(url + len, 'a', pad);
+  url[len + pad] = '\0';
+  if (ident_url_absolute(url, &af) >= 0)
+    printf("adr=%s fil=%s\n", af.adr, af.fil);
+  else
+    printf("error\n");
+  freet(url);
+  return 0;
+}
+
 /* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
   no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
   registering an already-crawled link (dedup/collision paths). */
@@ -2106,6 +2154,8 @@ static const struct selftest_entry {
    {"filtersize", "<size> <string> <filter>...",
     "size-aware filter verdict (negative size = unknown/scan time)",
     st_filtersize},
+    {"filterdual", "<string1> <string2> <filter>...",
+     "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
    {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
    {"stripquery", "", "--strip-query pattern/key stripping self-test",
     st_stripquery},
@@ -2132,6 +2182,7 @@ static const struct selftest_entry {
     st_relative},
    {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
     st_resolve},
+    {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
    {"header", "<raw-header-line> ...", "response header-line parsing",
     st_header},
    {"savename", "<fil> <content-type> [key=value ...]",
--- a/src/htsserver.c
+++ b/src/htsserver.c
@@ -265,7 +265,6 @@ T_SOC smallserver_init(int *port, char *adr) {
      SOCaddr_initport(server, *port);
      if (bind(soc, &SOCaddr_sockaddr(server), SOCaddr_size(server)) == 0) {
        if (listen(soc, 10) >= 0) {
-          // SOCaddr_inetntoa(adr, 128, server2);
          strcpy(adr, h_loc);
        } else {
 #ifdef _WIN32
@@ -439,7 +438,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
        meth = 10;
      } else {
 #ifdef _DEBUG
-        // assert(FALSE);
 #endif
      }
      if (meth) {
@@ -1360,7 +1358,6 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {

            StringCat(headers, error_hdr);
            StringCat(output, error);
-            //assert(file == NULL);
          }
        }
      } else {
@@ -1388,12 +1385,10 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                  != StringLength(output)))
        ) {
 #ifdef _DEBUG
-        //assert(FALSE);
 #endif
      }
    } else {
 #ifdef _DEBUG
-      // assert(FALSE);
 #endif
    }

@@ -1540,8 +1535,7 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
            } while(strnotempty(test));
          }

-          if (!strnotempty(test)) {     // éviter doublons
-            // conv_printf(key,key);
+          if (!strnotempty(test)) { // éviter doublons
            const size_t len = strlen(intkey);
            char *const buff = (char *) malloc(len + 1);

@@ -1632,8 +1626,6 @@ static int htslang_load(char *limit_to, size_t limit_size, const char *path) {
                    intkey = "";
                } else {
                  if (loops > 0) {
-                    //err_msg += intkey;
-                    //err_msg += " ";
                  }
                }
              }
@@ -1740,13 +1732,7 @@ static void LANG_DELETE(void) {
 }

 // sélection de la langue
-static void LANG_INIT(const char *path) {
-  //CWinApp* pApp = AfxGetApp();
-  //if (pApp) {
-  /* pApp->GetProfileInt("Language","IntId",0); */
-  LANG_T(path, 0 /*pApp->GetProfileInt("Language","IntId",0) */ );
-  //}
-}
+static void LANG_INIT(const char *path) { LANG_T(path, 0); }

 static int LANG_T(const char *path, int l) {
  if (l >= 0) {
--- a/src/htstools.c
+++ b/src/htstools.c
@@ -68,16 +68,6 @@ struct find_handle_struct {
  char path[2048];
 };
 #endif
-//#ifndef HTS_DEF_FWSTRUCT_topindex_chain
-//#define HTS_DEF_FWSTRUCT_topindex_chain
-//typedef struct topindex_chain topindex_chain;
-//#endif
-//struct topindex_chain {
-//  int level;                    /* sort level */
-//  char *category;               /* category */
-//  char name[2048];              /* path */
-//  struct topindex_chain *next;  /* next element */
-//};

 /* Tools */

@@ -293,7 +283,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
  char BIGSTK newcurr_fil[HTS_URLMAXSIZE * 2], newlink[HTS_URLMAXSIZE * 2];
  char *curr;

-  //int n=0;
  char *a;
  int slash = 0;

@@ -338,7 +327,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
    if (*curr == '/')
      curr++;
    l = link;
-    //c=curr;
    // couper ce qui est commun
    while((streql(*link, *curr)) && (*link != 0)) {
      link++;
@@ -350,8 +338,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
      link--;
      curr--;
    }
-    //if (*link=='/') link++;
-    //if (*curr=='/') curr++;
  }

  // calculer la profondeur du répertoire courant et remonter
@@ -362,7 +348,6 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
  while(*a)
    if (*(a++) == '/')
      strlcatbuff(s, "../", ssize);
-  //if (strlen(s)==0) strcatbuff(s,"/");

  if (slash)
    strlcatbuff(s, "/", ssize); // keep it absolute!
--- a/src/htsweb.c
+++ b/src/htsweb.c
@@ -639,8 +639,7 @@ int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_back * b
          for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) {        // no lien
            int i = (back_index + _i) % back_max;       // commencer par le "premier" (l'actuel)

-            if (back[i].status >= 0) {  // signifie "lien actif"
-              // int ok=0;  // OPTI
+            if (back[i].status >= 0) { // signifie "lien actif"
              ok = 0;
              switch (j) {
              case 0:          // prioritaire
--- a/src/htswizard.c
+++ b/src/htswizard.c
@@ -188,12 +188,10 @@ static int hts_acceptlink_(httrackp * opt, int ptr,

  /* Doit-on traiter les non html? */
  if ((opt->getmode & HTS_GETMODE_NONHTML) == 0) { // non on ne doit pas
-    if (!ishtml(opt, fil)) {    // non il ne faut pas
-      //adr[0]='\0';    // ne pas traiter ce lien, pas traiter
+    if (!ishtml(opt, fil)) {                       // non il ne faut pas
      forbidden_url = 1;        // interdire récupération du lien
      hts_log_print(opt, LOG_DEBUG, "non-html file ignored at %s : %s", adr,
                    fil);
-
    }
  }

@@ -352,7 +350,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,

  } else {                      // adresse différente, sortir?

-    //if (!opt->wizard) {    // mode non wizard
    // doit-on traiter ce lien?.. vérifier droits de sortie
    switch ((opt->travel & HTS_TRAVEL_SCOPE_MASK)) {
    case HTS_TRAVEL_SAME_ADDRESS:
@@ -380,7 +377,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
      if ((i > 0) && (j > 0)) {
        if (!strfield2(adr + i, urladr() + j)) { // !=
          if (!opt->wizard) {                    // mode non wizard
-            // printf("refused: %s\n",adr);
            forbidden_url = 1; // pas même domaine
            hts_log_print(opt, LOG_DEBUG, "foreign domain link canceled: %s%s",
                          adr, fil);
@@ -408,7 +404,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
      if ((i > 0) && (j > 0)) {
        if (!strfield2(adr + i, urladr() + j)) { // !-
          if (!opt->wizard) {                    // mode non wizard
-            // printf("refused: %s\n",adr);
            forbidden_url = 1; // pas même .xx
            hts_log_print(opt, LOG_DEBUG,
                          "foreign location link canceled: %s%s", adr, fil);
@@ -439,7 +434,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
  // récupérer les liens à côtés d'un lien (nearlink) (nvelle pos)
  if (forbidden_url != 0 && opt->nearlink) {
    if (!ishtml(opt, fil)) {    // non html
-      //printf("ok %s%s\n",ad,fil);
      forbidden_url = 0;        // autoriser
      may_set_prio_to = 1 + 1;  // set prio to 1 (parse but skip urls) if near is the winner
      hts_log_print(opt, LOG_DEBUG, "near link authorized: %s%s", adr, fil);
@@ -530,28 +524,11 @@ static int hts_acceptlink_(httrackp * opt, int ptr,

        // filters, 0=sait pas 1=ok -1=interdit
        {
-          int jokDepth1 = 0, jokDepth2 = 0;
-          int jok1 = 0, jok2 = 0;
+          int jokDepth = 0;

-          jok1 =
-            fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, NULL, NULL,
-                        &jokDepth1);
-          jok2 =
-            fa_strjoker( /*url */ 0, _FILTERS, *_FILTERS_PTR, l, NULL, NULL,
-                        &jokDepth2);
-          if (jok2 == 0) {      // #2 doesn't know
-            jok = jok1;         // then, use #1
-            mdepth = _FILTERS[jokDepth1];
-          } else if (jok1 == 0) {       // #1 doesn't know
-            jok = jok2;         // then, use #2
-            mdepth = _FILTERS[jokDepth2];
-          } else if (jokDepth1 >= jokDepth2) {  // #1 matching rule is "after" #2, then it is prioritary
-            jok = jok1;
-            mdepth = _FILTERS[jokDepth1];
-          } else {              // #2 matching rule is "after" #1, then it is prioritary
-            jok = jok2;
-            mdepth = _FILTERS[jokDepth2];
-          }
+          jok = fa_strjoker_dual(/*url */ 0, _FILTERS, *_FILTERS_PTR, lfull, l,
+                                 NULL, NULL, &jokDepth);
+          mdepth = _FILTERS[jokDepth];
        }

        if (jok == 1) {         // autorisé
@@ -887,7 +864,6 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
        }
      }
    }
-    //adr[0]='\0';  // cancel
  }
  // -------------------- FINAL PHASE --------------------
  // Test if the "Near" test won
@@ -972,34 +948,10 @@ int hts_testlinksize(httrackp * opt, const char *adr, const char *fil, LLint siz

      // filters, 0=sait pas 1=ok -1=interdit
      {
-        int jokDepth1 = 0, jokDepth2 = 0;
-        int jok1 = 0, jok2 = 0;
-        LLint sz1 = size, sz2 = size;
-        int size_flag1 = 0, size_flag2 = 0;
-
-        jok1 =
-          fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
-                      lfull, &sz1, &size_flag1, &jokDepth1);
-        jok2 =
-          fa_strjoker( /*url */ 0, *opt->filters.filters, *opt->filters.filptr,
-                      l, &sz2, &size_flag2, &jokDepth2);
-        if (jok2 == 0) {        // #2 doesn't know
-          jok = jok1;           // then, use #1
-          sz = sz1;
-          size_flag = size_flag1;
-        } else if (jok1 == 0) { // #1 doesn't know
-          jok = jok2;           // then, use #2
-          sz = sz2;
-          size_flag = size_flag2;
-        } else if (jokDepth1 >= jokDepth2) {    // #1 matching rule is "after" #2, then it is prioritary
-          jok = jok1;
-          sz = sz1;
-          size_flag = size_flag1;
-        } else {                // #2 matching rule is "after" #1, then it is prioritary
-          jok = jok2;
-          sz = sz2;
-          size_flag = size_flag2;
-        }
+        sz = size;
+        jok = fa_strjoker_dual(/*url */ 0, *opt->filters.filters,
+                               *opt->filters.filptr, lfull, l, &sz, &size_flag,
+                               NULL);
      }

      // log
--- a/src/htszlib.h
+++ b/src/htszlib.h
@@ -36,7 +36,6 @@ Please visit our Website: http://www.httrack.com

 /* ZLib */
 #include "zlib.h"
-//#include "zutil.h"

 /* MiniZip */
 #include "minizip/zip.h"
--- a/src/httrack.c
+++ b/src/httrack.c
@@ -54,7 +54,6 @@ static int linput(FILE * fp, char *s, int max);
 #include "htswrap.h"

 /* specific definitions */
-//#include "htsbase.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -454,8 +453,7 @@ static int __cdecl htsshow_loop(t_hts_callbackarg * carg, httrackp * opt, lien_b
          for(_i = 0 + k; (_i < max(back_max * k, 1)) && (index < NStatsBuffer); _i++) {        // no lien
            int i = (back_index + _i) % back_max;       // commencer par le "premier" (l'actuel)

-            if (back[i].status >= 0) {  // signifie "lien actif"
-              // int ok=0;  // OPTI
+            if (back[i].status >= 0) { // signifie "lien actif"
              ok = 0;
              switch (j) {
              case 0:          // prioritaire
@@ -773,28 +771,7 @@ static void sig_finish(int code) {      // finir et quitter
  fprintf(stderr, "\nExit requested to engine (signal %d)\n", code);
 }

-#ifdef _WIN32
-#if 0
-static void sig_ask(int code) { // demander
-  char s[256];
-
-  signal(code, sig_term);       // quitter si encore
-  printf("\nQuit program/Interrupt/Cancel? (Q/I/C) ");
-  fflush(stdout);
-  scanf("%s", s);
-  if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
-      || (s[0] == 'q') || (s[0] == 'Q'))
-    exit(0);                    // quitter
-  else if ((s[0] == 'i') || (s[0] == 'I')) {
-    if (global_opt != NULL) {
-      // ask for stop
-      global_opt->state.stop = 1;
-    }
-  }
-  signal(code, sig_ask);        // remettre signal
-}
-#endif
-#else
+#ifndef _WIN32
 static void sig_doback(int blind);
 static void sig_back(int code) {        // ignorer et mettre en backing 
  if (global_opt != NULL && !global_opt->background_on_suspend) {
@@ -809,36 +786,6 @@ static void sig_back(int code) {        // ignorer et mettre en backing
  }
 }

-#if 0
-static void sig_ask(int code) { // demander
-  char s[256];
-
-  signal(code, sig_term);       // quitter si encore
-  printf
-    ("\nQuit program/Interrupt/Background/bLind background/Cancel? (Q/I/B/L/C) ");
-  fflush(stdout);
-  scanf("%s", s);
-  if ((s[0] == 'y') || (s[0] == 'Y') || (s[0] == 'o') || (s[0] == 'O')
-      || (s[0] == 'q') || (s[0] == 'Q'))
-    exit(0);                    // quitter
-  else if ((s[0] == 'b') || (s[0] == 'B') || (s[0] == 'a') || (s[0] == 'A'))
-    sig_doback(0);              // arrière plan
-  else if ((s[0] == 'l') || (s[0] == 'L'))
-    sig_doback(1);              // arrière plan
-  else if ((s[0] == 'i') || (s[0] == 'I')) {
-    if (global_opt != NULL) {
-      // ask for stop
-      printf("finishing pending transfers.. please wait\n");
-      global_opt->state.stop = 1;
-    }
-    signal(code, sig_ask);      // remettre signal
-  } else {
-    printf("cancel..\n");
-    signal(code, sig_ask);      // remettre signal
-  }
-}
-#endif
-
 static void sig_brpipe(int code) {      // treat if necessary
  signal(code, sig_brpipe);
 }
@@ -951,23 +898,12 @@ static void sig_leave(int code) {

 static void signal_handlers(void) {
 #ifdef _WIN32
-#if 0                           /* BUG366763 */
-  signal(SIGINT, sig_ask);      // ^C
-#else
  signal(SIGINT, sig_leave);    // ^C
-#endif
  signal(SIGTERM, sig_finish);  // kill <process>
 #else
-#if 0                           /* BUG366763 */
-  signal(SIGHUP, sig_back);     // close window
-#endif
  signal(SIGTSTP, sig_back);    // ^Z
  signal(SIGTERM, sig_finish);  // kill <process>
-#if 0                           /* BUG366763 */
-  signal(SIGINT, sig_ask);      // ^C
-#else
  signal(SIGINT, sig_leave);    // ^C
-#endif
  signal(SIGPIPE, sig_brpipe);  // broken pipe (write into non-opened socket)
  signal(SIGCHLD, sig_ignore);  // child change status
 #endif
--- a/src/proxy/proxytrack.c
+++ b/src/proxy/proxytrack.c
@@ -250,13 +250,6 @@ static int gethost(const char *hostname, SOCaddr * server) {
    struct addrinfo hints;

    memset(&hints, 0, sizeof(hints));
-#if 0
-    if (IPV6_resolver == 1)     // V4 only (for bogus V6 entries)
-      hints.ai_family = PF_INET;
-    else if (IPV6_resolver == 2)        // V6 only (for testing V6 only)
-      hints.ai_family = PF_INET6;
-    else
-#endif
    hints.ai_family = PF_UNSPEC;
    hints.ai_socktype = SOCK_STREAM;
    hints.ai_protocol = IPPROTO_TCP;
@@ -340,12 +333,7 @@ int proxytrack_main(char *proxyAddr, int proxyPort, char *icpAddr, int icpPort,
  T_SOC socICP = smallserver_init(proxyAddr, icpPort, SOCK_DGRAM);

  if (soc != INVALID_SOCKET && socICP != INVALID_SOCKET) {
-    //char url[HTS_URLMAXSIZE * 2];
-    //char method[32];
-    //char data[32768];

-    //url[0] = method[0] = data[0] = '\0';
-    //
    printf("HTTP Proxy installed on %s:%d/\n", proxyAddr, proxyPort);
    printf("ICP Proxy installed on %s:%d/\n", icpAddr, icpPort);
 #ifndef _WIN32
@@ -950,7 +938,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
    char *command;
    char *proto;
    char *surl;
-    //int directHit = 0;
    int headRequest = 0;
    int listRequest = 0;

@@ -1092,10 +1079,8 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {

      /* Post-process request */
      if (link_has_authority(surl)) {
-        if (strncasecmp
-            (surl, "http://proxytrack/",
-             sizeof("http://proxytrack/") - 1) == 0) {
-          //directHit = 1;        /* Another direct hit hack */
+        if (strncasecmp(surl, "http://proxytrack/",
+                        sizeof("http://proxytrack/") - 1) == 0) {
        }
        StringCopy(url, surl);
      } else {
@@ -1116,7 +1101,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
              toHit += 7;
            }
            /* Direct hit */
-            //directHit = 1;
            StringCopy(url, "");
            if (!link_has_authority(toHit))
              StringCat(url, "http://");
@@ -1127,7 +1111,6 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
            const char *toHit = surl + sizeof("/proxytrack/") - 1;

            /* Direct hit */
-            //directHit = 1;
            StringCopy(url, "");
            if (!link_has_authority(toHit))
              StringCat(url, "http://");
@@ -1611,10 +1594,7 @@ static int proxytrack_start_ICP(PT_Indexes indexes, T_SOC soc) {
        unsigned char Opcode = buffer[0];
        unsigned char Version = buffer[1];
        unsigned short Message_Length = READ_NET16(&buffer[2]);
-        unsigned int Request_Number = READ_NET32(&buffer[4]);   /* Session ID */
-        //unsigned int Options = READ_NET32(&buffer[8]);
-        //unsigned int Option_Data = READ_NET32(&buffer[12]);     /* ICP_FLAG_SRC_RTT */
-        //unsigned int Sender_Host_Address = READ_NET32(&buffer[16]);     /* ignored */
+        unsigned int Request_Number = READ_NET32(&buffer[4]); /* Session ID */
        unsigned char *Payload = &buffer[20];

        buffer[bufferSize] = '\0';      /* Ensure payload is NULL terminated */
--- a/src/proxy/store.c
+++ b/src/proxy/store.c
@@ -1035,7 +1035,6 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
      if (unzOpenCurrentFile(index->zFile) == Z_OK) {
        char headerBuff[8192 + 2];
        int readSizeHeader;
-        //int totalHeader = 0;
        int dataincache = 0;

        /* For BIG comments */
@@ -1074,9 +1073,8 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
              ZIP_READFIELD_STRING(line, value, "Last-Modified", r->lastmodified);      // last-modified
              ZIP_READFIELD_STRING(line, value, "Etag", r->etag);       // Etag
              ZIP_READFIELD_STRING(line, value, "Location", r->location);       // 'location' pour moved
-              ZIP_READFIELD_STRING(line, value, "Content-Disposition", r->cdispo);      // Content-disposition
-              //ZIP_READFIELD_STRING(line, value, "X-Addr", ..);            // Original address
-              //ZIP_READFIELD_STRING(line, value, "X-Fil", ..);            // Original URI filename
+              ZIP_READFIELD_STRING(line, value, "Content-Disposition",
+                                   r->cdispo); // Content-disposition
              ZIP_READFIELD_STRING(line, value, "X-Save", previous_save_);      // Original save filename
              if (line[0] != '\0') {
                int len = r->headers ? ((int) strlen(r->headers)) : 0;
@@ -1090,8 +1088,7 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
                strcat(r->headers, "\r\n");
              }
            }
-          } while(offset < readSizeHeader && !lineEof);
-          //totalHeader = offset;
+          } while (offset < readSizeHeader && !lineEof);

          /* Previous entry */
          if (previous_save_[0] != '\0') {
@@ -1202,7 +1199,6 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
                      strcpy(r->msg, "Cache Read Error : Read Data");
                    } else
                      *(r->adr + r->size) = '\0';
-                    //printf(">%s status %d\n",back[p].r->contenttype,back[p].r->statuscode);
                  } else {      // erreur
                    r->statuscode = STATUSCODE_INVALID;
                    strcpy(r->msg, "Cache Memory Error");
@@ -1484,14 +1480,12 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
              a += cache_brstr(a, firstline);
              strcpy(cache->lastmodified, firstline);
            } else {
-              // fprintf(opt->errlog,"Cache: version 1.%d not supported, ignoring current cache"LF,cache->version);
              fclose(cache->dat);
              cache->dat = NULL;
              free(use);
              use = NULL;
            }
-          } else {              // non supporté
-            // fspc(opt->errlog,"error"); fprintf(opt->errlog,"Cache: %s not supported, ignoring current cache"LF,firstline);
+          } else { // non supporté
            fclose(cache->dat);
            cache->dat = NULL;
            free(use);
@@ -1500,7 +1494,6 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
          /* */
        } else {                // Vieille version du cache
          /* */
-          // hts_log_print(opt, LOG_WARNING, "Cache: importing old cache format");
          cache->version = 0;   // cache 1.0
          strcpy(cache->lastmodified, firstline);
        }
--- a/tests/01_engine-filter.test
+++ b/tests/01_engine-filter.test
@@ -12,6 +12,11 @@ match() {
 nomatch() {
    test "$(httrack -O /dev/null -#test=filter "$1" "$2")" == "$2 does NOT match $1" || exit 1
 }
+# single-arg call means an empty subject (unreachable as a CLI arg); '*(' used
+# to force max=1 and over-read past it.
+nomatch_empty() {
+    test "$(httrack -O /dev/null -#test=filter "$1")" == " does NOT match $1" || exit 1
+}

 # bare star matches everything
 match '*' 'anything/at/all'
@@ -122,3 +127,7 @@ nomatch '*[file]end' 'foo?xend'
 nomatch '*[name]X' 'abc?X'
 match '*[file]' 'foo?x=1' # trailing query: tolerated, as for *.aspx
 match '*.aspx' 'page.aspx?y=2'
+
+# empty subject against a unique-match allow-all pattern (heap over-read regress)
+nomatch_empty '*(('
+nomatch_empty '**(('
--- a/tests/01_engine-filterdual.test
+++ b/tests/01_engine-filterdual.test
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# fa_strjoker_dual: merged verdict on the two URL forms the engine tests;
+# the match latest in the filter list wins, an unknown verdict defers.
+
+fdual() {
+    local want="$1"
+    shift
+    test "$(httrack -O /dev/null -#test=filterdual "$@")" == "$want" || exit 1
+}
+
+fdual 'verdict=unknown rule=0' zzz yyy '+a*'     # neither form matches
+fdual 'verdict=forbidden rule=0' aaa zzz '-a*'   # only form 1 matches
+fdual 'verdict=forbidden rule=0' zzz aaa '-a*'   # only form 2 matches
+fdual 'verdict=forbidden rule=1' a b '+a*' '-b*' # later rule wins (form 2)
+fdual 'verdict=forbidden rule=1' b a '+a*' '-b*' # later rule wins (form 1)
+fdual 'verdict=allowed rule=0' a aa '+a*'        # tie: form 1 wins
--- a/tests/01_engine-identurl.test
+++ b/tests/01_engine-identurl.test
@@ -0,0 +1,27 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# ident_url_absolute splits a raw URL into (adr, fil), the first parser to touch
+# it. -#test=identurl <url> [pad] prints "adr=.. fil=.." or "error"; pad appends
+# N 'a's, reaching lengths a CLI arg can't.
+
+split() {
+    test "$(httrack -O /dev/null -#test=identurl "$1")" == "$2" || exit 1
+}
+
+split 'http://www.example.com/a/b/../c.html' 'adr=www.example.com fil=/a/c.html'
+split 'ftp://ftp.example.com/pub/file.txt' 'adr=ftp://ftp.example.com fil=/pub/file.txt'
+split 'file:///etc/hostname' 'adr=file:// fil=/etc/hostname'
+
+# empty-path file:// used to read one byte past the URL (p[1] with *p=0)
+split 'file://' 'adr=file:// fil=//'
+
+# a root-relative path is copied whole into fil[HTS_URLMAXSIZE*2], so a URL at
+# the buffer size is the tightest overflow; it must be rejected, not abort.
+split_pad() {
+    test "$(httrack -O /dev/null -#test=identurl "$1" "$2")" == "$3" || exit 1
+}
+split_pad '/' 2047 'error' # 1 + 2047 = 2048 = HTS_URLMAXSIZE*2
+split_pad 'http://h/' 3000 'error'
--- a/tests/01_engine-idna.test
+++ b/tests/01_engine-idna.test
@@ -8,8 +8,8 @@ set -euo pipefail

 enc() { test "$(httrack -O /dev/null -#test=idna-encode "$1")" == "$2" || exit 1; }
 dec() { test "$(httrack -O /dev/null -#test=idna-decode "$1")" == "$2" || exit 1; }
-# crash probe: malformed ACE input must exit cleanly, not abort.
-runs() { httrack -O /dev/null -#test=idna-decode "$1" >/dev/null 2>&1 || exit 1; }
+# a malformed encode input must be rejected (empty output), not encoded or leaked.
+enc_bad() { test -z "$(httrack -O /dev/null -#test=idna-encode "$1" 2>/dev/null)" || exit 1; }

 # encode
 enc 'www.café.com' 'www.xn--caf-dma.com'
@@ -33,6 +33,9 @@ enc 'xn--already-encoded.com' 'xn--already-encoded.com'
 # an empty punycode payload decodes back to the bare xn-- label
 dec 'xn--' 'xn--'

-# malformed ACE payloads (invalid base-36, garbage) must not crash
-runs 'xn--!!!'
-runs 'xn--already-encoded.com'
+# malformed ACE payloads decode to a stable (garbage) string, not a crash
+dec 'xn--!!!' '㚯'
+dec 'xn--already-encoded.com' 'ǮalǭǮreaǭǫdy.com'
+
+# second label has a truncated UTF-8 sequence: used to leak the segment buffer.
+enc_bad "$(printf 'b\xc3\xbcchev.\xe4\xbe\x8b\xe5\xad\x62\xc3\xbccheple')"
--- a/tests/01_engine-parse.test
+++ b/tests/01_engine-parse.test
@@ -106,8 +106,6 @@ grep -Eq 'srcset="j\.gif 2x"' "$saved" ||
 # inline style attribute, with the URL unquoted, double-quoted and single-quoted
 # (the quote style is preserved on rewrite). No-detect attributes (title, alt,
 # ...) are left untouched. Asserted by rewrite (deterministic), not download.
-# data-* (#201/#203) is omitted: its detection is currently nondeterministic and
-# can't be locked yet.
 site2="$tmp/attrs"
 mkdir -p "$site2"
 for f in xl ibg ibgs cex cexd cexs tt; do gif "$site2/$f.gif"; done
@@ -352,4 +350,64 @@ found "v.webm" "$out10"
 found "subs.vtt" "$out10"
 notfound "plain.gif" "$out10"

+# Unknown-attr (data-*) URLs (#201/#203): the script automaton state must reset at
+# </script>, or detection dies for the rest of the page after the first script.
+site11="$tmp/dataattr"
+mkdir -p "$site11"
+for f in pre post mid spdata handler handler2 jsdecoy jsdecoy2 nodecoy \
+    spalt glalt spxml jtail1 jtail2 textdecoy tagdecoy phantom; do gif "$site11/$f.gif"; done
+cat >"$site11/index.html" <<EOF
+<html><body>
+<img data-pre="file://$site11/pre.gif">
+<script>var x = 1;</script>
+<img data-post="file://$site11/post.gif">
+<img data-mid="file://$site11/mid.gif" alt="mid-tag attr, no delimiter after">
+<img data-sp = "file://$site11/spdata.gif" q=1>
+<img alt="nodecoy.gif" src="pre.gif">
+<img alt = "spalt.gif" id=x>
+<img src="pre.gif"alt="glalt.gif" id=x>
+<p xmlns:bar = "spxml.gif" id=x></p>
+<div data-json='["jtail1.gif","jtail2.gif"]' q=1></div>
+<p>t = "textdecoy.gif" q</p>
+<img ="tagdecoy.gif" q>
+<a onclick='q = 1; "h1'>x</a>
+<img data-h1="file://$site11/handler.gif">
+<a onclick='w = 1; "h2>x</a>
+<img data-h2="file://$site11/handler2.gif">
+<script>var s; s = 1; "jsdecoy.gif";</script>
+<script>var s2 = "jsdecoy2.gif" x;</script>
+<script>var f = "</script>
+<img alt="y" "phantom.gif">
+</body></html>
+EOF
+out11="$tmp/dataattr-out"
+crawl "$site11/index.html" "$out11"
+saved11=$(savedhtml "$out11")
+test -n "$saved11" || ! echo "FAIL: saved dataattr page not found" || exit 1
+grep -Fq 'data-pre="pre.gif"' "$saved11" ||
+    ! echo "FAIL #201: data-* URL before any script not detected/rewritten" || exit 1
+grep -Fq 'data-post="post.gif"' "$saved11" ||
+    ! echo "FAIL #201: data-* URL after a script not detected (state leak)" || exit 1
+grep -Fq 'data-mid="mid.gif"' "$saved11" ||
+    ! echo "FAIL #201: mid-tag data-* URL not detected" || exit 1
+grep -Fq 'data-sp = "spdata.gif"' "$saved11" ||
+    ! echo "FAIL #201: spaced-= data-* URL not detected" || exit 1
+found "handler.gif" "$out11"  # automaton reset at the handler-terminator exit
+found "handler2.gif" "$out11" # ... and at the '>' exit of an unterminated handler
+# a JS string not preceded by =/(/, is still ignored after the reset
+notfound "jsdecoy.gif" "$out11"
+# in-script, the strict follower gate still applies (intag is 1 in script bodies)
+notfound "jsdecoy2.gif" "$out11"
+# no-detect attrs stay exempt, incl. spaced-'=' and glued-attr forms
+notfound "nodecoy.gif" "$out11"
+notfound "spalt.gif" "$out11"
+notfound "glalt.gif" "$out11"
+notfound "spxml.gif" "$out11"
+# not '='-preceded: comma-list tails and out-of-tag text tokens stay ignored
+notfound "jtail2.gif" "$out11"
+notfound "textdecoy.gif" "$out11"
+notfound "tagdecoy.gif" "$out11" # a '=' glued to the tag name is not an attr
+# a </script> inside a JS string must not cause phantom fetches later
+notfound "phantom.gif" "$out11"
+
 exit 0
--- a/tests/40_local-why.test
+++ b/tests/40_local-why.test
@@ -0,0 +1,35 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# --why: report which +/- filter rule decides for a URL, without crawling.
+
+tmpdir="$(mktemp -d)"
+trap 'rm -rf "$tmpdir"' EXIT
+
+why() {
+    local want="$1" out
+    shift
+    out="$(httrack -O "$tmpdir/p" -q "$@" |
+        grep -E 'accepted|rejected|no filter|unable')"
+    test "$out" == "$want" || {
+        echo "want: $want"
+        echo "got : $out"
+        exit 1
+    }
+}
+
+base=http://www.example.com
+why "$base/x.zip: rejected by rule #2 (-*.zip)" \
+    --why "$base/x.zip" "$base/" '+*.png' '-*.zip'
+why "$base/a.png: accepted by rule #1 (+*.png)" \
+    --why "$base/a.png" "$base/" '+*.png' '-*.zip'
+why "$base/i.html: no filter rule matches; the wizard decides (same-site links are followed by default)" \
+    --why "$base/i.html" "$base/" '+*.png' '-*.zip'
+# scheme-less input gets the same http:// default as primary URLs
+why "$base/x.zip: rejected by rule #2 (-*.zip)" \
+    --why "www.example.com/x.zip" "$base/" '+*.png' '-*.zip'
+# the rule latest in the list decides
+why "$base/x.zip: accepted by rule #3 (+*x.zip)" \
+    --why "$base/x.zip" "$base/" '+*.png' '-*.zip' '+*x.zip'
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -35,11 +35,13 @@ TESTS = \
 	01_engine-entities.test \
 	01_engine-filelist.test \
 	01_engine-filter.test \
+	01_engine-filterdual.test \
 	01_engine-ftp-line.test \
 	01_engine-ftp-userpass.test \
 	01_engine-hashtable.test \
 	01_engine-header.test \
 	01_engine-idna.test \
+	01_engine-identurl.test \
 	01_engine-escape-room.test \
 	01_engine-inplace-escape.test \
 	01_engine-java.test \
@@ -104,6 +106,7 @@ TESTS = \
 	36_local-bigcrawl.test \
 	37_local-cache-outage.test \
 	38_local-update-304.test \
-	39_local-delayed-cancel.test
+	39_local-delayed-cancel.test \
+	40_local-why.test

 CLEANFILES = check-network_sh.cache
Author	SHA1	Message	Date
Xavier Roche	53a257448f	Add --why to explain which filter rule accepts or rejects a URL (#505 ) The top user-confusion class is filters silently not doing what their author expects. --why <url>, given the rest of the crawl command line, prints the deciding rule ("rejected by rule #2 (-*.zip)") or reports that no rule matches, then exits without crawling. It reuses the real option parsing and filter seeding end to end, and evaluates through the new fa_strjoker_dual(), which also replaces the two hand-rolled copies of the dual-form verdict merge in htswizard.c. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-07 20:09:25 +02:00
Xavier Roche	8ba3cb6c03	Enable _FORTIFY_SOURCE and -fstack-protector-strong (#504 ) The rest of the OpenSSF hardening set (stack-clash, cf-protection, relro/now, noexecstack, PIE, format-security) is already enabled; this fills the two gaps. Bundle autoconf-archive's AX_ADD_FORTIFY_SOURCE (serial 10: tries =3, falls back to =2, no-ops when the toolchain predefines it); distro copies can be too old (Debian trixie ships serial 4, =2 only). Skip fortify in -fsanitize builds, whose interceptors want the unfortified calls. Upgrade -fstack-protector to -strong, keeping the plain form as fallback. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-07 20:04:09 +02:00
Xavier Roche	b9afe755bd	Add a CodeQL C analysis workflow (#503 ) Scan on push to master, PRs, and a weekly cron, with the security-extended query suite; findings land in the code-scanning dashboard. Manual build mode so CodeQL traces the real autotools build. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-07 20:03:28 +02:00
Xavier Roche	407916e3f6	Add libFuzzer harnesses for the pure hostile-input parsers (#502 ) New fuzz/ tree with a libFuzzer harness per pure (buf,size) parser that ingests network-sourced bytes: the charset/UTF-8/IDNA codecs, the HTML <meta> charset scan, HTML entity and URL percent decoders, the wildcard filter matcher, and the URL splitter. Gated behind --enable-fuzzers (requires clang, forces --disable-shared so clang's static sanitizer runtimes link); a normal build is untouched. A CI job builds the harnesses under ASan+UBSan+LeakSanitizer and replays each seed corpus. Replay, not open-ended mutation, keeps the job deterministic and avoids strjoker's catastrophic backtracking (a data-dependent exponential in the wildcard matcher, tracked separately). Maintainers run the timed mutation mode locally for discovery; OSS-Fuzz is the next step. The corpora carry the reproducers for the four parser bugs these harnesses found, so the job also pins those regressions. It depends on the fixes in the preceding commit; merge that first. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-07 18:24:53 +02:00
Xavier Roche	15021cd470	Fix four fuzzer-found over-reads/leak in the filter, URL, and IDNA parsers (#499 ) libFuzzer harnesses over the pure hostile-input parsers surfaced four pre-existing defects, each reachable from crawled content: - strjoker (htsfilters.c): a unique-match pattern like (( forces max=1 even when the subject is empty, so the matcher recurses one byte past the subject's NUL. Clamp max to the subject length. - ident_url_absolute (htslib.c): a bare file:// URL leaves the path pointer on the terminating NUL, and p[1] then reads past it. Short-circuit on p == 0. - ident_url_absolute (htslib.c): the length guard bounds only the host, so a short-host/long-path URL overruns fil[HTS_URLMAXSIZE*2] and trips the htssafe abort (a would-be overflow). Reject an over-long URL up front. - hts_convertStringUTF8ToIDNA (htscharset.c): a malformed UTF-8 sequence in a multi-label host aborts the per-segment encode without freeing the segment integer buffer. Free it on the error path. Tests: new identurl self-test drives ident_url_absolute (file:// and an internally-built over-long URL, since the CLI caps arguments); filter gains empty-subject cases; idna gains a malformed-encode probe. The two over-reads and the abort are negative-controlled (they trap under ASan / abort on the unpatched engine); the IDNA leak's deterministic control is the LeakSanitizer fuzz job in the follow-up fuzzing PR. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-07 18:14:16 +02:00
Xavier Roche	5f7045c3a4	Remove dead code: #if 0 blocks and commented-out statements (#498 ) * Remove dead #if 0 blocks and their orphaned satellites All #if 0 code that has been disabled for years goes away: the sig_ask interactive signal handlers, check_rate, HTS_TOTAL_RECV_CHECK, the make_empty_index no-op body (function removed with its only call site), the htscharset sample main(), the dead FTP CWD sequence, and assorted disabled fragments in the parser and engine. Blocks with a live #else keep that branch. The htsweb.c DEBUG toggle stays: its #else arm is the live definition and the macro is used. The htsparse.c 3.43 note about not post-excluding authorized links is kept as a plain comment; commented-out declarations and calls that only served the removed blocks go with them. French comments rewrapped by the formatter were translated in passing. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * Remove commented-out code across the engine Sweep of ~650 lines of //-disabled statements, declarations, and calls (plus three /*-wrapped dead blocks in htscoremain.c and htscore.c and the MFC pApp remnant in htsserver.c). Prose comments stay, including labels that describe live code; usage examples and config-knob docs (HTS_TRACE_MALLOC, the htsweb.c DEBUG toggle) were kept. Insertions are clang-format rewraps of live lines that became touched by the deletions. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> --------- Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-06 21:36:02 +02:00
Xavier Roche	f66ec195aa	ParseAll URL detection dies after the first script and misses mid-tag attributes (#497 ) * Aggressive parser dies for the rest of the page after the first script The script automaton state (inscript_state_pos) is reset when a script or style element is entered, but never when it exits. The dirty-parser character tracker (parseall_lastc) only advances while that state is INSCRIPT_START, without checking inscript, so whatever state the automaton holds at </script> freezes for all following HTML. The '/' of </script> itself is fed to the automaton before the exit branch runs, so even a clean script deterministically parks it in INSCRIPT_SLASH: URL detection in unknown attributes (data-, content) goes dead after the first script on the page (#201, #203). A </script> inside a JS string or comment freezes a quote state with parseall_lastc stuck on '=', turning stray quoted tokens into phantom fetches instead. Reset the state at the three live exit sites, mirroring the entry resets. Regression test proven to fail on the unfixed parser. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> Quoted attribute values mid-tag were invisible to the aggressive parser Even with the automaton unfrozen, a quoted string only qualified when its next non-space character was one of ),;>/+ or end-of-line: in HTML terms, only the last attribute of a tag (or line) could be detected. The mid-tag attributes of #201 (data-gifsrc="x.gif" data-poster=...) were structurally unreachable, making detection depend on attribute order and source formatting. In a tag outside any script, a quoted value ends at its closing quote, so waive the follower requirement there. The arm resolves the owning attribute name itself and declines for no-detect/xmlns names and non-attribute quotes: the intag_startattr lookup is unreliable mid-tag (repointed at every in-tag whitespace, stale on glued attributes), and adversarial review showed a plain intag arm fetching alt/xmlns values that master suppressed. Script and event-handler contexts keep the strict gate. Tests pin each automaton-exit reset (handler fixtures freeze lastc on ';' so they stay live), the nodetect/xmlns bypass shapes, the frozen- quote phantom-fetch mode, and the '='-resolution decoys. Runtime audits in httrack-works/issue-201-203/. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> --------- Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-06 08:42:44 +02:00
Xavier Roche	7928db5729	Cancelling a crawl mid delayed-type-check orphans .delayed placeholders (#496 ) * Cancelling a crawl mid delayed-type-check orphaned .delayed placeholders hts_wait_delayed() patches url_sav to the final name while a direct-to-disk transfer may still be writing to the .delayed placeholder path (bound when a back_wait tick lands between the slot's unlock and the patch, e.g. url_savename's post-stop test branch pumping sockets). From then on every cleanup keyed on IS_DELAYED_EXT(url_sav) is blind to the on-disk name and a cancelled mirror leaves X.N.delayed partials behind (#483). Move the placeholder to the final name right before the patch (back_delayed_rename, reopening a mid-transfer stream in append mode), and drop it in the cache-miss re-add path where a cancel between back_maydelete() and the re-add dropped its last reference. The .delayed leftover audit in local-crawl.sh now runs for every crawl (--skip-delayed-audit is gone), and 39_local-delayed-cancel exercises the cancel window: 4/10 runs failing on master, 10/10 clean with the fix; 0 orphans in 35 direct-harness cancelled crawls against ~1 in 6 before. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * review: never unlink a pre-existing target on a failed placeholder move Windows rename() fails when the target exists; unlinking newname there would destroy a previous run's completed file during an update crawl. Drop only what the slot owns: the placeholder if the rename failed, the moved partial if the reopen did. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * review: comment conciseness pass Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> --------- Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-05 18:10:55 +02:00
				`@@ -0,0 +1 @@`
				`<html><head><meta charset="utf-8"></head><body>x</body></html>`
				`@@ -0,0 +1 @@`
				`<meta http-equiv="Content-Type" content="text/html; charset=iso-8859-1">`
				`@@ -0,0 +1 @@`
				`http://user:pass@www.example.com:8080/a/b/../c/./d.html?q=1#frag`
				`@@ -0,0 +1 @@`
				ftpumŠ[/../e0O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._f9O_i../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/.../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W//O/../f9O_i<5F>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/i<>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_im<69>W/.¿ù../O/../9O_imÙ<6D>Š<EFBFBD>W/../_/../../e0O/../f9O_./f9O_i<5F>W/../O/../90O/./e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_imÙ<6D>Š<EFBFBD>W/../O/.._i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9O_i/../O/../f9O_i<5F>W/../O/../90O/../itpumÙ/ftpumŠ[/../e0O/../itpumÙ<6D>Š[/../O/../f9O_i<5F>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/../f9OmÙ<6D>Š<EFBFBD>W/../O/../9O_imÙ<6D>Š<EFBFBD>W/../O/..O_imÙ<6D>Š<EFBFBD>W/../O/../../e0O/9O_imÙ<6D>Š<EFBFBD>W/../O/../