Merge origin/master into fix-630-pathlog-utf8

Resolve tests/Makefile.am: keep 66 (#632), 67 (#633) and this branch's 69.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/windows-build.yml

@@ -149,25 +149,36 @@ jobs:
           export HTTPS_SUPPORT=yes BROTLI_ENABLED=yes ZSTD_ENABLED=yes
           export LC_ALL=C.UTF-8
 
-          # A wedged crawl must not eat the job's whole timeout budget.
-          watchdog=""
-          command -v timeout >/dev/null && watchdog="timeout 600"
+          # A wedged crawl must not eat the job's timeout budget. timeout(1)'s
+          # signals can't reap a native httrack.exe (MSYS signals don't reach it),
+          # so a hang orphaned processes that starved the runner; run_with_timeout
+          # TerminateProcess-es the whole tree. 600s clears the slowest multi-pass
+          # crawl (a few passes at --max-time=120 each).
+          . ./testlib.sh
 
           pass=0 fail=0 skip=0 failed="" skipped=""
           for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
-                   *_local-*.test 13_crawl_proxy_https.test; do
+                   *_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
+                   60_crawl-log-salvage.test; do
             rc=0
-            # shellcheck disable=SC2086
-            $watchdog bash "$t" >"$t.log" 2>&1 || rc=$?
+            run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
             case "$rc" in
               0) pass=$((pass + 1)); echo "PASS $t" ;;
               77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
+              124)
+                fail=$((fail + 1)) failed="$failed $t"
+                echo "FAIL $t (timed out, tree killed)"
+                # Re-running a wedge traced would just hang again: salvage the
+                # killed crawl's own logs into the artifact instead.
+                dump_crawl_logs >>"$t.log"
+                tail -n 25 "$t.log" | sed 's/^/      /'
+                ;;
               *)
                 fail=$((fail + 1)) failed="$failed $t"
                 echo "FAIL $t (exit $rc)"
                 # These assert with `test "$(...)" == "..." || exit 1`, which
-                # says nothing at all on failure. Re-run traced.
-                bash -x "$t" >>"$t.log" 2>&1 || true
+                # says nothing at all on failure. Re-run traced, still bounded.
+                run_with_timeout 600 bash -x "$t" >>"$t.log" 2>&1 || true
                 tail -n 25 "$t.log" | sed 's/^/      /'
                 ;;
             esac

.gitignore

@@ -43,3 +43,6 @@ Makefile
 
 # Python bytecode (tests/local-server.py).
 __pycache__/
+
+# Per-checkout Claude Code rules (symlink into a local sandbox).
+/CLAUDE.local.md

AGENTS.md

@@ -67,8 +67,9 @@ Before pushing, and when reviewing others, don't skim for bugs:
 - **Co-Authored-By is mandatory for AI-assisted commits.** Carry a
   `Co-Authored-By:` trailer naming the assistant. Attribute there, never in a
   PR-body footer.
-- PRs land as a merge commit; every commit on the branch goes onto master, so
-  keep each commit message clean and meaningful.
+- PRs are squash-merged: one commit per PR lands on master, built from the PR
+  title and description, so those are what the history keeps. The branch's
+  intermediate commits are not preserved.
 
 ## PR descriptions
 - Plain concise prose; lead with what changed and why. No What/Why/How template.

CLAUDE.local.md

@@ -1 +0,0 @@
-/home/roche/git/httrack-works/CLAUDE.httrack.local.md

CONTRIBUTING.md

@@ -6,8 +6,8 @@ with an AI assistant? The operational checklist is [AGENTS.md](AGENTS.md).
 ## Pull requests
 
 - One change per PR. Small diffs merge fast.
-- PRs land as a merge commit, so the branch's commits go onto master as-is: keep
-  each commit message clean and explain *why*.
+- PRs are squash-merged: one commit per PR goes onto master, built from the PR
+  title and description, so those are the history. Explain *why* there.
 - Be terse in the PR title and description: name the problem, not the fix, don't
   restate the diff, and calibrate length to the change.
 - Add or update tests for engine changes (`tests/`), and keep CI green.

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.13], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,11 +29,13 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
-# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
-# so it stays soname .so.3; bump revision.
+# 3:5:0: 3.49.13 leaves every installed struct layout and the exported symbol set
+# untouched vs 3.49.12; the 2 GB (_stat64) and UTF-8 export widenings are _WIN32-only,
+# and INTsys widened on POSIX (#600) but sits in no installed struct. Stays soname
+# .so.3; bump revision. x32 alone changes layout (LLint was 4 bytes there, #524), and
+# is not a release architecture.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:4:0"
+VERSION_INFO="3:5:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 

debian/changelog

@@ -1,3 +1,18 @@
+httrack (3.49.13-1) unstable; urgency=medium
+
+  * New upstream release: SOCKS5 and CONNECT proxy support, brotli and zstd
+    content decoding, --update data-loss fixes, and hardening of several
+    network-facing parsers; full list in history.txt.
+  * Add libbrotli-dev and libzstd-dev to Build-Depends for the new content
+    codings.
+  * libhttrack3 and libhttrack-dev no longer ship libhtsjava.so.3*: the
+    obsolete Java-applet parser was removed upstream. It was a dlopen plugin
+    with no reverse dependencies, so this is a plain file drop on upgrade;
+    libhttrack.so.3 itself is unchanged and needs no rename.
+  * Drop the htsjava attribution from debian/copyright.
+
+ -- Xavier Roche <xavier@debian.org>  Fri, 17 Jul 2026 10:49:52 +0200
+
 httrack (3.49.12-1) unstable; urgency=medium
 
   * New upstream release: security and crawl-correctness fixes (remote stack

history.txt

@@ -4,6 +4,31 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
+3.49-13
++ New: SOCKS5 proxy support, with scheme-aware -P URLs (socks5://, socks5h://, connect://) and plain HTTP tunneled through a CONNECT-only proxy (#563, #564)
++ New: decode brotli and zstd content codings, advertised over TLS only as browsers do (#556)
++ New: webhttrack exposes the engine options added since 3.49-2, among them --cookies-file, --pause and --strip-query (#587)
++ Fixed: files of 2 GB or more were mishandled on Windows and on every 32-bit build (#569)
++ Fixed: --update destroyed a good local copy when the re-fetch returned an HTTP error, was aborted by -M/-E, failed to decode, or came in short (#176, #521, #557, #562)
++ Fixed: a self-redirect cookie wall was dropped instead of being re-fetched with the cookie (#15)
++ Fixed: a stalled TLS handshake ignored --timeout, and synchronous DNS resolution could wedge a crawl past --max-time (#607, #613)
++ Fixed: -M metered saved bytes rather than received volume, and overshot its cap under a slow server (#77, #520)
++ Fixed: several network-facing overflows and denial-of-service paths in the Content-Range, chunked-transfer, cookie, filter and ProxyTrack cache parsers
++ Fixed: a failed connect did not fall back to the next address on Windows (#579)
++ Fixed: -P took an out-of-range port as a garbage port, and scanned past an IPv6 literal's closing bracket (#598, #602)
++ Fixed: reject a port outside 1..65535 wherever one is parsed (a crawled URL, the htsserver and proxytrack listen arguments, an ftp:// URL), instead of letting a bare sscanf wrap a huge value into a plausible port and silently use it (#614, #626)
++ Fixed: a configured proxy still resolved and dialed the origin itself (#592)
++ Fixed: ~/ in the -O base path was never expanded (#270)
++ Fixed: a non-ASCII -O output path was double-encoded on Windows once argv became UTF-8 (#621)
++ Fixed: files under a non-ASCII project path were saved to a mangled directory on Windows (#217)
++ Fixed: --build-top-index (-%i) and --protocol (-@i) were taken for the -i continue flag, wiping the URL list and exiting on the usage screen (#615)
++ Fixed: webhttrack ignored LC_ALL/LC_MESSAGES and picked the wrong Chinese and Portuguese (#95)
++ Fixed: webhttrack wrote its base path and httrack.ini to the filesystem root when $HOME was empty (#625)
++ Fixed: crawls on a non-default port were slowed by a per-request pre-resolve (#181)
++ Changed: Windows builds moved to Visual Studio 2022 and OpenSSL 3.x, the VS2008 project files are retired, and the binaries carry a version resource
++ Changed: removed the obsolete Java-applet (.class) parser and the dead SWF module remnants
++ Changed: multiple internal hardening, test and CI improvements (Windows and macOS crawl suites, HTML-parser fuzzing, parallel make check)
+
 3.49-12
 + New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
 + Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)

html/server/option10.html

@@ -103,6 +103,7 @@ ${LANG_PROXYTYPE}
 			title='${html:LANG_PROXYTYPETIP}' onMouseOver="info('${html:LANG_PROXYTYPETIP}'); return true" onMouseOut="info(' '); return true"
 >
 <option value=""${ztest:proxytype: selected:}>HTTP</option>
+<option value="2"${test:proxytype::: selected}>HTTP (CONNECT tunnel)</option>
 <option value="1"${test:proxytype:: selected}>SOCKS5</option>
 </select>
 <br><br>

html/server/step4.html

@@ -192,7 +192,7 @@ ${do:end-if}
 	${test:logtype:::--extra-log:--debug-log}
 	${test:index:--index=0:}
 	${test:index2:--search-index=0:--search-index}
-	${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype:socks5\3A//}${do:end-if}${do:output-mode:html}${prox}${test:prox:\3A}${portprox}${test:prox:"}
+	${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype::socks5:connect}${test:proxytype:\3A//}${do:end-if}${do:output-mode:html}${prox}${test:prox:\3A}${portprox}${test:prox:"}
 	${test:ftpprox:--httpproxy-ftp=0:--httpproxy-ftp}
 </textarea>
 

lang.def

@@ -1009,7 +1009,7 @@ A fatal error has occurred during this mirror
 LANG_PROXYTYPE
 Proxy type:
 LANG_PROXYTYPETIP
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
 LANG_COOKIEFILE
 Load cookies from file:
 LANG_COOKIEFILETIP

lang.indexes

@@ -23,7 +23,7 @@ pt:7
 ro:25
 ru:8
 sk:20
-si:24
+sl:24
 sv:17
 tr:10
 uk:22

lang/Bulgarian.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Ôàòàëíà ãðåøêà ïðè ñúçäàâàíåòî íà òîçè îãëåäàëåí ñàéò
 Proxy type:
 Òèï íà ïðîêñè:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Ïðîòîêîë íà ïðîêñè: HTTP èëè SOCKS5 (ñòàíäàðòíèÿò ïîðò íà SOCKS5 å 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Ïðîòîêîë íà ïðîêñè. HTTP: ñòàíäàðòíî ïðîêñè. HTTP (CONNECT òóíåë): èçïðàùà âñÿêà çàÿâêà ïðåç CONNECT òóíåë, çà ïðîêñè ïîääúðæàùè ñàìî CONNECT êàòî HTTPTunnelPort íà Tor. SOCKS5: ïîðò ïî ïîäðàçáèðàíå 1080.
 Load cookies from file:
 Çàðåæäàíå íà 'áèñêâèòêè' îò ôàéë:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Castellano.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Ha ocurrido un error fatal durante esta copia
 Proxy type:
 Tipo de proxy:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protocolo del proxy: HTTP o SOCKS5 (el puerto SOCKS5 predeterminado es 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protocolo del proxy. HTTP: proxy estándar. HTTP (túnel CONNECT): envía cada petición a través de un túnel CONNECT, para proxys que solo admiten CONNECT como el HTTPTunnelPort de Tor. SOCKS5: puerto predeterminado 1080.
 Load cookies from file:
 Cargar cookies desde un archivo:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Cesky.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 
 Proxy type:
 Typ proxy:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protokol proxy: HTTP nebo SOCKS5 (výchozí port SOCKS5 je 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protokol proxy. HTTP: standardní proxy. HTTP (tunel CONNECT): odešle každý požadavek pøes tunel CONNECT, pro proxy podporující jen CONNECT jako HTTPTunnelPort v Toru. SOCKS5: výchozí port 1080.
 Load cookies from file:
 Naèíst cookies ze souboru:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Chinese-BIG5.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 這鏡像發生了不可回復的錯誤
 Proxy type:
 proxy 類型:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-proxy 協定: HTTP 或 SOCKS5 (SOCKS5 的預設連接埠為 1080)。
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+代理協定。HTTP：標準代理。HTTP（CONNECT 隧道）：透過 CONNECT 隧道傳送每個請求，用於僅支援 CONNECT 的代理，例如 Tor 的 HTTPTunnelPort。SOCKS5：預設連接埠 1080。
 Load cookies from file:
 從檔案載入 cookies:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Chinese-Simplified.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 
 Proxy type:
 代理类型:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-代理协议: HTTP 或 SOCKS5 (SOCKS5 的默认端口为 1080)。
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+代理协议。HTTP：标准代理。HTTP（CONNECT 隧道）：通过 CONNECT 隧道发送每个请求，用于仅支持 CONNECT 的代理，例如 Tor 的 HTTPTunnelPort。SOCKS5：默认端口 1080。
 Load cookies from file:
 从文件加载 cookies:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Croatian.txt

@@ -932,8 +932,8 @@ A fatal error has occurred during this mirror
 Tijekom ovog zrcaljenja je nastala fatalna pogreška
 Proxy type:
 Vrsta posrednika:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protokol posrednika: HTTP ili SOCKS5 (zadani port za SOCKS5 je 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protokol posrednika. HTTP: standardni posrednik. HTTP (CONNECT tunel): ¹alje svaki zahtjev kroz CONNECT tunel, za posrednike koji podr¾avaju samo CONNECT poput Torovog HTTPTunnelPorta. SOCKS5: zadani port 1080.
 Load cookies from file:
 Uèitati kolaèiæe iz datoteke:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Dansk.txt

@@ -978,8 +978,8 @@ HTTrack: could not save profile for '%s'!
 HTTrack: kunne ikke gemme profil for '%s'!
 Proxy type:
 Proxytype:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Proxyprotokol: HTTP eller SOCKS5 (SOCKS5 bruger som standard port 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Proxyprotokol. HTTP: standardproxy. HTTP (CONNECT-tunnel): sender hver forespørgsel gennem en CONNECT-tunnel, til proxyer der kun understøtter CONNECT som Tors HTTPTunnelPort. SOCKS5: standardport 1080.
 Load cookies from file:
 Indlæs cookies fra fil:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Deutsch.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Fataler Fehler während der Webseiten-Kopie
 Proxy type:
 Proxy-Typ:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Proxy-Protokoll: HTTP oder SOCKS5 (Standardport von SOCKS5 ist 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Proxy-Protokoll. HTTP: normaler Proxy. HTTP (CONNECT-Tunnel): sendet jede Anfrage durch einen CONNECT-Tunnel, für Proxys, die nur CONNECT unterstützen, wie Tors HTTPTunnelPort. SOCKS5: Standardport 1080.
 Load cookies from file:
 Cookies aus Datei laden:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Eesti.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 
 Proxy type:
 Proxy tüüp:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Proxy protokoll: HTTP või SOCKS5 (SOCKS5 vaikeport on 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Puhverserveri protokoll. HTTP: tavaline puhverserver. HTTP (CONNECT-tunnel): saadab iga päringu läbi CONNECT-tunneli, ainult CONNECT-it toetavate puhverserverite jaoks nagu Tori HTTPTunnelPort. SOCKS5: vaikeport 1080.
 Load cookies from file:
 Lae küpsised failist:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/English.txt

@@ -978,8 +978,8 @@ HTTrack: could not save profile for '%s'!
 HTTrack: could not save profile for '%s'!
 Proxy type:
 Proxy type:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
 Load cookies from file:
 Load cookies from file:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Finnish.txt

@@ -930,11 +930,10 @@ Server terminated
 Palvelin lopetettu
 A fatal error has occurred during this mirror
 Tällä peilillä tapahtui vakava virhe
-
 Proxy type:
 Välityspalvelimen tyyppi:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Välityspalvelimen protokolla: HTTP tai SOCKS5 (SOCKS5:n oletusportti on 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Välityspalvelimen protokolla. HTTP: tavallinen välityspalvelin. HTTP (CONNECT-tunneli): lähettää jokaisen pyynnön CONNECT-tunnelin kautta, vain CONNECT-yhteyksiä tukeville välityspalvelimille kuten Torin HTTPTunnelPort. SOCKS5: oletusportti 1080.
 Load cookies from file:
 Lataa evästeet tiedostosta:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Francais.txt

@@ -978,8 +978,8 @@ Default referer URL
 Champ referer par défaut
 Proxy type:
 Type de proxy :
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protocole du proxy : HTTP ou SOCKS5 (le port SOCKS5 par défaut est 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protocole du proxy. HTTP : proxy standard. HTTP (tunnel CONNECT) : envoie chaque requête via un tunnel CONNECT, pour les proxys acceptant uniquement CONNECT comme le HTTPTunnelPort de Tor. SOCKS5 : port par défaut 1080.
 Load cookies from file:
 Charger les cookies depuis un fichier :
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Greek.txt

@@ -932,8 +932,8 @@ A fatal error has occurred during this mirror
 Ένα καταστροφικό σφάλμα προκλήθηκε κατά την αντιγραφή αυτού του τόπου
 Proxy type:
 Τύπος proxy:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Πρωτόκολλο proxy: HTTP ή SOCKS5 (η προεπιλεγμένη θύρα του SOCKS5 είναι 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Πρωτόκολλο διαμεσολαβητή. HTTP: τυπικός διαμεσολαβητής. HTTP (σήραγγα CONNECT): στέλνει κάθε αίτημα μέσω σήραγγας CONNECT, για διαμεσολαβητές που υποστηρίζουν μόνο CONNECT όπως το HTTPTunnelPort του Tor. SOCKS5: προεπιλεγμένη θύρα 1080.
 Load cookies from file:
 Φόρτωση cookies από αρχείο:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Italiano.txt

@@ -374,7 +374,7 @@ Create error logging and report files
 Crea file di log per segnalare errori e informazioni
 Generate DOS 8-3 filenames ONLY
 Genera solo nomi di file in formato 8.3
-Generate ISO 9660 filenames ONLY for CDROM medias
+Generate ISO9660 filenames ONLY for CDROM medias
 Genera nomi di file in formato ISO9660 per i CDROM
 Do not create HTML error pages
 Non generare pagine d'errore HTML
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Si è verificato un errore fatale durante la copia
 Proxy type:
 Tipo di proxy:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protocollo del proxy: HTTP o SOCKS5 (la porta SOCKS5 predefinita è 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protocollo del proxy. HTTP: proxy standard. HTTP (tunnel CONNECT): invia ogni richiesta attraverso un tunnel CONNECT, per proxy che supportano solo CONNECT come l'HTTPTunnelPort di Tor. SOCKS5: porta predefinita 1080.
 Load cookies from file:
 Carica i cookie da un file:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Japanese.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 
 Proxy type:
 プロキシの種類:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-プロキシのプロトコル: HTTP または SOCKS5 (SOCKS5 の既定ポートは 1080)。
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+プロキシのプロトコル。HTTP: 標準のプロキシ。HTTP (CONNECT トンネル): すべてのリクエストを CONNECT トンネル経由で送信します。Tor の HTTPTunnelPort のような CONNECT のみのプロキシ用です。SOCKS5: 既定ポート 1080。
 Load cookies from file:
 クッキーをファイルから読み込む:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Macedonian.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Íàñòàíà ôàòàëíà ãðåøêà ïðè îâî¼ mirror
 Proxy type:
 ÂØß ÝÐ ßàÞÚáØ:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-¿àÞâÞÚÞÛ ÝÐ ßàÞÚáØ: HTTP ØÛØ SOCKS5 (áâÐÝÔÐàÔÝÐâÐ ßÞàâÐ ÝÐ SOCKS5 Õ 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+¿àÞâÞÚÞÛ ÝÐ ßàÞÚáØ. HTTP: áâÐÝÔÐàÔÕÝ ßàÞÚáØ. HTTP (âãÝÕÛ CONNECT): ÓÞ ØáßàÐüÐ áÕÚÞÕ ÑÐàÐúÕ ßàÕÚã âãÝÕÛ CONNECT, ×Ð ßàÞÚáØ èâÞ ßÞÔÔàÖãÒÐÐâ áÐÜÞ CONNECT ÚÐÚÞ HTTPTunnelPort ÝÐ Tor. SOCKS5: áâÐÝÔÐàÔÝÐ ßÞàâÐ 1080.
 Load cookies from file:
 ²çØâÐø ÚÞÛÐçØúÐ ÞÔ ÔÐâÞâÕÚÐ:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Magyar.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Végzetes hiba történt a tükrözés közben
 Proxy type:
 Proxy típusa:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Proxy protokoll: HTTP vagy SOCKS5 (a SOCKS5 alapértelmezett portja 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Proxy protokoll. HTTP: szabványos proxy. HTTP (CONNECT alagút): minden kérést CONNECT alagúton keresztül küld, a csak CONNECT-et támogató proxykhoz, mint a Tor HTTPTunnelPortja. SOCKS5: alapértelmezett port 1080.
 Load cookies from file:
 Sütik betöltése fájlból:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Nederlands.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Een fatale fout is opgetreden tijdens deze spiegeling
 Proxy type:
 Proxytype:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Proxyprotocol: HTTP of SOCKS5 (standaardpoort van SOCKS5 is 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Proxyprotocol. HTTP: standaardproxy. HTTP (CONNECT-tunnel): stuurt elke aanvraag via een CONNECT-tunnel, voor proxy's die alleen CONNECT ondersteunen zoals Tors HTTPTunnelPort. SOCKS5: standaardpoort 1080.
 Load cookies from file:
 Cookies uit bestand laden:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Norsk.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 
 Proxy type:
 Proxytype:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Proxyprotokoll: HTTP eller SOCKS5 (SOCKS5 bruker som standard port 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Proxyprotokoll. HTTP: standard proxy. HTTP (CONNECT-tunnel): sender hver forespørsel gjennom en CONNECT-tunnel, for proxyer som bare støtter CONNECT som Tors HTTPTunnelPort. SOCKS5: standardport 1080.
 Load cookies from file:
 Last inn cookies fra fil:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Polski.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Podczas tworzenia lustra wydarzyl sie fatalny blad.
 Proxy type:
 Typ proxy:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protokó³ proxy: HTTP lub SOCKS5 (domy¶lny port SOCKS5 to 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protokó³ proxy. HTTP: standardowe proxy. HTTP (tunel CONNECT): wysy³a ka¿de ¿±danie przez tunel CONNECT, dla proxy obs³uguj±cych tylko CONNECT, jak HTTPTunnelPort w Torze. SOCKS5: port domy¶lny 1080.
 Load cookies from file:
 Wczytaj ciasteczka z pliku:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Portugues-Brasil.txt

@@ -918,7 +918,7 @@ normal\nextended\ndebug
 normal\nextendido\ncorrigir
 Download web site(s)\nDownload web site(s) + questions\nGet individual files\nDownload all sites in pages (multiple mirror)\nTest links in pages (bookmark test)\n* Continue interrupted download\n* Update existing download
 Copiar site(s) da Web\nCopiar site(s) interativos da Web (perguntas)\nReceber arquivos específicos\nCopiar todas as páginas do site (alternação múltipla)\nTestar links nas páginas (testar indicador)\n* Retomar download interrompido\n* Atualizar download existente
-Relative URL / Absolute URL (default)\nAbsolute URL / Absolute URL\nAbsolute URI / Absolute URL\nOriginal URL / Original URL
+Relative URI / Absolute URL (default)\nAbsolute URL / Absolute URL\nAbsolute URI / Absolute URL\nOriginal URL / Original URL
 URI Relativa / URL Absoluta (padrão)\nURL Absoluta / URL Absoluta\nURI Absoluta / URL Absoluta\nURL Original / URL Original
 Open Source offline browser
 Abrir origem offline no navegador
@@ -978,8 +978,8 @@ HTTrack: could not save profile for '%s'!
 HTTrack: não foi possível salvar o perfil para '%s'!
 Proxy type:
 Tipo de proxy:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protocolo do proxy: HTTP ou SOCKS5 (a porta SOCKS5 padrão é 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protocolo do proxy. HTTP: proxy padrão. HTTP (túnel CONNECT): envia cada requisição através de um túnel CONNECT, para proxies que só suportam CONNECT como o HTTPTunnelPort do Tor. SOCKS5: porta padrão 1080.
 Load cookies from file:
 Carregar cookies de um arquivo:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Portugues.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 
 Proxy type:
 Tipo de proxy:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protocolo do proxy: HTTP ou SOCKS5 (a porta SOCKS5 predefinida é 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protocolo do proxy. HTTP: proxy padrão. HTTP (túnel CONNECT): envia cada pedido através de um túnel CONNECT, para proxies que só suportam CONNECT como o HTTPTunnelPort do Tor. SOCKS5: porta predefinida 1080.
 Load cookies from file:
 Carregar cookies de um ficheiro:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Romanian.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 A survenit o eroare fatală în timpul acestei clonări.
 Proxy type:
 Tip proxy:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protocol proxy: HTTP sau SOCKS5 (portul SOCKS5 implicit este 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protocol proxy. HTTP: proxy standard. HTTP (tunel CONNECT): trimite fiecare cerere printr-un tunel CONNECT, pentru proxy care accepta doar CONNECT precum HTTPTunnelPort al lui Tor. SOCKS5: port implicit 1080.
 Load cookies from file:
 Încarca cookies dintr-un fisier:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Russian.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Âî âðåìÿ òåêóùåé çàêà÷êè ïðîèçîøëà ôàòàëüíàÿ îøèáêà
 Proxy type:
 Òèï ïðîêñè:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Ïðîòîêîë ïðîêñè: HTTP èëè SOCKS5 (ïî óìîë÷àíèþ SOCKS5 èñïîëüçóåò ïîðò 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Ïðîòîêîë ïðîêñè. HTTP: îáû÷íûé ïðîêñè. HTTP (òóííåëü CONNECT): îòïðàâëÿåò êàæäûé çàïðîñ ÷åðåç òóííåëü CONNECT, äëÿ ïðîêñè, ïîääåðæèâàþùèõ òîëüêî CONNECT, íàïðèìåð HTTPTunnelPort â Tor. SOCKS5: ïîðò ïî óìîë÷àíèþ 1080.
 Load cookies from file:
 Çàãðóçèòü cookies èç ôàéëà:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Slovak.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 
 Proxy type:
 Typ proxy:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protokol proxy: HTTP alebo SOCKS5 (predvolený port SOCKS5 je 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protokol proxy. HTTP: ¹tandardné proxy. HTTP (tunel CONNECT): odo¹le ka¾dú po¾iadavku cez tunel CONNECT, pre proxy podporujúce len CONNECT ako HTTPTunnelPort v Tore. SOCKS5: predvolený port 1080.
 Load cookies from file:
 Naèíta» cookies zo súboru:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Slovenian.txt

@@ -3,7 +3,7 @@ Slovenian
 LANGUAGE_FILE
 Slovenian
 LANGUAGE_ISO
-si
+sl
 LANGUAGE_AUTHOR
 Jadran Rudec,iur.\r\njrudec@email.si \r\n
 LANGUAGE_CHARSET
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 
 Proxy type:
 Vrsta proxyja:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Protokol proxy: HTTP ali SOCKS5 (privzeta vrata SOCKS5 so 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Protokol proxy. HTTP: standardni proxy. HTTP (tunel CONNECT): poslje vsako zahtevo skozi tunel CONNECT, za posrednike, ki podpirajo samo CONNECT, kot je HTTPTunnelPort v Toru. SOCKS5: privzeta vrata 1080.
 Load cookies from file:
 Nalozi piskotke iz datoteke:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Svenska.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 
 Proxy type:
 Proxytyp:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Proxyprotokoll: HTTP eller SOCKS5 (SOCKS5 använder som standard port 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Proxyprotokoll. HTTP: standardproxy. HTTP (CONNECT-tunnel): skickar varje begäran genom en CONNECT-tunnel, för proxyservrar som bara stöder CONNECT som Tors HTTPTunnelPort. SOCKS5: standardport 1080.
 Load cookies from file:
 Läs in cookies från fil:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Turkish.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Bu yansýlama iþlemi sýrasýnda ölümcül bir hata oluþtu
 Proxy type:
 Proxy türü:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Proxy protokolü: HTTP veya SOCKS5 (SOCKS5 varsayýlan portu 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Proxy protokolü. HTTP: standart proxy. HTTP (CONNECT tüneli): her isteði bir CONNECT tüneli üzerinden gönderir; Tor'un HTTPTunnelPort gibi yalnýzca CONNECT destekleyen proxy'ler için. SOCKS5: varsayýlan baðlantý noktasý 1080.
 Load cookies from file:
 Çerezleri dosyadan yükle:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Ukrainian.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 
 Proxy type:
 Òèï ïðîêñ³:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Ïðîòîêîë ïðîêñ³: HTTP àáî SOCKS5 (çà çàìîâ÷óâàííÿì SOCKS5 âèêîðèñòîâóº ïîðò 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Ïðîòîêîë ïðîêñ³. HTTP: ñòàíäàðòíèé ïðîêñ³. HTTP (òóíåëü CONNECT): íàäñèëຠêîæåí çàïèò ÷åðåç òóíåëü CONNECT, äëÿ ïðîêñ³, ùî ï³äòðèìóþòü ëèøå CONNECT, ÿê-îò HTTPTunnelPort ó Tor. SOCKS5: ïîðò çà çàìîâ÷óâàííÿì 1080.
 Load cookies from file:
 Çàâàíòàæèòè cookies ç ôàéëó:
 Preload cookies from a Netscape cookies.txt file before crawling.

lang/Uzbek.txt

@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
 Joriy ko’chirish vaqtida jiddiy xatolik yuz berdi
 Proxy type:
 Proksi turi:
-Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
-Proksi protokoli: HTTP yoki SOCKS5 (SOCKS5 ning standart porti 1080).
+Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
+Proksi protokoli. HTTP: oddiy proksi. HTTP (CONNECT tuneli): har bir so'rovni CONNECT tuneli orqali yuboradi; faqat CONNECT'ni qo'llab-quvvatlaydigan proksilar uchun, masalan Tor HTTPTunnelPort. SOCKS5: standart port 1080.
 Load cookies from file:
 Fayldan cookies yuklamoq:
 Preload cookies from a Netscape cookies.txt file before crawling.

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "14 July 2026" "httrack website copier"
+.TH httrack 1 "16 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -135,7 +135,7 @@ continue an interrupted mirror using the cache (\-\-continue)
 mirror ALL links located in the first level pages (mirror links) (\-\-mirrorlinks)
 .SS Proxy options:
 .IP \-P
-proxy use (\-P [socks5://][user:pass@]proxy:port) (\-\-proxy <param>)
+proxy use (\-P [socks5://|connect://][user:pass@]proxy:port) (\-\-proxy <param>)
 .IP \-%f
 *use proxy for ftp (f0 don't use) (\-\-httpproxy\-ftp[=N])
 .IP \-%b
@@ -165,7 +165,7 @@ pause transfer if N bytes reached, and wait until lock file is deleted (\-\-max\
 .IP \-cN
 number of multiple connections (*c8) (\-\-sockets[=N])
 .IP \-TN
-timeout, number of seconds after a non\-responding link is shutdown (\-\-timeout[=N])
+timeout, number of seconds after a non\-responding link is shutdown; also bounds host name resolution (\-\-timeout[=N])
 .IP \-RN
 number of retries, in case of timeout or non\-fatal errors (*R1) (\-\-retries[=N])
 .IP \-JN

src/Makefile.am

@@ -46,9 +46,11 @@ htsserver_LDFLAGS = $(AM_LDFLAGS) $(LDFLAGS_PIE)
 
 lib_LTLIBRARIES = libhttrack.la
 
-htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h
+htsserver_SOURCES = htsserver.c htsserver.h htsweb.c htsweb.h \
+	htsurlport.c htsurlport.h
 proxytrack_SOURCES = proxy/main.c \
 	proxy/proxytrack.c proxy/store.c \
+	htsurlport.c htsurlport.h \
 	coucal/coucal.c htsmd5.c md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c
 
@@ -58,7 +60,7 @@ whttrackrun_SCRIPTS = webhttrack
 libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htscache_selftest.c htsdns_selftest.c htsselftest.c \
 	htscatchurl.c htsfilters.c htsftp.c htshash.c coucal/coucal.c \
-	htshelp.c htslib.c htscoremain.c \
+	htshelp.c htslib.c htsurlport.c htscoremain.c \
 	htsname.c htsrobots.c htstools.c htswizard.c \
 	htsalias.c htsthread.c htsindex.c htsbauth.c \
 	htsmd5.c htscodec.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
@@ -69,7 +71,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htsbasenet.h htsbauth.h htscache.h htscache_selftest.h htsdns_selftest.h htsselftest.h htscatchurl.h  \
 	htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h  \
 	htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
-	htshelp.h htsindex.h htslib.h htsmd5.h \
+	htshelp.h htsindex.h htslib.h htsurlport.h htsmd5.h \
 	htsmodules.h htsname.h htsnet.h htssniff.h \
 	htsopt.h htsrobots.h htsthread.h  \
 	htstools.h htswizard.h htswrap.h htscodec.h htsproxy.h htszlib.h  \

src/htsalias.c

@@ -567,24 +567,29 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
   return 0;
 }
 
-/* Get home directory, '.' if failed */
+/* Get home directory, '.' if unset or empty */
 /* example: /home/smith */
 const char *hts_gethome(void) {
   const char *home = getenv("HOME");
 
-  if (home)
-    return home;
-  else
-    return ".";
+  /* An empty $HOME would expand ~/foo into the absolute /foo */
+  return strnotempty(home) ? home : ".";
 }
 
-/* Convert ~/foo into /home/smith/foo */
+/* Convert ~/foo into /home/smith/foo (~user/ left alone: no getpwnam here) */
 void expand_home(String * str) {
-  if (StringSub(*str, 1) == '~') {
+  if (StringNotEmpty(*str) && StringSub(*str, 0) == '~' &&
+      (StringLength(*str) == 1 || StringSub(*str, 1) == '/')) {
     char BIGSTK tempo[HTS_URLMAXSIZE * 2];
+    const char *const home = hts_gethome();
+    const size_t homelen = strlen(home);
+    const size_t taillen = StringLength(*str) - 1;
 
-    strcpybuff(tempo, hts_gethome());
-    strcatbuff(tempo, StringBuff(*str) + 1);
-    StringCopy(*str, tempo);
+    /* Leave untouched rather than abort() in strcatbuff on a huge $HOME */
+    if (taillen < sizeof(tempo) && homelen < sizeof(tempo) - taillen) {
+      strcpybuff(tempo, home);
+      strcatbuff(tempo, StringBuff(*str) + 1);
+      StringCopy(*str, tempo);
+    }
   }
 }

src/htsback.c

@@ -2696,9 +2696,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         int dispo = 0;
 
         // probe the resolved address list once per fresh connect (cache hit:
-        // the host was resolved when this connect was opened)
+        // the host was resolved when this connect was opened). Not under a
+        // proxy: the socket dials the proxy, so resolving the origin here leaks
+        // its DNS and lets a proxy-connect failure fall back to dialing it
+        // direct.
         if (cf->addr_count < 0 && back[i].r.soc != INVALID_SOCKET &&
-            !back[i].r.is_file) {
+            !back[i].r.is_file && !back[i].r.req.proxy.active) {
           SOCaddr scratch[HTS_MAXADDRNUM];
 
           cf->addr_count = hts_dns_resolve_all(opt, back[i].url_adr, scratch,
@@ -2754,6 +2757,28 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
             }
           }
 
+          // plain http tunneled through a CONNECT-only proxy (#564)
+          if (back[i].r.req.proxy.active &&
+              hts_proxy_is_connect(back[i].r.req.proxy.name) &&
+              !back[i].r.keep_alive
+#if HTS_USEOPENSSL
+              && !back[i].r.ssl
+#endif
+          ) {
+            const int timeout = back[i].timeout > 0 ? back[i].timeout : 30;
+
+            if (!http_proxy_tunnel(opt, &back[i].r, back[i].url_adr, timeout)) {
+              if (!strnotempty(back[i].r.msg))
+                strcpybuff(back[i].r.msg, "proxy CONNECT failed");
+              deletehttp(&back[i].r);
+              back[i].r.soc = INVALID_SOCKET;
+              back[i].r.statuscode = STATUSCODE_NON_FATAL;
+              back[i].status = STATUS_READY;
+              back_set_finished(sback, i);
+              continue;
+            }
+          }
+
 #if HTS_USEOPENSSL
           /* SSL mode */
           if (back[i].r.ssl) {
@@ -2791,6 +2816,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 if (SSL_set_fd(back[i].r.ssl_con, (int) back[i].r.soc) == 1) {
                   SSL_set_connect_state(back[i].r.ssl_con);
                   back[i].status = STATUS_SSL_WAIT_HANDSHAKE;   /* handshake wait */
+                  // the handshake gets its own timeout window, as connect does
+                  if (back[i].timeout > 0)
+                    back[i].timeout_refresh = time_local();
                 } else
                   back[i].r.statuscode = STATUSCODE_SSL_HANDSHAKE;
               } else
@@ -2850,6 +2878,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       }
 #if HTS_USEOPENSSL
       else if (back[i].status == STATUS_SSL_WAIT_HANDSHAKE) {       // wait for SSL handshake
+        // a peer that never speaks TLS must be reaped by --timeout too (#607)
+        if (!gestion_timeout)
+          if (back[i].timeout > 0)
+            gestion_timeout = 1;
+
         /* SSL mode */
         if (back[i].r.ssl) {
           int conn_code;
@@ -4240,6 +4273,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 strcpybuff(back[i].r.msg, "Connect Time Out");
               else if (back[i].status == STATUS_WAIT_DNS)
                 strcpybuff(back[i].r.msg, "DNS Time Out");
+              else if (back[i].status == STATUS_SSL_WAIT_HANDSHAKE)
+                strcpybuff(back[i].r.msg, "SSL/TLS Handshake Time Out");
               else
                 strcpybuff(back[i].r.msg, "Receive Time Out");
               back[i].status = STATUS_READY;    // terminé

src/htscoremain.c

@@ -113,6 +113,7 @@ HTSEXT_API int hts_main(int argc, char **argv) {
 }
 
 static int hts_main_internal(int argc, char **argv, httrackp * opt);
+static hts_boolean cmdl_shortopt_has(const char *s, char c);
 
 // Main, récupère les paramètres et appelle le robot
 HTSEXT_API int hts_main2(int argc, char **argv, httrackp * opt) {
@@ -304,12 +305,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                      hts_get_version_info(opt));
               return 0;
             } else {
-              if (strncmp(tmp_argv[0], "--", 2)) {      /* pas */
-                if ((strchr(tmp_argv[0], 'q') != NULL))
-                  opt->quiet = 1;       // ne pas poser de questions! (nohup par exemple)
-                if ((strchr(tmp_argv[0], 'i') != NULL)) {       // doit.log!
-                  argv_url = -1;        /* forcer */
-                  opt->quiet = 1;
+              if (strncmp(tmp_argv[0], "--", 2)) { /* not a long option */
+                if (cmdl_shortopt_has(tmp_argv[0], 'q'))
+                  opt->quiet = HTS_TRUE; // never ask questions (nohup)
+                if (cmdl_shortopt_has(tmp_argv[0], 'i')) { // doit.log!
+                  argv_url = -1;
+                  opt->quiet = HTS_TRUE;
                 }
               } else if (strcmp(tmp_argv[0] + 2, "quiet") == 0) {
                 opt->quiet = 1; // ne pas poser de questions! (nohup par exemple)
@@ -424,25 +425,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
 
       }                         // for
 
-      // Convert path to UTF-8
-#ifdef _WIN32
-      {
-        char *const path =
-          hts_convertStringSystemToUTF8(StringBuff(opt->path_html),
-                                        (int) StringLength(opt->path_html));
-        if (path != NULL) {
-          StringCopy(opt->path_html_utf8, path);
-          free(path);
-        } else {
-          StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html),
-                      StringLength(opt->path_html));
-        }
-      }
-#else
-      // Assume UTF-8 filesystem.
+      // path_html is already UTF-8 (argv is UTF-8 on Windows via
+      // hts_argv_utf8), so no re-encoding.
       StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html),
                   StringLength(opt->path_html));
-#endif
 
       /* if doit.log exists, or if new URL(s) defined, 
          then DO NOT load standard config files */
@@ -2382,30 +2368,28 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       opt->errlog = stderr;
     } else if (httrack_logmode >= 2) {
       // deux fichiers log
-      structcheck(StringBuff(opt->path_log));
-      if (fexist
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt")))
-        remove(fconcat
-               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"));
-      if (fexist
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt")))
-        remove(fconcat
-               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"));
+      // path_log holds UTF-8 bytes (argv is UTF-8): the ANSI file calls would
+      // read them as the codepage and drop the logs into a mangled twin (#630).
+      structcheck_utf8(StringBuff(opt->path_log));
+      if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                              StringBuff(opt->path_log), "hts-log.txt")))
+        UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                       StringBuff(opt->path_log), "hts-log.txt"));
+      if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                              StringBuff(opt->path_log), "hts-err.txt")))
+        UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                       StringBuff(opt->path_log), "hts-err.txt"));
 
       /* Check FS directory structure created */
-      structcheck(StringBuff(opt->path_log));
+      structcheck_utf8(StringBuff(opt->path_log));
 
-      opt->log =
-        fopen(fconcat
-              (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"),
-              "w");
+      opt->log = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                               StringBuff(opt->path_log), "hts-log.txt"),
+                       "w");
       if (httrack_logmode == 2)
-        opt->errlog =
-          fopen(fconcat
-                (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"),
-                "w");
+        opt->errlog = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                    StringBuff(opt->path_log), "hts-err.txt"),
+                            "w");
       else
         opt->errlog = opt->log;
       if (opt->log == NULL) {
@@ -2485,9 +2469,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
          }
          } */
 
-      // vérifier existence de la structure
-      structcheck(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "/"));
-      structcheck(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "/"));
+      // vérifier existence de la structure (path_html/path_log are UTF-8, use
+      // the UTF-8 mkdir path)
+      structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                               StringBuff(opt->path_html), "/"));
+      structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                               StringBuff(opt->path_log), "/"));
 
       // reprise/update
       if (opt->cache) {
@@ -2807,6 +2794,32 @@ int check_path(String * s, char *defaultname) {
   return return_value;
 }
 
+/* Does the short-option cluster s carry c from the main option set (-i, -iC2,
+   -%Mi)? Walked as the parser does below: %, &, @ and # each take the letter
+   after them into another set, so the i of -%i is not the main-set -i. */
+static hts_boolean cmdl_shortopt_has(const char *s, char c) {
+  const char *com;
+
+  if (s[0] != '-' || s[1] == '-')
+    return HTS_FALSE;
+  for (com = s + 1; *com != '\0'; com++) {
+    switch (*com) {
+    case '%':
+    case '&':
+    case '@':
+    case '#':
+      if (*(com + 1) != '\0')
+        com++; /* skip the other set's letter */
+      break;
+    default:
+      if (*com == c)
+        return HTS_TRUE;
+      break;
+    }
+  }
+  return HTS_FALSE;
+}
+
 // détermine si l'argument est une option
 int cmdl_opt(char *s) {
   if (s[0] == '-') {            // c'est peut être une option

src/htsdns_selftest.c

@@ -63,9 +63,14 @@ typedef struct mock_host {
   int gai_err; /* non-zero: getaddrinfo returns this */
   int naddr;
   mock_addr addr[6];
-  int calls; /* times the backend resolved this host */
+  int calls;   /* times the backend resolved this host */
+  int slow_ms; /* non-zero: block this long, as a black-hole resolver would */
 } mock_host;
 
+/* Long enough to outlast the 1s --timeout the bounded resolve is checked
+   against, short enough to keep the self-test quick. */
+#define MOCK_SLOW_MS 3000
+
 static mock_host mock_hosts[] = {
     {"v4only.test", 0, 1, {{AF_INET, {1, 2, 3, 4}}}, 0},
     {"v6only.test", 0, 1, {{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 1}}}, 0},
@@ -95,8 +100,15 @@ static mock_host mock_hosts[] = {
       {AF_INET, {10, 0, 0, 6}}},
      0},
     {"nodns.test", EAI_NONAME, 0, {{0}}, 0},
+    /* resolves, but only well after --timeout: the #606 wedge */
+    {"slow.test", 0, 1, {{AF_INET, {127, 0, 0, 9}}}, 0, MOCK_SLOW_MS},
 };
 
+/* Serializes mock_host bookkeeping: a timed-out resolve is abandoned, so its
+   worker is still inside the backend while the test reads the counters. */
+static htsmutex mock_lock = HTSMUTEX_INIT;
+static int mock_finished = 0; /* backend calls that have returned */
+
 static mock_host *mock_find(const char *name) {
   for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++) {
     if (strcmp(mock_hosts[i].name, name) == 0)
@@ -106,8 +118,34 @@ static mock_host *mock_find(const char *name) {
 }
 
 static void mock_reset_calls(void) {
+  hts_mutexlock(&mock_lock);
   for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++)
     mock_hosts[i].calls = 0;
+  mock_finished = 0;
+  hts_mutexrelease(&mock_lock);
+}
+
+static int mock_read_calls(const char *name) {
+  int calls;
+
+  hts_mutexlock(&mock_lock);
+  calls = mock_find(name)->calls;
+  hts_mutexrelease(&mock_lock);
+  return calls;
+}
+
+/* Wait for n backend calls to return, ordering their writes against ours. */
+static void mock_wait_finished(int n) {
+  for (;;) {
+    hts_boolean done;
+
+    hts_mutexlock(&mock_lock);
+    done = (mock_finished >= n) ? HTS_TRUE : HTS_FALSE;
+    hts_mutexrelease(&mock_lock);
+    if (done)
+      break;
+    Sleep(10);
+  }
 }
 
 /* Build one addrinfo node owning its sockaddr (freed by mock_freeaddrinfo). */
@@ -133,10 +171,10 @@ static struct addrinfo *mock_mkai(const mock_addr *a) {
   return ai;
 }
 
-static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
-                                              const char *service,
-                                              const struct addrinfo *hints,
-                                              struct addrinfo **res) {
+static int HTS_RESOLVER_CALL mock_getaddrinfo_(const char *node,
+                                               const char *service,
+                                               const struct addrinfo *hints,
+                                               struct addrinfo **res) {
   mock_host *const h = mock_find(node);
   const int want = (hints != NULL) ? hints->ai_family : PF_UNSPEC;
   struct addrinfo *head = NULL, *tail = NULL;
@@ -145,7 +183,11 @@ static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
   *res = NULL;
   if (h == NULL)
     return EAI_NONAME;
+  hts_mutexlock(&mock_lock);
   h->calls++; /* a real backend hit; a cached host skips this */
+  hts_mutexrelease(&mock_lock);
+  if (h->slow_ms != 0)
+    Sleep(h->slow_ms);
   if (h->gai_err != 0)
     return h->gai_err;
   for (int i = 0; i < h->naddr; i++) {
@@ -165,6 +207,18 @@ static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
   return 0;
 }
 
+static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
+                                              const char *service,
+                                              const struct addrinfo *hints,
+                                              struct addrinfo **res) {
+  const int ret = mock_getaddrinfo_(node, service, hints, res);
+
+  hts_mutexlock(&mock_lock);
+  mock_finished++;
+  hts_mutexrelease(&mock_lock);
+  return ret;
+}
+
 static void HTS_RESOLVER_CALL mock_freeaddrinfo(struct addrinfo *res) {
   while (res != NULL) {
     struct addrinfo *const next = res->ai_next;
@@ -348,6 +402,50 @@ int dns_selftests(httrackp *opt) {
       deletesoc(s);
   }
 
+  /* A URL port outside 1..65535 must refuse the link, not fold into range and
+     connect elsewhere (#614). *addr_count discriminates: 0 only if refused
+     before the resolve, still 2 for one merely truncated or defaulted. */
+  {
+    /* an empty "dual.test:" means the default port (WHATWG, curl): keep it */
+    static const char *const good[] = {"dual.test:1",    "dual.test:80",
+                                       "dual.test:8080", "dual.test:65535",
+                                       "dual.test:080",  "dual.test:"};
+    /* 65616 and 4294967376 are load-bearing: both wrap to a plausible 80 */
+    static const char *const bad[] = {
+        "dual.test:0",          "dual.test:65536",      "dual.test:65616",
+        "dual.test:99999",      "dual.test:2147483648", "dual.test:4294967296",
+        "dual.test:4294967376", "dual.test:-1",         "dual.test:-23437",
+        "dual.test:80x",        "dual.test:+80",        "dual.test: 80",
+        "dual.test:0x50"};
+    size_t k;
+
+    for (k = 0; k < sizeof(good) / sizeof(good[0]); k++) {
+      htsblk r;
+      int count = -1;
+      T_SOC s;
+
+      hts_init_htsblk(&r);
+      s = newhttp_addr(opt, good[k], &r, -1, 0, 0, &count);
+      CHECK(count == 2); /* accepted: reached the resolve */
+      if (s != INVALID_SOCKET)
+        deletesoc(s);
+    }
+
+    for (k = 0; k < sizeof(bad) / sizeof(bad[0]); k++) {
+      htsblk r;
+      int count = -1;
+      T_SOC s;
+
+      hts_init_htsblk(&r);
+      s = newhttp_addr(opt, bad[k], &r, -1, 0, 0, &count);
+      CHECK(s == INVALID_SOCKET);
+      CHECK(count == 0); /* refused before resolving, not a failed connect */
+      CHECK(strstr(r.msg, "Invalid port") != NULL);
+      if (s != INVALID_SOCKET)
+        deletesoc(s);
+    }
+  }
+
   /* Connect-fallback decision (consumer of the multi-address list): when a
      stuck connect should abandon the current address for the next one. */
   {
@@ -370,6 +468,91 @@ int dns_selftests(httrackp *opt) {
   return failures;
 }
 
+/* Probes how long acquiring opt->state.lock takes while a resolve is in
+   flight. hts_has_stopped() takes that same lock and mutates nothing, and the
+   API promises it stays callable from another thread during a mirror. */
+typedef struct lock_probe {
+  httrackp *opt;
+  htsmutex lock;
+  hts_boolean done;
+  TStamp blocked_ms;
+} lock_probe;
+
+static void lock_probe_thread(void *arg) {
+  lock_probe *const p = (lock_probe *) arg;
+  TStamp start;
+
+  Sleep(MOCK_SLOW_MS / 10); /* let the resolve get under way first */
+  start = mtime_local();
+  (void) hts_has_stopped(p->opt);
+  hts_mutexlock(&p->lock);
+  p->blocked_ms = mtime_local() - start;
+  p->done = HTS_TRUE;
+  hts_mutexrelease(&p->lock);
+}
+
+int dns_timeout_selftests(httrackp *opt) {
+  SOCaddr addrs[HTS_MAXADDRNUM];
+  const char *err = NULL;
+  lock_probe probe;
+  TStamp start, elapsed;
+  int count;
+
+  failures = 0;
+  hts_dns_set_resolver_backend(&mock_backend);
+  IPV6_resolver = 0;
+  mock_reset_calls();
+  opt->timeout = 1; /* the bound under test */
+
+  memset(&probe, 0, sizeof(probe));
+  probe.opt = opt;
+  probe.lock = HTSMUTEX_INIT;
+  hts_mutexinit(&probe.lock);
+  CHECK(hts_newthread(lock_probe_thread, &probe) == 0);
+
+  start = mtime_local();
+  count = hts_dns_resolve_all(opt, "slow.test", addrs, HTS_MAXADDRNUM, &err);
+  elapsed = mtime_local() - start;
+
+  /* the resolve returns on opt->timeout, not when the resolver deigns to
+     answer: this is what lets --max-time and --timeout fire (#606). The bound
+     is derived from opt->timeout, never from the mock's sleep, or a resolve
+     that ignored opt->timeout would still pass under the mock. */
+  CHECK(elapsed < (TStamp) opt->timeout * 1000 + 500);
+  CHECK(count == 0); /* a timeout is reported as "does not resolve" */
+
+  /* state.lock is not held across the resolve; a concurrent stop query, which
+     the mirror API promises stays live, is not blocked behind it */
+  for (;;) {
+    hts_boolean done;
+    TStamp blocked;
+
+    hts_mutexlock(&probe.lock);
+    done = probe.done;
+    blocked = probe.blocked_ms;
+    hts_mutexrelease(&probe.lock);
+    if (done) {
+      CHECK(blocked < 500);
+      break;
+    }
+    Sleep(20);
+  }
+  hts_mutexfree(&probe.lock);
+
+  /* a timeout is not an answer, so it must not be negative-cached: the host is
+     resolved again rather than written off for the rest of the crawl */
+  CHECK(mock_read_calls("slow.test") == 1);
+  count = hts_dns_resolve_all(opt, "slow.test", addrs, HTS_MAXADDRNUM, &err);
+  CHECK(count == 0);
+  CHECK(mock_read_calls("slow.test") == 2); /* re-resolved, not cached */
+
+  /* Both resolves were abandoned mid-backend; wait for their workers to leave
+     it before returning. The backend stays installed: an abandoned worker
+     still reads it (to free its addrinfo) after the last call returns. */
+  mock_wait_finished(2);
+  return failures;
+}
+
 #else
 
 int dns_selftests(httrackp *opt) {

src/htsdns_selftest.h

@@ -46,6 +46,12 @@ typedef struct httrackp httrackp;
    Returns the number of failed checks (0 == success). */
 int dns_selftests(httrackp *opt);
 
+/* Drive a deliberately slow (mock) resolver, asserting that a resolve is
+   bounded by opt->timeout, does not hold opt->state.lock while it runs, and
+   does not cache a timeout as an answer (#606). Takes a few seconds.
+   Returns the number of failed checks (0 == success). */
+int dns_timeout_selftests(httrackp *opt);
+
 #endif
 
 #endif

src/htsftp.c

@@ -248,7 +248,13 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
     // port
     a = strchr(adr, ':');       // port
     if (a) {
-      sscanf(a + 1, "%d", &port);
+      // folding a nonsense port into 1..65535 fetches one the link never named;
+      // an empty "host:" just means the default (#614)
+      if (a[1] != '\0' && !hts_parse_url_port(a + 1, &port)) {
+        snprintf(back->r.msg, sizeof(back->r.msg), "Invalid port: %s", a + 1);
+        back->r.statuscode = STATUSCODE_INVALID; // permanent, unlike a DNS miss
+        _HALT_FTP return 0;
+      }
       strncatbuff(_adr, adr, (int) (a - adr));
     } else
       strcpybuff(_adr, adr);

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-12"
-#define HTTRACK_VERSIONID "3.49.12"
+#define HTTRACK_VERSION "3.49-13"
+#define HTTRACK_VERSIONID "3.49.13"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 
@@ -323,11 +323,9 @@ typedef int64_t TStamp;
 #define LLintP "%" PRId64
 
 /* Integer type for file offsets/sizes passed to the C library; INTsysP is its
-   printf conversion. FIXME: LFS_FLAG is a configure make variable, never a C
-   macro, so this test is dead and INTsys stays int on POSIX despite large-file
-   support (the real macro is HTS_LFS). Widening it there is an installed-header
-   type change, so it is left alone here. */
-#if defined(LFS_FLAG) || defined(_MSC_VER)
+   printf conversion. HTS_LFS is the large-file macro: LFS_FLAG is a configure
+   make variable carrying the -D flags, never itself defined. */
+#if defined(HTS_LFS) || defined(_MSC_VER)
 typedef LLint INTsys;
 
 #define INTsysP LLintP

src/htshelp.c

@@ -477,7 +477,7 @@ void help(const char *app, int more) {
     ("  Y   mirror ALL links located in the first level pages (mirror links)");
   infomsg("");
   infomsg("Proxy options:");
-  infomsg("  P  proxy use (-P [socks5://][user:pass@]proxy:port)");
+  infomsg("  P  proxy use (-P [socks5://|connect://][user:pass@]proxy:port)");
   infomsg(" %f *use proxy for ftp (f0 don't use)");
   infomsg(" %b  use this local hostname to make/send requests (-%b hostname)");
   infomsg("");
@@ -496,8 +496,8 @@ void help(const char *app, int more) {
   infomsg("");
   infomsg("Flow control:");
   infomsg("  cN number of multiple connections (*c8)");
-  infomsg
-    ("  TN timeout, number of seconds after a non-responding link is shutdown");
+  infomsg("  TN timeout, number of seconds after a non-responding link is"
+          " shutdown; also bounds host name resolution");
   infomsg
     ("  RN number of retries, in case of timeout or non-fatal errors (*R1)");
   infomsg

src/htslib.c

@@ -65,6 +65,7 @@ Please visit our Website: http://www.httrack.com
 #endif /* _WIN32 */
 #include <stdarg.h>
 
+#include <ctype.h>
 #include <string.h>
 #include <time.h>
 #include <stdarg.h>
@@ -947,9 +948,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
           /* widths bound method[256], url[HTS_URLMAXSIZE*2], protocol[256] */
           if (sscanf(line, "%255s %2047s %255s", method, url, protocol) == 3) {
             size_t ret;
-            // absolute-URI for an http proxy; a socks tunnel takes origin-form
+            // http proxy: absolute-URI; socks/CONNECT tunnel: origin-form
             if (retour->req.proxy.active &&
-                !hts_proxy_is_socks(retour->req.proxy.name)) {
+                !hts_proxy_is_socks(retour->req.proxy.name) &&
+                !hts_proxy_is_connect(retour->req.proxy.name)) {
               print_buffer(&bstr,
                       "%s http://%s%s %s\r\n", method, adr, url,
                       protocol);
@@ -985,9 +987,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
         print_buffer(&bstr, "HEAD ");
     }
 
-    // an http proxy needs an absolute URI; a socks tunnel does not
+    // an http proxy needs an absolute URI; a socks or CONNECT tunnel does not
     if (retour->req.proxy.active &&
         !hts_proxy_is_socks(retour->req.proxy.name) &&
+        !hts_proxy_is_connect(retour->req.proxy.name) &&
         (strncmp(adr, "https://", 8) != 0)) {
       if (!link_has_authority(adr)) {   // default http
 #if HDEBUG
@@ -1032,10 +1035,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
     if (xsend)
       print_buffer(&bstr, "%s", xsend);  // éventuelles autres lignes
 
-    // for https, auth rides the CONNECT (the tunneled GET would leak it); for
-    // socks, the handshake
+    // https/connect://: auth rides the CONNECT; socks: the handshake
     if (retour->req.proxy.active &&
         !hts_proxy_is_socks(retour->req.proxy.name) &&
+        !hts_proxy_is_connect(retour->req.proxy.name) &&
         strncmp(adr, "https://", 8) != 0) {
       if (link_has_authorization(retour->req.proxy.name)) {     // et hop, authentification proxy!
         const char *a = jump_identification_const(retour->req.proxy.name);
@@ -2164,15 +2167,18 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
 #endif
 
       if (a != NULL) {
-        int i = -1;
-
-        iadr2[0] = '\0';
-        sscanf(a + 1, "%d", &i);
-        if (i != -1) {
-          port = (unsigned short int) i;
+        // folding a nonsense port into 0..65535 crawls one neither the link nor
+        // a port filter named; an empty "host:" just means the default (#614)
+        if (a[1] != '\0' && !hts_parse_url_port(a + 1, &port)) {
+          if (retour != NULL) {
+            snprintf(retour->msg, sizeof(retour->msg), "Invalid port: %s",
+                     a + 1);
+          }
+          return INVALID_SOCKET;
         }
 
-        // adresse véritable (sans :xx)
+        iadr2[0] = '\0';
+        // the address itself, without the ":port"
         strncatbuff(iadr2, iadr, (int) (a - iadr));
         resolve_host = iadr2;
       }
@@ -3717,6 +3723,8 @@ const char *jump_protocol_const(const char *source) {
     source += p;
   else if ((p = strfield(source, "socks5:")))
     source += p;
+  else if ((p = strfield(source, "connect:")))
+    source += p;
   // net_path
   if (strncmp(source, "//", 2) == 0)
     source += 2;
@@ -3732,11 +3740,27 @@ hts_boolean hts_proxy_is_socks(const char *name) {
                                                                    : HTS_FALSE;
 }
 
+hts_boolean hts_proxy_is_connect(const char *name) {
+  if (name == NULL)
+    return HTS_FALSE;
+  return strfield(name, "connect:") ? HTS_TRUE : HTS_FALSE;
+}
+
 // default proxy port for a -P argument, keyed on the scheme
 static int proxy_default_port(const char *arg) {
   return hts_proxy_is_socks(arg) ? 1080 : 8080;
 }
 
+// port "a" of -P argument "arg": digits fitting TCP's 1..65535, else the scheme
+// default. Not sscanf("%d"): past INT_MAX it wraps to a garbage port (#602)
+static int parse_proxy_port(const char *a, const char *arg) {
+  int port;
+
+  if (!hts_parse_url_port(a, &port))
+    return proxy_default_port(arg);
+  return port;
+}
+
 void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
   const char *authority = strstr(arg, "://");
   const char *a;
@@ -3744,18 +3768,15 @@ void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
 
   if (name_size == 0)
     return;
-  // scan back to the port ':' (or userinfo '@'), but never past the authority,
-  // so a scheme's own colon is not read as a port separator; inspect a[-1] from
-  // one-past-end so no pointer below the string is ever formed
+  // scan back to the port ':' (or userinfo '@'), never past the authority (a
+  // scheme's own colon is not a port separator) nor into an IPv6 literal (']'
+  // stops it); inspect a[-1] from one-past-end so no pointer underflows
   authority = (authority != NULL) ? authority + 3 : arg;
   a = arg + strlen(arg);
-  while (a > authority && a[-1] != ':' && a[-1] != '@')
+  while (a > authority && a[-1] != ':' && a[-1] != '@' && a[-1] != ']')
     a--;
   if (a > authority && a[-1] == ':') {
-    int p = -1;
-
-    sscanf(a, "%d", &p);
-    *port = (p > 0) ? p : proxy_default_port(arg);
+    *port = parse_proxy_port(a, arg);
     namelen = (size_t) (a - 1 - arg);
   } else {
     *port = proxy_default_port(arg);
@@ -5074,50 +5095,164 @@ HTSEXT_API int check_hostname_dns(const char *const hostname) {
   return hts_dns_resolve_nocache(hostname, &buffer) != NULL;
 }
 
-// Needs locking
-// Internal DNS cache. Fill out[0..count-1] with up to max addresses for _iadr,
-// resolving (and caching the full list) on a miss. Returns the count.
-static int hts_dns_resolve_list_(httrackp *opt, const char *_iadr,
-                                 SOCaddr *const out, const int max,
-                                 const char **error) {
-  char BIGSTK iadr[HTS_URLMAXSIZE * 2];
-  coucal cache = hts_cache(opt); // le cache dns
+/* A resolve in flight. Refcounted: a timed-out resolve is abandoned, not
+   cancelled, so the last of caller/worker to leave frees the job. */
+typedef struct dns_resolve_job {
+  htsmutex lock;
+  int refcount;
+  hts_boolean done;
+  char *hostname;
+  SOCaddr addr[HTS_MAXADDRNUM];
   int count;
+  const char *error;
+} dns_resolve_job;
+
+/* Copy the first min(count, max) addresses of src into dest. */
+static void dns_copy_addrs(SOCaddr *dest, SOCaddr *src, int count, int max) {
+  int i;
+
+  for (i = 0; i < count && i < max; i++)
+    SOCaddr_copy_SOCaddr(dest[i], src[i]);
+}
+
+static void dns_job_release(dns_resolve_job *job) {
+  hts_boolean last;
+
+  hts_mutexlock(&job->lock);
+  last = (--job->refcount == 0) ? HTS_TRUE : HTS_FALSE;
+  hts_mutexrelease(&job->lock);
+  if (last) {
+    hts_mutexfree(&job->lock);
+    freet(job->hostname);
+    freet(job);
+  }
+}
+
+/* Outlives a timed-out resolve, so it writes only the job: never opt (freed
+   before the thread wait at exit) nor the DNS cache. */
+static void dns_resolve_thread(void *arg) {
+  dns_resolve_job *const job = (dns_resolve_job *) arg;
+  SOCaddr resolved[HTS_MAXADDRNUM];
+  const char *error = NULL;
+  const int count = hts_dns_resolve_nocache_list(job->hostname, resolved,
+                                                 HTS_MAXADDRNUM, &error);
+
+  hts_mutexlock(&job->lock);
+  dns_copy_addrs(job->addr, resolved, count, HTS_MAXADDRNUM);
+  job->count = count;
+  job->error = error;
+  job->done = HTS_TRUE; /* published last: gates the caller's read of addr[] */
+  hts_mutexrelease(&job->lock);
+  dns_job_release(job);
+}
+
+/* Resolve hostname on a worker thread, giving up after timeout seconds.
+   Returns the address count, or -1 on timeout -- distinct from 0 ("does not
+   resolve"), which is a real answer and gets negative-cached. */
+static int hts_dns_resolve_nocache_list_bounded(const char *hostname,
+                                                SOCaddr *const out,
+                                                const int max,
+                                                const int timeout,
+                                                const char **error) {
+  dns_resolve_job *job;
+  TStamp deadline;
+  int count = -1;
+  int poll_ms = 1;
+
+  if (timeout <= 0) /* no bound asked for (--timeout 0) */
+    return hts_dns_resolve_nocache_list(hostname, out, max, error);
+
+  job = calloct(1, sizeof(*job));
+  assertf(job != NULL);
+  hts_mutexinit(&job->lock);
+  job->hostname = strdupt(hostname);
+  job->refcount = 2; /* this caller + the worker */
+  if (hts_newthread(dns_resolve_thread, job) != 0) {
+    job->refcount = 1; /* no worker: fall back to resolving inline */
+    dns_job_release(job);
+    return hts_dns_resolve_nocache_list(hostname, out, max, error);
+  }
+
+  deadline = mtime_local() + (TStamp) timeout * 1000;
+  for (;;) {
+    hts_boolean done;
+
+    hts_mutexlock(&job->lock);
+    done = job->done;
+    if (done) {
+      count = job->count;
+      dns_copy_addrs(out, job->addr, count, max);
+      if (error != NULL)
+        *error = job->error;
+    }
+    hts_mutexrelease(&job->lock);
+    if (done || mtime_local() >= deadline)
+      break;
+    Sleep(poll_ms);
+    if (poll_ms < 50) /* short first polls keep a fast resolve fast */
+      poll_ms *= 2;
+  }
+  dns_job_release(job);
+  return count;
+}
+
+int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
+                        const char **error) {
+  char BIGSTK host[HTS_URLMAXSIZE * 2];
+  SOCaddr resolved[HTS_MAXADDRNUM];
+  coucal cache;
+  int count, i;
 
   assertf(opt != NULL);
-  assertf(_iadr != NULL);
   assertf(out != NULL);
+  if (!strnotempty(iadr) || max <= 0) {
+    return 0;
+  }
 
-  strcpybuff(iadr, jump_identification_const(_iadr));
-  // couper éventuel :
+  /* cache key and resolver input: identification and any ":port" stripped */
+  strcpybuff(host, jump_identification_const(iadr));
   {
     char *a;
 
-    if ((a = jump_toport(iadr)))
+    if ((a = jump_toport(host)))
       *a = '\0';
   }
 
-  /* get IP from the dns cache */
-  count = hts_ghbn_all(cache, iadr, out, max);
+  hts_mutexlock(&opt->state.lock);
+  cache = hts_cache(opt);
+#if HTS_INET6 != 0
+  hts_resolver_check_env(); /* settle the backend before a worker reads it */
+#endif
+  count = hts_ghbn_all(cache, host, out, max);
+  hts_mutexrelease(&opt->state.lock);
   if (count >= 0) { // cache hit (0 == negative-cached)
     return count;
-  } else { // non présent dans le cache dns, tester
-    SOCaddr resolved[HTS_MAXADDRNUM];
-    t_dnscache *record;
-    int i;
+  }
 
 #if DEBUGDNS
-    printf("resolving (not cached) %s\n", iadr);
+  printf("resolving (not cached) %s\n", host);
 #endif
 
-    count = hts_dns_resolve_nocache_list(iadr, resolved, HTS_MAXADDRNUM, error);
+  /* Resolve with no lock held: getaddrinfo can block for a long time, and
+     state.lock also gates the stop request (#606). */
+  count = hts_dns_resolve_nocache_list_bounded(host, resolved, HTS_MAXADDRNUM,
+                                               opt->timeout, error);
 
 #if HTS_WIDE_DEBUG
-    DEBUG_W("gethostbyname done\n");
+  DEBUG_W("gethostbyname done\n");
 #endif
 
-    /* attempt to store new entry (coucal owns it and dups the host key) */
-    record = malloct(sizeof(t_dnscache));
+  if (count < 0) { /* timed out: no answer to cache, and none to report */
+    if (error != NULL)
+      *error = "host name resolution timed out";
+    return 0;
+  }
+
+  hts_mutexlock(&opt->state.lock);
+  { /* store the full list (coucal owns the record and dups the host key; a
+       concurrent resolve of the same host replaces, and frees, this one) */
+    t_dnscache *const record = malloct(sizeof(t_dnscache));
+
     if (record != NULL) {
       memset(record, 0, sizeof(*record));
       record->host_count = count;
@@ -5127,28 +5262,13 @@ static int hts_dns_resolve_list_(httrackp *opt, const char *_iadr,
         memcpy(record->host_addr[i], &SOCaddr_sockaddr(resolved[i]),
                record->host_length[i]);
       }
-      coucal_add_pvoid(cache, iadr, record);
+      coucal_add_pvoid(cache, host, record);
     }
-
-    /* copy result to caller (cache store may have failed; result still valid)
-     */
-    for (i = 0; i < count && i < max; i++) {
-      SOCaddr_copy_SOCaddr(out[i], resolved[i]);
-    }
-    return count;
-  } // retour hp du cache
-}
-
-int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
-                        const char **error) {
-  int count;
-
-  if (!strnotempty(iadr) || max <= 0) {
-    return 0;
   }
-  hts_mutexlock(&opt->state.lock);
-  count = hts_dns_resolve_list_(opt, iadr, out, max, error);
   hts_mutexrelease(&opt->state.lock);
+
+  /* copy result to caller (cache store may have failed; result still valid) */
+  dns_copy_addrs(out, resolved, count, max);
   return count;
 }
 

src/htslib.h

@@ -61,6 +61,7 @@ typedef struct lien_adrfilsave lien_adrfilsave;
 
 /* définitions globales */
 #include "htsglobal.h"
+#include "htsurlport.h"
 
 /* basic net definitions */
 #include "htsbase.h"
@@ -210,6 +211,9 @@ int check_writeinput_t(T_SOC soc, int timeout);
 /* TRUE if this -P proxy name (which keeps its scheme) is a SOCKS5 proxy. */
 hts_boolean hts_proxy_is_socks(const char *name);
 
+/* TRUE if this -P proxy name is a "connect://" CONNECT-only proxy (#564). */
+hts_boolean hts_proxy_is_connect(const char *name);
+
 void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * retour,
                char *rcvd);
 void treatfirstline(htsblk * retour, const char *rcvd);
@@ -219,7 +223,8 @@ LLint http_xfread1(htsblk * r, int bufl);
 /* Cached resolver: fill out[0..count-1] with up to max addresses for iadr (in
    resolver order), returning the count (0 = does not resolve, negative-cached).
    Resolves once per host; later calls read the DNS cache. Must hold no lock
-   (brackets opt->state.lock itself). */
+   (brackets opt->state.lock itself, never across the resolve). A miss resolves
+   on a worker thread bounded by opt->timeout; a timeout reports 0, uncached. */
 int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
                         const char **error);
 HTS_INLINE SOCaddr *hts_dns_resolve2(httrackp *opt, const char *iadr,

src/htsname.c

@@ -1532,8 +1532,30 @@ int url_savename(lien_adrfilsave *const afs,
       // last segment
       wsave[j++] = '/';
 #define MAX_UTF8_SEQ_CHARS 4
-      for(i = lastSeg; wsave[i] != '\0' && j < maxLen; i++) {
-        wsave[j++] = wsave[i];
+      {
+        // #623: the ".delayed" placeholder marker sits at the tail; cutting
+        // through it drops IS_DELAYED_EXT, so the file is never renamed to its
+        // final name. Reserve the trailing ".<id>.delayed" across the cut.
+        size_t markStart = wsaveLen;
+        if (IS_DELAYED_EXT(afs->save)) {
+          const size_t extDot = wsaveLen - strlen("." DELAYED_EXT);
+          size_t p = extDot; /* walk back over a dot-separated ".<hexid>" tag */
+
+          while (p > lastSeg && ((wsave[p - 1] >= '0' && wsave[p - 1] <= '9') ||
+                                 (wsave[p - 1] >= 'a' && wsave[p - 1] <= 'f')))
+            p--;
+          // keep the tag only if truly ".<hexid>.delayed"; else the bare marker
+          // (a wholly-hex base, e.g. a hashed #133 name, must not be absorbed)
+          markStart = (p > lastSeg && p < extDot && wsave[p - 1] == '.')
+                          ? p - 1
+                          : extDot;
+        }
+        // head, bounded so the marker still fits, then the marker itself
+        for (i = lastSeg; i < markStart && j + (wsaveLen - markStart) < maxLen;
+             i++)
+          wsave[j++] = wsave[i];
+        for (i = markStart; i < wsaveLen && j < maxLen; i++)
+          wsave[j++] = wsave[i];
       }
       // terminating \0
       wsave[j++] = '\0';

src/htsparse.c

@@ -315,6 +315,43 @@ static void escape_url_parens(char *const s, const size_t size) {
   strlcpybuff(s, buff, size);
 }
 
+/* Strip a default ":80" from lien's authority in place. Any spelling that
+   range-parses to 80 (":80", ":080") is dropped by its matched length, not a
+   hardcoded 3 chars; a value that merely wraps to 80 as an int (#614) stays. */
+void hts_strip_default_port(char *lien, size_t size) {
+  char *a;
+
+  if (!link_has_authority(lien))
+    return;
+  a = strstr(lien, "//"); // "//" authority
+  if (a)
+    a += 2;
+  else
+    a = lien;
+  a = jump_toport(a);
+  if (a) { // port present
+    char *b = a + 1;
+    char saved;
+    int port;
+    hts_boolean is_default;
+
+    while (isdigit((unsigned char) *b))
+      b++;
+    saved = *b;
+    *b = '\0';
+    is_default = hts_parse_url_port(a + 1, &port) && port == 80;
+    *b = saved;
+    if (is_default) { // default port, strip it
+      char BIGSTK tempo[HTS_URLMAXSIZE * 2];
+
+      tempo[0] = '\0';
+      strncatbuff(tempo, lien, a - lien);
+      strcatbuff(tempo, b); // skip the whole matched :port
+      strlcpybuff(lien, tempo, size);
+    }
+  }
+}
+
 /* Main parser */
 int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
   char catbuff[CATBUFF_SIZE];
@@ -2138,38 +2175,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     } while((b != a) && (b));
                   }
                 }
-                // éliminer les éventuels :80 (port par défaut!)
-                if (link_has_authority(lien)) {
-                  char *a;
-
-                  a = strstr(lien, "//");       // "//" authority
-                  if (a)
-                    a += 2;
-                  else
-                    a = lien;
-                  a = jump_toport(a);
-                  if (a) {      // port
-                    int port = 0;
-                    int defport = 80;
-                    char *b = a + 1;
-
-#if HTS_USEOPENSSL
-#endif
-                    while(isdigit((unsigned char) *b)) {
-                      port *= 10;
-                      port += (int) (*b - '0');
-                      b++;
-                    }
-                    if (port == defport) {      // port 80, default - c'est débile
-                      char BIGSTK tempo[HTS_URLMAXSIZE * 2];
-
-                      tempo[0] = '\0';
-                      strncatbuff(tempo, lien, a - lien);
-                      strcatbuff(tempo, a + 3); // sauter :80
-                      strcpybuff(lien, tempo);
-                    }
-                  }
-                }
+                // drop a default :80 port from the authority
+                hts_strip_default_port(lien, sizeof(lien));
                 // filtrer les parazites (mailto & cie)
                 /*
                    if (strfield(lien,"mailto:")) {  // ne pas traiter

src/htsparse.h

@@ -106,6 +106,10 @@ struct htsmoduleStructExtended {
 */
 int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre);
 
+/* Strip a default ":80" (any spelling) from an absolute link's authority, in
+   place into a buffer of the given size. */
+void hts_strip_default_port(char *lien, size_t size);
+
 /*
   Check for 301,302.. errors ("moved") and handle them; re-isuue requests, make
   rediretc file, handle filters considerations..

src/htsproxy.c

@@ -90,12 +90,15 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
   if (soc == INVALID_SOCKET)
     return 0;
 
-  // CONNECT needs an explicit host:port; default the https port
+  // CONNECT needs an explicit host:port; default :80 for http, :443 for https
   authority[0] = '\0';
   if (portsep != NULL)
     strlcatbuff(authority, host, sizeof(authority)); // already host:port
-  else
-    snprintf(authority, sizeof(authority), "%s:%d", host, 443);
+  else {
+    const int defport = (strncmp(adr, "https://", 8) == 0) ? 443 : 80;
+
+    snprintf(authority, sizeof(authority), "%s:%d", host, defport);
+  }
 
   // backstop: never let a stray CR/LF in the host smuggle a second line into
   // the CONNECT request (the host is already sanitized upstream)
@@ -132,7 +135,7 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
   }
   strlcatbuff(req, H_CRLF, sizeof(req)); // end of request headers
 
-  // raw send: ssl is set, so sendc() would route to TLS
+  // raw send(): sendc() would route to TLS when ssl is set (https tunnel)
   {
     const char *p = req;
     size_t remain = strlen(req);
@@ -488,14 +491,11 @@ static int socks5_handshake_stream(httrackp *opt, socks5_stream *st,
     if ((unsigned char) host[i] < ' ')
       return socks5_fail(msg, msgsize, "SOCKS5: invalid origin host");
   }
-  if (portsep != NULL) {
-    int p = -1;
-
-    sscanf(portsep + 1, "%d", &p);
-    if (p <= 0 || p > 65535)
-      return socks5_fail(msg, msgsize, "SOCKS5: invalid origin port");
-    port = p;
-  }
+  // the old range check ran after sscanf("%d") had wrapped a huge value into a
+  // plausible port (#614). An empty "host:" stays refused here, unlike the
+  // direct path, as it was before #614.
+  if (portsep != NULL && !hts_parse_url_port(portsep + 1, &port))
+    return socks5_fail(msg, msgsize, "SOCKS5: invalid origin port");
   if (link_has_authorization(proxy_name)) {
     if (!socks5_credentials(opt, proxy_name, user, sizeof(user), &userlen, pass,
                             sizeof(pass), &passlen, msg, msgsize))

src/htsselftest.c

@@ -45,6 +45,7 @@ Please visit our Website: http://www.httrack.com
 #include "htscore.h"
 #include "htsdefines.h"
 #include "htslib.h"
+#include "htsalias.h"
 #include "htsparse.h"
 #include "htscache.h"
 #include "htscache_selftest.h"
@@ -825,6 +826,21 @@ static int st_simplify(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+static int st_expandhome(httrackp *opt, int argc, char **argv) {
+  String path = STRING_EMPTY;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "expandhome: needs a path\n");
+    return 1;
+  }
+  StringCopy(path, argv[0]);
+  expand_home(&path);
+  printf("expanded=%s\n", StringBuff(path));
+  StringFree(path);
+  return 0;
+}
+
 static int st_mime(httrackp *opt, int argc, char **argv) {
   char mime[256];
 
@@ -1365,9 +1381,12 @@ static int st_proxyurl(httrackp *opt, int argc, char **argv) {
     return 1;
   }
   hts_parse_proxy(argv[0], name, sizeof(name), &port);
-  // host= is what the connect actually resolves (scheme + user:pass stripped)
-  printf("name=%s port=%d host=%s\n", name, port,
-         jump_identification_const(name));
+  // host= is the resolved host (scheme/userinfo stripped); kind= the transport
+  printf("name=%s port=%d host=%s kind=%s\n", name, port,
+         jump_identification_const(name),
+         hts_proxy_is_socks(name)     ? "socks"
+         : hts_proxy_is_connect(name) ? "connect"
+                                      : "http");
   return 0;
 }
 
@@ -1450,6 +1469,25 @@ static int st_socks5(httrackp *opt, int argc, char **argv) {
   assertf(socks5_handshake_scripted(opt, "origin.test:8443", proxy, &io) == 1);
   assertf(memcmp(io.sent + io.sent_len - 2, "\x20\xfb", 2) == 0);
 
+  /* a bad origin port is refused before any byte goes out (#614). 4294967376 is
+     the case the old range check could not see: it overflowed the sscanf("%d")
+     into a plausible 80 and passed. 65616 would not prove anything here, since
+     it fits an int and the old check already caught it. */
+  {
+    static const char *const bad[] = {"origin.test:4294967376",
+                                      "origin.test:80x", "origin.test:+80",
+                                      "origin.test: 80", "origin.test:8.0"};
+    size_t k;
+
+    for (k = 0; k < sizeof(bad) / sizeof(bad[0]); k++) {
+      len = socks5_reply(script, 0x01, v4, sizeof(v4));
+      io.reply = script;
+      io.reply_len = len;
+      assertf(socks5_handshake_scripted(opt, bad[k], proxy, &io) == 0);
+      assertf(io.sent_len == 0);
+    }
+  }
+
   /* credentials: split on the first colon of the escaped userinfo, so %3a stays
      inside the username and a colon in the password is not a delimiter */
   {
@@ -1507,6 +1545,22 @@ static int st_socks5(httrackp *opt, int argc, char **argv) {
     assertf(io.sent_len == 0);
   }
 
+  /* the request is always ATYP=domain, which cannot carry an IPv6 literal: a
+     bracketed origin is rejected rather than sent as a bogus domain name. The
+     msg check pins the reason: a stricter host validator would also reject
+     these, but for the wrong cause. */
+  io.reply = script;
+  io.reply_len = len;
+  assertf(socks5_handshake_scripted(opt, "[::1]", proxy, &io) == 0);
+  assertf(io.sent_len == 0);
+  assertf(strstr(io.msg, "IPv6") != NULL);
+  io.reply = script;
+  io.reply_len = len;
+  assertf(socks5_handshake_scripted(opt, "[2001:db8::1]:8443", proxy, &io) ==
+          0);
+  assertf(io.sent_len == 0);
+  assertf(strstr(io.msg, "IPv6") != NULL);
+
   printf("socks5 self-test OK\n");
   return 0;
 }
@@ -1535,6 +1589,41 @@ static int st_identabs(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Default-port strip (#627): a genuine 80 (any spelling) is removed by its
+   matched length, host preserved; a non-80 port or one that only wraps to 80 as
+   a 32-bit int (#614) is left intact. Guards the old bug where ":080"/":0080"
+   dropped a hardcoded 3 chars and glued the leftover digits onto the host. */
+static int st_stripport(httrackp *opt, int argc, char **argv) {
+  static const struct {
+    const char *in, *out;
+  } cases[] = {
+      {"http://127.0.0.1:80/x", "http://127.0.0.1/x"},
+      {"http://127.0.0.1:080/x", "http://127.0.0.1/x"},
+      {"http://127.0.0.1:0080/x", "http://127.0.0.1/x"},
+      {"http://127.0.0.1:80", "http://127.0.0.1"},
+      {"http://127.0.0.1:0081/x", "http://127.0.0.1:0081/x"},
+      {"http://127.0.0.1:81/x", "http://127.0.0.1:81/x"},
+      {"http://127.0.0.1:8080/x", "http://127.0.0.1:8080/x"},
+      {"http://127.0.0.1:4294967376/x", "http://127.0.0.1:4294967376/x"},
+      {"http://127.0.0.1/x", "http://127.0.0.1/x"},
+  };
+
+  size_t k;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  for (k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) {
+    char BIGSTK buff[HTS_URLMAXSIZE * 2];
+
+    strcpybuff(buff, cases[k].in);
+    hts_strip_default_port(buff, sizeof(buff));
+    assertf(strcmp(buff, cases[k].out) == 0);
+  }
+  printf("stripport self-test OK\n");
+  return 0;
+}
+
 /* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
    no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
    registering an already-crawled link (dedup/collision paths). */
@@ -2090,6 +2179,15 @@ static int st_dns(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+static int st_dnstimeout(httrackp *opt, int argc, char **argv) {
+  const int err = dns_timeout_selftests(opt);
+
+  (void) argc;
+  (void) argv;
+  printf("dns-timeout-selftest: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 static int st_cookies(httrackp *opt, int argc, char **argv) {
   static t_cookie cookie;
   char hdr[1024];
@@ -2414,6 +2512,45 @@ static int st_makeindex(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+// hts_buildtopindex takes a system-charset path but verif_backblue() below it
+// expects utf-8, so on Windows a non-ASCII project dir gets the gifs written to
+// a mangled twin (issues #216/#217). argv[0] is a writable dir.
+static int st_topindex(httrackp *opt, int argc, char **argv) {
+  char projdir[HTS_URLMAXSIZE];
+  char path[HTS_URLMAXSIZE + 16]; /* projdir plus a basename */
+#ifdef _WIN32
+  /* the GUI hands hts_buildtopindex an ANSI path; mimic it. CP1252 'cafe' */
+  static const char *const projName = "caf\xE9";
+#else
+  /* POSIX system charset is UTF-8 */
+  static const char *const projName = "caf\xC3\xA9";
+#endif
+
+  assertf(argc >= 1);
+  snprintf(projdir, sizeof(projdir), "%s/%s", argv[0], projName);
+
+  /* structcheck(), not the utf-8 MKDIR family: same charset as buildtopindex */
+  snprintf(path, sizeof(path), "%s/", projdir);
+  assertf(structcheck(path) == 0);
+
+  /* returns 0 here: the dir holds no sub-project, only the gifs matter */
+  (void) hts_buildtopindex(opt, projdir, "");
+
+  /* the gifs must land in the project dir itself, not in a mangled sibling */
+  snprintf(path, sizeof(path), "%s/backblue.gif", projdir);
+  assertf(fexist(path));
+
+  /* raw unlink/rmdir: UNLINK is utf-8 on Windows, these paths aren't */
+  unlink(path);
+  snprintf(path, sizeof(path), "%s/fade.gif", projdir);
+  unlink(path);
+  snprintf(path, sizeof(path), "%s/index.html", projdir);
+  unlink(path);
+  rmdir(projdir);
+  printf("topindex self-test OK\n");
+  return 0;
+}
+
 /* Each inplace_escape_*() must equal escape_*() on a copy. */
 static int st_inplace_escape(httrackp *opt, int argc, char **argv) {
   /* >255 bytes forces the helper's malloct path, not the stack buffer */
@@ -3051,6 +3188,7 @@ static const struct selftest_entry {
     {"filterbounds", "", "matcher length/work caps reject hostile patterns",
      st_filterbounds},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
+    {"expandhome", "<path>", "expand a leading ~/ into $HOME", st_expandhome},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
     {"urlhack", "", "-%u url-hack sub-flag (www/slash/query) self-test",
@@ -3086,6 +3224,8 @@ static const struct selftest_entry {
      st_socks5},
     {"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
      st_identabs},
+    {"stripport", "", "default :80 port strip preserves host (#627)",
+     st_stripport},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
     {"headerlong", "[header-name:]",
@@ -3114,10 +3254,15 @@ static const struct selftest_entry {
     {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
      st_cache_corrupt},
     {"dns", "", "DNS resolver/cache self-test", st_dns},
+    {"dnstimeout", "", "a slow DNS resolve is bounded and holds no lock",
+     st_dnstimeout},
     {"cookies", "", "cookie request-header self-test", st_cookies},
     {"useragent", "", "default User-Agent self-test", st_useragent},
     {"makeindex", "[dir]", "hts_finish_makeindex footer/refresh self-test",
      st_makeindex},
+    {"topindex", "[dir]",
+     "hts_buildtopindex charset handling of a non-ASCII project dir",
+     st_topindex},
     {"inplace-escape", "", "inplace_escape_* vs escape_* equivalence self-test",
      st_inplace_escape},
     {"escape-room", "", "HT_ADD_HTMLESCAPED* reservation-factor self-test",

src/htsserver.h

@@ -164,13 +164,12 @@ HTS_UNUSED static int linputsoc_t(T_SOC soc, char *s, int max, int timeout) {
   return -1;
 }
 
+/* Same contract as hts_gethome(), which is hidden and out of reach from here */
 static const char *gethomedir(void) {
   const char *home = getenv("HOME");
 
-  if (home)
-    return home;
-  else
-    return ".";
+  /* An empty $HOME would put the base path and httrack.ini at the root */
+  return strnotempty(home) ? home : ".";
 }
 static int linput_cpp(FILE * fp, char *s, int max) {
   int rlen = 0;

src/htstools.c

@@ -896,7 +896,20 @@ HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
     if (fpo) {
       find_handle h;
 
-      verif_backblue(opt, concat(catbuff, sizeof(catbuff), rpath, "/")); // générer gif
+      // générer gif. verif_backblue() is utf-8, but our path is the system
+      // charset, so on Windows convert it or the gifs land in a mangled twin
+      // dir (#217). Elsewhere the system charset is already utf-8.
+#ifdef _WIN32
+      {
+        const char *const base = concat(catbuff, sizeof(catbuff), rpath, "/");
+        char *const base_utf8 =
+            hts_convertStringSystemToUTF8(base, strlen(base));
+        verif_backblue(opt, base_utf8 != NULL ? base_utf8 : base);
+        free(base_utf8);
+      }
+#else
+      verif_backblue(opt, concat(catbuff, sizeof(catbuff), rpath, "/"));
+#endif
       // Header
       hts_template_format(fpo, toptemplate_header,
               "<!-- Mirror and index made by HTTrack Website Copier/"

src/htsurlport.c

@@ -0,0 +1,50 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: TCP port parser, shared by the engine, htsserver and    */
+/*       proxytrack                                              */
+/* Author: Xavier Roche                                          */
+/* ------------------------------------------------------------ */
+
+#include "htsurlport.h"
+
+#include <ctype.h>
+#include <stdlib.h>
+
+hts_boolean hts_parse_url_port(const char *a, int *port) {
+  char *end;
+  long p;
+
+  if (!isdigit((unsigned char) *a)) // strtol would eat a sign or leading space
+    return HTS_FALSE;
+  p = strtol(a, &end, 10);
+  if (*end != '\0' || p < 1 || p > 65535) // ERANGE lands out of range too
+    return HTS_FALSE;
+  *port = (int) p;
+  return HTS_TRUE;
+}

src/htsurlport.h

@@ -0,0 +1,45 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: TCP port parser, shared by the engine, htsserver and    */
+/*       proxytrack                                              */
+/* Author: Xavier Roche                                          */
+/* ------------------------------------------------------------ */
+
+#ifndef HTSURLPORT_DEFH
+#define HTSURLPORT_DEFH
+
+#include "htsglobal.h"
+
+/* Parse the port text "a" (after the ':', up to the end of the string): TRUE
+   and *port set for a bare decimal in 1..65535, else FALSE and *port left
+   alone. Not sscanf("%d"), which range-checks nothing and wraps past INT_MAX.
+   Its own file so proxytrack, which does not link the library, can share it. */
+hts_boolean hts_parse_url_port(const char *a, int *port);
+
+#endif

src/htsweb.c

@@ -63,6 +63,7 @@ Please visit our Website: http://www.httrack.com
 #include "md5.c"
 
 #include "htsserver.h"
+#include "htsurlport.h"
 #include "htsweb.h"
 
 #if USE_BEGINTHREAD==0
@@ -257,8 +258,10 @@ int main(int argc, char *argv[]) {
   /* set commandline keys */
   for(i = 2; i < argc; i += 2) {
     if (strcmp(argv[i], "--port") == 0 && i + 1 < argc) {
-      if (sscanf(argv[i + 1], "%d", &defaultPort) != 1 || defaultPort < 0
-          || defaultPort >= 65535) {
+      // the range check ran after sscanf("%d") had wrapped a huge value into a
+      // plausible port, and listened there (#614). 0 (was the auto-pick) and
+      // 65535 (was refused, off by one) now both mean what they say.
+      if (!hts_parse_url_port(argv[i + 1], &defaultPort)) {
         fprintf(stderr, "couldn't set the port number to %s\n", argv[i + 1]);
         return -1;
       }

src/libhttrack.vcxproj

@@ -121,6 +121,7 @@
     <ClCompile Include="htshelp.c" />
     <ClCompile Include="htsindex.c" />
     <ClCompile Include="htslib.c" />
+    <ClCompile Include="htsurlport.c" />
     <ClCompile Include="htsmd5.c" />
     <ClCompile Include="htsmodules.c" />
     <ClCompile Include="htsname.c" />

src/proxy/main.c

@@ -47,7 +47,9 @@ static void sig_brpipe(int code) {
 }
 #endif
 
-static int scanHostPort(const char *str, char *host, int *port) {
+// split a "host:port" listen argument; FALSE sends the caller to the usage
+// screen. The port was unchecked, so a huge one wrapped into range (#614).
+static hts_boolean scanHostPort(const char *str, char *host, int *port) {
   char *pos = strrchr(str, ':');
 
   if (pos != NULL) {
@@ -56,12 +58,10 @@ static int scanHostPort(const char *str, char *host, int *port) {
     if (n < 256) {
       host[0] = '\0';
       strncat(host, str, n);
-      if (sscanf(pos + 1, "%d", port) == 1) {
-        return 1;
-      }
+      return hts_parse_url_port(pos + 1, port);
     }
   }
-  return 0;
+  return HTS_FALSE;
 }
 
 int main(int argc, char *argv[]) {

src/proxy/proxytrack.h

@@ -107,13 +107,12 @@ HTS_UNUSED static void proxytrack_print_log(const char *severity, const char *fo
 	"<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \
 	"<!-- End Disable IE Friendly HTTP Error Messages -->\r\n"
 
+/* Same contract as hts_gethome(): proxytrack does not link libhttrack */
 HTS_UNUSED static const char *gethomedir(void) {
   const char *home = getenv("HOME");
 
-  if (home)
-    return home;
-  else
-    return ".";
+  /* An empty $HOME would resolve a relative path against the root */
+  return strnotempty(home) ? home : ".";
 }
 
 HTS_UNUSED static int linput(FILE * fp, char *s, int max) {

src/proxytrack.vcxproj

@@ -99,6 +99,7 @@
     <ClCompile Include="proxy\main.c" />
     <ClCompile Include="proxy\proxytrack.c" />
     <ClCompile Include="proxy\store.c" />
+    <ClCompile Include="htsurlport.c" />
     <ClCompile Include="coucal\coucal.c" />
     <ClCompile Include="htsmd5.c" />
     <ClCompile Include="md5.c" />

src/version.rc

@@ -17,8 +17,8 @@
 #endif
 
 VS_VERSION_INFO VERSIONINFO
-  FILEVERSION 3, 49, 12, 0
-  PRODUCTVERSION 3, 49, 12, 0
+  FILEVERSION 3, 49, 13, 0
+  PRODUCTVERSION 3, 49, 13, 0
   FILEFLAGSMASK VS_FFI_FILEFLAGSMASK
 #ifdef _DEBUG
   FILEFLAGS VS_FF_DEBUG
@@ -35,12 +35,12 @@ BEGIN
     BEGIN
       VALUE "CompanyName", "Xavier Roche"
       VALUE "FileDescription", VER_FILE_DESCRIPTION
-      VALUE "FileVersion", "3.49.12"
+      VALUE "FileVersion", "3.49.13"
       VALUE "InternalName", VER_ORIGINAL_FILENAME
       VALUE "LegalCopyright", "Copyright (C) 1998-2026 Xavier Roche and other contributors. GNU GPL v3 or later."
       VALUE "OriginalFilename", VER_ORIGINAL_FILENAME
       VALUE "ProductName", "HTTrack Website Copier"
-      VALUE "ProductVersion", "3.49-12"
+      VALUE "ProductVersion", "3.49-13"
     END
   END
   BLOCK "VarFileInfo"

src/webhttrack

@@ -33,6 +33,19 @@ function log {
     return 0
 }
 
+# Map a POSIX locale ("zh_TW.UTF-8@euro") to its lang.indexes number, English (1) if unknown
+function lang_index {
+    local locale=$1 indexes=$2 tag n t
+    tag=$(echo "${locale}" | cut -f1 -d'.' | cut -f1 -d'@' | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z_')
+    # a few languages carry a region-specific entry (zh_tw, pt_br); else use the bare language
+    for t in "${tag}" "${tag%%_*}"; do
+        n=$(grep -E "^${t}:" "${indexes}" | cut -f2 -d':')
+        test -n "${n}" && break
+    done
+    test -n "${n}" || n=1
+    echo "${n}"
+}
+
 function launch_browser {
     log "Launching $1"
     browser=$1
@@ -59,13 +72,11 @@ test -f "${DISTPATH}/lang.indexes" || ! log "Could not find ${DISTPATH}/lang.ind
 test -d "${DISTPATH}/lang" || ! log "Could not find ${DISTPATH}/lang" || exit 1
 test -d "${DISTPATH}/html" || ! log "Could not find ${DISTPATH}/html" || exit 1
 
-# Locale
-HTSLANG="${LC_MESSAGES}"
-! test -n "${HTSLANG}" && HTSLANG="${LC_ALL}"
+# Locale: POSIX precedence, LC_ALL overrides LC_MESSAGES overrides LANG
+HTSLANG="${LC_ALL}"
+! test -n "${HTSLANG}" && HTSLANG="${LC_MESSAGES}"
 ! test -n "${HTSLANG}" && HTSLANG="${LANG}"
-HTSLANG="$(echo "$LANG" | cut -f1 -d'.' | cut -f1 -d'_')"
-LANGN=$(grep -E "^${HTSLANG}:" "${DISTPATH}/lang.indexes" | cut -f2 -d':')
-! test -n "${LANGN}" && LANGN=1
+LANGN=$(lang_index "${HTSLANG}" "${DISTPATH}/lang.indexes")
 
 # Find the browser
 # note: not all systems have sensible-browser or www-browser alternative

src/webhttrack.vcxproj

@@ -97,6 +97,7 @@
   <ItemGroup>
     <ClCompile Include="htsserver.c" />
     <ClCompile Include="htsweb.c" />
+    <ClCompile Include="htsurlport.c" />
   </ItemGroup>
 
   <ItemGroup>

tests/01_engine-cmdline.test

@@ -54,6 +54,13 @@ refused() {
         ! echo "FAIL: $1 (exit $RC)" || exit 1
 }
 
+# assert continue mode was entered: it drops the URL list, so with no cache to
+# resume the run ends on the usage screen rather than on any other error
+continued() {
+    { test "$RC" -ne 0 && grep -q 'usage:' "$1/.log"; } ||
+        ! echo "FAIL: $2 (exit $RC)" || exit 1
+}
+
 # a value past the old 126/256 caps but within the cap is accepted, on both the
 # short and long form of each option
 long=$(nchars 900)
@@ -102,4 +109,50 @@ for bad in nan nan:5 5:nan inf 10:5 99999; do
     refused "#185: invalid --pause '$bad' not refused cleanly"
 done
 
+# An option is not -i (continue) merely because its name contains an 'i' (#615).
+# These used to wipe the URL given before them and exit on the usage screen.
+run "$tmp/ord-bti" --build-top-index
+accepted "$tmp/ord-bti" "#615: --build-top-index after the URL wiped the URL list"
+run "$tmp/ord-bti-s" "-%i"
+accepted "$tmp/ord-bti-s" "#615: -%i after the URL wiped the URL list"
+run "$tmp/ord-proto" --protocol 2
+accepted "$tmp/ord-proto" "#615: --protocol after the URL wiped the URL list"
+run "$tmp/ord-proto-s" "-@i2"
+accepted "$tmp/ord-proto-s" "#615: -@i2 after the URL wiped the URL list"
+
+# %, &, @ and # take the letter after them into another option set, wherever
+# they sit in the cluster, so the i of -q%i is not the main-set -i either.
+run "$tmp/ord-qpi" "-q%i"
+accepted "$tmp/ord-qpi" "#615: -q%i after the URL wiped the URL list"
+run "$tmp/ord-qai" "-q@i"
+accepted "$tmp/ord-qai" "#615: -q@i after the URL wiped the URL list"
+
+# -%q only forces quiet mode, which nothing here can see: a run whose stdout is
+# not a tty is quiet from the start (htscoremain.c:174). Just check it crawls.
+run "$tmp/ord-iqs" "-%q"
+accepted "$tmp/ord-iqs" "#615: -%q after the URL broke the crawl"
+
+# The real -i still forces continue mode and drops the URL list, in every form:
+# a fix that merely stopped looking for 'i' would pass the checks above.
+run "$tmp/cont-s" -i
+continued "$tmp/cont-s" "#615: -i after the URL no longer forces continue mode"
+run "$tmp/cont-c" -iC2
+continued "$tmp/cont-c" "#615: -iC2 after the URL no longer forces continue mode"
+run "$tmp/cont-l" --continue
+continued "$tmp/cont-l" "#615: --continue after the URL no longer forces continue mode"
+run "$tmp/cont-u" --update
+continued "$tmp/cont-u" "#615: --update after the URL no longer forces continue mode"
+
+# ...including where another set's letter precedes it in the cluster.
+run "$tmp/cont-pq" "-%qi"
+continued "$tmp/cont-pq" "#615: -%qi after the URL no longer forces continue mode"
+run "$tmp/cont-pm" "-%Mi"
+continued "$tmp/cont-pm" "#615: -%Mi after the URL no longer forces continue mode"
+
+# Placed before the URL these always worked; they must keep working.
+run_only "$tmp/pre-bti" "-%i" "file://$tmp/index.html"
+accepted "$tmp/pre-bti" "#615: -%i before the URL broke the crawl"
+run_only "$tmp/pre-proto" "-@i2" "file://$tmp/index.html"
+accepted "$tmp/pre-proto" "#615: -@i2 before the URL broke the crawl"
+
 exit 0

tests/01_engine-dnstimeout.test

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# A resolve against a deliberately slow mock getaddrinfo (no network) must
+# return on --timeout, must not hold opt->state.lock meanwhile, and must not
+# cache the timeout as an answer (#606).
+# 'run' is an ignored placeholder argument.
+out=$(httrack -#test=dnstimeout run)
+
+test "$out" = "dns-timeout-selftest: OK" || {
+    echo "expected 'dns-timeout-selftest: OK', got: $out" >&2
+    exit 1
+}

tests/01_engine-expandhome.test

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+
+# SC2088: the literal, unexpanded '~' is the input under test.
+# shellcheck disable=SC2088
+
+set -euo pipefail
+
+# ~ expansion for the -O base path (expand_home). $HOME is pinned so the cases
+# cannot depend on the caller's, but MSYS rewrites it into a Windows path
+# before the native httrack.exe reads it: ask the engine what it saw rather
+# than hardcoding the value.
+ask() {
+    HOME=HTSHOME httrack -O /dev/null -#test=expandhome "$1"
+}
+exp() {
+    test "$(ask "$1")" == "expanded=$2" || exit 1
+}
+
+home=$(ask '~')
+home=${home#expanded=}
+
+# or every case below is vacuous: '~' means expansion never fired, '.' means it
+# fell back to the default without ever reading $HOME
+test "$home" != '~' || exit 1
+test "$home" != '.' || exit 1
+
+exp '~' "$home"
+exp '~/foo' "$home/foo"
+exp '~/foo/bar/#' "$home/foo/bar/#"
+exp '~/' "$home/"
+exp '~/../x' "$home/../x"
+
+# ~user/ needs getpwnam: left alone rather than mangled into $HOME + "user/"
+exp '~smith/foo' '~smith/foo'
+exp '~root' '~root'
+
+# only a leading '~' expands; anything else is a literal path component
+exp 'a~/x' 'a~/x'
+exp './~/x' './~/x'
+exp 'foo~bar' 'foo~bar'
+
+# an unset or empty $HOME must not turn ~/foo into the absolute /foo
+test "$(env -u HOME httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=./foo" || exit 1
+test "$(HOME='' httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=./foo" || exit 1
+
+# A $HOME past the 2 * HTS_URLMAXSIZE buffer is left alone: strcatbuff aborts
+# rather than truncates. Only meaningful where the engine sees the value we set:
+# MSYS rewrites HOME into a path of its own choosing, long or not.
+if test "$home" = HTSHOME; then
+    long=$(printf '%04096d' 0 | tr '0' 'A')
+    test "$(HOME="$long" httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=~/foo" || exit 1
+fi

tests/01_engine-proxyurl.test

@@ -5,40 +5,89 @@ set -euo pipefail
 
 # hts_parse_proxy splits a -P argument "[scheme://][user:pass@]host[:port]" into
 # the proxy host string and port. -#test=proxyurl <arg> prints
-# "name=.. port=.. host=..", where host= is what the connect resolves.
+# "name=.. port=.. host=.. kind=..", where host= is what the connect resolves and
+# kind= is the transport the scheme selects (http, socks or connect).
 
 p() {
     test "$(httrack -O /dev/null -#test=proxyurl "$1")" == "$2" || exit 1
 }
 
 # bare host, with and without an explicit port
-p 'proxy.example.com:8080' 'name=proxy.example.com port=8080 host=proxy.example.com'
-p 'proxy.example.com' 'name=proxy.example.com port=8080 host=proxy.example.com'
+p 'proxy.example.com:8080' 'name=proxy.example.com port=8080 host=proxy.example.com kind=http'
+p 'proxy.example.com' 'name=proxy.example.com port=8080 host=proxy.example.com kind=http'
 
 # user:pass@ is kept in name (for proxy auth) and stripped from host
-p 'user:pass@proxy:3128' 'name=user:pass@proxy port=3128 host=proxy'
-p 'user:pass@proxy' 'name=user:pass@proxy port=8080 host=proxy'
+p 'user:pass@proxy:3128' 'name=user:pass@proxy port=3128 host=proxy kind=http'
+p 'user:pass@proxy' 'name=user:pass@proxy port=8080 host=proxy kind=http'
 
 # a scheme colon must not be read as a port: http://host with no port used to
 # parse to name=http with a garbage port
-p 'http://proxy:8080' 'name=http://proxy port=8080 host=proxy'
-p 'http://proxy' 'name=http://proxy port=8080 host=proxy'
+p 'http://proxy:8080' 'name=http://proxy port=8080 host=proxy kind=http'
+p 'http://proxy' 'name=http://proxy port=8080 host=proxy kind=http'
 
 # socks5/socks5h default to 1080 and resolve to the bare host
-p 'socks5://host:1080' 'name=socks5://host port=1080 host=host'
-p 'socks5://host' 'name=socks5://host port=1080 host=host'
-p 'socks5h://host' 'name=socks5h://host port=1080 host=host'
+p 'socks5://host:1080' 'name=socks5://host port=1080 host=host kind=socks'
+p 'socks5://host' 'name=socks5://host port=1080 host=host kind=socks'
+p 'socks5h://host' 'name=socks5h://host port=1080 host=host kind=socks'
 # explicit port wins over the 1080 default, with and without userinfo
-p 'socks5://host:9050' 'name=socks5://host port=9050 host=host'
-p 'socks5://user:pass@host:9050' 'name=socks5://user:pass@host port=9050 host=host'
+p 'socks5://host:9050' 'name=socks5://host port=9050 host=host kind=socks'
+p 'socks5://user:pass@host:9050' 'name=socks5://user:pass@host port=9050 host=host kind=socks'
+
+# connect:// is an http proxy driven with CONNECT for every request (#564); it
+# defaults to the http 8080 port and resolves to the bare host
+p 'connect://host' 'name=connect://host port=8080 host=host kind=connect'
+p 'connect://host:8082' 'name=connect://host port=8082 host=host kind=connect'
+p 'connect://user:pass@host:8082' 'name=connect://user:pass@host port=8082 host=host kind=connect'
+# only the scheme selects connect mode: a host literally named "connect" is a
+# plain http proxy, not a tunnel
+p 'connect:8080' 'name=connect port=8080 host=connect kind=http'
+p 'connect.example.com:3128' 'name=connect.example.com port=3128 host=connect.example.com kind=http'
 
 # the split is byte-transparent: percent-escapes stay encoded (decoded later at
 # auth time), only structural ASCII ':'/'@' delimit, and the last '@' ends the
 # userinfo. So %40/%3A and UTF-8 bytes never mis-split host or port.
-p 'socks5://user:p%40ss@host:1080' 'name=socks5://user:p%40ss@host port=1080 host=host'
-p 'socks5://us%3Aer:pass@host:1080' 'name=socks5://us%3Aer:pass@host port=1080 host=host'
-p 'socks5://a@b:c@host:1080' 'name=socks5://a@b:c@host port=1080 host=host'
-p 'socks5://naïve:pass@héllo:1080' 'name=socks5://naïve:pass@héllo port=1080 host=héllo'
+p 'socks5://user:p%40ss@host:1080' 'name=socks5://user:p%40ss@host port=1080 host=host kind=socks'
+p 'socks5://us%3Aer:pass@host:1080' 'name=socks5://us%3Aer:pass@host port=1080 host=host kind=socks'
+p 'socks5://a@b:c@host:1080' 'name=socks5://a@b:c@host port=1080 host=host kind=socks'
+p 'socks5://naïve:pass@héllo:1080' 'name=socks5://naïve:pass@héllo port=1080 host=héllo kind=socks'
+
+# a bracketed IPv6 literal holds colons of its own: the backward scan must stop
+# at the ']', or a portless literal parses to a truncated name and port=1
+p 'http://[2001:db8::1]' 'name=http://[2001:db8::1] port=8080 host=[2001:db8::1] kind=http'
+p 'http://[2001:db8::1]:3128' 'name=http://[2001:db8::1] port=3128 host=[2001:db8::1] kind=http'
+p 'socks5://[::1]' 'name=socks5://[::1] port=1080 host=[::1] kind=socks'
+p 'socks5://[::1]:1080' 'name=socks5://[::1] port=1080 host=[::1] kind=socks'
+p 'socks5://user:pass@[::1]' 'name=socks5://user:pass@[::1] port=1080 host=[::1] kind=socks'
+p 'socks5://user:pass@[::1]:9050' 'name=socks5://user:pass@[::1] port=9050 host=[::1] kind=socks'
+p '[::1]' 'name=[::1] port=8080 host=[::1] kind=http'
+p '[::1]:3128' 'name=[::1] port=3128 host=[::1] kind=http'
+# the ']' only ends the scan: one outside a literal must eat neither the name nor
+# the port. The last two discriminate: when a real port follows the ']', a
+# truncating bug agrees by accident.
+p 'http://user:p]ass@host:80' 'name=http://user:p]ass@host port=80 host=host kind=http'
+p 'http://a]b:3128' 'name=http://a]b port=3128 host=a]b kind=http'
+p 'http://a]b' 'name=http://a]b port=8080 host=a]b kind=http'
+p 'http://a]b:c@host' 'name=http://a]b:c@host port=8080 host=host kind=http'
 
 # a lone ':' must not underflow the backward scan
-p ':' 'name= port=8080 host='
+p ':' 'name= port=8080 host= kind=http'
+
+# a port is *DIGIT in 1..65535, else the scheme default: sscanf("%d") used to
+# wrap past INT_MAX and hand back a garbage port (#602)
+p 'http://host:65535' 'name=http://host port=65535 host=host kind=http'
+p 'http://host:1' 'name=http://host port=1 host=host kind=http'
+p 'http://host:65536' 'name=http://host port=8080 host=host kind=http'
+p 'http://host:2147483648' 'name=http://host port=8080 host=host kind=http'
+p 'http://host:4294967296' 'name=http://host port=8080 host=host kind=http'
+p 'http://host:99999999999999' 'name=http://host port=8080 host=host kind=http'
+# discriminating: this one wrapped to a plausible 80, so range-checking the
+# sscanf result instead of rejecting the overflow still passes everything above
+p 'http://host:4294967376' 'name=http://host port=8080 host=host kind=http'
+p 'http://host:999999999999999999999999999999' 'name=http://host port=8080 host=host kind=http'
+p 'http://host:0' 'name=http://host port=8080 host=host kind=http'
+p 'http://host:-1' 'name=http://host port=8080 host=host kind=http'
+p 'http://host:80x' 'name=http://host port=8080 host=host kind=http'
+p 'http://host: 80' 'name=http://host port=8080 host=host kind=http'
+p 'http://host:' 'name=http://host port=8080 host=host kind=http'
+# the fallback is the scheme's own default, not a hardcoded 8080
+p 'socks5://host:99999999999999' 'name=socks5://host port=1080 host=host kind=socks'

tests/01_engine-topindex.test

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# hts_buildtopindex takes a system-charset path but verif_backblue below it
+# expects utf-8, mangling a non-ASCII project dir on Windows (#216, #217).
+dir=$(mktemp -d)
+trap 'rm -rf "$dir"' EXIT
+
+httrack -O /dev/null -#test=topindex "$dir" run |
+    grep -q "topindex self-test OK"

tests/01_engine-xfread.test

@@ -9,21 +9,23 @@ set -euo pipefail
 # drives both refusal paths and the accept path.
 
 out="$(httrack -O /dev/null -#test=xfread-limit)"
+# Match with here-strings, not `echo | grep -q`: under pipefail the early grep
+# exit SIGPIPEs echo and fails the pipeline even when the pattern matched.
 for case in bylen bygrow; do
-    echo "$out" | grep -q "${case}: refused=1 adr=null msg=In-memory content too large" || {
+    grep -q "${case}: refused=1 adr=null msg=In-memory content too large" <<<"$out" || {
         echo "FAIL ${case}: $out"
         exit 1
     }
 done
 
 # Exactly INT32_MAX must be refused too (the reallocs add 1).
-echo "$out" | grep -q 'boundary: msg=In-memory content too large' || {
+grep -q 'boundary: msg=In-memory content too large' <<<"$out" || {
     echo "FAIL boundary: $out"
     exit 1
 }
 
 # The guard must NOT fire for a legitimate small size.
-if echo "$out" | grep -q 'accept: msg=In-memory content too large'; then
+if grep -q 'accept: msg=In-memory content too large' <<<"$out"; then
     echo "FAIL accept (guard fired on a legit size): $out"
     exit 1
 fi

tests/52_local-socks5.test

@@ -165,6 +165,13 @@ auth="$tmpdir/auth"
 start_server "$auth" ok
 run_crawl "$auth/out" "http://${host}:${http_port}/" \
     "socks5://user:secret@127.0.0.1:${socks_port}"
+# name a rejected version byte here: the crawl check below only knows the
+# handshake failed, not why
+grep -q "^AUTHVER-BAD" "$auth/socks.log" && {
+    echo "FAIL: sub-negotiation version was not RFC 1929's 0x01" >&2
+    cat "$auth/socks.log" >&2
+    exit 1
+}
 grep -rq "ORIGIN-PAGE-563" "$auth/out" || {
     echo "FAIL: origin not downloaded through the authenticated socks proxy" >&2
     cat "$auth/out.log" "$auth/socks.log" >&2

tests/56_local-proxy-noleak.test

@@ -0,0 +1,93 @@
+#!/bin/bash
+#
+# A crawl through a proxy must never resolve or dial the origin itself: the proxy
+# does that. A dead proxy + a multi-address origin used to fall back to dialing
+# the origin direct (bypassing the proxy, leaking its DNS and IP). The decoy
+# origin here must therefore receive nothing.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
+if test "${V6_SUPPORT:-}" == "no"; then
+    echo "no IPv6 support (resolver override compiled out), skipping"
+    exit 77
+fi
+python=$(find_python) || {
+    echo "python3 missing, skipping"
+    exit 77
+}
+
+server=$(nativepath "$top_srcdir/tests/local-server.py")
+root=$(nativepath "$top_srcdir/tests/server-root")
+tmpdir=$(mktemp -d)
+serverpid=
+
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+    return 0
+}
+trap cleanup EXIT
+
+# decoy origin: it must stay silent. LOCAL_SERVER_VERBOSE logs any request it gets.
+LOCAL_SERVER_VERBOSE=1 "$python" "$server" --root "$root" --bind 127.0.0.1 \
+    >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$tmpdir/srv.out" 2>/dev/null || true)
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$tmpdir/srv.err")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+
+# a proxy port nothing listens on: grab a free one and let it close (refuses).
+deadproxy=$("$python" -c \
+    'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()')
+
+# origin resolves to two addresses so a fallback would have a second to dial;
+# 127.0.0.1 (the decoy) is the one the old bypass reached.
+out="$tmpdir/crawl"
+HTTRACK_DEBUG_RESOLVE="decoyhost:127.0.0.2,127.0.0.1" \
+    httrack "http://decoyhost:$port/simple/basic.html" -O "$out" \
+    -P "127.0.0.1:$deadproxy" -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z \
+    >"$tmpdir/log" 2>&1
+
+log="$out/hts-log.txt"
+
+# the origin must have seen no connection (no leak/bypass)
+if grep -qE '"(GET|POST|HEAD|CONNECT) ' "$tmpdir/srv.err"; then
+    echo "FAIL: origin was contacted directly, bypassing the proxy"
+    cat "$tmpdir/srv.err"
+    exit 1
+fi
+
+# the crawl must fail at the proxy (proves it really tried the proxy, not a no-op)
+grep -q 'Connect Error' "$log" || {
+    echo "FAIL: expected a proxy connect error"
+    cat "$log"
+    exit 1
+}
+
+# and it must not have tried an origin-address fallback under the proxy
+if grep -q "trying next address" "$log"; then
+    echo "FAIL: fell back to an origin address under a proxy"
+    cat "$log"
+    exit 1
+fi
+
+echo "OK: a dead proxy fails cleanly without dialing the origin"

tests/57_local-proxy-connect.test

@@ -0,0 +1,168 @@
+#!/bin/bash
+#
+# Issue #564: plain http through a CONNECT-only proxy (tor HTTPTunnelPort). With
+# a "connect://" proxy httrack must open a CONNECT tunnel to the origin and send
+# the request origin-form, like https already does, instead of the absolute-URI
+# form a CONNECT-only proxy rejects. Fully local: a plain-http origin behind a
+# proxy that answers CONNECT and 501s everything else.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
+python=$(find_python) || python=
+if test -z "$python"; then
+    echo "python3 missing, skipping"
+    exit 77
+fi
+
+server=$(nativepath "$top_srcdir/tests/proxy-connect-server.py")
+tmpdir=$(mktemp -d)
+pids=
+
+cleanup() {
+    for pid in $pids; do
+        stop_server "$pid"
+    done
+    rm -rf "$tmpdir"
+}
+trap cleanup EXIT
+
+# start_server <logdir> <mode>: launches a proxy+origin pair, sets $origin_port
+# and $proxy_port from its announced ephemeral ports.
+start_server() {
+    local dir="$1" mode="$2" ports
+    mkdir -p "$dir"
+    ports="$dir/ports.txt"
+    dir_native=$(nativepath "$dir")
+    : >"$ports"
+    "$python" "$server" "$dir_native" "$mode" \
+        >"$ports" 2>"$dir/server.err" &
+    pids="$pids $!"
+    for _ in $(seq 1 100); do
+        grep -q "^ready" "$ports" 2>/dev/null && break
+        sleep 0.1
+    done
+    grep -q "^ready" "$ports" 2>/dev/null || {
+        echo "server ($mode) did not start" >&2
+        cat "$dir/server.err" >&2
+        exit 1
+    }
+    origin_port=$(awk '/^ORIGIN/{print $2}' "$ports")
+    proxy_port=$(awk '/^PROXY/{print $2}' "$ports")
+}
+
+# Kill httrack after a deadline so a hang surfaces as $HANG_RC, not a stalled job
+# (a portable stand-in for `timeout`, which macOS lacks).
+HANG_RC=137 # 128 + SIGKILL
+run_crawl() {
+    local out="$1" proxy="$2" url="${4:-http://127.0.0.1:${3}/}"
+    rm -rf "$out"
+    httrack "$url" --proxy "$proxy" \
+        -O "$out" -r1 -s0 --timeout=10 >"$out.log" 2>&1 &
+    local pid=$!
+    (sleep 60 && kill -9 "$pid" 2>/dev/null) &
+    local guard=$!
+    local rc=0
+    wait "$pid" 2>/dev/null || rc=$?
+    kill "$guard" 2>/dev/null || true
+    wait "$guard" 2>/dev/null || true
+    return "$rc"
+}
+
+# --- working proxy ----------------------------------------------------------
+ok="$tmpdir/ok"
+start_server "$ok" ok
+
+# 1. page fetched through a CONNECT tunnel, request delivered origin-form
+run_crawl "$ok/out" "connect://127.0.0.1:${proxy_port}" "$origin_port"
+grep -rq "ORIGIN-PAGE-564" "$ok/out" || {
+    echo "FAIL: origin page not downloaded through the CONNECT tunnel" >&2
+    cat "$ok/out.log" >&2
+    exit 1
+}
+grep -q "^CONNECT 127.0.0.1:${origin_port} " "$ok/proxy.log" || {
+    echo "FAIL: proxy never received a CONNECT for the plain-http origin" >&2
+    cat "$ok/proxy.log" >&2
+    exit 1
+}
+grep -q "^GET / HTTP/1" "$ok/origin-headers.log" || {
+    echo "FAIL: origin did not see an origin-form request line" >&2
+    cat "$ok/origin-headers.log" >&2
+    exit 1
+}
+if grep -q "^GET http://" "$ok/origin-headers.log"; then
+    echo "FAIL: absolute-URI request leaked through the tunnel to the origin" >&2
+    exit 1
+fi
+echo "OK: plain http tunneled origin-form through CONNECT"
+
+# 2. default CONNECT port follows the origin scheme: a portless http origin
+# tunnels to :80 (not the https :443). Nothing serves :80 here so the fetch
+# fails; only the CONNECT target port is asserted. Reverting to a hardcoded 443
+# makes this log ":443" and fail.
+: >"$ok/proxy.log"
+run_crawl "$ok/out_port" "connect://127.0.0.1:${proxy_port}" "" "http://127.0.0.1/" || true
+grep -q "^CONNECT 127.0.0.1:80 " "$ok/proxy.log" || {
+    echo "FAIL: portless http origin did not CONNECT to the default port 80" >&2
+    cat "$ok/proxy.log" >&2
+    exit 1
+}
+echo "OK: portless http origin tunnels to the default port 80"
+
+# 3. teeth: without connect://, httrack sends the absolute-URI form, which the
+# CONNECT-only proxy rejects (501) -> nothing fetched. Proves the scheme is what
+# flips the behavior and that the proxy really is CONNECT-only.
+: >"$ok/proxy.log"
+: >"$ok/origin-headers.log"
+run_crawl "$ok/out_plain" "127.0.0.1:${proxy_port}" "$origin_port" || true
+grep -rq "ORIGIN-PAGE-564" "$ok/out_plain" && {
+    echo "FAIL: absolute-URI form unexpectedly fetched through a CONNECT-only proxy" >&2
+    exit 1
+}
+grep -q "^GET http://127.0.0.1:${origin_port}/" "$ok/proxy.log" || {
+    echo "FAIL: a plain proxy should have received an absolute-URI GET" >&2
+    cat "$ok/proxy.log" >&2
+    exit 1
+}
+echo "OK: absolute-URI form (no connect://) rejected by the CONNECT-only proxy"
+
+# 4. authenticated proxy: creds ride the CONNECT, never reach the origin
+: >"$ok/proxy.log"
+: >"$ok/origin-headers.log"
+run_crawl "$ok/out2" "connect://user:secret@127.0.0.1:${proxy_port}" "$origin_port"
+grep -rq "ORIGIN-PAGE-564" "$ok/out2" || {
+    echo "FAIL: origin page not downloaded through the authenticated proxy" >&2
+    exit 1
+}
+got=$(awk '/^AUTH Basic /{print $3}' "$ok/proxy.log" | head -1)
+# base64("user:secret"); compared literally to stay portable (no base64 -d)
+test "$got" == "dXNlcjpzZWNyZXQ=" || {
+    echo "FAIL: Proxy-Authorization not carried on CONNECT (got '$got')" >&2
+    cat "$ok/proxy.log" >&2
+    exit 1
+}
+if grep -qi "proxy-authorization" "$ok/origin-headers.log"; then
+    echo "FAIL: proxy credentials leaked to the origin through the tunnel" >&2
+    exit 1
+fi
+echo "OK: proxy credentials carried on CONNECT, not leaked to origin"
+
+# --- hostile proxy ----------------------------------------------------------
+# A proxy that answers 200 then streams headers forever must not hang the crawl.
+flood="$tmpdir/flood"
+start_server "$flood" flood
+rc=0
+run_crawl "$flood/out" "connect://127.0.0.1:${proxy_port}" "$origin_port" || rc=$?
+test "$rc" -ne "$HANG_RC" || {
+    echo "FAIL: crawl hung on a flooding proxy (bounded read missing)" >&2
+    exit 1
+}
+grep -rq "ORIGIN-PAGE-564" "$flood/out" 2>/dev/null && {
+    echo "FAIL: flooding proxy unexpectedly served the page" >&2
+    exit 1
+}
+echo "OK: bounded proxy response, no hang on a flooding proxy"

tests/58_watchdog.test

@@ -0,0 +1,73 @@
+#!/bin/bash
+#
+# Unit-tests the suite watchdog (run_with_timeout/kill_tree in testlib.sh): a
+# hung child must be killed tree-and-all and reported 124, never left to orphan
+# a spinning httrack.exe. Regression guard for the Windows x64 hangs that read as
+# "the hosted runner lost communication with the server".
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_wd.XXXXXX")
+trap 'rm -rf "$tmp"' EXIT
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+# Exit status passes through when the command finishes in time.
+rc=0 && run_with_timeout 30 sh -c 'exit 0' || rc=$?
+test "$rc" -eq 0 || fail "clean exit reported $rc"
+rc=0 && run_with_timeout 30 sh -c 'exit 7' || rc=$?
+test "$rc" -eq 7 || fail "exit 7 reported $rc"
+
+# An overrunning command is reported 124, near the deadline not the child's end.
+start=$SECONDS
+rc=0
+if is_windows; then
+    run_with_timeout 2 ping -n 30 127.0.0.1 || rc=$?
+else
+    run_with_timeout 2 sleep 30 || rc=$?
+fi
+test "$rc" -eq 124 || fail "hang reported $rc, want 124"
+test "$((SECONDS - start))" -lt 15 || fail "watchdog fired late"
+
+# The native descendant tree is really dead, not orphaned. On Windows an outer
+# MSYS bash with a native ping grandchild is the exact case signals can't reap;
+# on POSIX a surviving grandchild would touch the marker after we stop waiting.
+rc=0
+if is_windows; then
+    # Existence by exact Windows PID, not a global ping.exe count: the timing
+    # sub-test above leaves a still-dying ping that a count would race. Plain
+    # tasklist, no switches (the workflow's MSYS2_ARG_CONV_EXCL='*' mangles a
+    # //FI filter arg into a silent no-match); $2 is the PID column.
+    alive() { tasklist 2>/dev/null | awk -v p="$1" '$2 == p {f = 1} END {exit !f}'; }
+    # The grandchild ping records its own Windows PID: non-empty proves it ran
+    # (positive control, so the reap check can't pass vacuously), then it must go.
+    gw="$tmp/gcwinpid"
+    # `wait` keeps the bash alive on the ping past the deadline; without it the
+    # bash exits at once and run_with_timeout returns 0 instead of timing out.
+    # shellcheck disable=SC2016 # $1 expands in the bash -c child, by design
+    run_with_timeout 2 bash -c 'ping -n 30 127.0.0.1 >/dev/null 2>&1 & cat /proc/$!/winpid >"$1"; wait' _ "$gw" || rc=$?
+    test "$rc" -eq 124 || fail "grandchild hang reported $rc"
+    w=$(cat "$gw" 2>/dev/null)
+    test -n "$w" || fail "positive control: grandchild ping never started"
+    sleep 1
+    ! alive "$w" || fail "native ping grandchild (pid $w) survived the watchdog"
+else
+    started="$tmp/started" marker="$tmp/alive"
+    # The grandchild (backgrounded subshell) marks that it ran, then would touch
+    # the marker after the deadline; killing only the direct child orphans it, so
+    # the marker still appears. started present + marker absent = ran and reaped.
+    # shellcheck disable=SC2016 # $1/$2 expand in the sh -c child, by design
+    run_with_timeout 2 sh -c '{ : >"$1"; sleep 10; : >"$2"; } & wait' _ "$started" "$marker" || rc=$?
+    test "$rc" -eq 124 || fail "grandchild hang reported $rc"
+    test -e "$started" || fail "positive control: grandchild never ran"
+    sleep 12
+    test ! -e "$marker" || fail "watchdog left a grandchild running"
+fi
+
+echo "watchdog OK"

tests/59_local-tls-stall.test

@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Issue #607: a peer that accepts the TCP connect but never speaks TLS leaves the
+# slot stuck in the handshake. The per-slot --timeout must reap it; before the
+# fix only --max-time did, so these crawls ran until the kill guard.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
+if test "${HTTPS_SUPPORT:-}" == "no"; then
+    echo "no https support compiled, skipping"
+    exit 77
+fi
+python=$(find_python) || {
+    echo "python3 missing, skipping"
+    exit 77
+}
+
+server=$(nativepath "$top_srcdir/tests/tls-stall-server.py")
+tmpdir=$(mktemp -d)
+serverpid=
+
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+    return 0
+}
+trap cleanup EXIT
+
+# start_stall_server <tag> <mode-args...>: sets $port from the announced one.
+start_stall_server() {
+    local tag="$1"
+    shift
+    "$python" "$server" "$@" >"$tmpdir/$tag.out" 2>"$tmpdir/$tag.err" &
+    serverpid=$!
+    port=
+    for _ in $(seq 1 50); do
+        line=$(head -n1 "$tmpdir/$tag.out" 2>/dev/null || true)
+        if test "${line%% *}" == "PORT"; then
+            port="${line#PORT }"
+            return 0
+        fi
+        kill -0 "$serverpid" 2>/dev/null || {
+            echo "server exited early: $(cat "$tmpdir/$tag.err")"
+            exit 1
+        }
+        sleep 0.1
+    done
+    echo "could not discover server port"
+    exit 1
+}
+
+# no -E/--max-time on purpose: --timeout is the only thing that can end these.
+# The kill guard stands in for the hang, so a wall time near it means no reap.
+crawl_wall() {
+    local out="$tmpdir/$1"
+    shift
+    local start
+    start=$(date +%s)
+    run_with_timeout 60 httrack -O "$out" -c1 --robots=0 --retries=0 --quiet -Z \
+        "$@" >>"$tmpdir/log" 2>&1 || true
+    echo $(($(date +%s) - start))
+}
+
+# 1. handshake stalled from the first byte: reaped at --timeout, not before.
+start_stall_server direct direct
+wall=$(crawl_wall crawl1 "https://127.0.0.1:$port/" --timeout=5)
+if test "$wall" -ge 30 || test "$wall" -lt 3; then
+    echo "FAIL: stalled handshake reaped after ${wall}s, expected about 5s" >&2
+    cat "$tmpdir/log" >&2
+    exit 1
+fi
+grep -q 'Handshake Time Out' "$tmpdir/crawl1/hts-log.txt" || {
+    echo "FAIL: crawl ended in ${wall}s but not on a handshake timeout" >&2
+    cat "$tmpdir/crawl1/hts-log.txt" >&2
+    exit 1
+}
+echo "OK: stalled TLS handshake reaped by --timeout after ${wall}s"
+
+# 2. the handshake window is its own, not what the connect left over: a proxy
+# that takes 4s to answer CONNECT must still leave the full --timeout=5 for the
+# handshake (~9s total). Sharing the connect's clock would reap at ~5s.
+stop_server "$serverpid"
+start_stall_server proxy proxy 4
+wall=$(crawl_wall crawl2 "https://127.0.0.1:443/" -P "127.0.0.1:$port" --timeout=5)
+if test "$wall" -ge 20 || test "$wall" -lt 7; then
+    echo "FAIL: handshake after a slow connect reaped at ${wall}s, expected about 9s" >&2
+    cat "$tmpdir/log" >&2
+    exit 1
+fi
+echo "OK: handshake keeps its own timeout window after a slow connect (${wall}s)"

tests/60_crawl-log-salvage.test

@@ -0,0 +1,46 @@
+#!/bin/bash
+#
+# Unit-tests dump_crawl_logs (testlib.sh): the killed test's hts-log.txt must
+# reach the uploaded logs, or a Windows wedge past --max-time stays
+# undiagnosable (#605).
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/httrack_salv.XXXXXX")
+export TMPDIR
+trap 'rm -rf "$TMPDIR"' EXIT
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+verdict="More than 120 seconds passed.. giving up"
+
+# Vacuity control: with nothing left behind the assertions below cannot pass on
+# ambient output.
+test -z "$(dump_crawl_logs)" || fail "dumped something with no crawl tmpdir"
+
+# A killed local-crawl.sh leaves exactly this shape.
+d="${TMPDIR}/httrack_local.probe"
+mkdir -p "$d/crawl"
+echo "$verdict" >"$d/crawl/hts-log.txt"
+echo "httrack stdout marker" >"$d/log"
+
+# Append to an unterminated line, as a killed test's own log always ends.
+art="${TMPDIR}/artifact.log"
+printf '[running httrack ...] ..\t' >"$art"
+dump_crawl_logs >>"$art"
+
+grep -qF "$verdict" "$art" || fail "hts-log.txt verdict not salvaged"
+grep -qF "httrack stdout marker" "$art" || fail "httrack stdout not salvaged"
+grep -q '^--- .*hts-log.txt' "$art" || fail "dump glued its header onto the unterminated line"
+
+# Cleared, so the next timing-out test cannot re-report this crawl as its own.
+test ! -d "$d" || fail "salvaged tmpdir left behind"
+test -z "$(dump_crawl_logs)" || fail "salvaged crawl reported twice"
+
+echo "crawl log salvage OK"

tests/61_webhttrack-locale.test

@@ -0,0 +1,91 @@
+#!/bin/bash
+#
+# webhttrack must map the caller's locale to the right lang.indexes number.
+# Guards: LC_ALL/LC_MESSAGES were ignored, and zh_tw/pt_br were unreachable.
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+distdir=$(cd "${testdir}/.." && pwd)
+script="${distdir}/src/webhttrack"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+test -r "${script}" || fail "no ${script}"
+test -r "${distdir}/lang.indexes" || fail "no ${distdir}/lang.indexes"
+
+# Lift the locale code out of the launcher, so this tests the shipped logic itself.
+funcs=$(sed -n '/^function lang_index/,/^}/p' "${script}")
+# End on the next section, not a blank line: a blank line grown inside the block
+# would truncate the lift and silently stop testing everything below it.
+grep -q '^# Find the browser' "${script}" || fail "locale block terminator moved in ${script}"
+block=$(sed -n '/^# Locale/,/^# Find the browser/p' "${script}" | sed '$d')
+echo "${block}" | grep -q 'LANGN=.*lang_index' || fail "could not lift the whole locale block from ${script}"
+
+# Run the lifted block against whatever locale vars the caller exported.
+run_block() {
+    # shellcheck disable=SC2034 # read by the eval'd locale block
+    DISTPATH="${distdir}"
+    # the launcher runs under neither -e ("! test -n x && y" returns 1) nor -u
+    set +eu
+    eval "${funcs}"
+    eval "${block}"
+    # shellcheck disable=SC2153 # LANGN is set by the eval'd block, not a LANG typo
+    echo "${LANGN}"
+}
+
+# Resolve LANG/LC_ALL/LC_MESSAGES (empty = unset) to the language number.
+lang_of() {
+    (
+        unset LANG LC_ALL LC_MESSAGES
+        # only the value matters here; hide bash's setlocale gripe when the locale is not installed
+        # shellcheck disable=SC2030 # confining the locale to this subshell is the point
+        {
+            if test -n "$1"; then export LANG="$1"; fi
+            if test -n "$2"; then export LC_ALL="$2"; fi
+            if test -n "$3"; then export LC_MESSAGES="$3"; fi
+        } 2>/dev/null
+        run_block
+    )
+}
+
+check() {
+    local want=$1 got desc
+    shift
+    desc="LANG='$1' LC_ALL='$2' LC_MESSAGES='$3'"
+    got=$(lang_of "$1" "$2" "$3")
+    test "${got}" = "${want}" || fail "${desc}: want ${want}, got ${got}"
+    echo "ok: ${desc} -> ${got}"
+}
+
+#      want  LANG            LC_ALL          LC_MESSAGES
+check 2 "fr_FR.UTF-8" "" ""
+check 14 "zh_TW.UTF-8" "" ""                      # region-qualified: Chinese-BIG5, not Simplified (13)
+check 12 "pt_BR.UTF-8" "" ""                      # region-qualified: Portugues-Brasil, not Portugues (7)
+check 13 "zh_CN.UTF-8" "" ""                      # unlisted region falls back to the bare language
+check 24 "sl_SI.UTF-8" "" ""                      # Slovenian is ISO 639-1 "sl"
+check 4 "" "de_DE.UTF-8" ""                       # LC_ALL alone
+check 4 "" "" "de_DE.UTF-8"                       # LC_MESSAGES alone
+check 4 "fr_FR.UTF-8" "de_DE.UTF-8" ""            # LC_ALL wins over LANG
+check 4 "fr_FR.UTF-8" "" "de_DE.UTF-8"            # LC_MESSAGES wins over LANG
+check 2 "de_DE.UTF-8" "fr_FR.UTF-8" "de_DE.UTF-8" # LC_ALL wins over LC_MESSAGES
+check 2 "fr_FR@euro" "" ""                        # @modifier without a charset
+check 1 "C" "" ""                                 # C, unknown and unset all mean English
+check 1 "xx_YY.UTF-8" "" ""
+check 1 "" "" ""
+
+# Exported but empty is unset (POSIX), so it must not mask LANG. lang_of cannot
+# express this: it exports nothing for an empty argument.
+got=$(
+    unset LANG LC_ALL LC_MESSAGES
+    # shellcheck disable=SC2031 # confining the locale to this subshell is the point
+    { export LANG="fr_FR.UTF-8" LC_ALL="" LC_MESSAGES=""; } 2>/dev/null
+    run_block
+)
+test "${got}" = 2 || fail "empty LC_ALL/LC_MESSAGES must not mask LANG: want 2, got ${got}"
+echo "ok: LANG='fr_FR.UTF-8' with LC_ALL='' exported -> ${got}"
+
+echo "PASS"

tests/62_lang-integrity.test

@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# lang/*.txt are line pairs: an English msgid, then its translation. A stray
+# blank or a drifted msgid silently unhooks every translation that follows,
+# rather than failing, so assert the pairing and the join against English.txt.
+# Pairs are physical lines here; a msgid ending in \ continues onto the next one
+# for the engine (linput_cpp) but not for us. None do today.
+
+set -euo pipefail
+
+langdir="${top_srcdir:-..}/lang"
+def="${top_srcdir:-..}/lang.def"
+eng="$langdir/English.txt"
+for f in "$eng" "$def"; do
+    [ -f "$f" ] || {
+        echo "cannot find $f"
+        exit 1
+    }
+done
+
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+keys="$tmp/english.keys"
+
+# Byte-wise: each file is in its own declared legacy charset, lang.def is CRLF.
+LC_ALL=C awk 'NR%2==1 { sub(/\r$/, ""); print }' "$eng" >"$keys"
+
+# An empty parse would make every check below pass vacuously.
+nkeys=$(wc -l <"$keys")
+if [ "$nkeys" -lt 400 ]; then
+    echo "only $nkeys msgids parsed from $eng; the parse is broken"
+    exit 1
+fi
+
+nfiles=0
+fail=0
+for f in "$langdir"/*.txt; do
+    nfiles=$((nfiles + 1))
+    LC_ALL=C awk -v keys="$keys" -v name="${f##*/}" '
+        BEGIN { while ((getline k < keys) > 0) known[k] = 1 }
+        { sub(/\r$/, "") }
+        NR % 2 == 1 {
+            if ($0 == "")
+                printf "%s:%d: blank line where an English msgid belongs\n", name, NR
+            else if (!($0 in known))
+                printf "%s:%d: msgid absent from English.txt: %s\n", name, NR, $0
+            else
+                next
+            bad++
+        }
+        END {
+            if (NR % 2) {
+                printf "%s: odd line count (%d): msgid/translation pairing is broken\n", name, NR
+                bad++
+            }
+            exit(bad > 0)
+        }
+    ' "$f" || fail=1
+done
+
+if [ "$nfiles" -lt 25 ]; then
+    echo "only $nfiles language files found in $langdir"
+    exit 1
+fi
+
+# lang.def maps each LANG_* macro to the English string the GUI compiles in,
+# which is the lookup key into every lang/*.txt: no msgid, no translations.
+LC_ALL=C awk -v keys="$keys" '
+    BEGIN { while ((getline k < keys) > 0) known[k] = 1 }
+    { sub(/\r$/, "") }
+    NR % 2 == 1 { macro = $0; next }
+    macro ~ /^(LANG_|LISTDEF_)/ {
+        n++
+        if (!($0 in known)) {
+            printf "lang.def:%d: %s maps to \"%s\", which is not a msgid in English.txt\n", NR, macro, $0
+            bad++
+        }
+    }
+    END {
+        if (n < 400) {
+            printf "lang.def: only %d LANG_ entries parsed; the parse is broken\n", n
+            bad++
+        }
+        exit(bad > 0)
+    }
+' "$def" || fail=1
+
+[ "$fail" -eq 0 ] || exit 1
+
+echo "$nfiles language files consistent with $nkeys English msgids"

tests/63_webhttrack-home.test

@@ -0,0 +1,88 @@
+#!/bin/bash
+#
+# htsserver's $HOME lookup, through the web UI it actually renders. An exported
+# but empty $HOME must not resolve the default base path against the root.
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
+distdir=$(cd "${distdir}" && pwd)
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+command -v htsserver >/dev/null || fail "no htsserver in PATH"
+# Mirror local-crawl.sh's skip-with-77: the Debian buildd chroot has no python3.
+command -v python3 >/dev/null || {
+    echo "python3 not found; skipping" >&2
+    exit 77
+}
+
+srv=
+cleanup() {
+    test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
+}
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+log=$(mktemp)
+trap 'cleanup; rm -f "${log}"' EXIT
+
+# Ask htsserver for its rendered default base path, under the given $HOME
+# ("-" = leave $HOME unset).
+pathfor() {
+    local homeval=$1 port url got
+    port=$(python3 -c 'import socket
+s = socket.socket()
+s.bind(("127.0.0.1", 0))
+print(s.getsockname()[1])
+s.close()')
+
+    : >"${log}"
+    (
+        # htsserver ignores SIGTERM/SIGTTOU handling from the harness shell
+        trap '' TERM TTOU
+        if test "${homeval}" = -; then unset HOME; else export HOME="${homeval}"; fi
+        exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
+    ) &
+    srv=$!
+
+    for _ in $(seq 1 40); do
+        url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
+        kill -0 "${srv}" 2>/dev/null || break
+        sleep 0.25
+    done
+    test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
+
+    # The UI is served ISO-8859-1, so keep python out of text mode.
+    got=$(python3 -c 'import re, sys, urllib.request
+body = urllib.request.urlopen(sys.argv[1] + "server/step2.html", timeout=20).read()
+m = re.search(br"name=\"path\" value=\"([^\"]*)\"", body)
+print(m.group(1).decode("latin-1") if m else "")' "${url}")
+
+    kill -9 "${srv}" 2>/dev/null || true
+    wait "${srv}" 2>/dev/null || true
+    srv=
+    test -n "${got}" || fail "no path field in the rendered step2.html"
+    echo "${got}"
+}
+
+check() {
+    local desc=$1 homeval=$2 want=$3 got
+    got=$(pathfor "${homeval}")
+    test "${got}" = "${want}" || fail "${desc}: want ${want}, got ${got}"
+    echo "ok: ${desc} -> ${got}"
+}
+
+# A real HOME must reach the page, or the two cases below are vacuous: they
+# also pass when gethomedir() ignores the environment and always answers ".".
+check 'a set HOME is used' /nonexistent/hts-home /nonexistent/hts-home/websites
+
+# Exported but empty is the regression: getenv() returns "", not NULL, so a
+# bare NULL check let the base path collapse onto /websites.
+check 'an empty HOME' '' ./websites
+check 'an unset HOME' - ./websites
+
+echo "PASS"

tests/64_local-intl-outdir.test

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# A non-ASCII -O output dir must receive the mirror, not a double-encoded twin
+# (#621). argv is UTF-8 on Windows, so path_html was re-encoded as if it were
+# the ANSI codepage, prefixing every saved file with a mangled directory. Only
+# path_html is the non-ASCII "café"; path_log stays ASCII so the harness reads
+# hts-log.txt/hts-cache normally and this isolates the -O encoding path.
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --html-subdir 'café' --errors 0 --files 5 \
+    --found 'simple/basic.html' \
+    httrack 'BASEURL/simple/basic.html'

tests/65_port-siblings.test

@@ -0,0 +1,127 @@
+#!/bin/bash
+#
+
+# Port parsing outside the crawled-URL path (#614): the htsserver --port listen
+# port, proxytrack's host:port listen arguments, and an ftp:// URL port. All
+# three took the port from sscanf("%d"), which range-checks nothing and wraps
+# past INT_MAX, so a value naming no real port silently became a plausible one.
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# run_with_timeout: timeout(1) is absent on macOS, and the listen servers block.
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_portsib.XXXXXX") || exit 1
+trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+
+# A port the old code accepted only by wrapping: 4295009395 = 2^32 + 42099 and
+# 4294967376 = 2^32 + 80 both fold to a plausible listen port. They are the
+# load-bearing cases, since a value that merely exceeds 65535 while still
+# fitting an int (65616) was already caught by the htsserver range check.
+overflowing="4295009395"
+
+# Values sscanf("%d") took as a port that no operator meant: a trailing
+# character, a sign, leading whitespace, a decimal point. Each silently listened
+# on the prefix it managed to scan.
+malformed=("80x" "+80" " 80" "8.0" "")
+
+# --- htsserver --port ------------------------------------------------------
+# Its argv[1] is the html root; the option pairs follow. It blocks serving once
+# a port is accepted, so bound it; we assert on the output, not the exit status.
+
+websrv() {
+    run_with_timeout 3 htsserver "$tmp" --port "$1" >"$tmp/web.log" 2>&1 || true
+}
+
+web_refused() {
+    websrv "$1"
+    grep -q "couldn't set the port number" "$tmp/web.log" ||
+        ! echo "FAIL: #614: htsserver --port '$1' not refused" || exit 1
+    # it must not have reached the listen socket on some other port
+    ! grep -q "^URL=" "$tmp/web.log" ||
+        ! echo "FAIL: #614: htsserver --port '$1' listened anyway" || exit 1
+}
+
+web_accepted() {
+    local port=$1
+    : >"$tmp/web.log"
+    # A bound server blocks, so stop it once it announces, not on a fixed
+    # deadline: the announce trails a local-hostname resolve that stalls on macOS,
+    # where a 3s kill landed mid-resolve, before the URL= line ever printed.
+    local had_m=
+    case "$-" in *m*) had_m=1 ;; esac
+    is_windows || set -m
+    htsserver "$tmp" --port "$port" >"$tmp/web.log" 2>&1 &
+    local pid=$!
+    test -n "$had_m" || is_windows || set +m
+    local waited=0
+    until grep -qE "^URL=http://.*:$port/|couldn't set the port number|Unable to initialize a temporary server" "$tmp/web.log"; do
+        test "$waited" -lt 20 || break
+        sleep 1
+        waited=$((waited + 1))
+    done
+    kill_tree "$pid"
+    wait "$pid" 2>/dev/null || true
+    ! grep -q "couldn't set the port number" "$tmp/web.log" ||
+        ! echo "FAIL: #614: htsserver --port '$port' wrongly refused" || exit 1
+    grep -qE "^URL=http://.*:$port/|Unable to initialize a temporary server" "$tmp/web.log" ||
+        ! echo "FAIL: #614: htsserver --port '$port' never reached the listener" || exit 1
+}
+
+# this used to listen on 42099
+web_refused "$overflowing"
+for p in "${malformed[@]}" 0 65536 99999 -1; do
+    web_refused "$p"
+done
+
+# 65535 is a valid port the old "< 65535" bound refused
+web_accepted 65535
+
+# --- proxytrack <proxy-addr:port> <ICP-addr:port> --------------------------
+# A bad argument falls through to the usage screen; it had no range check at
+# all, so 65616 quietly listened on port 80. A valid one binds and blocks.
+
+proxy_refused() {
+    run_with_timeout 3 proxytrack "127.0.0.1:$1" 127.0.0.1:3130 >"$tmp/pt.log" 2>&1 || true
+    grep -q "^usage:" "$tmp/pt.log" ||
+        ! echo "FAIL: #614: proxytrack port '$1' not refused" || exit 1
+}
+
+proxy_accepted() {
+    run_with_timeout 3 proxytrack "127.0.0.1:$1" 127.0.0.1:3130 >"$tmp/pt.log" 2>&1 || true
+    ! grep -q "^usage:" "$tmp/pt.log" ||
+        ! echo "FAIL: #614: proxytrack port '$1' wrongly refused" || exit 1
+}
+
+for p in "${malformed[@]}" 65616 4294967376 "$overflowing" 0 -1; do
+    proxy_refused "$p"
+done
+proxy_accepted 45678
+
+# --- ftp:// URL port -------------------------------------------------------
+# A URL port, so it follows the crawled-URL policy: refuse the link rather than
+# fold it into range. An ftp:// link in scanned HTML is not followed, so the
+# seed has to come from the command line.
+
+ftp_port() {
+    local out="$tmp/ftp"
+
+    rm -rf "$out"
+    run_with_timeout 30 httrack "ftp://127.0.0.1:$1/x" -O "$out" --quiet -n \
+        >/dev/null 2>&1 || true
+    grep -c "Invalid port: $1" "$out/hts-log.txt" 2>/dev/null || true
+}
+
+# 65616 truncated to 80 and really did connect there
+for p in 65616 99999; do
+    test "$(ftp_port "$p")" -gt 0 ||
+        ! echo "FAIL: #614: ftp:// port $p not refused" || exit 1
+done
+
+# a valid port still reaches the connect attempt
+test "$(ftp_port 65535)" -eq 0 ||
+    ! echo "FAIL: #614: ftp:// port 65535 wrongly refused" || exit 1
+
+exit 0

tests/66_engine-port80-strip.test

@@ -0,0 +1,11 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Stripping a default :80 from a crawled link used to skip a hardcoded 3 chars
+# (#627): ":080"/":0080" lost only ":80" and glued the rest onto the host
+# (127.0.0.1:080/x -> 127.0.0.10/x), and a value wrapping to 80 as a 32-bit int
+# (#614) was stripped as if it were the default. All assertions live in the
+# engine self-test.
+httrack -O /dev/null -#test=stripport | grep -q "stripport self-test OK"

tests/67_engine-delayed-truncate.test

@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# #623: url_savename enforces the 236-char path ceiling by cutting the tail of
+# the last segment, where the mandatory ".delayed" placeholder marker lives. A
+# cut marker fails IS_DELAYED_EXT, so back_delayed_rename never renames the file
+# to its final name and the download is lost. The marker must survive the cut.
+
+# statuscode=302 status=-1 = a redirect answer still downloading: no type is
+# resolved, so the name gets a ".<id>.delayed" placeholder (see 01_engine-savename).
+CEIL=236
+
+httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
+scratch=$(mktemp -d)
+trap 'rm -rf "$scratch"' EXIT
+cd "$scratch"
+
+run() {
+    "$httrack_bin" -O /dev/null -#test=savename "$@" | sed -n 's/^savename: //p'
+}
+
+# A deep path ending in a long segment that overruns the ceiling.
+deep="/d1$(printf 'a%.0s' {1..40})/d2$(printf 'b%.0s' {1..40})"
+deep="$deep/d3$(printf 'c%.0s' {1..40})/d4$(printf 'd%.0s' {1..40})"
+long="$deep/$(printf 'z%.0s' {1..90})"
+
+out="$(run "$long" text/html statuscode=302 status=-1)"
+case "$out" in
+*.delayed) ;;
+*)
+    echo "FAIL: delayed marker cut by truncation: '$out'"
+    exit 1
+    ;;
+esac
+test "${#out}" -le "$CEIL" || {
+    echo "FAIL: truncated name ${#out} > $CEIL ceiling: '$out'"
+    exit 1
+}
+
+# #133-style hashed name: an all-hex last segment must keep the marker too (the
+# ".<id>." tag is not mistaken for part of the hash).
+hexseg=$(printf 'a1b2c3d4e5f60718%.0s' {1..8})
+hexpath="/d1$(printf 'x%.0s' {1..30})/d2$(printf 'y%.0s' {1..30})/$hexseg"
+out="$(run "$hexpath" text/html statuscode=302 status=-1)"
+case "$out" in
+*.delayed) ;;
+*)
+    echo "FAIL: hashed-name marker cut by truncation: '$out'"
+    exit 1
+    ;;
+esac
+test "${#out}" -le "$CEIL" || {
+    echo "FAIL: hashed name ${#out} > $CEIL ceiling: '$out'"
+    exit 1
+}
+
+# A non-delayed name of the same shape still truncates, with no marker to keep.
+out="$(run "$long.html" text/html)"
+test "${#out}" -le "$CEIL" || {
+    echo "FAIL: non-delayed name ${#out} > $CEIL ceiling: '$out'"
+    exit 1
+}
+case "$out" in
+*.delayed)
+    echo "FAIL: non-delayed name grew a .delayed marker: '$out'"
+    exit 1
+    ;;
+esac
+
+echo "delayed-truncate OK"

tests/69_local-intl-logdir.test

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# A single non-ASCII -O sets both path_html and path_log to "café" (#630). The
+# logs (hts-log.txt/hts-err.txt) must land there, not in an ANSI-mangled twin:
+# on Windows path_log holds UTF-8 bytes the raw file calls read as the codepage.
+# The --errors/--files/--log-found audits all grep logroot=café/hts-log.txt, so
+# a log written to the twin fails them. POSIX has no twin, so this bites on the
+# Windows CI leg (like test 64). The cache twin is a separate, larger fix.
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --outdir-intl 'café' --errors 0 --files 5 \
+    --found 'simple/basic.html' \
+    --log-found 'mirror complete in' \
+    httrack 'BASEURL/simple/basic.html'

tests/Makefile.am

@@ -2,7 +2,8 @@
 # explicitly: automake does not expand wildcards in EXTRA_DIST, so a glob would
 # silently drop it from the dist tarball and break "make distcheck".
 EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
-	proxy-https-server.py socks5-server.py \
+	proxy-https-server.py socks5-server.py proxy-connect-server.py \
+	proxytestlib.py tls-stall-server.py \
 	local-crawl.sh local-server.py testlib.sh server.crt server.key \
 	server-root/simple/basic.html server-root/simple/link.html \
 	server-root/stripquery/index.html server-root/stripquery/a.html \
@@ -34,6 +35,7 @@ TESTS = \
 	01_engine-copyopt.test \
 	01_engine-crange.test \
 	01_engine-dns.test \
+	01_engine-dnstimeout.test \
 	01_engine-doitlog.test \
 	01_engine-entities.test \
 	01_engine-filelist.test \
@@ -57,6 +59,7 @@ TESTS = \
 	01_engine-pause.test \
 	01_engine-rcfile.test \
 	01_engine-reconcile.test \
+	01_engine-expandhome.test \
 	01_engine-fsize.test \
 	01_engine-redirect.test \
 	01_engine-relative.test \
@@ -68,6 +71,7 @@ TESTS = \
 	01_engine-status.test \
 	01_engine-stripquery.test \
 	01_engine-strsafe.test \
+	01_engine-topindex.test \
 	01_engine-urlhack.test \
 	01_engine-unescape-bounds.test \
 	01_engine-useragent.test \
@@ -132,6 +136,19 @@ TESTS = \
 	52_local-socks5.test \
 	53_local-proxytrack-cache-corrupt.test \
 	54_local-update-truncate-purge.test \
-	55_local-chunked.test
+	55_local-chunked.test \
+	56_local-proxy-noleak.test \
+	57_local-proxy-connect.test \
+	58_watchdog.test \
+	59_local-tls-stall.test \
+	60_crawl-log-salvage.test \
+	61_webhttrack-locale.test \
+	62_lang-integrity.test \
+	63_webhttrack-home.test \
+	64_local-intl-outdir.test \
+	65_port-siblings.test \
+	66_engine-port80-strip.test \
+	67_engine-delayed-truncate.test \
+	69_local-intl-logdir.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -47,6 +47,8 @@ key="${testdir}/server.key"
 
 tls=
 verbose=
+html_subdir=
+outdir_intl=
 rerun=
 rerun_args=
 rerun_dead=
@@ -132,6 +134,20 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         rerun_args="${args[$pos]}"
         ;;
+    --html-subdir)
+        # Mirror into "$out/NAME" (path_html) but keep logs/cache in "$out"
+        # (path_log): a non-ASCII NAME exercises the -O path encoding (#621)
+        # without routing the harness-read hts-log.txt through a non-ASCII path.
+        pos=$((pos + 1))
+        html_subdir="${args[$pos]}"
+        ;;
+    --outdir-intl)
+        # Single non-ASCII -O "$out/NAME": path_html AND path_log are NAME, so
+        # the logs (and the harness reads of them) go through the non-ASCII path
+        # (#630). Distinct from --html-subdir, which keeps path_log ASCII.
+        pos=$((pos + 1))
+        outdir_intl="${args[$pos]}"
+        ;;
     --errors | --errors-content | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
@@ -207,12 +223,26 @@ test -n "$ver" || die "could not run httrack"
 
 out="${tmpdir}/crawl"
 mkdir "$out" || die "could not create $out"
+# path_html holds the mirror + index; path_log holds hts-cache/hts-log.txt.
+# Default: both are "$out". --html-subdir moves path_html to "$out/NAME" while
+# path_log (logroot) stays "$out"; --outdir-intl moves both to "$out/NAME".
+mirrorroot="$out"
+logroot="$out"
+odir="$out"
+if test -n "$html_subdir"; then
+    mirrorroot="${out}/${html_subdir}"
+    odir="${mirrorroot},${out}"
+elif test -n "$outdir_intl"; then
+    mirrorroot="${out}/${outdir_intl}"
+    logroot="$mirrorroot"
+    odir="$mirrorroot"
+fi
 # Localhost is fast; disable the rate/bandwidth safety limits but keep a
 # max-time backstop so a hang cannot wedge the suite.
 declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
 log="${tmpdir}/log"
 info "running httrack ${hts[*]}"
-httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
+httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
 crawlpid=$!
 wait "$crawlpid"
 crawlres=$?
@@ -224,12 +254,12 @@ test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || {
     exit 1
 }
 result "OK"
-grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
+grep -iE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt" >&2
 
 # --- optional second pass: re-mirror into the same dir (cache/update path) ----
 if test -n "$rerun"; then
     info "re-running httrack (update pass)"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -243,7 +273,7 @@ if test -n "$rerun"; then
     # The update summary reports "files updated"; a fresh crawl never does. Assert
     # it so a regression that bypasses the cache (re-crawls fresh) can't pass.
     info "checking update used the cache"
-    if grep -aqE "mirror complete in .*files updated" "${out}/hts-log.txt"; then
+    if grep -aqE "mirror complete in .*files updated" "${logroot}/hts-log.txt"; then
         result "OK"
     else
         result "update pass did not report cache activity"
@@ -257,7 +287,7 @@ fi
 if test -n "$rerun_args"; then
     read -ra extra <<<"$rerun_args"
     info "re-running httrack with ${rerun_args}"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -275,11 +305,11 @@ if test -n "$rerun_dead"; then
     zip="${out}/hts-cache/new.zip"
     test -s "$zip" || die "no cache was written by the first pass"
     cp "$zip" "${tmpdir}/cache-before.zip"
-    cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
+    cp "${logroot}/hts-log.txt" "${tmpdir}/log-before.txt"
     stop_server "$serverpid"
     serverpid=
     info "re-running httrack against the stopped server"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
     crawlpid=$!
     wait "$crawlpid" || true
@@ -288,7 +318,7 @@ if test -n "$rerun_dead"; then
     # The dead pass must have gone through the no-data rollback, not bailed out
     # before the mirror loop (which would leave the cache trivially untouched).
     info "checking the dead pass hit the rollback"
-    if grep -aq "No data seems to have been transferred" "${out}/hts-log.txt"; then
+    if grep -aq "No data seems to have been transferred" "${logroot}/hts-log.txt"; then
         result "OK"
     else
         result "rollback notice not found in hts-log.txt"
@@ -303,12 +333,12 @@ if test -n "$rerun_dead"; then
         exit 1
     fi
     # Audits below describe the healthy crawl, not the dead pass.
-    cp "${tmpdir}/log-before.txt" "${out}/hts-log.txt"
+    cp "${tmpdir}/log-before.txt" "${logroot}/hts-log.txt"
 fi
 
 # --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
 hostroot=
-for cand in "${out}/127.0.0.1_${port}" "${out}/127.0.0.1"; do
+for cand in "${mirrorroot}/127.0.0.1_${port}" "${mirrorroot}/127.0.0.1"; do
     if test -d "$cand"; then
         hostroot="$cand"
         break
@@ -333,19 +363,19 @@ while test "$i" -lt "${#audit[@]}"; do
     --errors)
         i=$((i + 1))
         assert_equals "checking errors" "${audit[$i]}" \
-            "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
+            "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")"
         ;;
     --errors-content)
         i=$((i + 1))
-        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
+        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")
         # transient network failures (statuscode -2..-6) flake on busy loopback;
         # the code parens are followed by " at link" or " after N retries at link"
-        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
+        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${logroot}/hts-log.txt" || true)
         assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
         ;;
     --files)
         i=$((i + 1))
-        nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
+        nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${logroot}/hts-log.txt" |
             sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
         assert_equals "checking files" "${audit[$i]}" "$nFiles"
         ;;
@@ -376,7 +406,7 @@ while test "$i" -lt "${#audit[@]}"; do
     --log-found)
         i=$((i + 1))
         info "checking log matches ${audit[$i]}"
-        if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then result "OK"; else
+        if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then result "OK"; else
             result "not in log"
             exit 1
         fi
@@ -384,7 +414,7 @@ while test "$i" -lt "${#audit[@]}"; do
     --log-not-found)
         i=$((i + 1))
         info "checking log lacks ${audit[$i]}"
-        if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then
+        if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then
             result "present in log"
             exit 1
         else result "OK"; fi

tests/proxy-connect-server.py

@@ -0,0 +1,37 @@
+#!/usr/bin/env python3
+"""Local CONNECT-only proxy + plain-HTTP origin for the issue #564 test.
+
+Models a tor HTTPTunnelPort: an HTTP proxy that honours CONNECT and rejects
+every other method with 501 (the classic absolute-URI form a normal HTTP proxy
+accepts). Fronting it is a plain-HTTP origin. The proxy logs each request line
+(and any Proxy-Authorization); the origin logs its request line and headers.
+That lets the test assert a plain-http crawl tunneled through CONNECT, arrived
+origin-form (not "GET http://host/..."), and never leaked proxy credentials.
+
+Proxy modes (argv[2], default "ok"):
+  ok    - honour CONNECT and tunnel to the origin
+  flood - answer 200 then stream headers forever with no blank line, to exercise
+          the client's bound on the proxy response (must not hang the crawl)
+
+Usage: proxy-connect-server.py <logdir> [mode]
+Prints "ORIGIN <port>", "PROXY <port>", then "ready" (one per line) on stdout.
+"""
+import os
+import sys
+
+# python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import proxytestlib  # noqa: E402
+
+ORIGIN_BODY = b"<html><body>ORIGIN-PAGE-564</body></html>"
+
+
+def main():
+    logdir = sys.argv[1]
+    mode = sys.argv[2] if len(sys.argv) > 2 else "ok"
+    proxytestlib.serve(logdir, ORIGIN_BODY, 80, mode)
+
+
+if __name__ == "__main__":
+    main()

tests/proxy-https-server.py

@@ -15,138 +15,21 @@ Proxy modes (argv[3], default "ok"):
 Usage: proxy-https-server.py <cert.pem> <logdir> [mode]
 Prints "ORIGIN <port>", "PROXY <port>", then "ready" (one per line) on stdout.
 """
-import http.server
 import os
-import socket
-import socketserver
-import ssl
 import sys
-import threading
+
+# python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+import proxytestlib  # noqa: E402
 
 ORIGIN_BODY = b"<html><body>ORIGIN-PAGE-85</body></html>"
-PROXY_LOG = "proxy.log"
-ORIGIN_LOG = "origin-headers.log"
-
-
-def make_origin(logdir):
-    class Origin(http.server.BaseHTTPRequestHandler):
-        def do_GET(self):
-            with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle:
-                for key in self.headers.keys():
-                    handle.write(key + "\n")
-            self.send_response(200)
-            self.send_header("Content-Type", "text/html")
-            self.send_header("Content-Length", str(len(ORIGIN_BODY)))
-            self.end_headers()
-            self.wfile.write(ORIGIN_BODY)
-
-        def log_message(self, *args):
-            pass
-
-    return Origin
-
-
-def start_origin(certfile, logdir):
-    httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir))
-    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
-    ctx.load_cert_chain(certfile)
-    httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
-    port = httpd.socket.getsockname()[1]
-    threading.Thread(target=httpd.serve_forever, daemon=True).start()
-    return port
-
-
-def pipe(src, dst):
-    try:
-        while True:
-            data = src.recv(65536)
-            if not data:
-                break
-            dst.sendall(data)
-    except OSError:
-        pass
-    finally:
-        for sock in (src, dst):
-            try:
-                sock.shutdown(socket.SHUT_RDWR)
-            except OSError:
-                pass
-
-
-def handle_client(conn, logdir, mode):
-    rfile = conn.makefile("rb")
-    request_line = rfile.readline().decode("latin-1").strip()
-    auth = None
-    while True:
-        line = rfile.readline().decode("latin-1")
-        if line in ("\r\n", "\n", ""):
-            break
-        key, _, value = line.partition(":")
-        if key.strip().lower() == "proxy-authorization":
-            auth = value.strip()
-    with open(os.path.join(logdir, PROXY_LOG), "a") as handle:
-        handle.write(request_line + "\n")
-        if auth is not None:
-            handle.write("AUTH " + auth + "\n")
-    parts = request_line.split()
-    if not (len(parts) >= 2 and parts[0] == "CONNECT"):
-        conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n")
-        conn.close()
-        return
-    if mode == "flood":
-        # 200, then an endless header stream with no terminating blank line: the
-        # client must bound this and give up, not hang.
-        try:
-            conn.sendall(b"HTTP/1.0 200 Connection established\r\n")
-            while True:
-                conn.sendall(b"X-Pad: 0123456789\r\n")
-        except OSError:
-            pass
-        conn.close()
-        return
-    host, _, port = parts[1].partition(":")
-    try:
-        upstream = socket.create_connection((host, int(port or 443)))
-    except OSError:
-        conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n")
-        conn.close()
-        return
-    conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
-    threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start()
-    pipe(upstream, conn)
-
-
-def start_proxy(logdir, mode):
-    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    srv.bind(("127.0.0.1", 0))
-    srv.listen(16)
-    port = srv.getsockname()[1]
-
-    def serve():
-        while True:
-            conn, _ = srv.accept()
-            threading.Thread(
-                target=handle_client, args=(conn, logdir, mode), daemon=True
-            ).start()
-
-    threading.Thread(target=serve, daemon=True).start()
-    return port
 
 
 def main():
     certfile, logdir = sys.argv[1], sys.argv[2]
     mode = sys.argv[3] if len(sys.argv) > 3 else "ok"
-    for name in (PROXY_LOG, ORIGIN_LOG):
-        open(os.path.join(logdir, name), "w").close()
-    origin_port = start_origin(certfile, logdir)
-    proxy_port = start_proxy(logdir, mode)
-    # Keep the port lines the caller parses LF: Windows would emit \r\n.
-    sys.stdout.reconfigure(newline="\n")
-    print("ORIGIN %d" % origin_port, flush=True)
-    print("PROXY %d" % proxy_port, flush=True)
-    print("ready", flush=True)
-    threading.Event().wait()
+    proxytestlib.serve(logdir, ORIGIN_BODY, 443, mode, certfile=certfile)
 
 
 if __name__ == "__main__":

tests/proxytestlib.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python3
+"""Shared helpers for the local proxy test servers.
+
+A CONNECT proxy in front of an origin, as used by proxy-https-server.py (TLS
+origin, #85) and proxy-connect-server.py (plain origin, #564). socks5-server.py
+reuses the relay only; its origin is specialised for keep-alive reuse.
+Importable because Python puts the running script's directory on sys.path.
+"""
+import http.server
+import os
+import socket
+import socketserver
+import ssl
+import sys
+import threading
+
+PROXY_LOG = "proxy.log"
+ORIGIN_LOG = "origin-headers.log"
+
+
+def bind_ephemeral():
+    """Listening socket on a free loopback port, and that port."""
+    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    srv.bind(("127.0.0.1", 0))
+    srv.listen(16)
+    return srv, srv.getsockname()[1]
+
+
+def pipe(src, dst):
+    """Relay bytes one way until EOF, then tear both ends down."""
+    try:
+        while True:
+            data = src.recv(65536)
+            if not data:
+                break
+            dst.sendall(data)
+    except OSError:
+        pass
+    finally:
+        for sock in (src, dst):
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+
+
+def make_origin(logdir, body):
+    class Origin(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):
+            # the request line proves origin-form vs absolute-URI (#564)
+            with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle:
+                handle.write(self.requestline + "\n")
+                for key in self.headers.keys():
+                    handle.write(key + "\n")
+            self.send_response(200)
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+
+        def log_message(self, *args):
+            pass
+
+    return Origin
+
+
+def start_origin(logdir, body, certfile=None):
+    """Serve body on an ephemeral port, over TLS when certfile is given."""
+    httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir, body))
+    if certfile is not None:
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        ctx.load_cert_chain(certfile)
+        httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
+    port = httpd.socket.getsockname()[1]
+    threading.Thread(target=httpd.serve_forever, daemon=True).start()
+    return port
+
+
+def handle_client(conn, logdir, mode, default_port):
+    rfile = conn.makefile("rb")
+    request_line = rfile.readline().decode("latin-1").strip()
+    auth = None
+    while True:
+        line = rfile.readline().decode("latin-1")
+        if line in ("\r\n", "\n", ""):
+            break
+        key, _, value = line.partition(":")
+        if key.strip().lower() == "proxy-authorization":
+            auth = value.strip()
+    with open(os.path.join(logdir, PROXY_LOG), "a") as handle:
+        handle.write(request_line + "\n")
+        if auth is not None:
+            handle.write("AUTH " + auth + "\n")
+    parts = request_line.split()
+    # CONNECT-only: reject the classic absolute-URI form a normal proxy accepts
+    if not (len(parts) >= 2 and parts[0] == "CONNECT"):
+        conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n")
+        conn.close()
+        return
+    if mode == "flood":
+        # 200, then endless headers with no blank line: the client must not hang
+        try:
+            conn.sendall(b"HTTP/1.0 200 Connection established\r\n")
+            while True:
+                conn.sendall(b"X-Pad: 0123456789\r\n")
+        except OSError:
+            pass
+        conn.close()
+        return
+    host, _, port = parts[1].partition(":")
+    try:
+        # default_port only backstops a portless CONNECT; httrack sends host:port
+        upstream = socket.create_connection((host, int(port or default_port)))
+    except OSError:
+        conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n")
+        conn.close()
+        return
+    conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
+    threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start()
+    pipe(upstream, conn)
+
+
+def start_proxy(logdir, mode, default_port):
+    srv, port = bind_ephemeral()
+
+    def accept_loop():
+        while True:
+            conn, _ = srv.accept()
+            threading.Thread(
+                target=handle_client,
+                args=(conn, logdir, mode, default_port),
+                daemon=True,
+            ).start()
+
+    threading.Thread(target=accept_loop, daemon=True).start()
+    return port
+
+
+def serve(logdir, origin_body, default_port, mode="ok", certfile=None):
+    """Start the origin+proxy pair, announce both ports, then block forever."""
+    for name in (PROXY_LOG, ORIGIN_LOG):
+        open(os.path.join(logdir, name), "w").close()
+    origin_port = start_origin(logdir, origin_body, certfile)
+    proxy_port = start_proxy(logdir, mode, default_port)
+    # Keep the port lines the caller parses LF: Windows would emit \r\n.
+    sys.stdout.reconfigure(newline="\n")
+    print("ORIGIN %d" % origin_port, flush=True)
+    print("PROXY %d" % proxy_port, flush=True)
+    print("ready", flush=True)
+    threading.Event().wait()

tests/socks5-server.py

@@ -16,9 +16,15 @@ import struct
 import sys
 import threading
 
+# python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from proxytestlib import bind_ephemeral, pipe  # noqa: E402
+
 # The one name the proxy answers for; a .invalid TLD never resolves (RFC 6761),
 # so a locally-resolving client could not reach us -- success proves remote DNS.
 REMOTE_HOST = b"socks-origin.invalid"
+AUTH_VERSION = 0x01  # RFC 1929 sub-negotiation version
 # index links the subpages so an -r3 crawl reuses one keep-alive socket
 LINKS = "".join('<a href="/p%d.html">%d</a>' % (i, i) for i in range(1, 6))
 ORIGIN_BODY = ("<html><body>ORIGIN-PAGE-563 " + LINKS + "</body></html>").encode()
@@ -85,23 +91,6 @@ def recvn(conn, n):
     return buf
 
 
-def pipe(src, dst):
-    try:
-        while True:
-            data = src.recv(65536)
-            if not data:
-                break
-            dst.sendall(data)
-    except OSError:
-        pass
-    finally:
-        for sock in (src, dst):
-            try:
-                sock.shutdown(socket.SHUT_RDWR)
-            except OSError:
-                pass
-
-
 def log(logdir, line):
     with open(os.path.join(logdir, SOCKS_LOG), "a") as handle:
         handle.write(line + "\n")
@@ -116,6 +105,12 @@ def negotiate_auth(conn, logdir):
     if 0x02 in methods:  # prefer user/pass so the auth test exercises RFC 1929
         conn.sendall(b"\x05\x02")
         (subver,) = recvn(conn, 1)
+        # RFC 1929's version byte is 0x01, not SOCKS5's 0x05: a real proxy
+        # rejects a mismatch instead of tunnelling anyway
+        if subver != AUTH_VERSION:
+            log(logdir, "AUTHVER-BAD %d" % subver)
+            conn.sendall(bytes([AUTH_VERSION, 0x01]))  # sub-negotiation failure
+            return False
         (ulen,) = recvn(conn, 1)
         uname = recvn(conn, ulen)
         (plen,) = recvn(conn, 1)
@@ -192,11 +187,7 @@ def handle_socks(conn, logdir, mode):
 
 
 def start_socks(logdir, mode):
-    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    srv.bind(("127.0.0.1", 0))
-    srv.listen(16)
-    port = srv.getsockname()[1]
+    srv, port = bind_ephemeral()
 
     def serve():
         while True:

tests/testlib.sh

@@ -43,3 +43,69 @@ stop_server() {
     wait "$1" 2>/dev/null || true
     return 0
 }
+
+# Dump and clear the crawl logs a hard-killed test leaves in TMPDIR (its cleanup
+# trap never ran): hts-log.txt alone records "More than N seconds passed.. giving
+# up", so a wedge past --max-time is undiagnosable without it (#605).
+dump_crawl_logs() {
+    local d f
+    for d in "${TMPDIR:-/tmp}"/httrack_local.*; do
+        test -d "$d" || continue
+        for f in "$d/crawl/hts-log.txt" "$d/log" "$d/log.2"; do
+            test -f "$f" || continue
+            # Leading newline: the killed test's last line has no terminator.
+            printf '\n--- %s (last 200 lines)\n' "$f"
+            tail -n 200 "$f"
+        done
+        # so a later test's dump cannot re-report this one; never fatal, the
+        # caller is already handling a failure and Windows may still hold a file
+        rm -rf "$d" || true
+    done
+}
+
+# Kill a backgrounded job and its whole descendant tree. POSIX: the caller must
+# have put the job in its own process group (run_with_timeout does) so we signal
+# the group; a bare kill would orphan the grandchildren. Windows: the tree is
+# native processes MSYS can't signal, so taskkill /T ends it by Windows PID.
+# Single-slash switches: the workflow sets MSYS_NO_PATHCONV/MSYS2_ARG_CONV_EXCL,
+# so args pass verbatim and a //T would reach taskkill unfolded and be rejected.
+kill_tree() {
+    local pid=$1
+    if is_windows; then
+        local winpid=
+        test -r "/proc/$pid/winpid" && winpid=$(cat "/proc/$pid/winpid" 2>/dev/null)
+        if test -n "$winpid"; then
+            taskkill /F /T /PID "$winpid" >/dev/null 2>&1 || true
+        else
+            # The offline suite runs serially, so no wanted process races this.
+            taskkill /F /IM httrack.exe >/dev/null 2>&1 || true
+            taskkill /F /IM python.exe >/dev/null 2>&1 || true
+        fi
+    fi
+    kill -9 -"$pid" 2>/dev/null || kill -9 "$pid" 2>/dev/null || true
+}
+
+# Run "$@" under a wall-clock deadline of $1 seconds; return its exit status, or
+# 124 if it overran and was killed. timeout(1) is unusable here: it's absent on
+# macOS and its signals can't reap httrack.exe on Windows. We poll and kill_tree.
+run_with_timeout() {
+    local secs=$1
+    shift
+    local had_m=
+    case "$-" in *m*) had_m=1 ;; esac
+    is_windows || set -m # own process group, so kill_tree can signal the group
+    "$@" &
+    local pid=$!
+    test -n "$had_m" || is_windows || set +m
+    local waited=0
+    while kill -0 "$pid" 2>/dev/null; do
+        if test "$waited" -ge "$secs"; then
+            kill_tree "$pid"
+            wait "$pid" 2>/dev/null || true
+            return 124
+        fi
+        sleep 1
+        waited=$((waited + 1))
+    done
+    wait "$pid"
+}

tests/tls-stall-server.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""Peers that accept a connection and never speak TLS (#607).
+
+Modes: "direct" stalls the handshake straight away; "proxy <secs>" answers a
+CONNECT after <secs> before stalling, so the handshake starts on a clock the
+connect has already eaten into. Prints "PORT <n>" once listening.
+Usage: tls-stall-server.py [direct | proxy <secs>]
+"""
+import os
+import sys
+import threading
+import time
+
+# python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from proxytestlib import bind_ephemeral  # noqa: E402
+
+held = []  # keep every socket open: a close would fail the handshake outright
+
+
+def stall(conn, delay):
+    if delay is not None:
+        rfile = conn.makefile("rb")
+        while rfile.readline() not in (b"\r\n", b"\n", b""):
+            pass
+        time.sleep(delay)
+        conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
+    held.append(conn)
+
+
+def main():
+    mode = sys.argv[1] if len(sys.argv) > 1 else "direct"
+    delay = float(sys.argv[2]) if mode == "proxy" else None
+    srv, port = bind_ephemeral()
+    sys.stdout.reconfigure(newline="\n")  # Windows would emit \r\n
+    print("PORT %d" % port, flush=True)
+    while True:
+        conn, _ = srv.accept()
+        threading.Thread(target=stall, args=(conn, delay), daemon=True).start()
+
+
+main()