Cover the handshake timeout window, and share the ephemeral-bind helper

Test 59 only bounded the crawl from above, so an engine that reaped every
handshake instantly passed it, and nothing exercised the timeout_refresh at the
handshake entry: on loopback the connect is instant, so the handshake's own
window is indistinguishable from the connect's.

Add a floor to the first case, and a second one behind a proxy that takes 4s to
answer CONNECT. The handshake must still get its full --timeout=5 from there
(~9s); sharing the connect's clock reaps at ~5s. Dropping the refresh now fails
the test, as does collapsing the window to zero.

The stall server grows a proxy mode for that, and takes its listening socket
from a new proxytestlib bind_ephemeral(), which replaces the same boilerplate
in the socks5 and proxy servers.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsback.c

@@ -2816,6 +2816,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 if (SSL_set_fd(back[i].r.ssl_con, (int) back[i].r.soc) == 1) {
                   SSL_set_connect_state(back[i].r.ssl_con);
                   back[i].status = STATUS_SSL_WAIT_HANDSHAKE;   /* handshake wait */
+                  // the handshake gets its own timeout window, as connect does
+                  if (back[i].timeout > 0)
+                    back[i].timeout_refresh = time_local();
                 } else
                   back[i].r.statuscode = STATUSCODE_SSL_HANDSHAKE;
               } else
@@ -2875,6 +2878,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       }
 #if HTS_USEOPENSSL
       else if (back[i].status == STATUS_SSL_WAIT_HANDSHAKE) {       // wait for SSL handshake
+        // a peer that never speaks TLS must be reaped by --timeout too (#607)
+        if (!gestion_timeout)
+          if (back[i].timeout > 0)
+            gestion_timeout = 1;
+
         /* SSL mode */
         if (back[i].r.ssl) {
           int conn_code;
@@ -4265,6 +4273,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 strcpybuff(back[i].r.msg, "Connect Time Out");
               else if (back[i].status == STATUS_WAIT_DNS)
                 strcpybuff(back[i].r.msg, "DNS Time Out");
+              else if (back[i].status == STATUS_SSL_WAIT_HANDSHAKE)
+                strcpybuff(back[i].r.msg, "SSL/TLS Handshake Time Out");
               else
                 strcpybuff(back[i].r.msg, "Receive Time Out");
               back[i].status = STATUS_READY;    // terminé

tests/59_local-tls-stall.test

@@ -0,0 +1,95 @@
+#!/bin/bash
+#
+# Issue #607: a peer that accepts the TCP connect but never speaks TLS leaves the
+# slot stuck in the handshake. The per-slot --timeout must reap it; before the
+# fix only --max-time did, so these crawls ran until the kill guard.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
+if test "${HTTPS_SUPPORT:-}" == "no"; then
+    echo "no https support compiled, skipping"
+    exit 77
+fi
+python=$(find_python) || {
+    echo "python3 missing, skipping"
+    exit 77
+}
+
+server=$(nativepath "$top_srcdir/tests/tls-stall-server.py")
+tmpdir=$(mktemp -d)
+serverpid=
+
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+    return 0
+}
+trap cleanup EXIT
+
+# start_stall_server <tag> <mode-args...>: sets $port from the announced one.
+start_stall_server() {
+    local tag="$1"
+    shift
+    "$python" "$server" "$@" >"$tmpdir/$tag.out" 2>"$tmpdir/$tag.err" &
+    serverpid=$!
+    port=
+    for _ in $(seq 1 50); do
+        line=$(head -n1 "$tmpdir/$tag.out" 2>/dev/null || true)
+        if test "${line%% *}" == "PORT"; then
+            port="${line#PORT }"
+            return 0
+        fi
+        kill -0 "$serverpid" 2>/dev/null || {
+            echo "server exited early: $(cat "$tmpdir/$tag.err")"
+            exit 1
+        }
+        sleep 0.1
+    done
+    echo "could not discover server port"
+    exit 1
+}
+
+# no -E/--max-time on purpose: --timeout is the only thing that can end these.
+# The kill guard stands in for the hang, so a wall time near it means no reap.
+crawl_wall() {
+    local out="$tmpdir/$1"
+    shift
+    local start
+    start=$(date +%s)
+    run_with_timeout 60 httrack -O "$out" -c1 --robots=0 --retries=0 --quiet -Z \
+        "$@" >>"$tmpdir/log" 2>&1 || true
+    echo $(($(date +%s) - start))
+}
+
+# 1. handshake stalled from the first byte: reaped at --timeout, not before.
+start_stall_server direct direct
+wall=$(crawl_wall crawl1 "https://127.0.0.1:$port/" --timeout=5)
+if test "$wall" -ge 30 || test "$wall" -lt 3; then
+    echo "FAIL: stalled handshake reaped after ${wall}s, expected about 5s" >&2
+    cat "$tmpdir/log" >&2
+    exit 1
+fi
+grep -q 'Handshake Time Out' "$tmpdir/crawl1/hts-log.txt" || {
+    echo "FAIL: crawl ended in ${wall}s but not on a handshake timeout" >&2
+    cat "$tmpdir/crawl1/hts-log.txt" >&2
+    exit 1
+}
+echo "OK: stalled TLS handshake reaped by --timeout after ${wall}s"
+
+# 2. the handshake window is its own, not what the connect left over: a proxy
+# that takes 4s to answer CONNECT must still leave the full --timeout=5 for the
+# handshake (~9s total). Sharing the connect's clock would reap at ~5s.
+stop_server "$serverpid"
+start_stall_server proxy proxy 4
+wall=$(crawl_wall crawl2 "https://127.0.0.1:443/" -P "127.0.0.1:$port" --timeout=5)
+if test "$wall" -ge 20 || test "$wall" -lt 7; then
+    echo "FAIL: handshake after a slow connect reaped at ${wall}s, expected about 9s" >&2
+    cat "$tmpdir/log" >&2
+    exit 1
+fi
+echo "OK: handshake keeps its own timeout window after a slow connect (${wall}s)"

tests/Makefile.am

@@ -3,7 +3,7 @@
 # silently drop it from the dist tarball and break "make distcheck".
 EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
 	proxy-https-server.py socks5-server.py proxy-connect-server.py \
-	proxytestlib.py \
+	proxytestlib.py tls-stall-server.py \
 	local-crawl.sh local-server.py testlib.sh server.crt server.key \
 	server-root/simple/basic.html server-root/simple/link.html \
 	server-root/stripquery/index.html server-root/stripquery/a.html \
@@ -136,6 +136,7 @@ TESTS = \
 	55_local-chunked.test \
 	56_local-proxy-noleak.test \
 	57_local-proxy-connect.test \
-	58_watchdog.test
+	58_watchdog.test \
+	59_local-tls-stall.test
 
 CLEANFILES = check-network_sh.cache

tests/proxytestlib.py

@@ -18,6 +18,15 @@ PROXY_LOG = "proxy.log"
 ORIGIN_LOG = "origin-headers.log"
 
 
+def bind_ephemeral():
+    """Listening socket on a free loopback port, and that port."""
+    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    srv.bind(("127.0.0.1", 0))
+    srv.listen(16)
+    return srv, srv.getsockname()[1]
+
+
 def pipe(src, dst):
     """Relay bytes one way until EOF, then tear both ends down."""
     try:
@@ -113,11 +122,7 @@ def handle_client(conn, logdir, mode, default_port):
 
 
 def start_proxy(logdir, mode, default_port):
-    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    srv.bind(("127.0.0.1", 0))
-    srv.listen(16)
-    port = srv.getsockname()[1]
+    srv, port = bind_ephemeral()
 
     def accept_loop():
         while True:

tests/socks5-server.py

@@ -19,7 +19,7 @@ import threading
 # python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-from proxytestlib import pipe  # noqa: E402
+from proxytestlib import bind_ephemeral, pipe  # noqa: E402
 
 # The one name the proxy answers for; a .invalid TLD never resolves (RFC 6761),
 # so a locally-resolving client could not reach us -- success proves remote DNS.
@@ -180,11 +180,7 @@ def handle_socks(conn, logdir, mode):
 
 
 def start_socks(logdir, mode):
-    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    srv.bind(("127.0.0.1", 0))
-    srv.listen(16)
-    port = srv.getsockname()[1]
+    srv, port = bind_ephemeral()
 
     def serve():
         while True:

tests/tls-stall-server.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+"""Peers that accept a connection and never speak TLS (#607).
+
+Modes: "direct" stalls the handshake straight away; "proxy <secs>" answers a
+CONNECT after <secs> before stalling, so the handshake starts on a clock the
+connect has already eaten into. Prints "PORT <n>" once listening.
+Usage: tls-stall-server.py [direct | proxy <secs>]
+"""
+import os
+import sys
+import threading
+import time
+
+# python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+from proxytestlib import bind_ephemeral  # noqa: E402
+
+held = []  # keep every socket open: a close would fail the handshake outright
+
+
+def stall(conn, delay):
+    if delay is not None:
+        rfile = conn.makefile("rb")
+        while rfile.readline() not in (b"\r\n", b"\n", b""):
+            pass
+        time.sleep(delay)
+        conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
+    held.append(conn)
+
+
+def main():
+    mode = sys.argv[1] if len(sys.argv) > 1 else "direct"
+    delay = float(sys.argv[2]) if mode == "proxy" else None
+    srv, port = bind_ephemeral()
+    sys.stdout.reconfigure(newline="\n")  # Windows would emit \r\n
+    print("PORT %d" % port, flush=True)
+    while True:
+        conn, _ = srv.accept()
+        threading.Thread(target=stall, args=(conn, delay), daemon=True).start()
+
+
+main()