Share the proxy test servers' duplicated code in proxytestlib.py

proxy-https-server.py (#85) and proxy-connect-server.py (#564) were ~90% identical: the CONNECT proxy, the relay and the origin were copies, with only the origin body, the TLS wrap and the argv shape differing. Move the common half into tests/proxytestlib.py; each server is now a docstring plus a serve() call. socks5-server.py reuses the relay too, but keeps its own origin, which is specialised for keep-alive reuse and subpages. The shared origin always logs the request line (it was proxy-connect only); harmless for #85, which only asserts no Proxy-Authorization reaches it. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-16 13:51:46 +03:00 · 2026-07-16 09:41:28 +02:00
5 changed files with 155 additions and 263 deletions
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -3,6 +3,7 @@
 # silently drop it from the dist tarball and break "make distcheck".
 EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
 	proxy-https-server.py socks5-server.py proxy-connect-server.py \
+	proxytestlib.py \
 	local-crawl.sh local-server.py testlib.sh server.crt server.key \
 	server-root/simple/basic.html server-root/simple/link.html \
 	server-root/stripquery/index.html server-root/stripquery/a.html \
--- a/tests/proxy-connect-server.py
+++ b/tests/proxy-connect-server.py
@@ -16,136 +16,17 @@ Proxy modes (argv[2], default "ok"):
 Usage: proxy-connect-server.py <logdir> [mode]
 Prints "ORIGIN <port>", "PROXY <port>", then "ready" (one per line) on stdout.
 """
-import http.server
-import os
-import socket
-import socketserver
 import sys
-import threading
+
+import proxytestlib

 ORIGIN_BODY = b"<html><body>ORIGIN-PAGE-564</body></html>"
-PROXY_LOG = "proxy.log"
-ORIGIN_LOG = "origin-headers.log"
-
-
-def make_origin(logdir):
-    class Origin(http.server.BaseHTTPRequestHandler):
-        def do_GET(self):
-            with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle:
-                handle.write(self.requestline + "\n")
-                for key in self.headers.keys():
-                    handle.write(key + "\n")
-            self.send_response(200)
-            self.send_header("Content-Type", "text/html")
-            self.send_header("Content-Length", str(len(ORIGIN_BODY)))
-            self.end_headers()
-            self.wfile.write(ORIGIN_BODY)
-
-        def log_message(self, *args):
-            pass
-
-    return Origin
-
-
-def start_origin(logdir):
-    httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir))
-    port = httpd.socket.getsockname()[1]
-    threading.Thread(target=httpd.serve_forever, daemon=True).start()
-    return port
-
-
-def pipe(src, dst):
-    try:
-        while True:
-            data = src.recv(65536)
-            if not data:
-                break
-            dst.sendall(data)
-    except OSError:
-        pass
-    finally:
-        for sock in (src, dst):
-            try:
-                sock.shutdown(socket.SHUT_RDWR)
-            except OSError:
-                pass
-
-
-def handle_client(conn, logdir, mode):
-    rfile = conn.makefile("rb")
-    request_line = rfile.readline().decode("latin-1").strip()
-    auth = None
-    while True:
-        line = rfile.readline().decode("latin-1")
-        if line in ("\r\n", "\n", ""):
-            break
-        key, _, value = line.partition(":")
-        if key.strip().lower() == "proxy-authorization":
-            auth = value.strip()
-    with open(os.path.join(logdir, PROXY_LOG), "a") as handle:
-        handle.write(request_line + "\n")
-        if auth is not None:
-            handle.write("AUTH " + auth + "\n")
-    parts = request_line.split()
-    # CONNECT-only: reject the classic absolute-URI form a normal proxy accepts
-    if not (len(parts) >= 2 and parts[0] == "CONNECT"):
-        conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n")
-        conn.close()
-        return
-    if mode == "flood":
-        # 200, then an endless header stream with no terminating blank line: the
-        # client must bound this and give up, not hang.
-        try:
-            conn.sendall(b"HTTP/1.0 200 Connection established\r\n")
-            while True:
-                conn.sendall(b"X-Pad: 0123456789\r\n")
-        except OSError:
-            pass
-        conn.close()
-        return
-    host, _, port = parts[1].partition(":")
-    try:
-        upstream = socket.create_connection((host, int(port or 80)))
-    except OSError:
-        conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n")
-        conn.close()
-        return
-    conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
-    threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start()
-    pipe(upstream, conn)
-
-
-def start_proxy(logdir, mode):
-    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    srv.bind(("127.0.0.1", 0))
-    srv.listen(16)
-    port = srv.getsockname()[1]
-
-    def serve():
-        while True:
-            conn, _ = srv.accept()
-            threading.Thread(
-                target=handle_client, args=(conn, logdir, mode), daemon=True
-            ).start()
-
-    threading.Thread(target=serve, daemon=True).start()
-    return port


 def main():
    logdir = sys.argv[1]
    mode = sys.argv[2] if len(sys.argv) > 2 else "ok"
-    for name in (PROXY_LOG, ORIGIN_LOG):
-        open(os.path.join(logdir, name), "w").close()
-    origin_port = start_origin(logdir)
-    proxy_port = start_proxy(logdir, mode)
-    # Keep the port lines the caller parses LF: Windows would emit \r\n.
-    sys.stdout.reconfigure(newline="\n")
-    print("ORIGIN %d" % origin_port, flush=True)
-    print("PROXY %d" % proxy_port, flush=True)
-    print("ready", flush=True)
-    threading.Event().wait()
+    proxytestlib.serve(logdir, ORIGIN_BODY, 80, mode)


 if __name__ == "__main__":
--- a/tests/proxy-https-server.py
+++ b/tests/proxy-https-server.py
@@ -15,138 +15,17 @@ Proxy modes (argv[3], default "ok"):
 Usage: proxy-https-server.py <cert.pem> <logdir> [mode]
 Prints "ORIGIN <port>", "PROXY <port>", then "ready" (one per line) on stdout.
 """
-import http.server
-import os
-import socket
-import socketserver
-import ssl
 import sys
-import threading
+
+import proxytestlib

 ORIGIN_BODY = b"<html><body>ORIGIN-PAGE-85</body></html>"
-PROXY_LOG = "proxy.log"
-ORIGIN_LOG = "origin-headers.log"
-
-
-def make_origin(logdir):
-    class Origin(http.server.BaseHTTPRequestHandler):
-        def do_GET(self):
-            with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle:
-                for key in self.headers.keys():
-                    handle.write(key + "\n")
-            self.send_response(200)
-            self.send_header("Content-Type", "text/html")
-            self.send_header("Content-Length", str(len(ORIGIN_BODY)))
-            self.end_headers()
-            self.wfile.write(ORIGIN_BODY)
-
-        def log_message(self, *args):
-            pass
-
-    return Origin
-
-
-def start_origin(certfile, logdir):
-    httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir))
-    ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
-    ctx.load_cert_chain(certfile)
-    httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
-    port = httpd.socket.getsockname()[1]
-    threading.Thread(target=httpd.serve_forever, daemon=True).start()
-    return port
-
-
-def pipe(src, dst):
-    try:
-        while True:
-            data = src.recv(65536)
-            if not data:
-                break
-            dst.sendall(data)
-    except OSError:
-        pass
-    finally:
-        for sock in (src, dst):
-            try:
-                sock.shutdown(socket.SHUT_RDWR)
-            except OSError:
-                pass
-
-
-def handle_client(conn, logdir, mode):
-    rfile = conn.makefile("rb")
-    request_line = rfile.readline().decode("latin-1").strip()
-    auth = None
-    while True:
-        line = rfile.readline().decode("latin-1")
-        if line in ("\r\n", "\n", ""):
-            break
-        key, _, value = line.partition(":")
-        if key.strip().lower() == "proxy-authorization":
-            auth = value.strip()
-    with open(os.path.join(logdir, PROXY_LOG), "a") as handle:
-        handle.write(request_line + "\n")
-        if auth is not None:
-            handle.write("AUTH " + auth + "\n")
-    parts = request_line.split()
-    if not (len(parts) >= 2 and parts[0] == "CONNECT"):
-        conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n")
-        conn.close()
-        return
-    if mode == "flood":
-        # 200, then an endless header stream with no terminating blank line: the
-        # client must bound this and give up, not hang.
-        try:
-            conn.sendall(b"HTTP/1.0 200 Connection established\r\n")
-            while True:
-                conn.sendall(b"X-Pad: 0123456789\r\n")
-        except OSError:
-            pass
-        conn.close()
-        return
-    host, _, port = parts[1].partition(":")
-    try:
-        upstream = socket.create_connection((host, int(port or 443)))
-    except OSError:
-        conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n")
-        conn.close()
-        return
-    conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
-    threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start()
-    pipe(upstream, conn)
-
-
-def start_proxy(logdir, mode):
-    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    srv.bind(("127.0.0.1", 0))
-    srv.listen(16)
-    port = srv.getsockname()[1]
-
-    def serve():
-        while True:
-            conn, _ = srv.accept()
-            threading.Thread(
-                target=handle_client, args=(conn, logdir, mode), daemon=True
-            ).start()
-
-    threading.Thread(target=serve, daemon=True).start()
-    return port


 def main():
    certfile, logdir = sys.argv[1], sys.argv[2]
    mode = sys.argv[3] if len(sys.argv) > 3 else "ok"
-    for name in (PROXY_LOG, ORIGIN_LOG):
-        open(os.path.join(logdir, name), "w").close()
-    origin_port = start_origin(certfile, logdir)
-    proxy_port = start_proxy(logdir, mode)
-    # Keep the port lines the caller parses LF: Windows would emit \r\n.
-    sys.stdout.reconfigure(newline="\n")
-    print("ORIGIN %d" % origin_port, flush=True)
-    print("PROXY %d" % proxy_port, flush=True)
-    print("ready", flush=True)
-    threading.Event().wait()
+    proxytestlib.serve(logdir, ORIGIN_BODY, 443, mode, certfile=certfile)


 if __name__ == "__main__":
--- a/tests/proxytestlib.py
+++ b/tests/proxytestlib.py
@@ -0,0 +1,146 @@
+#!/usr/bin/env python3
+"""Shared helpers for the local proxy test servers.
+
+A CONNECT proxy in front of an origin, as used by proxy-https-server.py (TLS
+origin, #85) and proxy-connect-server.py (plain origin, #564). socks5-server.py
+reuses the relay only; its origin is specialised for keep-alive reuse.
+Importable because Python puts the running script's directory on sys.path.
+"""
+import http.server
+import os
+import socket
+import socketserver
+import ssl
+import sys
+import threading
+
+PROXY_LOG = "proxy.log"
+ORIGIN_LOG = "origin-headers.log"
+
+
+def pipe(src, dst):
+    """Relay bytes one way until EOF, then tear both ends down."""
+    try:
+        while True:
+            data = src.recv(65536)
+            if not data:
+                break
+            dst.sendall(data)
+    except OSError:
+        pass
+    finally:
+        for sock in (src, dst):
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+
+
+def make_origin(logdir, body):
+    class Origin(http.server.BaseHTTPRequestHandler):
+        def do_GET(self):
+            # the request line proves origin-form vs absolute-URI (#564)
+            with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle:
+                handle.write(self.requestline + "\n")
+                for key in self.headers.keys():
+                    handle.write(key + "\n")
+            self.send_response(200)
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", str(len(body)))
+            self.end_headers()
+            self.wfile.write(body)
+
+        def log_message(self, *args):
+            pass
+
+    return Origin
+
+
+def start_origin(logdir, body, certfile=None):
+    """Serve body on an ephemeral port, over TLS when certfile is given."""
+    httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir, body))
+    if certfile is not None:
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        ctx.load_cert_chain(certfile)
+        httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
+    port = httpd.socket.getsockname()[1]
+    threading.Thread(target=httpd.serve_forever, daemon=True).start()
+    return port
+
+
+def handle_client(conn, logdir, mode, default_port):
+    rfile = conn.makefile("rb")
+    request_line = rfile.readline().decode("latin-1").strip()
+    auth = None
+    while True:
+        line = rfile.readline().decode("latin-1")
+        if line in ("\r\n", "\n", ""):
+            break
+        key, _, value = line.partition(":")
+        if key.strip().lower() == "proxy-authorization":
+            auth = value.strip()
+    with open(os.path.join(logdir, PROXY_LOG), "a") as handle:
+        handle.write(request_line + "\n")
+        if auth is not None:
+            handle.write("AUTH " + auth + "\n")
+    parts = request_line.split()
+    # CONNECT-only: reject the classic absolute-URI form a normal proxy accepts
+    if not (len(parts) >= 2 and parts[0] == "CONNECT"):
+        conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n")
+        conn.close()
+        return
+    if mode == "flood":
+        # 200, then endless headers with no blank line: the client must not hang
+        try:
+            conn.sendall(b"HTTP/1.0 200 Connection established\r\n")
+            while True:
+                conn.sendall(b"X-Pad: 0123456789\r\n")
+        except OSError:
+            pass
+        conn.close()
+        return
+    host, _, port = parts[1].partition(":")
+    try:
+        # default_port only backstops a portless CONNECT; httrack sends host:port
+        upstream = socket.create_connection((host, int(port or default_port)))
+    except OSError:
+        conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n")
+        conn.close()
+        return
+    conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
+    threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start()
+    pipe(upstream, conn)
+
+
+def start_proxy(logdir, mode, default_port):
+    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    srv.bind(("127.0.0.1", 0))
+    srv.listen(16)
+    port = srv.getsockname()[1]
+
+    def accept_loop():
+        while True:
+            conn, _ = srv.accept()
+            threading.Thread(
+                target=handle_client,
+                args=(conn, logdir, mode, default_port),
+                daemon=True,
+            ).start()
+
+    threading.Thread(target=accept_loop, daemon=True).start()
+    return port
+
+
+def serve(logdir, origin_body, default_port, mode="ok", certfile=None):
+    """Start the origin+proxy pair, announce both ports, then block forever."""
+    for name in (PROXY_LOG, ORIGIN_LOG):
+        open(os.path.join(logdir, name), "w").close()
+    origin_port = start_origin(logdir, origin_body, certfile)
+    proxy_port = start_proxy(logdir, mode, default_port)
+    # Keep the port lines the caller parses LF: Windows would emit \r\n.
+    sys.stdout.reconfigure(newline="\n")
+    print("ORIGIN %d" % origin_port, flush=True)
+    print("PROXY %d" % proxy_port, flush=True)
+    print("ready", flush=True)
+    threading.Event().wait()
--- a/tests/socks5-server.py
+++ b/tests/socks5-server.py
@@ -16,6 +16,8 @@ import struct
 import sys
 import threading

+from proxytestlib import pipe
+
 # The one name the proxy answers for; a .invalid TLD never resolves (RFC 6761),
 # so a locally-resolving client could not reach us -- success proves remote DNS.
 REMOTE_HOST = b"socks-origin.invalid"
@@ -85,23 +87,6 @@ def recvn(conn, n):
    return buf


-def pipe(src, dst):
-    try:
-        while True:
-            data = src.recv(65536)
-            if not data:
-                break
-            dst.sendall(data)
-    except OSError:
-        pass
-    finally:
-        for sock in (src, dst):
-            try:
-                sock.shutdown(socket.SHUT_RDWR)
-            except OSError:
-                pass
-
-
 def log(logdir, line):
    with open(os.path.join(logdir, SOCKS_LOG), "a") as handle:
        handle.write(line + "\n")