diff --git a/tests/Makefile.am b/tests/Makefile.am index b1ac829..5fc2df3 100644 --- a/tests/Makefile.am +++ b/tests/Makefile.am @@ -3,6 +3,7 @@ # silently drop it from the dist tarball and break "make distcheck". EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \ proxy-https-server.py socks5-server.py proxy-connect-server.py \ + proxytestlib.py \ local-crawl.sh local-server.py testlib.sh server.crt server.key \ server-root/simple/basic.html server-root/simple/link.html \ server-root/stripquery/index.html server-root/stripquery/a.html \ diff --git a/tests/proxy-connect-server.py b/tests/proxy-connect-server.py index 9583633..c6f8fcf 100644 --- a/tests/proxy-connect-server.py +++ b/tests/proxy-connect-server.py @@ -16,136 +16,17 @@ Proxy modes (argv[2], default "ok"): Usage: proxy-connect-server.py [mode] Prints "ORIGIN ", "PROXY ", then "ready" (one per line) on stdout. """ -import http.server -import os -import socket -import socketserver import sys -import threading + +import proxytestlib ORIGIN_BODY = b"ORIGIN-PAGE-564" -PROXY_LOG = "proxy.log" -ORIGIN_LOG = "origin-headers.log" - - -def make_origin(logdir): - class Origin(http.server.BaseHTTPRequestHandler): - def do_GET(self): - with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle: - handle.write(self.requestline + "\n") - for key in self.headers.keys(): - handle.write(key + "\n") - self.send_response(200) - self.send_header("Content-Type", "text/html") - self.send_header("Content-Length", str(len(ORIGIN_BODY))) - self.end_headers() - self.wfile.write(ORIGIN_BODY) - - def log_message(self, *args): - pass - - return Origin - - -def start_origin(logdir): - httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir)) - port = httpd.socket.getsockname()[1] - threading.Thread(target=httpd.serve_forever, daemon=True).start() - return port - - -def pipe(src, dst): - try: - while True: - data = src.recv(65536) - if not data: - break - dst.sendall(data) - except OSError: - pass - finally: - for sock in (src, dst): - try: - sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - - -def handle_client(conn, logdir, mode): - rfile = conn.makefile("rb") - request_line = rfile.readline().decode("latin-1").strip() - auth = None - while True: - line = rfile.readline().decode("latin-1") - if line in ("\r\n", "\n", ""): - break - key, _, value = line.partition(":") - if key.strip().lower() == "proxy-authorization": - auth = value.strip() - with open(os.path.join(logdir, PROXY_LOG), "a") as handle: - handle.write(request_line + "\n") - if auth is not None: - handle.write("AUTH " + auth + "\n") - parts = request_line.split() - # CONNECT-only: reject the classic absolute-URI form a normal proxy accepts - if not (len(parts) >= 2 and parts[0] == "CONNECT"): - conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n") - conn.close() - return - if mode == "flood": - # 200, then an endless header stream with no terminating blank line: the - # client must bound this and give up, not hang. - try: - conn.sendall(b"HTTP/1.0 200 Connection established\r\n") - while True: - conn.sendall(b"X-Pad: 0123456789\r\n") - except OSError: - pass - conn.close() - return - host, _, port = parts[1].partition(":") - try: - upstream = socket.create_connection((host, int(port or 80))) - except OSError: - conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n") - conn.close() - return - conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n") - threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start() - pipe(upstream, conn) - - -def start_proxy(logdir, mode): - srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - srv.bind(("127.0.0.1", 0)) - srv.listen(16) - port = srv.getsockname()[1] - - def serve(): - while True: - conn, _ = srv.accept() - threading.Thread( - target=handle_client, args=(conn, logdir, mode), daemon=True - ).start() - - threading.Thread(target=serve, daemon=True).start() - return port def main(): logdir = sys.argv[1] mode = sys.argv[2] if len(sys.argv) > 2 else "ok" - for name in (PROXY_LOG, ORIGIN_LOG): - open(os.path.join(logdir, name), "w").close() - origin_port = start_origin(logdir) - proxy_port = start_proxy(logdir, mode) - # Keep the port lines the caller parses LF: Windows would emit \r\n. - sys.stdout.reconfigure(newline="\n") - print("ORIGIN %d" % origin_port, flush=True) - print("PROXY %d" % proxy_port, flush=True) - print("ready", flush=True) - threading.Event().wait() + proxytestlib.serve(logdir, ORIGIN_BODY, 80, mode) if __name__ == "__main__": diff --git a/tests/proxy-https-server.py b/tests/proxy-https-server.py index eebf2de..9a4a171 100644 --- a/tests/proxy-https-server.py +++ b/tests/proxy-https-server.py @@ -15,138 +15,17 @@ Proxy modes (argv[3], default "ok"): Usage: proxy-https-server.py [mode] Prints "ORIGIN ", "PROXY ", then "ready" (one per line) on stdout. """ -import http.server -import os -import socket -import socketserver -import ssl import sys -import threading + +import proxytestlib ORIGIN_BODY = b"ORIGIN-PAGE-85" -PROXY_LOG = "proxy.log" -ORIGIN_LOG = "origin-headers.log" - - -def make_origin(logdir): - class Origin(http.server.BaseHTTPRequestHandler): - def do_GET(self): - with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle: - for key in self.headers.keys(): - handle.write(key + "\n") - self.send_response(200) - self.send_header("Content-Type", "text/html") - self.send_header("Content-Length", str(len(ORIGIN_BODY))) - self.end_headers() - self.wfile.write(ORIGIN_BODY) - - def log_message(self, *args): - pass - - return Origin - - -def start_origin(certfile, logdir): - httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir)) - ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) - ctx.load_cert_chain(certfile) - httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True) - port = httpd.socket.getsockname()[1] - threading.Thread(target=httpd.serve_forever, daemon=True).start() - return port - - -def pipe(src, dst): - try: - while True: - data = src.recv(65536) - if not data: - break - dst.sendall(data) - except OSError: - pass - finally: - for sock in (src, dst): - try: - sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - - -def handle_client(conn, logdir, mode): - rfile = conn.makefile("rb") - request_line = rfile.readline().decode("latin-1").strip() - auth = None - while True: - line = rfile.readline().decode("latin-1") - if line in ("\r\n", "\n", ""): - break - key, _, value = line.partition(":") - if key.strip().lower() == "proxy-authorization": - auth = value.strip() - with open(os.path.join(logdir, PROXY_LOG), "a") as handle: - handle.write(request_line + "\n") - if auth is not None: - handle.write("AUTH " + auth + "\n") - parts = request_line.split() - if not (len(parts) >= 2 and parts[0] == "CONNECT"): - conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n") - conn.close() - return - if mode == "flood": - # 200, then an endless header stream with no terminating blank line: the - # client must bound this and give up, not hang. - try: - conn.sendall(b"HTTP/1.0 200 Connection established\r\n") - while True: - conn.sendall(b"X-Pad: 0123456789\r\n") - except OSError: - pass - conn.close() - return - host, _, port = parts[1].partition(":") - try: - upstream = socket.create_connection((host, int(port or 443))) - except OSError: - conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n") - conn.close() - return - conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n") - threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start() - pipe(upstream, conn) - - -def start_proxy(logdir, mode): - srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) - srv.bind(("127.0.0.1", 0)) - srv.listen(16) - port = srv.getsockname()[1] - - def serve(): - while True: - conn, _ = srv.accept() - threading.Thread( - target=handle_client, args=(conn, logdir, mode), daemon=True - ).start() - - threading.Thread(target=serve, daemon=True).start() - return port def main(): certfile, logdir = sys.argv[1], sys.argv[2] mode = sys.argv[3] if len(sys.argv) > 3 else "ok" - for name in (PROXY_LOG, ORIGIN_LOG): - open(os.path.join(logdir, name), "w").close() - origin_port = start_origin(certfile, logdir) - proxy_port = start_proxy(logdir, mode) - # Keep the port lines the caller parses LF: Windows would emit \r\n. - sys.stdout.reconfigure(newline="\n") - print("ORIGIN %d" % origin_port, flush=True) - print("PROXY %d" % proxy_port, flush=True) - print("ready", flush=True) - threading.Event().wait() + proxytestlib.serve(logdir, ORIGIN_BODY, 443, mode, certfile=certfile) if __name__ == "__main__": diff --git a/tests/proxytestlib.py b/tests/proxytestlib.py new file mode 100644 index 0000000..4d5a227 --- /dev/null +++ b/tests/proxytestlib.py @@ -0,0 +1,146 @@ +#!/usr/bin/env python3 +"""Shared helpers for the local proxy test servers. + +A CONNECT proxy in front of an origin, as used by proxy-https-server.py (TLS +origin, #85) and proxy-connect-server.py (plain origin, #564). socks5-server.py +reuses the relay only; its origin is specialised for keep-alive reuse. +Importable because Python puts the running script's directory on sys.path. +""" +import http.server +import os +import socket +import socketserver +import ssl +import sys +import threading + +PROXY_LOG = "proxy.log" +ORIGIN_LOG = "origin-headers.log" + + +def pipe(src, dst): + """Relay bytes one way until EOF, then tear both ends down.""" + try: + while True: + data = src.recv(65536) + if not data: + break + dst.sendall(data) + except OSError: + pass + finally: + for sock in (src, dst): + try: + sock.shutdown(socket.SHUT_RDWR) + except OSError: + pass + + +def make_origin(logdir, body): + class Origin(http.server.BaseHTTPRequestHandler): + def do_GET(self): + # the request line proves origin-form vs absolute-URI (#564) + with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle: + handle.write(self.requestline + "\n") + for key in self.headers.keys(): + handle.write(key + "\n") + self.send_response(200) + self.send_header("Content-Type", "text/html") + self.send_header("Content-Length", str(len(body))) + self.end_headers() + self.wfile.write(body) + + def log_message(self, *args): + pass + + return Origin + + +def start_origin(logdir, body, certfile=None): + """Serve body on an ephemeral port, over TLS when certfile is given.""" + httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir, body)) + if certfile is not None: + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(certfile) + httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True) + port = httpd.socket.getsockname()[1] + threading.Thread(target=httpd.serve_forever, daemon=True).start() + return port + + +def handle_client(conn, logdir, mode, default_port): + rfile = conn.makefile("rb") + request_line = rfile.readline().decode("latin-1").strip() + auth = None + while True: + line = rfile.readline().decode("latin-1") + if line in ("\r\n", "\n", ""): + break + key, _, value = line.partition(":") + if key.strip().lower() == "proxy-authorization": + auth = value.strip() + with open(os.path.join(logdir, PROXY_LOG), "a") as handle: + handle.write(request_line + "\n") + if auth is not None: + handle.write("AUTH " + auth + "\n") + parts = request_line.split() + # CONNECT-only: reject the classic absolute-URI form a normal proxy accepts + if not (len(parts) >= 2 and parts[0] == "CONNECT"): + conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n") + conn.close() + return + if mode == "flood": + # 200, then endless headers with no blank line: the client must not hang + try: + conn.sendall(b"HTTP/1.0 200 Connection established\r\n") + while True: + conn.sendall(b"X-Pad: 0123456789\r\n") + except OSError: + pass + conn.close() + return + host, _, port = parts[1].partition(":") + try: + # default_port only backstops a portless CONNECT; httrack sends host:port + upstream = socket.create_connection((host, int(port or default_port))) + except OSError: + conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n") + conn.close() + return + conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n") + threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start() + pipe(upstream, conn) + + +def start_proxy(logdir, mode, default_port): + srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + srv.bind(("127.0.0.1", 0)) + srv.listen(16) + port = srv.getsockname()[1] + + def accept_loop(): + while True: + conn, _ = srv.accept() + threading.Thread( + target=handle_client, + args=(conn, logdir, mode, default_port), + daemon=True, + ).start() + + threading.Thread(target=accept_loop, daemon=True).start() + return port + + +def serve(logdir, origin_body, default_port, mode="ok", certfile=None): + """Start the origin+proxy pair, announce both ports, then block forever.""" + for name in (PROXY_LOG, ORIGIN_LOG): + open(os.path.join(logdir, name), "w").close() + origin_port = start_origin(logdir, origin_body, certfile) + proxy_port = start_proxy(logdir, mode, default_port) + # Keep the port lines the caller parses LF: Windows would emit \r\n. + sys.stdout.reconfigure(newline="\n") + print("ORIGIN %d" % origin_port, flush=True) + print("PROXY %d" % proxy_port, flush=True) + print("ready", flush=True) + threading.Event().wait() diff --git a/tests/socks5-server.py b/tests/socks5-server.py index 629e8f1..cd19c24 100644 --- a/tests/socks5-server.py +++ b/tests/socks5-server.py @@ -16,6 +16,8 @@ import struct import sys import threading +from proxytestlib import pipe + # The one name the proxy answers for; a .invalid TLD never resolves (RFC 6761), # so a locally-resolving client could not reach us -- success proves remote DNS. REMOTE_HOST = b"socks-origin.invalid" @@ -85,23 +87,6 @@ def recvn(conn, n): return buf -def pipe(src, dst): - try: - while True: - data = src.recv(65536) - if not data: - break - dst.sendall(data) - except OSError: - pass - finally: - for sock in (src, dst): - try: - sock.shutdown(socket.SHUT_RDWR) - except OSError: - pass - - def log(logdir, line): with open(os.path.join(logdir, SOCKS_LOG), "a") as handle: handle.write(line + "\n")