mirror of
https://github.com/xroche/httrack.git
synced 2026-07-17 14:19:48 +03:00
Share the proxy test servers' duplicated code in proxytestlib.py
proxy-https-server.py (#85) and proxy-connect-server.py (#564) were ~90% identical: the CONNECT proxy, the relay and the origin were copies, differing only in the origin body, the TLS wrap and the argv shape. The common half moves to tests/proxytestlib.py, leaving each server as a docstring plus a serve() call (153 and 152 lines become 32 and 33). socks5-server.py reuses the relay too, but keeps its own origin, which is specialized for keep-alive reuse and subpages. Behavior-preserving except that the shared origin always logs the request line, which previously only proxy-connect-server.py did. That is harmless for #85, whose origin-log assertion only checks that no Proxy-Authorization arrives. Tests 13, 52 and 57 pass unchanged, the full suite is identical at 101 pass / 7 skip, and the new module ships in the release tarball.
This commit is contained in:
Xavier Roche
@@ -15,138 +15,17 @@ Proxy modes (argv[3], default "ok"):
|
||||
Usage: proxy-https-server.py <cert.pem> <logdir> [mode]
|
||||
Prints "ORIGIN <port>", "PROXY <port>", then "ready" (one per line) on stdout.
|
||||
"""
|
||||
import http.server
|
||||
import os
|
||||
import socket
|
||||
import socketserver
|
||||
import ssl
|
||||
import sys
|
||||
import threading
|
||||
|
||||
import proxytestlib
|
||||
|
||||
ORIGIN_BODY = b"<html><body>ORIGIN-PAGE-85</body></html>"
|
||||
PROXY_LOG = "proxy.log"
|
||||
ORIGIN_LOG = "origin-headers.log"
|
||||
|
||||
|
||||
def make_origin(logdir):
|
||||
class Origin(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle:
|
||||
for key in self.headers.keys():
|
||||
handle.write(key + "\n")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.send_header("Content-Length", str(len(ORIGIN_BODY)))
|
||||
self.end_headers()
|
||||
self.wfile.write(ORIGIN_BODY)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
return Origin
|
||||
|
||||
|
||||
def start_origin(certfile, logdir):
|
||||
httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir))
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
ctx.load_cert_chain(certfile)
|
||||
httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
|
||||
port = httpd.socket.getsockname()[1]
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
return port
|
||||
|
||||
|
||||
def pipe(src, dst):
|
||||
try:
|
||||
while True:
|
||||
data = src.recv(65536)
|
||||
if not data:
|
||||
break
|
||||
dst.sendall(data)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
for sock in (src, dst):
|
||||
try:
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def handle_client(conn, logdir, mode):
|
||||
rfile = conn.makefile("rb")
|
||||
request_line = rfile.readline().decode("latin-1").strip()
|
||||
auth = None
|
||||
while True:
|
||||
line = rfile.readline().decode("latin-1")
|
||||
if line in ("\r\n", "\n", ""):
|
||||
break
|
||||
key, _, value = line.partition(":")
|
||||
if key.strip().lower() == "proxy-authorization":
|
||||
auth = value.strip()
|
||||
with open(os.path.join(logdir, PROXY_LOG), "a") as handle:
|
||||
handle.write(request_line + "\n")
|
||||
if auth is not None:
|
||||
handle.write("AUTH " + auth + "\n")
|
||||
parts = request_line.split()
|
||||
if not (len(parts) >= 2 and parts[0] == "CONNECT"):
|
||||
conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n")
|
||||
conn.close()
|
||||
return
|
||||
if mode == "flood":
|
||||
# 200, then an endless header stream with no terminating blank line: the
|
||||
# client must bound this and give up, not hang.
|
||||
try:
|
||||
conn.sendall(b"HTTP/1.0 200 Connection established\r\n")
|
||||
while True:
|
||||
conn.sendall(b"X-Pad: 0123456789\r\n")
|
||||
except OSError:
|
||||
pass
|
||||
conn.close()
|
||||
return
|
||||
host, _, port = parts[1].partition(":")
|
||||
try:
|
||||
upstream = socket.create_connection((host, int(port or 443)))
|
||||
except OSError:
|
||||
conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n")
|
||||
conn.close()
|
||||
return
|
||||
conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
|
||||
threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start()
|
||||
pipe(upstream, conn)
|
||||
|
||||
|
||||
def start_proxy(logdir, mode):
|
||||
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
srv.bind(("127.0.0.1", 0))
|
||||
srv.listen(16)
|
||||
port = srv.getsockname()[1]
|
||||
|
||||
def serve():
|
||||
while True:
|
||||
conn, _ = srv.accept()
|
||||
threading.Thread(
|
||||
target=handle_client, args=(conn, logdir, mode), daemon=True
|
||||
).start()
|
||||
|
||||
threading.Thread(target=serve, daemon=True).start()
|
||||
return port
|
||||
|
||||
|
||||
def main():
|
||||
certfile, logdir = sys.argv[1], sys.argv[2]
|
||||
mode = sys.argv[3] if len(sys.argv) > 3 else "ok"
|
||||
for name in (PROXY_LOG, ORIGIN_LOG):
|
||||
open(os.path.join(logdir, name), "w").close()
|
||||
origin_port = start_origin(certfile, logdir)
|
||||
proxy_port = start_proxy(logdir, mode)
|
||||
# Keep the port lines the caller parses LF: Windows would emit \r\n.
|
||||
sys.stdout.reconfigure(newline="\n")
|
||||
print("ORIGIN %d" % origin_port, flush=True)
|
||||
print("PROXY %d" % proxy_port, flush=True)
|
||||
print("ready", flush=True)
|
||||
threading.Event().wait()
|
||||
proxytestlib.serve(logdir, ORIGIN_BODY, 443, mode, certfile=certfile)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user