Don't resolve or dial the origin ourselves when a proxy is set

The connect-fallback probe resolved the origin locally and, on a proxy-connect
failure, fell back to dialing the origin direct — bypassing the proxy, leaking
the origin's DNS query, and sending it the request (with the full absolute URI)
over a direct connection. Under a proxy the socket dials the proxy, not the
origin, so an origin-address list is meaningless here; gate the probe on
!proxy.active. A dead proxy now fails cleanly instead of falling open to the
origin.

Test 55 drives a crawl through a dead proxy at a two-address origin decoy and
asserts the decoy is never contacted; it fails on the pre-fix engine.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/codeql.yml

@@ -46,6 +46,12 @@ jobs:
           # fopen's umask-controlled 0666 is intended for mirror/cache/log
           # output; the one credential file (cookies.txt) is kept 0600 on Unix.
           config: |
+            paths-ignore:
+              # Demo callback samples, not part of libhttrack.
+              - libtest
+              # ProxyTrack: a separate legacy binary with no auth surface; its
+              # recv/cache-parse code trips taint queries by design.
+              - src/proxy
             query-filters:
               - exclude:
                   id: cpp/world-writable-file-creation

fuzz/Makefile.am

@@ -39,6 +39,7 @@ EXTRA_DIST = README.md run-fuzzers.sh \
 	corpus/filters/filter.bin corpus/filters/filter-size.bin \
 	corpus/filters/regress-empty-subject-unique.bin \
 	corpus/filters/redos-star-classes.bin \
+	corpus/filters/regress-classdepth-timeout.bin \
 	corpus/url/http-url.txt corpus/url/relative-path.txt \
 	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
 	corpus/header/full-response.txt corpus/header/redirect.txt \

html/server/option10.html

@@ -98,7 +98,16 @@ ${do:end-if}
 <input type="hidden" name="redirect" value="">
 <input type="hidden" name="closeme" value="">
 
-${LANG_IOPT10}: 
+${LANG_PROXYTYPE}
+<select name="proxytype"
+			onMouseOver="info('${html:LANG_PROXYTYPETIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+>
+<option value=""${ztest:proxytype: selected:}>HTTP</option>
+<option value="1"${test:proxytype:: selected}>SOCKS5</option>
+</select>
+<br><br>
+
+${LANG_IOPT10}:
 <input name="prox" value="${prox}" size="32"
 			onMouseOver="info('${html:LANG_G14}'); return true" onMouseOut="info('&nbsp;'); return true"
 >:

html/server/option5.html

@@ -148,6 +148,14 @@ ${LANG_I65}
 >
 </td></tr>
 
+<tr><td>
+${LANG_PAUSEFILES}
+</td><td>
+<input name="pausefiles" value="${pausefiles}" size="8"
+			onMouseOver="info('${html:LANG_PAUSEFILESTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+>
+</td></tr>
+
 <tr><td>
 ${LANG_I52}
 </td><td>

html/server/option8.html

@@ -103,6 +103,12 @@ ${do:end-if}
 > ${LANG_I58}
 <br><br>
 
+${LANG_COOKIEFILE}
+<input name="cookiesfile" value="${cookiesfile}" size="40"
+			onMouseOver="info('${html:LANG_COOKIEFILETIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+>
+<br><br>
+
 ${LANG_I59}
 <br>
 <select name="checktype"
@@ -136,6 +142,27 @@ ${listid:robots:LISTDEF_8}
 > ${LANG_I62b2}
 <br><br>
 
+<input type="checkbox" name="keepwww" ${checked:keepwww}
+			onMouseOver="info('${html:LANG_KEEPWWWTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+> ${LANG_KEEPWWW}
+<br><br>
+
+<input type="checkbox" name="keepslashes" ${checked:keepslashes}
+			onMouseOver="info('${html:LANG_KEEPSLASHESTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+> ${LANG_KEEPSLASHES}
+<br><br>
+
+<input type="checkbox" name="keepqueryorder" ${checked:keepqueryorder}
+			onMouseOver="info('${html:LANG_KEEPQUERYORDERTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+> ${LANG_KEEPQUERYORDER}
+<br><br>
+
+${LANG_STRIPQUERY}
+<input name="stripquery" value="${stripquery}" size="40"
+			onMouseOver="info('${html:LANG_STRIPQUERYTIP}'); return true" onMouseOut="info('&nbsp;'); return true"
+>
+<br><br>
+
 <input type="checkbox" name="toler" ${checked:toler}
 			onMouseOver="info('${html:LANG_I1i}'); return true" onMouseOut="info('&nbsp;'); return true"
 > ${LANG_I62}

html/server/step2.html

@@ -129,15 +129,21 @@ ${do:copy:NoPwdInPages:hidepwd}
 ${do:copy:NoQueryStrings:hidequery}
 ${do:copy:NoPurgeOldFiles:nopurge}
 ${do:copy:Cookies:cookies}
+${do:copy:CookiesFile:cookiesfile}
 ${do:copy:CheckType:checktype}
 ${do:copy:ParseJava:parsejava}
 ${do:copy:HTTP10:http10}
 ${do:copy:TolerantRequests:toler}
 ${do:copy:UpdateHack:updhack}
 ${do:copy:URLHack:urlhack}
+${do:copy:KeepWww:keepwww}
+${do:copy:KeepSlashes:keepslashes}
+${do:copy:KeepQueryOrder:keepqueryorder}
+${do:copy:StripQuery:stripquery}
 ${do:copy:StoreAllInCache:cache2}
 ${do:copy:LogType:logtype}
 ${do:copy:UseHTTPProxyForFTP:ftpprox}
+${do:copy:ProxyType:proxytype}
 ${do:copy:Build:build}
 ${do:copy:PrimaryScan:filter}
 ${do:copy:Travel:travel}
@@ -148,6 +154,7 @@ ${do:copy:MaxHtml:maxhtml}
 ${do:copy:MaxOther:othermax}
 ${do:copy:MaxAll:sizemax}
 ${do:copy:MaxWait:pausebytes}
+${do:copy:PauseFiles:pausefiles}
 ${do:copy:Sockets:connexion}
 ${do:copy:Retry:retry}
 ${do:copy:MaxTime:maxtime}

html/server/step4.html

@@ -178,6 +178,12 @@ ${do:end-if}
 	${test:parsejava:--parse-java=0:}
 	${test:updhack:--updatehack}
 	${test:urlhack:--urlhack=0:--urlhack}
+	${test:keepwww:--keep-www-prefix}
+	${test:keepslashes:--keep-double-slashes}
+	${test:keepqueryorder:--keep-query-order}
+	${test:cookiesfile:--cookies-file "}${html:cookiesfile}${test:cookiesfile:"}
+	${test:pausefiles:--pause "}${pausefiles}${test:pausefiles:"}
+	${test:stripquery:--strip-query "}${html:stripquery}${test:stripquery:"}
 	${test:toler:--tolerant}
 	${test:http10:--http-10}
 	${test:cache2:--store-all-in-cache}
@@ -186,7 +192,7 @@ ${do:end-if}
 	${test:logtype:::--extra-log:--debug-log}
 	${test:index:--index=0:}
 	${test:index2:--search-index=0:--search-index}
-	${test:prox:--proxy "}${prox}${test:prox:\3A}${portprox}${test:prox:"}
+	${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype:socks5\3A//}${do:end-if}${do:output-mode:html}${prox}${test:prox:\3A}${portprox}${test:prox:"}
 	${test:ftpprox:--httpproxy-ftp=0:--httpproxy-ftp}
 </textarea>
 
@@ -219,15 +225,21 @@ NoPwdInPages=${ztest:hidepwd:0:1}
 NoQueryStrings=${ztest:hidequery:0:1}
 NoPurgeOldFiles=${ztest:nopurge:0:1}
 Cookies=${ztest:cookies:0:1}
+CookiesFile=${cookiesfile}
 CheckType=${ztest:checktype:0:1:2}
 ParseJava=${ztest:parsejava:0:1}
 HTTP10=${ztest:http10:0:1}
 TolerantRequests=${ztest:toler:0:1}
 UpdateHack=${ztest:updhack:0:1}
 URLHack=${ztest:urlhack:0:1}
+KeepWww=${ztest:keepwww:0:1}
+KeepSlashes=${ztest:keepslashes:0:1}
+KeepQueryOrder=${ztest:keepqueryorder:0:1}
+StripQuery=${stripquery}
 StoreAllInCache=${ztest:cache2:0:1}
 LogType=${logtype}
 UseHTTPProxyForFTP=${ztest:ftpprox:0:1}
+ProxyType=${proxytype}
 Build=${build}
 PrimaryScan=${filter}
 Travel=${travel}
@@ -238,6 +250,7 @@ MaxHtml=${maxhtml}
 MaxOther=${othermax}
 MaxAll=${sizemax}
 MaxWait=${pausebytes}
+PauseFiles=${pausefiles}
 Sockets=${connexion}
 Retry=${retry}
 MaxTime=${maxtime}

src/htsback.c

@@ -2696,9 +2696,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         int dispo = 0;
 
         // probe the resolved address list once per fresh connect (cache hit:
-        // the host was resolved when this connect was opened)
+        // the host was resolved when this connect was opened). Not under a
+        // proxy: the socket dials the proxy, so resolving the origin here leaks
+        // its DNS and lets a proxy-connect failure fall back to dialing it
+        // direct.
         if (cf->addr_count < 0 && back[i].r.soc != INVALID_SOCKET &&
-            !back[i].r.is_file) {
+            !back[i].r.is_file && !back[i].r.req.proxy.active) {
           SOCaddr scratch[HTS_MAXADDRNUM];
 
           cf->addr_count = hts_dns_resolve_all(opt, back[i].url_adr, scratch,
@@ -3310,16 +3313,26 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                         back[i].chunk_blocksize = -1;   /* ending */
                       back[i].r.totalsize += chunk_size;        // noter taille
                       if (back[i].r.adr != NULL || !back[i].r.is_write) {       // Not to disk
-                        back[i].r.adr =
-                          (char *) realloct(back[i].r.adr,
-                                            (size_t) back[i].r.totalsize + 1);
-                        if (!back[i].r.adr) {
-                          if (cache->log != NULL) {
-                            hts_log_print(opt, LOG_ERROR,
-                                          "not enough memory (" LLintP
-                                          ") for %s%s",
-                                          (LLint) back[i].r.totalsize,
-                                          back[i].url_adr, back[i].url_fil);
+                        // totalsize sums attacker-declared chunk sizes; past
+                        // 2GB the (size_t) cast below truncates on 32-bit and
+                        // under-allocates. Mark the chunk invalid so the shared
+                        // error path tears the transfer down.
+                        if (back[i].r.totalsize > INT32_MAX) {
+                          hts_log_print(opt, LOG_WARNING,
+                                        "Chunked resource too large for %s%s",
+                                        back[i].url_adr, back[i].url_fil);
+                          chunk_size = -1;
+                        } else {
+                          back[i].r.adr = (char *) realloct(
+                              back[i].r.adr, (size_t) back[i].r.totalsize + 1);
+                          if (!back[i].r.adr) {
+                            if (cache->log != NULL) {
+                              hts_log_print(opt, LOG_ERROR,
+                                            "not enough memory (" LLintP
+                                            ") for %s%s",
+                                            (LLint) back[i].r.totalsize,
+                                            back[i].url_adr, back[i].url_fil);
+                            }
                           }
                         }
                       }

src/htsfilters.c

@@ -154,12 +154,14 @@ static const char *strjoker_rec(strjoker_memo *memo, const char *chaine,
 
   if (depth > memo->maxdepth)
     memo->maxdepth = depth;
+  /* Charge the budget before the depth cap: cut disables the memo, so uncounted
+     deep calls would let the sub-cap search explode (OSS-Fuzz 535114376). */
+  if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
+    return NULL; /* work budget spent: fail the match safely */
   if (depth >= STRJOKER_MAXDEPTH) {
     memo->cut = HTS_TRUE;
     return NULL; /* nesting beyond any real filter: fail the branch safely */
   }
-  if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
-    return NULL; /* work budget spent: fail the match safely */
   if (memo->failed) {
     bit = (size_t) (chaine - memo->chaine0) * memo->stride +
           (size_t) (joker - memo->joker0);

src/htsselftest.c

@@ -794,6 +794,20 @@ static int st_filterbounds(httrackp *opt, int argc, char **argv) {
   memset(subj, 'a', 32);
   subj[32] = '\0';
   assertf(strjoker(subj, pat, NULL, NULL) != NULL);
+  /* Same pin for the class-branch shape users actually write (*[..]), against a
+     long subject: it must match with room to spare under the work cap. */
+  {
+    const char *seg = "*[A-Z,a-z,0-9]";
+    const size_t seglen = strlen(seg), nseg = 16;
+
+    for (i = 0; i < (int) nseg; i++)
+      memcpy(pat + i * seglen, seg, seglen);
+    pat[nseg * seglen] = '\0';
+    memset(subj, 'a', 512);
+    subj[512] = '\0';
+    assertf(strjoker_bounds(subj, pat, &steps, &maxsteps, NULL, NULL) != NULL);
+    assertf(steps < maxsteps);
+  }
   freet(pat);
   freet(subj);
   printf("filterbounds: OK\n");

tests/55_local-chunked.test

@@ -0,0 +1,18 @@
+#!/bin/bash
+# A well-formed Transfer-Encoding: chunked response mirrors intact: the chunk
+# automaton (htsback.c) joins the bodies and the 2GB in-RAM cap stays quiet.
+# The 76-char run spans a 64-byte chunk boundary (only contiguous once joined);
+# a CR anywhere would be leftover framing; '</body>' proves no truncation.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
+    --found 'chunked/page.html' \
+    --file-matches 'chunked/page.html' \
+    'chunk-body chunk-body chunk-body chunk-body chunk-body chunk-body chunk-body' \
+    --file-matches 'chunked/page.html' '</body>' \
+    --file-not-matches 'chunked/page.html' $'\r' \
+    --log-not-found 'too large' \
+    httrack 'BASEURL/chunked/index.html'

tests/56_local-proxy-noleak.test

@@ -0,0 +1,93 @@
+#!/bin/bash
+#
+# A crawl through a proxy must never resolve or dial the origin itself: the proxy
+# does that. A dead proxy + a multi-address origin used to fall back to dialing
+# the origin direct (bypassing the proxy, leaking its DNS and IP). The decoy
+# origin here must therefore receive nothing.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
+if test "${V6_SUPPORT:-}" == "no"; then
+    echo "no IPv6 support (resolver override compiled out), skipping"
+    exit 77
+fi
+python=$(find_python) || {
+    echo "python3 missing, skipping"
+    exit 77
+}
+
+server=$(nativepath "$top_srcdir/tests/local-server.py")
+root=$(nativepath "$top_srcdir/tests/server-root")
+tmpdir=$(mktemp -d)
+serverpid=
+
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+    return 0
+}
+trap cleanup EXIT
+
+# decoy origin: it must stay silent. LOCAL_SERVER_VERBOSE logs any request it gets.
+LOCAL_SERVER_VERBOSE=1 "$python" "$server" --root "$root" --bind 127.0.0.1 \
+    >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$tmpdir/srv.out" 2>/dev/null || true)
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$tmpdir/srv.err")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+
+# a proxy port nothing listens on: grab a free one and let it close (refuses).
+deadproxy=$("$python" -c \
+    'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()')
+
+# origin resolves to two addresses so a fallback would have a second to dial;
+# 127.0.0.1 (the decoy) is the one the old bypass reached.
+out="$tmpdir/crawl"
+HTTRACK_DEBUG_RESOLVE="decoyhost:127.0.0.2,127.0.0.1" \
+    httrack "http://decoyhost:$port/simple/basic.html" -O "$out" \
+    -P "127.0.0.1:$deadproxy" -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z \
+    >"$tmpdir/log" 2>&1
+
+log="$out/hts-log.txt"
+
+# the origin must have seen no connection (no leak/bypass)
+if grep -qE '"(GET|POST|HEAD|CONNECT) ' "$tmpdir/srv.err"; then
+    echo "FAIL: origin was contacted directly, bypassing the proxy"
+    cat "$tmpdir/srv.err"
+    exit 1
+fi
+
+# the crawl must fail at the proxy (proves it really tried the proxy, not a no-op)
+grep -q 'Connect Error' "$log" || {
+    echo "FAIL: expected a proxy connect error"
+    cat "$log"
+    exit 1
+}
+
+# and it must not have tried an origin-address fallback under the proxy
+if grep -q "trying next address" "$log"; then
+    echo "FAIL: fell back to an origin address under a proxy"
+    cat "$log"
+    exit 1
+fi
+
+echo "OK: a dead proxy fails cleanly without dialing the origin"

tests/Makefile.am

@@ -131,6 +131,8 @@ TESTS = \
 	51_local-update-codec.test \
 	52_local-socks5.test \
 	53_local-proxytrack-cache-corrupt.test \
-	54_local-update-truncate-purge.test
+	54_local-update-truncate-purge.test \
+	55_local-chunked.test \
+	56_local-proxy-noleak.test
 
 CLEANFILES = check-network_sh.cache

tests/local-server.py

@@ -1215,6 +1215,28 @@ class Handler(SimpleHTTPRequestHandler):
         if self.command != "HEAD":
             self.wfile.write(body)
 
+    def route_chunked_index(self):
+        self.send_html('\t<a href="page.html">chunked</a>\n')
+
+    def route_chunked_page(self):
+        # Transfer-Encoding: chunked over many small chunks: drives the engine's
+        # chunk automaton (htsback.c). The mirrored file must equal the joined
+        # chunk bodies, so the 2GB in-RAM cap doesn't fire on ordinary traffic.
+        blob = big_html("chunked", "<p>" + "chunk-body " * 300 + "</p>")
+        self.protocol_version = "HTTP/1.1"
+        self.send_response(200)
+        self.send_header("Content-Type", "text/html; charset=utf-8")
+        self.send_header("Transfer-Encoding", "chunked")
+        self.send_header("Connection", "close")
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        step = 64
+        for off in range(0, len(blob), step):
+            piece = blob[off : off + step]
+            self.wfile.write(b"%X\r\n" % len(piece) + piece + b"\r\n")
+        self.wfile.write(b"0\r\n\r\n")
+
     # Content-Disposition naming: the attachment filename replaces the
     # URL-derived name; path components in it are stripped (RFC 2616).
     CDISPO_NAMES = {
@@ -1554,6 +1576,8 @@ class Handler(SimpleHTTPRequestHandler):
         "/crange206mem/blob.bin": route_crange206mem,
         "/size/index.html": route_size_index,
         "/size/oversize.bin": route_size_oversize,
+        "/chunked/index.html": route_chunked_index,
+        "/chunked/page.html": route_chunked_page,
         "/errpage/index.html": route_errpage_index,
         "/errpage/good.html": route_errpage_good,
         "/errpage/missing.html": route_errpage_missing,