mirror of
https://github.com/xroche/httrack.git
synced 2026-07-17 06:10:42 +03:00
Compare commits
1 Commits
fix-602-pr
...
webhttrack
| Author | SHA1 | Date | |
|---|---|---|---|
|
6fd2c6255d |
19
.github/workflows/windows-build.yml
vendored
19
.github/workflows/windows-build.yml
vendored
@@ -149,28 +149,25 @@ jobs:
|
||||
export HTTPS_SUPPORT=yes BROTLI_ENABLED=yes ZSTD_ENABLED=yes
|
||||
export LC_ALL=C.UTF-8
|
||||
|
||||
# A wedged crawl must not eat the job's timeout budget. timeout(1)'s
|
||||
# signals can't reap a native httrack.exe (MSYS signals don't reach it),
|
||||
# so a hang orphaned processes that starved the runner; run_with_timeout
|
||||
# TerminateProcess-es the whole tree. 600s clears the slowest multi-pass
|
||||
# crawl (a few passes at --max-time=120 each).
|
||||
. ./testlib.sh
|
||||
# A wedged crawl must not eat the job's whole timeout budget.
|
||||
watchdog=""
|
||||
command -v timeout >/dev/null && watchdog="timeout 600"
|
||||
|
||||
pass=0 fail=0 skip=0 failed="" skipped=""
|
||||
for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
|
||||
*_local-*.test 13_crawl_proxy_https.test 58_watchdog.test; do
|
||||
*_local-*.test 13_crawl_proxy_https.test; do
|
||||
rc=0
|
||||
run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
|
||||
# shellcheck disable=SC2086
|
||||
$watchdog bash "$t" >"$t.log" 2>&1 || rc=$?
|
||||
case "$rc" in
|
||||
0) pass=$((pass + 1)); echo "PASS $t" ;;
|
||||
77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
|
||||
124) fail=$((fail + 1)) failed="$failed $t"; echo "FAIL $t (timed out, tree killed)" ;;
|
||||
*)
|
||||
fail=$((fail + 1)) failed="$failed $t"
|
||||
echo "FAIL $t (exit $rc)"
|
||||
# These assert with `test "$(...)" == "..." || exit 1`, which
|
||||
# says nothing at all on failure. Re-run traced, still bounded.
|
||||
run_with_timeout 600 bash -x "$t" >>"$t.log" 2>&1 || true
|
||||
# says nothing at all on failure. Re-run traced.
|
||||
bash -x "$t" >>"$t.log" 2>&1 || true
|
||||
tail -n 25 "$t.log" | sed 's/^/ /'
|
||||
;;
|
||||
esac
|
||||
|
||||
3
.gitignore
vendored
3
.gitignore
vendored
@@ -43,6 +43,3 @@ Makefile
|
||||
|
||||
# Python bytecode (tests/local-server.py).
|
||||
__pycache__/
|
||||
|
||||
# Per-checkout Claude Code rules (symlink into a local sandbox).
|
||||
/CLAUDE.local.md
|
||||
|
||||
@@ -67,9 +67,8 @@ Before pushing, and when reviewing others, don't skim for bugs:
|
||||
- **Co-Authored-By is mandatory for AI-assisted commits.** Carry a
|
||||
`Co-Authored-By:` trailer naming the assistant. Attribute there, never in a
|
||||
PR-body footer.
|
||||
- PRs are squash-merged: one commit per PR lands on master, built from the PR
|
||||
title and description, so those are what the history keeps. The branch's
|
||||
intermediate commits are not preserved.
|
||||
- PRs land as a merge commit; every commit on the branch goes onto master, so
|
||||
keep each commit message clean and meaningful.
|
||||
|
||||
## PR descriptions
|
||||
- Plain concise prose; lead with what changed and why. No What/Why/How template.
|
||||
|
||||
1
CLAUDE.local.md
Symbolic link
1
CLAUDE.local.md
Symbolic link
@@ -0,0 +1 @@
|
||||
/home/roche/git/httrack-works/CLAUDE.httrack.local.md
|
||||
@@ -6,8 +6,8 @@ with an AI assistant? The operational checklist is [AGENTS.md](AGENTS.md).
|
||||
## Pull requests
|
||||
|
||||
- One change per PR. Small diffs merge fast.
|
||||
- PRs are squash-merged: one commit per PR goes onto master, built from the PR
|
||||
title and description, so those are the history. Explain *why* there.
|
||||
- PRs land as a merge commit, so the branch's commits go onto master as-is: keep
|
||||
each commit message clean and explain *why*.
|
||||
- Be terse in the PR title and description: name the problem, not the fix, don't
|
||||
restate the diff, and calibrate length to the change.
|
||||
- Add or update tests for engine changes (`tests/`), and keep CI green.
|
||||
|
||||
@@ -103,7 +103,6 @@ ${LANG_PROXYTYPE}
|
||||
title='${html:LANG_PROXYTYPETIP}' onMouseOver="info('${html:LANG_PROXYTYPETIP}'); return true" onMouseOut="info(' '); return true"
|
||||
>
|
||||
<option value=""${ztest:proxytype: selected:}>HTTP</option>
|
||||
<option value="2"${test:proxytype::: selected}>HTTP (CONNECT tunnel)</option>
|
||||
<option value="1"${test:proxytype:: selected}>SOCKS5</option>
|
||||
</select>
|
||||
<br><br>
|
||||
|
||||
@@ -192,7 +192,7 @@ ${do:end-if}
|
||||
${test:logtype:::--extra-log:--debug-log}
|
||||
${test:index:--index=0:}
|
||||
${test:index2:--search-index=0:--search-index}
|
||||
${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype::socks5:connect}${test:proxytype:\3A//}${do:end-if}${do:output-mode:html}${prox}${test:prox:\3A}${portprox}${test:prox:"}
|
||||
${test:prox:--proxy "}${do:if-not-empty:prox}${test:proxytype:socks5\3A//}${do:end-if}${do:output-mode:html}${prox}${test:prox:\3A}${portprox}${test:prox:"}
|
||||
${test:ftpprox:--httpproxy-ftp=0:--httpproxy-ftp}
|
||||
</textarea>
|
||||
|
||||
|
||||
2
lang.def
2
lang.def
@@ -1009,7 +1009,7 @@ A fatal error has occurred during this mirror
|
||||
LANG_PROXYTYPE
|
||||
Proxy type:
|
||||
LANG_PROXYTYPETIP
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
LANG_COOKIEFILE
|
||||
Load cookies from file:
|
||||
LANG_COOKIEFILETIP
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Ôàòàëíà ãðåøêà ïðè ñúçäàâàíåòî íà òîçè îãëåäàëåí ñàéò
|
||||
Proxy type:
|
||||
Òèï íà ïðîêñè:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Ïðîòîêîë íà ïðîêñè. HTTP: ñòàíäàðòíî ïðîêñè. HTTP (CONNECT òóíåë): èçïðàùà âñÿêà çàÿâêà ïðåç CONNECT òóíåë, çà ïðîêñè ïîääúðæàùè ñàìî CONNECT êàòî HTTPTunnelPort íà Tor. SOCKS5: ïîðò ïî ïîäðàçáèðàíå 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Ïðîòîêîë íà ïðîêñè: HTTP èëè SOCKS5 (ñòàíäàðòíèÿò ïîðò íà SOCKS5 å 1080).
|
||||
Load cookies from file:
|
||||
Çàðåæäàíå íà 'áèñêâèòêè' îò ôàéë:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Ha ocurrido un error fatal durante esta copia
|
||||
Proxy type:
|
||||
Tipo de proxy:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protocolo del proxy. HTTP: proxy estándar. HTTP (túnel CONNECT): envía cada petición a través de un túnel CONNECT, para proxys que solo admiten CONNECT como el HTTPTunnelPort de Tor. SOCKS5: puerto predeterminado 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protocolo del proxy: HTTP o SOCKS5 (el puerto SOCKS5 predeterminado es 1080).
|
||||
Load cookies from file:
|
||||
Cargar cookies desde un archivo:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
|
||||
Proxy type:
|
||||
Typ proxy:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protokol proxy. HTTP: standardní proxy. HTTP (tunel CONNECT): odešle každý požadavek pøes tunel CONNECT, pro proxy podporující jen CONNECT jako HTTPTunnelPort v Toru. SOCKS5: výchozí port 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protokol proxy: HTTP nebo SOCKS5 (výchozí port SOCKS5 je 1080).
|
||||
Load cookies from file:
|
||||
Naèíst cookies ze souboru:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
這鏡像發生了不可回復的錯誤
|
||||
Proxy type:
|
||||
proxy 類型:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
代理協定。HTTP：標準代理。HTTP（CONNECT 隧道）：透過 CONNECT 隧道傳送每個請求，用於僅支援 CONNECT 的代理，例如 Tor 的 HTTPTunnelPort。SOCKS5：預設連接埠 1080。
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
proxy 協定: HTTP 或 SOCKS5 (SOCKS5 的預設連接埠為 1080)。
|
||||
Load cookies from file:
|
||||
從檔案載入 cookies:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
|
||||
Proxy type:
|
||||
代理类型:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
代理协议。HTTP：标准代理。HTTP（CONNECT 隧道）：通过 CONNECT 隧道发送每个请求，用于仅支持 CONNECT 的代理，例如 Tor 的 HTTPTunnelPort。SOCKS5：默认端口 1080。
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
代理协议: HTTP 或 SOCKS5 (SOCKS5 的默认端口为 1080)。
|
||||
Load cookies from file:
|
||||
从文件加载 cookies:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -932,8 +932,8 @@ A fatal error has occurred during this mirror
|
||||
Tijekom ovog zrcaljenja je nastala fatalna pogreška
|
||||
Proxy type:
|
||||
Vrsta posrednika:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protokol posrednika. HTTP: standardni posrednik. HTTP (CONNECT tunel): ¹alje svaki zahtjev kroz CONNECT tunel, za posrednike koji podr¾avaju samo CONNECT poput Torovog HTTPTunnelPorta. SOCKS5: zadani port 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protokol posrednika: HTTP ili SOCKS5 (zadani port za SOCKS5 je 1080).
|
||||
Load cookies from file:
|
||||
Uèitati kolaèiæe iz datoteke:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -978,8 +978,8 @@ HTTrack: could not save profile for '%s'!
|
||||
HTTrack: kunne ikke gemme profil for '%s'!
|
||||
Proxy type:
|
||||
Proxytype:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proxyprotokol. HTTP: standardproxy. HTTP (CONNECT-tunnel): sender hver forespørgsel gennem en CONNECT-tunnel, til proxyer der kun understøtter CONNECT som Tors HTTPTunnelPort. SOCKS5: standardport 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Proxyprotokol: HTTP eller SOCKS5 (SOCKS5 bruger som standard port 1080).
|
||||
Load cookies from file:
|
||||
Indlæs cookies fra fil:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Fataler Fehler während der Webseiten-Kopie
|
||||
Proxy type:
|
||||
Proxy-Typ:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proxy-Protokoll. HTTP: normaler Proxy. HTTP (CONNECT-Tunnel): sendet jede Anfrage durch einen CONNECT-Tunnel, für Proxys, die nur CONNECT unterstützen, wie Tors HTTPTunnelPort. SOCKS5: Standardport 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Proxy-Protokoll: HTTP oder SOCKS5 (Standardport von SOCKS5 ist 1080).
|
||||
Load cookies from file:
|
||||
Cookies aus Datei laden:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
|
||||
Proxy type:
|
||||
Proxy tüüp:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Puhverserveri protokoll. HTTP: tavaline puhverserver. HTTP (CONNECT-tunnel): saadab iga päringu läbi CONNECT-tunneli, ainult CONNECT-it toetavate puhverserverite jaoks nagu Tori HTTPTunnelPort. SOCKS5: vaikeport 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Proxy protokoll: HTTP või SOCKS5 (SOCKS5 vaikeport on 1080).
|
||||
Load cookies from file:
|
||||
Lae küpsised failist:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -978,8 +978,8 @@ HTTrack: could not save profile for '%s'!
|
||||
HTTrack: could not save profile for '%s'!
|
||||
Proxy type:
|
||||
Proxy type:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Load cookies from file:
|
||||
Load cookies from file:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -933,8 +933,8 @@ T
|
||||
|
||||
Proxy type:
|
||||
Välityspalvelimen tyyppi:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Välityspalvelimen protokolla. HTTP: tavallinen välityspalvelin. HTTP (CONNECT-tunneli): lähettää jokaisen pyynnön CONNECT-tunnelin kautta, vain CONNECT-yhteyksiä tukeville välityspalvelimille kuten Torin HTTPTunnelPort. SOCKS5: oletusportti 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Välityspalvelimen protokolla: HTTP tai SOCKS5 (SOCKS5:n oletusportti on 1080).
|
||||
Load cookies from file:
|
||||
Lataa evästeet tiedostosta:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -978,8 +978,8 @@ Default referer URL
|
||||
Champ referer par défaut
|
||||
Proxy type:
|
||||
Type de proxy :
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protocole du proxy. HTTP : proxy standard. HTTP (tunnel CONNECT) : envoie chaque requête via un tunnel CONNECT, pour les proxys acceptant uniquement CONNECT comme le HTTPTunnelPort de Tor. SOCKS5 : port par défaut 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protocole du proxy : HTTP ou SOCKS5 (le port SOCKS5 par défaut est 1080).
|
||||
Load cookies from file:
|
||||
Charger les cookies depuis un fichier :
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -932,8 +932,8 @@ A fatal error has occurred during this mirror
|
||||
Ένα καταστροφικό σφάλμα προκλήθηκε κατά την αντιγραφή αυτού του τόπου
|
||||
Proxy type:
|
||||
Τύπος proxy:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Πρωτόκολλο διαμεσολαβητή. HTTP: τυπικός διαμεσολαβητής. HTTP (σήραγγα CONNECT): στέλνει κάθε αίτημα μέσω σήραγγας CONNECT, για διαμεσολαβητές που υποστηρίζουν μόνο CONNECT όπως το HTTPTunnelPort του Tor. SOCKS5: προεπιλεγμένη θύρα 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Πρωτόκολλο proxy: HTTP ή SOCKS5 (η προεπιλεγμένη θύρα του SOCKS5 είναι 1080).
|
||||
Load cookies from file:
|
||||
Φόρτωση cookies από αρχείο:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Si è verificato un errore fatale durante la copia
|
||||
Proxy type:
|
||||
Tipo di proxy:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protocollo del proxy. HTTP: proxy standard. HTTP (tunnel CONNECT): invia ogni richiesta attraverso un tunnel CONNECT, per proxy che supportano solo CONNECT come l'HTTPTunnelPort di Tor. SOCKS5: porta predefinita 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protocollo del proxy: HTTP o SOCKS5 (la porta SOCKS5 predefinita è 1080).
|
||||
Load cookies from file:
|
||||
Carica i cookie da un file:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
|
||||
Proxy type:
|
||||
プロキシの種類:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
プロキシのプロトコル。HTTP: 標準のプロキシ。HTTP (CONNECT トンネル): すべてのリクエストを CONNECT トンネル経由で送信します。Tor の HTTPTunnelPort のような CONNECT のみのプロキシ用です。SOCKS5: 既定ポート 1080。
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
プロキシのプロトコル: HTTP または SOCKS5 (SOCKS5 の既定ポートは 1080)。
|
||||
Load cookies from file:
|
||||
クッキーをファイルから読み込む:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Íàñòàíà ôàòàëíà ãðåøêà ïðè îâî¼ mirror
|
||||
Proxy type:
|
||||
ÂØß ÝÐ ßàÞÚáØ:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
¿àÞâÞÚÞÛ ÝÐ ßàÞÚáØ. HTTP: áâÐÝÔÐàÔÕÝ ßàÞÚáØ. HTTP (âãÝÕÛ CONNECT): ÓÞ ØáßàÐüÐ áÕÚÞÕ ÑÐàÐúÕ ßàÕÚã âãÝÕÛ CONNECT, ×Ð ßàÞÚáØ èâÞ ßÞÔÔàÖãÒÐÐâ áÐÜÞ CONNECT ÚÐÚÞ HTTPTunnelPort ÝÐ Tor. SOCKS5: áâÐÝÔÐàÔÝÐ ßÞàâÐ 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
¿àÞâÞÚÞÛ ÝÐ ßàÞÚáØ: HTTP ØÛØ SOCKS5 (áâÐÝÔÐàÔÝÐâÐ ßÞàâÐ ÝÐ SOCKS5 Õ 1080).
|
||||
Load cookies from file:
|
||||
²çØâÐø ÚÞÛÐçØúÐ ÞÔ ÔÐâÞâÕÚÐ:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Végzetes hiba történt a tükrözés közben
|
||||
Proxy type:
|
||||
Proxy típusa:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proxy protokoll. HTTP: szabványos proxy. HTTP (CONNECT alagút): minden kérést CONNECT alagúton keresztül küld, a csak CONNECT-et támogató proxykhoz, mint a Tor HTTPTunnelPortja. SOCKS5: alapértelmezett port 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Proxy protokoll: HTTP vagy SOCKS5 (a SOCKS5 alapértelmezett portja 1080).
|
||||
Load cookies from file:
|
||||
Sütik betöltése fájlból:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Een fatale fout is opgetreden tijdens deze spiegeling
|
||||
Proxy type:
|
||||
Proxytype:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proxyprotocol. HTTP: standaardproxy. HTTP (CONNECT-tunnel): stuurt elke aanvraag via een CONNECT-tunnel, voor proxy's die alleen CONNECT ondersteunen zoals Tors HTTPTunnelPort. SOCKS5: standaardpoort 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Proxyprotocol: HTTP of SOCKS5 (standaardpoort van SOCKS5 is 1080).
|
||||
Load cookies from file:
|
||||
Cookies uit bestand laden:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
|
||||
Proxy type:
|
||||
Proxytype:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proxyprotokoll. HTTP: standard proxy. HTTP (CONNECT-tunnel): sender hver forespørsel gjennom en CONNECT-tunnel, for proxyer som bare støtter CONNECT som Tors HTTPTunnelPort. SOCKS5: standardport 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Proxyprotokoll: HTTP eller SOCKS5 (SOCKS5 bruker som standard port 1080).
|
||||
Load cookies from file:
|
||||
Last inn cookies fra fil:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Podczas tworzenia lustra wydarzyl sie fatalny blad.
|
||||
Proxy type:
|
||||
Typ proxy:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protokó³ proxy. HTTP: standardowe proxy. HTTP (tunel CONNECT): wysy³a ka¿de ¿±danie przez tunel CONNECT, dla proxy obs³uguj±cych tylko CONNECT, jak HTTPTunnelPort w Torze. SOCKS5: port domy¶lny 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protokó³ proxy: HTTP lub SOCKS5 (domy¶lny port SOCKS5 to 1080).
|
||||
Load cookies from file:
|
||||
Wczytaj ciasteczka z pliku:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -978,8 +978,8 @@ HTTrack: could not save profile for '%s'!
|
||||
HTTrack: não foi possível salvar o perfil para '%s'!
|
||||
Proxy type:
|
||||
Tipo de proxy:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protocolo do proxy. HTTP: proxy padrão. HTTP (túnel CONNECT): envia cada requisição através de um túnel CONNECT, para proxies que só suportam CONNECT como o HTTPTunnelPort do Tor. SOCKS5: porta padrão 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protocolo do proxy: HTTP ou SOCKS5 (a porta SOCKS5 padrão é 1080).
|
||||
Load cookies from file:
|
||||
Carregar cookies de um arquivo:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
|
||||
Proxy type:
|
||||
Tipo de proxy:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protocolo do proxy. HTTP: proxy padrão. HTTP (túnel CONNECT): envia cada pedido através de um túnel CONNECT, para proxies que só suportam CONNECT como o HTTPTunnelPort do Tor. SOCKS5: porta predefinida 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protocolo do proxy: HTTP ou SOCKS5 (a porta SOCKS5 predefinida é 1080).
|
||||
Load cookies from file:
|
||||
Carregar cookies de um ficheiro:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
A survenit o eroare fatală în timpul acestei clonări.
|
||||
Proxy type:
|
||||
Tip proxy:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protocol proxy. HTTP: proxy standard. HTTP (tunel CONNECT): trimite fiecare cerere printr-un tunel CONNECT, pentru proxy care accepta doar CONNECT precum HTTPTunnelPort al lui Tor. SOCKS5: port implicit 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protocol proxy: HTTP sau SOCKS5 (portul SOCKS5 implicit este 1080).
|
||||
Load cookies from file:
|
||||
Încarca cookies dintr-un fisier:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Âî âðåìÿ òåêóùåé çàêà÷êè ïðîèçîøëà ôàòàëüíàÿ îøèáêà
|
||||
Proxy type:
|
||||
Òèï ïðîêñè:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Ïðîòîêîë ïðîêñè. HTTP: îáû÷íûé ïðîêñè. HTTP (òóííåëü CONNECT): îòïðàâëÿåò êàæäûé çàïðîñ ÷åðåç òóííåëü CONNECT, äëÿ ïðîêñè, ïîääåðæèâàþùèõ òîëüêî CONNECT, íàïðèìåð HTTPTunnelPort â Tor. SOCKS5: ïîðò ïî óìîë÷àíèþ 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Ïðîòîêîë ïðîêñè: HTTP èëè SOCKS5 (ïî óìîë÷àíèþ SOCKS5 èñïîëüçóåò ïîðò 1080).
|
||||
Load cookies from file:
|
||||
Çàãðóçèòü cookies èç ôàéëà:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
|
||||
Proxy type:
|
||||
Typ proxy:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protokol proxy. HTTP: ¹tandardné proxy. HTTP (tunel CONNECT): odo¹le ka¾dú po¾iadavku cez tunel CONNECT, pre proxy podporujúce len CONNECT ako HTTPTunnelPort v Tore. SOCKS5: predvolený port 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protokol proxy: HTTP alebo SOCKS5 (predvolený port SOCKS5 je 1080).
|
||||
Load cookies from file:
|
||||
Naèíta» cookies zo súboru:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
|
||||
Proxy type:
|
||||
Vrsta proxyja:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Protokol proxy. HTTP: standardni proxy. HTTP (tunel CONNECT): poslje vsako zahtevo skozi tunel CONNECT, za posrednike, ki podpirajo samo CONNECT, kot je HTTPTunnelPort v Toru. SOCKS5: privzeta vrata 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Protokol proxy: HTTP ali SOCKS5 (privzeta vrata SOCKS5 so 1080).
|
||||
Load cookies from file:
|
||||
Nalozi piskotke iz datoteke:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
|
||||
Proxy type:
|
||||
Proxytyp:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proxyprotokoll. HTTP: standardproxy. HTTP (CONNECT-tunnel): skickar varje begäran genom en CONNECT-tunnel, för proxyservrar som bara stöder CONNECT som Tors HTTPTunnelPort. SOCKS5: standardport 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Proxyprotokoll: HTTP eller SOCKS5 (SOCKS5 använder som standard port 1080).
|
||||
Load cookies from file:
|
||||
Läs in cookies från fil:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Bu yansýlama iþlemi sýrasýnda ölümcül bir hata oluþtu
|
||||
Proxy type:
|
||||
Proxy türü:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proxy protokolü. HTTP: standart proxy. HTTP (CONNECT tüneli): her isteði bir CONNECT tüneli üzerinden gönderir; Tor'un HTTPTunnelPort gibi yalnýzca CONNECT destekleyen proxy'ler için. SOCKS5: varsayýlan baðlantý noktasý 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Proxy protokolü: HTTP veya SOCKS5 (SOCKS5 varsayýlan portu 1080).
|
||||
Load cookies from file:
|
||||
Çerezleri dosyadan yükle:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
|
||||
Proxy type:
|
||||
Òèï ïðîêñ³:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Ïðîòîêîë ïðîêñ³. HTTP: ñòàíäàðòíèé ïðîêñ³. HTTP (òóíåëü CONNECT): íàäñèëຠêîæåí çàïèò ÷åðåç òóíåëü CONNECT, äëÿ ïðîêñ³, ùî ï³äòðèìóþòü ëèøå CONNECT, ÿê-îò HTTPTunnelPort ó Tor. SOCKS5: ïîðò çà çàìîâ÷óâàííÿì 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Ïðîòîêîë ïðîêñ³: HTTP àáî SOCKS5 (çà çàìîâ÷óâàííÿì SOCKS5 âèêîðèñòîâóº ïîðò 1080).
|
||||
Load cookies from file:
|
||||
Çàâàíòàæèòè cookies ç ôàéëó:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -930,8 +930,8 @@ A fatal error has occurred during this mirror
|
||||
Joriy ko’chirish vaqtida jiddiy xatolik yuz berdi
|
||||
Proxy type:
|
||||
Proksi turi:
|
||||
Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.
|
||||
Proksi protokoli. HTTP: oddiy proksi. HTTP (CONNECT tuneli): har bir so'rovni CONNECT tuneli orqali yuboradi; faqat CONNECT'ni qo'llab-quvvatlaydigan proksilar uchun, masalan Tor HTTPTunnelPort. SOCKS5: standart port 1080.
|
||||
Proxy protocol: HTTP, or SOCKS5 (SOCKS5 default port is 1080).
|
||||
Proksi protokoli: HTTP yoki SOCKS5 (SOCKS5 ning standart porti 1080).
|
||||
Load cookies from file:
|
||||
Fayldan cookies yuklamoq:
|
||||
Preload cookies from a Netscape cookies.txt file before crawling.
|
||||
|
||||
@@ -135,7 +135,7 @@ continue an interrupted mirror using the cache (\-\-continue)
|
||||
mirror ALL links located in the first level pages (mirror links) (\-\-mirrorlinks)
|
||||
.SS Proxy options:
|
||||
.IP \-P
|
||||
proxy use (\-P [socks5://|connect://][user:pass@]proxy:port) (\-\-proxy <param>)
|
||||
proxy use (\-P [socks5://][user:pass@]proxy:port) (\-\-proxy <param>)
|
||||
.IP \-%f
|
||||
*use proxy for ftp (f0 don't use) (\-\-httpproxy\-ftp[=N])
|
||||
.IP \-%b
|
||||
|
||||
@@ -2696,12 +2696,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
int dispo = 0;
|
||||
|
||||
// probe the resolved address list once per fresh connect (cache hit:
|
||||
// the host was resolved when this connect was opened). Not under a
|
||||
// proxy: the socket dials the proxy, so resolving the origin here leaks
|
||||
// its DNS and lets a proxy-connect failure fall back to dialing it
|
||||
// direct.
|
||||
// the host was resolved when this connect was opened)
|
||||
if (cf->addr_count < 0 && back[i].r.soc != INVALID_SOCKET &&
|
||||
!back[i].r.is_file && !back[i].r.req.proxy.active) {
|
||||
!back[i].r.is_file) {
|
||||
SOCaddr scratch[HTS_MAXADDRNUM];
|
||||
|
||||
cf->addr_count = hts_dns_resolve_all(opt, back[i].url_adr, scratch,
|
||||
@@ -2757,28 +2754,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
}
|
||||
}
|
||||
|
||||
// plain http tunneled through a CONNECT-only proxy (#564)
|
||||
if (back[i].r.req.proxy.active &&
|
||||
hts_proxy_is_connect(back[i].r.req.proxy.name) &&
|
||||
!back[i].r.keep_alive
|
||||
#if HTS_USEOPENSSL
|
||||
&& !back[i].r.ssl
|
||||
#endif
|
||||
) {
|
||||
const int timeout = back[i].timeout > 0 ? back[i].timeout : 30;
|
||||
|
||||
if (!http_proxy_tunnel(opt, &back[i].r, back[i].url_adr, timeout)) {
|
||||
if (!strnotempty(back[i].r.msg))
|
||||
strcpybuff(back[i].r.msg, "proxy CONNECT failed");
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
back[i].r.statuscode = STATUSCODE_NON_FATAL;
|
||||
back[i].status = STATUS_READY;
|
||||
back_set_finished(sback, i);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
/* SSL mode */
|
||||
if (back[i].r.ssl) {
|
||||
|
||||
@@ -323,9 +323,11 @@ typedef int64_t TStamp;
|
||||
#define LLintP "%" PRId64
|
||||
|
||||
/* Integer type for file offsets/sizes passed to the C library; INTsysP is its
|
||||
printf conversion. HTS_LFS is the large-file macro: LFS_FLAG is a configure
|
||||
make variable carrying the -D flags, never itself defined. */
|
||||
#if defined(HTS_LFS) || defined(_MSC_VER)
|
||||
printf conversion. FIXME: LFS_FLAG is a configure make variable, never a C
|
||||
macro, so this test is dead and INTsys stays int on POSIX despite large-file
|
||||
support (the real macro is HTS_LFS). Widening it there is an installed-header
|
||||
type change, so it is left alone here. */
|
||||
#if defined(LFS_FLAG) || defined(_MSC_VER)
|
||||
typedef LLint INTsys;
|
||||
|
||||
#define INTsysP LLintP
|
||||
|
||||
@@ -477,7 +477,7 @@ void help(const char *app, int more) {
|
||||
(" Y mirror ALL links located in the first level pages (mirror links)");
|
||||
infomsg("");
|
||||
infomsg("Proxy options:");
|
||||
infomsg(" P proxy use (-P [socks5://|connect://][user:pass@]proxy:port)");
|
||||
infomsg(" P proxy use (-P [socks5://][user:pass@]proxy:port)");
|
||||
infomsg(" %f *use proxy for ftp (f0 don't use)");
|
||||
infomsg(" %b use this local hostname to make/send requests (-%b hostname)");
|
||||
infomsg("");
|
||||
|
||||
48
src/htslib.c
48
src/htslib.c
@@ -65,7 +65,6 @@ Please visit our Website: http://www.httrack.com
|
||||
#endif /* _WIN32 */
|
||||
#include <stdarg.h>
|
||||
|
||||
#include <ctype.h>
|
||||
#include <string.h>
|
||||
#include <time.h>
|
||||
#include <stdarg.h>
|
||||
@@ -948,10 +947,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
/* widths bound method[256], url[HTS_URLMAXSIZE*2], protocol[256] */
|
||||
if (sscanf(line, "%255s %2047s %255s", method, url, protocol) == 3) {
|
||||
size_t ret;
|
||||
// http proxy: absolute-URI; socks/CONNECT tunnel: origin-form
|
||||
// absolute-URI for an http proxy; a socks tunnel takes origin-form
|
||||
if (retour->req.proxy.active &&
|
||||
!hts_proxy_is_socks(retour->req.proxy.name) &&
|
||||
!hts_proxy_is_connect(retour->req.proxy.name)) {
|
||||
!hts_proxy_is_socks(retour->req.proxy.name)) {
|
||||
print_buffer(&bstr,
|
||||
"%s http://%s%s %s\r\n", method, adr, url,
|
||||
protocol);
|
||||
@@ -987,10 +985,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
print_buffer(&bstr, "HEAD ");
|
||||
}
|
||||
|
||||
// an http proxy needs an absolute URI; a socks or CONNECT tunnel does not
|
||||
// an http proxy needs an absolute URI; a socks tunnel does not
|
||||
if (retour->req.proxy.active &&
|
||||
!hts_proxy_is_socks(retour->req.proxy.name) &&
|
||||
!hts_proxy_is_connect(retour->req.proxy.name) &&
|
||||
(strncmp(adr, "https://", 8) != 0)) {
|
||||
if (!link_has_authority(adr)) { // default http
|
||||
#if HDEBUG
|
||||
@@ -1035,10 +1032,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
if (xsend)
|
||||
print_buffer(&bstr, "%s", xsend); // éventuelles autres lignes
|
||||
|
||||
// https/connect://: auth rides the CONNECT; socks: the handshake
|
||||
// for https, auth rides the CONNECT (the tunneled GET would leak it); for
|
||||
// socks, the handshake
|
||||
if (retour->req.proxy.active &&
|
||||
!hts_proxy_is_socks(retour->req.proxy.name) &&
|
||||
!hts_proxy_is_connect(retour->req.proxy.name) &&
|
||||
strncmp(adr, "https://", 8) != 0) {
|
||||
if (link_has_authorization(retour->req.proxy.name)) { // et hop, authentification proxy!
|
||||
const char *a = jump_identification_const(retour->req.proxy.name);
|
||||
@@ -3720,8 +3717,6 @@ const char *jump_protocol_const(const char *source) {
|
||||
source += p;
|
||||
else if ((p = strfield(source, "socks5:")))
|
||||
source += p;
|
||||
else if ((p = strfield(source, "connect:")))
|
||||
source += p;
|
||||
// net_path
|
||||
if (strncmp(source, "//", 2) == 0)
|
||||
source += 2;
|
||||
@@ -3737,31 +3732,11 @@ hts_boolean hts_proxy_is_socks(const char *name) {
|
||||
: HTS_FALSE;
|
||||
}
|
||||
|
||||
hts_boolean hts_proxy_is_connect(const char *name) {
|
||||
if (name == NULL)
|
||||
return HTS_FALSE;
|
||||
return strfield(name, "connect:") ? HTS_TRUE : HTS_FALSE;
|
||||
}
|
||||
|
||||
// default proxy port for a -P argument, keyed on the scheme
|
||||
static int proxy_default_port(const char *arg) {
|
||||
return hts_proxy_is_socks(arg) ? 1080 : 8080;
|
||||
}
|
||||
|
||||
// port "a" of -P argument "arg": digits fitting TCP's 1..65535, else the scheme
|
||||
// default. Not sscanf("%d"): past INT_MAX it wraps to a garbage port (#602)
|
||||
static int parse_proxy_port(const char *a, const char *arg) {
|
||||
char *end;
|
||||
long p;
|
||||
|
||||
if (!isdigit((unsigned char) *a)) // strtol would eat a sign or leading space
|
||||
return proxy_default_port(arg);
|
||||
p = strtol(a, &end, 10);
|
||||
if (*end != '\0' || p < 1 || p > 65535) // ERANGE lands out of range too
|
||||
return proxy_default_port(arg);
|
||||
return (int) p;
|
||||
}
|
||||
|
||||
void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
|
||||
const char *authority = strstr(arg, "://");
|
||||
const char *a;
|
||||
@@ -3769,15 +3744,18 @@ void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
|
||||
|
||||
if (name_size == 0)
|
||||
return;
|
||||
// scan back to the port ':' (or userinfo '@'), never past the authority (a
|
||||
// scheme's own colon is not a port separator) nor into an IPv6 literal (']'
|
||||
// stops it); inspect a[-1] from one-past-end so no pointer underflows
|
||||
// scan back to the port ':' (or userinfo '@'), but never past the authority,
|
||||
// so a scheme's own colon is not read as a port separator; inspect a[-1] from
|
||||
// one-past-end so no pointer below the string is ever formed
|
||||
authority = (authority != NULL) ? authority + 3 : arg;
|
||||
a = arg + strlen(arg);
|
||||
while (a > authority && a[-1] != ':' && a[-1] != '@' && a[-1] != ']')
|
||||
while (a > authority && a[-1] != ':' && a[-1] != '@')
|
||||
a--;
|
||||
if (a > authority && a[-1] == ':') {
|
||||
*port = parse_proxy_port(a, arg);
|
||||
int p = -1;
|
||||
|
||||
sscanf(a, "%d", &p);
|
||||
*port = (p > 0) ? p : proxy_default_port(arg);
|
||||
namelen = (size_t) (a - 1 - arg);
|
||||
} else {
|
||||
*port = proxy_default_port(arg);
|
||||
|
||||
@@ -210,9 +210,6 @@ int check_writeinput_t(T_SOC soc, int timeout);
|
||||
/* TRUE if this -P proxy name (which keeps its scheme) is a SOCKS5 proxy. */
|
||||
hts_boolean hts_proxy_is_socks(const char *name);
|
||||
|
||||
/* TRUE if this -P proxy name is a "connect://" CONNECT-only proxy (#564). */
|
||||
hts_boolean hts_proxy_is_connect(const char *name);
|
||||
|
||||
void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * retour,
|
||||
char *rcvd);
|
||||
void treatfirstline(htsblk * retour, const char *rcvd);
|
||||
|
||||
@@ -90,15 +90,12 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
|
||||
if (soc == INVALID_SOCKET)
|
||||
return 0;
|
||||
|
||||
// CONNECT needs an explicit host:port; default :80 for http, :443 for https
|
||||
// CONNECT needs an explicit host:port; default the https port
|
||||
authority[0] = '\0';
|
||||
if (portsep != NULL)
|
||||
strlcatbuff(authority, host, sizeof(authority)); // already host:port
|
||||
else {
|
||||
const int defport = (strncmp(adr, "https://", 8) == 0) ? 443 : 80;
|
||||
|
||||
snprintf(authority, sizeof(authority), "%s:%d", host, defport);
|
||||
}
|
||||
else
|
||||
snprintf(authority, sizeof(authority), "%s:%d", host, 443);
|
||||
|
||||
// backstop: never let a stray CR/LF in the host smuggle a second line into
|
||||
// the CONNECT request (the host is already sanitized upstream)
|
||||
@@ -135,7 +132,7 @@ int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
|
||||
}
|
||||
strlcatbuff(req, H_CRLF, sizeof(req)); // end of request headers
|
||||
|
||||
// raw send(): sendc() would route to TLS when ssl is set (https tunnel)
|
||||
// raw send: ssl is set, so sendc() would route to TLS
|
||||
{
|
||||
const char *p = req;
|
||||
size_t remain = strlen(req);
|
||||
|
||||
@@ -1365,12 +1365,9 @@ static int st_proxyurl(httrackp *opt, int argc, char **argv) {
|
||||
return 1;
|
||||
}
|
||||
hts_parse_proxy(argv[0], name, sizeof(name), &port);
|
||||
// host= is the resolved host (scheme/userinfo stripped); kind= the transport
|
||||
printf("name=%s port=%d host=%s kind=%s\n", name, port,
|
||||
jump_identification_const(name),
|
||||
hts_proxy_is_socks(name) ? "socks"
|
||||
: hts_proxy_is_connect(name) ? "connect"
|
||||
: "http");
|
||||
// host= is what the connect actually resolves (scheme + user:pass stripped)
|
||||
printf("name=%s port=%d host=%s\n", name, port,
|
||||
jump_identification_const(name));
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -1510,22 +1507,6 @@ static int st_socks5(httrackp *opt, int argc, char **argv) {
|
||||
assertf(io.sent_len == 0);
|
||||
}
|
||||
|
||||
/* the request is always ATYP=domain, which cannot carry an IPv6 literal: a
|
||||
bracketed origin is rejected rather than sent as a bogus domain name. The
|
||||
msg check pins the reason: a stricter host validator would also reject
|
||||
these, but for the wrong cause. */
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, "[::1]", proxy, &io) == 0);
|
||||
assertf(io.sent_len == 0);
|
||||
assertf(strstr(io.msg, "IPv6") != NULL);
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, "[2001:db8::1]:8443", proxy, &io) ==
|
||||
0);
|
||||
assertf(io.sent_len == 0);
|
||||
assertf(strstr(io.msg, "IPv6") != NULL);
|
||||
|
||||
printf("socks5 self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -5,89 +5,40 @@ set -euo pipefail
|
||||
|
||||
# hts_parse_proxy splits a -P argument "[scheme://][user:pass@]host[:port]" into
|
||||
# the proxy host string and port. -#test=proxyurl <arg> prints
|
||||
# "name=.. port=.. host=.. kind=..", where host= is what the connect resolves and
|
||||
# kind= is the transport the scheme selects (http, socks or connect).
|
||||
# "name=.. port=.. host=..", where host= is what the connect resolves.
|
||||
|
||||
p() {
|
||||
test "$(httrack -O /dev/null -#test=proxyurl "$1")" == "$2" || exit 1
|
||||
}
|
||||
|
||||
# bare host, with and without an explicit port
|
||||
p 'proxy.example.com:8080' 'name=proxy.example.com port=8080 host=proxy.example.com kind=http'
|
||||
p 'proxy.example.com' 'name=proxy.example.com port=8080 host=proxy.example.com kind=http'
|
||||
p 'proxy.example.com:8080' 'name=proxy.example.com port=8080 host=proxy.example.com'
|
||||
p 'proxy.example.com' 'name=proxy.example.com port=8080 host=proxy.example.com'
|
||||
|
||||
# user:pass@ is kept in name (for proxy auth) and stripped from host
|
||||
p 'user:pass@proxy:3128' 'name=user:pass@proxy port=3128 host=proxy kind=http'
|
||||
p 'user:pass@proxy' 'name=user:pass@proxy port=8080 host=proxy kind=http'
|
||||
p 'user:pass@proxy:3128' 'name=user:pass@proxy port=3128 host=proxy'
|
||||
p 'user:pass@proxy' 'name=user:pass@proxy port=8080 host=proxy'
|
||||
|
||||
# a scheme colon must not be read as a port: http://host with no port used to
|
||||
# parse to name=http with a garbage port
|
||||
p 'http://proxy:8080' 'name=http://proxy port=8080 host=proxy kind=http'
|
||||
p 'http://proxy' 'name=http://proxy port=8080 host=proxy kind=http'
|
||||
p 'http://proxy:8080' 'name=http://proxy port=8080 host=proxy'
|
||||
p 'http://proxy' 'name=http://proxy port=8080 host=proxy'
|
||||
|
||||
# socks5/socks5h default to 1080 and resolve to the bare host
|
||||
p 'socks5://host:1080' 'name=socks5://host port=1080 host=host kind=socks'
|
||||
p 'socks5://host' 'name=socks5://host port=1080 host=host kind=socks'
|
||||
p 'socks5h://host' 'name=socks5h://host port=1080 host=host kind=socks'
|
||||
p 'socks5://host:1080' 'name=socks5://host port=1080 host=host'
|
||||
p 'socks5://host' 'name=socks5://host port=1080 host=host'
|
||||
p 'socks5h://host' 'name=socks5h://host port=1080 host=host'
|
||||
# explicit port wins over the 1080 default, with and without userinfo
|
||||
p 'socks5://host:9050' 'name=socks5://host port=9050 host=host kind=socks'
|
||||
p 'socks5://user:pass@host:9050' 'name=socks5://user:pass@host port=9050 host=host kind=socks'
|
||||
|
||||
# connect:// is an http proxy driven with CONNECT for every request (#564); it
|
||||
# defaults to the http 8080 port and resolves to the bare host
|
||||
p 'connect://host' 'name=connect://host port=8080 host=host kind=connect'
|
||||
p 'connect://host:8082' 'name=connect://host port=8082 host=host kind=connect'
|
||||
p 'connect://user:pass@host:8082' 'name=connect://user:pass@host port=8082 host=host kind=connect'
|
||||
# only the scheme selects connect mode: a host literally named "connect" is a
|
||||
# plain http proxy, not a tunnel
|
||||
p 'connect:8080' 'name=connect port=8080 host=connect kind=http'
|
||||
p 'connect.example.com:3128' 'name=connect.example.com port=3128 host=connect.example.com kind=http'
|
||||
p 'socks5://host:9050' 'name=socks5://host port=9050 host=host'
|
||||
p 'socks5://user:pass@host:9050' 'name=socks5://user:pass@host port=9050 host=host'
|
||||
|
||||
# the split is byte-transparent: percent-escapes stay encoded (decoded later at
|
||||
# auth time), only structural ASCII ':'/'@' delimit, and the last '@' ends the
|
||||
# userinfo. So %40/%3A and UTF-8 bytes never mis-split host or port.
|
||||
p 'socks5://user:p%40ss@host:1080' 'name=socks5://user:p%40ss@host port=1080 host=host kind=socks'
|
||||
p 'socks5://us%3Aer:pass@host:1080' 'name=socks5://us%3Aer:pass@host port=1080 host=host kind=socks'
|
||||
p 'socks5://a@b:c@host:1080' 'name=socks5://a@b:c@host port=1080 host=host kind=socks'
|
||||
p 'socks5://naïve:pass@héllo:1080' 'name=socks5://naïve:pass@héllo port=1080 host=héllo kind=socks'
|
||||
|
||||
# a bracketed IPv6 literal holds colons of its own: the backward scan must stop
|
||||
# at the ']', or a portless literal parses to a truncated name and port=1
|
||||
p 'http://[2001:db8::1]' 'name=http://[2001:db8::1] port=8080 host=[2001:db8::1] kind=http'
|
||||
p 'http://[2001:db8::1]:3128' 'name=http://[2001:db8::1] port=3128 host=[2001:db8::1] kind=http'
|
||||
p 'socks5://[::1]' 'name=socks5://[::1] port=1080 host=[::1] kind=socks'
|
||||
p 'socks5://[::1]:1080' 'name=socks5://[::1] port=1080 host=[::1] kind=socks'
|
||||
p 'socks5://user:pass@[::1]' 'name=socks5://user:pass@[::1] port=1080 host=[::1] kind=socks'
|
||||
p 'socks5://user:pass@[::1]:9050' 'name=socks5://user:pass@[::1] port=9050 host=[::1] kind=socks'
|
||||
p '[::1]' 'name=[::1] port=8080 host=[::1] kind=http'
|
||||
p '[::1]:3128' 'name=[::1] port=3128 host=[::1] kind=http'
|
||||
# the ']' only ends the scan: one outside a literal must eat neither the name nor
|
||||
# the port. The last two discriminate: when a real port follows the ']', a
|
||||
# truncating bug agrees by accident.
|
||||
p 'http://user:p]ass@host:80' 'name=http://user:p]ass@host port=80 host=host kind=http'
|
||||
p 'http://a]b:3128' 'name=http://a]b port=3128 host=a]b kind=http'
|
||||
p 'http://a]b' 'name=http://a]b port=8080 host=a]b kind=http'
|
||||
p 'http://a]b:c@host' 'name=http://a]b:c@host port=8080 host=host kind=http'
|
||||
p 'socks5://user:p%40ss@host:1080' 'name=socks5://user:p%40ss@host port=1080 host=host'
|
||||
p 'socks5://us%3Aer:pass@host:1080' 'name=socks5://us%3Aer:pass@host port=1080 host=host'
|
||||
p 'socks5://a@b:c@host:1080' 'name=socks5://a@b:c@host port=1080 host=host'
|
||||
p 'socks5://naïve:pass@héllo:1080' 'name=socks5://naïve:pass@héllo port=1080 host=héllo'
|
||||
|
||||
# a lone ':' must not underflow the backward scan
|
||||
p ':' 'name= port=8080 host= kind=http'
|
||||
|
||||
# a port is *DIGIT in 1..65535, else the scheme default: sscanf("%d") used to
|
||||
# wrap past INT_MAX and hand back a garbage port (#602)
|
||||
p 'http://host:65535' 'name=http://host port=65535 host=host kind=http'
|
||||
p 'http://host:1' 'name=http://host port=1 host=host kind=http'
|
||||
p 'http://host:65536' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:2147483648' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:4294967296' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:99999999999999' 'name=http://host port=8080 host=host kind=http'
|
||||
# discriminating: this one wrapped to a plausible 80, so range-checking the
|
||||
# sscanf result instead of rejecting the overflow still passes everything above
|
||||
p 'http://host:4294967376' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:999999999999999999999999999999' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:0' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:-1' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:80x' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host: 80' 'name=http://host port=8080 host=host kind=http'
|
||||
p 'http://host:' 'name=http://host port=8080 host=host kind=http'
|
||||
# the fallback is the scheme's own default, not a hardcoded 8080
|
||||
p 'socks5://host:99999999999999' 'name=socks5://host port=1080 host=host kind=socks'
|
||||
p ':' 'name= port=8080 host='
|
||||
|
||||
@@ -1,93 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# A crawl through a proxy must never resolve or dial the origin itself: the proxy
|
||||
# does that. A dead proxy + a multi-address origin used to fall back to dialing
|
||||
# the origin direct (bypassing the proxy, leaking its DNS and IP). The decoy
|
||||
# origin here must therefore receive nothing.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$top_srcdir/tests/testlib.sh"
|
||||
|
||||
if test "${V6_SUPPORT:-}" == "no"; then
|
||||
echo "no IPv6 support (resolver override compiled out), skipping"
|
||||
exit 77
|
||||
fi
|
||||
python=$(find_python) || {
|
||||
echo "python3 missing, skipping"
|
||||
exit 77
|
||||
}
|
||||
|
||||
server=$(nativepath "$top_srcdir/tests/local-server.py")
|
||||
root=$(nativepath "$top_srcdir/tests/server-root")
|
||||
tmpdir=$(mktemp -d)
|
||||
serverpid=
|
||||
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
return 0
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# decoy origin: it must stay silent. LOCAL_SERVER_VERBOSE logs any request it gets.
|
||||
LOCAL_SERVER_VERBOSE=1 "$python" "$server" --root "$root" --bind 127.0.0.1 \
|
||||
>"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$tmpdir/srv.out" 2>/dev/null || true)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$tmpdir/srv.err")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# a proxy port nothing listens on: grab a free one and let it close (refuses).
|
||||
deadproxy=$("$python" -c \
|
||||
'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()')
|
||||
|
||||
# origin resolves to two addresses so a fallback would have a second to dial;
|
||||
# 127.0.0.1 (the decoy) is the one the old bypass reached.
|
||||
out="$tmpdir/crawl"
|
||||
HTTRACK_DEBUG_RESOLVE="decoyhost:127.0.0.2,127.0.0.1" \
|
||||
httrack "http://decoyhost:$port/simple/basic.html" -O "$out" \
|
||||
-P "127.0.0.1:$deadproxy" -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z \
|
||||
>"$tmpdir/log" 2>&1
|
||||
|
||||
log="$out/hts-log.txt"
|
||||
|
||||
# the origin must have seen no connection (no leak/bypass)
|
||||
if grep -qE '"(GET|POST|HEAD|CONNECT) ' "$tmpdir/srv.err"; then
|
||||
echo "FAIL: origin was contacted directly, bypassing the proxy"
|
||||
cat "$tmpdir/srv.err"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# the crawl must fail at the proxy (proves it really tried the proxy, not a no-op)
|
||||
grep -q 'Connect Error' "$log" || {
|
||||
echo "FAIL: expected a proxy connect error"
|
||||
cat "$log"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# and it must not have tried an origin-address fallback under the proxy
|
||||
if grep -q "trying next address" "$log"; then
|
||||
echo "FAIL: fell back to an origin address under a proxy"
|
||||
cat "$log"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "OK: a dead proxy fails cleanly without dialing the origin"
|
||||
@@ -1,168 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Issue #564: plain http through a CONNECT-only proxy (tor HTTPTunnelPort). With
|
||||
# a "connect://" proxy httrack must open a CONNECT tunnel to the origin and send
|
||||
# the request origin-form, like https already does, instead of the absolute-URI
|
||||
# form a CONNECT-only proxy rejects. Fully local: a plain-http origin behind a
|
||||
# proxy that answers CONNECT and 501s everything else.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$top_srcdir/tests/testlib.sh"
|
||||
|
||||
python=$(find_python) || python=
|
||||
if test -z "$python"; then
|
||||
echo "python3 missing, skipping"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
server=$(nativepath "$top_srcdir/tests/proxy-connect-server.py")
|
||||
tmpdir=$(mktemp -d)
|
||||
pids=
|
||||
|
||||
cleanup() {
|
||||
for pid in $pids; do
|
||||
stop_server "$pid"
|
||||
done
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# start_server <logdir> <mode>: launches a proxy+origin pair, sets $origin_port
|
||||
# and $proxy_port from its announced ephemeral ports.
|
||||
start_server() {
|
||||
local dir="$1" mode="$2" ports
|
||||
mkdir -p "$dir"
|
||||
ports="$dir/ports.txt"
|
||||
dir_native=$(nativepath "$dir")
|
||||
: >"$ports"
|
||||
"$python" "$server" "$dir_native" "$mode" \
|
||||
>"$ports" 2>"$dir/server.err" &
|
||||
pids="$pids $!"
|
||||
for _ in $(seq 1 100); do
|
||||
grep -q "^ready" "$ports" 2>/dev/null && break
|
||||
sleep 0.1
|
||||
done
|
||||
grep -q "^ready" "$ports" 2>/dev/null || {
|
||||
echo "server ($mode) did not start" >&2
|
||||
cat "$dir/server.err" >&2
|
||||
exit 1
|
||||
}
|
||||
origin_port=$(awk '/^ORIGIN/{print $2}' "$ports")
|
||||
proxy_port=$(awk '/^PROXY/{print $2}' "$ports")
|
||||
}
|
||||
|
||||
# Kill httrack after a deadline so a hang surfaces as $HANG_RC, not a stalled job
|
||||
# (a portable stand-in for `timeout`, which macOS lacks).
|
||||
HANG_RC=137 # 128 + SIGKILL
|
||||
run_crawl() {
|
||||
local out="$1" proxy="$2" url="${4:-http://127.0.0.1:${3}/}"
|
||||
rm -rf "$out"
|
||||
httrack "$url" --proxy "$proxy" \
|
||||
-O "$out" -r1 -s0 --timeout=10 >"$out.log" 2>&1 &
|
||||
local pid=$!
|
||||
(sleep 60 && kill -9 "$pid" 2>/dev/null) &
|
||||
local guard=$!
|
||||
local rc=0
|
||||
wait "$pid" 2>/dev/null || rc=$?
|
||||
kill "$guard" 2>/dev/null || true
|
||||
wait "$guard" 2>/dev/null || true
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# --- working proxy ----------------------------------------------------------
|
||||
ok="$tmpdir/ok"
|
||||
start_server "$ok" ok
|
||||
|
||||
# 1. page fetched through a CONNECT tunnel, request delivered origin-form
|
||||
run_crawl "$ok/out" "connect://127.0.0.1:${proxy_port}" "$origin_port"
|
||||
grep -rq "ORIGIN-PAGE-564" "$ok/out" || {
|
||||
echo "FAIL: origin page not downloaded through the CONNECT tunnel" >&2
|
||||
cat "$ok/out.log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -q "^CONNECT 127.0.0.1:${origin_port} " "$ok/proxy.log" || {
|
||||
echo "FAIL: proxy never received a CONNECT for the plain-http origin" >&2
|
||||
cat "$ok/proxy.log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -q "^GET / HTTP/1" "$ok/origin-headers.log" || {
|
||||
echo "FAIL: origin did not see an origin-form request line" >&2
|
||||
cat "$ok/origin-headers.log" >&2
|
||||
exit 1
|
||||
}
|
||||
if grep -q "^GET http://" "$ok/origin-headers.log"; then
|
||||
echo "FAIL: absolute-URI request leaked through the tunnel to the origin" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: plain http tunneled origin-form through CONNECT"
|
||||
|
||||
# 2. default CONNECT port follows the origin scheme: a portless http origin
|
||||
# tunnels to :80 (not the https :443). Nothing serves :80 here so the fetch
|
||||
# fails; only the CONNECT target port is asserted. Reverting to a hardcoded 443
|
||||
# makes this log ":443" and fail.
|
||||
: >"$ok/proxy.log"
|
||||
run_crawl "$ok/out_port" "connect://127.0.0.1:${proxy_port}" "" "http://127.0.0.1/" || true
|
||||
grep -q "^CONNECT 127.0.0.1:80 " "$ok/proxy.log" || {
|
||||
echo "FAIL: portless http origin did not CONNECT to the default port 80" >&2
|
||||
cat "$ok/proxy.log" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK: portless http origin tunnels to the default port 80"
|
||||
|
||||
# 3. teeth: without connect://, httrack sends the absolute-URI form, which the
|
||||
# CONNECT-only proxy rejects (501) -> nothing fetched. Proves the scheme is what
|
||||
# flips the behavior and that the proxy really is CONNECT-only.
|
||||
: >"$ok/proxy.log"
|
||||
: >"$ok/origin-headers.log"
|
||||
run_crawl "$ok/out_plain" "127.0.0.1:${proxy_port}" "$origin_port" || true
|
||||
grep -rq "ORIGIN-PAGE-564" "$ok/out_plain" && {
|
||||
echo "FAIL: absolute-URI form unexpectedly fetched through a CONNECT-only proxy" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -q "^GET http://127.0.0.1:${origin_port}/" "$ok/proxy.log" || {
|
||||
echo "FAIL: a plain proxy should have received an absolute-URI GET" >&2
|
||||
cat "$ok/proxy.log" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK: absolute-URI form (no connect://) rejected by the CONNECT-only proxy"
|
||||
|
||||
# 4. authenticated proxy: creds ride the CONNECT, never reach the origin
|
||||
: >"$ok/proxy.log"
|
||||
: >"$ok/origin-headers.log"
|
||||
run_crawl "$ok/out2" "connect://user:secret@127.0.0.1:${proxy_port}" "$origin_port"
|
||||
grep -rq "ORIGIN-PAGE-564" "$ok/out2" || {
|
||||
echo "FAIL: origin page not downloaded through the authenticated proxy" >&2
|
||||
exit 1
|
||||
}
|
||||
got=$(awk '/^AUTH Basic /{print $3}' "$ok/proxy.log" | head -1)
|
||||
# base64("user:secret"); compared literally to stay portable (no base64 -d)
|
||||
test "$got" == "dXNlcjpzZWNyZXQ=" || {
|
||||
echo "FAIL: Proxy-Authorization not carried on CONNECT (got '$got')" >&2
|
||||
cat "$ok/proxy.log" >&2
|
||||
exit 1
|
||||
}
|
||||
if grep -qi "proxy-authorization" "$ok/origin-headers.log"; then
|
||||
echo "FAIL: proxy credentials leaked to the origin through the tunnel" >&2
|
||||
exit 1
|
||||
fi
|
||||
echo "OK: proxy credentials carried on CONNECT, not leaked to origin"
|
||||
|
||||
# --- hostile proxy ----------------------------------------------------------
|
||||
# A proxy that answers 200 then streams headers forever must not hang the crawl.
|
||||
flood="$tmpdir/flood"
|
||||
start_server "$flood" flood
|
||||
rc=0
|
||||
run_crawl "$flood/out" "connect://127.0.0.1:${proxy_port}" "$origin_port" || rc=$?
|
||||
test "$rc" -ne "$HANG_RC" || {
|
||||
echo "FAIL: crawl hung on a flooding proxy (bounded read missing)" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -rq "ORIGIN-PAGE-564" "$flood/out" 2>/dev/null && {
|
||||
echo "FAIL: flooding proxy unexpectedly served the page" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK: bounded proxy response, no hang on a flooding proxy"
|
||||
@@ -1,73 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Unit-tests the suite watchdog (run_with_timeout/kill_tree in testlib.sh): a
|
||||
# hung child must be killed tree-and-all and reported 124, never left to orphan
|
||||
# a spinning httrack.exe. Regression guard for the Windows x64 hangs that read as
|
||||
# "the hosted runner lost communication with the server".
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_wd.XXXXXX")
|
||||
trap 'rm -rf "$tmp"' EXIT
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
# Exit status passes through when the command finishes in time.
|
||||
rc=0 && run_with_timeout 30 sh -c 'exit 0' || rc=$?
|
||||
test "$rc" -eq 0 || fail "clean exit reported $rc"
|
||||
rc=0 && run_with_timeout 30 sh -c 'exit 7' || rc=$?
|
||||
test "$rc" -eq 7 || fail "exit 7 reported $rc"
|
||||
|
||||
# An overrunning command is reported 124, near the deadline not the child's end.
|
||||
start=$SECONDS
|
||||
rc=0
|
||||
if is_windows; then
|
||||
run_with_timeout 2 ping -n 30 127.0.0.1 || rc=$?
|
||||
else
|
||||
run_with_timeout 2 sleep 30 || rc=$?
|
||||
fi
|
||||
test "$rc" -eq 124 || fail "hang reported $rc, want 124"
|
||||
test "$((SECONDS - start))" -lt 15 || fail "watchdog fired late"
|
||||
|
||||
# The native descendant tree is really dead, not orphaned. On Windows an outer
|
||||
# MSYS bash with a native ping grandchild is the exact case signals can't reap;
|
||||
# on POSIX a surviving grandchild would touch the marker after we stop waiting.
|
||||
rc=0
|
||||
if is_windows; then
|
||||
# Existence by exact Windows PID, not a global ping.exe count: the timing
|
||||
# sub-test above leaves a still-dying ping that a count would race. Plain
|
||||
# tasklist, no switches (the workflow's MSYS2_ARG_CONV_EXCL='*' mangles a
|
||||
# //FI filter arg into a silent no-match); $2 is the PID column.
|
||||
alive() { tasklist 2>/dev/null | awk -v p="$1" '$2 == p {f = 1} END {exit !f}'; }
|
||||
# The grandchild ping records its own Windows PID: non-empty proves it ran
|
||||
# (positive control, so the reap check can't pass vacuously), then it must go.
|
||||
gw="$tmp/gcwinpid"
|
||||
# `wait` keeps the bash alive on the ping past the deadline; without it the
|
||||
# bash exits at once and run_with_timeout returns 0 instead of timing out.
|
||||
# shellcheck disable=SC2016 # $1 expands in the bash -c child, by design
|
||||
run_with_timeout 2 bash -c 'ping -n 30 127.0.0.1 >/dev/null 2>&1 & cat /proc/$!/winpid >"$1"; wait' _ "$gw" || rc=$?
|
||||
test "$rc" -eq 124 || fail "grandchild hang reported $rc"
|
||||
w=$(cat "$gw" 2>/dev/null)
|
||||
test -n "$w" || fail "positive control: grandchild ping never started"
|
||||
sleep 1
|
||||
! alive "$w" || fail "native ping grandchild (pid $w) survived the watchdog"
|
||||
else
|
||||
started="$tmp/started" marker="$tmp/alive"
|
||||
# The grandchild (backgrounded subshell) marks that it ran, then would touch
|
||||
# the marker after the deadline; killing only the direct child orphans it, so
|
||||
# the marker still appears. started present + marker absent = ran and reaped.
|
||||
# shellcheck disable=SC2016 # $1/$2 expand in the sh -c child, by design
|
||||
run_with_timeout 2 sh -c '{ : >"$1"; sleep 10; : >"$2"; } & wait' _ "$started" "$marker" || rc=$?
|
||||
test "$rc" -eq 124 || fail "grandchild hang reported $rc"
|
||||
test -e "$started" || fail "positive control: grandchild never ran"
|
||||
sleep 12
|
||||
test ! -e "$marker" || fail "watchdog left a grandchild running"
|
||||
fi
|
||||
|
||||
echo "watchdog OK"
|
||||
@@ -2,8 +2,7 @@
|
||||
# explicitly: automake does not expand wildcards in EXTRA_DIST, so a glob would
|
||||
# silently drop it from the dist tarball and break "make distcheck".
|
||||
EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
|
||||
proxy-https-server.py socks5-server.py proxy-connect-server.py \
|
||||
proxytestlib.py \
|
||||
proxy-https-server.py socks5-server.py \
|
||||
local-crawl.sh local-server.py testlib.sh server.crt server.key \
|
||||
server-root/simple/basic.html server-root/simple/link.html \
|
||||
server-root/stripquery/index.html server-root/stripquery/a.html \
|
||||
@@ -133,9 +132,6 @@ TESTS = \
|
||||
52_local-socks5.test \
|
||||
53_local-proxytrack-cache-corrupt.test \
|
||||
54_local-update-truncate-purge.test \
|
||||
55_local-chunked.test \
|
||||
56_local-proxy-noleak.test \
|
||||
57_local-proxy-connect.test \
|
||||
58_watchdog.test
|
||||
55_local-chunked.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -1,37 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local CONNECT-only proxy + plain-HTTP origin for the issue #564 test.
|
||||
|
||||
Models a tor HTTPTunnelPort: an HTTP proxy that honours CONNECT and rejects
|
||||
every other method with 501 (the classic absolute-URI form a normal HTTP proxy
|
||||
accepts). Fronting it is a plain-HTTP origin. The proxy logs each request line
|
||||
(and any Proxy-Authorization); the origin logs its request line and headers.
|
||||
That lets the test assert a plain-http crawl tunneled through CONNECT, arrived
|
||||
origin-form (not "GET http://host/..."), and never leaked proxy credentials.
|
||||
|
||||
Proxy modes (argv[2], default "ok"):
|
||||
ok - honour CONNECT and tunnel to the origin
|
||||
flood - answer 200 then stream headers forever with no blank line, to exercise
|
||||
the client's bound on the proxy response (must not hang the crawl)
|
||||
|
||||
Usage: proxy-connect-server.py <logdir> [mode]
|
||||
Prints "ORIGIN <port>", "PROXY <port>", then "ready" (one per line) on stdout.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
# python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import proxytestlib # noqa: E402
|
||||
|
||||
ORIGIN_BODY = b"<html><body>ORIGIN-PAGE-564</body></html>"
|
||||
|
||||
|
||||
def main():
|
||||
logdir = sys.argv[1]
|
||||
mode = sys.argv[2] if len(sys.argv) > 2 else "ok"
|
||||
proxytestlib.serve(logdir, ORIGIN_BODY, 80, mode)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -15,21 +15,138 @@ Proxy modes (argv[3], default "ok"):
|
||||
Usage: proxy-https-server.py <cert.pem> <logdir> [mode]
|
||||
Prints "ORIGIN <port>", "PROXY <port>", then "ready" (one per line) on stdout.
|
||||
"""
|
||||
import http.server
|
||||
import os
|
||||
import socket
|
||||
import socketserver
|
||||
import ssl
|
||||
import sys
|
||||
|
||||
# python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
import proxytestlib # noqa: E402
|
||||
import threading
|
||||
|
||||
ORIGIN_BODY = b"<html><body>ORIGIN-PAGE-85</body></html>"
|
||||
PROXY_LOG = "proxy.log"
|
||||
ORIGIN_LOG = "origin-headers.log"
|
||||
|
||||
|
||||
def make_origin(logdir):
|
||||
class Origin(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle:
|
||||
for key in self.headers.keys():
|
||||
handle.write(key + "\n")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.send_header("Content-Length", str(len(ORIGIN_BODY)))
|
||||
self.end_headers()
|
||||
self.wfile.write(ORIGIN_BODY)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
return Origin
|
||||
|
||||
|
||||
def start_origin(certfile, logdir):
|
||||
httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir))
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
ctx.load_cert_chain(certfile)
|
||||
httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
|
||||
port = httpd.socket.getsockname()[1]
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
return port
|
||||
|
||||
|
||||
def pipe(src, dst):
|
||||
try:
|
||||
while True:
|
||||
data = src.recv(65536)
|
||||
if not data:
|
||||
break
|
||||
dst.sendall(data)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
for sock in (src, dst):
|
||||
try:
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def handle_client(conn, logdir, mode):
|
||||
rfile = conn.makefile("rb")
|
||||
request_line = rfile.readline().decode("latin-1").strip()
|
||||
auth = None
|
||||
while True:
|
||||
line = rfile.readline().decode("latin-1")
|
||||
if line in ("\r\n", "\n", ""):
|
||||
break
|
||||
key, _, value = line.partition(":")
|
||||
if key.strip().lower() == "proxy-authorization":
|
||||
auth = value.strip()
|
||||
with open(os.path.join(logdir, PROXY_LOG), "a") as handle:
|
||||
handle.write(request_line + "\n")
|
||||
if auth is not None:
|
||||
handle.write("AUTH " + auth + "\n")
|
||||
parts = request_line.split()
|
||||
if not (len(parts) >= 2 and parts[0] == "CONNECT"):
|
||||
conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n")
|
||||
conn.close()
|
||||
return
|
||||
if mode == "flood":
|
||||
# 200, then an endless header stream with no terminating blank line: the
|
||||
# client must bound this and give up, not hang.
|
||||
try:
|
||||
conn.sendall(b"HTTP/1.0 200 Connection established\r\n")
|
||||
while True:
|
||||
conn.sendall(b"X-Pad: 0123456789\r\n")
|
||||
except OSError:
|
||||
pass
|
||||
conn.close()
|
||||
return
|
||||
host, _, port = parts[1].partition(":")
|
||||
try:
|
||||
upstream = socket.create_connection((host, int(port or 443)))
|
||||
except OSError:
|
||||
conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n")
|
||||
conn.close()
|
||||
return
|
||||
conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
|
||||
threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start()
|
||||
pipe(upstream, conn)
|
||||
|
||||
|
||||
def start_proxy(logdir, mode):
|
||||
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
srv.bind(("127.0.0.1", 0))
|
||||
srv.listen(16)
|
||||
port = srv.getsockname()[1]
|
||||
|
||||
def serve():
|
||||
while True:
|
||||
conn, _ = srv.accept()
|
||||
threading.Thread(
|
||||
target=handle_client, args=(conn, logdir, mode), daemon=True
|
||||
).start()
|
||||
|
||||
threading.Thread(target=serve, daemon=True).start()
|
||||
return port
|
||||
|
||||
|
||||
def main():
|
||||
certfile, logdir = sys.argv[1], sys.argv[2]
|
||||
mode = sys.argv[3] if len(sys.argv) > 3 else "ok"
|
||||
proxytestlib.serve(logdir, ORIGIN_BODY, 443, mode, certfile=certfile)
|
||||
for name in (PROXY_LOG, ORIGIN_LOG):
|
||||
open(os.path.join(logdir, name), "w").close()
|
||||
origin_port = start_origin(certfile, logdir)
|
||||
proxy_port = start_proxy(logdir, mode)
|
||||
# Keep the port lines the caller parses LF: Windows would emit \r\n.
|
||||
sys.stdout.reconfigure(newline="\n")
|
||||
print("ORIGIN %d" % origin_port, flush=True)
|
||||
print("PROXY %d" % proxy_port, flush=True)
|
||||
print("ready", flush=True)
|
||||
threading.Event().wait()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -1,146 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shared helpers for the local proxy test servers.
|
||||
|
||||
A CONNECT proxy in front of an origin, as used by proxy-https-server.py (TLS
|
||||
origin, #85) and proxy-connect-server.py (plain origin, #564). socks5-server.py
|
||||
reuses the relay only; its origin is specialised for keep-alive reuse.
|
||||
Importable because Python puts the running script's directory on sys.path.
|
||||
"""
|
||||
import http.server
|
||||
import os
|
||||
import socket
|
||||
import socketserver
|
||||
import ssl
|
||||
import sys
|
||||
import threading
|
||||
|
||||
PROXY_LOG = "proxy.log"
|
||||
ORIGIN_LOG = "origin-headers.log"
|
||||
|
||||
|
||||
def pipe(src, dst):
|
||||
"""Relay bytes one way until EOF, then tear both ends down."""
|
||||
try:
|
||||
while True:
|
||||
data = src.recv(65536)
|
||||
if not data:
|
||||
break
|
||||
dst.sendall(data)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
for sock in (src, dst):
|
||||
try:
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def make_origin(logdir, body):
|
||||
class Origin(http.server.BaseHTTPRequestHandler):
|
||||
def do_GET(self):
|
||||
# the request line proves origin-form vs absolute-URI (#564)
|
||||
with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle:
|
||||
handle.write(self.requestline + "\n")
|
||||
for key in self.headers.keys():
|
||||
handle.write(key + "\n")
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
return Origin
|
||||
|
||||
|
||||
def start_origin(logdir, body, certfile=None):
|
||||
"""Serve body on an ephemeral port, over TLS when certfile is given."""
|
||||
httpd = socketserver.TCPServer(("127.0.0.1", 0), make_origin(logdir, body))
|
||||
if certfile is not None:
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
ctx.load_cert_chain(certfile)
|
||||
httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
|
||||
port = httpd.socket.getsockname()[1]
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
return port
|
||||
|
||||
|
||||
def handle_client(conn, logdir, mode, default_port):
|
||||
rfile = conn.makefile("rb")
|
||||
request_line = rfile.readline().decode("latin-1").strip()
|
||||
auth = None
|
||||
while True:
|
||||
line = rfile.readline().decode("latin-1")
|
||||
if line in ("\r\n", "\n", ""):
|
||||
break
|
||||
key, _, value = line.partition(":")
|
||||
if key.strip().lower() == "proxy-authorization":
|
||||
auth = value.strip()
|
||||
with open(os.path.join(logdir, PROXY_LOG), "a") as handle:
|
||||
handle.write(request_line + "\n")
|
||||
if auth is not None:
|
||||
handle.write("AUTH " + auth + "\n")
|
||||
parts = request_line.split()
|
||||
# CONNECT-only: reject the classic absolute-URI form a normal proxy accepts
|
||||
if not (len(parts) >= 2 and parts[0] == "CONNECT"):
|
||||
conn.sendall(b"HTTP/1.0 501 Not Implemented\r\n\r\n")
|
||||
conn.close()
|
||||
return
|
||||
if mode == "flood":
|
||||
# 200, then endless headers with no blank line: the client must not hang
|
||||
try:
|
||||
conn.sendall(b"HTTP/1.0 200 Connection established\r\n")
|
||||
while True:
|
||||
conn.sendall(b"X-Pad: 0123456789\r\n")
|
||||
except OSError:
|
||||
pass
|
||||
conn.close()
|
||||
return
|
||||
host, _, port = parts[1].partition(":")
|
||||
try:
|
||||
# default_port only backstops a portless CONNECT; httrack sends host:port
|
||||
upstream = socket.create_connection((host, int(port or default_port)))
|
||||
except OSError:
|
||||
conn.sendall(b"HTTP/1.0 502 Bad Gateway\r\n\r\n")
|
||||
conn.close()
|
||||
return
|
||||
conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
|
||||
threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start()
|
||||
pipe(upstream, conn)
|
||||
|
||||
|
||||
def start_proxy(logdir, mode, default_port):
|
||||
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
srv.bind(("127.0.0.1", 0))
|
||||
srv.listen(16)
|
||||
port = srv.getsockname()[1]
|
||||
|
||||
def accept_loop():
|
||||
while True:
|
||||
conn, _ = srv.accept()
|
||||
threading.Thread(
|
||||
target=handle_client,
|
||||
args=(conn, logdir, mode, default_port),
|
||||
daemon=True,
|
||||
).start()
|
||||
|
||||
threading.Thread(target=accept_loop, daemon=True).start()
|
||||
return port
|
||||
|
||||
|
||||
def serve(logdir, origin_body, default_port, mode="ok", certfile=None):
|
||||
"""Start the origin+proxy pair, announce both ports, then block forever."""
|
||||
for name in (PROXY_LOG, ORIGIN_LOG):
|
||||
open(os.path.join(logdir, name), "w").close()
|
||||
origin_port = start_origin(logdir, origin_body, certfile)
|
||||
proxy_port = start_proxy(logdir, mode, default_port)
|
||||
# Keep the port lines the caller parses LF: Windows would emit \r\n.
|
||||
sys.stdout.reconfigure(newline="\n")
|
||||
print("ORIGIN %d" % origin_port, flush=True)
|
||||
print("PROXY %d" % proxy_port, flush=True)
|
||||
print("ready", flush=True)
|
||||
threading.Event().wait()
|
||||
@@ -16,11 +16,6 @@ import struct
|
||||
import sys
|
||||
import threading
|
||||
|
||||
# python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
|
||||
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
from proxytestlib import pipe # noqa: E402
|
||||
|
||||
# The one name the proxy answers for; a .invalid TLD never resolves (RFC 6761),
|
||||
# so a locally-resolving client could not reach us -- success proves remote DNS.
|
||||
REMOTE_HOST = b"socks-origin.invalid"
|
||||
@@ -90,6 +85,23 @@ def recvn(conn, n):
|
||||
return buf
|
||||
|
||||
|
||||
def pipe(src, dst):
|
||||
try:
|
||||
while True:
|
||||
data = src.recv(65536)
|
||||
if not data:
|
||||
break
|
||||
dst.sendall(data)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
for sock in (src, dst):
|
||||
try:
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def log(logdir, line):
|
||||
with open(os.path.join(logdir, SOCKS_LOG), "a") as handle:
|
||||
handle.write(line + "\n")
|
||||
|
||||
@@ -43,50 +43,3 @@ stop_server() {
|
||||
wait "$1" 2>/dev/null || true
|
||||
return 0
|
||||
}
|
||||
|
||||
# Kill a backgrounded job and its whole descendant tree. POSIX: the caller must
|
||||
# have put the job in its own process group (run_with_timeout does) so we signal
|
||||
# the group; a bare kill would orphan the grandchildren. Windows: the tree is
|
||||
# native processes MSYS can't signal, so taskkill /T ends it by Windows PID.
|
||||
# Single-slash switches: the workflow sets MSYS_NO_PATHCONV/MSYS2_ARG_CONV_EXCL,
|
||||
# so args pass verbatim and a //T would reach taskkill unfolded and be rejected.
|
||||
kill_tree() {
|
||||
local pid=$1
|
||||
if is_windows; then
|
||||
local winpid=
|
||||
test -r "/proc/$pid/winpid" && winpid=$(cat "/proc/$pid/winpid" 2>/dev/null)
|
||||
if test -n "$winpid"; then
|
||||
taskkill /F /T /PID "$winpid" >/dev/null 2>&1 || true
|
||||
else
|
||||
# The offline suite runs serially, so no wanted process races this.
|
||||
taskkill /F /IM httrack.exe >/dev/null 2>&1 || true
|
||||
taskkill /F /IM python.exe >/dev/null 2>&1 || true
|
||||
fi
|
||||
fi
|
||||
kill -9 -"$pid" 2>/dev/null || kill -9 "$pid" 2>/dev/null || true
|
||||
}
|
||||
|
||||
# Run "$@" under a wall-clock deadline of $1 seconds; return its exit status, or
|
||||
# 124 if it overran and was killed. timeout(1) is unusable here: it's absent on
|
||||
# macOS and its signals can't reap httrack.exe on Windows. We poll and kill_tree.
|
||||
run_with_timeout() {
|
||||
local secs=$1
|
||||
shift
|
||||
local had_m=
|
||||
case "$-" in *m*) had_m=1 ;; esac
|
||||
is_windows || set -m # own process group, so kill_tree can signal the group
|
||||
"$@" &
|
||||
local pid=$!
|
||||
test -n "$had_m" || is_windows || set +m
|
||||
local waited=0
|
||||
while kill -0 "$pid" 2>/dev/null; do
|
||||
if test "$waited" -ge "$secs"; then
|
||||
kill_tree "$pid"
|
||||
wait "$pid" 2>/dev/null || true
|
||||
return 124
|
||||
fi
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
wait "$pid"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user