mirror of
https://github.com/xroche/httrack.git
synced 2026-07-15 13:21:14 +03:00
Compare commits
10 Commits
fix/proxy-
...
export-con
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
45f56e6194 | ||
|
|
d9ff780d0f | ||
|
|
b026c80bb6 | ||
|
|
e0648beff9 | ||
|
|
ee1aebd935 | ||
|
|
b4957f0625 | ||
|
|
c9d36c82c7 | ||
|
|
1fa0184468 | ||
|
|
79e46eeceb | ||
|
|
436ac53793 |
5
.gitattributes
vendored
5
.gitattributes
vendored
@@ -1,3 +1,8 @@
|
||||
# Resource scripts are Windows tooling input and must stay CRLF: the engine has no
|
||||
# encoding guard, so an autocrlf checkout could otherwise flatten them silently.
|
||||
*.rc text eol=crlf
|
||||
|
||||
# The test scripts run under Git Bash on the Windows CI; a CRLF checkout makes
|
||||
# bash die on $'\r' on every line of them.
|
||||
*.test text eol=lf
|
||||
*.sh text eol=lf
|
||||
|
||||
69
.github/workflows/windows-build.yml
vendored
69
.github/workflows/windows-build.yml
vendored
@@ -119,22 +119,61 @@ jobs:
|
||||
throw "CRT mismatch across the boundary: libhttrack.dll=$a httrack.exe=$b"
|
||||
}
|
||||
|
||||
# The engine self-tests exercise the codecs the DLL was linked with: a
|
||||
# missing brotli/zstd decoder shows up here, not just at link time.
|
||||
- name: Run the content-coding self-tests
|
||||
shell: pwsh
|
||||
# The engine ships ~90 tests and none had ever run on Windows: "make check"
|
||||
# is Linux/macOS only. These are the offline ones, driven from Git Bash
|
||||
# against the native httrack.exe. They subsume the self-tests this step
|
||||
# used to run inline (codecs, cache, fsize).
|
||||
- name: Run the engine test suite (offline tests)
|
||||
shell: bash
|
||||
working-directory: tests
|
||||
run: |
|
||||
$ErrorActionPreference = "Stop"
|
||||
$exe = "src\${{ matrix.platform }}\${{ matrix.configuration }}\httrack.exe"
|
||||
$tmp = New-Item -ItemType Directory -Path (Join-Path $env:RUNNER_TEMP "st")
|
||||
foreach ($t in @("acceptencoding", "contentcodings")) {
|
||||
$out = & $exe -O NUL "-#test=$t" $tmp.FullName run
|
||||
if ($LASTEXITCODE -ne 0) { throw "$t failed" }
|
||||
if (-not ($out | Select-String -SimpleMatch "$t self-test OK")) {
|
||||
throw "$t did not report OK: $out"
|
||||
}
|
||||
Write-Host ($out -join "`n")
|
||||
}
|
||||
set -u
|
||||
bin="$(cygpath -u "$GITHUB_WORKSPACE")/src/${{ matrix.platform }}/${{ matrix.configuration }}"
|
||||
export PATH="$bin:$PATH"
|
||||
command -v httrack >/dev/null || { echo "::error::no httrack.exe in $bin"; exit 1; }
|
||||
|
||||
# httrack.exe is native, so MSYS rewrites any argument shaped like a
|
||||
# POSIX path, and a URL path is shaped exactly like one: "/a/b.html"
|
||||
# reached the engine as "C:/Program Files/Git/a/b.html". Switch that
|
||||
# off, and hand the tests a TMPDIR that is already a Windows path.
|
||||
export MSYS_NO_PATHCONV=1
|
||||
export MSYS2_ARG_CONV_EXCL='*'
|
||||
TMPDIR="$(cygpath -m "$RUNNER_TEMP")"
|
||||
export TMPDIR
|
||||
|
||||
pass=0 fail=0 skip=0 failed=""
|
||||
for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test; do
|
||||
rc=0
|
||||
bash "$t" >"$t.log" 2>&1 || rc=$?
|
||||
case "$rc" in
|
||||
0) pass=$((pass + 1)); echo "PASS $t" ;;
|
||||
77) skip=$((skip + 1)); echo "SKIP $t" ;;
|
||||
*)
|
||||
fail=$((fail + 1)) failed="$failed $t"
|
||||
echo "FAIL $t (exit $rc)"
|
||||
# These assert with `test "$(...)" == "..." || exit 1`, which
|
||||
# says nothing at all on failure. Re-run traced.
|
||||
bash -x "$t" >>"$t.log" 2>&1 || true
|
||||
tail -n 25 "$t.log" | sed 's/^/ /'
|
||||
;;
|
||||
esac
|
||||
done
|
||||
echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
|
||||
tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Every gate in these scripts exits 77, so a suite that degraded to
|
||||
# all-skipped would report green having tested nothing: assert a floor
|
||||
# on what actually ran, not just the absence of failures.
|
||||
[ "$pass" -ge 45 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
- name: Upload the test logs
|
||||
if: always()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: engine-tests-${{ matrix.platform }}-${{ matrix.configuration }}
|
||||
path: tests/*.log
|
||||
if-no-files-found: warn
|
||||
|
||||
- name: Upload MSBuild logs
|
||||
if: always()
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
.\"
|
||||
.\" This file is generated by man/makeman.sh; do not edit by hand.
|
||||
.\" SPDX-License-Identifier: GPL-3.0-or-later
|
||||
.TH httrack 1 "12 July 2026" "httrack website copier"
|
||||
.TH httrack 1 "14 July 2026" "httrack website copier"
|
||||
.SH NAME
|
||||
httrack \- offline browser : copy websites to a local directory
|
||||
.SH SYNOPSIS
|
||||
@@ -135,7 +135,7 @@ continue an interrupted mirror using the cache (\-\-continue)
|
||||
mirror ALL links located in the first level pages (mirror links) (\-\-mirrorlinks)
|
||||
.SS Proxy options:
|
||||
.IP \-P
|
||||
proxy use (\-P proxy:port or \-P user:pass@proxy:port) (\-\-proxy <param>)
|
||||
proxy use (\-P [socks5://][user:pass@]proxy:port) (\-\-proxy <param>)
|
||||
.IP \-%f
|
||||
*use proxy for ftp (f0 don't use) (\-\-httpproxy\-ftp[=N])
|
||||
.IP \-%b
|
||||
|
||||
@@ -61,7 +61,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htshelp.c htslib.c htscoremain.c \
|
||||
htsname.c htsrobots.c htstools.c htswizard.c \
|
||||
htsalias.c htsthread.c htsindex.c htsbauth.c \
|
||||
htsmd5.c htscodec.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmd5.c htscodec.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
|
||||
htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
|
||||
md5.c \
|
||||
minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
|
||||
@@ -72,7 +72,7 @@ libhttrack_la_SOURCES = htscore.c htsparse.c htsback.c htscache.c \
|
||||
htshelp.h htsindex.h htslib.h htsmd5.h \
|
||||
htsmodules.h htsname.h htsnet.h htssniff.h \
|
||||
htsopt.h htsrobots.h htsthread.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htszlib.h \
|
||||
htstools.h htswizard.h htswrap.h htscodec.h htsproxy.h htszlib.h \
|
||||
htsstrings.h htsarrays.h httrack-library.h \
|
||||
htscharset.h punycode.h htsencoding.h \
|
||||
htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \
|
||||
|
||||
@@ -45,6 +45,7 @@ Please visit our Website: http://www.httrack.com
|
||||
|
||||
#include "htsftp.h"
|
||||
#include "htscodec.h"
|
||||
#include "htsproxy.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#else
|
||||
@@ -139,8 +140,9 @@ int back_connect_fallback_due(int addr_index, int addr_count, int elapsed,
|
||||
|
||||
/* Pending-connect result for a non-blocking socket reported ready by select():
|
||||
0 = connected, >0 = the connect errno (refused, unreachable, ...), -1 if the
|
||||
probe itself failed. A failed connect is reported writable too, so this is
|
||||
how success is told from failure without blocking. */
|
||||
probe itself failed. A failed connect is reported ready as well (writable on
|
||||
posix, exception set on winsock), so this is how success is told from failure
|
||||
without blocking. */
|
||||
static int connect_socket_error(T_SOC soc) {
|
||||
int soerr = 0;
|
||||
socklen_t len = (socklen_t) sizeof(soerr);
|
||||
@@ -775,7 +777,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
|
||||
if (back[p].r.statuscode == HTTP_OK) { // OK (ou 304 en backing)
|
||||
if (back[p].r.is_write) { // Written file
|
||||
if (may_be_hypertext_mime(opt, back[p].r.contenttype, back[p].url_fil)) { // to parse!
|
||||
off_t sz;
|
||||
LLint sz;
|
||||
|
||||
sz = fsize_utf8(back[p].url_sav);
|
||||
if (sz > 0) { // ok, exists!
|
||||
@@ -1952,7 +1954,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
/* Not in cache ; maybe in temporary cache ? Warning: non-movable
|
||||
"url_sav" */
|
||||
else if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
|
||||
const off_t file_size = fsize_utf8(itemback->url_sav);
|
||||
const LLint file_size = fsize_utf8(itemback->url_sav);
|
||||
|
||||
/* Found file on disk */
|
||||
if (file_size > 0) {
|
||||
@@ -1987,7 +1989,7 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
}
|
||||
/* Not in cache or temporary cache ; found on disk ? (hack) */
|
||||
else if (fexist_utf8(save)) {
|
||||
const off_t sz = fsize_utf8(save);
|
||||
const LLint sz = fsize_utf8(save);
|
||||
|
||||
// Bon, là il est possible que le fichier ait été partiellement transféré
|
||||
// (s'il l'avait été en totalité il aurait été inscrit dans le cache ET existerait sur disque)
|
||||
@@ -2111,6 +2113,17 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
|
||||
"error: forbidden test with ftp link for back_add");
|
||||
return -1; // erreur pas de test permis
|
||||
}
|
||||
// the ftp client dials the origin itself: over socks that would bypass
|
||||
// the proxy, so fail the link rather than leak the connection (#563)
|
||||
if (back[p].r.req.proxy.active &&
|
||||
hts_proxy_is_socks(back[p].r.req.proxy.name)) {
|
||||
back[p].r.statuscode = STATUSCODE_NON_FATAL;
|
||||
strcpybuff(back[p].r.msg,
|
||||
"ftp:// is not supported over a SOCKS proxy");
|
||||
back[p].status = STATUS_READY;
|
||||
back_set_finished(sback, p);
|
||||
return 0;
|
||||
}
|
||||
if (!(back[p].r.req.proxy.active && opt->ftp_proxy)) { // connexion directe, gérée en thread
|
||||
FTPDownloadStruct *str =
|
||||
(FTPDownloadStruct *) malloc(sizeof(FTPDownloadStruct));
|
||||
@@ -2646,7 +2659,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
for (i_mod = 0; i_mod < (unsigned int) back_max; i_mod++) {
|
||||
unsigned int i = (i_mod + mod_random) % (back_max);
|
||||
|
||||
if (back[i].status > 0) {
|
||||
// winsock flags a failed connect in the exception set only: leave a
|
||||
// connecting slot to the connect handler, which can still fall back
|
||||
if (back[i].status > 0 && back[i].status != STATUS_CONNECTING) {
|
||||
if (!back[i].r.is_file) { // not file..
|
||||
if (back[i].r.soc != INVALID_SOCKET) { // hey, you never know..
|
||||
int err = FD_ISSET(back[i].r.soc, &fds_e);
|
||||
@@ -2660,10 +2675,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
}
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
back[i].r.statuscode = STATUSCODE_CONNERROR;
|
||||
if (back[i].status == STATUS_CONNECTING)
|
||||
strcpybuff(back[i].r.msg, "Connect Error");
|
||||
else
|
||||
strcpybuff(back[i].r.msg, "Receive Error");
|
||||
strcpybuff(back[i].r.msg, "Receive Error");
|
||||
if (back[i].status == STATUS_ALIVE) { /* Keep-alive socket */
|
||||
back_delete(opt, cache, sback, i);
|
||||
} else {
|
||||
@@ -2696,10 +2708,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
gestion_timeout = 1;
|
||||
|
||||
// connecté?
|
||||
dispo = FD_ISSET(back[i].r.soc, &fds_c);
|
||||
dispo = back[i].r.soc != INVALID_SOCKET &&
|
||||
(FD_ISSET(back[i].r.soc, &fds_c) ||
|
||||
FD_ISSET(back[i].r.soc, &fds_e));
|
||||
if (dispo) { // socket ready: connect() finished (ok or failed)
|
||||
// a refused/failed connect is reported writable too; probe SO_ERROR
|
||||
// and, on failure, fall back to the next address (or fail the slot)
|
||||
// probe SO_ERROR and, on failure, fall back to the next address
|
||||
if (connect_socket_error(back[i].r.soc) != 0) {
|
||||
if (!back_connect_next(opt, sback, i)) {
|
||||
deletehttp(&back[i].r);
|
||||
@@ -2713,13 +2726,40 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
}
|
||||
busy_state = 1;
|
||||
|
||||
// socks5: tunnel to the origin before anything is written, for http
|
||||
// as well as https. Skip on a reused keep-alive socket (already
|
||||
// tunneled) and on the post-TLS re-entry (ssl_con set) (#563).
|
||||
if (back[i].r.req.proxy.active &&
|
||||
hts_proxy_is_socks(back[i].r.req.proxy.name) &&
|
||||
!back[i].r.keep_alive
|
||||
#if HTS_USEOPENSSL
|
||||
&& back[i].r.ssl_con == NULL
|
||||
#endif
|
||||
) {
|
||||
const int timeout = back[i].timeout > 0 ? back[i].timeout : 30;
|
||||
|
||||
if (!socks5_handshake(opt, &back[i].r, back[i].url_adr, timeout)) {
|
||||
if (!strnotempty(back[i].r.msg))
|
||||
strcpybuff(back[i].r.msg, "SOCKS5 handshake failed");
|
||||
deletehttp(&back[i].r);
|
||||
back[i].r.soc = INVALID_SOCKET;
|
||||
back[i].r.statuscode = STATUSCODE_NON_FATAL;
|
||||
back[i].status = STATUS_READY;
|
||||
back_set_finished(sback, i);
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
#if HTS_USEOPENSSL
|
||||
/* SSL mode */
|
||||
if (back[i].r.ssl) {
|
||||
int tunnel_ok = 1;
|
||||
|
||||
// https via proxy: CONNECT-tunnel before TLS (#85)
|
||||
if (back[i].r.req.proxy.active && back[i].r.ssl_con == NULL) {
|
||||
// https via an http proxy: CONNECT-tunnel before TLS (#85); socks
|
||||
// already carries the origin connection
|
||||
if (back[i].r.req.proxy.active &&
|
||||
!hts_proxy_is_socks(back[i].r.req.proxy.name) &&
|
||||
back[i].r.ssl_con == NULL) {
|
||||
const int timeout = back[i].timeout > 0 ? back[i].timeout : 30;
|
||||
|
||||
tunnel_ok =
|
||||
@@ -3656,7 +3696,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) { // 'OK'
|
||||
if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) { // not HTML
|
||||
if (strnotempty(back[i].url_sav)) { // target found
|
||||
off_t size = fsize_utf8(back[i].url_sav);
|
||||
LLint size = fsize_utf8(back[i].url_sav);
|
||||
|
||||
if (size >= 0) {
|
||||
if (back[i].r.totalsize == size) { // same size!
|
||||
@@ -3875,7 +3915,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
// traiter 206 (partial content)
|
||||
// xxc SI CHUNK VERIFIER QUE CA MARCHE??
|
||||
if (back[i].r.statuscode == 206) { // on nous envoie un morceau (la fin) coz une partie sur disque!
|
||||
off_t sz = fsize_utf8(back[i].url_sav);
|
||||
LLint sz = fsize_utf8(back[i].url_sav);
|
||||
/* RFC 7233: resume at the server's Content-Range start,
|
||||
not the offset we requested; a server may resume
|
||||
earlier and appending the overlap duplicates bytes
|
||||
|
||||
@@ -356,7 +356,7 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
|
||||
FILE *fp;
|
||||
|
||||
// On recopie le fichier->.
|
||||
off_t file_size = fsize_utf8(fconv(catbuff, sizeof(catbuff), url_save));
|
||||
LLint file_size = fsize_utf8(fconv(catbuff, sizeof(catbuff), url_save));
|
||||
|
||||
if (file_size >= 0) {
|
||||
fp = FOPEN(fconv(catbuff, sizeof(catbuff), url_save), "rb");
|
||||
@@ -1241,23 +1241,27 @@ char *readfile(const char *fil) {
|
||||
char *readfile2(const char *fil, LLint * size) {
|
||||
char *adr = NULL;
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
INTsys len = 0;
|
||||
const LLint len = fsize(fil);
|
||||
|
||||
len = fsize(fil);
|
||||
if (len >= 0) { // exists
|
||||
/* a size too large for size_t (32-bit Windows/i386) must fail closed: it
|
||||
would wrap malloct() short while fread() still read the untruncated len */
|
||||
const size_t buflen = len >= 0 ? llint_to_size_t(len) : (size_t) -1;
|
||||
|
||||
if (buflen != (size_t) -1) { // exists, and is addressable
|
||||
FILE *fp;
|
||||
|
||||
fp = fopen(fconv(catbuff, sizeof(catbuff), fil), "rb");
|
||||
if (fp != NULL) { // n'existe pas (!)
|
||||
adr = (char *) malloct(len + 1);
|
||||
adr = (char *) malloct(buflen + 1);
|
||||
if (size != NULL)
|
||||
*size = len;
|
||||
if (adr != NULL) {
|
||||
if (len > 0 && fread(adr, 1, len, fp) != len) { // fichier endommagé ?
|
||||
if (buflen > 0 &&
|
||||
fread(adr, 1, buflen, fp) != buflen) { // fichier endommagé ?
|
||||
freet(adr);
|
||||
adr = NULL;
|
||||
} else
|
||||
*(adr + len) = '\0';
|
||||
*(adr + buflen) = '\0';
|
||||
}
|
||||
fclose(fp);
|
||||
}
|
||||
@@ -1269,19 +1273,21 @@ char *readfile2(const char *fil, LLint * size) {
|
||||
char *readfile_utf8(const char *fil) {
|
||||
char *adr = NULL;
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
const off_t len = fsize_utf8(fil);
|
||||
const LLint len = fsize_utf8(fil);
|
||||
const size_t buflen = len >= 0 ? llint_to_size_t(len) : (size_t) -1;
|
||||
|
||||
if (len >= 0) { // exists
|
||||
if (buflen != (size_t) -1) { // exists, and is addressable (see readfile2)
|
||||
FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), fil), "rb");
|
||||
|
||||
if (fp != NULL) { // n'existe pas (!)
|
||||
adr = (char *) malloct(len + 1);
|
||||
adr = (char *) malloct(buflen + 1);
|
||||
if (adr != NULL) {
|
||||
if (len > 0 && fread(adr, 1, len, fp) != len) { // fichier endommagé ?
|
||||
if (buflen > 0 &&
|
||||
fread(adr, 1, buflen, fp) != buflen) { // fichier endommagé ?
|
||||
freet(adr);
|
||||
adr = NULL;
|
||||
} else {
|
||||
adr[len] = '\0';
|
||||
adr[buflen] = '\0';
|
||||
}
|
||||
}
|
||||
fclose(fp);
|
||||
|
||||
@@ -232,11 +232,10 @@ static int disk_fallback_selftest(httrackp *opt) {
|
||||
static const char body[] = "BINARY-on-disk-body-0123456789-no-trailing-nul";
|
||||
const size_t body_len = sizeof(body) - 1;
|
||||
|
||||
/* X-Save must start with path_html_utf8 so the reader resolves it verbatim
|
||||
(otherwise it re-roots it as a pre-3.40 relative path); then the body we
|
||||
create at fconv(save) is exactly where cache_readex looks for it. */
|
||||
fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
|
||||
"example.com/blob.bin");
|
||||
/* a DOS-ified X-Save loses the path_html_utf8 prefix the reader matches on,
|
||||
and gets re-rooted as a pre-3.40 relative path; fconv() only on access */
|
||||
concat(save, sizeof(save), StringBuff(opt->path_html_utf8),
|
||||
"example.com/blob.bin");
|
||||
|
||||
/* write only the header (X-In-Cache: 0); the body stays on disk */
|
||||
selftest_open_for_write(&cache, opt);
|
||||
@@ -939,9 +938,9 @@ static void reconcile_put(httrackp *opt, const char *name, size_t size) {
|
||||
}
|
||||
|
||||
/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
|
||||
static int reconcile_expect(httrackp *opt, const char *name, off_t size,
|
||||
static int reconcile_expect(httrackp *opt, const char *name, LLint size,
|
||||
const char *what) {
|
||||
const off_t got = fsize(reconcile_st_path(opt, name));
|
||||
const LLint got = fsize(reconcile_st_path(opt, name));
|
||||
|
||||
if (got != size) {
|
||||
fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
|
||||
@@ -955,7 +954,7 @@ int cache_reconcile_selftest(httrackp *opt, const char *dir) {
|
||||
int failures = 0;
|
||||
|
||||
/* around the interrupted-run thresholds (new < 32768, old > 65536) */
|
||||
static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
|
||||
static const LLint TINY = 1024, MID = 40000, SOLID = 131072;
|
||||
|
||||
golden_setup(opt, dir);
|
||||
#ifdef _WIN32
|
||||
@@ -1253,8 +1252,9 @@ static void corrupt_build_disk(httrackp *opt) {
|
||||
memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
|
||||
remove(reconcile_st_path(opt, "hts-cache/new.zip"));
|
||||
remove(reconcile_st_path(opt, "hts-cache/old.zip"));
|
||||
fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
|
||||
CORRUPT_ADR "/victim.bin");
|
||||
/* X-Save stored verbatim: see disk_fallback_selftest */
|
||||
concat(save, sizeof(save), StringBuff(opt->path_html_utf8),
|
||||
CORRUPT_ADR "/victim.bin");
|
||||
selftest_open_for_write(&cache, opt);
|
||||
store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
|
||||
"OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
|
||||
|
||||
@@ -31,6 +31,9 @@ Please visit our Website: http://www.httrack.com
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#include "htscharset.h"
|
||||
#ifdef _WIN32
|
||||
#include <shellapi.h>
|
||||
#endif
|
||||
#include "htsbase.h"
|
||||
#include "punycode.h"
|
||||
#include "htssafe.h"
|
||||
@@ -382,10 +385,44 @@ char *hts_convertStringFromUTF8(const char *s, size_t size, const char *charset)
|
||||
return hts_convertStringCPFromUTF8(s, size, cp);
|
||||
}
|
||||
|
||||
char *hts_convertStringSystemToUTF8(const char *s, size_t size) {
|
||||
HTSEXT_API char *hts_convertStringSystemToUTF8(const char *s, size_t size) {
|
||||
return hts_convertStringCPToUTF8(s, size, GetACP());
|
||||
}
|
||||
|
||||
HTSEXT_API void hts_argv_utf8(int *pargc, char ***pargv) {
|
||||
int wargc = 0;
|
||||
LPWSTR *const wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
|
||||
char **argv;
|
||||
int i;
|
||||
|
||||
// On any failure keep the CRT's ANSI argv: lossy, but never half-converted.
|
||||
if (wargv == NULL)
|
||||
return;
|
||||
if (wargc <= 0 ||
|
||||
(argv = calloct((size_t) wargc + 1, sizeof(char *))) == NULL) {
|
||||
LocalFree(wargv);
|
||||
return;
|
||||
}
|
||||
for (i = 0; i < wargc; i++) {
|
||||
const int wsize = (int) wcslen(wargv[i]);
|
||||
|
||||
// hts_convertUCS2StringToUTF8() returns NULL on an empty string
|
||||
argv[i] =
|
||||
wsize != 0 ? hts_convertUCS2StringToUTF8(wargv[i], wsize) : strdupt("");
|
||||
if (argv[i] == NULL) {
|
||||
while (i-- != 0)
|
||||
freet(argv[i]);
|
||||
freet(argv);
|
||||
LocalFree(wargv);
|
||||
return;
|
||||
}
|
||||
}
|
||||
argv[wargc] = NULL; // callers may rely on argv[argc] == NULL
|
||||
LocalFree(wargv);
|
||||
*pargc = wargc;
|
||||
*pargv = argv; // never freed: argv lives for the process
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
#include <errno.h>
|
||||
|
||||
@@ -34,6 +34,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#define HTS_CHARSET_DEFH
|
||||
|
||||
/** Standard includes. **/
|
||||
#include "htsglobal.h"
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#ifdef _WIN32
|
||||
@@ -174,7 +175,16 @@ extern char *hts_convertUCS2StringToUTF8(LPWSTR woutput, int wsize);
|
||||
* Convert current system codepage to UTF-8.
|
||||
* This function is WIN32 specific.
|
||||
**/
|
||||
extern char *hts_convertStringSystemToUTF8(const char *s, size_t size);
|
||||
HTSEXT_API char *hts_convertStringSystemToUTF8(const char *s, size_t size);
|
||||
|
||||
/**
|
||||
* Replace the CRT's ANSI argv by a UTF-8 one decoded from the real UTF-16
|
||||
* command line: every char* is UTF-8 on Windows (FOPEN, STAT, ... convert at
|
||||
* the syscall boundary). Keeps the CRT's argv on failure; the new array is
|
||||
* writable, NULL-terminated, and lives for the process.
|
||||
* This function is WIN32 specific.
|
||||
**/
|
||||
HTSEXT_API void hts_argv_utf8(int *pargc, char ***pargv);
|
||||
#endif
|
||||
|
||||
#endif
|
||||
|
||||
@@ -855,7 +855,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
char *filelist_buff = NULL;
|
||||
size_t filelist_sz = 0;
|
||||
const char *filelist_err = NULL; /* failure reason, NULL on success */
|
||||
const off_t fs = fsize(StringBuff(opt->filelist));
|
||||
const LLint fs = fsize(StringBuff(opt->filelist));
|
||||
|
||||
if (fs < 0) {
|
||||
/* fsize() hides the cause; redo stat() for a precise errno (#49) */
|
||||
@@ -863,7 +863,7 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
filelist_err = stat(StringBuff(opt->filelist), &st) != 0
|
||||
? strerror(errno)
|
||||
: "not a regular file";
|
||||
} else if ((filelist_sz = off_t_to_size_t(fs)) == (size_t) -1) {
|
||||
} else if ((filelist_sz = llint_to_size_t(fs)) == (size_t) -1) {
|
||||
filelist_err = "file too large";
|
||||
filelist_sz = 0;
|
||||
} else {
|
||||
@@ -2089,10 +2089,9 @@ int httpmirror(char *url1, httrackp * opt) {
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/old.lst"), "rb");
|
||||
if (old_lst) {
|
||||
const size_t sz =
|
||||
off_t_to_size_t(fsize(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
"hts-cache/new.lst")));
|
||||
const size_t sz = llint_to_size_t(
|
||||
fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
|
||||
StringBuff(opt->path_log), "hts-cache/new.lst")));
|
||||
new_lst =
|
||||
fopen(fconcat
|
||||
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
|
||||
|
||||
@@ -1509,7 +1509,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
htsmain_free();
|
||||
return -1;
|
||||
} else {
|
||||
off_t fz;
|
||||
LLint fz;
|
||||
|
||||
na++;
|
||||
fz = fsize(argv[na]);
|
||||
@@ -2584,21 +2584,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
|
||||
hts_get_version_info(opt), t, url);
|
||||
fprintf(opt->log, "(");
|
||||
for(i = 0; i < argc; i++) {
|
||||
#ifdef _WIN32
|
||||
char *carg =
|
||||
hts_convertStringSystemToUTF8(argv[i], (int) strlen(argv[i]));
|
||||
char *arg = carg != NULL ? carg : argv[i];
|
||||
#else
|
||||
const char *arg = argv[i];
|
||||
#endif
|
||||
const char *arg = argv[i]; // already UTF-8 on every platform
|
||||
if (strchr(arg, ' ') == NULL || strchr(arg, '\"') != NULL)
|
||||
fprintf(opt->log, "%s ", arg);
|
||||
else // entre "" (si espace(s) et pas déja de ")
|
||||
fprintf(opt->log, "\"%s\" ", arg);
|
||||
#ifdef _WIN32
|
||||
if (carg != NULL)
|
||||
free(carg);
|
||||
#endif
|
||||
}
|
||||
fprintf(opt->log, ")" LF);
|
||||
fprintf(opt->log, LF);
|
||||
|
||||
@@ -122,8 +122,11 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
|
||||
}
|
||||
|
||||
/* Real filters/URLs fit HTS_URLMAXSIZE*2; past it a hostile pattern recurses
|
||||
O(len) deep or runs O(n^2*stars). Cap length (depth) and steps (work). */
|
||||
O(len) deep or runs O(n^2*stars). Cap length, recursion depth and steps
|
||||
(work): the length cap alone still allows ~2000 frames, ~900KB of stack,
|
||||
which overflows the 1MB a Windows thread gets (#574). */
|
||||
#define STRJOKER_MAXLEN (HTS_URLMAXSIZE * 2)
|
||||
#define STRJOKER_MAXDEPTH 256u
|
||||
#define STRJOKER_MAXSTEPS 2000000u
|
||||
|
||||
/* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
|
||||
@@ -133,20 +136,28 @@ typedef struct strjoker_memo {
|
||||
size_t stride; /* strlen(joker0) + 1 */
|
||||
unsigned char *failed; /* failed-pair bitmap; NULL: no memo */
|
||||
size_t *nsteps; /* shared work counter; NULL: unbounded (oracle) */
|
||||
size_t maxdepth; /* deepest recursion reached */
|
||||
hts_boolean cut; /* a branch hit the depth cap: its failures are not final */
|
||||
} strjoker_memo;
|
||||
|
||||
static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
|
||||
const char *joker, LLint *size,
|
||||
int *size_flag);
|
||||
static const char *strjoker_impl(strjoker_memo *memo, const char *chaine,
|
||||
const char *joker, LLint *size, int *size_flag,
|
||||
size_t depth);
|
||||
|
||||
/* A pair that failed once fails forever (*size is only written on the success
|
||||
path), so record NULL results and cut the re-exploration. */
|
||||
static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
|
||||
const char *joker, LLint *size,
|
||||
int *size_flag) {
|
||||
static const char *strjoker_rec(strjoker_memo *memo, const char *chaine,
|
||||
const char *joker, LLint *size, int *size_flag,
|
||||
size_t depth) {
|
||||
size_t bit = 0;
|
||||
const char *adr;
|
||||
|
||||
if (depth > memo->maxdepth)
|
||||
memo->maxdepth = depth;
|
||||
if (depth >= STRJOKER_MAXDEPTH) {
|
||||
memo->cut = HTS_TRUE;
|
||||
return NULL; /* nesting beyond any real filter: fail the branch safely */
|
||||
}
|
||||
if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
|
||||
return NULL; /* work budget spent: fail the match safely */
|
||||
if (memo->failed) {
|
||||
@@ -155,8 +166,9 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
|
||||
if (memo->failed[bit >> 3] & (unsigned char) (1u << (bit & 7)))
|
||||
return NULL;
|
||||
}
|
||||
adr = strjoker_impl(memo, chaine, joker, size, size_flag);
|
||||
if (adr == NULL && memo->failed)
|
||||
adr = strjoker_impl(memo, chaine, joker, size, size_flag, depth + 1);
|
||||
/* a cut branch may fail here yet match when reached at a shallower depth */
|
||||
if (adr == NULL && memo->failed && !memo->cut)
|
||||
memo->failed[bit >> 3] |= (unsigned char) (1u << (bit & 7));
|
||||
return adr;
|
||||
}
|
||||
@@ -164,13 +176,15 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
|
||||
/* Match chaine against joker with a shared work budget *nsteps (a strjokerfind
|
||||
sweep passes one counter, bounding the whole scan). */
|
||||
static const char *strjoker_bounded(const char *chaine, const char *joker,
|
||||
LLint *size, int *size_flag,
|
||||
size_t *nsteps) {
|
||||
strjoker_memo memo = {chaine, joker, 0, NULL, nsteps};
|
||||
LLint *size, int *size_flag, size_t *nsteps,
|
||||
size_t *maxdepth) {
|
||||
strjoker_memo memo = {chaine, joker, 0, NULL, nsteps, 0, HTS_FALSE};
|
||||
unsigned char stackbits[2048];
|
||||
hts_boolean onheap = HTS_FALSE;
|
||||
const char *adr;
|
||||
|
||||
if (maxdepth != NULL)
|
||||
*maxdepth = 0;
|
||||
if (chaine != NULL && joker != NULL) {
|
||||
const size_t l1 = strlen(chaine), l2 = strlen(joker);
|
||||
|
||||
@@ -189,9 +203,11 @@ static const char *strjoker_bounded(const char *chaine, const char *joker,
|
||||
}
|
||||
}
|
||||
}
|
||||
adr = strjoker_rec(&memo, chaine, joker, size, size_flag);
|
||||
adr = strjoker_rec(&memo, chaine, joker, size, size_flag, 0);
|
||||
if (onheap)
|
||||
freet(memo.failed);
|
||||
if (maxdepth != NULL)
|
||||
*maxdepth = memo.maxdepth;
|
||||
return adr;
|
||||
}
|
||||
|
||||
@@ -202,34 +218,39 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
|
||||
LLint *size, int *size_flag) {
|
||||
size_t nsteps = 0;
|
||||
|
||||
return strjoker_bounded(chaine, joker, size, size_flag, &nsteps);
|
||||
return strjoker_bounded(chaine, joker, size, size_flag, &nsteps, NULL);
|
||||
}
|
||||
|
||||
/* Test-only oracle: the same matcher without the failure memo. */
|
||||
const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
|
||||
int *size_flag) {
|
||||
const strjoker_memo memo = {chaine, joker, 0, NULL};
|
||||
strjoker_memo memo = {chaine, joker, 0, NULL, NULL, 0, HTS_FALSE};
|
||||
|
||||
return strjoker_rec(&memo, chaine, joker, size, size_flag);
|
||||
return strjoker_rec(&memo, chaine, joker, size, size_flag, 0);
|
||||
}
|
||||
|
||||
/* Test-only: strjoker() reporting the work-budget steps spent and the cap, so a
|
||||
self-test can prove the budget bounds a hostile pattern's work. */
|
||||
const char *strjoker_steps(const char *chaine, const char *joker,
|
||||
size_t *nsteps_out, size_t *maxsteps_out) {
|
||||
size_t nsteps = 0;
|
||||
const char *r = strjoker_bounded(chaine, joker, NULL, NULL, &nsteps);
|
||||
/* Test-only: strjoker() reporting the work and depth spent and their caps, so a
|
||||
self-test can prove both bound a hostile pattern. */
|
||||
const char *strjoker_bounds(const char *chaine, const char *joker,
|
||||
size_t *nsteps_out, size_t *maxsteps_out,
|
||||
size_t *depth_out, size_t *maxdepth_out) {
|
||||
size_t nsteps = 0, depth = 0;
|
||||
const char *r = strjoker_bounded(chaine, joker, NULL, NULL, &nsteps, &depth);
|
||||
|
||||
if (nsteps_out != NULL)
|
||||
*nsteps_out = nsteps;
|
||||
if (maxsteps_out != NULL)
|
||||
*maxsteps_out = STRJOKER_MAXSTEPS;
|
||||
if (depth_out != NULL)
|
||||
*depth_out = depth;
|
||||
if (maxdepth_out != NULL)
|
||||
*maxdepth_out = STRJOKER_MAXDEPTH;
|
||||
return r;
|
||||
}
|
||||
|
||||
static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
|
||||
const char *joker, LLint *size,
|
||||
int *size_flag) {
|
||||
static const char *strjoker_impl(strjoker_memo *memo, const char *chaine,
|
||||
const char *joker, LLint *size, int *size_flag,
|
||||
size_t depth) {
|
||||
if (strnotempty(joker) == 0) { // fin de chaine joker
|
||||
if (strnotempty(chaine) == 0) // fin aussi pour la chaine: ok
|
||||
return chaine;
|
||||
@@ -403,7 +424,8 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
|
||||
|
||||
// tester sans le joker (pas ()+ mais ()*)
|
||||
if (!unique) {
|
||||
if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag))) {
|
||||
if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag,
|
||||
depth))) {
|
||||
return adr;
|
||||
}
|
||||
}
|
||||
@@ -416,7 +438,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
|
||||
while(i < (int) max) {
|
||||
if (pass[(int) (unsigned char) chaine[i]]) { // caractère autorisé
|
||||
if ((adr = strjoker_rec(memo, chaine + i + 1, joker + jmp, size,
|
||||
size_flag))) {
|
||||
size_flag, depth))) {
|
||||
return adr;
|
||||
}
|
||||
i++;
|
||||
@@ -427,7 +449,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
|
||||
// tester chaîne vide
|
||||
if (i != max + 2) // avant c'est ok
|
||||
if ((adr = strjoker_rec(memo, chaine + max, joker + jmp, size,
|
||||
size_flag)))
|
||||
size_flag, depth)))
|
||||
return adr;
|
||||
|
||||
return NULL; // perdu
|
||||
@@ -449,7 +471,8 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
|
||||
// comparaison ok?
|
||||
if (ok) {
|
||||
// continuer la comparaison.
|
||||
if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag))
|
||||
if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag,
|
||||
depth))
|
||||
return chaine; // retourner 1e lettre
|
||||
}
|
||||
|
||||
@@ -471,8 +494,8 @@ const char *strjokerfind(const char *chaine, const char *joker) {
|
||||
if (chaine != NULL && strlen(chaine) > STRJOKER_MAXLEN)
|
||||
return NULL;
|
||||
while(*chaine) {
|
||||
if ((adr = strjoker_bounded(chaine, joker, NULL, NULL,
|
||||
&nsteps))) { // ok trouvé
|
||||
if ((adr = strjoker_bounded(chaine, joker, NULL, NULL, &nsteps,
|
||||
NULL))) { // ok trouvé
|
||||
return adr;
|
||||
}
|
||||
if (nsteps > STRJOKER_MAXSTEPS) // scan budget spent: no match
|
||||
|
||||
@@ -52,10 +52,12 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
|
||||
oracle for the memoized matcher. */
|
||||
const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
|
||||
int *size_flag);
|
||||
/* strjoker() reporting the work-budget steps it spent and the cap; test-only,
|
||||
lets a self-test assert the budget bounds a hostile pattern's work. */
|
||||
const char *strjoker_steps(const char *chaine, const char *joker,
|
||||
size_t *nsteps_out, size_t *maxsteps_out);
|
||||
/* strjoker() reporting the work-budget steps and the recursion depth it spent,
|
||||
with their caps; test-only, lets a self-test assert both bound a hostile
|
||||
pattern (depth bounds the stack: an uncapped one overflows Windows' 1MB). */
|
||||
const char *strjoker_bounds(const char *chaine, const char *joker,
|
||||
size_t *nsteps_out, size_t *maxsteps_out,
|
||||
size_t *depth_out, size_t *maxdepth_out);
|
||||
const char *strjokerfind(const char *chaine, const char *joker);
|
||||
#endif
|
||||
|
||||
|
||||
@@ -322,16 +322,15 @@ typedef int64_t TStamp;
|
||||
/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
|
||||
#define LLintP "%" PRId64
|
||||
|
||||
/* Integer type for file offsets/sizes passed to the C library. Widens to
|
||||
LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain
|
||||
int otherwise; INTsysP is its printf conversion. */
|
||||
#ifdef LFS_FLAG
|
||||
/* Integer type for file offsets/sizes passed to the C library; INTsysP is its
|
||||
printf conversion. FIXME: LFS_FLAG is a configure make variable, never a C
|
||||
macro, so this test is dead and INTsys stays int on POSIX despite large-file
|
||||
support (the real macro is HTS_LFS). Widening it there is an installed-header
|
||||
type change, so it is left alone here. */
|
||||
#if defined(LFS_FLAG) || defined(_MSC_VER)
|
||||
typedef LLint INTsys;
|
||||
|
||||
#define INTsysP LLintP
|
||||
#ifdef __linux
|
||||
#define HTS_FSEEKO
|
||||
#endif
|
||||
#else
|
||||
typedef int INTsys;
|
||||
|
||||
|
||||
@@ -477,7 +477,7 @@ void help(const char *app, int more) {
|
||||
(" Y mirror ALL links located in the first level pages (mirror links)");
|
||||
infomsg("");
|
||||
infomsg("Proxy options:");
|
||||
infomsg(" P proxy use (-P proxy:port or -P user:pass@proxy:port)");
|
||||
infomsg(" P proxy use (-P [socks5://][user:pass@]proxy:port)");
|
||||
infomsg(" %f *use proxy for ftp (f0 don't use)");
|
||||
infomsg(" %b use this local hostname to make/send requests (-%b hostname)");
|
||||
infomsg("");
|
||||
|
||||
@@ -318,7 +318,7 @@ void index_finish(const char *indexpath, int mode) {
|
||||
char catbuff[CATBUFF_SIZE];
|
||||
char **tab;
|
||||
char *blk;
|
||||
off_t size = fpsize(fp_tmpproject);
|
||||
LLint size = fpsize(fp_tmpproject);
|
||||
|
||||
if (size > 0) {
|
||||
if (fp_tmpproject) {
|
||||
|
||||
228
src/htslib.c
228
src/htslib.c
@@ -649,165 +649,6 @@ T_SOC http_fopen(httrackp * opt, const char *adr, const char *fil, htsblk * reto
|
||||
return http_xfopen(opt, 0, 1, 1, NULL, adr, fil, retour);
|
||||
}
|
||||
|
||||
// Read a CRLF line from a non-blocking socket (waits up to timeout per recv).
|
||||
// Returns the line length (0 = empty), or -1 on timeout/EOF/error.
|
||||
static int proxy_getline(T_SOC soc, char *s, int max, int timeout) {
|
||||
int j = 0;
|
||||
|
||||
for (;;) {
|
||||
unsigned char ch;
|
||||
int n;
|
||||
|
||||
if (!check_readinput_t(soc, timeout))
|
||||
return -1; // timed out waiting for data
|
||||
n = (int) recv(soc, &ch, 1, 0);
|
||||
if (n == 1) {
|
||||
if (ch == 13) // CR
|
||||
continue;
|
||||
if (ch == 10) // LF: end of line
|
||||
break;
|
||||
if (j >= max - 1)
|
||||
return -1; // line too long: bound the read against a hostile proxy
|
||||
s[j++] = (char) ch;
|
||||
} else if (n == 0) {
|
||||
return -1; // connection closed
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
if (WSAGetLastError() == WSAEWOULDBLOCK)
|
||||
continue;
|
||||
#else
|
||||
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
|
||||
continue;
|
||||
#endif
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
s[j] = '\0';
|
||||
return j;
|
||||
}
|
||||
|
||||
int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
|
||||
int timeout) {
|
||||
const T_SOC soc = retour->soc;
|
||||
const char *const host = jump_identification_const(adr); // host[:port]
|
||||
const char *const portsep = jump_toport_const(adr); // ":port" or NULL
|
||||
char BIGSTK authority[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK req[HTS_URLMAXSIZE * 4 + 1100];
|
||||
char line[1024];
|
||||
int code;
|
||||
|
||||
if (soc == INVALID_SOCKET)
|
||||
return 0;
|
||||
|
||||
// CONNECT needs an explicit host:port; default the https port
|
||||
authority[0] = '\0';
|
||||
if (portsep != NULL)
|
||||
strlcatbuff(authority, host, sizeof(authority)); // already host:port
|
||||
else
|
||||
snprintf(authority, sizeof(authority), "%s:%d", host, 443);
|
||||
|
||||
// backstop: never let a stray CR/LF in the host smuggle a second line into
|
||||
// the CONNECT request (the host is already sanitized upstream)
|
||||
{
|
||||
const char *c;
|
||||
|
||||
for (c = authority; *c != '\0'; c++) {
|
||||
if ((unsigned char) *c < ' ') {
|
||||
strcpybuff(retour->msg, "proxy CONNECT: invalid host");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
snprintf(req, sizeof(req), "CONNECT %s HTTP/1.0" H_CRLF "Host: %s" H_CRLF,
|
||||
authority, authority);
|
||||
|
||||
// creds go on the CONNECT, not the tunneled origin request
|
||||
if (link_has_authorization(retour->req.proxy.name)) {
|
||||
const char *a = jump_identification_const(retour->req.proxy.name);
|
||||
const char *astart = jump_protocol_const(retour->req.proxy.name);
|
||||
char autorisation[1100];
|
||||
char user_pass[256];
|
||||
|
||||
autorisation[0] = user_pass[0] = '\0';
|
||||
strncatbuff(user_pass, astart, (int) (a - astart) - 1);
|
||||
strcpybuff(user_pass, unescape_http(OPT_GET_BUFF(opt),
|
||||
OPT_GET_BUFF_SIZE(opt), user_pass));
|
||||
code64((unsigned char *) user_pass, (int) strlen(user_pass),
|
||||
(unsigned char *) autorisation, 0);
|
||||
strlcatbuff(req, "Proxy-Authorization: Basic ", sizeof(req));
|
||||
strlcatbuff(req, autorisation, sizeof(req));
|
||||
strlcatbuff(req, H_CRLF, sizeof(req));
|
||||
}
|
||||
strlcatbuff(req, H_CRLF, sizeof(req)); // end of request headers
|
||||
|
||||
// raw send: ssl is set, so sendc() would route to TLS
|
||||
{
|
||||
const char *p = req;
|
||||
size_t remain = strlen(req);
|
||||
int stalls = 0;
|
||||
|
||||
while (remain > 0) {
|
||||
const int n = (int) send(soc, p, (int) remain, 0);
|
||||
|
||||
if (n > 0) {
|
||||
p += n;
|
||||
remain -= (size_t) n;
|
||||
stalls = 0;
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
const int wouldblock = (WSAGetLastError() == WSAEWOULDBLOCK);
|
||||
#else
|
||||
const int wouldblock =
|
||||
(errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR);
|
||||
#endif
|
||||
// don't spin forever on a fatal error or an unwritable socket
|
||||
if (!wouldblock || !check_writeinput_t(soc, timeout) ||
|
||||
++stalls > 100) {
|
||||
strcpybuff(retour->msg, "proxy CONNECT: write error");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// proxy status line: "HTTP/1.x <code> ..."
|
||||
if (proxy_getline(soc, line, sizeof(line), timeout) < 0) {
|
||||
strcpybuff(retour->msg, "proxy CONNECT: no response");
|
||||
return 0;
|
||||
}
|
||||
if (sscanf(line, "HTTP/%*d.%*d %d", &code) < 1)
|
||||
code = 0;
|
||||
if (code < 200 || code >= 300) {
|
||||
snprintf(retour->msg, sizeof(retour->msg), "proxy CONNECT refused: %s",
|
||||
strnotempty(line) ? line : "(no status)");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// drain headers to the blank line; cap the count so a flooding proxy can't
|
||||
// stall the crawl
|
||||
{
|
||||
int headers = 0;
|
||||
|
||||
for (;;) {
|
||||
const int n = proxy_getline(soc, line, sizeof(line), timeout);
|
||||
|
||||
if (n < 0) {
|
||||
strcpybuff(retour->msg, "proxy CONNECT: truncated response");
|
||||
return 0;
|
||||
}
|
||||
if (n == 0)
|
||||
break; // blank line: tunnel ready
|
||||
if (++headers > 64) {
|
||||
strcpybuff(retour->msg, "proxy CONNECT: too many response headers");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
// ouverture d'une liaison http, envoi d'une requète
|
||||
// mode: 0 GET 1 HEAD [2 POST]
|
||||
// treat: traiter header?
|
||||
@@ -1106,8 +947,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
/* widths bound method[256], url[HTS_URLMAXSIZE*2], protocol[256] */
|
||||
if (sscanf(line, "%255s %2047s %255s", method, url, protocol) == 3) {
|
||||
size_t ret;
|
||||
// selon que l'on a ou pas un proxy
|
||||
if (retour->req.proxy.active) {
|
||||
// absolute-URI for an http proxy; a socks tunnel takes origin-form
|
||||
if (retour->req.proxy.active &&
|
||||
!hts_proxy_is_socks(retour->req.proxy.name)) {
|
||||
print_buffer(&bstr,
|
||||
"%s http://%s%s %s\r\n", method, adr, url,
|
||||
protocol);
|
||||
@@ -1143,8 +985,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
print_buffer(&bstr, "HEAD ");
|
||||
}
|
||||
|
||||
// si on gère un proxy, il faut une Absolute URI: on ajoute avant http://www.adr.dom
|
||||
if (retour->req.proxy.active && (strncmp(adr, "https://", 8) != 0)) {
|
||||
// an http proxy needs an absolute URI; a socks tunnel does not
|
||||
if (retour->req.proxy.active &&
|
||||
!hts_proxy_is_socks(retour->req.proxy.name) &&
|
||||
(strncmp(adr, "https://", 8) != 0)) {
|
||||
if (!link_has_authority(adr)) { // default http
|
||||
#if HDEBUG
|
||||
printf("Proxy Use: for %s%s proxy %d port %d\n", adr, fil,
|
||||
@@ -1188,8 +1032,11 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
|
||||
if (xsend)
|
||||
print_buffer(&bstr, "%s", xsend); // éventuelles autres lignes
|
||||
|
||||
// for https, auth rides the CONNECT (the tunneled GET would leak it)
|
||||
if (retour->req.proxy.active && strncmp(adr, "https://", 8) != 0) {
|
||||
// for https, auth rides the CONNECT (the tunneled GET would leak it); for
|
||||
// socks, the handshake
|
||||
if (retour->req.proxy.active &&
|
||||
!hts_proxy_is_socks(retour->req.proxy.name) &&
|
||||
strncmp(adr, "https://", 8) != 0) {
|
||||
if (link_has_authorization(retour->req.proxy.name)) { // et hop, authentification proxy!
|
||||
const char *a = jump_identification_const(retour->req.proxy.name);
|
||||
const char *astart = jump_protocol_const(retour->req.proxy.name);
|
||||
@@ -3878,11 +3725,16 @@ const char *jump_protocol_const(const char *source) {
|
||||
|
||||
DECLARE_NON_CONST_VERSION(jump_protocol)
|
||||
|
||||
hts_boolean hts_proxy_is_socks(const char *name) {
|
||||
if (name == NULL)
|
||||
return HTS_FALSE;
|
||||
return (strfield(name, "socks5h:") || strfield(name, "socks5:")) ? HTS_TRUE
|
||||
: HTS_FALSE;
|
||||
}
|
||||
|
||||
// default proxy port for a -P argument, keyed on the scheme
|
||||
static int proxy_default_port(const char *arg) {
|
||||
if (strfield(arg, "socks5h:") || strfield(arg, "socks5:"))
|
||||
return 1080;
|
||||
return 8080;
|
||||
return hts_proxy_is_socks(arg) ? 1080 : 8080;
|
||||
}
|
||||
|
||||
void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
|
||||
@@ -4739,26 +4591,31 @@ int fexist_utf8(const char *s) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Taille d'un fichier, -1 si n'existe pas */
|
||||
/* File size, -1 if absent */
|
||||
/* Note: NOT utf-8 */
|
||||
off_t fsize(const char *s) {
|
||||
struct stat st;
|
||||
LLint fsize(const char *s) {
|
||||
STRUCT_STAT st;
|
||||
|
||||
if (!strnotempty(s)) // nom vide: erreur
|
||||
if (!strnotempty(s)) // empty name: error
|
||||
return -1;
|
||||
/* _stat64, not stat(): the latter is 32-bit on MSVC */
|
||||
#ifdef _WIN32
|
||||
if (_stat64(s, &st) == 0 && S_ISREG(st.st_mode)) {
|
||||
#else
|
||||
if (stat(s, &st) == 0 && S_ISREG(st.st_mode)) {
|
||||
#endif
|
||||
return st.st_size;
|
||||
} else {
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
|
||||
/* Taille d'un fichier, -1 si n'existe pas */
|
||||
/* File size, -1 if absent */
|
||||
/* Note: utf-8 */
|
||||
off_t fsize_utf8(const char *s) {
|
||||
LLint fsize_utf8(const char *s) {
|
||||
STRUCT_STAT st;
|
||||
|
||||
if (!strnotempty(s)) // nom vide: erreur
|
||||
if (!strnotempty(s)) // empty name: error
|
||||
return -1;
|
||||
if (STAT(s, &st) == 0 && S_ISREG(st.st_mode)) {
|
||||
return st.st_size;
|
||||
@@ -4767,24 +4624,17 @@ off_t fsize_utf8(const char *s) {
|
||||
}
|
||||
}
|
||||
|
||||
off_t fpsize(FILE * fp) {
|
||||
off_t oldpos, size;
|
||||
/* fseeko/ftello, never fseek/ftell: ftell returns long, which cannot carry an
|
||||
offset past 2GB on a 32-bit platform and fails with EOVERFLOW there. */
|
||||
LLint fpsize(FILE *fp) {
|
||||
LLint oldpos, size;
|
||||
|
||||
if (!fp)
|
||||
return -1;
|
||||
#ifdef HTS_FSEEKO
|
||||
oldpos = ftello(fp);
|
||||
#else
|
||||
oldpos = ftell(fp);
|
||||
#endif
|
||||
fseek(fp, 0, SEEK_END);
|
||||
#ifdef HTS_FSEEKO
|
||||
fseeko(fp, 0, SEEK_END);
|
||||
size = ftello(fp);
|
||||
fseeko(fp, oldpos, SEEK_SET);
|
||||
#else
|
||||
size = ftell(fp);
|
||||
fseek(fp, oldpos, SEEK_SET);
|
||||
#endif
|
||||
return size;
|
||||
}
|
||||
|
||||
@@ -6548,13 +6398,13 @@ int hts_stat_utf8(const char *path, STRUCT_STAT * buf) {
|
||||
LPWSTR wpath = hts_convertUTF8StringToUCS2(path, (int) strlen(path), NULL);
|
||||
|
||||
if (wpath != NULL) {
|
||||
const int result = _wstat(wpath, buf);
|
||||
const int result = _wstat64(wpath, buf);
|
||||
|
||||
free(wpath);
|
||||
return result;
|
||||
} else {
|
||||
// Fallback on conversion error.
|
||||
return _stat(path, buf);
|
||||
return _stat64(path, buf);
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
28
src/htslib.h
28
src/htslib.h
@@ -207,15 +207,9 @@ int check_readinput(htsblk * r);
|
||||
int check_readinput_t(T_SOC soc, int timeout);
|
||||
int check_writeinput_t(T_SOC soc, int timeout);
|
||||
|
||||
/* Open an HTTP CONNECT tunnel through the active proxy for an https request:
|
||||
`retour->soc` must already be TCP-connected to the proxy, and `adr` is the
|
||||
origin authority (url_adr, e.g. "https://host:port"). Sends the CONNECT
|
||||
request (with Proxy-Authorization when the proxy carries credentials) and
|
||||
reads the proxy's status line, so the caller's TLS handshake then runs
|
||||
end-to-end with the origin. Blocks up to `timeout` seconds. Returns 1 on a
|
||||
2xx tunnel, 0 on failure (retour->msg/statuscode set). */
|
||||
int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
|
||||
int timeout);
|
||||
/* TRUE if this -P proxy name (which keeps its scheme) is a SOCKS5 proxy. */
|
||||
hts_boolean hts_proxy_is_socks(const char *name);
|
||||
|
||||
void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * retour,
|
||||
char *rcvd);
|
||||
void treatfirstline(htsblk * retour, const char *rcvd);
|
||||
@@ -316,10 +310,11 @@ void cut_path(char *fullpath, char *path, size_t path_size, char *pname,
|
||||
int fexist(const char *s);
|
||||
int fexist_utf8(const char *s);
|
||||
|
||||
/*LLint fsize(const char* s); */
|
||||
off_t fpsize(FILE * fp);
|
||||
off_t fsize(const char *s);
|
||||
off_t fsize_utf8(const char *s);
|
||||
/* File size in bytes, -1 if absent or not a regular file. LLint, not off_t:
|
||||
the latter is a 32-bit long on MSVC, truncating any file of 2GB or more. */
|
||||
LLint fpsize(FILE *fp);
|
||||
LLint fsize(const char *s);
|
||||
LLint fsize_utf8(const char *s);
|
||||
|
||||
// Threads
|
||||
typedef void *(*beginthread_type) (void *);
|
||||
@@ -399,7 +394,8 @@ void *hts_get_callback(t_hts_htmlcheck_callbacks * callbacks,
|
||||
HTSEXT_API FILE *hts_fopen_utf8(const char *path, const char *mode);
|
||||
|
||||
#define STAT hts_stat_utf8
|
||||
typedef struct _stat STRUCT_STAT;
|
||||
/* _stat64: _stat's st_size is a 32-bit long, even in x64 builds */
|
||||
typedef struct _stat64 STRUCT_STAT;
|
||||
HTSEXT_API int hts_stat_utf8(const char *path, STRUCT_STAT * buf);
|
||||
|
||||
#define UNLINK hts_unlink_utf8
|
||||
@@ -597,9 +593,9 @@ HTS_STATIC int compare_mime(httrackp * opt, const char *mime, const char *file,
|
||||
#endif
|
||||
|
||||
// returns (size_t) -1 upon error
|
||||
static HTS_UNUSED size_t off_t_to_size_t(off_t o) {
|
||||
static HTS_UNUSED size_t llint_to_size_t(LLint o) {
|
||||
const size_t so = (size_t) o;
|
||||
if ((off_t) so == o) {
|
||||
if ((LLint) so == o) {
|
||||
return so;
|
||||
} else {
|
||||
return (size_t) -1;
|
||||
|
||||
539
src/htsproxy.c
Normal file
539
src/htsproxy.c
Normal file
@@ -0,0 +1,539 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: Proxy tunneling (HTTP CONNECT, SOCKS5) */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
/* Internal engine bytecode */
|
||||
#define HTS_INTERNAL_BYTECODE
|
||||
|
||||
#include "htscore.h"
|
||||
|
||||
#include "htslib.h"
|
||||
#include "htsproxy.h"
|
||||
|
||||
#include <string.h>
|
||||
|
||||
// Read a CRLF line from a non-blocking socket (waits up to timeout per recv).
|
||||
// Returns the line length (0 = empty), or -1 on timeout/EOF/error.
|
||||
static int proxy_getline(T_SOC soc, char *s, int max, int timeout) {
|
||||
int j = 0;
|
||||
|
||||
for (;;) {
|
||||
unsigned char ch;
|
||||
int n;
|
||||
|
||||
if (!check_readinput_t(soc, timeout))
|
||||
return -1; // timed out waiting for data
|
||||
n = (int) recv(soc, &ch, 1, 0);
|
||||
if (n == 1) {
|
||||
if (ch == 13) // CR
|
||||
continue;
|
||||
if (ch == 10) // LF: end of line
|
||||
break;
|
||||
if (j >= max - 1)
|
||||
return -1; // line too long: bound the read against a hostile proxy
|
||||
s[j++] = (char) ch;
|
||||
} else if (n == 0) {
|
||||
return -1; // connection closed
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
if (WSAGetLastError() == WSAEWOULDBLOCK)
|
||||
continue;
|
||||
#else
|
||||
if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
|
||||
continue;
|
||||
#endif
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
s[j] = '\0';
|
||||
return j;
|
||||
}
|
||||
|
||||
int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
|
||||
int timeout) {
|
||||
const T_SOC soc = retour->soc;
|
||||
const char *const host = jump_identification_const(adr); // host[:port]
|
||||
const char *const portsep = jump_toport_const(adr); // ":port" or NULL
|
||||
char BIGSTK authority[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK req[HTS_URLMAXSIZE * 4 + 1100];
|
||||
char line[1024];
|
||||
int code;
|
||||
|
||||
if (soc == INVALID_SOCKET)
|
||||
return 0;
|
||||
|
||||
// CONNECT needs an explicit host:port; default the https port
|
||||
authority[0] = '\0';
|
||||
if (portsep != NULL)
|
||||
strlcatbuff(authority, host, sizeof(authority)); // already host:port
|
||||
else
|
||||
snprintf(authority, sizeof(authority), "%s:%d", host, 443);
|
||||
|
||||
// backstop: never let a stray CR/LF in the host smuggle a second line into
|
||||
// the CONNECT request (the host is already sanitized upstream)
|
||||
{
|
||||
const char *c;
|
||||
|
||||
for (c = authority; *c != '\0'; c++) {
|
||||
if ((unsigned char) *c < ' ') {
|
||||
strcpybuff(retour->msg, "proxy CONNECT: invalid host");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
snprintf(req, sizeof(req), "CONNECT %s HTTP/1.0" H_CRLF "Host: %s" H_CRLF,
|
||||
authority, authority);
|
||||
|
||||
// creds go on the CONNECT, not the tunneled origin request
|
||||
if (link_has_authorization(retour->req.proxy.name)) {
|
||||
const char *a = jump_identification_const(retour->req.proxy.name);
|
||||
const char *astart = jump_protocol_const(retour->req.proxy.name);
|
||||
char autorisation[1100];
|
||||
char user_pass[256];
|
||||
|
||||
autorisation[0] = user_pass[0] = '\0';
|
||||
strncatbuff(user_pass, astart, (int) (a - astart) - 1);
|
||||
strcpybuff(user_pass, unescape_http(OPT_GET_BUFF(opt),
|
||||
OPT_GET_BUFF_SIZE(opt), user_pass));
|
||||
code64((unsigned char *) user_pass, (int) strlen(user_pass),
|
||||
(unsigned char *) autorisation, 0);
|
||||
strlcatbuff(req, "Proxy-Authorization: Basic ", sizeof(req));
|
||||
strlcatbuff(req, autorisation, sizeof(req));
|
||||
strlcatbuff(req, H_CRLF, sizeof(req));
|
||||
}
|
||||
strlcatbuff(req, H_CRLF, sizeof(req)); // end of request headers
|
||||
|
||||
// raw send: ssl is set, so sendc() would route to TLS
|
||||
{
|
||||
const char *p = req;
|
||||
size_t remain = strlen(req);
|
||||
int stalls = 0;
|
||||
|
||||
while (remain > 0) {
|
||||
const int n = (int) send(soc, p, (int) remain, 0);
|
||||
|
||||
if (n > 0) {
|
||||
p += n;
|
||||
remain -= (size_t) n;
|
||||
stalls = 0;
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
const int wouldblock = (WSAGetLastError() == WSAEWOULDBLOCK);
|
||||
#else
|
||||
const int wouldblock =
|
||||
(errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR);
|
||||
#endif
|
||||
// don't spin forever on a fatal error or an unwritable socket
|
||||
if (!wouldblock || !check_writeinput_t(soc, timeout) ||
|
||||
++stalls > 100) {
|
||||
strcpybuff(retour->msg, "proxy CONNECT: write error");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// proxy status line: "HTTP/1.x <code> ..."
|
||||
if (proxy_getline(soc, line, sizeof(line), timeout) < 0) {
|
||||
strcpybuff(retour->msg, "proxy CONNECT: no response");
|
||||
return 0;
|
||||
}
|
||||
if (sscanf(line, "HTTP/%*d.%*d %d", &code) < 1)
|
||||
code = 0;
|
||||
if (code < 200 || code >= 300) {
|
||||
snprintf(retour->msg, sizeof(retour->msg), "proxy CONNECT refused: %s",
|
||||
strnotempty(line) ? line : "(no status)");
|
||||
return 0;
|
||||
}
|
||||
|
||||
// drain headers to the blank line; cap the count so a flooding proxy can't
|
||||
// stall the crawl
|
||||
{
|
||||
int headers = 0;
|
||||
|
||||
for (;;) {
|
||||
const int n = proxy_getline(soc, line, sizeof(line), timeout);
|
||||
|
||||
if (n < 0) {
|
||||
strcpybuff(retour->msg, "proxy CONNECT: truncated response");
|
||||
return 0;
|
||||
}
|
||||
if (n == 0)
|
||||
break; // blank line: tunnel ready
|
||||
if (++headers > 64) {
|
||||
strcpybuff(retour->msg, "proxy CONNECT: too many response headers");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 1;
|
||||
}
|
||||
|
||||
/* SOCKS5 client (RFC 1928, RFC 1929 auth), hostname mode only: the proxy
|
||||
resolves the origin name (remote DNS, curl's socks5h). The stream is the
|
||||
proxy socket, or a scripted buffer under -#test=socks5. */
|
||||
|
||||
#define SOCKS5_VERSION 0x05
|
||||
#define SOCKS5_AUTH_VERSION 0x01
|
||||
#define SOCKS5_METHOD_NONE 0x00
|
||||
#define SOCKS5_METHOD_USERPASS 0x02
|
||||
#define SOCKS5_METHOD_NOACCEPTABLE 0xFF
|
||||
#define SOCKS5_CMD_CONNECT 0x01
|
||||
#define SOCKS5_ATYP_IPV4 0x01
|
||||
#define SOCKS5_ATYP_DOMAIN 0x03
|
||||
#define SOCKS5_ATYP_IPV6 0x04
|
||||
#define SOCKS5_MAXFIELD 255 /* one length byte: host, user and password */
|
||||
|
||||
typedef struct socks5_stream {
|
||||
T_SOC soc; /* INVALID_SOCKET when scripted (self-test) */
|
||||
int timeout;
|
||||
socks5_test_io *io;
|
||||
} socks5_stream;
|
||||
|
||||
static int socks5_fail(char *msg, size_t msgsize, const char *text) {
|
||||
if (msgsize != 0)
|
||||
strlcpybuff(msg, text, msgsize);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Read exactly n bytes, or fail: SOCKS frames are not self-delimiting, so a
|
||||
short read would desync the stream shared with the origin traffic. */
|
||||
static int socks5_read_n(socks5_stream *st, unsigned char *buf, size_t n) {
|
||||
size_t got = 0;
|
||||
int stalls = 0;
|
||||
|
||||
if (st->soc == INVALID_SOCKET) { /* scripted */
|
||||
socks5_test_io *const io = st->io;
|
||||
|
||||
if (n > io->reply_len - io->consumed)
|
||||
return 0;
|
||||
memcpy(buf, io->reply + io->consumed, n);
|
||||
io->consumed += n;
|
||||
return 1;
|
||||
}
|
||||
while (got < n) {
|
||||
const int r = (int) recv(st->soc, (char *) buf + got, (int) (n - got), 0);
|
||||
|
||||
if (r > 0) {
|
||||
got += (size_t) r;
|
||||
stalls = 0;
|
||||
} else if (r == 0) {
|
||||
return 0; // proxy closed mid-frame
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
const int wouldblock = (WSAGetLastError() == WSAEWOULDBLOCK);
|
||||
#else
|
||||
const int wouldblock =
|
||||
(errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR);
|
||||
#endif
|
||||
if (!wouldblock || !check_readinput_t(st->soc, st->timeout) ||
|
||||
++stalls > 100)
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int socks5_write_all(socks5_stream *st, const unsigned char *buf,
|
||||
size_t len) {
|
||||
size_t remain = len;
|
||||
int stalls = 0;
|
||||
|
||||
if (st->soc == INVALID_SOCKET) { /* scripted */
|
||||
socks5_test_io *const io = st->io;
|
||||
|
||||
if (len > sizeof(io->sent) - io->sent_len)
|
||||
return 0;
|
||||
memcpy(io->sent + io->sent_len, buf, len);
|
||||
io->sent_len += len;
|
||||
return 1;
|
||||
}
|
||||
while (remain > 0) {
|
||||
// raw send: the socket is still plain here, sendc() would route to TLS
|
||||
const int n = (int) send(st->soc, (const char *) buf, (int) remain, 0);
|
||||
|
||||
if (n > 0) {
|
||||
buf += n;
|
||||
remain -= (size_t) n;
|
||||
stalls = 0;
|
||||
} else {
|
||||
#ifdef _WIN32
|
||||
const int wouldblock = (WSAGetLastError() == WSAEWOULDBLOCK);
|
||||
#else
|
||||
const int wouldblock =
|
||||
(errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR);
|
||||
#endif
|
||||
if (!wouldblock || !check_writeinput_t(st->soc, st->timeout) ||
|
||||
++stalls > 100)
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
return 1;
|
||||
}
|
||||
|
||||
static const char *socks5_rep_message(unsigned char rep) {
|
||||
switch (rep) {
|
||||
case 0x01:
|
||||
return "general failure";
|
||||
case 0x02:
|
||||
return "connection not allowed by ruleset";
|
||||
case 0x03:
|
||||
return "network unreachable";
|
||||
case 0x04:
|
||||
return "host unreachable";
|
||||
case 0x05:
|
||||
return "connection refused";
|
||||
case 0x06:
|
||||
return "TTL expired";
|
||||
case 0x07:
|
||||
return "command not supported";
|
||||
case 0x08:
|
||||
return "address type not supported";
|
||||
default:
|
||||
return "unknown error";
|
||||
}
|
||||
}
|
||||
|
||||
/* The reply's BND.ADDR is variable-length: consume exactly what ATYP says, or
|
||||
the leftover bytes prepend to the first origin (or TLS) read. */
|
||||
static int socks5_read_reply(socks5_stream *st, char *msg, size_t msgsize) {
|
||||
unsigned char head[4];
|
||||
unsigned char addr[SOCKS5_MAXFIELD + 2];
|
||||
size_t addrlen;
|
||||
|
||||
if (!socks5_read_n(st, head, sizeof(head)))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: no reply from proxy");
|
||||
if (head[0] != SOCKS5_VERSION)
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: bad version in reply");
|
||||
if (head[1] != 0x00) {
|
||||
snprintf(msg, msgsize, "SOCKS5 connect failed: %s",
|
||||
socks5_rep_message(head[1]));
|
||||
return 0;
|
||||
}
|
||||
switch (head[3]) {
|
||||
case SOCKS5_ATYP_IPV4:
|
||||
addrlen = 4;
|
||||
break;
|
||||
case SOCKS5_ATYP_IPV6:
|
||||
addrlen = 16;
|
||||
break;
|
||||
case SOCKS5_ATYP_DOMAIN: {
|
||||
unsigned char len;
|
||||
|
||||
if (!socks5_read_n(st, &len, 1))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: truncated reply");
|
||||
addrlen = len; // <= 255, always fits addr[]
|
||||
break;
|
||||
}
|
||||
default: // unknown length: the stream cannot be resynchronized
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: unknown address type in reply");
|
||||
}
|
||||
if (addrlen != 0 && !socks5_read_n(st, addr, addrlen))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: truncated reply address");
|
||||
if (!socks5_read_n(st, addr, 2)) // BND.PORT, unused
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: truncated reply port");
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int socks5_negotiate(socks5_stream *st, const char *host, size_t hostlen,
|
||||
int port, const char *user, size_t userlen,
|
||||
const char *pass, size_t passlen, int want_auth,
|
||||
char *msg, size_t msgsize) {
|
||||
unsigned char frame[3 + SOCKS5_MAXFIELD + SOCKS5_MAXFIELD];
|
||||
unsigned char rep[2];
|
||||
|
||||
/* greeting */
|
||||
frame[0] = SOCKS5_VERSION;
|
||||
frame[1] = (unsigned char) (want_auth ? 2 : 1);
|
||||
frame[2] = SOCKS5_METHOD_NONE;
|
||||
frame[3] = SOCKS5_METHOD_USERPASS;
|
||||
if (!socks5_write_all(st, frame, (size_t) 2 + frame[1]))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: write error");
|
||||
if (!socks5_read_n(st, rep, sizeof(rep)))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: no method reply from proxy");
|
||||
if (rep[0] != SOCKS5_VERSION)
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: bad version in method reply");
|
||||
switch (rep[1]) {
|
||||
case SOCKS5_METHOD_NONE:
|
||||
break;
|
||||
case SOCKS5_METHOD_USERPASS:
|
||||
if (!want_auth)
|
||||
return socks5_fail(msg, msgsize,
|
||||
"SOCKS5: proxy requires authentication, none given");
|
||||
/* RFC 1929 sub-negotiation; its version byte is 0x01, not 0x05 */
|
||||
frame[0] = SOCKS5_AUTH_VERSION;
|
||||
frame[1] = (unsigned char) userlen;
|
||||
memcpy(frame + 2, user, userlen);
|
||||
frame[2 + userlen] = (unsigned char) passlen;
|
||||
memcpy(frame + 3 + userlen, pass, passlen);
|
||||
if (!socks5_write_all(st, frame, 3 + userlen + passlen))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: write error");
|
||||
if (!socks5_read_n(st, rep, sizeof(rep)))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: no authentication reply");
|
||||
if (rep[1] != 0x00)
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: authentication failed");
|
||||
break;
|
||||
case SOCKS5_METHOD_NOACCEPTABLE:
|
||||
return socks5_fail(
|
||||
msg, msgsize,
|
||||
"SOCKS5: proxy accepts no authentication method we offer");
|
||||
default:
|
||||
return socks5_fail(msg, msgsize,
|
||||
"SOCKS5: proxy selected an unknown method");
|
||||
}
|
||||
|
||||
/* CONNECT to the origin, by name */
|
||||
frame[0] = SOCKS5_VERSION;
|
||||
frame[1] = SOCKS5_CMD_CONNECT;
|
||||
frame[2] = 0x00; // RSV
|
||||
frame[3] = SOCKS5_ATYP_DOMAIN;
|
||||
frame[4] = (unsigned char) hostlen;
|
||||
memcpy(frame + 5, host, hostlen);
|
||||
frame[5 + hostlen] = (unsigned char) (port >> 8);
|
||||
frame[6 + hostlen] = (unsigned char) (port & 0xFF);
|
||||
if (!socks5_write_all(st, frame, 7 + hostlen))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: write error");
|
||||
|
||||
return socks5_read_reply(st, msg, msgsize);
|
||||
}
|
||||
|
||||
/* Decode the proxy userinfo into the two RFC 1929 fields. Split on the first
|
||||
colon of the still-escaped string, so a %3A stays inside the username. */
|
||||
static int socks5_credentials(httrackp *opt, const char *proxy_name, char *user,
|
||||
size_t user_size, size_t *userlen, char *pass,
|
||||
size_t pass_size, size_t *passlen, char *msg,
|
||||
size_t msgsize) {
|
||||
const char *const a = jump_identification_const(proxy_name); // past the '@'
|
||||
const char *const astart = jump_protocol_const(proxy_name);
|
||||
char userinfo[1024];
|
||||
char *colon;
|
||||
|
||||
if (a <= astart || (size_t) (a - astart) - 1 >= sizeof(userinfo))
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: credentials too long");
|
||||
userinfo[0] = '\0';
|
||||
strncatbuff(userinfo, astart, (int) (a - astart) - 1);
|
||||
colon = strchr(userinfo, ':');
|
||||
if (colon != NULL)
|
||||
*colon++ = '\0';
|
||||
strlcpybuff(
|
||||
user, unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), userinfo),
|
||||
user_size);
|
||||
strlcpybuff(pass,
|
||||
colon != NULL ? unescape_http(OPT_GET_BUFF(opt),
|
||||
OPT_GET_BUFF_SIZE(opt), colon)
|
||||
: "",
|
||||
pass_size);
|
||||
*userlen = strlen(user);
|
||||
*passlen = strlen(pass);
|
||||
// reject, never truncate: a clipped secret would authenticate as another one
|
||||
if (*userlen > SOCKS5_MAXFIELD || *passlen > SOCKS5_MAXFIELD)
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: credentials too long");
|
||||
if (*userlen == 0)
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: empty proxy username");
|
||||
return 1;
|
||||
}
|
||||
|
||||
static int socks5_handshake_stream(httrackp *opt, socks5_stream *st,
|
||||
const char *adr, const char *proxy_name,
|
||||
int ssl, char *msg, size_t msgsize) {
|
||||
const char *const host = jump_identification_const(adr);
|
||||
const char *const portsep = jump_toport_const(adr);
|
||||
const size_t hostlen =
|
||||
portsep != NULL ? (size_t) (portsep - host) : strlen(host);
|
||||
// sized for the whole userinfo: decoding shrinks, so an over-long credential
|
||||
// lands intact and is rejected below rather than silently clipped
|
||||
char user[1024];
|
||||
char pass[1024];
|
||||
size_t userlen = 0, passlen = 0;
|
||||
int want_auth = 0;
|
||||
int port = ssl ? 443 : 80;
|
||||
size_t i;
|
||||
|
||||
if (hostlen == 0 || hostlen > SOCKS5_MAXFIELD)
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: invalid origin host");
|
||||
if (host[0] == '[') // ATYP=domain cannot carry an IPv6 literal
|
||||
return socks5_fail(msg, msgsize,
|
||||
"SOCKS5: IPv6 literal origin is not supported");
|
||||
for (i = 0; i < hostlen; i++) {
|
||||
if ((unsigned char) host[i] < ' ')
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: invalid origin host");
|
||||
}
|
||||
if (portsep != NULL) {
|
||||
int p = -1;
|
||||
|
||||
sscanf(portsep + 1, "%d", &p);
|
||||
if (p <= 0 || p > 65535)
|
||||
return socks5_fail(msg, msgsize, "SOCKS5: invalid origin port");
|
||||
port = p;
|
||||
}
|
||||
if (link_has_authorization(proxy_name)) {
|
||||
if (!socks5_credentials(opt, proxy_name, user, sizeof(user), &userlen, pass,
|
||||
sizeof(pass), &passlen, msg, msgsize))
|
||||
return 0;
|
||||
want_auth = 1;
|
||||
}
|
||||
return socks5_negotiate(st, host, hostlen, port, user, userlen, pass, passlen,
|
||||
want_auth, msg, msgsize);
|
||||
}
|
||||
|
||||
int socks5_handshake(httrackp *opt, htsblk *retour, const char *adr,
|
||||
int timeout) {
|
||||
socks5_stream st;
|
||||
#if HTS_USEOPENSSL
|
||||
const int ssl = retour->ssl;
|
||||
#else
|
||||
const int ssl = 0;
|
||||
#endif
|
||||
|
||||
if (retour->soc == INVALID_SOCKET)
|
||||
return 0;
|
||||
st.soc = retour->soc;
|
||||
st.timeout = timeout;
|
||||
st.io = NULL;
|
||||
return socks5_handshake_stream(opt, &st, adr, retour->req.proxy.name, ssl,
|
||||
retour->msg, sizeof(retour->msg));
|
||||
}
|
||||
|
||||
int socks5_handshake_scripted(httrackp *opt, const char *adr,
|
||||
const char *proxy_name, socks5_test_io *io) {
|
||||
socks5_stream st;
|
||||
|
||||
st.soc = INVALID_SOCKET;
|
||||
st.timeout = 0;
|
||||
st.io = io;
|
||||
io->consumed = 0;
|
||||
io->sent_len = 0;
|
||||
io->msg[0] = '\0';
|
||||
return socks5_handshake_stream(opt, &st, adr, proxy_name, 0, io->msg,
|
||||
sizeof(io->msg));
|
||||
}
|
||||
82
src/htsproxy.h
Normal file
82
src/htsproxy.h
Normal file
@@ -0,0 +1,82 @@
|
||||
/* ------------------------------------------------------------ */
|
||||
/*
|
||||
HTTrack Website Copier, Offline Browser for Windows and Unix
|
||||
Copyright (C) 2026 Xavier Roche and other contributors
|
||||
|
||||
SPDX-License-Identifier: GPL-3.0-or-later
|
||||
|
||||
This program is free software: you can redistribute it and/or modify
|
||||
it under the terms of the GNU General Public License as published by
|
||||
the Free Software Foundation, either version 3 of the License, or
|
||||
(at your option) any later version.
|
||||
|
||||
This program is distributed in the hope that it will be useful,
|
||||
but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
GNU General Public License for more details.
|
||||
|
||||
You should have received a copy of the GNU General Public License
|
||||
along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
Ethical use: we kindly ask that you NOT use this software to harvest email
|
||||
addresses or to collect any other private information about people. Doing so
|
||||
would dishonor our work and waste the many hours we have spent on it.
|
||||
|
||||
Please visit our Website: http://www.httrack.com
|
||||
*/
|
||||
|
||||
/* ------------------------------------------------------------ */
|
||||
/* File: Proxy tunneling (HTTP CONNECT, SOCKS5) .h */
|
||||
/* Author: Xavier Roche */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
#ifndef HTS_DEFPROXY
|
||||
#define HTS_DEFPROXY
|
||||
|
||||
#include "htsglobal.h"
|
||||
|
||||
/* Forward definitions */
|
||||
#ifndef HTS_DEF_FWSTRUCT_httrackp
|
||||
#define HTS_DEF_FWSTRUCT_httrackp
|
||||
typedef struct httrackp httrackp;
|
||||
#endif
|
||||
#ifndef HTS_DEF_FWSTRUCT_htsblk
|
||||
#define HTS_DEF_FWSTRUCT_htsblk
|
||||
typedef struct htsblk htsblk;
|
||||
#endif
|
||||
|
||||
/* Open an HTTP CONNECT tunnel through the active proxy for an https request:
|
||||
`retour->soc` must already be TCP-connected to the proxy, and `adr` is the
|
||||
origin authority (url_adr, e.g. "https://host:port"). Sends the CONNECT
|
||||
request (with Proxy-Authorization when the proxy carries credentials) and
|
||||
reads the proxy's status line, so the caller's TLS handshake then runs
|
||||
end-to-end with the origin. Blocks up to `timeout` seconds. Returns 1 on a
|
||||
2xx tunnel, 0 on failure (retour->msg/statuscode set). */
|
||||
int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
|
||||
int timeout);
|
||||
|
||||
/* SOCKS5 (RFC 1928, RFC 1929 auth) handshake on the raw proxy socket, before
|
||||
any TLS: `retour->soc` must be TCP-connected to the proxy and `adr` is the
|
||||
origin (url_adr). The origin name is sent verbatim (ATYP=domain), so the
|
||||
proxy resolves it; a bracketed IPv6 literal origin is rejected. Blocks up to
|
||||
`timeout` seconds. Returns 1 with the stream positioned on the first origin
|
||||
byte, 0 on failure (retour->msg set). */
|
||||
int socks5_handshake(httrackp *opt, htsblk *retour, const char *adr,
|
||||
int timeout);
|
||||
|
||||
/* Self-test hook (-#test=socks5): run the handshake against `reply`, a scripted
|
||||
server byte stream, instead of a socket. `sent` captures the client frames,
|
||||
`consumed` the reply bytes eaten (the frame-desync guard). */
|
||||
typedef struct socks5_test_io {
|
||||
const unsigned char *reply;
|
||||
size_t reply_len;
|
||||
size_t consumed;
|
||||
unsigned char sent[1024];
|
||||
size_t sent_len;
|
||||
char msg[256];
|
||||
} socks5_test_io;
|
||||
|
||||
int socks5_handshake_scripted(httrackp *opt, const char *adr,
|
||||
const char *proxy_name, socks5_test_io *io);
|
||||
|
||||
#endif
|
||||
@@ -55,6 +55,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htsmd5.h"
|
||||
#include "htssniff.h"
|
||||
#include "htscodec.h"
|
||||
#include "htsproxy.h"
|
||||
#if HTS_USEZLIB
|
||||
#include "htszlib.h"
|
||||
#endif
|
||||
@@ -64,6 +65,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "coucal/coucal.h"
|
||||
|
||||
#include <ctype.h>
|
||||
#include <errno.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
@@ -71,6 +73,9 @@ Please visit our Website: http://www.httrack.com
|
||||
#ifndef _WIN32
|
||||
#include <sys/socket.h>
|
||||
#include <unistd.h>
|
||||
#else
|
||||
#include <io.h> /* _get_osfhandle, for the sparse-file hint */
|
||||
#include <winioctl.h> /* FSCTL_SET_SPARSE */
|
||||
#endif
|
||||
|
||||
/* very minimalistic internal tests */
|
||||
@@ -752,7 +757,7 @@ static int st_filterbounds(httrackp *opt, int argc, char **argv) {
|
||||
const size_t subjlen = 2048;
|
||||
char *subj = malloct(big + 1);
|
||||
char *pat = malloct(2 * stars + 2);
|
||||
size_t steps = 0, maxsteps = 0, i;
|
||||
size_t steps = 0, maxsteps = 0, depth = 0, maxdepth = 0, i;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
@@ -773,9 +778,22 @@ static int st_filterbounds(httrackp *opt, int argc, char **argv) {
|
||||
subj[subjlen] = '\0';
|
||||
/* Budget must fire and hold: steps > cap (deleting the budget zeroes the
|
||||
counter that is the enforcement), steps < 10*cap (unbudgeted ~1.26e9). */
|
||||
assertf(strjoker_steps(subj, pat, &steps, &maxsteps) == NULL);
|
||||
assertf(strjoker_bounds(subj, pat, &steps, &maxsteps, &depth, &maxdepth) ==
|
||||
NULL);
|
||||
assertf(steps > maxsteps && steps < 10 * maxsteps);
|
||||
/* Depth caps the stack: uncapped this recurses 2046 frames, ~900KB (#574). */
|
||||
assertf(depth == maxdepth);
|
||||
assertf(strjokerfind(subj, pat) == NULL);
|
||||
/* Pin the cap from below: 32 segments must still match, so a cap set so low
|
||||
it would break real multi-segment filters (which use far fewer) fails. */
|
||||
for (i = 0; i < 32; i++) {
|
||||
pat[2 * i] = '*';
|
||||
pat[2 * i + 1] = 'a';
|
||||
}
|
||||
pat[64] = '\0';
|
||||
memset(subj, 'a', 32);
|
||||
subj[32] = '\0';
|
||||
assertf(strjoker(subj, pat, NULL, NULL) != NULL);
|
||||
freet(pat);
|
||||
freet(subj);
|
||||
printf("filterbounds: OK\n");
|
||||
@@ -812,7 +830,11 @@ static int st_mime(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
static size_t st_decode_body(const char *arg, char *buf, size_t size);
|
||||
|
||||
static int st_charset(httrackp *opt, int argc, char **argv) {
|
||||
char buf[512];
|
||||
size_t len;
|
||||
char *s;
|
||||
|
||||
(void) opt;
|
||||
@@ -820,7 +842,8 @@ static int st_charset(httrackp *opt, int argc, char **argv) {
|
||||
fprintf(stderr, "charset: needs a charset and a string\n");
|
||||
return 1;
|
||||
}
|
||||
s = hts_convertStringToUTF8(argv[1], strlen(argv[1]), argv[0]);
|
||||
len = st_decode_body(argv[1], buf, sizeof(buf));
|
||||
s = hts_convertStringToUTF8(buf, len, argv[0]);
|
||||
if (s != NULL) {
|
||||
printf("%s\n", s);
|
||||
freet(s);
|
||||
@@ -845,16 +868,22 @@ static int st_metacharset(httrackp *opt, int argc, char **argv) {
|
||||
}
|
||||
|
||||
static int st_isutf8(httrackp *opt, int argc, char **argv) {
|
||||
char buf[512];
|
||||
size_t len;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "isutf8: needs a string\n");
|
||||
return 1;
|
||||
}
|
||||
printf("%d\n", hts_isStringUTF8(argv[0], strlen(argv[0])) ? 1 : 0);
|
||||
len = st_decode_body(argv[0], buf, sizeof(buf));
|
||||
printf("%d\n", hts_isStringUTF8(buf, len) ? 1 : 0);
|
||||
return 0;
|
||||
}
|
||||
|
||||
static int st_idna_encode(httrackp *opt, int argc, char **argv) {
|
||||
char buf[512];
|
||||
size_t len;
|
||||
char *s;
|
||||
|
||||
(void) opt;
|
||||
@@ -862,7 +891,8 @@ static int st_idna_encode(httrackp *opt, int argc, char **argv) {
|
||||
fprintf(stderr, "idna-encode: needs a hostname\n");
|
||||
return 1;
|
||||
}
|
||||
s = hts_convertStringUTF8ToIDNA(argv[0], strlen(argv[0]));
|
||||
len = st_decode_body(argv[0], buf, sizeof(buf));
|
||||
s = hts_convertStringUTF8ToIDNA(buf, len);
|
||||
if (s != NULL) {
|
||||
printf("%s\n", s);
|
||||
freet(s);
|
||||
@@ -1024,7 +1054,7 @@ static int st_hashtable(httrackp *opt, int argc, char **argv) {
|
||||
|
||||
/* produce random patterns, or read from a file */
|
||||
if (sscanf(snum, "%lu", &count) != 1) {
|
||||
const off_t size = fsize(snum);
|
||||
const LLint size = fsize(snum);
|
||||
FILE *fp = fopen(snum, "rb");
|
||||
if (fp != NULL) {
|
||||
buff = malloct(size);
|
||||
@@ -1124,7 +1154,10 @@ static int st_strsafe(httrackp *opt, int argc, char **argv) {
|
||||
comes from argv so its length is opaque to the compiler (no static
|
||||
-Wstringop-overflow, genuine runtime check). "overflow-buff" exercises
|
||||
htsbuff. */
|
||||
char small[4];
|
||||
/* Not sizeof(char*): on ILP32 a char[4] equals the pointer size, and the
|
||||
MSVC array-vs-pointer heuristic (sizeof(A) != sizeof(char*)) then reads
|
||||
it as a pointer and silently skips the bound. */
|
||||
char small[6];
|
||||
const char *const src = (argc >= 2) ? argv[1] : "overflowing";
|
||||
|
||||
if (strcmp(argv[0], "overflow-buff") == 0) {
|
||||
@@ -1324,6 +1357,146 @@ static int st_proxyurl(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Scripted SOCKS5 server: no-auth method reply, then a success CONNECT reply
|
||||
carrying the given ATYP address, then a sentinel origin byte the handshake
|
||||
must leave unread (an off-by-one in the reply framing eats it). */
|
||||
static size_t socks5_reply(unsigned char *buf, unsigned char atyp,
|
||||
const unsigned char *addr, size_t addrlen) {
|
||||
size_t n = 0;
|
||||
|
||||
buf[n++] = 0x05;
|
||||
buf[n++] = 0x00; /* method: no authentication */
|
||||
buf[n++] = 0x05;
|
||||
buf[n++] = 0x00; /* REP: succeeded */
|
||||
buf[n++] = 0x00;
|
||||
buf[n++] = atyp;
|
||||
memcpy(buf + n, addr, addrlen);
|
||||
n += addrlen;
|
||||
buf[n++] = 0x1f;
|
||||
buf[n++] = 0x90; /* BND.PORT */
|
||||
buf[n++] = 0xAA; /* sentinel */
|
||||
return n;
|
||||
}
|
||||
|
||||
#define SOCKS5_SENTINEL_LEFT(io, len) ((io).consumed == (len) - 1)
|
||||
|
||||
static int st_socks5(httrackp *opt, int argc, char **argv) {
|
||||
static const unsigned char v4[4] = {127, 0, 0, 1};
|
||||
static const unsigned char v6[16] = {0};
|
||||
static const unsigned char domain[6] = {5, 'p', 'r', 'o', 'x', 'y'};
|
||||
const char *const proxy = "socks5://127.0.0.1";
|
||||
unsigned char script[64];
|
||||
socks5_test_io io;
|
||||
size_t len;
|
||||
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
/* each reply address type is drained exactly, sentinel untouched */
|
||||
len = socks5_reply(script, 0x01, v4, sizeof(v4));
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 1);
|
||||
assertf(SOCKS5_SENTINEL_LEFT(io, len));
|
||||
/* the greeting offers no-auth only, and the origin goes out by name, port 80
|
||||
*/
|
||||
assertf(io.sent_len == 3 + 7 + 11);
|
||||
assertf(memcmp(io.sent, "\x05\x01\x00", 3) == 0);
|
||||
assertf(memcmp(io.sent + 3, "\x05\x01\x00\x03\x0borigin.test\x00\x50", 18) ==
|
||||
0);
|
||||
|
||||
len = socks5_reply(script, 0x04, v6, sizeof(v6));
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 1);
|
||||
assertf(SOCKS5_SENTINEL_LEFT(io, len));
|
||||
|
||||
len = socks5_reply(script, 0x03, domain, sizeof(domain));
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 1);
|
||||
assertf(SOCKS5_SENTINEL_LEFT(io, len));
|
||||
|
||||
/* an unknown address type has no known length: fail, never guess */
|
||||
len = socks5_reply(script, 0x02, v4, sizeof(v4));
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 0);
|
||||
|
||||
/* truncated frames (here: no BND.PORT) fail instead of over-reading */
|
||||
len = socks5_reply(script, 0x01, v4, sizeof(v4)) - 3;
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 0);
|
||||
|
||||
/* explicit origin port is encoded big-endian (8443 = 0x20fb) */
|
||||
len = socks5_reply(script, 0x01, v4, sizeof(v4));
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test:8443", proxy, &io) == 1);
|
||||
assertf(memcmp(io.sent + io.sent_len - 2, "\x20\xfb", 2) == 0);
|
||||
|
||||
/* credentials: split on the first colon of the escaped userinfo, so %3a stays
|
||||
inside the username and a colon in the password is not a delimiter */
|
||||
{
|
||||
static const unsigned char auth_script[] = {
|
||||
0x05, 0x02, /* method: user/pass */
|
||||
0x01, 0x00, /* auth: success */
|
||||
0x05, 0x00, 0x00, 0x01, /* REP: succeeded, ATYP ipv4 */
|
||||
127, 0, 0, 1, 0x1f, 0x90, 0xAA};
|
||||
|
||||
io.reply = auth_script;
|
||||
io.reply_len = sizeof(auth_script);
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test",
|
||||
"socks5://us%3aer:p:ass@127.0.0.1",
|
||||
&io) == 1);
|
||||
assertf(SOCKS5_SENTINEL_LEFT(io, sizeof(auth_script)));
|
||||
assertf(memcmp(io.sent, "\x05\x02\x00\x02", 4) == 0);
|
||||
assertf(memcmp(io.sent + 4, "\x01\x05us:er\x05p:ass", 13) == 0);
|
||||
}
|
||||
|
||||
/* a proxy demanding auth we cannot provide, and one refusing every method */
|
||||
io.reply = (const unsigned char *) "\x05\x02";
|
||||
io.reply_len = 2;
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 0);
|
||||
io.reply = (const unsigned char *) "\x05\xff";
|
||||
io.reply_len = 2;
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 0);
|
||||
|
||||
/* a refused CONNECT is an error, not a tunnel */
|
||||
io.reply = (const unsigned char
|
||||
*) "\x05\x00\x05\x05\x00\x01\x7f\x00\x00\x01\x1f\x90";
|
||||
io.reply_len = 12;
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 0);
|
||||
|
||||
/* over-long host or credentials are rejected before anything is sent */
|
||||
{
|
||||
char host[512];
|
||||
char name[1024];
|
||||
size_t i;
|
||||
|
||||
host[0] = '\0';
|
||||
for (i = 0; i < 256; i++)
|
||||
strcatbuff(host, "a");
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, host, proxy, &io) == 0);
|
||||
assertf(io.sent_len == 0);
|
||||
|
||||
strcpybuff(name, "socks5://user:");
|
||||
for (i = 0; i < 256; i++)
|
||||
strcatbuff(name, "p");
|
||||
strcatbuff(name, "@127.0.0.1");
|
||||
io.reply = script;
|
||||
io.reply_len = len;
|
||||
assertf(socks5_handshake_scripted(opt, "origin.test", name, &io) == 0);
|
||||
assertf(io.sent_len == 0);
|
||||
}
|
||||
|
||||
printf("socks5 self-test OK\n");
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Regression for the one-byte fil[] overflow: a 2047-byte hostless "?"-URL used
|
||||
to abort in strncat_safe_ when the missing leading '/' pushed fil to 2048. */
|
||||
static int st_identabs(httrackp *opt, int argc, char **argv) {
|
||||
@@ -1468,7 +1641,8 @@ static int st_crange(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Decode a body argument ("hex:FFD8.." or literal text) into buf. */
|
||||
/* Decode an argument ("hex:FFD8.." or literal text) into buf. Raw non-UTF-8
|
||||
bytes cannot survive a Windows command line, so hostile inputs go as hex. */
|
||||
static size_t st_decode_body(const char *arg, char *buf, size_t size) {
|
||||
size_t n = 0;
|
||||
|
||||
@@ -1508,6 +1682,94 @@ static int st_sniff(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* fsize()/fsize_utf8()/fpsize() must report a size past 4GB: 32-bit wraps both
|
||||
ways there (MSVC's off_t and struct _stat st_size are long, 32-bit even on
|
||||
x64), and a size under 4GB would survive an *unsigned* 32-bit truncation. */
|
||||
static int st_fsize(httrackp *opt, int argc, char **argv) {
|
||||
const LLint expected = 5 * 1024 * 1024 * 1024LL;
|
||||
char BIGSTK path[HTS_URLMAXSIZE * 2];
|
||||
char BIGSTK absent[HTS_URLMAXSIZE * 2];
|
||||
/* both variants: only fsize() feeds the >2GB readers (file://, --list) */
|
||||
const int width = (int) sizeof(fsize(""));
|
||||
const int width_utf8 = (int) sizeof(fsize_utf8(""));
|
||||
FILE *fp;
|
||||
LLint got, got_utf8, gotp, gone;
|
||||
int rc = 0;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "fsize: needs a directory\n");
|
||||
return 1;
|
||||
}
|
||||
concat(path, sizeof(path), argv[0], "/sparse-5g.bin");
|
||||
concat(absent, sizeof(absent), argv[0], "/no-such-file.bin");
|
||||
|
||||
/* sparse: seek past 4GB and write the last byte */
|
||||
fp = FOPEN(path, "wb");
|
||||
if (fp == NULL) {
|
||||
fprintf(stderr, "fsize: cannot create '%s': %s\n", path, strerror(errno));
|
||||
return 1;
|
||||
}
|
||||
#ifdef _WIN32
|
||||
{
|
||||
/* NTFS allocates the hole unless asked not to; POSIX gives it for free.
|
||||
Best-effort: a non-sparse file still measures the same, just costs 5GB.
|
||||
*/
|
||||
HANDLE h = (HANDLE) _get_osfhandle(_fileno(fp));
|
||||
DWORD ret;
|
||||
|
||||
if (h != INVALID_HANDLE_VALUE)
|
||||
(void) DeviceIoControl(h, FSCTL_SET_SPARSE, NULL, 0, NULL, 0, &ret, NULL);
|
||||
}
|
||||
#endif
|
||||
if (fseeko(fp, expected - 1, SEEK_SET) != 0 || fputc(0, fp) == EOF ||
|
||||
fclose(fp) != 0) {
|
||||
fprintf(stderr, "fsize: cannot extend '%s' to " LLintP ": %s\n", path,
|
||||
expected, strerror(errno));
|
||||
UNLINK(path);
|
||||
return 1;
|
||||
}
|
||||
|
||||
got = fsize(path);
|
||||
got_utf8 = fsize_utf8(path);
|
||||
fp = FOPEN(path, "rb");
|
||||
if (fp == NULL) {
|
||||
fprintf(stderr, "fsize: cannot reopen '%s': %s\n", path, strerror(errno));
|
||||
gotp = -1;
|
||||
} else {
|
||||
gotp = fpsize(fp);
|
||||
fclose(fp);
|
||||
}
|
||||
UNLINK(path);
|
||||
gone = fsize(absent); /* contract: -1, not 0, when absent */
|
||||
|
||||
printf("fsize: width=%d,%d size=" LLintP "," LLintP " psize=" LLintP
|
||||
" absent=" LLintP "\n",
|
||||
width, width_utf8, got, got_utf8, gotp, gone);
|
||||
if (width != 8 || width_utf8 != 8) {
|
||||
fprintf(stderr, "fsize: return types are %d/%d bytes, expected 8\n", width,
|
||||
width_utf8);
|
||||
rc = 1;
|
||||
}
|
||||
if (got != expected || got_utf8 != expected) {
|
||||
fprintf(stderr,
|
||||
"fsize: fsize/fsize_utf8 are " LLintP "/" LLintP
|
||||
", expected " LLintP "\n",
|
||||
got, got_utf8, expected);
|
||||
rc = 1;
|
||||
}
|
||||
if (gotp != expected) {
|
||||
fprintf(stderr, "fsize: fpsize is " LLintP ", expected " LLintP "\n", gotp,
|
||||
expected);
|
||||
rc = 1;
|
||||
}
|
||||
if (gone != -1) {
|
||||
fprintf(stderr, "fsize: absent file is " LLintP ", expected -1\n", gone);
|
||||
rc = 1;
|
||||
}
|
||||
return rc;
|
||||
}
|
||||
|
||||
static int st_savename(httrackp *opt, int argc, char **argv) {
|
||||
lien_adrfilsave afs;
|
||||
cache_back cache;
|
||||
@@ -2654,11 +2916,43 @@ static int st_robots(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Connected stream pair over loopback; Windows has no socketpair(). */
|
||||
static int st_socketpair(T_SOC sv[2]) {
|
||||
struct sockaddr_in sa;
|
||||
socklen_t len = sizeof(sa);
|
||||
T_SOC srv, cli, acc;
|
||||
|
||||
memset(&sa, 0, sizeof(sa));
|
||||
sa.sin_family = AF_INET;
|
||||
sa.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
|
||||
if ((srv = (T_SOC) socket(AF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET)
|
||||
return -1;
|
||||
if (bind(srv, (struct sockaddr *) &sa, sizeof(sa)) != 0 ||
|
||||
listen(srv, 1) != 0 ||
|
||||
getsockname(srv, (struct sockaddr *) &sa, &len) != 0) {
|
||||
deletesoc(srv);
|
||||
return -1;
|
||||
}
|
||||
if ((cli = (T_SOC) socket(AF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET) {
|
||||
deletesoc(srv);
|
||||
return -1;
|
||||
}
|
||||
if (connect(cli, (struct sockaddr *) &sa, sizeof(sa)) != 0 ||
|
||||
(acc = (T_SOC) accept(srv, NULL, NULL)) == INVALID_SOCKET) {
|
||||
deletesoc(cli);
|
||||
deletesoc(srv);
|
||||
return -1;
|
||||
}
|
||||
deletesoc(srv);
|
||||
sv[0] = acc;
|
||||
sv[1] = cli;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* get_ftp_line must bound a hostile, CRLF-less reply into its internal
|
||||
1024-byte buffer; ASan turns the pre-fix overflow into an abort here. */
|
||||
#ifndef _WIN32
|
||||
static int st_ftpline(httrackp *opt, int argc, char **argv) {
|
||||
int sv[2];
|
||||
T_SOC sv[2];
|
||||
char line[2048];
|
||||
char flood[4096];
|
||||
|
||||
@@ -2666,19 +2960,19 @@ static int st_ftpline(httrackp *opt, int argc, char **argv) {
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
memset(flood, 'x', sizeof(flood));
|
||||
assertf(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0);
|
||||
assertf(write(sv[1], "220 ", 4) == 4); // valid 3-digit code
|
||||
assertf(write(sv[1], flood, sizeof(flood)) == (ssize_t) sizeof(flood));
|
||||
assertf(write(sv[1], "\r\n", 2) == 2); // end the line so we return
|
||||
close(sv[1]);
|
||||
assertf(st_socketpair(sv) == 0);
|
||||
// the 4102-byte reply fits the loopback send buffer, so no reader is needed
|
||||
assertf(send(sv[1], "220 ", 4, 0) == 4); // valid 3-digit code
|
||||
assertf(send(sv[1], flood, (int) sizeof(flood), 0) == (int) sizeof(flood));
|
||||
assertf(send(sv[1], "\r\n", 2, 0) == 2); // end the line so we return
|
||||
deletesoc(sv[1]);
|
||||
line[0] = '\0';
|
||||
get_ftp_line(sv[0], line, sizeof(line), 5);
|
||||
close(sv[0]);
|
||||
deletesoc(sv[0]);
|
||||
printf("ftp-line self-test OK (bounded %d-byte reply)\n",
|
||||
(int) sizeof(flood));
|
||||
return 0;
|
||||
}
|
||||
#endif
|
||||
|
||||
/* ftp_split_userpass: well-formed split, plus a hostile over-long userinfo
|
||||
that pre-fix overran user[256]/pass[256]. */
|
||||
@@ -2750,11 +3044,11 @@ static const struct selftest_entry {
|
||||
{"redirect-samefile", "", "same-file redirect detection self-test (#159)",
|
||||
st_redirect_samefile},
|
||||
{"mime", "<filename>", "MIME type for a filename", st_mime},
|
||||
{"charset", "<charset> <string>",
|
||||
{"charset", "<charset> <hex:..|string>",
|
||||
"convert a string to UTF-8 from a charset", st_charset},
|
||||
{"metacharset", "<html>", "extract the <meta> charset from an HTML page",
|
||||
st_metacharset},
|
||||
{"isutf8", "<string>", "is the string valid UTF-8 (1/0)", st_isutf8},
|
||||
{"isutf8", "<hex:..|string>", "is the string valid UTF-8 (1/0)", st_isutf8},
|
||||
{"idna-encode", "<host>", "encode a hostname to IDNA/punycode",
|
||||
st_idna_encode},
|
||||
{"idna-decode", "<host>", "decode an IDNA/punycode hostname",
|
||||
@@ -2774,6 +3068,8 @@ static const struct selftest_entry {
|
||||
{"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
|
||||
{"proxyurl", "<proxy-arg>", "parse a -P proxy URL into host/port",
|
||||
st_proxyurl},
|
||||
{"socks5", "", "SOCKS5 handshake framing and credential self-test",
|
||||
st_socks5},
|
||||
{"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
|
||||
st_identabs},
|
||||
{"header", "<raw-header-line> ...", "response header-line parsing",
|
||||
@@ -2789,6 +3085,7 @@ static const struct selftest_entry {
|
||||
"local save-name for a URL", st_savename},
|
||||
{"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
|
||||
st_sniff},
|
||||
{"fsize", "<dir>", "file size past the 2GB signed-32-bit wrap", st_fsize},
|
||||
{"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
|
||||
{"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
|
||||
st_cacheindex},
|
||||
@@ -2819,10 +3116,8 @@ static const struct selftest_entry {
|
||||
st_contentcodings},
|
||||
{"robots", "", "robots.txt RFC 9309 Allow/Disallow precedence self-test",
|
||||
st_robots},
|
||||
#ifndef _WIN32
|
||||
{"ftp-line", "", "get_ftp_line bounds a hostile FTP reply line",
|
||||
st_ftpline},
|
||||
#endif
|
||||
{"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
|
||||
};
|
||||
|
||||
|
||||
@@ -743,7 +743,8 @@ HTSEXT_API hts_boolean hts_findissystem(find_handle find);
|
||||
HTSEXT_API FILE *hts_fopen_utf8(const char *path, const char *mode);
|
||||
|
||||
#define STAT hts_stat_utf8
|
||||
typedef struct _stat STRUCT_STAT;
|
||||
/* _stat64: _stat's st_size is a 32-bit long, even in x64 builds */
|
||||
typedef struct _stat64 STRUCT_STAT;
|
||||
|
||||
HTSEXT_API int hts_stat_utf8(const char *path, STRUCT_STAT *buf);
|
||||
|
||||
|
||||
@@ -45,6 +45,7 @@ Please visit our Website: http://www.httrack.com
|
||||
#include "htsdefines.h"
|
||||
#include "httrack.h"
|
||||
#include "htslib.h"
|
||||
#include "htscharset.h" // after htslib.h: winsock2.h must precede windows.h
|
||||
|
||||
/* Static definitions */
|
||||
static int fexist(const char *s);
|
||||
@@ -206,6 +207,7 @@ int main(int argc, char **argv) {
|
||||
httrackp *opt;
|
||||
|
||||
#ifdef _WIN32
|
||||
hts_argv_utf8(&argc, &argv);
|
||||
{
|
||||
WORD wVersionRequested; // requested version WinSock API
|
||||
WSADATA wsadata; // Windows Sockets API data
|
||||
|
||||
@@ -70,7 +70,7 @@
|
||||
<Link>
|
||||
<!-- vcpkg auto-links libssl/libcrypto/zlib/brotli/zstd; only the OS import
|
||||
lib is explicit. -->
|
||||
<AdditionalDependencies>Ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<AdditionalDependencies>Ws2_32.lib;Shell32.lib;%(AdditionalDependencies)</AdditionalDependencies>
|
||||
<GenerateDebugInformation>true</GenerateDebugInformation>
|
||||
</Link>
|
||||
</ItemDefinitionGroup>
|
||||
@@ -125,6 +125,7 @@
|
||||
<ClCompile Include="htsmodules.c" />
|
||||
<ClCompile Include="htsname.c" />
|
||||
<ClCompile Include="htsparse.c" />
|
||||
<ClCompile Include="htsproxy.c" />
|
||||
<ClCompile Include="htsrobots.c" />
|
||||
<ClCompile Include="htsselftest.c" />
|
||||
<ClCompile Include="htssniff.c" />
|
||||
|
||||
@@ -577,9 +577,11 @@ PT_Index PT_LoadCache(const char *filename) {
|
||||
chain = coucal_enum_next(&en);
|
||||
}
|
||||
if (chain != NULL) {
|
||||
if (!link_has_authority(chain->name))
|
||||
strcat(index->slots.common.startUrl, "http://");
|
||||
strcat(index->slots.common.startUrl, chain->name);
|
||||
const char *scheme = link_has_authority(chain->name) ? "" : "http://";
|
||||
|
||||
snprintf(index->slots.common.startUrl,
|
||||
sizeof(index->slots.common.startUrl), "%s%s", scheme,
|
||||
(const char *) chain->name);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -632,7 +634,7 @@ int PT_EnumCache(PT_Indexes indexes,
|
||||
const long int index_id = (long int) chain->value.intg;
|
||||
const char *const url = chain->name;
|
||||
|
||||
if (index_id >= 0 && index_id <= indexes->index_size) {
|
||||
if (index_id >= 0 && index_id < indexes->index_size) {
|
||||
PT_Element item =
|
||||
PT_ReadCache(indexes->index[index_id], url,
|
||||
FETCH_HEADERS | FETCH_BODY);
|
||||
@@ -757,7 +759,7 @@ PT_Element PT_ReadIndex(PT_Indexes indexes, const char *url, int flags) {
|
||||
if (strncmp(url, "http://", 7) == 0)
|
||||
url += 7;
|
||||
if (coucal_read(indexes->cil, url, &index_id)) {
|
||||
if (index_id >= 0 && index_id <= indexes->index_size) {
|
||||
if (index_id >= 0 && index_id < indexes->index_size) {
|
||||
PT_Element item = PT_ReadCache(indexes->index[index_id], url, flags);
|
||||
|
||||
if (item != NULL) {
|
||||
@@ -780,7 +782,7 @@ int PT_LookupIndex(PT_Indexes indexes, const char *url) {
|
||||
if (strncmp(url, "http://", 7) == 0)
|
||||
url += 7;
|
||||
if (coucal_read(indexes->cil, url, &index_id)) {
|
||||
if (index_id >= 0 && index_id <= indexes->index_size) {
|
||||
if (index_id >= 0 && index_id < indexes->index_size) {
|
||||
return 1;
|
||||
} else {
|
||||
proxytrack_print_log(CRITICAL,
|
||||
@@ -967,10 +969,12 @@ int PT_LoadCache__New(PT_Index index_, const char *filename) {
|
||||
/* First link as starting URL */
|
||||
if (!firstSeen) {
|
||||
if (strstr(filenameIndex, "/robots.txt") == NULL) {
|
||||
const char *scheme =
|
||||
link_has_authority(filenameIndex) ? "" : "http://";
|
||||
|
||||
firstSeen = 1;
|
||||
if (!link_has_authority(filenameIndex))
|
||||
strcat(index->startUrl, "http://");
|
||||
strcat(index->startUrl, filenameIndex);
|
||||
snprintf(index->startUrl, sizeof(index->startUrl), "%s%s",
|
||||
scheme, filenameIndex);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
@@ -1102,7 +1106,8 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
&& (!isalpha(previous_save_[0]) || previous_save_[1] != ':')) // c:/home/foo/bar.gif
|
||||
) {
|
||||
index->safeCache = 1;
|
||||
sprintf(previous_save, "%s%s", index->path, previous_save_);
|
||||
snprintf(previous_save, sizeof(previous_save), "%s%s",
|
||||
index->path, previous_save_);
|
||||
}
|
||||
// bogus format (includes buggy absolute path)
|
||||
else {
|
||||
@@ -1127,15 +1132,16 @@ static PT_Element PT_ReadCache__New_u(PT_Index index_, const char *url,
|
||||
int saveLen = (int) strlen(previous_save_);
|
||||
|
||||
if (index->fixedPath < saveLen) {
|
||||
sprintf(previous_save, "%s%s", index->path,
|
||||
previous_save_ + index->fixedPath);
|
||||
snprintf(previous_save, sizeof(previous_save), "%s%s",
|
||||
index->path, previous_save_ + index->fixedPath);
|
||||
} else {
|
||||
snprintf(r->msg, sizeof(r->msg), "Bogus fixePath prefix for %s (prefixLen=%d)",
|
||||
previous_save_, (int) index->fixedPath);
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
sprintf(previous_save, "%s%s", index->path, previous_save_);
|
||||
snprintf(previous_save, sizeof(previous_save), "%s%s",
|
||||
index->path, previous_save_);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1348,45 +1354,71 @@ static int PT_SaveCache__New(PT_Indexes indexes, const char *filename) {
|
||||
/* Old HTTrack cache (dat/ndx) format */
|
||||
/* ------------------------------------------------------------ */
|
||||
|
||||
static int cache_brstr(char *adr, char *s) {
|
||||
static int cache_brstr(char *adr, char *s, size_t s_size) {
|
||||
int i;
|
||||
int off;
|
||||
char buff[256 + 1];
|
||||
char buff[256 + 4];
|
||||
|
||||
off = binput(adr, buff, 256);
|
||||
adr += off;
|
||||
sscanf(buff, "%d", &i);
|
||||
if (i > 0)
|
||||
strncpy(s, adr, i);
|
||||
*(s + i) = '\0';
|
||||
off += i;
|
||||
/* binput stops at adr's terminating NUL; a value only follows a real line
|
||||
terminator, so never step past end-of-buffer. */
|
||||
if (adr[off - 1] == '\0') {
|
||||
s[0] = '\0';
|
||||
return off - 1;
|
||||
}
|
||||
/* an empty/non-numeric field leaves i unset: treat as length 0 */
|
||||
if (sscanf(buff, "%d", &i) != 1 || i < 0 || i > 32768)
|
||||
i = 0;
|
||||
if (i > 0) {
|
||||
/* a corrupt cache may declare a length past the buffer end; bound the copy
|
||||
and the advance to the bytes actually present */
|
||||
const size_t avail = strnlen(adr + off, (size_t) i);
|
||||
const size_t store = avail < s_size ? avail : s_size - 1;
|
||||
|
||||
memcpy(s, adr + off, store);
|
||||
s[store] = '\0';
|
||||
off += (int) avail;
|
||||
} else {
|
||||
s[0] = '\0';
|
||||
}
|
||||
return off;
|
||||
}
|
||||
|
||||
static void cache_rstr(FILE * fp, char *s) {
|
||||
INTsys i;
|
||||
static void cache_rstr(FILE *fp, char *s, size_t s_size) {
|
||||
INTsys i = 0;
|
||||
char buff[256 + 4];
|
||||
|
||||
linput(fp, buff, 256);
|
||||
sscanf(buff, INTsysP, &i);
|
||||
if (i < 0 || i > 32768) /* error, something nasty happened */
|
||||
/* an unmatched sscanf leaves i untouched: length 0, not a stack value */
|
||||
if (sscanf(buff, INTsysP, &i) != 1 || i < 0 || i > 32768)
|
||||
i = 0;
|
||||
if (i > 0) {
|
||||
if ((int) fread(s, 1, i, fp) != i) {
|
||||
/* store at most s_size-1 bytes, but consume all i bytes so the next field
|
||||
stays aligned when a tampered cache declares more than the destination
|
||||
holds */
|
||||
const size_t want = (size_t) i;
|
||||
const size_t store = want < s_size ? want : s_size - 1;
|
||||
|
||||
if (fread(s, 1, store, fp) != store) {
|
||||
assertf(! "fread_cache_failed");
|
||||
}
|
||||
if (want > store && fseek(fp, (long) (want - store), SEEK_CUR) != 0) {
|
||||
assertf(!"fseek_cache_failed");
|
||||
}
|
||||
s[store] = '\0';
|
||||
} else {
|
||||
s[0] = '\0';
|
||||
}
|
||||
*(s + i) = '\0';
|
||||
}
|
||||
|
||||
static char *cache_rstr_addr(FILE * fp) {
|
||||
INTsys i;
|
||||
INTsys i = 0;
|
||||
char *addr = NULL;
|
||||
char buff[256 + 4];
|
||||
|
||||
linput(fp, buff, 256);
|
||||
sscanf(buff, "%d", &i);
|
||||
if (i < 0 || i > 32768) /* error, something nasty happened */
|
||||
/* an unmatched sscanf leaves i untouched: length 0, not a stack value */
|
||||
if (sscanf(buff, INTsysP, &i) != 1 || i < 0 || i > 32768)
|
||||
i = 0;
|
||||
if (i > 0) {
|
||||
addr = malloc(i + 1);
|
||||
@@ -1400,19 +1432,31 @@ static char *cache_rstr_addr(FILE * fp) {
|
||||
return addr;
|
||||
}
|
||||
|
||||
/* Allocate n+1 bytes for an n-byte cache body, rejecting a hostile/corrupt
|
||||
length that would demand an absurd allocation or wrap n+1. NULL on refuse. */
|
||||
static char *cache_alloc_body(size_t n) {
|
||||
if (n >= (size_t) INT32_MAX)
|
||||
return NULL;
|
||||
return (char *) malloc(n + 1);
|
||||
}
|
||||
|
||||
static void cache_rint(FILE * fp, int *i) {
|
||||
char s[256];
|
||||
|
||||
cache_rstr(fp, s);
|
||||
sscanf(s, "%d", i);
|
||||
cache_rstr(fp, s, sizeof(s));
|
||||
if (sscanf(s, "%d", i) != 1)
|
||||
*i = 0;
|
||||
}
|
||||
|
||||
static void cache_rLLint(FILE * fp, unsigned long *i) {
|
||||
int l;
|
||||
char s[256];
|
||||
|
||||
cache_rstr(fp, s);
|
||||
sscanf(s, "%d", &l);
|
||||
cache_rstr(fp, s, sizeof(s));
|
||||
/* reject a non-numeric or negative field: a negative length cast to unsigned
|
||||
would wrap a later malloc(size+1) to a tiny buffer */
|
||||
if (sscanf(s, "%d", &l) != 1 || l < 0)
|
||||
l = 0;
|
||||
*i = (unsigned long) l;
|
||||
}
|
||||
|
||||
@@ -1472,12 +1516,12 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
char *a = use;
|
||||
|
||||
use[ndxSize] = '\0';
|
||||
a += cache_brstr(a, firstline);
|
||||
a += cache_brstr(a, firstline, sizeof(firstline));
|
||||
if (strncmp(firstline, "CACHE-", 6) == 0) { // Nouvelle version du cache
|
||||
if (strncmp(firstline, "CACHE-1.", 8) == 0) { // Version 1.1x
|
||||
cache->version = (int) (firstline[8] - '0'); // cache 1.x
|
||||
if (cache->version <= 5) {
|
||||
a += cache_brstr(a, firstline);
|
||||
a += cache_brstr(a, firstline, sizeof(firstline));
|
||||
strcpy(cache->lastmodified, firstline);
|
||||
} else {
|
||||
fclose(cache->dat);
|
||||
@@ -1523,11 +1567,12 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
|
||||
if (!firstSeen) {
|
||||
if (strstr(line, "/robots.txt") == NULL) {
|
||||
PT_Index__Old index = cache;
|
||||
const char *scheme =
|
||||
link_has_authority(line) ? "" : "http://";
|
||||
|
||||
firstSeen = 1;
|
||||
if (!link_has_authority(line))
|
||||
strcat(index->startUrl, "http://");
|
||||
strcat(index->startUrl, line);
|
||||
snprintf(index->startUrl, sizeof(index->startUrl), "%s%s",
|
||||
scheme, line);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1623,9 +1668,13 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
String urlDecoded;
|
||||
|
||||
r->statuscode = old_r.statuscode;
|
||||
r->size = old_r.size; // taille fichier
|
||||
strcpy(r->msg, old_r.msg);
|
||||
strcpy(r->contenttype, old_r.contenttype);
|
||||
r->size = old_r.size > 0 ? (size_t) old_r.size : 0; // taille fichier
|
||||
/* these fixed fields come straight from fread and carry no guaranteed
|
||||
NUL; terminate them before the bounded copy */
|
||||
old_r.msg[sizeof(old_r.msg) - 1] = '\0';
|
||||
old_r.contenttype[sizeof(old_r.contenttype) - 1] = '\0';
|
||||
strcpybuff(r->msg, old_r.msg);
|
||||
strcpybuff(r->contenttype, old_r.contenttype);
|
||||
|
||||
/* Guess the destination filename.. this sucks, because this method is not reliable.
|
||||
Yes, the old 1.0 cache format was *that* bogus. /rx */
|
||||
@@ -1666,26 +1715,27 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
cache_rint(cache->dat, &r->statuscode);
|
||||
cache_rLLint(cache->dat, &size_);
|
||||
r->size = (size_t) size_;
|
||||
cache_rstr(cache->dat, r->msg);
|
||||
cache_rstr(cache->dat, r->contenttype);
|
||||
cache_rstr(cache->dat, r->msg, sizeof(r->msg));
|
||||
cache_rstr(cache->dat, r->contenttype, sizeof(r->contenttype));
|
||||
if (cache->version >= 3)
|
||||
cache_rstr(cache->dat, r->charset);
|
||||
cache_rstr(cache->dat, r->lastmodified);
|
||||
cache_rstr(cache->dat, r->etag);
|
||||
cache_rstr(cache->dat, r->location);
|
||||
cache_rstr(cache->dat, r->charset, sizeof(r->charset));
|
||||
cache_rstr(cache->dat, r->lastmodified, sizeof(r->lastmodified));
|
||||
cache_rstr(cache->dat, r->etag, sizeof(r->etag));
|
||||
cache_rstr(cache->dat, r->location, sizeof(location_default));
|
||||
if (cache->version >= 2)
|
||||
cache_rstr(cache->dat, r->cdispo);
|
||||
cache_rstr(cache->dat, r->cdispo, sizeof(r->cdispo));
|
||||
if (cache->version >= 4) {
|
||||
cache_rstr(cache->dat, previous_save_); // adr
|
||||
cache_rstr(cache->dat, previous_save_); // fil
|
||||
cache_rstr(cache->dat, previous_save_, sizeof(previous_save_)); // adr
|
||||
cache_rstr(cache->dat, previous_save_, sizeof(previous_save_)); // fil
|
||||
previous_save[0] = '\0';
|
||||
cache_rstr(cache->dat, previous_save_); // save
|
||||
cache_rstr(cache->dat, previous_save_,
|
||||
sizeof(previous_save_)); // save
|
||||
}
|
||||
if (cache->version >= 5) {
|
||||
r->headers = cache_rstr_addr(cache->dat);
|
||||
}
|
||||
//
|
||||
cache_rstr(cache->dat, check);
|
||||
cache_rstr(cache->dat, check, sizeof(check));
|
||||
if (strcmp(check, "HTS") == 0) { /* intégrité OK */
|
||||
ok = 1;
|
||||
}
|
||||
@@ -1714,7 +1764,8 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
&& (!isalpha(previous_save_[0]) || previous_save_[1] != ':')) // c:/home/foo/bar.gif
|
||||
) {
|
||||
index->safeCache = 1;
|
||||
sprintf(previous_save, "%s%s", index->path, previous_save_);
|
||||
snprintf(previous_save, sizeof(previous_save), "%s%s", index->path,
|
||||
previous_save_);
|
||||
}
|
||||
// bogus format (includes buggy absolute path)
|
||||
else {
|
||||
@@ -1739,15 +1790,16 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
int saveLen = (int) strlen(previous_save_);
|
||||
|
||||
if (index->fixedPath < saveLen) {
|
||||
sprintf(previous_save, "%s%s", index->path,
|
||||
previous_save_ + index->fixedPath);
|
||||
snprintf(previous_save, sizeof(previous_save), "%s%s",
|
||||
index->path, previous_save_ + index->fixedPath);
|
||||
} else {
|
||||
snprintf(r->msg, sizeof(r->msg), "Bogus fixePath prefix for %s (prefixLen=%d)",
|
||||
previous_save_, (int) index->fixedPath);
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
}
|
||||
} else {
|
||||
sprintf(previous_save, "%s%s", index->path, previous_save_);
|
||||
snprintf(previous_save, sizeof(previous_save), "%s%s",
|
||||
index->path, previous_save_);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1764,7 +1816,7 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
FILE *fp = fopen(previous_save, "rb");
|
||||
|
||||
if (fp != NULL) {
|
||||
r->adr = (char *) malloc(r->size + 1);
|
||||
r->adr = cache_alloc_body(r->size);
|
||||
if (r->adr != NULL) {
|
||||
if (r->size > 0 && fread(r->adr, 1, r->size, fp) != r->size) {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
@@ -1784,7 +1836,7 @@ static PT_Element PT_ReadCache__Old_u(PT_Index index_, const char *url,
|
||||
} else {
|
||||
// lire fichier (d'un coup)
|
||||
if (flags & FETCH_BODY) {
|
||||
r->adr = (char *) malloc(r->size + 1);
|
||||
r->adr = cache_alloc_body(r->size);
|
||||
if (r->adr != NULL) {
|
||||
if (fread(r->adr, 1, r->size, cache->dat) != r->size) { // erreur
|
||||
free(r->adr);
|
||||
|
||||
@@ -23,7 +23,7 @@ conv 'iso-8859-1' 'café' 'café'
|
||||
|
||||
# windows-1252 is NOT latin-1: 0x80 is the euro sign, not U+0080. This is the
|
||||
# case that actually exercises the cp1252 table (the 0x80-0x9F range).
|
||||
conv 'windows-1252' $'\x80' '€'
|
||||
conv 'windows-1252' 'hex:80' '€'
|
||||
|
||||
# pure ASCII is charset-invariant
|
||||
conv 'us-ascii' 'hello' 'hello'
|
||||
@@ -34,8 +34,8 @@ conv 'no-such-charset-xyz' 'abc' 'abc'
|
||||
test "$(httrack -O /dev/null -#test=charset 'no-such-charset-xyz' 'café' 2>/dev/null)" == "" || exit 1
|
||||
|
||||
# malformed UTF-8 (lone continuation byte, truncated lead byte) must not crash
|
||||
runs 'utf-8' $'\x80'
|
||||
runs 'utf-8' $'\xc3'
|
||||
runs 'utf-8' 'hex:80'
|
||||
runs 'utf-8' 'hex:c3'
|
||||
|
||||
# hts_getCharsetFromMeta: <meta> charset extraction, HTML5 and legacy forms.
|
||||
# -#test=metacharset <html> prints the charset, or "(none)".
|
||||
@@ -89,11 +89,11 @@ utf8() {
|
||||
utf8 'plain' '1'
|
||||
utf8 'café' '1'
|
||||
utf8 '统计' '1'
|
||||
utf8 "$(printf 'caf\xe9')" '0' # latin-1 accent
|
||||
utf8 "$(printf '\xc3')" '0' # truncated lead byte
|
||||
utf8 "$(printf '\xa9')" '0' # lone continuation byte
|
||||
utf8 "$(printf '\xf0\x9f\x98\x80')" '1' # 4-byte emoji
|
||||
utf8 "$(printf '\xc0\xaf')" '0' # overlong (latin-1 "À¯" must stay convertible)
|
||||
utf8 "$(printf '\xed\xa0\x80')" '0' # UTF-16 surrogate
|
||||
utf8 "$(printf '\xf4\x90\x80\x80')" '0' # above U+10FFFF
|
||||
utf8 "$(printf '\xf8\x88\x80\x80\x80')" '0' # legacy 5-byte form
|
||||
utf8 'hex:636166e9' '0' # latin-1 accent
|
||||
utf8 'hex:c3' '0' # truncated lead byte
|
||||
utf8 'hex:a9' '0' # lone continuation byte
|
||||
utf8 'hex:f09f9880' '1' # 4-byte emoji
|
||||
utf8 'hex:c0af' '0' # overlong (latin-1 "À¯" must stay convertible)
|
||||
utf8 'hex:eda080' '0' # UTF-16 surrogate
|
||||
utf8 'hex:f4908080' '0' # above U+10FFFF
|
||||
utf8 'hex:f888808080' '0' # legacy 5-byte form
|
||||
|
||||
@@ -52,14 +52,15 @@ run "$tmp/dir" "$tmp/adir"
|
||||
loghas 'Could not include URL list "[^"]+": not a regular file'
|
||||
|
||||
# unreadable regular file: the fopen() errno arm fires, distinct from the
|
||||
# directory branch. Root bypasses mode 000, so skip it there.
|
||||
if test "$(id -u)" -ne 0; then
|
||||
: >"$tmp/noperm.txt"
|
||||
chmod 000 "$tmp/noperm.txt"
|
||||
# directory branch. Needs an OS that enforces mode 000: root bypasses it, and
|
||||
# Windows ignores chmod entirely, so probe rather than assume.
|
||||
: >"$tmp/noperm.txt"
|
||||
chmod 000 "$tmp/noperm.txt"
|
||||
if ! cat "$tmp/noperm.txt" >/dev/null 2>&1; then
|
||||
run "$tmp/perm" "$tmp/noperm.txt"
|
||||
chmod 644 "$tmp/noperm.txt"
|
||||
loghas 'Could not include URL list "[^"]+": .+'
|
||||
lognot 'not a regular file'
|
||||
fi
|
||||
chmod 644 "$tmp/noperm.txt"
|
||||
|
||||
exit 0
|
||||
|
||||
@@ -193,5 +193,13 @@ test "$(with_timeout httrack -O /dev/null -#test=filter "$pat" "$subj")" == "$su
|
||||
match '*[aX]X*[a]Y' 'aXaXaY'
|
||||
nomatch '*[aX]X*[a]Y' 'aXbXaY'
|
||||
|
||||
# hostile patterns must not stack-overflow or hang: length + work caps
|
||||
# hostile patterns must not stack-overflow or hang: length + depth + work caps
|
||||
test "$(with_timeout httrack -O /dev/null -#test=filterbounds)" == "filterbounds: OK" || exit 1
|
||||
|
||||
# and the depth cap must hold on the 1MB stack a Windows thread gets (#574):
|
||||
# uncapped, the same pattern recursed ~900KB deep. Skipped where 512K is too
|
||||
# small to even start the binary.
|
||||
if (ulimit -s 512 && httrack --version >/dev/null 2>&1); then
|
||||
out=$(ulimit -s 512 && with_timeout httrack -O /dev/null -#test=filterbounds)
|
||||
test "$out" == "filterbounds: OK" || exit 1
|
||||
fi
|
||||
|
||||
18
tests/01_engine-fsize.test
Normal file
18
tests/01_engine-fsize.test
Normal file
@@ -0,0 +1,18 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# File sizes past the 32-bit wrap ('httrack -#test=fsize <dir>'). 5GB, so a
|
||||
# signed *and* an unsigned truncation both show up; sparse, so it costs no disk.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
out=$(httrack -#test=fsize "$dir")
|
||||
echo "$out"
|
||||
|
||||
want="fsize: width=8,8 size=5368709120,5368709120 psize=5368709120 absent=-1"
|
||||
test "$out" == "$want" || {
|
||||
echo "FAIL: unexpected size report (want '$want')"
|
||||
exit 1
|
||||
}
|
||||
@@ -38,4 +38,7 @@ dec 'xn--!!!' '㚯'
|
||||
dec 'xn--already-encoded.com' 'ǮalǭǮreaǭǫdy.com'
|
||||
|
||||
# second label has a truncated UTF-8 sequence: used to leak the segment buffer.
|
||||
enc_bad "$(printf 'b\xc3\xbcchev.\xe4\xbe\x8b\xe5\xad\x62\xc3\xbccheple')"
|
||||
# Fed as hex: the raw bytes must reach the encoder intact, but invalid UTF-8
|
||||
# cannot survive a Windows command line (it round-trips through UTF-16 to valid
|
||||
# U+FFFD, which then encodes fine instead of being rejected).
|
||||
enc_bad 'hex:62c3bc636865762ee4be8be5ad62c3bc636865706c65'
|
||||
|
||||
@@ -32,6 +32,13 @@ esac
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_rcfile.XXXXXX") || exit 1
|
||||
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# HTS_HTTRACKRC is ".httrackrc" on POSIX but "httrackrc" on Windows: write both,
|
||||
# each platform reads the one it knows.
|
||||
write_rc() {
|
||||
cat >"$1/.httrackrc"
|
||||
cp "$1/.httrackrc" "$1/httrackrc"
|
||||
}
|
||||
|
||||
# --- 1. alias token survives the bound intact -------------------------------
|
||||
d1="$tmp/intact"
|
||||
mkdir -p "$d1"
|
||||
@@ -40,7 +47,7 @@ echo '<html><body>hello</body></html>' >"$d1/index.html"
|
||||
# optinclude_file() lowercases each config line, so the marker is lowercase to
|
||||
# survive the comparison verbatim.
|
||||
marker='zzz_rcfile_marker_0123456789_abcdefghijklmnopqrstuvwxyz_intact'
|
||||
printf 'user-agent=%s\n' "$marker" >"$d1/.httrackrc"
|
||||
printf 'user-agent=%s\n' "$marker" | write_rc "$d1"
|
||||
|
||||
# Run with no -O so the working-directory .httrackrc is loaded (an -O path makes
|
||||
# the engine skip the rc files). Output lands in the temp dir. Guard the run so a
|
||||
@@ -73,7 +80,7 @@ echo '<html><body>hi</body></html>' >"$d2/index.html"
|
||||
val=$(printf 'a%.0s' $(seq 1 200))
|
||||
for _ in $(seq 1 400); do
|
||||
printf 'user-agent=%s\n' "$val"
|
||||
done >"$d2/.httrackrc"
|
||||
done | write_rc "$d2"
|
||||
|
||||
# The process aborts (httrack turns the fatal signal into exit 134 either way),
|
||||
# so the exit code does not distinguish the bounded abort from a heap overflow;
|
||||
|
||||
@@ -35,7 +35,8 @@ chk image/avif hex:0000001C6674797061766966 "$yes"
|
||||
chk image/avif hex:0000001C6674797068656963 "$no" # heic brand is not avif
|
||||
chk image/heic hex:0000001C6674797068656963 "$yes"
|
||||
chk image/svg+xml '<svg xmlns="x">' "$yes"
|
||||
chk image/svg+xml $'\xef\xbb\xbf <?xml version="1.0"?>' "$yes" # BOM+ws skip
|
||||
# BOM+ws skip; hex, as Windows argv cannot carry the raw BOM through the ANSI codepage
|
||||
chk image/svg+xml hex:EFBBBF20203C3F786D6C2076657273696F6E3D22312E30223F3E "$yes"
|
||||
|
||||
# audio / video
|
||||
chk audio/mpeg 'ID3xxx' "$yes"
|
||||
|
||||
9
tests/01_engine-socks5.test
Normal file
9
tests/01_engine-socks5.test
Normal file
@@ -0,0 +1,9 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# The SOCKS5 handshake framing and credential split, driven against scripted
|
||||
# server replies (frame draining, oversize rejects, RFC 1929 fields).
|
||||
|
||||
httrack -O /dev/null '-#test=socks5' | grep -q "socks5 self-test OK"
|
||||
@@ -32,10 +32,10 @@ case "$err" in
|
||||
esac
|
||||
|
||||
# Same guarantee for the htsbuff builder. The source is exactly the buffer
|
||||
# capacity (4 bytes into a 4-byte buffer), so this also pins the boundary: a
|
||||
# capacity (6 bytes into a 6-byte buffer), so this also pins the boundary: a
|
||||
# '<=' off-by-one in the capacity check would let it through (and print "NOT
|
||||
# aborted"). Match the specific htsbuff abort message, not just any assert.
|
||||
err=$(httrack -#test=strsafe overflow-buff "abcd" 2>&1) || true
|
||||
err=$(httrack -#test=strsafe overflow-buff "abcdef" 2>&1) || true
|
||||
case "$err" in
|
||||
*"strsafe: NOT aborted"*)
|
||||
echo "htsbuff over-capacity write was NOT caught" >&2
|
||||
|
||||
211
tests/52_local-socks5.test
Normal file
211
tests/52_local-socks5.test
Normal file
@@ -0,0 +1,211 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Issue #563: a SOCKS5 proxy must carry both http and https, resolve the origin
|
||||
# name REMOTELY (ATYP=domain, never a local lookup), and authenticate per RFC
|
||||
# 1929 without leaking the credentials to the origin.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
if test "${HTTPS_SUPPORT:-}" == "no"; then
|
||||
echo "no https support compiled, skipping"
|
||||
exit 77
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1 || ! command -v openssl >/dev/null 2>&1; then
|
||||
echo "python3/openssl missing, skipping"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
# a .invalid name never resolves (RFC 6761): reaching the page at all proves the
|
||||
# proxy did the DNS
|
||||
host="socks-origin.invalid"
|
||||
server="$top_srcdir/tests/socks5-server.py"
|
||||
tmpdir=$(mktemp -d)
|
||||
pids=
|
||||
|
||||
cleanup() {
|
||||
for pid in $pids; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
done
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
openssl req -x509 -newkey rsa:2048 -keyout "$tmpdir/key.pem" \
|
||||
-out "$tmpdir/cert.pem" -days 2 -nodes -subj "/CN=127.0.0.1" \
|
||||
>/dev/null 2>&1
|
||||
cat "$tmpdir/key.pem" "$tmpdir/cert.pem" >"$tmpdir/both.pem"
|
||||
|
||||
# start_server <logdir> <mode>: sets $tls_port/$http_port/$socks_port
|
||||
start_server() {
|
||||
local dir="$1" mode="$2" ports
|
||||
mkdir -p "$dir"
|
||||
ports="$dir/ports.txt"
|
||||
python3 "$server" "$tmpdir/both.pem" "$dir" "$mode" \
|
||||
>"$ports" 2>"$dir/server.err" &
|
||||
pids="$pids $!"
|
||||
for _ in $(seq 1 100); do
|
||||
grep -q "^ready" "$ports" 2>/dev/null && break
|
||||
sleep 0.1
|
||||
done
|
||||
grep -q "^ready" "$ports" 2>/dev/null || {
|
||||
echo "server ($mode) did not start" >&2
|
||||
cat "$dir/server.err" >&2
|
||||
exit 1
|
||||
}
|
||||
tls_port=$(awk '/^TLS/{print $2}' "$ports")
|
||||
http_port=$(awk '/^HTTP/{print $2}' "$ports")
|
||||
socks_port=$(awk '/^SOCKS/{print $2}' "$ports")
|
||||
}
|
||||
|
||||
# kill a hung httrack so a missing read bound surfaces as $HANG_RC instead of
|
||||
# stalling the job (macOS has no `timeout`)
|
||||
HANG_RC=137 # 128 + SIGKILL
|
||||
run_crawl() {
|
||||
local out="$1" url="$2" proxy="$3"
|
||||
shift 3
|
||||
local extra="$*"
|
||||
test -n "$extra" || extra="-r1 -s0"
|
||||
rm -rf "$out"
|
||||
# shellcheck disable=SC2086
|
||||
httrack "$url" --proxy "$proxy" \
|
||||
-O "$out" $extra --timeout=10 >"$out.log" 2>&1 &
|
||||
local pid=$!
|
||||
(sleep 60 && kill -9 "$pid" 2>/dev/null) &
|
||||
local guard=$!
|
||||
local rc=0
|
||||
wait "$pid" 2>/dev/null || rc=$?
|
||||
kill "$guard" 2>/dev/null || true
|
||||
wait "$guard" 2>/dev/null || true
|
||||
return "$rc"
|
||||
}
|
||||
|
||||
# --- https over socks, remote DNS -------------------------------------------
|
||||
ok="$tmpdir/ok"
|
||||
start_server "$ok" ok
|
||||
|
||||
run_crawl "$ok/out" "https://${host}:${tls_port}/" "socks5://127.0.0.1:${socks_port}"
|
||||
grep -rq "ORIGIN-PAGE-563" "$ok/out" || {
|
||||
echo "FAIL: origin page not downloaded through the socks proxy" >&2
|
||||
cat "$ok/out.log" "$ok/server.err" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -q "^ATYP domain ${host}\$" "$ok/socks.log" || {
|
||||
echo "FAIL: proxy did not receive ATYP=domain ${host} (no remote DNS)" >&2
|
||||
cat "$ok/socks.log" >&2
|
||||
exit 1
|
||||
}
|
||||
# a client that resolved locally would have sent an address, not a name
|
||||
grep -q "^ATYP ipv4" "$ok/socks.log" && {
|
||||
echo "FAIL: httrack resolved locally and sent an IPv4 to the proxy" >&2
|
||||
cat "$ok/socks.log" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK: https tunneled through socks5 with remote DNS"
|
||||
|
||||
# --- plain http over socks --------------------------------------------------
|
||||
: >"$ok/origin-headers.log" # the https crawl above also logged a request line
|
||||
run_crawl "$ok/outh" "http://${host}:${http_port}/" "socks5://127.0.0.1:${socks_port}"
|
||||
grep -rq "ORIGIN-PAGE-563" "$ok/outh" || {
|
||||
echo "FAIL: plain http not tunneled through the socks proxy" >&2
|
||||
cat "$ok/outh.log" "$ok/socks.log" >&2
|
||||
exit 1
|
||||
}
|
||||
# the tunneled request is origin-form, as on a direct connection: the http-proxy
|
||||
# absolute URI would reach the origin as "GET http://host/"
|
||||
grep -q "^REQUEST GET / " "$ok/origin-headers.log" || {
|
||||
echo "FAIL: tunneled request is not origin-form" >&2
|
||||
cat "$ok/origin-headers.log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -q "^REQUEST GET http" "$ok/origin-headers.log" && {
|
||||
echo "FAIL: absolute-URI request sent through the socks tunnel" >&2
|
||||
cat "$ok/origin-headers.log" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK: plain http tunneled through socks5"
|
||||
|
||||
# --- keep-alive socket reuse must NOT re-run the handshake ------------------
|
||||
# A reused keep-alive socket is already tunneled; re-injecting the SOCKS
|
||||
# greeting corrupts the origin stream. -c1 + the origin's max>1 advert makes
|
||||
# httrack serve the whole plain-http crawl over one tunnel: exactly one
|
||||
# handshake, every subpage intact.
|
||||
ka="$tmpdir/ka"
|
||||
start_server "$ka" ok
|
||||
run_crawl "$ka/out" "http://${host}:${http_port}/" "socks5://127.0.0.1:${socks_port}" -r3 -c1
|
||||
missing=0
|
||||
for i in 1 2 3 4 5; do
|
||||
grep -rq "SUBPAGE-563 /p${i}.html" "$ka/out" || missing=1
|
||||
done
|
||||
test "$missing" -eq 0 || {
|
||||
echo "FAIL: a keep-alive subpage was lost (re-handshake corrupted the reused socket)" >&2
|
||||
cat "$ka/out.log" "$ka/socks.log" >&2
|
||||
exit 1
|
||||
}
|
||||
handshakes=$(grep -c "^CMD " "$ka/socks.log" || true)
|
||||
test "$handshakes" -eq 1 || {
|
||||
echo "FAIL: expected 1 SOCKS handshake for the reused socket, got $handshakes" >&2
|
||||
cat "$ka/socks.log" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK: keep-alive socket reused over one socks tunnel, no re-handshake"
|
||||
|
||||
# --- authenticated socks (RFC 1929) -----------------------------------------
|
||||
# over plain http: that is the request an http proxy would decorate with
|
||||
# Proxy-Authorization, so the no-leak check below has teeth
|
||||
auth="$tmpdir/auth"
|
||||
start_server "$auth" ok
|
||||
run_crawl "$auth/out" "http://${host}:${http_port}/" \
|
||||
"socks5://user:secret@127.0.0.1:${socks_port}"
|
||||
grep -rq "ORIGIN-PAGE-563" "$auth/out" || {
|
||||
echo "FAIL: origin not downloaded through the authenticated socks proxy" >&2
|
||||
cat "$auth/out.log" "$auth/socks.log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -q "^METHOD userpass\$" "$auth/socks.log" || {
|
||||
echo "FAIL: user/pass method not negotiated (auth ignored)" >&2
|
||||
cat "$auth/socks.log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -q "^AUTH user secret\$" "$auth/socks.log" || {
|
||||
echo "FAIL: wrong socks credentials sent" >&2
|
||||
cat "$auth/socks.log" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -qi "secret\|proxy-authorization" "$auth/origin-headers.log" && {
|
||||
echo "FAIL: socks credentials leaked into the origin request" >&2
|
||||
cat "$auth/origin-headers.log" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK: socks5 user/pass negotiated, creds not leaked to origin"
|
||||
|
||||
# --- hostile: refusing proxy ------------------------------------------------
|
||||
ref="$tmpdir/refuse"
|
||||
start_server "$ref" refuse
|
||||
rc=0
|
||||
run_crawl "$ref/out" "https://${host}:${tls_port}/" "socks5://127.0.0.1:${socks_port}" || rc=$?
|
||||
test "$rc" -ne "$HANG_RC" || {
|
||||
echo "FAIL: crawl hung on a refusing socks proxy" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -rq "ORIGIN-PAGE-563" "$ref/out" 2>/dev/null && {
|
||||
echo "FAIL: refusing proxy unexpectedly served the page" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK: refusing socks proxy fails cleanly, no hang"
|
||||
|
||||
# --- hostile: truncated handshake reply -------------------------------------
|
||||
trunc="$tmpdir/trunc"
|
||||
start_server "$trunc" truncate
|
||||
rc=0
|
||||
run_crawl "$trunc/out" "https://${host}:${tls_port}/" "socks5://127.0.0.1:${socks_port}" || rc=$?
|
||||
test "$rc" -ne "$HANG_RC" || {
|
||||
echo "FAIL: crawl hung on a truncated socks reply (bounded read missing)" >&2
|
||||
exit 1
|
||||
}
|
||||
grep -rq "ORIGIN-PAGE-563" "$trunc/out" 2>/dev/null && {
|
||||
echo "FAIL: truncated-reply proxy unexpectedly served the page" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK: bounded socks handshake, no hang on a truncated reply"
|
||||
21
tests/53_local-proxytrack-cache-corrupt.test
Normal file
21
tests/53_local-proxytrack-cache-corrupt.test
Normal file
@@ -0,0 +1,21 @@
|
||||
#!/bin/bash
|
||||
# A corrupt proxytrack .ndx must not overflow the fixed cache-read buffers. The
|
||||
# sanitizer CI build turns any regression here into a hard stack/heap overflow.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
dir=$(mktemp -d)
|
||||
trap 'rm -rf "$dir"' EXIT
|
||||
|
||||
# The first length-prefixed field declares far more than firstline[256] holds;
|
||||
# cache_brstr must clamp the copy to the destination, not the declared length.
|
||||
{
|
||||
printf '4000\n'
|
||||
head -c 4000 /dev/zero | tr '\0' 'A'
|
||||
} >"$dir/foo.ndx"
|
||||
: >"$dir/foo.dat" # cache_brstr runs only when the sibling .dat opens
|
||||
|
||||
proxytrack --convert "$dir/out.arc" "$dir/foo.ndx" >/dev/null 2>&1 || {
|
||||
echo "FAIL: proxytrack crashed/errored on a corrupt cache index"
|
||||
exit 1
|
||||
}
|
||||
@@ -2,7 +2,7 @@
|
||||
# explicitly: automake does not expand wildcards in EXTRA_DIST, so a glob would
|
||||
# silently drop it from the dist tarball and break "make distcheck".
|
||||
EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
|
||||
proxy-https-server.py \
|
||||
proxy-https-server.py socks5-server.py \
|
||||
local-crawl.sh local-server.py server.crt server.key \
|
||||
server-root/simple/basic.html server-root/simple/link.html \
|
||||
server-root/stripquery/index.html server-root/stripquery/a.html \
|
||||
@@ -47,6 +47,7 @@ TESTS = \
|
||||
01_engine-idna.test \
|
||||
01_engine-identurl.test \
|
||||
01_engine-proxyurl.test \
|
||||
01_engine-socks5.test \
|
||||
01_engine-identabs.test \
|
||||
01_engine-escape-room.test \
|
||||
01_engine-inplace-escape.test \
|
||||
@@ -56,6 +57,7 @@ TESTS = \
|
||||
01_engine-pause.test \
|
||||
01_engine-rcfile.test \
|
||||
01_engine-reconcile.test \
|
||||
01_engine-fsize.test \
|
||||
01_engine-redirect.test \
|
||||
01_engine-relative.test \
|
||||
01_engine-robots.test \
|
||||
@@ -126,6 +128,8 @@ TESTS = \
|
||||
48_local-crange-memresume.test \
|
||||
49_local-cookiewall.test \
|
||||
50_local-contentcodings.test \
|
||||
51_local-update-codec.test
|
||||
51_local-update-codec.test \
|
||||
52_local-socks5.test \
|
||||
53_local-proxytrack-cache-corrupt.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
228
tests/socks5-server.py
Normal file
228
tests/socks5-server.py
Normal file
@@ -0,0 +1,228 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Local SOCKS5 proxy (RFC 1928/1929) + TLS and plain HTTP origins for #563.
|
||||
|
||||
Logs the auth method, any user/pass, and the CONNECT ATYP + literal domain, so
|
||||
the test proves httrack sent a hostname (remote DNS) and negotiated auth.
|
||||
Modes (argv[3]): ok | refuse (REP=5) | truncate (partial reply, must not hang).
|
||||
Usage: socks5-server.py <cert.pem> <logdir> [mode]
|
||||
Prints "TLS <port>", "HTTP <port>", "SOCKS <port>", "ready" on stdout.
|
||||
"""
|
||||
import http.server
|
||||
import os
|
||||
import socket
|
||||
import socketserver
|
||||
import ssl
|
||||
import struct
|
||||
import sys
|
||||
import threading
|
||||
|
||||
# The one name the proxy answers for; a .invalid TLD never resolves (RFC 6761),
|
||||
# so a locally-resolving client could not reach us -- success proves remote DNS.
|
||||
REMOTE_HOST = b"socks-origin.invalid"
|
||||
# index links the subpages so an -r3 crawl reuses one keep-alive socket
|
||||
LINKS = "".join('<a href="/p%d.html">%d</a>' % (i, i) for i in range(1, 6))
|
||||
ORIGIN_BODY = ("<html><body>ORIGIN-PAGE-563 " + LINKS + "</body></html>").encode()
|
||||
SOCKS_LOG = "socks.log"
|
||||
ORIGIN_LOG = "origin-headers.log"
|
||||
|
||||
|
||||
def make_origin(logdir):
|
||||
class Origin(http.server.BaseHTTPRequestHandler):
|
||||
# HTTP/1.1 + a max>1 advertisement lets httrack keep the socket alive
|
||||
protocol_version = "HTTP/1.1"
|
||||
|
||||
def do_GET(self):
|
||||
with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle:
|
||||
handle.write(
|
||||
"REQUEST %s %s %s\n"
|
||||
% (self.command, self.path, self.request_version)
|
||||
)
|
||||
for key in self.headers.keys():
|
||||
handle.write(key + ": " + self.headers[key] + "\n")
|
||||
if "p" in self.path and ".html" in self.path:
|
||||
body = (
|
||||
b"<html><body>SUBPAGE-563 " + self.path.encode() + b"</body></html>"
|
||||
)
|
||||
else:
|
||||
body = ORIGIN_BODY
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.send_header("Keep-Alive", "timeout=15, max=100")
|
||||
self.send_header("Connection", "keep-alive")
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
return Origin
|
||||
|
||||
|
||||
class ThreadingTCPServer(socketserver.ThreadingTCPServer):
|
||||
allow_reuse_address = True
|
||||
daemon_threads = True
|
||||
|
||||
|
||||
def start_origin(logdir, certfile=None):
|
||||
httpd = ThreadingTCPServer(("127.0.0.1", 0), make_origin(logdir))
|
||||
if certfile is not None:
|
||||
ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
|
||||
ctx.load_cert_chain(certfile)
|
||||
httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
|
||||
port = httpd.socket.getsockname()[1]
|
||||
threading.Thread(target=httpd.serve_forever, daemon=True).start()
|
||||
return port
|
||||
|
||||
|
||||
def recvn(conn, n):
|
||||
buf = b""
|
||||
while len(buf) < n:
|
||||
chunk = conn.recv(n - len(buf))
|
||||
if not chunk:
|
||||
raise OSError("short read")
|
||||
buf += chunk
|
||||
return buf
|
||||
|
||||
|
||||
def pipe(src, dst):
|
||||
try:
|
||||
while True:
|
||||
data = src.recv(65536)
|
||||
if not data:
|
||||
break
|
||||
dst.sendall(data)
|
||||
except OSError:
|
||||
pass
|
||||
finally:
|
||||
for sock in (src, dst):
|
||||
try:
|
||||
sock.shutdown(socket.SHUT_RDWR)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def log(logdir, line):
|
||||
with open(os.path.join(logdir, SOCKS_LOG), "a") as handle:
|
||||
handle.write(line + "\n")
|
||||
|
||||
|
||||
def negotiate_auth(conn, logdir):
|
||||
"""RFC 1928 greeting + RFC 1929 sub-negotiation. Returns True to proceed."""
|
||||
ver, nmethods = recvn(conn, 2)
|
||||
if ver != 0x05:
|
||||
return False
|
||||
methods = recvn(conn, nmethods)
|
||||
if 0x02 in methods: # prefer user/pass so the auth test exercises RFC 1929
|
||||
conn.sendall(b"\x05\x02")
|
||||
(subver,) = recvn(conn, 1)
|
||||
(ulen,) = recvn(conn, 1)
|
||||
uname = recvn(conn, ulen)
|
||||
(plen,) = recvn(conn, 1)
|
||||
passwd = recvn(conn, plen)
|
||||
log(logdir, "METHOD userpass")
|
||||
log(logdir, "AUTH %s %s" % (uname.decode(), passwd.decode()))
|
||||
conn.sendall(b"\x01\x00") # RFC 1929 success
|
||||
elif 0x00 in methods:
|
||||
conn.sendall(b"\x05\x00")
|
||||
log(logdir, "METHOD noauth")
|
||||
else:
|
||||
conn.sendall(b"\x05\xff") # no acceptable method
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
def read_request(conn, logdir):
|
||||
"""RFC 1928 CONNECT. Logs CMD, ATYP, domain/addr, port. Returns dest port."""
|
||||
ver, cmd, _rsv, atyp = recvn(conn, 4)
|
||||
log(logdir, "CMD %d" % cmd)
|
||||
if atyp == 0x01:
|
||||
addr = socket.inet_ntoa(recvn(conn, 4))
|
||||
log(logdir, "ATYP ipv4 %s" % addr)
|
||||
elif atyp == 0x03:
|
||||
(dlen,) = recvn(conn, 1)
|
||||
domain = recvn(conn, dlen)
|
||||
log(logdir, "ATYP domain %s" % domain.decode())
|
||||
elif atyp == 0x04:
|
||||
recvn(conn, 16)
|
||||
log(logdir, "ATYP ipv6")
|
||||
domain = b""
|
||||
else:
|
||||
return None, None
|
||||
(port,) = struct.unpack(">H", recvn(conn, 2))
|
||||
log(logdir, "PORT %d" % port)
|
||||
name = domain if atyp == 0x03 else b""
|
||||
return name, port
|
||||
|
||||
|
||||
def reply(conn, rep):
|
||||
conn.sendall(bytes([0x05, rep, 0x00, 0x01]) + b"\x00\x00\x00\x00" + b"\x00\x00")
|
||||
|
||||
|
||||
def handle_socks(conn, logdir, mode):
|
||||
try:
|
||||
if not negotiate_auth(conn, logdir):
|
||||
conn.close()
|
||||
return
|
||||
if mode == "truncate":
|
||||
# one byte of the 10-byte reply, then hold the socket: the client
|
||||
# must bound this handshake read and give up, not hang.
|
||||
conn.sendall(b"\x05")
|
||||
threading.Event().wait()
|
||||
return
|
||||
name, port = read_request(conn, logdir)
|
||||
if mode == "refuse" or name != REMOTE_HOST or port is None:
|
||||
reply(conn, 0x05) # connection refused
|
||||
conn.close()
|
||||
return
|
||||
try:
|
||||
upstream = socket.create_connection(("127.0.0.1", port))
|
||||
except OSError:
|
||||
reply(conn, 0x05)
|
||||
conn.close()
|
||||
return
|
||||
reply(conn, 0x00) # succeeded
|
||||
threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start()
|
||||
pipe(upstream, conn)
|
||||
except OSError:
|
||||
try:
|
||||
conn.close()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
def start_socks(logdir, mode):
|
||||
srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
|
||||
srv.bind(("127.0.0.1", 0))
|
||||
srv.listen(16)
|
||||
port = srv.getsockname()[1]
|
||||
|
||||
def serve():
|
||||
while True:
|
||||
conn, _ = srv.accept()
|
||||
threading.Thread(
|
||||
target=handle_socks, args=(conn, logdir, mode), daemon=True
|
||||
).start()
|
||||
|
||||
threading.Thread(target=serve, daemon=True).start()
|
||||
return port
|
||||
|
||||
|
||||
def main():
|
||||
certfile, logdir = sys.argv[1], sys.argv[2]
|
||||
mode = sys.argv[3] if len(sys.argv) > 3 else "ok"
|
||||
for name in (SOCKS_LOG, ORIGIN_LOG):
|
||||
open(os.path.join(logdir, name), "w").close()
|
||||
tls_port = start_origin(logdir, certfile)
|
||||
http_port = start_origin(logdir, None)
|
||||
socks_port = start_socks(logdir, mode)
|
||||
print("TLS %d" % tls_port, flush=True)
|
||||
print("HTTP %d" % http_port, flush=True)
|
||||
print("SOCKS %d" % socks_port, flush=True)
|
||||
print("ready", flush=True)
|
||||
threading.Event().wait()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user