mirror of
https://github.com/xroche/httrack.git
synced 2026-07-12 03:46:58 +03:00
Compare commits
2 Commits
master
...
fix-c7-304
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
9e04bc4be3 | ||
|
|
a40265fab6 |
@@ -3611,7 +3611,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) { // 'OK'
|
||||
if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) { // not HTML
|
||||
if (strnotempty(back[i].url_sav)) { // target found
|
||||
off_t size = fsize_utf8(back[i].url_sav);
|
||||
int size = fsize_utf8(back[i].url_sav); // target size
|
||||
|
||||
if (size >= 0) {
|
||||
if (back[i].r.totalsize == size) { // same size!
|
||||
@@ -3839,7 +3839,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
const hts_boolean range_ok =
|
||||
back[i].r.crange > 0 && resume >= 0 &&
|
||||
resume <= (LLint) sz &&
|
||||
back[i].r.crange_end == back[i].r.crange - 1 &&
|
||||
back[i].r.crange_end + 1 == back[i].r.crange &&
|
||||
(back[i].r.totalsize < 0 ||
|
||||
back[i].r.totalsize ==
|
||||
back[i].r.crange_end - resume + 1);
|
||||
@@ -3908,20 +3908,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
|
||||
if (fp) {
|
||||
LLint alloc_mem = resume + 1;
|
||||
|
||||
// Bound the in-memory buffer to a 32-bit size (real
|
||||
// in-RAM resources are far smaller); a hostile
|
||||
// Content-Length that would overflow the add or the
|
||||
// (size_t) cast is dropped and refetched instead.
|
||||
if (back[i].r.totalsize > INT32_MAX - alloc_mem) {
|
||||
url_savename_refname_remove(opt, back[i].url_adr,
|
||||
back[i].url_fil);
|
||||
UNLINK(back[i].url_sav);
|
||||
alloc_mem = -1;
|
||||
} else if (back[i].r.totalsize >= 0)
|
||||
if (back[i].r.totalsize >= 0)
|
||||
alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
|
||||
if (alloc_mem >= 0 && deleteaddr(&back[i].r) &&
|
||||
(back[i].r.adr =
|
||||
(char *) malloct((size_t) alloc_mem))) {
|
||||
if (deleteaddr(&back[i].r)
|
||||
&& (back[i].r.adr =
|
||||
(char *) malloct((size_t) alloc_mem))) {
|
||||
back[i].r.size = resume;
|
||||
if (back[i].r.totalsize >= 0)
|
||||
back[i].r.totalsize += resume; // -> full size
|
||||
|
||||
17
src/htslib.c
17
src/htslib.c
@@ -1586,7 +1586,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
a = strchr(rcvd + p, '/');
|
||||
if (a != NULL) {
|
||||
a++;
|
||||
if (sscanf(a, LLintP, &retour->crange) == 1 && retour->crange >= 0) {
|
||||
if (sscanf(a, LLintP, &retour->crange) == 1) {
|
||||
retour->crange_start = 0;
|
||||
retour->crange_end = retour->crange - 1;
|
||||
} else {
|
||||
@@ -1594,12 +1594,6 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
|
||||
}
|
||||
}
|
||||
}
|
||||
// A valid Content-Range has no negative field; reject hostile values so
|
||||
// the crange +/- 1 arithmetic downstream cannot sign-overflow (UB).
|
||||
if (retour->crange_start < 0 || retour->crange_end < 0 ||
|
||||
retour->crange < 0) {
|
||||
retour->crange_start = retour->crange_end = retour->crange = 0;
|
||||
}
|
||||
}
|
||||
} else if ((p = strfield(rcvd, "Connection:")) != 0) {
|
||||
char *a = rcvd + p;
|
||||
@@ -2004,15 +1998,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
|
||||
|
||||
if (bufl > 0) {
|
||||
if (!r->is_write) { // stocker en mémoire
|
||||
// In-memory content must fit a 32-bit index (allocs below add 1, reads
|
||||
// use int offsets): reject a hostile Content-Length or endless stream.
|
||||
const LLint inmem_want =
|
||||
(r->totalsize >= 0) ? r->totalsize : (r->size + bufl);
|
||||
if (inmem_want >= INT32_MAX) {
|
||||
r->statuscode = STATUSCODE_INVALID;
|
||||
strcpybuff(r->msg, "In-memory content too large");
|
||||
return READ_ERROR;
|
||||
}
|
||||
if (r->totalsize >= 0) { // totalsize déterminé ET ALLOUE
|
||||
if (r->adr == NULL) {
|
||||
r->adr = (char *) malloct((size_t) r->totalsize + 1);
|
||||
|
||||
@@ -1310,82 +1310,6 @@ static int st_headerlong(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* http_xfread1 must refuse an in-memory buffer whose size would exceed a 32-bit
|
||||
index (hostile Content-Length or endless stream) rather than allocate it.
|
||||
The guard returns before any socket read, so no real connection is needed. */
|
||||
static int st_xfread_limit(httrackp *opt, int argc, char **argv) {
|
||||
htsblk r;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
|
||||
// Content-Length just over 2 GiB.
|
||||
memset(&r, 0, sizeof(r));
|
||||
r.soc = INVALID_SOCKET;
|
||||
r.totalsize = (LLint) INT32_MAX + 1;
|
||||
printf("bylen: refused=%d adr=%s msg=%s\n",
|
||||
http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
|
||||
r.msg);
|
||||
if (r.adr != NULL)
|
||||
freet(r.adr);
|
||||
|
||||
// Unknown length, buffer already at the limit: the next read would exceed it.
|
||||
memset(&r, 0, sizeof(r));
|
||||
r.soc = INVALID_SOCKET;
|
||||
r.totalsize = -1;
|
||||
r.size = (LLint) INT32_MAX;
|
||||
printf("bygrow: refused=%d adr=%s msg=%s\n",
|
||||
http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
|
||||
r.msg);
|
||||
if (r.adr != NULL)
|
||||
freet(r.adr);
|
||||
|
||||
// Exactly at the 2 GiB index (size + bufl == INT32_MAX): must also be
|
||||
// refused, since the reallocs below add 1 (a `> INT32_MAX` check would let
|
||||
// this through and overflow the int realloc size).
|
||||
memset(&r, 0, sizeof(r));
|
||||
r.soc = INVALID_SOCKET;
|
||||
r.totalsize = -1;
|
||||
r.size = (LLint) INT32_MAX - 8192;
|
||||
http_xfread1(&r, 8192);
|
||||
printf("boundary: msg=%s\n", r.msg);
|
||||
|
||||
// A legitimate small size must NOT be refused by the guard (the read then
|
||||
// fails on the invalid socket, but the size-too-large msg must not be set).
|
||||
memset(&r, 0, sizeof(r));
|
||||
r.soc = INVALID_SOCKET;
|
||||
r.totalsize = 1000;
|
||||
http_xfread1(&r, 8192);
|
||||
printf("accept: msg=%s\n", r.msg);
|
||||
if (r.adr != NULL)
|
||||
freet(r.adr);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Parse a Content-Range header and print the sanitized triple. A hostile value
|
||||
(negative or INT64 extreme) must clamp to 0 without signed-overflow UB. */
|
||||
static int st_crange(httrackp *opt, int argc, char **argv) {
|
||||
int i;
|
||||
|
||||
(void) opt;
|
||||
if (argc < 1) {
|
||||
fprintf(stderr, "crange: needs at least one raw Content-Range line\n");
|
||||
return 1;
|
||||
}
|
||||
for (i = 0; i < argc; i++) {
|
||||
htsblk r;
|
||||
char BIGSTK line[HTS_URLMAXSIZE * 2];
|
||||
|
||||
memset(&r, 0, sizeof(r));
|
||||
strcpybuff(line, argv[i]);
|
||||
treathead(NULL, "www.example.com", "/", &r, line);
|
||||
printf("crange_start=" LLintP " crange_end=" LLintP " crange=" LLintP "\n",
|
||||
(LLint) r.crange_start, (LLint) r.crange_end, (LLint) r.crange);
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Decode a body argument ("hex:FFD8.." or literal text) into buf. */
|
||||
static size_t st_decode_body(const char *arg, char *buf, size_t size) {
|
||||
size_t n = 0;
|
||||
@@ -2572,10 +2496,6 @@ static const struct selftest_entry {
|
||||
{"headerlong", "[header-name:]",
|
||||
"over-long header value must not overflow the parse scratch",
|
||||
st_headerlong},
|
||||
{"crange", "<raw-content-range-line> ...",
|
||||
"Content-Range parse integer safety", st_crange},
|
||||
{"xfread-limit", "", "in-memory receive buffer size bound",
|
||||
st_xfread_limit},
|
||||
{"savename", "<fil> <content-type> [key=value ...]",
|
||||
"local save-name for a URL", st_savename},
|
||||
{"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
|
||||
|
||||
@@ -1,34 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Content-Range parse (treathead via -#test=crange). Hostile values must clamp
|
||||
# to 0, and the crange +/- 1 arithmetic must not sign-overflow (UBSan aborts on
|
||||
# the pre-fix binary at the INT64_MIN cases).
|
||||
|
||||
cr() {
|
||||
local want="$1"
|
||||
shift
|
||||
out="$(httrack -O /dev/null -#test=crange "$@")"
|
||||
test "$out" == "$want" || {
|
||||
echo "FAIL: $* -> '$out' (want '$want')"
|
||||
exit 1
|
||||
}
|
||||
}
|
||||
|
||||
# Normal range and the '*/N' fallback both parse through unchanged.
|
||||
cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes 0-70870/70871'
|
||||
cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes */70871'
|
||||
|
||||
# INT64_MIN total in the fallback would make crange-1 overflow: clamp to 0.
|
||||
cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes */-9223372036854775808'
|
||||
|
||||
# Any negative field in the 3-field form clamps the whole triple to 0 (each
|
||||
# field independently: a leading negative start/end alone still zeroes all).
|
||||
cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -1--1/-1'
|
||||
cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -5-10/20'
|
||||
|
||||
# INT64_MAX is non-negative: it passes through, the parse must not mangle it.
|
||||
cr 'crange_start=0 crange_end=9223372036854775807 crange=9223372036854775807' \
|
||||
'Content-Range: bytes 0-9223372036854775807/9223372036854775807'
|
||||
@@ -1,29 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# http_xfread1 must refuse an in-memory receive buffer that would exceed a
|
||||
# 32-bit index (hostile Content-Length, or an endless stream) rather than
|
||||
# allocate it, while still accepting a normal small size. -#test=xfread-limit
|
||||
# drives both refusal paths and the accept path.
|
||||
|
||||
out="$(httrack -O /dev/null -#test=xfread-limit)"
|
||||
for case in bylen bygrow; do
|
||||
echo "$out" | grep -q "${case}: refused=1 adr=null msg=In-memory content too large" || {
|
||||
echo "FAIL ${case}: $out"
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
|
||||
# Exactly INT32_MAX must be refused too (the reallocs add 1).
|
||||
echo "$out" | grep -q 'boundary: msg=In-memory content too large' || {
|
||||
echo "FAIL boundary: $out"
|
||||
exit 1
|
||||
}
|
||||
|
||||
# The guard must NOT fire for a legitimate small size.
|
||||
if echo "$out" | grep -q 'accept: msg=In-memory content too large'; then
|
||||
echo "FAIL accept (guard fired on a legit size): $out"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1,96 +0,0 @@
|
||||
#!/bin/bash
|
||||
# C2: a resume whose 206 carries a Content-Range end of INT64_MAX must not
|
||||
# sign-overflow the crange+1 range check (UBSan aborts on the pre-fix binary).
|
||||
# Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets the hostile 206, and
|
||||
# must reject the range and refetch the whole file rather than overflow. Teeth
|
||||
# are UBSan-only: a normal build wraps to the same reject+restart.
|
||||
set -u
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
server="${testdir}/local-server.py"
|
||||
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crange.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
fi
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
serverlog="${tmpdir}/server.log"
|
||||
python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$serverlog" 2>/dev/null)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$serverlog")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
base="http://127.0.0.1:${port}"
|
||||
|
||||
which httrack >/dev/null || {
|
||||
echo "could not find httrack"
|
||||
exit 1
|
||||
}
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out"
|
||||
common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
|
||||
refdir="${out}/hts-cache/ref"
|
||||
|
||||
# --- pass 1: crawl, interrupt once the blob download is underway -------------
|
||||
printf '[pass 1: interrupt mid-download] ..\t'
|
||||
httrack "${common[@]}" "${base}/crange206/index.html" >"${tmpdir}/log1" 2>&1 &
|
||||
crawlpid=$!
|
||||
for _ in $(seq 1 300); do
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
|
||||
kill -0 "$crawlpid" 2>/dev/null || break
|
||||
sleep 0.1
|
||||
done
|
||||
sleep 0.3
|
||||
kill -TERM "$crawlpid" 2>/dev/null
|
||||
wait "$crawlpid" 2>/dev/null
|
||||
crawlpid=
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
|
||||
echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (temp-ref present)"
|
||||
|
||||
# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
|
||||
printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
|
||||
httrack "${common[@]}" --continue "${base}/crange206/index.html" >"${tmpdir}/log2" 2>&1
|
||||
echo "OK (terminated)"
|
||||
|
||||
blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
|
||||
full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
|
||||
|
||||
printf '[file recovered whole] ..\t'
|
||||
test -s "$blob" || {
|
||||
echo "FAIL: blob.bin missing after the hostile 206"
|
||||
exit 1
|
||||
}
|
||||
got=$(wc -c <"$blob")
|
||||
test "$got" -eq "$full" || {
|
||||
echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (${got} bytes)"
|
||||
@@ -1,96 +0,0 @@
|
||||
#!/bin/bash
|
||||
# C2 (memory branch): a resume whose 206 lies text/html with a matching
|
||||
# INT64_MAX Content-Length must not overflow the resume buffer-size add
|
||||
# (UBSan aborts on the pre-fix binary at alloc_mem += totalsize). Pass 1 leaves
|
||||
# a partial + temp-ref; pass 2 resumes and must reject and refetch the whole
|
||||
# file. Teeth are UBSan-only: a normal build wraps to the same reject+restart.
|
||||
set -u
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
server="${testdir}/local-server.py"
|
||||
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangemem.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
fi
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
serverlog="${tmpdir}/server.log"
|
||||
python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
line=$(head -n1 "$serverlog" 2>/dev/null)
|
||||
if test "${line%% *}" == "PORT"; then
|
||||
port="${line#PORT }"
|
||||
break
|
||||
fi
|
||||
kill -0 "$serverpid" 2>/dev/null || {
|
||||
echo "server exited early: $(cat "$serverlog")"
|
||||
exit 1
|
||||
}
|
||||
sleep 0.1
|
||||
done
|
||||
test -n "$port" || {
|
||||
echo "could not discover server port"
|
||||
exit 1
|
||||
}
|
||||
base="http://127.0.0.1:${port}"
|
||||
|
||||
which httrack >/dev/null || {
|
||||
echo "could not find httrack"
|
||||
exit 1
|
||||
}
|
||||
out="${tmpdir}/crawl"
|
||||
mkdir "$out"
|
||||
common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
|
||||
refdir="${out}/hts-cache/ref"
|
||||
|
||||
# --- pass 1: crawl, interrupt once the blob download is underway -------------
|
||||
printf '[pass 1: interrupt mid-download] ..\t'
|
||||
httrack "${common[@]}" "${base}/crange206mem/index.html" >"${tmpdir}/log1" 2>&1 &
|
||||
crawlpid=$!
|
||||
for _ in $(seq 1 300); do
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
|
||||
kill -0 "$crawlpid" 2>/dev/null || break
|
||||
sleep 0.1
|
||||
done
|
||||
sleep 0.3
|
||||
kill -TERM "$crawlpid" 2>/dev/null
|
||||
wait "$crawlpid" 2>/dev/null
|
||||
crawlpid=
|
||||
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
|
||||
echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (temp-ref present)"
|
||||
|
||||
# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
|
||||
printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
|
||||
httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1
|
||||
echo "OK (terminated)"
|
||||
|
||||
blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
|
||||
full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
|
||||
|
||||
printf '[file recovered whole] ..\t'
|
||||
test -s "$blob" || {
|
||||
echo "FAIL: blob.bin missing after the hostile 206"
|
||||
exit 1
|
||||
}
|
||||
got=$(wc -c <"$blob")
|
||||
test "$got" -eq "$full" || {
|
||||
echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
|
||||
exit 1
|
||||
}
|
||||
echo "OK (${got} bytes)"
|
||||
@@ -30,7 +30,6 @@ TESTS = \
|
||||
01_engine-cmdline.test \
|
||||
01_engine-cookies.test \
|
||||
01_engine-copyopt.test \
|
||||
01_engine-crange.test \
|
||||
01_engine-dns.test \
|
||||
01_engine-doitlog.test \
|
||||
01_engine-entities.test \
|
||||
@@ -66,7 +65,6 @@ TESTS = \
|
||||
01_engine-urlhack.test \
|
||||
01_engine-unescape-bounds.test \
|
||||
01_engine-useragent.test \
|
||||
01_engine-xfread.test \
|
||||
01_zlib-acceptencoding.test \
|
||||
01_zlib-cache.test \
|
||||
01_zlib-cache-corrupt.test \
|
||||
@@ -117,8 +115,6 @@ TESTS = \
|
||||
43_local-update-truncate.test \
|
||||
44_local-update-errormask.test \
|
||||
45_local-maxsize-recv.test \
|
||||
46_local-update-304-resume.test \
|
||||
47_local-crange-overflow.test \
|
||||
48_local-crange-memresume.test
|
||||
46_local-update-304-resume.test
|
||||
|
||||
CLEANFILES = check-network_sh.cache
|
||||
|
||||
@@ -105,7 +105,7 @@ function start-crawl {
|
||||
log="${tmp}/log"
|
||||
debug starting httrack -O "${tmp}" "${moreargs[@]}" "${@:pos}"
|
||||
info "running httrack ${*:pos}"
|
||||
httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -mrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
|
||||
httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -omrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
|
||||
crawlpid="$!"
|
||||
debug "started cralwer on pid $crawlpid"
|
||||
wait "$crawlpid"
|
||||
|
||||
@@ -213,7 +213,7 @@ mkdir "$out" || die "could not create $out"
|
||||
declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
|
||||
log="${tmpdir}/log"
|
||||
info "running httrack ${hts[*]}"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid"
|
||||
crawlres=$?
|
||||
@@ -230,7 +230,7 @@ grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
|
||||
# --- optional second pass: re-mirror into the same dir (cache/update path) ----
|
||||
if test -n "$rerun"; then
|
||||
info "re-running httrack (update pass)"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid"
|
||||
@@ -258,7 +258,7 @@ fi
|
||||
if test -n "$rerun_args"; then
|
||||
read -ra extra <<<"$rerun_args"
|
||||
info "re-running httrack with ${rerun_args}"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid"
|
||||
@@ -281,7 +281,7 @@ if test -n "$rerun_dead"; then
|
||||
wait "$serverpid" 2>/dev/null
|
||||
serverpid=
|
||||
info "re-running httrack against the stopped server"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
|
||||
"${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
|
||||
crawlpid=$!
|
||||
wait "$crawlpid" || true
|
||||
|
||||
@@ -905,94 +905,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(part)
|
||||
|
||||
# C2: a resume answered with a 206 whose Content-Range end is INT64_MAX would
|
||||
# sign-overflow the crange+1 range check (UBSan abort). Stall first (partial +
|
||||
# ref), then answer the resume Range with that hostile 206; httrack must reject
|
||||
# the range and refetch, never overflow.
|
||||
CRANGE206_BODY = b"CR206DAT" + bytes((i * 5 + 1) % 256 for i in range(6000))
|
||||
_crange206_started = False
|
||||
|
||||
def route_crange206_index(self):
|
||||
self.send_html('\t<a href="blob.bin">blob</a>')
|
||||
|
||||
def route_crange206(self):
|
||||
rng = self.headers.get("Range")
|
||||
if not Handler._crange206_started:
|
||||
Handler._crange206_started = True
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
|
||||
self.send_header("Accept-Ranges", "bytes")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.CRANGE206_BODY[:3000])
|
||||
self.wfile.flush()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
except OSError:
|
||||
pass
|
||||
return
|
||||
if rng is not None: # resume: hostile 206, Content-Range end = INT64_MAX
|
||||
self.send_response(206, "Partial Content")
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
|
||||
self.send_header("Content-Range", "bytes 0-9223372036854775807/1")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.CRANGE206_BODY)
|
||||
return
|
||||
# range-less refetch after the bad range is rejected: whole file.
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.CRANGE206_BODY)
|
||||
|
||||
# C2 memory branch: a resume answered with a 206 that lies text/html (so the
|
||||
# resume buffers in memory) plus a matching INT64_MAX Content-Length would
|
||||
# overflow the buffer-size add. Stall first, then send that hostile 206.
|
||||
_crange206mem_started = False
|
||||
|
||||
def route_crange206mem_index(self):
|
||||
self.send_html('\t<a href="blob.bin">blob</a>')
|
||||
|
||||
def route_crange206mem(self):
|
||||
rng = self.headers.get("Range")
|
||||
if not Handler._crange206mem_started:
|
||||
Handler._crange206mem_started = True
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
|
||||
self.send_header("Accept-Ranges", "bytes")
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.CRANGE206_BODY[:3000])
|
||||
self.wfile.flush()
|
||||
try:
|
||||
while True:
|
||||
time.sleep(3600)
|
||||
except OSError:
|
||||
pass
|
||||
return
|
||||
if rng is not None: # resume: text/html + matching INT64_MAX Content-Length
|
||||
self.send_response(206, "Partial Content")
|
||||
self.send_header("Content-Type", "text/html")
|
||||
self.send_header("Content-Length", "9223372036854775807")
|
||||
self.send_header(
|
||||
"Content-Range", "bytes 0-9223372036854775806/9223372036854775807"
|
||||
)
|
||||
self.end_headers()
|
||||
return # the overflow is computed before any body read
|
||||
# range-less refetch after the resume is rejected: whole file.
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/octet-stream")
|
||||
self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
|
||||
self.end_headers()
|
||||
if self.command != "HEAD":
|
||||
self.wfile.write(self.CRANGE206_BODY)
|
||||
|
||||
# error pages / 0-byte files (#17): -o0 ("no error pages") must keep 4xx/5xx
|
||||
# bodies off disk; a genuine 0-byte 200 is a valid file and stays.
|
||||
def route_errpage_index(self):
|
||||
@@ -1292,10 +1204,6 @@ class Handler(SimpleHTTPRequestHandler):
|
||||
"/overlap/index.html": route_overlap_index,
|
||||
"/overlap/flaky.bin": route_overlap,
|
||||
"/overlap/full.bin": route_overlap_full,
|
||||
"/crange206/index.html": route_crange206_index,
|
||||
"/crange206/blob.bin": route_crange206,
|
||||
"/crange206mem/index.html": route_crange206mem_index,
|
||||
"/crange206mem/blob.bin": route_crange206mem,
|
||||
"/size/index.html": route_size_index,
|
||||
"/size/oversize.bin": route_size_oversize,
|
||||
"/errpage/index.html": route_errpage_index,
|
||||
|
||||
Reference in New Issue
Block a user