mirror of
https://github.com/xroche/httrack.git
synced 2026-07-23 17:19:17 +03:00
1cab56bfe4
The option parser's `case 'K':` had no `break`, so a bare -K (no trailing digit) fell through into `case 'c':`, hit its no-digit branch, and forced the socket count back to the default of 4. `-c8 -K` ended up with 4 connections, not 8; the -K rewrite mode quietly overrode an earlier -c. Only a trailing bare -K triggered it (`-K -c8` and `-Kc8` were fine), which is why it went unnoticed. Add the missing break. The regression test drives the observable that maxsoc has: a socket count above 8 trips the "limited to 8" security warning in hts-log.txt, so `-c16` warns and `-c16 -K` warns only if the 16 survives. It fails on the pre-fix build and passes after. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
177 lines
7.3 KiB
Bash
Executable File
177 lines
7.3 KiB
Bash
Executable File
#!/bin/bash
|
|
#
|
|
|
|
# Offline command-line option tests (no network). The -F user-agent and -%X
|
|
# raw-header values used to be rejected past 126 / 256 bytes (#152); they are
|
|
# now bounded only by the general per-argument cap (HTS_CDLMAXSIZE). A value up
|
|
# to that cap is accepted on both the short (-F, -%X) and long (--user-agent,
|
|
# --headers) forms, and an over-cap value is refused cleanly rather than
|
|
# overrunning a fixed scratch buffer.
|
|
|
|
set -euo pipefail
|
|
|
|
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_cmdline.XXXXXX") || exit 1
|
|
trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
|
|
|
|
echo '<html><body>hello</body></html>' >"$tmp/index.html"
|
|
|
|
# a string of N repeated 'A' characters
|
|
nchars() {
|
|
printf 'A%.0s' $(seq 1 "$1")
|
|
}
|
|
|
|
# crawl the local fixture with the given extra args; leaves the exit status in RC
|
|
run() {
|
|
local out="$1"
|
|
shift
|
|
rm -rf "$out"
|
|
mkdir -p "$out"
|
|
RC=0
|
|
httrack "file://$tmp/index.html" -O "$out" --quiet -n "$@" >"$out/.log" 2>&1 || RC=$?
|
|
}
|
|
|
|
# crawl using exactly the given args as the only URL(s), no implicit primary URL;
|
|
# leaves the exit status in RC
|
|
run_only() {
|
|
local out="$1"
|
|
shift
|
|
rm -rf "$out"
|
|
mkdir -p "$out"
|
|
RC=0
|
|
httrack -O "$out" --quiet -n "$@" >"$out/.log" 2>&1 || RC=$?
|
|
}
|
|
|
|
# assert the value was accepted: clean exit and the fixture was mirrored
|
|
accepted() {
|
|
{ test "$RC" -eq 0 && test -n "$(find "$1" -type f -path '*/index.html' -print -quit)"; } ||
|
|
! echo "FAIL: $2 (exit $RC)" || exit 1
|
|
}
|
|
|
|
# assert the value was refused cleanly: a normal error exit, never a crash
|
|
# (a SIGABRT from an overflowed scratch buffer would surface as exit 134)
|
|
refused() {
|
|
{ test "$RC" -ne 0 && test "$RC" -ne 134; } ||
|
|
! echo "FAIL: $1 (exit $RC)" || exit 1
|
|
}
|
|
|
|
# assert continue mode was entered: it drops the URL list, so with no cache to
|
|
# resume the run ends on the usage screen rather than on any other error
|
|
continued() {
|
|
{ test "$RC" -ne 0 && grep -q 'usage:' "$1/.log"; } ||
|
|
! echo "FAIL: $2 (exit $RC)" || exit 1
|
|
}
|
|
|
|
# a value past the old 126/256 caps but within the cap is accepted, on both the
|
|
# short and long form of each option
|
|
long=$(nchars 900)
|
|
run "$tmp/ua-s" -F "$long"
|
|
accepted "$tmp/ua-s" "#152: long -F user-agent rejected or crashed"
|
|
run "$tmp/ua-l" --user-agent "$long"
|
|
accepted "$tmp/ua-l" "#152: long --user-agent rejected or crashed"
|
|
run "$tmp/hd-s" "-%X" "X-A: $long"
|
|
accepted "$tmp/hd-s" "#152: long -%X header rejected or crashed"
|
|
run "$tmp/hd-l" --headers "X-B: $long"
|
|
accepted "$tmp/hd-l" "#152: long --headers rejected or crashed"
|
|
|
|
# a value just under the cap (>1000) must not overflow the long-form alias
|
|
# scratch buffer (the param[] copy in optalias_check)
|
|
run "$tmp/ua-n" --user-agent "$(nchars 1010)"
|
|
accepted "$tmp/ua-n" "#152: near-cap --user-agent overflowed the param[] buffer"
|
|
|
|
# a value over the cap is refused cleanly (graceful error, not a SIGABRT), on
|
|
# both forms
|
|
over=$(nchars 1100)
|
|
run "$tmp/ov-s" -F "$over"
|
|
refused "#152: over-cap -F not refused cleanly"
|
|
run "$tmp/ov-l" --user-agent "$over"
|
|
refused "#152: over-cap --user-agent not refused cleanly"
|
|
|
|
# Quote handling on the sole URL (run_only, so the quoted arg is the only URL and
|
|
# can't be masked by an implicit one). A fully "-quoted URL has its surrounding
|
|
# quotes stripped in place and is mirrored; a dangling opening quote, and a lone
|
|
# quote (empty after the opening "), are refused cleanly and never crash.
|
|
run_only "$tmp/q-ok" "\"file://$tmp/index.html\""
|
|
accepted "$tmp/q-ok" "quoted URL not stripped/mirrored"
|
|
run_only "$tmp/q-bad" '"foo'
|
|
refused "dangling-quote argument not refused cleanly"
|
|
run_only "$tmp/q-lone" '"'
|
|
refused "lone-quote argument not refused cleanly"
|
|
|
|
# --pause (#185): valid MIN[:MAX] accepted; malformed, reversed, over-range and
|
|
# non-finite values refused cleanly. NaN defeats naive `<`/`>` checks (it
|
|
# compares false to everything), so it must not slip through to the int cast.
|
|
run "$tmp/pause-ok" --pause 0.2:0.4
|
|
accepted "$tmp/pause-ok" "#185: valid --pause range rejected"
|
|
run "$tmp/pause-fix" --pause 0.2
|
|
accepted "$tmp/pause-fix" "#185: valid fixed --pause rejected"
|
|
for bad in nan nan:5 5:nan inf 10:5 99999; do
|
|
run "$tmp/pause-bad" --pause "$bad"
|
|
refused "#185: invalid --pause '$bad' not refused cleanly"
|
|
done
|
|
|
|
# An option is not -i (continue) merely because its name contains an 'i' (#615).
|
|
# These used to wipe the URL given before them and exit on the usage screen.
|
|
run "$tmp/ord-bti" --build-top-index
|
|
accepted "$tmp/ord-bti" "#615: --build-top-index after the URL wiped the URL list"
|
|
run "$tmp/ord-bti-s" "-%i"
|
|
accepted "$tmp/ord-bti-s" "#615: -%i after the URL wiped the URL list"
|
|
run "$tmp/ord-proto" --protocol 2
|
|
accepted "$tmp/ord-proto" "#615: --protocol after the URL wiped the URL list"
|
|
run "$tmp/ord-proto-s" "-@i2"
|
|
accepted "$tmp/ord-proto-s" "#615: -@i2 after the URL wiped the URL list"
|
|
|
|
# %, &, @ and # take the letter after them into another option set, wherever
|
|
# they sit in the cluster, so the i of -q%i is not the main-set -i either.
|
|
run "$tmp/ord-qpi" "-q%i"
|
|
accepted "$tmp/ord-qpi" "#615: -q%i after the URL wiped the URL list"
|
|
run "$tmp/ord-qai" "-q@i"
|
|
accepted "$tmp/ord-qai" "#615: -q@i after the URL wiped the URL list"
|
|
|
|
# -%q only forces quiet mode, which nothing here can see: a run whose stdout is
|
|
# not a tty is quiet from the start (htscoremain.c:174). Just check it crawls.
|
|
run "$tmp/ord-iqs" "-%q"
|
|
accepted "$tmp/ord-iqs" "#615: -%q after the URL broke the crawl"
|
|
|
|
# The real -i still forces continue mode and drops the URL list, in every form:
|
|
# a fix that merely stopped looking for 'i' would pass the checks above.
|
|
run "$tmp/cont-s" -i
|
|
continued "$tmp/cont-s" "#615: -i after the URL no longer forces continue mode"
|
|
run "$tmp/cont-c" -iC2
|
|
continued "$tmp/cont-c" "#615: -iC2 after the URL no longer forces continue mode"
|
|
run "$tmp/cont-l" --continue
|
|
continued "$tmp/cont-l" "#615: --continue after the URL no longer forces continue mode"
|
|
run "$tmp/cont-u" --update
|
|
continued "$tmp/cont-u" "#615: --update after the URL no longer forces continue mode"
|
|
|
|
# ...including where another set's letter precedes it in the cluster.
|
|
run "$tmp/cont-pq" "-%qi"
|
|
continued "$tmp/cont-pq" "#615: -%qi after the URL no longer forces continue mode"
|
|
run "$tmp/cont-pm" "-%Mi"
|
|
continued "$tmp/cont-pm" "#615: -%Mi after the URL no longer forces continue mode"
|
|
|
|
# Placed before the URL these always worked; they must keep working.
|
|
run_only "$tmp/pre-bti" "-%i" "file://$tmp/index.html"
|
|
accepted "$tmp/pre-bti" "#615: -%i before the URL broke the crawl"
|
|
run_only "$tmp/pre-proto" "-@i2" "file://$tmp/index.html"
|
|
accepted "$tmp/pre-proto" "#615: -@i2 before the URL broke the crawl"
|
|
|
|
# -K must not reset the -c socket count (its case fell through into 'c'). maxsoc
|
|
# is only observable via the ">8, limited to 8" warning: -c16 trips it, and would
|
|
# stop tripping it if a trailing -K reset the count back to the default 4.
|
|
warned() {
|
|
grep -q 'simultaneous connections limited to' "$1/hts-log.txt" ||
|
|
! echo "FAIL: $2" || exit 1
|
|
}
|
|
not_warned() {
|
|
! grep -q 'simultaneous connections limited to' "$1/hts-log.txt" ||
|
|
! echo "FAIL: $2" || exit 1
|
|
}
|
|
run "$tmp/soc-c16" -c16
|
|
warned "$tmp/soc-c16" "-c16 did not trip the socket-count warning (probe blind)"
|
|
run "$tmp/soc-c16k" -c16 -K
|
|
warned "$tmp/soc-c16k" "-K reset the -c socket count (fell through into 'c')"
|
|
run "$tmp/soc-c8k" -c8 -K
|
|
not_warned "$tmp/soc-c8k" "-c8 -K spuriously warned"
|
|
|
|
exit 0
|