mirror of
https://github.com/xroche/httrack.git
synced 2026-07-15 05:11:03 +03:00
Compare commits
8 Commits
master
...
windows-ti
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f97eac119f | ||
|
|
f8cf843d1a | ||
|
|
807edcef76 | ||
|
|
05814ddd5f | ||
|
|
5580a1496a | ||
|
|
f1f70710f5 | ||
|
|
962af51d3a | ||
|
|
7478f34c27 |
31
.github/workflows/windows-build.yml
vendored
31
.github/workflows/windows-build.yml
vendored
@@ -123,9 +123,12 @@ jobs:
|
||||
# is Linux/macOS only. These are the offline ones, driven from Git Bash
|
||||
# against the native httrack.exe. They subsume the self-tests this step
|
||||
# used to run inline (codecs, cache, fsize).
|
||||
# The *_local-* ones crawl the bundled Python server over loopback: the real
|
||||
# TLS handshake, cache and file writer, which nothing else on Windows covers.
|
||||
- name: Run the engine test suite (offline tests)
|
||||
shell: bash
|
||||
working-directory: tests
|
||||
timeout-minutes: 45
|
||||
run: |
|
||||
set -u
|
||||
bin="$(cygpath -u "$GITHUB_WORKSPACE")/src/${{ matrix.platform }}/${{ matrix.configuration }}"
|
||||
@@ -141,13 +144,24 @@ jobs:
|
||||
TMPDIR="$(cygpath -m "$RUNNER_TEMP")"
|
||||
export TMPDIR
|
||||
|
||||
pass=0 fail=0 skip=0 failed=""
|
||||
for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test; do
|
||||
# Mirror what configure hands the suite. LC_ALL sets the codeset MSYS maps
|
||||
# a UTF-8 mirror name onto UTF-16 with, which the intl crawls "test -f".
|
||||
export HTTPS_SUPPORT=yes BROTLI_ENABLED=yes ZSTD_ENABLED=yes
|
||||
export LC_ALL=C.UTF-8
|
||||
|
||||
# A wedged crawl must not eat the job's whole timeout budget.
|
||||
watchdog=""
|
||||
command -v timeout >/dev/null && watchdog="timeout 600"
|
||||
|
||||
pass=0 fail=0 skip=0 failed="" skipped=""
|
||||
for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
|
||||
*_local-*.test 13_crawl_proxy_https.test; do
|
||||
rc=0
|
||||
bash "$t" >"$t.log" 2>&1 || rc=$?
|
||||
# shellcheck disable=SC2086
|
||||
$watchdog bash "$t" >"$t.log" 2>&1 || rc=$?
|
||||
case "$rc" in
|
||||
0) pass=$((pass + 1)); echo "PASS $t" ;;
|
||||
77) skip=$((skip + 1)); echo "SKIP $t" ;;
|
||||
77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
|
||||
*)
|
||||
fail=$((fail + 1)) failed="$failed $t"
|
||||
echo "FAIL $t (exit $rc)"
|
||||
@@ -161,10 +175,11 @@ jobs:
|
||||
echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
|
||||
tee -a "$GITHUB_STEP_SUMMARY"
|
||||
|
||||
# Every gate in these scripts exits 77, so a suite that degraded to
|
||||
# all-skipped would report green having tested nothing: assert a floor
|
||||
# on what actually ran, not just the absence of failures.
|
||||
[ "$pass" -ge 45 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
# Every gate here exits 77, so an all-skipped suite would report green having
|
||||
# tested nothing: pin the skips, and floor the passes in case the glob empties.
|
||||
expected_skips=" 19_local-connect-fallback.test" # pending #579
|
||||
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
|
||||
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
|
||||
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
|
||||
|
||||
- name: Upload the test logs
|
||||
|
||||
@@ -8,22 +8,26 @@ set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$top_srcdir/tests/testlib.sh"
|
||||
|
||||
if test "${HTTPS_SUPPORT:-}" == "no"; then
|
||||
echo "no https support compiled, skipping"
|
||||
exit 77
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1 || ! command -v openssl >/dev/null 2>&1; then
|
||||
python=$(find_python) || python=
|
||||
if test -z "$python" || ! command -v openssl >/dev/null 2>&1; then
|
||||
echo "python3/openssl missing, skipping"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
server="$top_srcdir/tests/proxy-https-server.py"
|
||||
server=$(nativepath "$top_srcdir/tests/proxy-https-server.py")
|
||||
tmpdir=$(mktemp -d)
|
||||
pids=
|
||||
|
||||
cleanup() {
|
||||
for pid in $pids; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
stop_server "$pid"
|
||||
done
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
@@ -38,10 +42,13 @@ cat "$tmpdir/key.pem" "$tmpdir/cert.pem" >"$tmpdir/both.pem"
|
||||
# start_server <logdir> <mode>: launches a proxy+origin pair, sets $origin_port
|
||||
# and $proxy_port from its announced ephemeral ports.
|
||||
start_server() {
|
||||
local dir="$1" mode="$2" ports
|
||||
local dir="$1" mode="$2" ports pem
|
||||
mkdir -p "$dir"
|
||||
ports="$dir/ports.txt"
|
||||
python3 "$server" "$tmpdir/both.pem" "$dir" "$mode" \
|
||||
pem=$(nativepath "$tmpdir/both.pem")
|
||||
dir_native=$(nativepath "$dir")
|
||||
: >"$ports"
|
||||
"$python" "$server" "$pem" "$dir_native" "$mode" \
|
||||
>"$ports" 2>"$dir/server.err" &
|
||||
pids="$pids $!"
|
||||
for _ in $(seq 1 100); do
|
||||
|
||||
@@ -12,14 +12,17 @@ set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$top_srcdir/tests/testlib.sh"
|
||||
|
||||
if test "${V6_SUPPORT:-}" == "no"; then
|
||||
echo "no IPv6 support (resolver list/override is IPv6-only), skipping"
|
||||
exit 77
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1; then
|
||||
python=$(find_python) || {
|
||||
echo "python3 missing, skipping"
|
||||
exit 77
|
||||
fi
|
||||
}
|
||||
# The fixture needs a second loopback IP (dead 127.0.0.2 + live 127.0.0.1) for
|
||||
# the fallback to have a target; GNU/Hurd has only 127.0.0.1, so skip there.
|
||||
case "$(uname -s)" in
|
||||
@@ -28,24 +31,26 @@ GNU | GNU/*)
|
||||
exit 77
|
||||
;;
|
||||
esac
|
||||
# TODO: drop once https://github.com/xroche/httrack/issues/579 is fixed.
|
||||
if is_windows; then
|
||||
echo "Windows: the connect fallback never runs (#579), skipping"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
server="$top_srcdir/tests/local-server.py"
|
||||
root="$top_srcdir/tests/server-root"
|
||||
server=$(nativepath "$top_srcdir/tests/local-server.py")
|
||||
root=$(nativepath "$top_srcdir/tests/server-root")
|
||||
tmpdir=$(mktemp -d)
|
||||
serverpid=
|
||||
|
||||
cleanup() {
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null || true
|
||||
wait "$serverpid" 2>/dev/null || true
|
||||
fi
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
return 0
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# bind the live server to 127.0.0.1 only, so 127.0.0.2 refuses the connect
|
||||
python3 "$server" --root "$root" --bind 127.0.0.1 >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &
|
||||
"$python" "$server" --root "$root" --bind 127.0.0.1 >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
|
||||
@@ -5,19 +5,19 @@ set -u
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
server="${testdir}/local-server.py"
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server=$(nativepath "${testdir}/local-server.py")
|
||||
root=$(nativepath "${testdir}/server-root")
|
||||
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_206.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
fi
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
@@ -25,8 +25,9 @@ trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
# --- start the server, discover its ephemeral port --------------------------
|
||||
# RESUME_COUNTER gets a byte per /resume/blob.txt request (pass-2 delta bounds re-gets).
|
||||
serverlog="${tmpdir}/server.log"
|
||||
: >"$serverlog"
|
||||
counter="${tmpdir}/blobcount"
|
||||
RESUME_COUNTER="$counter" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
|
||||
RESUME_COUNTER="$counter" "$python" "$server" --root "$root" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
|
||||
@@ -9,29 +9,30 @@ set -eu
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
server="${testdir}/local-server.py"
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server=$(nativepath "${testdir}/local-server.py")
|
||||
root=$(nativepath "${testdir}/server-root")
|
||||
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_198.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
if test -n "$crawlpid"; then kill -9 "$crawlpid" 2>/dev/null || true; fi
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null || true
|
||||
wait "$serverpid" 2>/dev/null || true
|
||||
fi
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# OVERLAP_COUNTER gets a byte per flaky.bin request so pass 1 knows when to interrupt.
|
||||
serverlog="${tmpdir}/server.log"
|
||||
: >"$serverlog"
|
||||
counter="${tmpdir}/hits"
|
||||
resumed="${tmpdir}/resumed" # gets a byte when the server serves a resume 206
|
||||
OVERLAP_COUNTER="$counter" OVERLAP_RESUMED="$resumed" \
|
||||
python3 "$server" --root "${testdir}/server-root" \
|
||||
"$python" "$server" --root "$root" \
|
||||
>"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
|
||||
@@ -9,9 +9,12 @@ set -e
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$top_srcdir/tests/testlib.sh"
|
||||
|
||||
# python3 runs the local server (mirror local-crawl.sh); skip when absent, else
|
||||
# run() swallows its exit-77 and the serverless 0s/0s crawl looks like a fail.
|
||||
command -v python3 >/dev/null || {
|
||||
find_python >/dev/null || {
|
||||
echo "python3 not found; skipping local crawl tests"
|
||||
exit 77
|
||||
}
|
||||
|
||||
@@ -7,27 +7,28 @@ set -u
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
server="${testdir}/local-server.py"
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server=$(nativepath "${testdir}/local-server.py")
|
||||
root=$(nativepath "${testdir}/server-root")
|
||||
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_c7.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
fi
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
serverlog="${tmpdir}/server.log"
|
||||
: >"$serverlog"
|
||||
counter="${tmpdir}/reqcount"
|
||||
mark="${tmpdir}/got304"
|
||||
RESUME304_COUNTER="$counter" RESUME304_MARK="$mark" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
|
||||
RESUME304_COUNTER="$counter" RESUME304_MARK="$mark" "$python" "$server" --root "$root" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
|
||||
@@ -8,25 +8,26 @@ set -u
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
server="${testdir}/local-server.py"
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server=$(nativepath "${testdir}/local-server.py")
|
||||
root=$(nativepath "${testdir}/server-root")
|
||||
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crange.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
fi
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
serverlog="${tmpdir}/server.log"
|
||||
python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
|
||||
: >"$serverlog"
|
||||
"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
|
||||
@@ -8,25 +8,26 @@ set -u
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
server="${testdir}/local-server.py"
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server=$(nativepath "${testdir}/local-server.py")
|
||||
root=$(nativepath "${testdir}/server-root")
|
||||
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangemem.XXXXXX") || exit 1
|
||||
serverpid=
|
||||
crawlpid=
|
||||
cleanup() {
|
||||
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
fi
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
serverlog="${tmpdir}/server.log"
|
||||
python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
|
||||
: >"$serverlog"
|
||||
"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=
|
||||
for _ in $(seq 1 50); do
|
||||
@@ -77,7 +78,14 @@ echo "OK (temp-ref present)"
|
||||
|
||||
# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
|
||||
printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
|
||||
httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1
|
||||
rc=0
|
||||
httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1 || rc=$?
|
||||
# a crash here would otherwise read as a clean "terminated"
|
||||
test "$rc" -eq 0 || {
|
||||
echo "FAIL: httrack exited $rc"
|
||||
cat "${tmpdir}/log2" >&2
|
||||
exit 1
|
||||
}
|
||||
echo "OK (terminated)"
|
||||
|
||||
blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
|
||||
@@ -86,6 +94,9 @@ full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
|
||||
printf '[file recovered whole] ..\t'
|
||||
test -s "$blob" || {
|
||||
echo "FAIL: blob.bin missing after the hostile 206"
|
||||
tail -40 "${out}/hts-log.txt" >&2
|
||||
cat "${tmpdir}/log2" >&2
|
||||
find "$out" -type f >&2
|
||||
exit 1
|
||||
}
|
||||
got=$(wc -c <"$blob")
|
||||
|
||||
@@ -8,11 +8,15 @@ set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$top_srcdir/tests/testlib.sh"
|
||||
|
||||
if test "${HTTPS_SUPPORT:-}" == "no"; then
|
||||
echo "no https support compiled, skipping"
|
||||
exit 77
|
||||
fi
|
||||
if ! command -v python3 >/dev/null 2>&1 || ! command -v openssl >/dev/null 2>&1; then
|
||||
python=$(find_python) || python=
|
||||
if test -z "$python" || ! command -v openssl >/dev/null 2>&1; then
|
||||
echo "python3/openssl missing, skipping"
|
||||
exit 77
|
||||
fi
|
||||
@@ -20,13 +24,13 @@ fi
|
||||
# a .invalid name never resolves (RFC 6761): reaching the page at all proves the
|
||||
# proxy did the DNS
|
||||
host="socks-origin.invalid"
|
||||
server="$top_srcdir/tests/socks5-server.py"
|
||||
server=$(nativepath "$top_srcdir/tests/socks5-server.py")
|
||||
tmpdir=$(mktemp -d)
|
||||
pids=
|
||||
|
||||
cleanup() {
|
||||
for pid in $pids; do
|
||||
kill "$pid" 2>/dev/null || true
|
||||
stop_server "$pid"
|
||||
done
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
@@ -39,10 +43,13 @@ cat "$tmpdir/key.pem" "$tmpdir/cert.pem" >"$tmpdir/both.pem"
|
||||
|
||||
# start_server <logdir> <mode>: sets $tls_port/$http_port/$socks_port
|
||||
start_server() {
|
||||
local dir="$1" mode="$2" ports
|
||||
local dir="$1" mode="$2" ports pem
|
||||
mkdir -p "$dir"
|
||||
ports="$dir/ports.txt"
|
||||
python3 "$server" "$tmpdir/both.pem" "$dir" "$mode" \
|
||||
pem=$(nativepath "$tmpdir/both.pem")
|
||||
dir_native=$(nativepath "$dir")
|
||||
: >"$ports"
|
||||
"$python" "$server" "$pem" "$dir_native" "$mode" \
|
||||
>"$ports" 2>"$dir/server.err" &
|
||||
pids="$pids $!"
|
||||
for _ in $(seq 1 100); do
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
# silently drop it from the dist tarball and break "make distcheck".
|
||||
EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
|
||||
proxy-https-server.py socks5-server.py \
|
||||
local-crawl.sh local-server.py server.crt server.key \
|
||||
local-crawl.sh local-server.py testlib.sh server.crt server.key \
|
||||
server-root/simple/basic.html server-root/simple/link.html \
|
||||
server-root/stripquery/index.html server-root/stripquery/a.html \
|
||||
server-root/fraglink/index.html server-root/fraglink/target.html \
|
||||
|
||||
@@ -38,6 +38,8 @@
|
||||
set -u
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
server="${testdir}/local-server.py"
|
||||
root="${LOCAL_SERVER_ROOT:-${testdir}/server-root}"
|
||||
cert="${testdir}/server.crt"
|
||||
@@ -72,12 +74,8 @@ function cleanup {
|
||||
kill -9 "$crawlpid" 2>/dev/null
|
||||
crawlpid=
|
||||
fi
|
||||
if test -n "$serverpid"; then
|
||||
kill "$serverpid" 2>/dev/null
|
||||
# Reap it so the port is released before we rm the tmpdir/log.
|
||||
wait "$serverpid" 2>/dev/null
|
||||
serverpid=
|
||||
fi
|
||||
stop_server "$serverpid"
|
||||
serverpid=
|
||||
if test -n "$tmpdir" && test -d "$tmpdir"; then
|
||||
test -n "$nopurge" || rm -rf "$tmpdir"
|
||||
fi
|
||||
@@ -96,7 +94,7 @@ nopurge=
|
||||
trap cleanup EXIT HUP INT QUIT PIPE TERM
|
||||
|
||||
# python3 is required; mirror check-network.sh's skip-with-77 convention.
|
||||
command -v python3 >/dev/null || ! echo "python3 not found; skipping local crawl tests" || exit 77
|
||||
python=$(find_python) || ! echo "python3 not found; skipping local crawl tests" >&2 || exit 77
|
||||
|
||||
tmptopdir=${TMPDIR:-/tmp}
|
||||
test -d "$tmptopdir" || mkdir -p "$tmptopdir" || die "no temporary directory; set TMPDIR"
|
||||
@@ -158,12 +156,12 @@ done
|
||||
# --- start the server --------------------------------------------------------
|
||||
test -r "$server" || die "cannot read $server"
|
||||
serverlog="${tmpdir}/server.log"
|
||||
serverargs=(--root "$root")
|
||||
serverargs=(--root "$(nativepath "$root")")
|
||||
if test -n "$tls"; then
|
||||
serverargs+=(--tls --cert "$cert" --key "$key")
|
||||
serverargs+=(--tls --cert "$(nativepath "$cert")" --key "$(nativepath "$key")")
|
||||
fi
|
||||
debug "starting python3 $server ${serverargs[*]}"
|
||||
python3 "$server" "${serverargs[@]}" >"$serverlog" 2>&1 &
|
||||
debug "starting $python $server ${serverargs[*]}"
|
||||
"$python" "$(nativepath "$server")" "${serverargs[@]}" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
|
||||
# Wait for the "PORT <n>" line (server prints it once bound).
|
||||
@@ -203,7 +201,7 @@ if test "${#cookies[@]}" -gt 0; then
|
||||
fi
|
||||
|
||||
# --- run httrack -------------------------------------------------------------
|
||||
which httrack >/dev/null || die "could not find httrack"
|
||||
command -v httrack >/dev/null || die "could not find httrack"
|
||||
ver=$(httrack -O /dev/null --version | sed -e 's/HTTrack version //')
|
||||
test -n "$ver" || die "could not run httrack"
|
||||
|
||||
@@ -278,8 +276,7 @@ if test -n "$rerun_dead"; then
|
||||
test -s "$zip" || die "no cache was written by the first pass"
|
||||
cp "$zip" "${tmpdir}/cache-before.zip"
|
||||
cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
|
||||
kill "$serverpid" 2>/dev/null
|
||||
wait "$serverpid" 2>/dev/null
|
||||
stop_server "$serverpid"
|
||||
serverpid=
|
||||
info "re-running httrack against the stopped server"
|
||||
httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
|
||||
@@ -441,12 +438,18 @@ while test "$i" -lt "${#audit[@]}"; do
|
||||
--file-mode)
|
||||
path="${audit[$((i + 1))]}"
|
||||
i=$((i + 2))
|
||||
mode=$(stat -c '%a' "${hostroot}/${path}" 2>/dev/null ||
|
||||
stat -f '%Lp' "${hostroot}/${path}" 2>/dev/null)
|
||||
info "checking ${path} mode ${mode:-none} is ${audit[$i]}"
|
||||
if test "$mode" = "${audit[$i]}"; then result "OK"; else
|
||||
result "wrong mode"
|
||||
exit 1
|
||||
if is_windows; then
|
||||
# No POSIX modes, and the engine only chmods #ifndef _WIN32.
|
||||
info "checking ${path} mode"
|
||||
result "SKIP (no POSIX modes)"
|
||||
else
|
||||
mode=$(stat -c '%a' "${hostroot}/${path}" 2>/dev/null ||
|
||||
stat -f '%Lp' "${hostroot}/${path}" 2>/dev/null)
|
||||
info "checking ${path} mode ${mode:-none} is ${audit[$i]}"
|
||||
if test "$mode" = "${audit[$i]}"; then result "OK"; else
|
||||
result "wrong mode"
|
||||
exit 1
|
||||
fi
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
|
||||
@@ -18,6 +18,7 @@ import base64
|
||||
import gzip
|
||||
import hashlib
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
|
||||
from urllib.parse import quote, unquote, urlsplit
|
||||
@@ -1817,7 +1818,8 @@ def main():
|
||||
httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
|
||||
|
||||
port = httpd.socket.getsockname()[1]
|
||||
# The launcher reads this line to discover the ephemeral port.
|
||||
# Keep the port line the launcher parses LF: Windows would emit \r\n.
|
||||
sys.stdout.reconfigure(newline="\n")
|
||||
print(f"PORT {port}", flush=True)
|
||||
|
||||
try:
|
||||
|
||||
@@ -141,6 +141,8 @@ def main():
|
||||
open(os.path.join(logdir, name), "w").close()
|
||||
origin_port = start_origin(certfile, logdir)
|
||||
proxy_port = start_proxy(logdir, mode)
|
||||
# Keep the port lines the caller parses LF: Windows would emit \r\n.
|
||||
sys.stdout.reconfigure(newline="\n")
|
||||
print("ORIGIN %d" % origin_port, flush=True)
|
||||
print("PROXY %d" % proxy_port, flush=True)
|
||||
print("ready", flush=True)
|
||||
|
||||
@@ -217,6 +217,8 @@ def main():
|
||||
tls_port = start_origin(logdir, certfile)
|
||||
http_port = start_origin(logdir, None)
|
||||
socks_port = start_socks(logdir, mode)
|
||||
# Keep the port lines the caller parses LF: Windows would emit \r\n.
|
||||
sys.stdout.reconfigure(newline="\n")
|
||||
print("TLS %d" % tls_port, flush=True)
|
||||
print("HTTP %d" % http_port, flush=True)
|
||||
print("SOCKS %d" % socks_port, flush=True)
|
||||
|
||||
45
tests/testlib.sh
Normal file
45
tests/testlib.sh
Normal file
@@ -0,0 +1,45 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Helpers shared by the crawl tests. Sourced, not run.
|
||||
|
||||
# Python 3 interpreter, or empty: Windows only installs python.exe, and a bare
|
||||
# "python" may be 2.x or the Store stub.
|
||||
find_python() {
|
||||
local py
|
||||
for py in "${PYTHON:-}" python3 python; do
|
||||
test -n "$py" || continue
|
||||
"$py" -c 'import sys; sys.exit(sys.version_info[0] != 3)' 2>/dev/null || continue
|
||||
printf '%s\n' "$py"
|
||||
return 0
|
||||
done
|
||||
return 1
|
||||
}
|
||||
|
||||
# Native form of a path: a non-MSYS binary cannot resolve Git Bash's /d/a/... ones.
|
||||
nativepath() {
|
||||
if is_windows && command -v cygpath >/dev/null 2>&1; then
|
||||
cygpath -m "$1"
|
||||
else
|
||||
printf '%s\n' "$1"
|
||||
fi
|
||||
}
|
||||
|
||||
is_windows() {
|
||||
case "$(uname -s)" in
|
||||
MINGW* | MSYS* | CYGWIN*) return 0 ;;
|
||||
*) return 1 ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Stop a backgrounded server and reap it; MSYS cannot signal a native python.exe,
|
||||
# so only -9 lands. Every step is "|| true": callers run under set -e, and reaping
|
||||
# a server we just signalled makes wait return 143.
|
||||
stop_server() {
|
||||
test -n "${1:-}" || return 0
|
||||
kill "$1" 2>/dev/null || true
|
||||
if is_windows; then
|
||||
kill -9 "$1" 2>/dev/null || true
|
||||
fi
|
||||
wait "$1" 2>/dev/null || true
|
||||
return 0
|
||||
}
|
||||
Reference in New Issue
Block a user