4 Commits

Author SHA1 Message Date
Xavier Roche
6547eaca7e Merge remote-tracking branch 'origin/master' into issue-1032 2026-08-06 14:16:40 +02:00
Xavier Roche
fccc3e246e tests: bound the fixture-server reap, and drop a fail-open pipe from the FTP login test
stop_server ran a bare `wait` from an EXIT trap, so a fixture the kill never
reached blocked a test that had already passed until the harness timed it out
and called it a failure. reap_bounded gives up after REAP_GRACE, and costs
nothing when the child is already gone.

The new login test read the mirrored file through `find | head -n1`: head
exits on the first line, find takes SIGPIPE, and under pipefail that 141 aborts
the test through set -e with nothing printed at all. The mirror path is
deterministic, so assert it directly.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-08-06 13:53:52 +02:00
Xavier Roche
d6f4d71a38 tests: exercise the colon-less split, and the credentials on the wire
The ftp-userpass sweep always wrote a ':', so the branch where only the '@'
bounds the user name was seen once, three bytes long, yet that is the one
reaching memcpy() with an unbounded length. Sweep it as a third case, and add
the bare over-long and at-boundary URLs to the crawl loops.

The crawl half probes a dead port, so no login is attempted and a clip applied
after the split went unseen. A new local-ftp test drives the test server and
asserts the exact USER/PASS bytes on the control channel; it needs the server
to answer 331, since httrack sends PASS only on a 3xx.

Signed-off-by: Xavier Roche <roche@httrack.com>
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-08-06 11:31:51 +02:00
Xavier Roche
186476cab9 FTP: refuse over-long URL userinfo instead of logging in as another account
ftp_split_userpass() clipped a URL's "user[:pass]@" into user[256]/pass[256],
so a URL naming one account put a shortened name on the wire and httrack
mirrored whatever that account served, exiting clean. The split now reports a
field that does not fit and the link fails with STATUSCODE_INVALID, the
refuse-don't-clip contract ftp_command() follows for an over-long path (#1019).
A compile-time assertion pins both buffers to what a "USER <user>" control line
holds, so widening one cannot reintroduce the clip in the command formatter.

Nothing bounded the user field at the '@' either, only at the first ':' or the
end of the string, so ftp://user@host:21/f logged in as "user@host".

The ftp-userpass self-test asserted the truncation as intended behaviour.

Closes #1032

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-08-06 11:12:29 +02:00
15 changed files with 250 additions and 986 deletions

View File

@@ -14,9 +14,6 @@ on:
permissions:
contents: read
# The suite watchdog reports as a commit status, the only channel that
# outlives the runner it is reporting on (#795).
statuses: write
# Cancel superseded runs on the same branch or PR.
concurrency:
@@ -40,9 +37,6 @@ jobs:
- uses: actions/checkout@v7
with:
submodules: recursive # coucal lives in src/coucal
# Or the token, which can now post statuses, sits in .git/config for
# the whole job, where every test the suite runs can read it.
persist-credentials: false
# Located through vswhere rather than microsoft/setup-msbuild: the repo
# only allows GitHub-owned actions.
@@ -177,14 +171,6 @@ jobs:
shell: bash
working-directory: tests
timeout-minutes: 45
env:
# Through the environment, never argv, which the process list exposes.
WATCHDOG_TOKEN: ${{ secrets.GITHUB_TOKEN }}
WATCHDOG_REPO: ${{ github.repository }}
# A pull_request's merge commit, so these stay out of the PR's checks UI.
WATCHDOG_SHA: ${{ github.sha }}
WATCHDOG_CONTEXT: windows-suite (${{ matrix.platform }}, ${{ matrix.configuration }})
WATCHDOG_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
run: |
bash ./ci-windows-suite.sh \
"$(cygpath -u "$GITHUB_WORKSPACE")/src/${{ matrix.platform }}/${{ matrix.configuration }}"

View File

@@ -122,31 +122,30 @@ void launch_ftp(FTPDownloadStruct * params) {
return 0; \
}
/* Bounded split of a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
void ftp_split_userpass(const char *src, const char *end, char *user,
size_t user_size, char *pass, size_t pass_size) {
size_t n = 0;
/* Split a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
hts_boolean ftp_split_userpass(const char *src, const char *end, char *user,
size_t user_size, char *pass, size_t pass_size) {
size_t len = 0, user_len, pass_len;
const char *colon;
assertf(user_size > 0 && pass_size > 0); /* the size-1 math underflows on 0 */
assertf(end > src); /* end is one past the '@' */
while (src[n] != '\0' && src[n] != ':') {
if (n < user_size - 1)
user[n] = src[n];
n++;
}
user[n < user_size ? n : user_size - 1] = '\0';
pass[0] = '\0';
if (src[n] == ':') { // password follows the colon
const size_t base = n + 1;
size_t k = 0;
while (&src[base + k + 1] < end && src[base + k] != '\0') {
if (k < pass_size - 1)
pass[k] = src[base + k];
k++;
}
pass[k < pass_size ? k : pass_size - 1] = '\0';
}
user[0] = pass[0] = '\0'; // fail safe for a caller that ignores the result
while (len < (size_t) (end - src) - 1 && src[len] != '\0')
len++;
colon = memchr(src, ':', len);
user_len = colon != NULL ? (size_t) (colon - src) : len;
pass_len = colon != NULL ? len - user_len - 1 : 0;
/* a clipped name is another account's, so refuse rather than log in as it */
if (user_len >= user_size || pass_len >= pass_size)
return HTS_FALSE;
memcpy(user, src, user_len);
user[user_len] = '\0';
if (pass_len != 0)
memcpy(pass, colon + 1, pass_len);
pass[pass_len] = '\0';
return HTS_TRUE;
}
/* Build "<verb> <path>" (see htsftp.h). */
@@ -258,7 +257,11 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
while(*real_adr == '/')
real_adr++; // sauter /
if ((adr = jump_identification(real_adr)) != real_adr) { // user
ftp_split_userpass(real_adr, adr, user, sizeof(user), pass, sizeof(pass));
if (!ftp_split_userpass_buf(real_adr, adr, user, pass)) {
strcpybuff(back->r.msg, "FTP user name or password too long");
back->r.statuscode = STATUSCODE_INVALID;
_HALT_FTP return 0;
}
}
// Calculer RETR <nom>
{

View File

@@ -74,10 +74,18 @@ int run_launch_ftp(FTPDownloadStruct * params);
int send_line(T_SOC soc, const char *data);
int get_ftp_line(T_SOC soc, char *line, size_t line_size, int timeout);
/* Split a "user[:pass]@" prefix (end = jump_identification result) into
bounded, NUL-terminated user/pass buffers, truncating to fit.
NUL-terminated user/pass buffers. Returns HTS_FALSE and empties both when a
field does not fit, as a clipped one would name another account.
Both sizes must be nonzero. */
void ftp_split_userpass(const char *src, const char *end, char *user,
size_t user_size, char *pass, size_t pass_size);
hts_boolean ftp_split_userpass(const char *src, const char *end, char *user,
size_t user_size, char *pass, size_t pass_size);
/* ftp_split_userpass() into the caller's fixed buffers; a buffer too wide for
its "USER <user>" line would be clipped again when the command is built. */
#define ftp_split_userpass_buf(src, end, user, pass) \
(HTS_COMPILE_ASSERT(sizeof(user) + sizeof("USER ") - 1 <= FTP_LINE_SIZE && \
sizeof(pass) + sizeof("PASS ") - 1 <= FTP_LINE_SIZE), \
ftp_split_userpass((src), (end), (user), sizeof(user), (pass), \
sizeof(pass)))
/* Build "<verb> <path>" into line[line_size]. The path is quoted whenever a
bare one would give the server a second token; it must already have been
screened for control bytes. Returns HTS_FALSE and empties line when the

View File

@@ -4501,40 +4501,79 @@ static int st_ftpline(httrackp *opt, int argc, char **argv) {
return 0;
}
/* ftp_split_userpass: well-formed split, plus a hostile over-long userinfo
that pre-fix overran user[256]/pass[256]. */
/* ftp_split_userpass: the split itself, and userinfo refused rather than
clipped into another account's name (#1032). */
static int st_ftpuser(httrackp *opt, int argc, char **argv) {
char user[256], pass[256];
char in[1200];
static const size_t caps[] = {16, 256}; /* asymmetric: a shared bound shows */
char ubuf[256 + 32], pbuf[sizeof(ubuf)], poison[sizeof(ubuf)];
char in[2 * 256 + 8];
size_t c, over;
(void) opt;
(void) argc;
(void) argv;
memset(poison, '#', sizeof(poison));
{
const char ok[] = "bob:secret@host/f"; // '@' at index 10
ftp_split_userpass(ok, ok + 11, user, sizeof(user), pass, sizeof(pass));
assertf(strcmp(user, "bob") == 0);
assertf(strcmp(pass, "secret") == 0);
assertf(ftp_split_userpass(ok, ok + 11, ubuf, sizeof(ubuf), pbuf,
sizeof(pbuf)) == HTS_TRUE);
assertf(strcmp(ubuf, "bob") == 0);
assertf(strcmp(pbuf, "secret") == 0);
}
memset(in, 'u', 400);
in[400] = ':';
memset(in + 401, 'p', 400);
in[801] = '@';
in[802] = '\0';
ftp_split_userpass(in, in + 802, user, sizeof(user), pass, sizeof(pass));
assertf(strlen(user) == sizeof(user) - 1);
assertf(strlen(pass) == sizeof(pass) - 1);
{
/* tight sizes + guard byte catch an off-by-one the 256 case can't */
char ubuf[16], pbuf[16];
const char ok[] = "bob@host/f"; // no password: the '@' still ends the user
memset(ubuf, 'Z', sizeof(ubuf));
memset(pbuf, 'Z', sizeof(pbuf));
ftp_split_userpass(in, in + 802, ubuf, 8, pbuf, 8);
assertf(strcmp(ubuf, "uuuuuuu") == 0);
assertf(strcmp(pbuf, "ppppppp") == 0);
assertf(ubuf[8] == 'Z' && pbuf[8] == 'Z');
assertf(ftp_split_userpass(ok, ok + 4, ubuf, sizeof(ubuf), pbuf,
sizeof(pbuf)) == HTS_TRUE);
assertf(strcmp(ubuf, "bob") == 0);
assertf(pbuf[0] == '\0');
}
{
const char ok[] = "u@relay:pw@gw/f"; // only the last '@' ends the userinfo
assertf(ftp_split_userpass(ok, ok + 11, ubuf, sizeof(ubuf), pbuf,
sizeof(pbuf)) == HTS_TRUE);
assertf(strcmp(ubuf, "u@relay") == 0);
assertf(strcmp(pbuf, "pw") == 0);
}
for (c = 0; c < sizeof(caps) / sizeof(caps[0]); c++) {
const size_t ucap = caps[c], pcap = caps[1 - c];
/* overshoot the user, the pass, then a bare name bounded only by '@' */
for (over = 0; over <= 2; over++) {
const size_t cap = over == 1 ? pcap : ucap;
size_t len;
for (len = cap - 2; len <= cap + 1; len++) {
const size_t user_len = over == 1 ? 1 : len;
const size_t pass_len = over == 0 ? 1 : (over == 1 ? len : 0);
const size_t total = user_len + pass_len + (over == 2 ? 1 : 2);
const hts_boolean fits = len < cap ? HTS_TRUE : HTS_FALSE;
memset(in, 'u', user_len);
if (over != 2) {
in[user_len] = ':';
memset(in + user_len + 1, 'p', pass_len);
}
in[total - 1] = '@';
in[total] = '\0';
memcpy(ubuf, poison, sizeof(ubuf)); /* a zero canary would hide a NUL */
memcpy(pbuf, poison, sizeof(pbuf));
assertf(ftp_split_userpass(in, in + total, ubuf, ucap, pbuf, pcap) ==
fits);
if (fits) {
assertf(strlen(ubuf) == user_len && strlen(pbuf) == pass_len);
assertf(ubuf[user_len - 1] == 'u');
assertf(pass_len == 0 || pbuf[pass_len - 1] == 'p');
} else {
assertf(ubuf[0] == '\0' && pbuf[0] == '\0'); /* fail safe */
}
/* the whole tail: one canary byte misses a write just past it */
assertf(memcmp(ubuf + ucap, poison, sizeof(ubuf) - ucap) == 0);
assertf(memcmp(pbuf + pcap, poison, sizeof(pbuf) - pcap) == 0);
}
}
}
printf("ftp-userpass self-test OK\n");
return 0;

View File

@@ -1,8 +1,43 @@
#!/bin/bash
#
# Over-long FTP userinfo must fail the link, never log in under a clipped name
# (#1032).
set -euo pipefail
# ftp_split_userpass bounds an over-long user:pass@ from a hostile ftp:// URL.
out=$(httrack -O /dev/null -#test=ftp-userpass run)
grep -q "ftp-userpass self-test OK" <<<"$out"
fail() {
echo "FAIL: $*" >&2
exit 1
}
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftpuser.XXXXXX") || exit 1
trap 'set +e; rm -rf "${tmpdir}"' EXIT
trap 'exit 1' HUP INT QUIT TERM
out=$(httrack -O "${tmpdir}/st" "-#test=ftp-userpass" run 2>&1) ||
fail "self-test exited non-zero: ${out}"
grep -q "ftp-userpass self-test OK" <<<"${out}" || fail "unexpected output: ${out}"
# The reachable half: a URL carries the userinfo straight into user[256]/pass[256].
rep() { awk -v n="$1" 'BEGIN { while (i++ < n) printf "a" }'; }
probe() {
rm -rf "${tmpdir}/mir"
httrack "ftp://$1@127.0.0.1:1/f.txt" -O "${tmpdir}/mir" -q >/dev/null 2>&1 ||
fail "an FTP URL with ${#1}-byte userinfo crashed the engine"
cat "${tmpdir}/mir/hts-log.txt"
}
# The bare name has no ':' to bound it, only the '@', and is the worse branch.
for u in "$(rep 256):pw" "u:$(rep 256)" "$(rep 256)"; do
log=$(probe "${u}")
grep -q "FTP user name or password too long" <<<"${log}" ||
fail "over-long userinfo was not refused: ${log}"
done
# 255 still fits, and a plain login is the outsider a widened gate would refuse.
for u in "$(rep 255):pw" "u:$(rep 255)" "$(rep 255)" "bob:secret"; do
log=$(probe "${u}")
grep -q "Unable to connect to the server" <<<"${log}" ||
fail "userinfo that fits did not reach the connect: ${log}"
done

View File

@@ -138,9 +138,7 @@ test "$killed" -le 10080 || fail "killed at $killed, want 9960 within a tick"
# taskkill is a grandchild of its own target, so a leaves-first /T would reap the
# watchdog before the root (#953): kill_tree here never returns, and the stubs
# record the order, which the code under test cannot write to. A taskkill runs no
# trap, so this path posts the verdict itself, after silencing the reporter that
# would otherwise overwrite it.
# record the order, which the code under test cannot write to.
rec="$tmp/killed"
: >"$rec"
vnow=9000
@@ -148,8 +146,6 @@ ticks=0
printf 'RUN 96_wedged.test at 0s\n' >"$progress"
rc=0
(
watchdog=777
ci_post_final_status() { echo "POST $1" >>"$rec"; }
kill_pid() { echo "DIRECT $1" >>"$rec"; }
kill_tree() {
echo "TREE $1" >>"$rec"
@@ -158,10 +154,10 @@ rc=0
ci_suite_heartbeat 960 360 "$progress" 900 4242 >"$tmp/hedge" 2>&1
) || rc=$?
test "$rc" -eq 9 || fail "the tree kill never fired: watchdog returned $rc"
want='DIRECT 777
POST 1
DIRECT 4242
TREE 4242'
test "$(cat "$rec")" = "$want" || fail "the kill path did: $(tr '\n' '/' <"$rec")"
test "$(sed -n 1p "$rec")" = "DIRECT 4242" ||
fail "the target was not signalled directly ahead of the tree walk: $(tr '\n' '/' <"$rec")"
test "$(sed -n 2p "$rec")" = "TREE 4242" ||
fail "the tree was not killed after the direct signal: $(tr '\n' '/' <"$rec")"
test "$(sed -n '$=' "$rec")" -eq 2 || fail "extra kills: $(tr '\n' '/' <"$rec")"
echo "heartbeat OK"

View File

@@ -39,8 +39,7 @@ grep -q usage <<<"$out" || fail "argless run said: $out"
# sibling's.
run=$tmp/run
mkdir -p "$run"
cp "$driver" "$testdir/testlib.sh" "$testdir/test-timeout.sh" \
"$testdir/ci-windows-watchdog.ps1" "$run/"
cp "$driver" "$testdir/testlib.sh" "$testdir/test-timeout.sh" "$run/"
chmod u+w "$run"/*.sh # distcheck's srcdir is read-only, and cp carries that over
cat >>"$run/testlib.sh" <<'EOF'
reap_leftover_processes() { return 0; }
@@ -52,23 +51,6 @@ printf '#!/bin/sh\nexit 0\n' >"$bin/httrack"
# shellcheck disable=SC2016 # $2 is the stub's own argument
printf '#!/bin/sh\necho "$2"\n' >"$bin/cygpath"
chmod +x "$bin/httrack" "$bin/cygpath"
# The driver puts this bindir on PATH, so a stub here is the interpreter it finds
# where there is no real one; a real PowerShell drives the real script instead.
wdargv=$tmp/wdargv
: >"$wdargv"
if ! command -v pwsh >/dev/null 2>&1 && ! command -v powershell.exe >/dev/null 2>&1; then
cat >"$bin/pwsh" <<EOF
#!/bin/sh
{ echo "TOKEN=\${WATCHDOG_TOKEN:-}"; printf '%s\n' "\$@"; } >>"$wdargv"
case " \$* " in
*" -Post failure "*) echo "final status failure"; exit 0 ;;
*" -Post "*) echo "final status other"; exit 0 ;;
esac
echo "watchdog ready"
exec sleep 60
EOF
chmod +x "$bin/pwsh"
fi
test -x "$bin/httrack" || {
echo "SKIP: ${TMPDIR:-/tmp} is noexec, the stub bindir cannot be run"
exit 77
@@ -76,21 +58,16 @@ test -x "$bin/httrack" || {
# One per glob the driver enumerates: an empty category is counted as a failing
# test named after the unexpanded pattern, which would drown the accounting.
for t in 00_runnable 13_zlib-pass 14_local-pass 15_watchdog-pass \
for t in 00_runnable 10_engine-pass 13_zlib-pass 14_local-pass 15_watchdog-pass \
16_crawl_proxy_https 17_crawl-log-salvage; do
printf '#!/bin/sh\nexit 0\n' >"$run/$t.test"
done
# One test reports back what it inherited: the token the step is handed can post
# commit statuses, and a forged one has already reached a commit under test.
# shellcheck disable=SC2016 # the stub expands it, not this shell
printf '#!/bin/sh\necho "TOKEN=${WATCHDOG_TOKEN:-}" >%s\nexit 0\n' "$tmp/childtoken" \
>"$run/10_engine-pass.test"
printf '#!/bin/sh\nexit 77\n' >"$run/11_engine-skip.test"
printf '#!/bin/sh\nexit 3\n' >"$run/12_engine-fail.test"
cd "$run"
rc=0
out=$(RUNNER_TEMP="$tmp" GITHUB_STEP_SUMMARY="$tmp/summary" WATCHDOG_TOKEN=s3cr3t \
out=$(RUNNER_TEMP="$tmp" GITHUB_STEP_SUMMARY="$tmp/summary" \
bash ./ci-windows-suite.sh "$bin" 2>&1) || rc=$?
# The skip set and the pass floor are pinned to the real suite, so a stub run
# ends on the floor; what is under test is the tally that reaches it.
@@ -100,18 +77,6 @@ grep -q '^ran=9 pass=7 fail=1 skip=1$' <<<"$out" ||
grep -q '::error::only 7 tests passed (1 skipped)' <<<"$out" ||
fail "the pass floor did not report the count"
grep -q 'FAIL 12_engine-fail.test (exit 3)' <<<"$out" || fail "the failing test was not named"
# The reporter as the real driver launches it, rather than through a test
# sourcing the helper: launched, ended, and holding a token nothing else sees.
grep -q '^watchdog ready$' watchdog.log || fail "no watchdog was launched: $(cat watchdog.log)"
grep -q 'final status failure' watchdog.log ||
fail "the driver's own verdict was never posted: $(cat watchdog.log)"
test -r "$tmp/childtoken" || fail "the test that reads its environment did not run"
grep -qx 'TOKEN=' "$tmp/childtoken" ||
fail "a test inherited the status token: $(cat "$tmp/childtoken")"
if test -s "$wdargv"; then
grep -qx 'TOKEN=s3cr3t' "$wdargv" || fail "the watchdog was launched without its token"
fi
# Positive control for the gate below: with every category matched it stays silent.
grep -q 'matched no tests' <<<"$out" && fail "a full suite reported an empty category"

View File

@@ -1,528 +0,0 @@
#!/bin/bash
#
# The off-box suite watchdog (#795): its pure decisions through -SelfTest, the
# process APIs it must never name, and the driver wiring no .ps1 can check for
# itself. It only reports, so there is no kill path here to exercise.
set -euo pipefail
# First, before anything here can start an interpreter: a token reaching one
# posts forged statuses onto the commit under test.
unset WATCHDOG_TOKEN WATCHDOG_REPO WATCHDOG_SHA WATCHDOG_URL
export WATCHDOG_CONTEXT='226_watchdog-native (never posted)'
testdir=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=tests/testlib.sh
. "$testdir/testlib.sh"
top=${abs_top_srcdir:-$(cd "$testdir/.." && pwd)}
wdscript="$testdir/ci-windows-watchdog.ps1"
driver="$testdir/ci-windows-suite.sh"
workflow="$top/.github/workflows/windows-build.yml"
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_wdnat.XXXXXX")
trap 'set +e; stop_server "${sinkpid:-}"; rm -rf "$tmp"' EXIT
fail() {
echo "FAIL: $*" >&2
exit 1
}
test -r "$wdscript" || fail "no $wdscript"
test -z "${WATCHDOG_TOKEN:-}${WATCHDOG_SHA:-}" || fail "a credential survived the scrub"
# The prose in the script names the very calls the audits forbid. A trailing
# comment goes only on a quote-free line: cutting at any '#' would hide a call
# sitting after an issue number inside a string.
uncommented() { sed -e 's/^[[:space:]]*#.*$//' -e "/['\"]/! s/[[:space:]]#.*$//" "$1"; }
# Telemetry only, so every spawn and kill idiom is out, aliases and short forms
# included: PowerShell needs neither the System. prefix nor a space after '&'.
forbidden_calls() {
uncommented "$1" | grep -nEi \
'Start-Process|Start-Job|Start-ThreadJob|Start-Service|Stop-Process|Invoke-Expression|Invoke-Item|Invoke-Command|Invoke-CimMethod|Get-WmiObject|Register-ScheduledJob|WScript\.Shell|taskkill|cmd\.exe|wmic|schtasks|Diagnostics\.Process|\.Kill\(|(^|[^-a-zA-Z0-9_])(iex|saps|spps|spjb|sajb|start)([^-a-zA-Z0-9_]|$)|(^|[^-a-zA-Z0-9_])&[^-a-zA-Z0-9_=&]'
}
if hits=$(forbidden_calls "$wdscript"); then
fail "the watchdog names a process API: $hits"
fi
# One control per idiom: a denylist is worth only the entries that actually fire.
# shellcheck disable=SC2016 # PowerShell source, not this shell's expansions
for probe in 'Start-Process notepad.exe' 'Stop-Process -Id 4 -Force' \
'[Diagnostics.Process]::Start("x")' '(Get-Process -Name httrack).Kill()' \
'Invoke-Expression $cmd' '& "C:\\x.exe" -y' 'Start-Job { 1 }' 'taskkill /F /IM x.exe' \
'iex $cmd' '$cmd | iex' '&"C:\\x.exe"' '&$exe' 'saps notepad.exe' 'start notepad.exe' \
'spps -Name httrack' 'spjb { 1 }' 'sajb { 1 }' 'Get-WmiObject Win32_Process' \
'Invoke-CimMethod -ClassName Win32_Process -MethodName Create' \
'Invoke-Command -ScriptBlock { 1 }' 'Register-ScheduledJob -Name x -ScriptBlock { 1 }' \
'(New-Object -ComObject WScript.Shell).Run("x")' \
'Write-Host "#795"; Start-Process x'; do
{
cat "$wdscript"
echo "$probe"
} >"$tmp/probe.ps1"
if ! forbidden_calls "$tmp/probe.ps1" >/dev/null; then
fail "the process-API audit cannot see: $probe"
fi
done
# shellcheck disable=SC2016 # $env: is PowerShell's, not this shell's
grep -q '\$env:WATCHDOG_TOKEN' "$wdscript" ||
fail "the watchdog does not read its token from the environment"
if sed -n '/^param(/,/^)/p' "$wdscript" | grep -qi 'token'; then
fail "the watchdog takes its token as a parameter, where the process list exposes it"
fi
grep -q "ApiBase = 'https://api.github.com'" "$wdscript" ||
fail "the watchdog's default endpoint is not the GitHub API"
# The token stays in the driver's own shell: exported, every test it runs would
# inherit a credential that can post statuses. 172 drives that end to end.
grep -q '^unset WATCHDOG_TOKEN$' "$driver" ||
fail "the driver leaves its token in the environment its tests inherit"
# A background holder of the step's stdout keeps the step open past the suite (#949).
grep -q '>>watchdog.log 2>&1' "$driver" ||
fail "the watchdog's output is not redirected off the step's stdout"
if test -r "$workflow"; then
# The heartbeat still owns the kill and must beat the step timeout. Anchored,
# so an ordinary workflow edit mentioning the key in prose is not a second one.
n=$(grep -cE '^[[:space:]]*timeout-minutes:[[:space:]]*[0-9]+[[:space:]]*$' "$workflow")
test "$n" -eq 1 || fail "$n timeout-minutes in the workflow, cannot tell which bounds the suite"
cap=$(($(awk '/^[[:space:]]*timeout-minutes:/ { print $2; exit }' "$workflow") * 60))
stuck=$(sed -n 's/^stuck=\([0-9]*\)$/\1/p' "$driver")
deadline=$(sed -n 's/^suite_deadline=\([0-9]*\)$/\1/p' "$driver")
test -n "$stuck" || fail "cannot read the heartbeat's static window"
test -n "$deadline" || fail "cannot read the driver's suite deadline"
# A test may start one second under the deadline and never report again.
test $((deadline + stuck + 240)) -le "$cap" ||
fail "a kill at ${deadline}+${stuck}s leaves under 240s before the ${cap}s step timeout"
# The watchdog's own stop is its MaxSeconds, which must land inside the step.
# shellcheck disable=SC2016 # $MaxSeconds is PowerShell's
maxsecs=$(sed -n 's/^[[:space:]]*\[int\]\$MaxSeconds = \([0-9]*\).*$/\1/p' "$wdscript")
test -n "$maxsecs" || fail "cannot read the watchdog's own deadline"
test "$maxsecs" -le "$cap" || fail "the watchdog runs ${maxsecs}s past the ${cap}s step timeout"
# Indentation-agnostic: pinning it would block ever scoping these to the job.
grep -Eq '^[[:space:]]+statuses:[[:space:]]*write$' "$workflow" ||
fail "the workflow token cannot post a commit status"
grep -Eq '^[[:space:]]+contents:[[:space:]]*read$' "$workflow" ||
fail "contents: read was dropped from the workflow"
# A token that may post statuses must not also be left in .git/config.
grep -Eq '^[[:space:]]+persist-credentials:[[:space:]]*false$' "$workflow" ||
fail "the checkout persists the job token for the whole job"
for v in WATCHDOG_TOKEN WATCHDOG_REPO WATCHDOG_SHA WATCHDOG_CONTEXT; do
grep -q "$v:" "$workflow" || fail "$v is not passed to the suite step"
done
elif test -d "$top/.github"; then
fail "$workflow is gone but $top/.github is not"
else
echo "note: no .github under $top (dist tarball), the workflow audits did not run"
fi
# PATH holds MSYS-style entries only, and TMPDIR on Windows is a D:/... path
# that would split it on the drive colon.
posixtmp=$tmp
if is_windows && command -v cygpath >/dev/null 2>&1; then
posixtmp=$(cygpath -u "$tmp")
fi
bin="$posixtmp/bin"
mkdir -p "$bin"
# Renamed into place, so the poll below cannot read a half-written file.
printf '#!/bin/sh\nprintf "%%s\\n" "$@" >%s/argv.part\nmv %s/argv.part %s/argv\necho "watchdog ready"\n' \
"$posixtmp" "$posixtmp" "$posixtmp" >"$bin/argvstub"
printf '#!/bin/sh\nexit 127\n' >"$bin/deadstub"
chmod +x "$bin/argvstub" "$bin/deadstub"
usestub() { cp "$bin/$1" "$bin/pwsh" && cp "$bin/$1" "$bin/powershell.exe"; }
usestub argvstub
if test -x "$bin/pwsh"; then
(
cd "$posixtmp"
# shellcheck disable=SC2030 # local to this subshell is the point
PATH="$bin:$PATH"
export PATH
# An unshadowed pwsh would drive the real watchdog, and every assertion
# below would still read as a plain launch failure.
test "$(command -v pwsh)" = "$bin/pwsh" ||
fail "the stub does not shadow PowerShell: $(command -v pwsh)"
# shellcheck source=tests/ci-windows-suite.sh
. "$driver"
ci_watchdog_pid='' ci_watchdog_exe=''
ci_start_native_watchdog "$posixtmp/progress.log" ||
fail "the launch reported no watchdog with a PowerShell on PATH"
test -n "$ci_watchdog_pid" || fail "the launch set no pid"
# wait only reaps a child, which is what the teardown has to signal.
wait "$ci_watchdog_pid" || fail "the watchdog is not a child of the driver"
# A pid comes from the fork, so a dead interpreter still yields one.
usestub deadstub
ci_watchdog_pid='dangling'
rc=0
ci_start_native_watchdog "$posixtmp/progress.log" || rc=$?
test "$rc" -ne 0 || fail "an interpreter exiting 127 was reported as launched"
test -z "$ci_watchdog_pid" || fail "a failed launch left pid $ci_watchdog_pid behind"
)
for _ in $(seq 1 100); do
test -r "$posixtmp/argv" && break
sleep 0.1
done
test -r "$posixtmp/argv" || fail "the watchdog was never launched"
grep -qx -- '-SelfTest' "$posixtmp/argv" && fail "the production launch runs the self-test"
grep -qx -- '-Post' "$posixtmp/argv" && fail "the production launch is the one-shot mode"
# Bypass is load-bearing: Windows PowerShell refuses an unsigned .ps1 without it.
for flag in -NoProfile -NonInteractive -ExecutionPolicy Bypass -File; do
grep -qx -- "$flag" "$posixtmp/argv" ||
fail "the launch dropped $flag: $(tr '\n' ' ' <"$posixtmp/argv")"
done
grep -qx -- '-ApiBase' "$posixtmp/argv" &&
fail "the production launch redirects the status API away from GitHub"
# Native form, because a non-MSYS interpreter cannot resolve a /d/a/... path.
grep -qx -- "$(nativepath "$wdscript")" "$posixtmp/argv" ||
fail "the launch did not name the script: $(tr '\n' ' ' <"$posixtmp/argv")"
grep -qx -- "$(nativepath "$posixtmp/progress.log")" "$posixtmp/argv" ||
fail "the launch did not pass the progress log"
# Both outcomes: a pending status that always dangles cannot mean anything.
usestub argvstub
for probe in 0:success 3:failure; do
rm -f "$posixtmp/argv"
(
cd "$posixtmp"
# shellcheck source=tests/ci-windows-suite.sh
. "$driver"
# shellcheck disable=SC2034 # read by the driver's ci_post_final_status
ci_watchdog_exe="$bin/pwsh"
ci_post_final_status "${probe%%:*}"
)
test -r "$posixtmp/argv" || fail "no final status posted for rc ${probe%%:*}"
grep -qx -- "${probe#*:}" "$posixtmp/argv" ||
fail "rc ${probe%%:*} posted $(tr '\n' ' ' <"$posixtmp/argv"), want ${probe#*:}"
done
# The runner cancels a superseded or overrunning step by signal, and bash
# reaches its EXIT trap from one with $? reading 0.
sigs=()
for sig in TERM HUP INT; do
# A shell that inherited a signal ignored can neither trap nor take it,
# which is what a nohup'd "make check" hands us for HUP.
if test -z "$(trap -p "$sig")"; then
sigs+=("$sig")
fi
done
test "${#sigs[@]}" -ge 1 || fail "every signal is ignored here, the teardown was not driven"
for sig in "${sigs[@]}"; do
rm -f "$posixtmp/argv"
(
cd "$posixtmp"
# shellcheck source=tests/ci-windows-suite.sh
. "$driver"
sleep 30 &
# shellcheck disable=SC2034 # all three are read by the driver's teardown
watchdog=$! heartbeat='' ci_watchdog_exe="$bin/pwsh"
ci_install_traps
kill -"$sig" "$BASHPID"
sleep 5
) >/dev/null 2>&1 || true
test -r "$posixtmp/argv" || fail "a $sig-killed suite posted nothing"
grep -qx -- failure "$posixtmp/argv" ||
fail "a $sig-killed suite posted $(tr '\n' ' ' <"$posixtmp/argv"), want failure"
done
else
echo "note: $tmp is noexec, the driver wiring was not driven"
fi
psruns=()
for c in pwsh powershell.exe powershell; do
command -v "$c" >/dev/null 2>&1 || continue
# powershell and powershell.exe are one interpreter under two names.
seen=0
for p in ${psruns[@]+"${psruns[@]}"}; do
test "${p%.exe}" != "${c%.exe}" || seen=1
done
test "$seen" -eq 1 || psruns+=("$c")
done
if test "${#psruns[@]}" -eq 0; then
echo "SKIP: no PowerShell here, the watchdog's own self-test cannot run"
exit 77
fi
# Bounded: a mutant that loops forever would otherwise be caught only by the step
# timeout this work exists to beat. -NoPost layers over the scrub above.
run_wd() {
local secs=$1
shift
# -NoPost last: everything after -File <script> belongs to the script.
run_with_timeout "$secs" "$psrun" "${psargs[@]}" "$@" -NoPost 2>&1
}
# The legs below grade what reached the API. Printing and posting are decoupled,
# so counting what the loop wrote can see neither the request rate nor the backoff.
py=$(find_python || true)
sink="$tmp/sink.py"
cat >"$sink" <<'PY'
import http.server
import sys
code, rec, portfile = int(sys.argv[1]), sys.argv[2], sys.argv[3]
class Handler(http.server.BaseHTTPRequestHandler):
def do_POST(self):
n = int(self.headers.get("Content-Length") or 0)
with open(rec, "ab") as f:
f.write(self.rfile.read(n) + b"\n")
self.send_response(code)
self.send_header("Content-Length", "2")
self.end_headers()
self.wfile.write(b"{}")
def log_message(self, *args):
pass
srv = http.server.HTTPServer(("127.0.0.1", 0), Handler)
with open(portfile, "w") as f:
f.write(str(srv.server_port))
srv.serve_forever()
PY
posts="$tmp/posts"
portfile="$tmp/port"
legout="$tmp/leg.out"
port='' sinkpid=''
sink_start() {
: >"$posts"
rm -f "$portfile"
"$py" "$sink" "$1" "$posts" "$portfile" &
sinkpid=$!
for _ in $(seq 1 100); do
test -s "$portfile" && break
sleep 0.1
done
test -s "$portfile" || fail "the status sink never bound a port"
port=$(cat "$portfile")
}
sink_stop() {
stop_server "$sinkpid"
sinkpid=''
}
# Ours, so a leg that lost its -ApiBase reaches GitHub with a credential it
# refuses rather than with whatever the step was handed.
export WATCHDOG_TOKEN=dummy WATCHDOG_REPO=octo/nowhere WATCHDOG_SHA=0000000
run_posting() {
local secs=$1
shift
run_with_timeout "$secs" "$psrun" "${psargs[@]}" "$@" -ApiBase "http://127.0.0.1:$port"
}
# t and q of each posted status, in order and whatever key order the JSON took.
posted_pairs() {
sed -n 's/.*t=\([0-9]*\)s q=\([0-9?]*\)s.*/\1 \2/p' "$posts"
}
# One property per leg, each returning its diagnosis so a mutant can be required
# to trip it.
cadence_leg() {
local n
sink_start 201
printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
run_posting 60 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
-IntervalSeconds 100 -PollSeconds 1 -MaxSeconds 4 >"$legout" 2>&1 || {
sink_stop
echo "the loop exited nonzero: $(cat "$legout")"
return 1
}
sink_stop
n=$(grep -c . "$posts" || true)
test "$n" -eq 1 || {
echo "$n API calls at a 100s cadence over 4s, want 1"
return 1
}
grep -q '36_local-bigcrawl.test' "$posts" || {
echo "the posted status does not name the test in flight: $(cat "$posts")"
return 1
}
}
staticness_leg() {
local stop moved lastq legpid
sink_start 201
printf 'RUN 36_local-bigcrawl.test at 0s\n' >"$tmp/progress.log"
run_posting 90 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
-IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 12 >"$legout" 2>&1 &
legpid=$!
# Six seconds of a log being written, then silence. Staticness has to follow
# the log through both, and elapsed time follows neither.
stop=$((SECONDS + 6))
while test "$SECONDS" -lt "$stop"; do
printf 'RUN 36_local-bigcrawl.test at %ss\n' "$SECONDS" >>"$tmp/progress.log"
sleep 0.4
done
wait "$legpid" || {
sink_stop
echo "the loop exited nonzero: $(cat "$legout")"
return 1
}
sink_stop
moved=$(posted_pairs | awk '$2 ~ /^[0-9]+$/ && $1 >= 2 && $2 <= 1 { n++ } END { print n + 0 }')
test "$moved" -ge 1 || {
echo "no post read the log as moving while it was being written: $(posted_pairs | tr '\n' '/')"
return 1
}
lastq=$(posted_pairs | awk 'END { print $2 }')
case $lastq in
'' | *[!0-9]*)
echo "the last post carries no staticness: $(posted_pairs | tr '\n' '/')"
return 1
;;
esac
test "$lastq" -ge 3 || {
echo "staticness stalled at ${lastq}s over six seconds of silence: $(posted_pairs | tr '\n' '/')"
return 1
}
}
backoff_leg() {
local calls lines
sink_start 403
printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
run_posting 60 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
-IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 8 >"$legout" 2>&1 || {
sink_stop
echo "the loop exited nonzero: $(cat "$legout")"
return 1
}
sink_stop
calls=$(grep -c . "$posts" || true)
lines=$(grep -c 't=[0-9]*s q=' "$legout" || true)
test "$calls" -ge 2 || {
echo "a rejecting API was called $calls times: the loop stopped trying"
return 1
}
test "$calls" -le 5 || {
echo "$calls calls to an API that rejects every one: nothing backs off"
return 1
}
test "$lines" -ge $((2 * calls)) || {
echo "$lines status lines for $calls calls: the artifact throttles with the API"
return 1
}
}
fullstdout_leg() {
local n
sink_start 201
printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
run_posting 60 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
-IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 5 >/dev/full 2>&1 || true
sink_stop
n=$(grep -c . "$posts" || true)
test "$n" -ge 3 || {
echo "$n posts with a full stdout: a failed log write took the loop with it"
return 1
}
}
first=1
for psrun in "${psruns[@]}"; do
psargs=(-NoProfile -NonInteractive)
case $psrun in
powershell*) psargs+=(-ExecutionPolicy Bypass) ;;
esac
out=$(run_wd 120 -File "$(nativepath "$wdscript")" -SelfTest) ||
fail "the $psrun self-test failed: $out"
grep -q 'watchdog self-test OK' <<<"$out" || fail "the $psrun self-test said: $out"
# A mutant the script survives is a property nothing is checking.
mutate() {
local name=$1 expr=$2 m="$tmp/$1.ps1" rc=0 said
sed "$expr" "$wdscript" >"$m"
cmp -s "$wdscript" "$m" && fail "mutant $name changed nothing, so it proves nothing"
said=$(run_wd 120 -File "$(nativepath "$m")" -SelfTest) || rc=$?
test "$rc" -ne 0 || fail "mutant $name passed the $psrun self-test: $said"
}
# shellcheck disable=SC2016 # PowerShell variables, quoted for sed
mutate cadence-off-by-one 's/-ge \$Interval/-gt \$Interval/'
# shellcheck disable=SC2016
mutate backoff-uncapped 's/\[Math\]::Min(\$Current \* 2, 32)/$Current * 2/'
# shellcheck disable=SC2016
mutate status-unclipped '/if (\$s.Length -gt 140)/d'
# shellcheck disable=SC2016
mutate tail-always-ok 's/\$r = @{ Ok = \$false;/$r = @{ Ok = $true;/'
# Timed and counted by this shell, never read out of what the watchdog
# writes: the cadence and the deadline are both mine to set.
printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
out=$(run_wd 60 -File "$(nativepath "$wdscript")" \
-ProgressLog "$(nativepath "$tmp/progress.log")" \
-IntervalSeconds 100 -PollSeconds 1 -MaxSeconds 5) ||
fail "the $psrun slow-cadence loop exited nonzero: $out"
slow=$(grep -c 't=[0-9]*s q=' <<<"$out" || true)
test "$slow" -eq 1 || fail "$slow status lines at a 100s cadence over 5s, want 1: $out"
began=$SECONDS
out=$(run_wd 60 -File "$(nativepath "$wdscript")" \
-ProgressLog "$(nativepath "$tmp/progress.log")" \
-IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 5) ||
fail "the $psrun loop exited nonzero: $out"
spent=$((SECONDS - began))
test "$spent" -ge 4 || fail "a 5s watchdog returned after ${spent}s: MaxSeconds is not seconds"
test "$spent" -le 20 || fail "a 5s watchdog ran ${spent}s: it does not stop on its own"
fast=$(grep -c 't=[0-9]*s q=' <<<"$out" || true)
# Against the slow leg, not a floor: a probe tick is not instant.
test "$fast" -gt "$slow" || fail "$fast status lines at 1s against $slow at 100s: $out"
grep -q '36_local-bigcrawl.test' <<<"$out" || fail "the status does not name the test in flight: $out"
grep -q '95_local-sitemap' <<<"$out" && fail "the status named a test that was never in flight"
# Trips if the scrub and -NoPost were both lost: only a real attempt logs this.
grep -q 'status post failed' <<<"$out" && fail "a test leg reached the status API"
# An unreadable log must not read as a wedged one.
out=$(run_wd 60 -File "$(nativepath "$wdscript")" \
-ProgressLog "$(nativepath "$tmp/no-such.log")" \
-IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 3) ||
fail "the $psrun loop exited nonzero on a missing log: $out"
grep -q 'q=?s' <<<"$out" || fail "a missing log reported a staticness: $out"
# The legs below read the sink, not the loop, and they exercise the script's
# decisions rather than the interpreter's: one flavour is enough.
test "$first" -eq 1 || continue
first=0
if test -z "$py"; then
echo "note: no python3 here, the posting legs did not run"
continue
fi
legs=(cadence staticness backoff)
# ENOSPC on stdout, which is a state the sick runner reaches.
test -w /dev/full && legs+=(fullstdout)
for leg in "${legs[@]}"; do
said=$("${leg}_leg" "$(nativepath "$wdscript")") ||
fail "the $psrun watchdog fails the $leg leg: $said"
done
# Same shape as mutate() above, graded by a leg instead of the self-test.
mutate_leg() {
local name=$1 expr=$2 leg=$3 m="$tmp/$1.ps1" said
sed "$expr" "$wdscript" >"$m"
cmp -s "$wdscript" "$m" && fail "mutant $name changed nothing, so it proves nothing"
said=$("${leg}_leg" "$(nativepath "$m")") &&
fail "mutant $name passed the $leg leg"
case $said in
*'sink never bound'*) fail "the status sink failed under mutant $name" ;;
esac
}
mutate_leg status-every-poll \
"s/if ((Get-WatchdogAction \$now \$postedAt \$IntervalSeconds) -eq 'post')/if (\$true)/" cadence
# shellcheck disable=SC2016 # PowerShell variables, quoted for sed
mutate_leg static-frozen 's/\$static = \$now - \$movedAt/$static = 0/' staticness
# shellcheck disable=SC2016
mutate_leg static-unknown 's/\$static = \$now - \$movedAt/$static = -1/' staticness
# shellcheck disable=SC2016
mutate_leg static-elapsed 's/\$static = \$now - \$movedAt/$static = $now/' staticness
# shellcheck disable=SC2016
mutate_leg backoff-never-skips 's/if (\$skip -gt 0)/if ($false)/' backoff
if test -w /dev/full; then
mutate_leg log-write-fatal 's/try { \(Write-Host .*\) } catch { }/\1/' fullstdout
fi
done
echo "native watchdog OK (${psruns[*]})"

View File

@@ -0,0 +1,88 @@
#!/bin/bash
#
# The credentials an URL carries must reach the control channel whole: a clip
# logged in as a different account (#1032). The engine test only ever probes a
# dead port, so nothing there reads what the login actually sends.
set -euo pipefail
: "${top_srcdir:=..}"
testdir=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=tests/testlib.sh
. "${testdir}/testlib.sh"
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
command -v httrack >/dev/null || {
echo "could not find httrack" >&2
exit 1
}
server=$(nativepath "${testdir}/ftp-server.py")
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftpuser.XXXXXX")
serverpid=
cleanup() {
stop_server "$serverpid"
rm -rf "$tmpdir"
}
trap 'set +e; cleanup' EXIT
trap cleanup HUP INT QUIT PIPE TERM
root="${tmpdir}/root"
out="${tmpdir}/crawl"
cmds="${tmpdir}/cmds"
mkdir -p "$root" "$out"
printf 'body\n' >"${root}/f.txt"
serverlog="${tmpdir}/server.out"
"$python" "$server" --root "$(nativepath "$root")" --require-pass \
--log "$(nativepath "$cmds")" >"$serverlog" 2>&1 &
serverpid=$!
port=$(discover_server_port "$serverlog" "$serverpid") || exit 1
fail() {
echo "FAIL: $*" >&2
exit 1
}
ok() { echo "OK: $*"; }
rep() { awk -v n="$2" -v c="$1" 'BEGIN { while (i++ < n) printf "%s", c }'; }
crawl() {
: >"$cmds"
rm -rf "${out:?}"
mkdir -p "$out"
run_with_timeout 60 httrack "ftp://$1@127.0.0.1:${port}/f.txt" -O "$out" \
--quiet --disable-security-limits --robots=0 --timeout=20 \
--max-time=45 --retries=1 -c1 >"${tmpdir}/log" 2>&1
}
# --- a plain login goes out unchanged ----------------------------------------
crawl "bob:secret" || fail "the plain-login crawl never finished"
sent=$(cat "$cmds")
grep -qxF "USER bob" <<<"$sent" || fail "no USER bob on the wire: ${sent}"
grep -qxF "PASS secret" <<<"$sent" || fail "no PASS secret on the wire: ${sent}"
cmp -s "${out}/127.0.0.1_${port}/f.txt" "${root}/f.txt" ||
fail "the login did not bring the body back; mirror holds $(find "$out" -type f)"
ok "bob:secret reaches the server verbatim, and mirrors"
# --- the longest userinfo that fits, byte for byte ---------------------------
# 255 of each: a 256th byte is refused, so this is what the wire must carry.
user=$(rep a 255)
pass=$(rep b 255)
crawl "${user}:${pass}" || fail "the 255-byte login crawl never finished"
sent=$(cat "$cmds")
grep -qxF "USER ${user}" <<<"$sent" ||
fail "the wire user is not the 255 bytes the URL named: $(grep -a '^USER' <<<"$sent")"
grep -qxF "PASS ${pass}" <<<"$sent" ||
fail "the wire pass is not the 255 bytes the URL named: $(grep -a '^PASS' <<<"$sent")"
ok "a 255-byte user and password reach the server unclipped"
# --- over-long userinfo never opens a session --------------------------------
# The bare name has no ':' to bound it, only the '@', and is the worse branch.
for u in "$(rep a 256):pw" "$(rep a 256)"; do
crawl "$u" || fail "the over-long crawl never finished"
grep -aq "FTP user name or password too long" "${out}/hts-log.txt" ||
fail "over-long userinfo was not refused: $(cat "${out}/hts-log.txt")"
sent=$(cat "$cmds")
test -z "$sent" || fail "the engine logged in under a clipped name: ${sent}"
done
ok "an over-long name is refused before anything reaches the control channel"

View File

@@ -10,7 +10,7 @@ EXTRA_DIST = $(TESTS) renamefail.c threadattrfail.c nobacktrace.c altstackprobe.
header-injection-server.py header-injection-check.py \
pty-resize.py test-timeout.sh png-colors.py \
local-crawl.sh local-server.py ftp-server.py testlib.sh \
ci-windows-suite.sh ci-windows-watchdog.ps1 \
ci-windows-suite.sh \
server.crt server.key \
server-root/simple/basic.html server-root/simple/link.html \
server-root/stripquery/index.html server-root/stripquery/a.html \

View File

@@ -29,81 +29,6 @@ ci_annotate() {
# schedule the workflow really passes off a virtual clock instead of waiting it out.
hb_now() { echo "$SECONDS"; }
# Start the off-box telemetry over the progress log $1, setting ci_watchdog_pid
# and ci_watchdog_exe; 1 with no PowerShell. Forks nothing and kills nothing, so
# it still reports where the heartbeat below cannot (#795).
ci_start_native_watchdog() {
local progress=$1 ps1 c waited=0
ps1="$testdir/ci-windows-watchdog.ps1"
test -r "$ps1" || return 1
ci_watchdog_exe=''
for c in pwsh powershell.exe; do
if command -v "$c" >/dev/null 2>&1; then
ci_watchdog_exe=$c
break
fi
done
test -n "$ci_watchdog_exe" || return 1
# Never the step's stdout: a background holder of that pipe keeps the step
# open past the suite (#949), and tests/*.log reaches the artifact anyway.
: >watchdog.log
WATCHDOG_TOKEN="${ci_watchdog_token:-}" \
"$ci_watchdog_exe" -NoProfile -NonInteractive -ExecutionPolicy Bypass \
-File "$(nativepath "$ps1")" \
-ProgressLog "$(nativepath "$progress")" \
>>watchdog.log 2>&1 &
ci_watchdog_pid=$!
# $! comes from the fork, not the exec: wait for it to actually speak.
while test "$waited" -lt 30; do
grep -q '^watchdog ready$' watchdog.log && return 0
kill -0 "$ci_watchdog_pid" 2>/dev/null || break
sleep 1
waited=$((waited + 1))
done
kill_pid "$ci_watchdog_pid"
ci_watchdog_pid=''
return 1
}
# Post the suite's own verdict from exit status $1, described by $2: the watchdog
# cannot see the driver exit, and a pending status that never resolves says nothing.
ci_post_final_status() {
local rc=$1 msg=${2:-"suite step ended rc=$1"} state=success
test -n "${ci_watchdog_exe:-}" || return 0
test "$rc" -eq 0 || state=failure
WATCHDOG_TOKEN="${ci_watchdog_token:-}" \
"$ci_watchdog_exe" -NoProfile -NonInteractive -ExecutionPolicy Bypass \
-File "$(nativepath "$testdir/ci-windows-watchdog.ps1")" \
-Post "$state" -Message "$msg" \
>>watchdog.log 2>&1 || true
}
# End both watchers and post the suite's verdict from exit status $1.
ci_stop_watchers() {
local rc=$1 msg="suite step ended rc=$1"
test -z "${ci_signal:-}" || msg="suite step killed by SIG$ci_signal"
kill "${heartbeat:-}" 2>/dev/null
# Silenced before the verdict, or its next tick overwrites it.
test -z "${watchdog:-}" || kill_pid "$watchdog"
# Posted even for a launch given up on: it may have posted a pending first.
ci_post_final_status "$rc" "$msg"
return 0
}
ci_on_signal() {
ci_signal=$1
exit $((128 + $2))
}
# A shell killed by a signal reaches its EXIT trap with $? = 0, so a cancelled or
# timed-out step would report success: the verdict has to come from the handler.
ci_install_traps() {
trap 'ci_on_signal INT 2' INT
trap 'ci_on_signal TERM 15' TERM
trap 'ci_on_signal HUP 1' HUP
trap 'ci_rc=$?; set +e; ci_stop_watchers "$ci_rc" || true' EXIT
}
ci_suite_heartbeat() {
local quiet=$1 every=$2 progress=$3 stuck=$4 main=$5
local tick=$2 begin now line said moved last=''
@@ -123,10 +48,6 @@ ci_suite_heartbeat() {
# outlive the step's own timeout before being caught.
if test $((now - moved)) -ge "$stuck"; then
ci_annotate error "suite watchdog" "killing the step: $((now - moved))s without progress, in flight: $last"
# A taskkill runs no trap, so this is the only verdict the kill path
# can leave; the reporter goes first or its next tick overwrites it.
test -z "${watchdog:-}" || kill_pid "$watchdog"
ci_post_final_status 1 "killed after $((now - moved))s without progress, in flight: $last"
# Direct first: kill_tree may reap this watchdog before its own root (#953).
kill_pid "$main"
kill_tree "$main"
@@ -198,21 +119,9 @@ export HTTRACK_PROGRESS_LOG="$PWD/$progress"
# its own terms keeps its log. Quiet past 16 min, clear of the 13 a healthy
# run measures here, and a kill 900s after the last progress line clears
# the longest legitimate gap, one $per_test.
stuck=900
# Orthogonal to that heartbeat rather than a spare of it: the heartbeat needs
# 960s of quiet and 900s of static log, and every #795 death measured so far
# lands inside the first 750s of the step.
watchdog='' ci_watchdog_pid='' ci_watchdog_exe='' ci_signal='' ci_rc=0
# Held here, never exported: a token every test child inherits is how four forged
# statuses once reached a commit under test.
ci_watchdog_token=${WATCHDOG_TOKEN:-}
unset WATCHDOG_TOKEN
ci_start_native_watchdog "$PWD/$progress" && watchdog=$ci_watchdog_pid
test -n "$watchdog" || echo "no off-box watchdog: no usable PowerShell"
# Started after it, so the kill path can silence and outlive the reporter.
ci_suite_heartbeat 960 360 "$progress" "$stuck" $$ &
ci_suite_heartbeat 960 360 "$progress" 900 $$ &
heartbeat=$!
ci_install_traps
trap 'kill "$heartbeat" 2>/dev/null || true' EXIT
pass=0 fail=0 skip=0 failed="" skipped="" deadline=0
# label:pattern, globbed rather than enumerated so a new NNN_engine-*.test or

View File

@@ -1,243 +0,0 @@
# Off-box telemetry for the Windows suite step (#795). It spawns nothing and
# kills nothing, so nothing a wedge takes away can disarm it, and it reports as a
# commit status, the only channel that outlives a dead runner.
param(
[string]$ProgressLog = '',
[int]$IntervalSeconds = 30,
[int]$PollSeconds = 5,
# Cannot outlive the step, whatever the caller forgets to kill.
[int]$MaxSeconds = 2700,
# One-shot mode: post this state and exit, so the driver reports its own end.
[string]$Post = '',
[string]$Message = '',
# Seam for the suite's own test, which points it at a sink it can count.
[string]$ApiBase = 'https://api.github.com',
[switch]$NoPost,
[switch]$SelfTest
)
$ErrorActionPreference = 'Stop'
# Windows PowerShell renders a progress bar per web call otherwise.
$ProgressPreference = 'SilentlyContinue'
# --- decisions, kept pure so -SelfTest can drive them by return value ---------
function Get-WatchdogAction {
param([int]$Now, [int]$Posted, [int]$Interval)
if (($Now - $Posted) -ge $Interval) { return 'post' }
return 'wait'
}
# Posts to skip after a rejected one: a fork PR's token is read-only for the run.
function Get-NextBackoff {
param([int]$Current)
if ($Current -lt 1) { return 1 }
return [Math]::Min($Current * 2, 32)
}
# GitHub truncates a description at 140 chars, so the counters take the cut, not
# the fields a wedge is read for. $Static below zero is unknown.
function Format-WatchdogStatus {
param([int]$Elapsed, [int]$Static, [string]$InFlight, [string]$Counters)
$q = '?'
if ($Static -ge 0) { $q = [string]$Static }
$t = ($InFlight -replace '\s+', ' ').Trim()
if ($t.Length -gt 46) { $t = $t.Substring(0, 46) }
$s = 't={0}s q={1}s {2} | {3}' -f $Elapsed, $q, $t, $Counters
if ($s.Length -gt 140) { $s = $s.Substring(0, 140) }
return $s
}
# --- probes ------------------------------------------------------------------
# One try/catch per counter: a probe that fails costs its own field, not the loop.
function Get-WatchdogCounters {
$f = New-Object System.Collections.ArrayList
try {
$os = Get-CimInstance -ClassName Win32_OperatingSystem -OperationTimeoutSec 5
[void]$f.Add('m={0}' -f [int]($os.FreePhysicalMemory / 1KB))
[void]$f.Add('v={0}' -f [int]($os.FreeVirtualMemory / 1KB))
} catch { [void]$f.Add('m=? v=?') }
try {
$ps = @(Get-Process)
[void]$f.Add('p={0}' -f $ps.Count)
[void]$f.Add('h={0}' -f (($ps | Measure-Object -Property Handles -Sum).Sum))
} catch { [void]$f.Add('p=? h=?') }
try {
$drive = New-Object System.IO.DriveInfo($env:SystemDrive + '\')
[void]$f.Add('d={0}' -f [int]($drive.AvailableFreeSpace / 1MB))
} catch { [void]$f.Add('d=?') }
return ($f -join ' ')
}
# Ok separates "nothing moved" from "could not read it", which would otherwise report
# a wedge for an unreadable file. Share flags: the driver appends as we read.
function Get-ProgressTail {
param([string]$Path)
$r = @{ Ok = $false; Signature = ''; Line = '' }
if (-not $Path) { return $r }
try {
$share = [System.IO.FileShare]::ReadWrite -bor [System.IO.FileShare]::Delete
$fs = New-Object System.IO.FileStream($Path, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, $share)
try {
$sr = New-Object System.IO.StreamReader($fs)
$text = $sr.ReadToEnd()
} finally { $fs.Dispose() }
$lines = @($text -split "`r?`n" | Where-Object { $_ -ne '' })
if ($lines.Count -gt 0) { $r.Line = $lines[-1] }
$r.Signature = '{0}|{1}' -f $text.Length, $r.Line
$r.Ok = $true
} catch { }
return $r
}
function Write-WatchdogLog {
param([string]$Message)
# A full disk is a state the sick runner reaches, and a log line lost to it
# must not take the loop reporting off-box with it.
try { Write-Host ('[watchdog {0:HH:mm:ss}] {1}' -f (Get-Date), $Message) } catch { }
}
# --- reporting ---------------------------------------------------------------
$script:Repo = $env:WATCHDOG_REPO
$script:Sha = $env:WATCHDOG_SHA
$script:Token = $env:WATCHDOG_TOKEN
$script:TargetUrl = $env:WATCHDOG_URL
$script:Context = $env:WATCHDOG_CONTEXT
if (-not $script:Context) { $script:Context = 'windows-suite-watchdog' }
# -NoPost is the test's hard stop: no request, whatever the environment holds.
function Send-WatchdogStatus {
param([string]$State, [string]$Description)
if ($NoPost -or $SelfTest) { return $false }
if (-not $script:Token -or -not $script:Repo -or -not $script:Sha) { return $false }
$body = @{ state = $State; context = $script:Context; description = $Description }
if ($script:TargetUrl) { $body['target_url'] = $script:TargetUrl }
try {
Invoke-RestMethod -Method Post -TimeoutSec 20 `
-Uri ('{0}/repos/{1}/statuses/{2}' -f $ApiBase.TrimEnd('/'), $script:Repo, $script:Sha) `
-UserAgent 'httrack-windows-suite-watchdog' `
-Headers @{
Authorization = ('Bearer {0}' -f $script:Token)
Accept = 'application/vnd.github+json'
} -ContentType 'application/json' -Body ($body | ConvertTo-Json -Compress) | Out-Null
return $true
} catch {
Write-WatchdogLog ('status post failed: {0}' -f $_.Exception.Message)
return $false
}
}
# --- self-test ----------------------------------------------------------------
function Invoke-WatchdogSelfTest {
$bad = New-Object System.Collections.ArrayList
function Assert-That($cond, $what) { if (-not $cond) { [void]$bad.Add($what) } }
Assert-That ((Get-WatchdogAction 30 0 30) -eq 'post') 'a post due exactly on the interval was skipped'
Assert-That ((Get-WatchdogAction 29 0 30) -eq 'wait') 'posted ahead of the interval'
Assert-That ((Get-WatchdogAction 5000 4990 30) -eq 'wait') 'posted off cadence'
Assert-That ((Get-NextBackoff 0) -eq 1) 'the first rejection does not back off'
Assert-That ((Get-NextBackoff 1) -eq 2) 'the backoff does not grow'
Assert-That ((Get-NextBackoff 32) -eq 32) 'the backoff is not capped'
$long = '43_local-update-truncate-with-a-very-long-name-indeed.test'
$line = Format-WatchdogStatus 812 41 $long 'm=9012 v=14022 p=118 h=41230 d=13210'
Assert-That ($line.Length -le 140) ('status description is {0} characters' -f $line.Length)
Assert-That ($line -like 't=812s q=41s 43_local-update-truncate*') ('status leads with the wrong fields: {0}' -f $line)
Assert-That ($line -like '*d=13210') 'the counters did not survive a long test name'
# -match, not -like: '?' is a wildcard there, so q=0s would satisfy it too.
Assert-That ((Format-WatchdogStatus 8 -1 'x' 'y') -match '^t=8s q=\?s x \| y$') 'an unknown staticness reads as a number'
$clip = Format-WatchdogStatus 1 2 ('x' * 80) 'c'
Assert-That ($clip -match '^t=1s q=2s x{46} \| c$') ('the in-flight name was not clipped to 46: {0}' -f $clip)
$wide = Format-WatchdogStatus 1 2 ('x' * 300) ('y' * 300)
Assert-That ($wide.Length -le 140) ('an oversized status was not clipped: {0}' -f $wide.Length)
$gone = Get-ProgressTail -Path ('no-such-progress-log-{0}.tmp' -f $PID)
Assert-That (-not $gone.Ok) 'an unreadable log reads as one that was read'
$f = New-Object System.IO.FileInfo([System.IO.Path]::GetTempFileName())
try {
[System.IO.File]::WriteAllText($f.FullName, "first`nRUN 42_probe.test at 7s`n")
$tail = Get-ProgressTail -Path $f.FullName
Assert-That ($tail.Ok) 'a readable log reads as unreadable'
Assert-That ($tail.Line -eq 'RUN 42_probe.test at 7s') ('the tail is not the last line: {0}' -f $tail.Line)
} finally { [System.IO.File]::Delete($f.FullName) }
Assert-That (-not (Send-WatchdogStatus 'success' 'self-test')) 'the self-test can reach the API'
if ($bad.Count -gt 0) {
foreach ($b in $bad) { Write-Host ('self-test FAIL: {0}' -f $b) }
exit 1
}
Write-Host 'watchdog self-test OK'
exit 0
}
if ($SelfTest) { Invoke-WatchdogSelfTest }
# --- one-shot: the driver's own verdict ---------------------------------------
# Windows PowerShell still defaults below TLS 1.2, which api.github.com refuses.
try { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 } catch { }
if ($Post) {
# One transient 5xx here is the difference between a resolved status and a
# finished suite left reading `pending`.
for ($try = 1; $try -le 3; $try++) {
if (Send-WatchdogStatus $Post $Message) { break }
if ($NoPost -or -not $script:Token) { break }
Start-Sleep -Seconds (2 * $try)
}
Write-WatchdogLog ('final status {0}: {1}' -f $Post, $Message)
exit 0
}
# --- main loop ----------------------------------------------------------------
$sw = [System.Diagnostics.Stopwatch]::StartNew()
$lastSig = ''
$movedAt = 0
# Negative, so the first tick posts: an early status is itself a datum.
$postedAt = -$IntervalSeconds
$backoff = 0
$skip = 0
# Guarded like the rest; the launcher waits for this exact line.
try { Write-Host 'watchdog ready' } catch { }
Write-WatchdogLog ('watching {0} every {1}s' -f $ProgressLog, $IntervalSeconds)
while ($sw.Elapsed.TotalSeconds -lt $MaxSeconds) {
# Measured, never accumulated: starvation is what makes a sleep overshoot.
$now = [int]$sw.Elapsed.TotalSeconds
try {
$tail = Get-ProgressTail -Path $ProgressLog
if ($tail.Ok -and $tail.Signature -ne $lastSig) {
$lastSig = $tail.Signature
$movedAt = $now
}
if ((Get-WatchdogAction $now $postedAt $IntervalSeconds) -eq 'post') {
$postedAt = $now
$static = -1
if ($tail.Ok) { $static = $now - $movedAt }
$desc = Format-WatchdogStatus $now $static $tail.Line (Get-WatchdogCounters)
# Logged whatever the backoff decides: it throttles the API, not the
# artifact, which is all a run whose token cannot post will leave.
Write-WatchdogLog $desc
if ($skip -gt 0) {
$skip--
} elseif (Send-WatchdogStatus 'pending' $desc) {
$backoff = 0
} else {
$backoff = Get-NextBackoff $backoff
$skip = $backoff
}
}
} catch {
Write-WatchdogLog ('tick failed: {0}' -f $_.Exception.Message)
}
Start-Sleep -Seconds $PollSeconds
}
Write-WatchdogLog ('stopping after {0}s' -f [int]$sw.Elapsed.TotalSeconds)

View File

@@ -10,6 +10,8 @@ directory). Each line is "<path> <mode>...", path "*" matching everything:
empty open the data connection and send nothing
norest answer REST with 500, so the client re-fetches from scratch
nomdtm answer MDTM with 500, like a server predating RFC 3659
--require-pass answers USER with 331, the only way the client's PASS is sent.
"""
import argparse
@@ -25,12 +27,13 @@ def reply(conn, text):
class Session(threading.Thread):
def __init__(self, conn, root, mode_file, log):
def __init__(self, conn, root, mode_file, log, require_pass=False):
threading.Thread.__init__(self, daemon=True)
self.conn = conn
self.root = root
self.mode_file = mode_file
self.log = log
self.require_pass = require_pass
self.pasv = None
self.rest = 0
self.path = "/" # named by SIZE/RETR; REST carries no path
@@ -132,7 +135,9 @@ class Session(threading.Thread):
def dispatch(self, verb, arg):
conn = self.conn
if verb in ("USER", "PASS", "TYPE", "NOOP"):
if verb == "USER" and self.require_pass:
reply(conn, "331 password required") # the client sends PASS only on a 3xx
elif verb in ("USER", "PASS", "TYPE", "NOOP"):
reply(conn, "200 ok")
elif verb == "SYST":
reply(conn, "215 UNIX Type: L8")
@@ -209,6 +214,7 @@ def main():
ap.add_argument("--root", required=True)
ap.add_argument("--mode-file")
ap.add_argument("--log")
ap.add_argument("--require-pass", action="store_true")
args = ap.parse_args()
logfp = open(args.log, "a", encoding="utf-8") if args.log else None
@@ -231,7 +237,7 @@ def main():
root = os.path.abspath(args.root)
while True:
conn, _ = srv.accept()
Session(conn, root, args.mode_file, log).start()
Session(conn, root, args.mode_file, log, args.require_pass).start()
if __name__ == "__main__":

View File

@@ -72,13 +72,13 @@ is_windows() {
}
# On Windows MSYS can't signal a native python.exe, so kill_tree ends the whole
# tree (a bare kill -9 leaves children). "|| true" throughout: callers run under
# set -e and the reap makes wait return 143.
# tree (a bare kill -9 leaves children). Bounded because this runs from an EXIT
# trap, where a survivor would turn a passing test into a harness timeout.
stop_server() {
test -n "${1:-}" || return 0
kill "$1" 2>/dev/null || true
if is_windows; then kill_tree "$1"; fi
wait "$1" 2>/dev/null || true
reap_bounded "$1" || true
return 0
}

View File

@@ -260,6 +260,6 @@ TESTS += 221_local-ftp-ctrlchars.test
TESTS += 218_crash-nopie-frames.test
TESTS += 222_pkgconfig-consumer.test
TESTS += 228_icon-small-flat.test
TESTS += 226_watchdog-native.test
TESTS += 224_engine-ftp-cmdlen.test
TESTS += 230_local-ftp-userpass.test
TESTS += 225_install-manifest.test