mirror of
https://github.com/xroche/httrack.git
synced 2026-08-06 15:56:26 +03:00
Compare commits
4 Commits
fix-795-ru
...
issue-1032
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
6547eaca7e | ||
|
|
fccc3e246e | ||
|
|
d6f4d71a38 | ||
|
|
186476cab9 |
14
.github/workflows/windows-build.yml
vendored
14
.github/workflows/windows-build.yml
vendored
@@ -14,9 +14,6 @@ on:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
# The suite watchdog reports as a commit status, the only channel that
|
||||
# outlives the runner it is reporting on (#795).
|
||||
statuses: write
|
||||
|
||||
# Cancel superseded runs on the same branch or PR.
|
||||
concurrency:
|
||||
@@ -40,9 +37,6 @@ jobs:
|
||||
- uses: actions/checkout@v7
|
||||
with:
|
||||
submodules: recursive # coucal lives in src/coucal
|
||||
# Or the token, which can now post statuses, sits in .git/config for
|
||||
# the whole job, where every test the suite runs can read it.
|
||||
persist-credentials: false
|
||||
|
||||
# Located through vswhere rather than microsoft/setup-msbuild: the repo
|
||||
# only allows GitHub-owned actions.
|
||||
@@ -177,14 +171,6 @@ jobs:
|
||||
shell: bash
|
||||
working-directory: tests
|
||||
timeout-minutes: 45
|
||||
env:
|
||||
# Through the environment, never argv, which the process list exposes.
|
||||
WATCHDOG_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
WATCHDOG_REPO: ${{ github.repository }}
|
||||
# A pull_request's merge commit, so these stay out of the PR's checks UI.
|
||||
WATCHDOG_SHA: ${{ github.sha }}
|
||||
WATCHDOG_CONTEXT: windows-suite (${{ matrix.platform }}, ${{ matrix.configuration }})
|
||||
WATCHDOG_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
|
||||
run: |
|
||||
bash ./ci-windows-suite.sh \
|
||||
"$(cygpath -u "$GITHUB_WORKSPACE")/src/${{ matrix.platform }}/${{ matrix.configuration }}"
|
||||
|
||||
49
src/htsftp.c
49
src/htsftp.c
@@ -122,31 +122,30 @@ void launch_ftp(FTPDownloadStruct * params) {
|
||||
return 0; \
|
||||
}
|
||||
|
||||
/* Bounded split of a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
|
||||
void ftp_split_userpass(const char *src, const char *end, char *user,
|
||||
size_t user_size, char *pass, size_t pass_size) {
|
||||
size_t n = 0;
|
||||
/* Split a hostile-URL "user[:pass]@" prefix (see htsftp.h). */
|
||||
hts_boolean ftp_split_userpass(const char *src, const char *end, char *user,
|
||||
size_t user_size, char *pass, size_t pass_size) {
|
||||
size_t len = 0, user_len, pass_len;
|
||||
const char *colon;
|
||||
|
||||
assertf(user_size > 0 && pass_size > 0); /* the size-1 math underflows on 0 */
|
||||
assertf(end > src); /* end is one past the '@' */
|
||||
|
||||
while (src[n] != '\0' && src[n] != ':') {
|
||||
if (n < user_size - 1)
|
||||
user[n] = src[n];
|
||||
n++;
|
||||
}
|
||||
user[n < user_size ? n : user_size - 1] = '\0';
|
||||
pass[0] = '\0';
|
||||
if (src[n] == ':') { // password follows the colon
|
||||
const size_t base = n + 1;
|
||||
size_t k = 0;
|
||||
|
||||
while (&src[base + k + 1] < end && src[base + k] != '\0') {
|
||||
if (k < pass_size - 1)
|
||||
pass[k] = src[base + k];
|
||||
k++;
|
||||
}
|
||||
pass[k < pass_size ? k : pass_size - 1] = '\0';
|
||||
}
|
||||
user[0] = pass[0] = '\0'; // fail safe for a caller that ignores the result
|
||||
while (len < (size_t) (end - src) - 1 && src[len] != '\0')
|
||||
len++;
|
||||
colon = memchr(src, ':', len);
|
||||
user_len = colon != NULL ? (size_t) (colon - src) : len;
|
||||
pass_len = colon != NULL ? len - user_len - 1 : 0;
|
||||
/* a clipped name is another account's, so refuse rather than log in as it */
|
||||
if (user_len >= user_size || pass_len >= pass_size)
|
||||
return HTS_FALSE;
|
||||
memcpy(user, src, user_len);
|
||||
user[user_len] = '\0';
|
||||
if (pass_len != 0)
|
||||
memcpy(pass, colon + 1, pass_len);
|
||||
pass[pass_len] = '\0';
|
||||
return HTS_TRUE;
|
||||
}
|
||||
|
||||
/* Build "<verb> <path>" (see htsftp.h). */
|
||||
@@ -258,7 +257,11 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
|
||||
while(*real_adr == '/')
|
||||
real_adr++; // sauter /
|
||||
if ((adr = jump_identification(real_adr)) != real_adr) { // user
|
||||
ftp_split_userpass(real_adr, adr, user, sizeof(user), pass, sizeof(pass));
|
||||
if (!ftp_split_userpass_buf(real_adr, adr, user, pass)) {
|
||||
strcpybuff(back->r.msg, "FTP user name or password too long");
|
||||
back->r.statuscode = STATUSCODE_INVALID;
|
||||
_HALT_FTP return 0;
|
||||
}
|
||||
}
|
||||
// Calculer RETR <nom>
|
||||
{
|
||||
|
||||
14
src/htsftp.h
14
src/htsftp.h
@@ -74,10 +74,18 @@ int run_launch_ftp(FTPDownloadStruct * params);
|
||||
int send_line(T_SOC soc, const char *data);
|
||||
int get_ftp_line(T_SOC soc, char *line, size_t line_size, int timeout);
|
||||
/* Split a "user[:pass]@" prefix (end = jump_identification result) into
|
||||
bounded, NUL-terminated user/pass buffers, truncating to fit.
|
||||
NUL-terminated user/pass buffers. Returns HTS_FALSE and empties both when a
|
||||
field does not fit, as a clipped one would name another account.
|
||||
Both sizes must be nonzero. */
|
||||
void ftp_split_userpass(const char *src, const char *end, char *user,
|
||||
size_t user_size, char *pass, size_t pass_size);
|
||||
hts_boolean ftp_split_userpass(const char *src, const char *end, char *user,
|
||||
size_t user_size, char *pass, size_t pass_size);
|
||||
/* ftp_split_userpass() into the caller's fixed buffers; a buffer too wide for
|
||||
its "USER <user>" line would be clipped again when the command is built. */
|
||||
#define ftp_split_userpass_buf(src, end, user, pass) \
|
||||
(HTS_COMPILE_ASSERT(sizeof(user) + sizeof("USER ") - 1 <= FTP_LINE_SIZE && \
|
||||
sizeof(pass) + sizeof("PASS ") - 1 <= FTP_LINE_SIZE), \
|
||||
ftp_split_userpass((src), (end), (user), sizeof(user), (pass), \
|
||||
sizeof(pass)))
|
||||
/* Build "<verb> <path>" into line[line_size]. The path is quoted whenever a
|
||||
bare one would give the server a second token; it must already have been
|
||||
screened for control bytes. Returns HTS_FALSE and empties line when the
|
||||
|
||||
@@ -4501,40 +4501,79 @@ static int st_ftpline(httrackp *opt, int argc, char **argv) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* ftp_split_userpass: well-formed split, plus a hostile over-long userinfo
|
||||
that pre-fix overran user[256]/pass[256]. */
|
||||
/* ftp_split_userpass: the split itself, and userinfo refused rather than
|
||||
clipped into another account's name (#1032). */
|
||||
static int st_ftpuser(httrackp *opt, int argc, char **argv) {
|
||||
char user[256], pass[256];
|
||||
char in[1200];
|
||||
static const size_t caps[] = {16, 256}; /* asymmetric: a shared bound shows */
|
||||
char ubuf[256 + 32], pbuf[sizeof(ubuf)], poison[sizeof(ubuf)];
|
||||
char in[2 * 256 + 8];
|
||||
size_t c, over;
|
||||
|
||||
(void) opt;
|
||||
(void) argc;
|
||||
(void) argv;
|
||||
memset(poison, '#', sizeof(poison));
|
||||
{
|
||||
const char ok[] = "bob:secret@host/f"; // '@' at index 10
|
||||
|
||||
ftp_split_userpass(ok, ok + 11, user, sizeof(user), pass, sizeof(pass));
|
||||
assertf(strcmp(user, "bob") == 0);
|
||||
assertf(strcmp(pass, "secret") == 0);
|
||||
assertf(ftp_split_userpass(ok, ok + 11, ubuf, sizeof(ubuf), pbuf,
|
||||
sizeof(pbuf)) == HTS_TRUE);
|
||||
assertf(strcmp(ubuf, "bob") == 0);
|
||||
assertf(strcmp(pbuf, "secret") == 0);
|
||||
}
|
||||
memset(in, 'u', 400);
|
||||
in[400] = ':';
|
||||
memset(in + 401, 'p', 400);
|
||||
in[801] = '@';
|
||||
in[802] = '\0';
|
||||
ftp_split_userpass(in, in + 802, user, sizeof(user), pass, sizeof(pass));
|
||||
assertf(strlen(user) == sizeof(user) - 1);
|
||||
assertf(strlen(pass) == sizeof(pass) - 1);
|
||||
{
|
||||
/* tight sizes + guard byte catch an off-by-one the 256 case can't */
|
||||
char ubuf[16], pbuf[16];
|
||||
const char ok[] = "bob@host/f"; // no password: the '@' still ends the user
|
||||
|
||||
memset(ubuf, 'Z', sizeof(ubuf));
|
||||
memset(pbuf, 'Z', sizeof(pbuf));
|
||||
ftp_split_userpass(in, in + 802, ubuf, 8, pbuf, 8);
|
||||
assertf(strcmp(ubuf, "uuuuuuu") == 0);
|
||||
assertf(strcmp(pbuf, "ppppppp") == 0);
|
||||
assertf(ubuf[8] == 'Z' && pbuf[8] == 'Z');
|
||||
assertf(ftp_split_userpass(ok, ok + 4, ubuf, sizeof(ubuf), pbuf,
|
||||
sizeof(pbuf)) == HTS_TRUE);
|
||||
assertf(strcmp(ubuf, "bob") == 0);
|
||||
assertf(pbuf[0] == '\0');
|
||||
}
|
||||
{
|
||||
const char ok[] = "u@relay:pw@gw/f"; // only the last '@' ends the userinfo
|
||||
|
||||
assertf(ftp_split_userpass(ok, ok + 11, ubuf, sizeof(ubuf), pbuf,
|
||||
sizeof(pbuf)) == HTS_TRUE);
|
||||
assertf(strcmp(ubuf, "u@relay") == 0);
|
||||
assertf(strcmp(pbuf, "pw") == 0);
|
||||
}
|
||||
for (c = 0; c < sizeof(caps) / sizeof(caps[0]); c++) {
|
||||
const size_t ucap = caps[c], pcap = caps[1 - c];
|
||||
|
||||
/* overshoot the user, the pass, then a bare name bounded only by '@' */
|
||||
for (over = 0; over <= 2; over++) {
|
||||
const size_t cap = over == 1 ? pcap : ucap;
|
||||
size_t len;
|
||||
|
||||
for (len = cap - 2; len <= cap + 1; len++) {
|
||||
const size_t user_len = over == 1 ? 1 : len;
|
||||
const size_t pass_len = over == 0 ? 1 : (over == 1 ? len : 0);
|
||||
const size_t total = user_len + pass_len + (over == 2 ? 1 : 2);
|
||||
const hts_boolean fits = len < cap ? HTS_TRUE : HTS_FALSE;
|
||||
|
||||
memset(in, 'u', user_len);
|
||||
if (over != 2) {
|
||||
in[user_len] = ':';
|
||||
memset(in + user_len + 1, 'p', pass_len);
|
||||
}
|
||||
in[total - 1] = '@';
|
||||
in[total] = '\0';
|
||||
memcpy(ubuf, poison, sizeof(ubuf)); /* a zero canary would hide a NUL */
|
||||
memcpy(pbuf, poison, sizeof(pbuf));
|
||||
assertf(ftp_split_userpass(in, in + total, ubuf, ucap, pbuf, pcap) ==
|
||||
fits);
|
||||
if (fits) {
|
||||
assertf(strlen(ubuf) == user_len && strlen(pbuf) == pass_len);
|
||||
assertf(ubuf[user_len - 1] == 'u');
|
||||
assertf(pass_len == 0 || pbuf[pass_len - 1] == 'p');
|
||||
} else {
|
||||
assertf(ubuf[0] == '\0' && pbuf[0] == '\0'); /* fail safe */
|
||||
}
|
||||
/* the whole tail: one canary byte misses a write just past it */
|
||||
assertf(memcmp(ubuf + ucap, poison, sizeof(ubuf) - ucap) == 0);
|
||||
assertf(memcmp(pbuf + pcap, poison, sizeof(pbuf) - pcap) == 0);
|
||||
}
|
||||
}
|
||||
}
|
||||
printf("ftp-userpass self-test OK\n");
|
||||
return 0;
|
||||
|
||||
@@ -1,8 +1,43 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# Over-long FTP userinfo must fail the link, never log in under a clipped name
|
||||
# (#1032).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# ftp_split_userpass bounds an over-long user:pass@ from a hostile ftp:// URL.
|
||||
out=$(httrack -O /dev/null -#test=ftp-userpass run)
|
||||
grep -q "ftp-userpass self-test OK" <<<"$out"
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftpuser.XXXXXX") || exit 1
|
||||
trap 'set +e; rm -rf "${tmpdir}"' EXIT
|
||||
trap 'exit 1' HUP INT QUIT TERM
|
||||
|
||||
out=$(httrack -O "${tmpdir}/st" "-#test=ftp-userpass" run 2>&1) ||
|
||||
fail "self-test exited non-zero: ${out}"
|
||||
grep -q "ftp-userpass self-test OK" <<<"${out}" || fail "unexpected output: ${out}"
|
||||
|
||||
# The reachable half: a URL carries the userinfo straight into user[256]/pass[256].
|
||||
rep() { awk -v n="$1" 'BEGIN { while (i++ < n) printf "a" }'; }
|
||||
|
||||
probe() {
|
||||
rm -rf "${tmpdir}/mir"
|
||||
httrack "ftp://$1@127.0.0.1:1/f.txt" -O "${tmpdir}/mir" -q >/dev/null 2>&1 ||
|
||||
fail "an FTP URL with ${#1}-byte userinfo crashed the engine"
|
||||
cat "${tmpdir}/mir/hts-log.txt"
|
||||
}
|
||||
|
||||
# The bare name has no ':' to bound it, only the '@', and is the worse branch.
|
||||
for u in "$(rep 256):pw" "u:$(rep 256)" "$(rep 256)"; do
|
||||
log=$(probe "${u}")
|
||||
grep -q "FTP user name or password too long" <<<"${log}" ||
|
||||
fail "over-long userinfo was not refused: ${log}"
|
||||
done
|
||||
|
||||
# 255 still fits, and a plain login is the outsider a widened gate would refuse.
|
||||
for u in "$(rep 255):pw" "u:$(rep 255)" "$(rep 255)" "bob:secret"; do
|
||||
log=$(probe "${u}")
|
||||
grep -q "Unable to connect to the server" <<<"${log}" ||
|
||||
fail "userinfo that fits did not reach the connect: ${log}"
|
||||
done
|
||||
|
||||
@@ -138,9 +138,7 @@ test "$killed" -le 10080 || fail "killed at $killed, want 9960 within a tick"
|
||||
|
||||
# taskkill is a grandchild of its own target, so a leaves-first /T would reap the
|
||||
# watchdog before the root (#953): kill_tree here never returns, and the stubs
|
||||
# record the order, which the code under test cannot write to. A taskkill runs no
|
||||
# trap, so this path posts the verdict itself, after silencing the reporter that
|
||||
# would otherwise overwrite it.
|
||||
# record the order, which the code under test cannot write to.
|
||||
rec="$tmp/killed"
|
||||
: >"$rec"
|
||||
vnow=9000
|
||||
@@ -148,8 +146,6 @@ ticks=0
|
||||
printf 'RUN 96_wedged.test at 0s\n' >"$progress"
|
||||
rc=0
|
||||
(
|
||||
watchdog=777
|
||||
ci_post_final_status() { echo "POST $1" >>"$rec"; }
|
||||
kill_pid() { echo "DIRECT $1" >>"$rec"; }
|
||||
kill_tree() {
|
||||
echo "TREE $1" >>"$rec"
|
||||
@@ -158,10 +154,10 @@ rc=0
|
||||
ci_suite_heartbeat 960 360 "$progress" 900 4242 >"$tmp/hedge" 2>&1
|
||||
) || rc=$?
|
||||
test "$rc" -eq 9 || fail "the tree kill never fired: watchdog returned $rc"
|
||||
want='DIRECT 777
|
||||
POST 1
|
||||
DIRECT 4242
|
||||
TREE 4242'
|
||||
test "$(cat "$rec")" = "$want" || fail "the kill path did: $(tr '\n' '/' <"$rec")"
|
||||
test "$(sed -n 1p "$rec")" = "DIRECT 4242" ||
|
||||
fail "the target was not signalled directly ahead of the tree walk: $(tr '\n' '/' <"$rec")"
|
||||
test "$(sed -n 2p "$rec")" = "TREE 4242" ||
|
||||
fail "the tree was not killed after the direct signal: $(tr '\n' '/' <"$rec")"
|
||||
test "$(sed -n '$=' "$rec")" -eq 2 || fail "extra kills: $(tr '\n' '/' <"$rec")"
|
||||
|
||||
echo "heartbeat OK"
|
||||
|
||||
@@ -39,8 +39,7 @@ grep -q usage <<<"$out" || fail "argless run said: $out"
|
||||
# sibling's.
|
||||
run=$tmp/run
|
||||
mkdir -p "$run"
|
||||
cp "$driver" "$testdir/testlib.sh" "$testdir/test-timeout.sh" \
|
||||
"$testdir/ci-windows-watchdog.ps1" "$run/"
|
||||
cp "$driver" "$testdir/testlib.sh" "$testdir/test-timeout.sh" "$run/"
|
||||
chmod u+w "$run"/*.sh # distcheck's srcdir is read-only, and cp carries that over
|
||||
cat >>"$run/testlib.sh" <<'EOF'
|
||||
reap_leftover_processes() { return 0; }
|
||||
@@ -52,23 +51,6 @@ printf '#!/bin/sh\nexit 0\n' >"$bin/httrack"
|
||||
# shellcheck disable=SC2016 # $2 is the stub's own argument
|
||||
printf '#!/bin/sh\necho "$2"\n' >"$bin/cygpath"
|
||||
chmod +x "$bin/httrack" "$bin/cygpath"
|
||||
# The driver puts this bindir on PATH, so a stub here is the interpreter it finds
|
||||
# where there is no real one; a real PowerShell drives the real script instead.
|
||||
wdargv=$tmp/wdargv
|
||||
: >"$wdargv"
|
||||
if ! command -v pwsh >/dev/null 2>&1 && ! command -v powershell.exe >/dev/null 2>&1; then
|
||||
cat >"$bin/pwsh" <<EOF
|
||||
#!/bin/sh
|
||||
{ echo "TOKEN=\${WATCHDOG_TOKEN:-}"; printf '%s\n' "\$@"; } >>"$wdargv"
|
||||
case " \$* " in
|
||||
*" -Post failure "*) echo "final status failure"; exit 0 ;;
|
||||
*" -Post "*) echo "final status other"; exit 0 ;;
|
||||
esac
|
||||
echo "watchdog ready"
|
||||
exec sleep 60
|
||||
EOF
|
||||
chmod +x "$bin/pwsh"
|
||||
fi
|
||||
test -x "$bin/httrack" || {
|
||||
echo "SKIP: ${TMPDIR:-/tmp} is noexec, the stub bindir cannot be run"
|
||||
exit 77
|
||||
@@ -76,21 +58,16 @@ test -x "$bin/httrack" || {
|
||||
|
||||
# One per glob the driver enumerates: an empty category is counted as a failing
|
||||
# test named after the unexpanded pattern, which would drown the accounting.
|
||||
for t in 00_runnable 13_zlib-pass 14_local-pass 15_watchdog-pass \
|
||||
for t in 00_runnable 10_engine-pass 13_zlib-pass 14_local-pass 15_watchdog-pass \
|
||||
16_crawl_proxy_https 17_crawl-log-salvage; do
|
||||
printf '#!/bin/sh\nexit 0\n' >"$run/$t.test"
|
||||
done
|
||||
# One test reports back what it inherited: the token the step is handed can post
|
||||
# commit statuses, and a forged one has already reached a commit under test.
|
||||
# shellcheck disable=SC2016 # the stub expands it, not this shell
|
||||
printf '#!/bin/sh\necho "TOKEN=${WATCHDOG_TOKEN:-}" >%s\nexit 0\n' "$tmp/childtoken" \
|
||||
>"$run/10_engine-pass.test"
|
||||
printf '#!/bin/sh\nexit 77\n' >"$run/11_engine-skip.test"
|
||||
printf '#!/bin/sh\nexit 3\n' >"$run/12_engine-fail.test"
|
||||
|
||||
cd "$run"
|
||||
rc=0
|
||||
out=$(RUNNER_TEMP="$tmp" GITHUB_STEP_SUMMARY="$tmp/summary" WATCHDOG_TOKEN=s3cr3t \
|
||||
out=$(RUNNER_TEMP="$tmp" GITHUB_STEP_SUMMARY="$tmp/summary" \
|
||||
bash ./ci-windows-suite.sh "$bin" 2>&1) || rc=$?
|
||||
# The skip set and the pass floor are pinned to the real suite, so a stub run
|
||||
# ends on the floor; what is under test is the tally that reaches it.
|
||||
@@ -100,18 +77,6 @@ grep -q '^ran=9 pass=7 fail=1 skip=1$' <<<"$out" ||
|
||||
grep -q '::error::only 7 tests passed (1 skipped)' <<<"$out" ||
|
||||
fail "the pass floor did not report the count"
|
||||
grep -q 'FAIL 12_engine-fail.test (exit 3)' <<<"$out" || fail "the failing test was not named"
|
||||
|
||||
# The reporter as the real driver launches it, rather than through a test
|
||||
# sourcing the helper: launched, ended, and holding a token nothing else sees.
|
||||
grep -q '^watchdog ready$' watchdog.log || fail "no watchdog was launched: $(cat watchdog.log)"
|
||||
grep -q 'final status failure' watchdog.log ||
|
||||
fail "the driver's own verdict was never posted: $(cat watchdog.log)"
|
||||
test -r "$tmp/childtoken" || fail "the test that reads its environment did not run"
|
||||
grep -qx 'TOKEN=' "$tmp/childtoken" ||
|
||||
fail "a test inherited the status token: $(cat "$tmp/childtoken")"
|
||||
if test -s "$wdargv"; then
|
||||
grep -qx 'TOKEN=s3cr3t' "$wdargv" || fail "the watchdog was launched without its token"
|
||||
fi
|
||||
# Positive control for the gate below: with every category matched it stays silent.
|
||||
grep -q 'matched no tests' <<<"$out" && fail "a full suite reported an empty category"
|
||||
|
||||
|
||||
@@ -1,528 +0,0 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# The off-box suite watchdog (#795): its pure decisions through -SelfTest, the
|
||||
# process APIs it must never name, and the driver wiring no .ps1 can check for
|
||||
# itself. It only reports, so there is no kill path here to exercise.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# First, before anything here can start an interpreter: a token reaching one
|
||||
# posts forged statuses onto the commit under test.
|
||||
unset WATCHDOG_TOKEN WATCHDOG_REPO WATCHDOG_SHA WATCHDOG_URL
|
||||
export WATCHDOG_CONTEXT='226_watchdog-native (never posted)'
|
||||
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "$testdir/testlib.sh"
|
||||
|
||||
top=${abs_top_srcdir:-$(cd "$testdir/.." && pwd)}
|
||||
wdscript="$testdir/ci-windows-watchdog.ps1"
|
||||
driver="$testdir/ci-windows-suite.sh"
|
||||
workflow="$top/.github/workflows/windows-build.yml"
|
||||
|
||||
tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_wdnat.XXXXXX")
|
||||
trap 'set +e; stop_server "${sinkpid:-}"; rm -rf "$tmp"' EXIT
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
test -r "$wdscript" || fail "no $wdscript"
|
||||
test -z "${WATCHDOG_TOKEN:-}${WATCHDOG_SHA:-}" || fail "a credential survived the scrub"
|
||||
|
||||
# The prose in the script names the very calls the audits forbid. A trailing
|
||||
# comment goes only on a quote-free line: cutting at any '#' would hide a call
|
||||
# sitting after an issue number inside a string.
|
||||
uncommented() { sed -e 's/^[[:space:]]*#.*$//' -e "/['\"]/! s/[[:space:]]#.*$//" "$1"; }
|
||||
|
||||
# Telemetry only, so every spawn and kill idiom is out, aliases and short forms
|
||||
# included: PowerShell needs neither the System. prefix nor a space after '&'.
|
||||
forbidden_calls() {
|
||||
uncommented "$1" | grep -nEi \
|
||||
'Start-Process|Start-Job|Start-ThreadJob|Start-Service|Stop-Process|Invoke-Expression|Invoke-Item|Invoke-Command|Invoke-CimMethod|Get-WmiObject|Register-ScheduledJob|WScript\.Shell|taskkill|cmd\.exe|wmic|schtasks|Diagnostics\.Process|\.Kill\(|(^|[^-a-zA-Z0-9_])(iex|saps|spps|spjb|sajb|start)([^-a-zA-Z0-9_]|$)|(^|[^-a-zA-Z0-9_])&[^-a-zA-Z0-9_=&]'
|
||||
}
|
||||
if hits=$(forbidden_calls "$wdscript"); then
|
||||
fail "the watchdog names a process API: $hits"
|
||||
fi
|
||||
# One control per idiom: a denylist is worth only the entries that actually fire.
|
||||
# shellcheck disable=SC2016 # PowerShell source, not this shell's expansions
|
||||
for probe in 'Start-Process notepad.exe' 'Stop-Process -Id 4 -Force' \
|
||||
'[Diagnostics.Process]::Start("x")' '(Get-Process -Name httrack).Kill()' \
|
||||
'Invoke-Expression $cmd' '& "C:\\x.exe" -y' 'Start-Job { 1 }' 'taskkill /F /IM x.exe' \
|
||||
'iex $cmd' '$cmd | iex' '&"C:\\x.exe"' '&$exe' 'saps notepad.exe' 'start notepad.exe' \
|
||||
'spps -Name httrack' 'spjb { 1 }' 'sajb { 1 }' 'Get-WmiObject Win32_Process' \
|
||||
'Invoke-CimMethod -ClassName Win32_Process -MethodName Create' \
|
||||
'Invoke-Command -ScriptBlock { 1 }' 'Register-ScheduledJob -Name x -ScriptBlock { 1 }' \
|
||||
'(New-Object -ComObject WScript.Shell).Run("x")' \
|
||||
'Write-Host "#795"; Start-Process x'; do
|
||||
{
|
||||
cat "$wdscript"
|
||||
echo "$probe"
|
||||
} >"$tmp/probe.ps1"
|
||||
if ! forbidden_calls "$tmp/probe.ps1" >/dev/null; then
|
||||
fail "the process-API audit cannot see: $probe"
|
||||
fi
|
||||
done
|
||||
|
||||
# shellcheck disable=SC2016 # $env: is PowerShell's, not this shell's
|
||||
grep -q '\$env:WATCHDOG_TOKEN' "$wdscript" ||
|
||||
fail "the watchdog does not read its token from the environment"
|
||||
if sed -n '/^param(/,/^)/p' "$wdscript" | grep -qi 'token'; then
|
||||
fail "the watchdog takes its token as a parameter, where the process list exposes it"
|
||||
fi
|
||||
grep -q "ApiBase = 'https://api.github.com'" "$wdscript" ||
|
||||
fail "the watchdog's default endpoint is not the GitHub API"
|
||||
# The token stays in the driver's own shell: exported, every test it runs would
|
||||
# inherit a credential that can post statuses. 172 drives that end to end.
|
||||
grep -q '^unset WATCHDOG_TOKEN$' "$driver" ||
|
||||
fail "the driver leaves its token in the environment its tests inherit"
|
||||
# A background holder of the step's stdout keeps the step open past the suite (#949).
|
||||
grep -q '>>watchdog.log 2>&1' "$driver" ||
|
||||
fail "the watchdog's output is not redirected off the step's stdout"
|
||||
|
||||
if test -r "$workflow"; then
|
||||
# The heartbeat still owns the kill and must beat the step timeout. Anchored,
|
||||
# so an ordinary workflow edit mentioning the key in prose is not a second one.
|
||||
n=$(grep -cE '^[[:space:]]*timeout-minutes:[[:space:]]*[0-9]+[[:space:]]*$' "$workflow")
|
||||
test "$n" -eq 1 || fail "$n timeout-minutes in the workflow, cannot tell which bounds the suite"
|
||||
cap=$(($(awk '/^[[:space:]]*timeout-minutes:/ { print $2; exit }' "$workflow") * 60))
|
||||
stuck=$(sed -n 's/^stuck=\([0-9]*\)$/\1/p' "$driver")
|
||||
deadline=$(sed -n 's/^suite_deadline=\([0-9]*\)$/\1/p' "$driver")
|
||||
test -n "$stuck" || fail "cannot read the heartbeat's static window"
|
||||
test -n "$deadline" || fail "cannot read the driver's suite deadline"
|
||||
# A test may start one second under the deadline and never report again.
|
||||
test $((deadline + stuck + 240)) -le "$cap" ||
|
||||
fail "a kill at ${deadline}+${stuck}s leaves under 240s before the ${cap}s step timeout"
|
||||
|
||||
# The watchdog's own stop is its MaxSeconds, which must land inside the step.
|
||||
# shellcheck disable=SC2016 # $MaxSeconds is PowerShell's
|
||||
maxsecs=$(sed -n 's/^[[:space:]]*\[int\]\$MaxSeconds = \([0-9]*\).*$/\1/p' "$wdscript")
|
||||
test -n "$maxsecs" || fail "cannot read the watchdog's own deadline"
|
||||
test "$maxsecs" -le "$cap" || fail "the watchdog runs ${maxsecs}s past the ${cap}s step timeout"
|
||||
|
||||
# Indentation-agnostic: pinning it would block ever scoping these to the job.
|
||||
grep -Eq '^[[:space:]]+statuses:[[:space:]]*write$' "$workflow" ||
|
||||
fail "the workflow token cannot post a commit status"
|
||||
grep -Eq '^[[:space:]]+contents:[[:space:]]*read$' "$workflow" ||
|
||||
fail "contents: read was dropped from the workflow"
|
||||
# A token that may post statuses must not also be left in .git/config.
|
||||
grep -Eq '^[[:space:]]+persist-credentials:[[:space:]]*false$' "$workflow" ||
|
||||
fail "the checkout persists the job token for the whole job"
|
||||
for v in WATCHDOG_TOKEN WATCHDOG_REPO WATCHDOG_SHA WATCHDOG_CONTEXT; do
|
||||
grep -q "$v:" "$workflow" || fail "$v is not passed to the suite step"
|
||||
done
|
||||
elif test -d "$top/.github"; then
|
||||
fail "$workflow is gone but $top/.github is not"
|
||||
else
|
||||
echo "note: no .github under $top (dist tarball), the workflow audits did not run"
|
||||
fi
|
||||
|
||||
# PATH holds MSYS-style entries only, and TMPDIR on Windows is a D:/... path
|
||||
# that would split it on the drive colon.
|
||||
posixtmp=$tmp
|
||||
if is_windows && command -v cygpath >/dev/null 2>&1; then
|
||||
posixtmp=$(cygpath -u "$tmp")
|
||||
fi
|
||||
bin="$posixtmp/bin"
|
||||
mkdir -p "$bin"
|
||||
# Renamed into place, so the poll below cannot read a half-written file.
|
||||
printf '#!/bin/sh\nprintf "%%s\\n" "$@" >%s/argv.part\nmv %s/argv.part %s/argv\necho "watchdog ready"\n' \
|
||||
"$posixtmp" "$posixtmp" "$posixtmp" >"$bin/argvstub"
|
||||
printf '#!/bin/sh\nexit 127\n' >"$bin/deadstub"
|
||||
chmod +x "$bin/argvstub" "$bin/deadstub"
|
||||
usestub() { cp "$bin/$1" "$bin/pwsh" && cp "$bin/$1" "$bin/powershell.exe"; }
|
||||
usestub argvstub
|
||||
|
||||
if test -x "$bin/pwsh"; then
|
||||
(
|
||||
cd "$posixtmp"
|
||||
# shellcheck disable=SC2030 # local to this subshell is the point
|
||||
PATH="$bin:$PATH"
|
||||
export PATH
|
||||
# An unshadowed pwsh would drive the real watchdog, and every assertion
|
||||
# below would still read as a plain launch failure.
|
||||
test "$(command -v pwsh)" = "$bin/pwsh" ||
|
||||
fail "the stub does not shadow PowerShell: $(command -v pwsh)"
|
||||
# shellcheck source=tests/ci-windows-suite.sh
|
||||
. "$driver"
|
||||
ci_watchdog_pid='' ci_watchdog_exe=''
|
||||
ci_start_native_watchdog "$posixtmp/progress.log" ||
|
||||
fail "the launch reported no watchdog with a PowerShell on PATH"
|
||||
test -n "$ci_watchdog_pid" || fail "the launch set no pid"
|
||||
# wait only reaps a child, which is what the teardown has to signal.
|
||||
wait "$ci_watchdog_pid" || fail "the watchdog is not a child of the driver"
|
||||
|
||||
# A pid comes from the fork, so a dead interpreter still yields one.
|
||||
usestub deadstub
|
||||
ci_watchdog_pid='dangling'
|
||||
rc=0
|
||||
ci_start_native_watchdog "$posixtmp/progress.log" || rc=$?
|
||||
test "$rc" -ne 0 || fail "an interpreter exiting 127 was reported as launched"
|
||||
test -z "$ci_watchdog_pid" || fail "a failed launch left pid $ci_watchdog_pid behind"
|
||||
)
|
||||
for _ in $(seq 1 100); do
|
||||
test -r "$posixtmp/argv" && break
|
||||
sleep 0.1
|
||||
done
|
||||
test -r "$posixtmp/argv" || fail "the watchdog was never launched"
|
||||
grep -qx -- '-SelfTest' "$posixtmp/argv" && fail "the production launch runs the self-test"
|
||||
grep -qx -- '-Post' "$posixtmp/argv" && fail "the production launch is the one-shot mode"
|
||||
# Bypass is load-bearing: Windows PowerShell refuses an unsigned .ps1 without it.
|
||||
for flag in -NoProfile -NonInteractive -ExecutionPolicy Bypass -File; do
|
||||
grep -qx -- "$flag" "$posixtmp/argv" ||
|
||||
fail "the launch dropped $flag: $(tr '\n' ' ' <"$posixtmp/argv")"
|
||||
done
|
||||
grep -qx -- '-ApiBase' "$posixtmp/argv" &&
|
||||
fail "the production launch redirects the status API away from GitHub"
|
||||
# Native form, because a non-MSYS interpreter cannot resolve a /d/a/... path.
|
||||
grep -qx -- "$(nativepath "$wdscript")" "$posixtmp/argv" ||
|
||||
fail "the launch did not name the script: $(tr '\n' ' ' <"$posixtmp/argv")"
|
||||
grep -qx -- "$(nativepath "$posixtmp/progress.log")" "$posixtmp/argv" ||
|
||||
fail "the launch did not pass the progress log"
|
||||
|
||||
# Both outcomes: a pending status that always dangles cannot mean anything.
|
||||
usestub argvstub
|
||||
for probe in 0:success 3:failure; do
|
||||
rm -f "$posixtmp/argv"
|
||||
(
|
||||
cd "$posixtmp"
|
||||
# shellcheck source=tests/ci-windows-suite.sh
|
||||
. "$driver"
|
||||
# shellcheck disable=SC2034 # read by the driver's ci_post_final_status
|
||||
ci_watchdog_exe="$bin/pwsh"
|
||||
ci_post_final_status "${probe%%:*}"
|
||||
)
|
||||
test -r "$posixtmp/argv" || fail "no final status posted for rc ${probe%%:*}"
|
||||
grep -qx -- "${probe#*:}" "$posixtmp/argv" ||
|
||||
fail "rc ${probe%%:*} posted $(tr '\n' ' ' <"$posixtmp/argv"), want ${probe#*:}"
|
||||
done
|
||||
|
||||
# The runner cancels a superseded or overrunning step by signal, and bash
|
||||
# reaches its EXIT trap from one with $? reading 0.
|
||||
sigs=()
|
||||
for sig in TERM HUP INT; do
|
||||
# A shell that inherited a signal ignored can neither trap nor take it,
|
||||
# which is what a nohup'd "make check" hands us for HUP.
|
||||
if test -z "$(trap -p "$sig")"; then
|
||||
sigs+=("$sig")
|
||||
fi
|
||||
done
|
||||
test "${#sigs[@]}" -ge 1 || fail "every signal is ignored here, the teardown was not driven"
|
||||
for sig in "${sigs[@]}"; do
|
||||
rm -f "$posixtmp/argv"
|
||||
(
|
||||
cd "$posixtmp"
|
||||
# shellcheck source=tests/ci-windows-suite.sh
|
||||
. "$driver"
|
||||
sleep 30 &
|
||||
# shellcheck disable=SC2034 # all three are read by the driver's teardown
|
||||
watchdog=$! heartbeat='' ci_watchdog_exe="$bin/pwsh"
|
||||
ci_install_traps
|
||||
kill -"$sig" "$BASHPID"
|
||||
sleep 5
|
||||
) >/dev/null 2>&1 || true
|
||||
test -r "$posixtmp/argv" || fail "a $sig-killed suite posted nothing"
|
||||
grep -qx -- failure "$posixtmp/argv" ||
|
||||
fail "a $sig-killed suite posted $(tr '\n' ' ' <"$posixtmp/argv"), want failure"
|
||||
done
|
||||
else
|
||||
echo "note: $tmp is noexec, the driver wiring was not driven"
|
||||
fi
|
||||
|
||||
psruns=()
|
||||
for c in pwsh powershell.exe powershell; do
|
||||
command -v "$c" >/dev/null 2>&1 || continue
|
||||
# powershell and powershell.exe are one interpreter under two names.
|
||||
seen=0
|
||||
for p in ${psruns[@]+"${psruns[@]}"}; do
|
||||
test "${p%.exe}" != "${c%.exe}" || seen=1
|
||||
done
|
||||
test "$seen" -eq 1 || psruns+=("$c")
|
||||
done
|
||||
if test "${#psruns[@]}" -eq 0; then
|
||||
echo "SKIP: no PowerShell here, the watchdog's own self-test cannot run"
|
||||
exit 77
|
||||
fi
|
||||
|
||||
# Bounded: a mutant that loops forever would otherwise be caught only by the step
|
||||
# timeout this work exists to beat. -NoPost layers over the scrub above.
|
||||
run_wd() {
|
||||
local secs=$1
|
||||
shift
|
||||
# -NoPost last: everything after -File <script> belongs to the script.
|
||||
run_with_timeout "$secs" "$psrun" "${psargs[@]}" "$@" -NoPost 2>&1
|
||||
}
|
||||
|
||||
# The legs below grade what reached the API. Printing and posting are decoupled,
|
||||
# so counting what the loop wrote can see neither the request rate nor the backoff.
|
||||
py=$(find_python || true)
|
||||
sink="$tmp/sink.py"
|
||||
cat >"$sink" <<'PY'
|
||||
import http.server
|
||||
import sys
|
||||
|
||||
code, rec, portfile = int(sys.argv[1]), sys.argv[2], sys.argv[3]
|
||||
|
||||
|
||||
class Handler(http.server.BaseHTTPRequestHandler):
|
||||
def do_POST(self):
|
||||
n = int(self.headers.get("Content-Length") or 0)
|
||||
with open(rec, "ab") as f:
|
||||
f.write(self.rfile.read(n) + b"\n")
|
||||
self.send_response(code)
|
||||
self.send_header("Content-Length", "2")
|
||||
self.end_headers()
|
||||
self.wfile.write(b"{}")
|
||||
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
|
||||
srv = http.server.HTTPServer(("127.0.0.1", 0), Handler)
|
||||
with open(portfile, "w") as f:
|
||||
f.write(str(srv.server_port))
|
||||
srv.serve_forever()
|
||||
PY
|
||||
|
||||
posts="$tmp/posts"
|
||||
portfile="$tmp/port"
|
||||
legout="$tmp/leg.out"
|
||||
port='' sinkpid=''
|
||||
sink_start() {
|
||||
: >"$posts"
|
||||
rm -f "$portfile"
|
||||
"$py" "$sink" "$1" "$posts" "$portfile" &
|
||||
sinkpid=$!
|
||||
for _ in $(seq 1 100); do
|
||||
test -s "$portfile" && break
|
||||
sleep 0.1
|
||||
done
|
||||
test -s "$portfile" || fail "the status sink never bound a port"
|
||||
port=$(cat "$portfile")
|
||||
}
|
||||
sink_stop() {
|
||||
stop_server "$sinkpid"
|
||||
sinkpid=''
|
||||
}
|
||||
# Ours, so a leg that lost its -ApiBase reaches GitHub with a credential it
|
||||
# refuses rather than with whatever the step was handed.
|
||||
export WATCHDOG_TOKEN=dummy WATCHDOG_REPO=octo/nowhere WATCHDOG_SHA=0000000
|
||||
|
||||
run_posting() {
|
||||
local secs=$1
|
||||
shift
|
||||
run_with_timeout "$secs" "$psrun" "${psargs[@]}" "$@" -ApiBase "http://127.0.0.1:$port"
|
||||
}
|
||||
# t and q of each posted status, in order and whatever key order the JSON took.
|
||||
posted_pairs() {
|
||||
sed -n 's/.*t=\([0-9]*\)s q=\([0-9?]*\)s.*/\1 \2/p' "$posts"
|
||||
}
|
||||
|
||||
# One property per leg, each returning its diagnosis so a mutant can be required
|
||||
# to trip it.
|
||||
cadence_leg() {
|
||||
local n
|
||||
sink_start 201
|
||||
printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
|
||||
run_posting 60 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
|
||||
-IntervalSeconds 100 -PollSeconds 1 -MaxSeconds 4 >"$legout" 2>&1 || {
|
||||
sink_stop
|
||||
echo "the loop exited nonzero: $(cat "$legout")"
|
||||
return 1
|
||||
}
|
||||
sink_stop
|
||||
n=$(grep -c . "$posts" || true)
|
||||
test "$n" -eq 1 || {
|
||||
echo "$n API calls at a 100s cadence over 4s, want 1"
|
||||
return 1
|
||||
}
|
||||
grep -q '36_local-bigcrawl.test' "$posts" || {
|
||||
echo "the posted status does not name the test in flight: $(cat "$posts")"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
staticness_leg() {
|
||||
local stop moved lastq legpid
|
||||
sink_start 201
|
||||
printf 'RUN 36_local-bigcrawl.test at 0s\n' >"$tmp/progress.log"
|
||||
run_posting 90 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
|
||||
-IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 12 >"$legout" 2>&1 &
|
||||
legpid=$!
|
||||
# Six seconds of a log being written, then silence. Staticness has to follow
|
||||
# the log through both, and elapsed time follows neither.
|
||||
stop=$((SECONDS + 6))
|
||||
while test "$SECONDS" -lt "$stop"; do
|
||||
printf 'RUN 36_local-bigcrawl.test at %ss\n' "$SECONDS" >>"$tmp/progress.log"
|
||||
sleep 0.4
|
||||
done
|
||||
wait "$legpid" || {
|
||||
sink_stop
|
||||
echo "the loop exited nonzero: $(cat "$legout")"
|
||||
return 1
|
||||
}
|
||||
sink_stop
|
||||
moved=$(posted_pairs | awk '$2 ~ /^[0-9]+$/ && $1 >= 2 && $2 <= 1 { n++ } END { print n + 0 }')
|
||||
test "$moved" -ge 1 || {
|
||||
echo "no post read the log as moving while it was being written: $(posted_pairs | tr '\n' '/')"
|
||||
return 1
|
||||
}
|
||||
lastq=$(posted_pairs | awk 'END { print $2 }')
|
||||
case $lastq in
|
||||
'' | *[!0-9]*)
|
||||
echo "the last post carries no staticness: $(posted_pairs | tr '\n' '/')"
|
||||
return 1
|
||||
;;
|
||||
esac
|
||||
test "$lastq" -ge 3 || {
|
||||
echo "staticness stalled at ${lastq}s over six seconds of silence: $(posted_pairs | tr '\n' '/')"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
backoff_leg() {
|
||||
local calls lines
|
||||
sink_start 403
|
||||
printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
|
||||
run_posting 60 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
|
||||
-IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 8 >"$legout" 2>&1 || {
|
||||
sink_stop
|
||||
echo "the loop exited nonzero: $(cat "$legout")"
|
||||
return 1
|
||||
}
|
||||
sink_stop
|
||||
calls=$(grep -c . "$posts" || true)
|
||||
lines=$(grep -c 't=[0-9]*s q=' "$legout" || true)
|
||||
test "$calls" -ge 2 || {
|
||||
echo "a rejecting API was called $calls times: the loop stopped trying"
|
||||
return 1
|
||||
}
|
||||
test "$calls" -le 5 || {
|
||||
echo "$calls calls to an API that rejects every one: nothing backs off"
|
||||
return 1
|
||||
}
|
||||
test "$lines" -ge $((2 * calls)) || {
|
||||
echo "$lines status lines for $calls calls: the artifact throttles with the API"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
fullstdout_leg() {
|
||||
local n
|
||||
sink_start 201
|
||||
printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
|
||||
run_posting 60 -File "$1" -ProgressLog "$(nativepath "$tmp/progress.log")" \
|
||||
-IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 5 >/dev/full 2>&1 || true
|
||||
sink_stop
|
||||
n=$(grep -c . "$posts" || true)
|
||||
test "$n" -ge 3 || {
|
||||
echo "$n posts with a full stdout: a failed log write took the loop with it"
|
||||
return 1
|
||||
}
|
||||
}
|
||||
|
||||
first=1
|
||||
for psrun in "${psruns[@]}"; do
|
||||
psargs=(-NoProfile -NonInteractive)
|
||||
case $psrun in
|
||||
powershell*) psargs+=(-ExecutionPolicy Bypass) ;;
|
||||
esac
|
||||
|
||||
out=$(run_wd 120 -File "$(nativepath "$wdscript")" -SelfTest) ||
|
||||
fail "the $psrun self-test failed: $out"
|
||||
grep -q 'watchdog self-test OK' <<<"$out" || fail "the $psrun self-test said: $out"
|
||||
|
||||
# A mutant the script survives is a property nothing is checking.
|
||||
mutate() {
|
||||
local name=$1 expr=$2 m="$tmp/$1.ps1" rc=0 said
|
||||
sed "$expr" "$wdscript" >"$m"
|
||||
cmp -s "$wdscript" "$m" && fail "mutant $name changed nothing, so it proves nothing"
|
||||
said=$(run_wd 120 -File "$(nativepath "$m")" -SelfTest) || rc=$?
|
||||
test "$rc" -ne 0 || fail "mutant $name passed the $psrun self-test: $said"
|
||||
}
|
||||
# shellcheck disable=SC2016 # PowerShell variables, quoted for sed
|
||||
mutate cadence-off-by-one 's/-ge \$Interval/-gt \$Interval/'
|
||||
# shellcheck disable=SC2016
|
||||
mutate backoff-uncapped 's/\[Math\]::Min(\$Current \* 2, 32)/$Current * 2/'
|
||||
# shellcheck disable=SC2016
|
||||
mutate status-unclipped '/if (\$s.Length -gt 140)/d'
|
||||
# shellcheck disable=SC2016
|
||||
mutate tail-always-ok 's/\$r = @{ Ok = \$false;/$r = @{ Ok = $true;/'
|
||||
|
||||
# Timed and counted by this shell, never read out of what the watchdog
|
||||
# writes: the cadence and the deadline are both mine to set.
|
||||
printf 'RUN 36_local-bigcrawl.test at 41s\n' >"$tmp/progress.log"
|
||||
out=$(run_wd 60 -File "$(nativepath "$wdscript")" \
|
||||
-ProgressLog "$(nativepath "$tmp/progress.log")" \
|
||||
-IntervalSeconds 100 -PollSeconds 1 -MaxSeconds 5) ||
|
||||
fail "the $psrun slow-cadence loop exited nonzero: $out"
|
||||
slow=$(grep -c 't=[0-9]*s q=' <<<"$out" || true)
|
||||
test "$slow" -eq 1 || fail "$slow status lines at a 100s cadence over 5s, want 1: $out"
|
||||
|
||||
began=$SECONDS
|
||||
out=$(run_wd 60 -File "$(nativepath "$wdscript")" \
|
||||
-ProgressLog "$(nativepath "$tmp/progress.log")" \
|
||||
-IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 5) ||
|
||||
fail "the $psrun loop exited nonzero: $out"
|
||||
spent=$((SECONDS - began))
|
||||
test "$spent" -ge 4 || fail "a 5s watchdog returned after ${spent}s: MaxSeconds is not seconds"
|
||||
test "$spent" -le 20 || fail "a 5s watchdog ran ${spent}s: it does not stop on its own"
|
||||
fast=$(grep -c 't=[0-9]*s q=' <<<"$out" || true)
|
||||
# Against the slow leg, not a floor: a probe tick is not instant.
|
||||
test "$fast" -gt "$slow" || fail "$fast status lines at 1s against $slow at 100s: $out"
|
||||
grep -q '36_local-bigcrawl.test' <<<"$out" || fail "the status does not name the test in flight: $out"
|
||||
grep -q '95_local-sitemap' <<<"$out" && fail "the status named a test that was never in flight"
|
||||
# Trips if the scrub and -NoPost were both lost: only a real attempt logs this.
|
||||
grep -q 'status post failed' <<<"$out" && fail "a test leg reached the status API"
|
||||
|
||||
# An unreadable log must not read as a wedged one.
|
||||
out=$(run_wd 60 -File "$(nativepath "$wdscript")" \
|
||||
-ProgressLog "$(nativepath "$tmp/no-such.log")" \
|
||||
-IntervalSeconds 1 -PollSeconds 1 -MaxSeconds 3) ||
|
||||
fail "the $psrun loop exited nonzero on a missing log: $out"
|
||||
grep -q 'q=?s' <<<"$out" || fail "a missing log reported a staticness: $out"
|
||||
|
||||
# The legs below read the sink, not the loop, and they exercise the script's
|
||||
# decisions rather than the interpreter's: one flavour is enough.
|
||||
test "$first" -eq 1 || continue
|
||||
first=0
|
||||
if test -z "$py"; then
|
||||
echo "note: no python3 here, the posting legs did not run"
|
||||
continue
|
||||
fi
|
||||
|
||||
legs=(cadence staticness backoff)
|
||||
# ENOSPC on stdout, which is a state the sick runner reaches.
|
||||
test -w /dev/full && legs+=(fullstdout)
|
||||
for leg in "${legs[@]}"; do
|
||||
said=$("${leg}_leg" "$(nativepath "$wdscript")") ||
|
||||
fail "the $psrun watchdog fails the $leg leg: $said"
|
||||
done
|
||||
|
||||
# Same shape as mutate() above, graded by a leg instead of the self-test.
|
||||
mutate_leg() {
|
||||
local name=$1 expr=$2 leg=$3 m="$tmp/$1.ps1" said
|
||||
sed "$expr" "$wdscript" >"$m"
|
||||
cmp -s "$wdscript" "$m" && fail "mutant $name changed nothing, so it proves nothing"
|
||||
said=$("${leg}_leg" "$(nativepath "$m")") &&
|
||||
fail "mutant $name passed the $leg leg"
|
||||
case $said in
|
||||
*'sink never bound'*) fail "the status sink failed under mutant $name" ;;
|
||||
esac
|
||||
}
|
||||
mutate_leg status-every-poll \
|
||||
"s/if ((Get-WatchdogAction \$now \$postedAt \$IntervalSeconds) -eq 'post')/if (\$true)/" cadence
|
||||
# shellcheck disable=SC2016 # PowerShell variables, quoted for sed
|
||||
mutate_leg static-frozen 's/\$static = \$now - \$movedAt/$static = 0/' staticness
|
||||
# shellcheck disable=SC2016
|
||||
mutate_leg static-unknown 's/\$static = \$now - \$movedAt/$static = -1/' staticness
|
||||
# shellcheck disable=SC2016
|
||||
mutate_leg static-elapsed 's/\$static = \$now - \$movedAt/$static = $now/' staticness
|
||||
# shellcheck disable=SC2016
|
||||
mutate_leg backoff-never-skips 's/if (\$skip -gt 0)/if ($false)/' backoff
|
||||
if test -w /dev/full; then
|
||||
mutate_leg log-write-fatal 's/try { \(Write-Host .*\) } catch { }/\1/' fullstdout
|
||||
fi
|
||||
done
|
||||
|
||||
echo "native watchdog OK (${psruns[*]})"
|
||||
88
tests/230_local-ftp-userpass.test
Executable file
88
tests/230_local-ftp-userpass.test
Executable file
@@ -0,0 +1,88 @@
|
||||
#!/bin/bash
|
||||
#
|
||||
# The credentials an URL carries must reach the control channel whole: a clip
|
||||
# logged in as a different account (#1032). The engine test only ever probes a
|
||||
# dead port, so nothing there reads what the login actually sends.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
: "${top_srcdir:=..}"
|
||||
testdir=$(cd "$(dirname "$0")" && pwd)
|
||||
# shellcheck source=tests/testlib.sh
|
||||
. "${testdir}/testlib.sh"
|
||||
|
||||
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
|
||||
command -v httrack >/dev/null || {
|
||||
echo "could not find httrack" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
server=$(nativepath "${testdir}/ftp-server.py")
|
||||
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftpuser.XXXXXX")
|
||||
serverpid=
|
||||
cleanup() {
|
||||
stop_server "$serverpid"
|
||||
rm -rf "$tmpdir"
|
||||
}
|
||||
trap 'set +e; cleanup' EXIT
|
||||
trap cleanup HUP INT QUIT PIPE TERM
|
||||
|
||||
root="${tmpdir}/root"
|
||||
out="${tmpdir}/crawl"
|
||||
cmds="${tmpdir}/cmds"
|
||||
mkdir -p "$root" "$out"
|
||||
printf 'body\n' >"${root}/f.txt"
|
||||
|
||||
serverlog="${tmpdir}/server.out"
|
||||
"$python" "$server" --root "$(nativepath "$root")" --require-pass \
|
||||
--log "$(nativepath "$cmds")" >"$serverlog" 2>&1 &
|
||||
serverpid=$!
|
||||
port=$(discover_server_port "$serverlog" "$serverpid") || exit 1
|
||||
|
||||
fail() {
|
||||
echo "FAIL: $*" >&2
|
||||
exit 1
|
||||
}
|
||||
ok() { echo "OK: $*"; }
|
||||
rep() { awk -v n="$2" -v c="$1" 'BEGIN { while (i++ < n) printf "%s", c }'; }
|
||||
|
||||
crawl() {
|
||||
: >"$cmds"
|
||||
rm -rf "${out:?}"
|
||||
mkdir -p "$out"
|
||||
run_with_timeout 60 httrack "ftp://$1@127.0.0.1:${port}/f.txt" -O "$out" \
|
||||
--quiet --disable-security-limits --robots=0 --timeout=20 \
|
||||
--max-time=45 --retries=1 -c1 >"${tmpdir}/log" 2>&1
|
||||
}
|
||||
|
||||
# --- a plain login goes out unchanged ----------------------------------------
|
||||
crawl "bob:secret" || fail "the plain-login crawl never finished"
|
||||
sent=$(cat "$cmds")
|
||||
grep -qxF "USER bob" <<<"$sent" || fail "no USER bob on the wire: ${sent}"
|
||||
grep -qxF "PASS secret" <<<"$sent" || fail "no PASS secret on the wire: ${sent}"
|
||||
cmp -s "${out}/127.0.0.1_${port}/f.txt" "${root}/f.txt" ||
|
||||
fail "the login did not bring the body back; mirror holds $(find "$out" -type f)"
|
||||
ok "bob:secret reaches the server verbatim, and mirrors"
|
||||
|
||||
# --- the longest userinfo that fits, byte for byte ---------------------------
|
||||
# 255 of each: a 256th byte is refused, so this is what the wire must carry.
|
||||
user=$(rep a 255)
|
||||
pass=$(rep b 255)
|
||||
crawl "${user}:${pass}" || fail "the 255-byte login crawl never finished"
|
||||
sent=$(cat "$cmds")
|
||||
grep -qxF "USER ${user}" <<<"$sent" ||
|
||||
fail "the wire user is not the 255 bytes the URL named: $(grep -a '^USER' <<<"$sent")"
|
||||
grep -qxF "PASS ${pass}" <<<"$sent" ||
|
||||
fail "the wire pass is not the 255 bytes the URL named: $(grep -a '^PASS' <<<"$sent")"
|
||||
ok "a 255-byte user and password reach the server unclipped"
|
||||
|
||||
# --- over-long userinfo never opens a session --------------------------------
|
||||
# The bare name has no ':' to bound it, only the '@', and is the worse branch.
|
||||
for u in "$(rep a 256):pw" "$(rep a 256)"; do
|
||||
crawl "$u" || fail "the over-long crawl never finished"
|
||||
grep -aq "FTP user name or password too long" "${out}/hts-log.txt" ||
|
||||
fail "over-long userinfo was not refused: $(cat "${out}/hts-log.txt")"
|
||||
sent=$(cat "$cmds")
|
||||
test -z "$sent" || fail "the engine logged in under a clipped name: ${sent}"
|
||||
done
|
||||
ok "an over-long name is refused before anything reaches the control channel"
|
||||
@@ -10,7 +10,7 @@ EXTRA_DIST = $(TESTS) renamefail.c threadattrfail.c nobacktrace.c altstackprobe.
|
||||
header-injection-server.py header-injection-check.py \
|
||||
pty-resize.py test-timeout.sh png-colors.py \
|
||||
local-crawl.sh local-server.py ftp-server.py testlib.sh \
|
||||
ci-windows-suite.sh ci-windows-watchdog.ps1 \
|
||||
ci-windows-suite.sh \
|
||||
server.crt server.key \
|
||||
server-root/simple/basic.html server-root/simple/link.html \
|
||||
server-root/stripquery/index.html server-root/stripquery/a.html \
|
||||
|
||||
@@ -29,81 +29,6 @@ ci_annotate() {
|
||||
# schedule the workflow really passes off a virtual clock instead of waiting it out.
|
||||
hb_now() { echo "$SECONDS"; }
|
||||
|
||||
# Start the off-box telemetry over the progress log $1, setting ci_watchdog_pid
|
||||
# and ci_watchdog_exe; 1 with no PowerShell. Forks nothing and kills nothing, so
|
||||
# it still reports where the heartbeat below cannot (#795).
|
||||
ci_start_native_watchdog() {
|
||||
local progress=$1 ps1 c waited=0
|
||||
ps1="$testdir/ci-windows-watchdog.ps1"
|
||||
test -r "$ps1" || return 1
|
||||
ci_watchdog_exe=''
|
||||
for c in pwsh powershell.exe; do
|
||||
if command -v "$c" >/dev/null 2>&1; then
|
||||
ci_watchdog_exe=$c
|
||||
break
|
||||
fi
|
||||
done
|
||||
test -n "$ci_watchdog_exe" || return 1
|
||||
# Never the step's stdout: a background holder of that pipe keeps the step
|
||||
# open past the suite (#949), and tests/*.log reaches the artifact anyway.
|
||||
: >watchdog.log
|
||||
WATCHDOG_TOKEN="${ci_watchdog_token:-}" \
|
||||
"$ci_watchdog_exe" -NoProfile -NonInteractive -ExecutionPolicy Bypass \
|
||||
-File "$(nativepath "$ps1")" \
|
||||
-ProgressLog "$(nativepath "$progress")" \
|
||||
>>watchdog.log 2>&1 &
|
||||
ci_watchdog_pid=$!
|
||||
# $! comes from the fork, not the exec: wait for it to actually speak.
|
||||
while test "$waited" -lt 30; do
|
||||
grep -q '^watchdog ready$' watchdog.log && return 0
|
||||
kill -0 "$ci_watchdog_pid" 2>/dev/null || break
|
||||
sleep 1
|
||||
waited=$((waited + 1))
|
||||
done
|
||||
kill_pid "$ci_watchdog_pid"
|
||||
ci_watchdog_pid=''
|
||||
return 1
|
||||
}
|
||||
|
||||
# Post the suite's own verdict from exit status $1, described by $2: the watchdog
|
||||
# cannot see the driver exit, and a pending status that never resolves says nothing.
|
||||
ci_post_final_status() {
|
||||
local rc=$1 msg=${2:-"suite step ended rc=$1"} state=success
|
||||
test -n "${ci_watchdog_exe:-}" || return 0
|
||||
test "$rc" -eq 0 || state=failure
|
||||
WATCHDOG_TOKEN="${ci_watchdog_token:-}" \
|
||||
"$ci_watchdog_exe" -NoProfile -NonInteractive -ExecutionPolicy Bypass \
|
||||
-File "$(nativepath "$testdir/ci-windows-watchdog.ps1")" \
|
||||
-Post "$state" -Message "$msg" \
|
||||
>>watchdog.log 2>&1 || true
|
||||
}
|
||||
|
||||
# End both watchers and post the suite's verdict from exit status $1.
|
||||
ci_stop_watchers() {
|
||||
local rc=$1 msg="suite step ended rc=$1"
|
||||
test -z "${ci_signal:-}" || msg="suite step killed by SIG$ci_signal"
|
||||
kill "${heartbeat:-}" 2>/dev/null
|
||||
# Silenced before the verdict, or its next tick overwrites it.
|
||||
test -z "${watchdog:-}" || kill_pid "$watchdog"
|
||||
# Posted even for a launch given up on: it may have posted a pending first.
|
||||
ci_post_final_status "$rc" "$msg"
|
||||
return 0
|
||||
}
|
||||
|
||||
ci_on_signal() {
|
||||
ci_signal=$1
|
||||
exit $((128 + $2))
|
||||
}
|
||||
|
||||
# A shell killed by a signal reaches its EXIT trap with $? = 0, so a cancelled or
|
||||
# timed-out step would report success: the verdict has to come from the handler.
|
||||
ci_install_traps() {
|
||||
trap 'ci_on_signal INT 2' INT
|
||||
trap 'ci_on_signal TERM 15' TERM
|
||||
trap 'ci_on_signal HUP 1' HUP
|
||||
trap 'ci_rc=$?; set +e; ci_stop_watchers "$ci_rc" || true' EXIT
|
||||
}
|
||||
|
||||
ci_suite_heartbeat() {
|
||||
local quiet=$1 every=$2 progress=$3 stuck=$4 main=$5
|
||||
local tick=$2 begin now line said moved last=''
|
||||
@@ -123,10 +48,6 @@ ci_suite_heartbeat() {
|
||||
# outlive the step's own timeout before being caught.
|
||||
if test $((now - moved)) -ge "$stuck"; then
|
||||
ci_annotate error "suite watchdog" "killing the step: $((now - moved))s without progress, in flight: $last"
|
||||
# A taskkill runs no trap, so this is the only verdict the kill path
|
||||
# can leave; the reporter goes first or its next tick overwrites it.
|
||||
test -z "${watchdog:-}" || kill_pid "$watchdog"
|
||||
ci_post_final_status 1 "killed after $((now - moved))s without progress, in flight: $last"
|
||||
# Direct first: kill_tree may reap this watchdog before its own root (#953).
|
||||
kill_pid "$main"
|
||||
kill_tree "$main"
|
||||
@@ -198,21 +119,9 @@ export HTTRACK_PROGRESS_LOG="$PWD/$progress"
|
||||
# its own terms keeps its log. Quiet past 16 min, clear of the 13 a healthy
|
||||
# run measures here, and a kill 900s after the last progress line clears
|
||||
# the longest legitimate gap, one $per_test.
|
||||
stuck=900
|
||||
# Orthogonal to that heartbeat rather than a spare of it: the heartbeat needs
|
||||
# 960s of quiet and 900s of static log, and every #795 death measured so far
|
||||
# lands inside the first 750s of the step.
|
||||
watchdog='' ci_watchdog_pid='' ci_watchdog_exe='' ci_signal='' ci_rc=0
|
||||
# Held here, never exported: a token every test child inherits is how four forged
|
||||
# statuses once reached a commit under test.
|
||||
ci_watchdog_token=${WATCHDOG_TOKEN:-}
|
||||
unset WATCHDOG_TOKEN
|
||||
ci_start_native_watchdog "$PWD/$progress" && watchdog=$ci_watchdog_pid
|
||||
test -n "$watchdog" || echo "no off-box watchdog: no usable PowerShell"
|
||||
# Started after it, so the kill path can silence and outlive the reporter.
|
||||
ci_suite_heartbeat 960 360 "$progress" "$stuck" $$ &
|
||||
ci_suite_heartbeat 960 360 "$progress" 900 $$ &
|
||||
heartbeat=$!
|
||||
ci_install_traps
|
||||
trap 'kill "$heartbeat" 2>/dev/null || true' EXIT
|
||||
|
||||
pass=0 fail=0 skip=0 failed="" skipped="" deadline=0
|
||||
# label:pattern, globbed rather than enumerated so a new NNN_engine-*.test or
|
||||
|
||||
@@ -1,243 +0,0 @@
|
||||
# Off-box telemetry for the Windows suite step (#795). It spawns nothing and
|
||||
# kills nothing, so nothing a wedge takes away can disarm it, and it reports as a
|
||||
# commit status, the only channel that outlives a dead runner.
|
||||
param(
|
||||
[string]$ProgressLog = '',
|
||||
[int]$IntervalSeconds = 30,
|
||||
[int]$PollSeconds = 5,
|
||||
# Cannot outlive the step, whatever the caller forgets to kill.
|
||||
[int]$MaxSeconds = 2700,
|
||||
# One-shot mode: post this state and exit, so the driver reports its own end.
|
||||
[string]$Post = '',
|
||||
[string]$Message = '',
|
||||
# Seam for the suite's own test, which points it at a sink it can count.
|
||||
[string]$ApiBase = 'https://api.github.com',
|
||||
[switch]$NoPost,
|
||||
[switch]$SelfTest
|
||||
)
|
||||
|
||||
$ErrorActionPreference = 'Stop'
|
||||
# Windows PowerShell renders a progress bar per web call otherwise.
|
||||
$ProgressPreference = 'SilentlyContinue'
|
||||
|
||||
# --- decisions, kept pure so -SelfTest can drive them by return value ---------
|
||||
|
||||
function Get-WatchdogAction {
|
||||
param([int]$Now, [int]$Posted, [int]$Interval)
|
||||
if (($Now - $Posted) -ge $Interval) { return 'post' }
|
||||
return 'wait'
|
||||
}
|
||||
|
||||
# Posts to skip after a rejected one: a fork PR's token is read-only for the run.
|
||||
function Get-NextBackoff {
|
||||
param([int]$Current)
|
||||
if ($Current -lt 1) { return 1 }
|
||||
return [Math]::Min($Current * 2, 32)
|
||||
}
|
||||
|
||||
# GitHub truncates a description at 140 chars, so the counters take the cut, not
|
||||
# the fields a wedge is read for. $Static below zero is unknown.
|
||||
function Format-WatchdogStatus {
|
||||
param([int]$Elapsed, [int]$Static, [string]$InFlight, [string]$Counters)
|
||||
$q = '?'
|
||||
if ($Static -ge 0) { $q = [string]$Static }
|
||||
$t = ($InFlight -replace '\s+', ' ').Trim()
|
||||
if ($t.Length -gt 46) { $t = $t.Substring(0, 46) }
|
||||
$s = 't={0}s q={1}s {2} | {3}' -f $Elapsed, $q, $t, $Counters
|
||||
if ($s.Length -gt 140) { $s = $s.Substring(0, 140) }
|
||||
return $s
|
||||
}
|
||||
|
||||
# --- probes ------------------------------------------------------------------
|
||||
|
||||
# One try/catch per counter: a probe that fails costs its own field, not the loop.
|
||||
function Get-WatchdogCounters {
|
||||
$f = New-Object System.Collections.ArrayList
|
||||
try {
|
||||
$os = Get-CimInstance -ClassName Win32_OperatingSystem -OperationTimeoutSec 5
|
||||
[void]$f.Add('m={0}' -f [int]($os.FreePhysicalMemory / 1KB))
|
||||
[void]$f.Add('v={0}' -f [int]($os.FreeVirtualMemory / 1KB))
|
||||
} catch { [void]$f.Add('m=? v=?') }
|
||||
try {
|
||||
$ps = @(Get-Process)
|
||||
[void]$f.Add('p={0}' -f $ps.Count)
|
||||
[void]$f.Add('h={0}' -f (($ps | Measure-Object -Property Handles -Sum).Sum))
|
||||
} catch { [void]$f.Add('p=? h=?') }
|
||||
try {
|
||||
$drive = New-Object System.IO.DriveInfo($env:SystemDrive + '\')
|
||||
[void]$f.Add('d={0}' -f [int]($drive.AvailableFreeSpace / 1MB))
|
||||
} catch { [void]$f.Add('d=?') }
|
||||
return ($f -join ' ')
|
||||
}
|
||||
|
||||
# Ok separates "nothing moved" from "could not read it", which would otherwise report
|
||||
# a wedge for an unreadable file. Share flags: the driver appends as we read.
|
||||
function Get-ProgressTail {
|
||||
param([string]$Path)
|
||||
$r = @{ Ok = $false; Signature = ''; Line = '' }
|
||||
if (-not $Path) { return $r }
|
||||
try {
|
||||
$share = [System.IO.FileShare]::ReadWrite -bor [System.IO.FileShare]::Delete
|
||||
$fs = New-Object System.IO.FileStream($Path, [System.IO.FileMode]::Open, [System.IO.FileAccess]::Read, $share)
|
||||
try {
|
||||
$sr = New-Object System.IO.StreamReader($fs)
|
||||
$text = $sr.ReadToEnd()
|
||||
} finally { $fs.Dispose() }
|
||||
$lines = @($text -split "`r?`n" | Where-Object { $_ -ne '' })
|
||||
if ($lines.Count -gt 0) { $r.Line = $lines[-1] }
|
||||
$r.Signature = '{0}|{1}' -f $text.Length, $r.Line
|
||||
$r.Ok = $true
|
||||
} catch { }
|
||||
return $r
|
||||
}
|
||||
|
||||
function Write-WatchdogLog {
|
||||
param([string]$Message)
|
||||
# A full disk is a state the sick runner reaches, and a log line lost to it
|
||||
# must not take the loop reporting off-box with it.
|
||||
try { Write-Host ('[watchdog {0:HH:mm:ss}] {1}' -f (Get-Date), $Message) } catch { }
|
||||
}
|
||||
|
||||
# --- reporting ---------------------------------------------------------------
|
||||
|
||||
$script:Repo = $env:WATCHDOG_REPO
|
||||
$script:Sha = $env:WATCHDOG_SHA
|
||||
$script:Token = $env:WATCHDOG_TOKEN
|
||||
$script:TargetUrl = $env:WATCHDOG_URL
|
||||
$script:Context = $env:WATCHDOG_CONTEXT
|
||||
if (-not $script:Context) { $script:Context = 'windows-suite-watchdog' }
|
||||
|
||||
# -NoPost is the test's hard stop: no request, whatever the environment holds.
|
||||
function Send-WatchdogStatus {
|
||||
param([string]$State, [string]$Description)
|
||||
if ($NoPost -or $SelfTest) { return $false }
|
||||
if (-not $script:Token -or -not $script:Repo -or -not $script:Sha) { return $false }
|
||||
$body = @{ state = $State; context = $script:Context; description = $Description }
|
||||
if ($script:TargetUrl) { $body['target_url'] = $script:TargetUrl }
|
||||
try {
|
||||
Invoke-RestMethod -Method Post -TimeoutSec 20 `
|
||||
-Uri ('{0}/repos/{1}/statuses/{2}' -f $ApiBase.TrimEnd('/'), $script:Repo, $script:Sha) `
|
||||
-UserAgent 'httrack-windows-suite-watchdog' `
|
||||
-Headers @{
|
||||
Authorization = ('Bearer {0}' -f $script:Token)
|
||||
Accept = 'application/vnd.github+json'
|
||||
} -ContentType 'application/json' -Body ($body | ConvertTo-Json -Compress) | Out-Null
|
||||
return $true
|
||||
} catch {
|
||||
Write-WatchdogLog ('status post failed: {0}' -f $_.Exception.Message)
|
||||
return $false
|
||||
}
|
||||
}
|
||||
|
||||
# --- self-test ----------------------------------------------------------------
|
||||
|
||||
function Invoke-WatchdogSelfTest {
|
||||
$bad = New-Object System.Collections.ArrayList
|
||||
function Assert-That($cond, $what) { if (-not $cond) { [void]$bad.Add($what) } }
|
||||
|
||||
Assert-That ((Get-WatchdogAction 30 0 30) -eq 'post') 'a post due exactly on the interval was skipped'
|
||||
Assert-That ((Get-WatchdogAction 29 0 30) -eq 'wait') 'posted ahead of the interval'
|
||||
Assert-That ((Get-WatchdogAction 5000 4990 30) -eq 'wait') 'posted off cadence'
|
||||
|
||||
Assert-That ((Get-NextBackoff 0) -eq 1) 'the first rejection does not back off'
|
||||
Assert-That ((Get-NextBackoff 1) -eq 2) 'the backoff does not grow'
|
||||
Assert-That ((Get-NextBackoff 32) -eq 32) 'the backoff is not capped'
|
||||
|
||||
$long = '43_local-update-truncate-with-a-very-long-name-indeed.test'
|
||||
$line = Format-WatchdogStatus 812 41 $long 'm=9012 v=14022 p=118 h=41230 d=13210'
|
||||
Assert-That ($line.Length -le 140) ('status description is {0} characters' -f $line.Length)
|
||||
Assert-That ($line -like 't=812s q=41s 43_local-update-truncate*') ('status leads with the wrong fields: {0}' -f $line)
|
||||
Assert-That ($line -like '*d=13210') 'the counters did not survive a long test name'
|
||||
# -match, not -like: '?' is a wildcard there, so q=0s would satisfy it too.
|
||||
Assert-That ((Format-WatchdogStatus 8 -1 'x' 'y') -match '^t=8s q=\?s x \| y$') 'an unknown staticness reads as a number'
|
||||
$clip = Format-WatchdogStatus 1 2 ('x' * 80) 'c'
|
||||
Assert-That ($clip -match '^t=1s q=2s x{46} \| c$') ('the in-flight name was not clipped to 46: {0}' -f $clip)
|
||||
$wide = Format-WatchdogStatus 1 2 ('x' * 300) ('y' * 300)
|
||||
Assert-That ($wide.Length -le 140) ('an oversized status was not clipped: {0}' -f $wide.Length)
|
||||
|
||||
$gone = Get-ProgressTail -Path ('no-such-progress-log-{0}.tmp' -f $PID)
|
||||
Assert-That (-not $gone.Ok) 'an unreadable log reads as one that was read'
|
||||
$f = New-Object System.IO.FileInfo([System.IO.Path]::GetTempFileName())
|
||||
try {
|
||||
[System.IO.File]::WriteAllText($f.FullName, "first`nRUN 42_probe.test at 7s`n")
|
||||
$tail = Get-ProgressTail -Path $f.FullName
|
||||
Assert-That ($tail.Ok) 'a readable log reads as unreadable'
|
||||
Assert-That ($tail.Line -eq 'RUN 42_probe.test at 7s') ('the tail is not the last line: {0}' -f $tail.Line)
|
||||
} finally { [System.IO.File]::Delete($f.FullName) }
|
||||
|
||||
Assert-That (-not (Send-WatchdogStatus 'success' 'self-test')) 'the self-test can reach the API'
|
||||
|
||||
if ($bad.Count -gt 0) {
|
||||
foreach ($b in $bad) { Write-Host ('self-test FAIL: {0}' -f $b) }
|
||||
exit 1
|
||||
}
|
||||
Write-Host 'watchdog self-test OK'
|
||||
exit 0
|
||||
}
|
||||
|
||||
if ($SelfTest) { Invoke-WatchdogSelfTest }
|
||||
|
||||
# --- one-shot: the driver's own verdict ---------------------------------------
|
||||
|
||||
# Windows PowerShell still defaults below TLS 1.2, which api.github.com refuses.
|
||||
try { [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12 } catch { }
|
||||
|
||||
if ($Post) {
|
||||
# One transient 5xx here is the difference between a resolved status and a
|
||||
# finished suite left reading `pending`.
|
||||
for ($try = 1; $try -le 3; $try++) {
|
||||
if (Send-WatchdogStatus $Post $Message) { break }
|
||||
if ($NoPost -or -not $script:Token) { break }
|
||||
Start-Sleep -Seconds (2 * $try)
|
||||
}
|
||||
Write-WatchdogLog ('final status {0}: {1}' -f $Post, $Message)
|
||||
exit 0
|
||||
}
|
||||
|
||||
# --- main loop ----------------------------------------------------------------
|
||||
|
||||
$sw = [System.Diagnostics.Stopwatch]::StartNew()
|
||||
$lastSig = ''
|
||||
$movedAt = 0
|
||||
# Negative, so the first tick posts: an early status is itself a datum.
|
||||
$postedAt = -$IntervalSeconds
|
||||
$backoff = 0
|
||||
$skip = 0
|
||||
|
||||
# Guarded like the rest; the launcher waits for this exact line.
|
||||
try { Write-Host 'watchdog ready' } catch { }
|
||||
Write-WatchdogLog ('watching {0} every {1}s' -f $ProgressLog, $IntervalSeconds)
|
||||
|
||||
while ($sw.Elapsed.TotalSeconds -lt $MaxSeconds) {
|
||||
# Measured, never accumulated: starvation is what makes a sleep overshoot.
|
||||
$now = [int]$sw.Elapsed.TotalSeconds
|
||||
try {
|
||||
$tail = Get-ProgressTail -Path $ProgressLog
|
||||
if ($tail.Ok -and $tail.Signature -ne $lastSig) {
|
||||
$lastSig = $tail.Signature
|
||||
$movedAt = $now
|
||||
}
|
||||
if ((Get-WatchdogAction $now $postedAt $IntervalSeconds) -eq 'post') {
|
||||
$postedAt = $now
|
||||
$static = -1
|
||||
if ($tail.Ok) { $static = $now - $movedAt }
|
||||
$desc = Format-WatchdogStatus $now $static $tail.Line (Get-WatchdogCounters)
|
||||
# Logged whatever the backoff decides: it throttles the API, not the
|
||||
# artifact, which is all a run whose token cannot post will leave.
|
||||
Write-WatchdogLog $desc
|
||||
if ($skip -gt 0) {
|
||||
$skip--
|
||||
} elseif (Send-WatchdogStatus 'pending' $desc) {
|
||||
$backoff = 0
|
||||
} else {
|
||||
$backoff = Get-NextBackoff $backoff
|
||||
$skip = $backoff
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
Write-WatchdogLog ('tick failed: {0}' -f $_.Exception.Message)
|
||||
}
|
||||
Start-Sleep -Seconds $PollSeconds
|
||||
}
|
||||
|
||||
Write-WatchdogLog ('stopping after {0}s' -f [int]$sw.Elapsed.TotalSeconds)
|
||||
@@ -10,6 +10,8 @@ directory). Each line is "<path> <mode>...", path "*" matching everything:
|
||||
empty open the data connection and send nothing
|
||||
norest answer REST with 500, so the client re-fetches from scratch
|
||||
nomdtm answer MDTM with 500, like a server predating RFC 3659
|
||||
|
||||
--require-pass answers USER with 331, the only way the client's PASS is sent.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
@@ -25,12 +27,13 @@ def reply(conn, text):
|
||||
|
||||
|
||||
class Session(threading.Thread):
|
||||
def __init__(self, conn, root, mode_file, log):
|
||||
def __init__(self, conn, root, mode_file, log, require_pass=False):
|
||||
threading.Thread.__init__(self, daemon=True)
|
||||
self.conn = conn
|
||||
self.root = root
|
||||
self.mode_file = mode_file
|
||||
self.log = log
|
||||
self.require_pass = require_pass
|
||||
self.pasv = None
|
||||
self.rest = 0
|
||||
self.path = "/" # named by SIZE/RETR; REST carries no path
|
||||
@@ -132,7 +135,9 @@ class Session(threading.Thread):
|
||||
|
||||
def dispatch(self, verb, arg):
|
||||
conn = self.conn
|
||||
if verb in ("USER", "PASS", "TYPE", "NOOP"):
|
||||
if verb == "USER" and self.require_pass:
|
||||
reply(conn, "331 password required") # the client sends PASS only on a 3xx
|
||||
elif verb in ("USER", "PASS", "TYPE", "NOOP"):
|
||||
reply(conn, "200 ok")
|
||||
elif verb == "SYST":
|
||||
reply(conn, "215 UNIX Type: L8")
|
||||
@@ -209,6 +214,7 @@ def main():
|
||||
ap.add_argument("--root", required=True)
|
||||
ap.add_argument("--mode-file")
|
||||
ap.add_argument("--log")
|
||||
ap.add_argument("--require-pass", action="store_true")
|
||||
args = ap.parse_args()
|
||||
|
||||
logfp = open(args.log, "a", encoding="utf-8") if args.log else None
|
||||
@@ -231,7 +237,7 @@ def main():
|
||||
root = os.path.abspath(args.root)
|
||||
while True:
|
||||
conn, _ = srv.accept()
|
||||
Session(conn, root, args.mode_file, log).start()
|
||||
Session(conn, root, args.mode_file, log, args.require_pass).start()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -72,13 +72,13 @@ is_windows() {
|
||||
}
|
||||
|
||||
# On Windows MSYS can't signal a native python.exe, so kill_tree ends the whole
|
||||
# tree (a bare kill -9 leaves children). "|| true" throughout: callers run under
|
||||
# set -e and the reap makes wait return 143.
|
||||
# tree (a bare kill -9 leaves children). Bounded because this runs from an EXIT
|
||||
# trap, where a survivor would turn a passing test into a harness timeout.
|
||||
stop_server() {
|
||||
test -n "${1:-}" || return 0
|
||||
kill "$1" 2>/dev/null || true
|
||||
if is_windows; then kill_tree "$1"; fi
|
||||
wait "$1" 2>/dev/null || true
|
||||
reap_bounded "$1" || true
|
||||
return 0
|
||||
}
|
||||
|
||||
|
||||
@@ -260,6 +260,6 @@ TESTS += 221_local-ftp-ctrlchars.test
|
||||
TESTS += 218_crash-nopie-frames.test
|
||||
TESTS += 222_pkgconfig-consumer.test
|
||||
TESTS += 228_icon-small-flat.test
|
||||
TESTS += 226_watchdog-native.test
|
||||
TESTS += 224_engine-ftp-cmdlen.test
|
||||
TESTS += 230_local-ftp-userpass.test
|
||||
TESTS += 225_install-manifest.test
|
||||
|
||||
Reference in New Issue
Block a user