Compare commits

..

1 Commits

Author SHA1 Message Date
Xavier Roche
d99d5db33e Signed-shift UB in zip-repair local-header read
unzRepair parses the local file header of a damaged cache zip during
repair. READ_32 combined two int-typed READ_16 halves as
READ_16(adr) | (READ_16((adr)+2) << 16); when the high half has bit 15
set (>= 0x8000), shifting it left by 16 exceeds INT_MAX, which is signed
overflow. UBSan aborts on the CRC/size fields of a header whose high
16-bit word has that bit set. Repair runs on hostile input (a corrupt or
foreign new.zip). Cast the halves to uLong before the shift, matching the
sibling minizip readers in unzip.c and zip.c.

Closes #639

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-18 08:18:55 +02:00
13 changed files with 42 additions and 402 deletions

View File

@@ -188,7 +188,7 @@ jobs:
# Every gate here exits 77, so an all-skipped suite would report green having
# tested nothing: pin the skips, and floor the passes in case the glob empties.
expected_skips=" 48_local-crange-memresume.test 71_local-crange-repaircache.test" # pending #581
expected_skips=" 48_local-crange-memresume.test" # pending #581
[ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
[ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
[ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }

View File

@@ -373,8 +373,7 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
return NULL;
}
HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
const char *charset) {
char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
const UINT cp = hts_getCodepage(charset);
return hts_convertStringCPToUTF8(s, size, cp);
@@ -555,8 +554,7 @@ static char *hts_convertStringCharset(const char *s, size_t size,
return NULL;
}
HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
const char *charset) {
char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
/* Empty string ? */
if (size == 0) {
return strdup("");

View File

@@ -51,8 +51,8 @@ typedef unsigned int hts_UCS4;
* Convert the string "s" from charset "charset" to UTF-8.
* Return NULL upon error.
**/
HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
const char *charset);
extern char *hts_convertStringToUTF8(const char *s, size_t size,
const char *charset);
/**
* Convert the string "s" from UTF-8 to charset "charset".

View File

@@ -2368,28 +2368,30 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
opt->errlog = stderr;
} else if (httrack_logmode >= 2) {
// deux fichiers log
// path_log holds UTF-8 bytes (argv is UTF-8): the ANSI file calls would
// read them as the codepage and drop the logs into a mangled twin (#630).
structcheck_utf8(StringBuff(opt->path_log));
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log), "hts-log.txt")))
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log), "hts-log.txt"));
if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log), "hts-err.txt")))
UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log), "hts-err.txt"));
structcheck(StringBuff(opt->path_log));
if (fexist
(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt")))
remove(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"));
if (fexist
(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt")))
remove(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"));
/* Check FS directory structure created */
structcheck_utf8(StringBuff(opt->path_log));
structcheck(StringBuff(opt->path_log));
opt->log = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log), "hts-log.txt"),
"w");
opt->log =
fopen(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"),
"w");
if (httrack_logmode == 2)
opt->errlog = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
StringBuff(opt->path_log), "hts-err.txt"),
"w");
opt->errlog =
fopen(fconcat
(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"),
"w");
else
opt->errlog = opt->log;
if (opt->log == NULL) {

View File

@@ -1532,30 +1532,8 @@ int url_savename(lien_adrfilsave *const afs,
// last segment
wsave[j++] = '/';
#define MAX_UTF8_SEQ_CHARS 4
{
// #623: the ".delayed" placeholder marker sits at the tail; cutting
// through it drops IS_DELAYED_EXT, so the file is never renamed to its
// final name. Reserve the trailing ".<id>.delayed" across the cut.
size_t markStart = wsaveLen;
if (IS_DELAYED_EXT(afs->save)) {
const size_t extDot = wsaveLen - strlen("." DELAYED_EXT);
size_t p = extDot; /* walk back over a dot-separated ".<hexid>" tag */
while (p > lastSeg && ((wsave[p - 1] >= '0' && wsave[p - 1] <= '9') ||
(wsave[p - 1] >= 'a' && wsave[p - 1] <= 'f')))
p--;
// keep the tag only if truly ".<hexid>.delayed"; else the bare marker
// (a wholly-hex base, e.g. a hashed #133 name, must not be absorbed)
markStart = (p > lastSeg && p < extDot && wsave[p - 1] == '.')
? p - 1
: extDot;
}
// head, bounded so the marker still fits, then the marker itself
for (i = lastSeg; i < markStart && j + (wsaveLen - markStart) < maxLen;
i++)
wsave[j++] = wsave[i];
for (i = markStart; i < wsaveLen && j < maxLen; i++)
wsave[j++] = wsave[i];
for(i = lastSeg; wsave[i] != '\0' && j < maxLen; i++) {
wsave[j++] = wsave[i];
}
// terminating \0
wsave[j++] = '\0';

View File

@@ -40,7 +40,6 @@ Please visit our Website: http://www.httrack.com
#include "htsnet.h"
#include "htslib.h"
#include "htscharset.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -750,14 +749,7 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
RUN THE SERVER
*/
if (strcmp((char *) adrcd, "start") == 0) {
/* POST body is in the form's charset, not the
UTF-8 argv the engine now assumes (#629). */
char *const cmdl = (char *) adr + p;
char *cmdlUtf8 = hts_convertStringToUTF8(
cmdl, strlen(cmdl), LANGSEL("LANGUAGE_CHARSET"));
webhttrack_main(cmdlUtf8 != NULL ? cmdlUtf8 : cmdl);
freet(cmdlUtf8);
webhttrack_main((char *) adr + p);
} else {
commandRunning = 0;
commandEnd = 1;

View File

@@ -65,7 +65,6 @@ Please visit our Website: http://www.httrack.com
#include "htsserver.h"
#include "htsurlport.h"
#include "htsweb.h"
#include "htscharset.h"
#if USE_BEGINTHREAD==0
#error fatal: no threads support
@@ -157,7 +156,6 @@ int main(int argc, char *argv[]) {
printf("Initializing the server..\n");
#ifdef _WIN32
hts_argv_utf8(&argc, &argv);
{
WORD wVersionRequested; // requested version WinSock API
WSADATA wsadata; // Windows Sockets API data

View File

@@ -1,72 +0,0 @@
#!/bin/bash
#
set -euo pipefail
# #623: url_savename enforces the 236-char path ceiling by cutting the tail of
# the last segment, where the mandatory ".delayed" placeholder marker lives. A
# cut marker fails IS_DELAYED_EXT, so back_delayed_rename never renames the file
# to its final name and the download is lost. The marker must survive the cut.
# statuscode=302 status=-1 = a redirect answer still downloading: no type is
# resolved, so the name gets a ".<id>.delayed" placeholder (see 01_engine-savename).
CEIL=236
httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
scratch=$(mktemp -d)
trap 'rm -rf "$scratch"' EXIT
cd "$scratch"
run() {
"$httrack_bin" -O /dev/null -#test=savename "$@" | sed -n 's/^savename: //p'
}
# A deep path ending in a long segment that overruns the ceiling.
deep="/d1$(printf 'a%.0s' {1..40})/d2$(printf 'b%.0s' {1..40})"
deep="$deep/d3$(printf 'c%.0s' {1..40})/d4$(printf 'd%.0s' {1..40})"
long="$deep/$(printf 'z%.0s' {1..90})"
out="$(run "$long" text/html statuscode=302 status=-1)"
case "$out" in
*.delayed) ;;
*)
echo "FAIL: delayed marker cut by truncation: '$out'"
exit 1
;;
esac
test "${#out}" -le "$CEIL" || {
echo "FAIL: truncated name ${#out} > $CEIL ceiling: '$out'"
exit 1
}
# #133-style hashed name: an all-hex last segment must keep the marker too (the
# ".<id>." tag is not mistaken for part of the hash).
hexseg=$(printf 'a1b2c3d4e5f60718%.0s' {1..8})
hexpath="/d1$(printf 'x%.0s' {1..30})/d2$(printf 'y%.0s' {1..30})/$hexseg"
out="$(run "$hexpath" text/html statuscode=302 status=-1)"
case "$out" in
*.delayed) ;;
*)
echo "FAIL: hashed-name marker cut by truncation: '$out'"
exit 1
;;
esac
test "${#out}" -le "$CEIL" || {
echo "FAIL: hashed name ${#out} > $CEIL ceiling: '$out'"
exit 1
}
# A non-delayed name of the same shape still truncates, with no marker to keep.
out="$(run "$long.html" text/html)"
test "${#out}" -le "$CEIL" || {
echo "FAIL: non-delayed name ${#out} > $CEIL ceiling: '$out'"
exit 1
}
case "$out" in
*.delayed)
echo "FAIL: non-delayed name grew a .delayed marker: '$out'"
exit 1
;;
esac
echo "delayed-truncate OK"

View File

@@ -1,90 +0,0 @@
#!/bin/bash
#
# webhttrack's POST body arrives in the form's declared charset, not UTF-8, yet
# the engine now assumes UTF-8 argv. A non-ASCII -O output dir submitted through
# the web UI must land under the UTF-8 directory, not an ISO-8859-1 twin (#629).
# Drives the real htsserver over HTTP; the default (English) form is served
# ISO-8859-1, so 'café' travels as the single byte 0xE9. The crawl target is a
# dead port: only the -O directory name is under test, and htsserver creates it
# from the decoded path before any fetch.
set -euo pipefail
testdir=$(cd "$(dirname "$0")" && pwd)
distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
distdir=$(cd "${distdir}" && pwd)
# shellcheck source=tests/testlib.sh
. "${testdir}/testlib.sh"
fail() {
echo "FAIL: $*" >&2
exit 1
}
command -v htsserver >/dev/null || fail "no htsserver in PATH"
python=$(find_python) || {
echo "python3 not found; skipping" >&2
exit 77
}
work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_charset.XXXXXX") || fail "no tmpdir"
srvlog=$(mktemp)
srv=
cleanup() {
# htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
rm -rf "${work}" "${srvlog}"
}
trap cleanup EXIT HUP INT QUIT PIPE TERM
# webhttrack server on a pre-picked port; an isolated HOME keeps a stray
# ~/.httrack.ini out of it.
sport=$("${python}" -c 'import socket
s = socket.socket()
s.bind(("127.0.0.1", 0))
print(s.getsockname()[1])
s.close()')
(
trap '' TERM TTOU
export HOME="${work}"
exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
) &
srv=$!
for _ in $(seq 1 40); do
url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break
kill -0 "${srv}" 2>/dev/null || break
sleep 0.25
done
test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
# Post a "start" whose -O dir is 'café' in the form's ISO-8859-1 charset.
"${python}" - "${url}" "${work}" <<'PY'
import sys, urllib.parse, urllib.request
url, work = sys.argv[1], sys.argv[2]
outdir = work + "/caf\xe9" # 'café' as the single byte the browser would send
# Port 1 refuses at once: only the decoded -O dir matters, not the fetch.
cmd = "httrack --quiet --robots=0 http://127.0.0.1:1/x.html -O " + outdir
fields = [("path", work), ("projname", "proj"), ("command_do", "start"),
("winprofile", "x"), ("command", cmd)]
body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="", encoding="latin-1"))
for k, v in fields)
req = urllib.request.Request(url + "step4.html", data=body.encode("latin-1"),
method="POST")
urllib.request.urlopen(req, timeout=20).read()
PY
# The crawl runs inside htsserver; wait for the -O dir, then check it chose the
# UTF-8 name and not the ISO-8859-1 twin.
utf8dir="${work}/café"
latin1dir="${work}/caf"$'\xe9'
for _ in $(seq 1 80); do
test -e "${utf8dir}" && break
kill -0 "${srv}" 2>/dev/null || break
sleep 0.25
done
test -e "${utf8dir}" || fail "-O dir never landed as UTF-8 café/: $(find "${work}" -maxdepth 2 2>/dev/null)"
test ! -e "${latin1dir}" || fail "-O dir created as the ISO-8859-1 twin instead"
echo "PASS"

View File

@@ -1,15 +0,0 @@
#!/bin/bash
#
# A single non-ASCII -O sets both path_html and path_log to "café" (#630). The
# logs (hts-log.txt/hts-err.txt) must land there, not in an ANSI-mangled twin:
# on Windows path_log holds UTF-8 bytes the raw file calls read as the codepage.
# The --errors/--files/--log-found audits all grep logroot=café/hts-log.txt, so
# a log written to the twin fails them. POSIX has no twin, so this bites on the
# Windows CI leg (like test 64). The cache twin is a separate, larger fix.
: "${top_srcdir:=..}"
bash "$top_srcdir/tests/local-crawl.sh" --outdir-intl 'café' --errors 0 --files 5 \
--found 'simple/basic.html' \
--log-found 'mirror complete in' \
httrack 'BASEURL/simple/basic.html'

View File

@@ -1,134 +0,0 @@
#!/bin/bash
# #581: an interrupted mirror can leave a truncated new.zip that pass 2 must
# repair before resuming. The repaired-cache resume then hits the same hostile
# 206 as test 48, so restart-whole must still drop the partial and refetch the
# whole file. Pass 1 leaves a partial + temp-ref; we truncate new.zip past its
# last local entry (dropping the central directory) so unzOpen fails and the
# repair path runs; pass 2 resumes, rejects the range, and refetches whole.
set -u
: "${top_srcdir:=..}"
testdir=$(cd "$(dirname "$0")" && pwd)
# shellcheck source=tests/testlib.sh
. "${testdir}/testlib.sh"
server=$(nativepath "${testdir}/local-server.py")
root=$(nativepath "${testdir}/server-root")
python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
# On Windows the pass-1 interrupt is a hard kill (MSYS can't signal a native
# exe) and the restart-whole path fails on the repaired cache (#581) -- the very
# bug this exercises; skip until the engine fix lands.
if is_windows; then
echo "Windows: restart-whole fails on a repaired cache (#581), skipping"
exit 77
fi
tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangerep.XXXXXX") || exit 1
serverpid=
crawlpid=
cleanup() {
test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
stop_server "$serverpid"
rm -rf "$tmpdir"
}
trap cleanup EXIT HUP INT QUIT PIPE TERM
serverlog="${tmpdir}/server.log"
: >"$serverlog"
"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
serverpid=$!
port=
for _ in $(seq 1 50); do
line=$(head -n1 "$serverlog" 2>/dev/null)
if test "${line%% *}" == "PORT"; then
port="${line#PORT }"
break
fi
kill -0 "$serverpid" 2>/dev/null || {
echo "server exited early: $(cat "$serverlog")"
exit 1
}
sleep 0.1
done
test -n "$port" || {
echo "could not discover server port"
exit 1
}
base="http://127.0.0.1:${port}"
which httrack >/dev/null || {
echo "could not find httrack"
exit 1
}
out="${tmpdir}/crawl"
mkdir "$out"
common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
refdir="${out}/hts-cache/ref"
newzip="${out}/hts-cache/new.zip"
# --- pass 1: crawl, interrupt once the blob download is underway -------------
printf '[pass 1: interrupt mid-download] ..\t'
httrack "${common[@]}" "${base}/crange206mem/index.html" >"${tmpdir}/log1" 2>&1 &
crawlpid=$!
for _ in $(seq 1 300); do
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
kill -0 "$crawlpid" 2>/dev/null || break
sleep 0.1
done
sleep 0.3
kill -TERM "$crawlpid" 2>/dev/null
wait "$crawlpid" 2>/dev/null
crawlpid=
test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
exit 1
}
echo "OK (temp-ref present)"
# --- damage the cache: drop the central directory a hard kill never wrote ----
printf '[damage new.zip -> forces repair] ..\t'
"$python" - "$newzip" <<'PY' || exit 1
import os, sys
path = sys.argv[1]
data = open(path, "rb").read()
cut = data.find(b"PK\x01\x02") # first central-directory header
if cut <= 0:
sys.exit("no central directory to drop in %s" % path)
os.truncate(path, cut)
PY
echo "OK"
# --- pass 2: --continue -> repair -> resume -> hostile 206 -> refetch whole ---
printf '[pass 2: repair, reject range, refetch] ..\t'
rc=0
httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1 || rc=$?
test "$rc" -eq 0 || {
echo "FAIL: httrack exited $rc"
cat "${tmpdir}/log2" >&2
exit 1
}
echo "OK (terminated)"
# The repair path must actually have run, else this is just a copy of test 48.
printf '[cache repair fired] ..\t'
grep -q 'damaged cache' "${out}/hts-log.txt" 2>/dev/null || {
echo "FAIL: repair path did not run; damage was ineffective"
exit 1
}
echo "OK"
blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
printf '[file recovered whole] ..\t'
test -s "$blob" || {
echo "FAIL: blob.bin missing after the repaired-cache resume"
exit 1
}
got=$(wc -c <"$blob")
test "$got" -eq "$full" || {
echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
exit 1
}
echo "OK (${got} bytes)"

View File

@@ -148,10 +148,6 @@ TESTS = \
63_webhttrack-home.test \
64_local-intl-outdir.test \
65_port-siblings.test \
66_engine-port80-strip.test \
67_engine-delayed-truncate.test \
68_webhttrack-outdir-charset.test \
69_local-intl-logdir.test \
71_local-crange-repaircache.test
66_engine-port80-strip.test
CLEANFILES = check-network_sh.cache

View File

@@ -48,7 +48,6 @@ key="${testdir}/server.key"
tls=
verbose=
html_subdir=
outdir_intl=
rerun=
rerun_args=
rerun_dead=
@@ -141,13 +140,6 @@ while test "$pos" -lt "$nargs"; do
pos=$((pos + 1))
html_subdir="${args[$pos]}"
;;
--outdir-intl)
# Single non-ASCII -O "$out/NAME": path_html AND path_log are NAME, so
# the logs (and the harness reads of them) go through the non-ASCII path
# (#630). Distinct from --html-subdir, which keeps path_log ASCII.
pos=$((pos + 1))
outdir_intl="${args[$pos]}"
;;
--errors | --errors-content | --files)
audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
pos=$((pos + 1))
@@ -224,18 +216,13 @@ test -n "$ver" || die "could not run httrack"
out="${tmpdir}/crawl"
mkdir "$out" || die "could not create $out"
# path_html holds the mirror + index; path_log holds hts-cache/hts-log.txt.
# Default: both are "$out". --html-subdir moves path_html to "$out/NAME" while
# path_log (logroot) stays "$out"; --outdir-intl moves both to "$out/NAME".
# Default: both are "$out". With --html-subdir, path_html becomes "$out/NAME"
# (the mirror root the audits inspect) while path_log stays "$out".
mirrorroot="$out"
logroot="$out"
odir="$out"
if test -n "$html_subdir"; then
mirrorroot="${out}/${html_subdir}"
odir="${mirrorroot},${out}"
elif test -n "$outdir_intl"; then
mirrorroot="${out}/${outdir_intl}"
logroot="$mirrorroot"
odir="$mirrorroot"
fi
# Localhost is fast; disable the rate/bandwidth safety limits but keep a
# max-time backstop so a hang cannot wedge the suite.
@@ -254,7 +241,7 @@ test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || {
exit 1
}
result "OK"
grep -iE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt" >&2
grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
# --- optional second pass: re-mirror into the same dir (cache/update path) ----
if test -n "$rerun"; then
@@ -273,7 +260,7 @@ if test -n "$rerun"; then
# The update summary reports "files updated"; a fresh crawl never does. Assert
# it so a regression that bypasses the cache (re-crawls fresh) can't pass.
info "checking update used the cache"
if grep -aqE "mirror complete in .*files updated" "${logroot}/hts-log.txt"; then
if grep -aqE "mirror complete in .*files updated" "${out}/hts-log.txt"; then
result "OK"
else
result "update pass did not report cache activity"
@@ -305,7 +292,7 @@ if test -n "$rerun_dead"; then
zip="${out}/hts-cache/new.zip"
test -s "$zip" || die "no cache was written by the first pass"
cp "$zip" "${tmpdir}/cache-before.zip"
cp "${logroot}/hts-log.txt" "${tmpdir}/log-before.txt"
cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
stop_server "$serverpid"
serverpid=
info "re-running httrack against the stopped server"
@@ -318,7 +305,7 @@ if test -n "$rerun_dead"; then
# The dead pass must have gone through the no-data rollback, not bailed out
# before the mirror loop (which would leave the cache trivially untouched).
info "checking the dead pass hit the rollback"
if grep -aq "No data seems to have been transferred" "${logroot}/hts-log.txt"; then
if grep -aq "No data seems to have been transferred" "${out}/hts-log.txt"; then
result "OK"
else
result "rollback notice not found in hts-log.txt"
@@ -333,7 +320,7 @@ if test -n "$rerun_dead"; then
exit 1
fi
# Audits below describe the healthy crawl, not the dead pass.
cp "${tmpdir}/log-before.txt" "${logroot}/hts-log.txt"
cp "${tmpdir}/log-before.txt" "${out}/hts-log.txt"
fi
# --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
@@ -363,19 +350,19 @@ while test "$i" -lt "${#audit[@]}"; do
--errors)
i=$((i + 1))
assert_equals "checking errors" "${audit[$i]}" \
"$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")"
"$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
;;
--errors-content)
i=$((i + 1))
total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")
total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
# transient network failures (statuscode -2..-6) flake on busy loopback;
# the code parens are followed by " at link" or " after N retries at link"
transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${logroot}/hts-log.txt" || true)
transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
;;
--files)
i=$((i + 1))
nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${logroot}/hts-log.txt" |
nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
assert_equals "checking files" "${audit[$i]}" "$nFiles"
;;
@@ -406,7 +393,7 @@ while test "$i" -lt "${#audit[@]}"; do
--log-found)
i=$((i + 1))
info "checking log matches ${audit[$i]}"
if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then result "OK"; else
if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then result "OK"; else
result "not in log"
exit 1
fi
@@ -414,7 +401,7 @@ while test "$i" -lt "${#audit[@]}"; do
--log-not-found)
i=$((i + 1))
info "checking log lacks ${audit[$i]}"
if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then
if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then
result "present in log"
exit 1
else result "OK"; fi