Compare commits

..

8 Commits

Author SHA1 Message Date
Xavier Roche
3563d4fbe0 Merge origin/master (#763 landed)
Signed-off-by: Xavier Roche <roche@httrack.com>

# Conflicts:
#	src/htsback.c
#	tests/96_local-refetch-keep.test
2026-07-27 09:48:55 +02:00
Xavier Roche
45630f63b4 Merge fix/refetch-debris
Signed-off-by: Xavier Roche <roche@httrack.com>

# Conflicts:
#	tests/96_local-refetch-keep.test
2026-07-27 09:26:02 +02:00
Xavier Roche
2ffe8d5582 tests: assert the retry exhaustion, not the per-platform failure message
The message a cut connection produces depends on whether any bytes arrived, so
matching it would fail on a runner that sees none.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-27 09:25:38 +02:00
Xavier Roche
14cc7a13fc Keep the copy for every failure, not just the five retryable codes
A malformed status line, an oversized declared length or a mid-flight abort all
land on STATUSCODE_INVALID, outside the retryable set, and the purge still ate
the file. Invert the test: any failure keeps the copy except the codes that
mean the engine passed the resource over on purpose.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-27 09:23:16 +02:00
Xavier Roche
1a7effce49 --purge-old deleted a file whose re-fetch never got a response
A transfer that dies on the wire leaves the previously mirrored copy in place,
but back_finalize() returned without noting it, so the URL fell out of new.lst
and the end-of-update purge treated the file like a page that had vanished from
the site. Note the surviving copy, the way the incomplete-transfer branch above
already does, for the connection-level failures htsparse.c retries on. A
deliberate skip (too big, MIME-excluded, cancelled) keeps its current fate.

Closes #746

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-27 09:19:14 +02:00
Xavier Roche
6457828b72 tests: make the re-fetch assertions catch a crawl that never re-fetched
The new test passed with no second pass at all, so a regression that stopped
re-fetching would have looked green. Assert the failure the fixture provokes,
give stay.bin a fresh pass-2 body so a fix that stopped overwriting anything
fails, and compare reset.bin by checksum rather than by length.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-27 09:18:27 +02:00
Xavier Roche
f118cd86d3 tests: assert the failed re-fetch keeps its bytes on every platform
Test 93 filtered reset.bin out of its bucket lists because a connection killed
before the status line surfaced differently per platform. It no longer does, so
assert the resource like any other: unchanged, with the bytes pass 1 mirrored.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-27 09:05:56 +02:00
Xavier Roche
1373530f17 A failed re-fetch overwrote the mirrored file with the aborted read's debris
A transfer that dies before a complete response has no body, yet the save path
still consulted r.adr, which at that point holds whatever the aborted header
read left behind: raw status-line bytes, or an empty buffer that truncated the
file to zero on macOS. Require a successful transfer, as the empty-body half of
the condition already did.

Closes #748

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>
2026-07-27 08:59:43 +02:00
9 changed files with 58 additions and 249 deletions

View File

@@ -585,6 +585,19 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
return 0;
}
/* Did the fetch fail to produce a response, as opposed to the engine
deliberately passing the resource over? Only the latter may be purged. */
static hts_boolean back_transfer_failed(const int statuscode) {
switch (statuscode) {
case STATUSCODE_TOO_BIG:
case STATUSCODE_EXCLUDED:
case STATUSCODE_TEST_OK:
return HTS_FALSE;
default:
return statuscode <= 0 ? HTS_TRUE : HTS_FALSE;
}
}
/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
existing file moved the good copy to back->tmpfile before truncating url_sav.
commit keeps the new file and drops the backup; else restore it so an aborted
@@ -1031,6 +1044,14 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
/* Aborted, error, or not ready: url_sav (if written) is broken; restore the
previous copy from the backup. */
back_finalize_backup(opt, &back[p], HTS_FALSE);
/* Note the surviving copy, or the end-of-update purge drops what this run
never managed to replace (#746). */
if (!back[p].testmode && back_transfer_failed(back[p].r.statuscode) &&
back[p].url_sav[0] != '\0' && fexist_utf8(back[p].url_sav)) {
filenote(&opt->state.strc, back[p].url_sav, NULL);
file_notify(opt, back[p].url_adr, back[p].url_fil, back[p].url_sav, 0, 0,
back[p].r.notmodified);
}
return -1;
}

View File

@@ -57,9 +57,6 @@ Please visit our Website: http://www.httrack.com
/* opt->state.warc value meaning "open failed once, do not retry". */
#define WARC_DISABLED ((void *) ~(uintptr_t) 0)
/* Suffix of the in-progress archive when a previous one must survive it. */
#define WARC_TMP_SUFFIX ".tmp"
struct warc_writer {
FILE *f;
httrackp *opt; /* kept for close-time logging (warc_wacz_package) */
@@ -89,11 +86,6 @@ struct warc_writer {
char *base_path; /* resolved archive path minus .warc[.gz] suffix */
const char *base_ext; /* ".warc.gz" or ".warc" (static) */
char *arc_path; /* full single-file archive path (NULL under rotation) */
/* A re-run must not destroy an archive it cannot replace (#759). */
hts_boolean protect_prev; /* previous archive present: build in a temp */
hts_boolean opened; /* open completed; a failed one swaps nothing */
hts_boolean failed; /* a record or segment was lost: swap nothing */
uint64_t unbacked_revisits; /* revisits whose payload no file here holds */
char **page_lines; /* one JSON page line per 200 text/html response, owned */
size_t page_count;
size_t page_cap;
@@ -1137,17 +1129,13 @@ static int warc_emit(warc_writer *w, const char *type, const char *content_type,
never split a record, and never rotate a warcinfo (it opens a segment). */
if (w->max_size > 0 && w->seg_base != NULL && w->offset >= w->max_size &&
strcmp(type, "warcinfo") != 0) {
if (warc_rotate(w) != 0) {
w->failed = HTS_TRUE;
if (warc_rotate(w) != 0)
return -1;
}
}
/* F4: overflow-safe block length; http_hdr_len+sep is provably small. */
if (payload > (size_t) -1 - http_hdr_len - sep) {
w->failed = HTS_TRUE;
if (payload > (size_t) -1 - http_hdr_len - sep)
return -1;
}
block_len = http_hdr_len + sep + payload;
memset(&hdr, 0, sizeof(hdr));
@@ -1253,23 +1241,11 @@ static int warc_emit(warc_writer *w, const char *type, const char *content_type,
rc = 0;
done:
wbuf_free(&hdr);
if (rc != 0)
w->failed = HTS_TRUE; /* a truncated run must not replace a whole one */
return rc;
}
/* ---- segment rotation (--warc-max-size) ---- */
/* Path to open for the segment whose final path is `final`: that path itself,
or a sibling temp while a previous archive must survive until close. */
static const char *warc_open_path(warc_writer *w, const char *final, char *buf,
size_t bufsz) {
if (!w->protect_prev)
return final;
snprintf(buf, bufsz, "%s" WARC_TMP_SUFFIX, final);
return buf;
}
/* Emit the warcinfo that heads a segment; sets w->info_id for its records. */
static int warc_write_warcinfo_record(warc_writer *w) {
w->info_id[0] = '\0'; /* warcinfo itself carries no WARC-Warcinfo-ID */
@@ -1281,23 +1257,17 @@ static int warc_write_warcinfo_record(warc_writer *w) {
static int warc_rotate(warc_writer *w) {
char namebuf[HTS_URLMAXSIZE * 2];
char openbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
char catbuff[CATBUFF_SIZE];
const unsigned next = w->seg + 1;
FILE *f;
snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, next,
w->seg_ext);
/* Open before advancing: w->seg must only ever name a segment that exists,
or close-time packaging and swapping would work on a missing file. */
f = FOPEN(fconv(catbuff, sizeof(catbuff),
warc_open_path(w, namebuf, openbuf, sizeof(openbuf))),
"wb");
if (f == NULL)
return -1;
if (w->f != NULL)
if (w->f != NULL) {
fclose(w->f);
w->f = f;
w->seg = next;
w->f = NULL;
}
w->seg++;
snprintf(namebuf, sizeof(namebuf), "%s-%05u%s", w->seg_base, w->seg,
w->seg_ext);
w->f = FOPEN(fconv(catbuff, sizeof(catbuff), namebuf), "wb");
if (w->f == NULL)
return -1;
w->offset = 0;
if (w->cdx_on) {
freet(w->cur_seg);
@@ -1354,7 +1324,6 @@ void warc_adopt_rawspool(htsblk *r, const char *tmpfile_path) {
warc_writer *warc_open(httrackp *opt, const char *path) {
warc_writer *w;
char namebuf[HTS_URLMAXSIZE * 2];
char openbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
char catbuff[CATBUFF_SIZE];
wbuf info;
const char *robots;
@@ -1505,105 +1474,35 @@ warc_writer *warc_open(httrackp *opt, const char *path) {
path = namebuf;
}
if (w->max_size == 0 && (w->arc_path = strdupt(path)) == NULL) {
warc_close(w);
return NULL;
}
/* Set only once arc_path is recorded, so a half-built writer never swaps. */
w->protect_prev = fsize_utf8(path) > 0 ? HTS_TRUE : HTS_FALSE;
w->f = FOPEN(fconv(catbuff, sizeof(catbuff),
warc_open_path(w, path, openbuf, sizeof(openbuf))),
"wb");
w->f = FOPEN(fconv(catbuff, sizeof(catbuff), path), "wb");
if (w->f == NULL) {
warc_close(w);
return NULL;
}
if (w->cdx_on)
w->cur_seg = path_basename_dup(path);
if (w->wacz_on && w->max_size == 0)
w->arc_path = strdupt(path); /* single-file: package this exact path */
if (warc_write_warcinfo_record(w) != 0) {
warc_close(w);
return NULL;
}
w->opened = HTS_TRUE;
return w;
}
/* Final path of segment s (the run's only archive when rotation is off). */
static const char *warc_seg_path(warc_writer *w, unsigned s, char *buf,
size_t bufsz) {
if (w->max_size == 0)
return w->arc_path;
snprintf(buf, bufsz, "%s-%05u%s", w->seg_base, s, w->seg_ext);
return buf;
}
/* Swap this run's archive into place, unless it only holds revisits naming
bodies the previous archive still has and this one does not (#759).
HTS_FALSE: the previous archive was kept, so leave its .cdx and .wacz too. */
static hts_boolean warc_commit(warc_writer *w) {
char finalbuf[HTS_URLMAXSIZE * 2];
char tmpbuf[HTS_URLMAXSIZE * 2 + sizeof(WARC_TMP_SUFFIX)];
char catbuff[CATBUFF_SIZE];
const unsigned nseg = (w->max_size > 0) ? w->seg + 1 : 1;
hts_boolean swap;
unsigned s;
if (!w->protect_prev)
return HTS_TRUE; /* nothing was there to lose: written in place */
/* hts_rename_over unlinks its destination when the source is missing, so
every segment has to be on disk before the first rename. */
swap = w->opened && !w->failed && w->unbacked_revisits == 0;
for (s = 0; s < nseg && swap; s++) {
snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX,
warc_seg_path(w, s, finalbuf, sizeof(finalbuf)));
swap = fsize_utf8(tmpbuf) > 0;
}
if (swap) {
for (s = 0; s < nseg; s++) {
const char *final = warc_seg_path(w, s, finalbuf, sizeof(finalbuf));
snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX, final);
if (!hts_rename_over(tmpbuf, final)) {
hts_log_print(w->opt, LOG_ERROR | LOG_ERRNO,
"WARC: could not replace %s", final);
return HTS_FALSE;
}
}
return HTS_TRUE;
}
for (s = 0; s < nseg; s++) {
snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX,
warc_seg_path(w, s, finalbuf, sizeof(finalbuf)));
(void) UNLINK(fconv(catbuff, sizeof(catbuff), tmpbuf));
}
if (w->unbacked_revisits > 0)
hts_log_print(
w->opt, LOG_ERROR,
"WARC: this pass revisited %llu URL(s) without re-downloading them, so "
"its archive would name bodies no file holds; kept the previous %s "
"(re-run with -C0, or --warc-file with a name of its own)",
(unsigned long long) w->unbacked_revisits,
warc_seg_path(w, 0, finalbuf, sizeof(finalbuf)));
return HTS_FALSE;
}
void warc_close(warc_writer *w) {
size_t i;
if (w == NULL)
return;
warc_cdx_flush(w); /* sort + write <base>.cdx before tearing down */
if (w->f != NULL)
fclose(w->f);
w->f = NULL;
if (warc_commit(w)) {
warc_cdx_flush(w); /* sort + write <base>.cdx beside the archive */
#if HTS_USEOPENSSL
if (w->wacz_on) /* package once the segment(s) + .cdx are closed on disk */
warc_wacz_package(w);
if (w->wacz_on) /* package once the segment(s) + .cdx are closed on disk */
warc_wacz_package(w);
#endif
}
if (w->seen != NULL)
coucal_delete(&w->seen);
for (i = 0; i < w->cdx_count; i++)
@@ -1684,8 +1583,6 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
if (is_update_unchanged) {
is_revisit = 1;
profile = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
/* Served from cache: the payload sits in the previous archive, not here. */
w->unbacked_revisits++;
} else if (have_pdig && w->seen != NULL) {
void *prev = NULL;
if (coucal_read_pvoid(w->seen, pdig, &prev) && prev != NULL) {

View File

@@ -1,14 +1,11 @@
#!/bin/bash
#
# A --warc crawl writes a standards-conformant WARC/1.1 archive, and a re-crawl
# of an unchanged (all-304) site emits revisit records. The stdlib validator (no
# warcio) is the real gate: it byte-compares the fresh page.html response body
# against what the server served, asserts the encoding headers were stripped and
# the payload digest matches, then checks the second pass turned the unchanged
# assets into revisits (no full response).
#
# That second pass archives under a name of its own, which is also the recovery
# the engine points at when it refuses to strand a previous archive (#759).
# A --warc crawl writes a standards-conformant WARC/1.1 archive, and an
# --update re-crawl of an unchanged (all-304) site emits revisit records.
# The stdlib validator (no warcio) is the real gate: it byte-compares the
# fresh page.html response body against what the server served, asserts the
# encoding headers were stripped and the payload digest matches, then checks
# the update pass turned the unchanged assets into revisits (no full response).
set -eu
@@ -17,10 +14,8 @@ set -eu
# page.html body served by tests/local-server.py (route_mini304_page).
export WARC_VALIDATE_BODY="page.html=3c68746d6c3e3c626f64793e74696e7920636163686561626c6520706167653c2f626f64793e3c2f68746d6c3e0a"
export WARC_VALIDATE_NORESP="index.html page.html"
export WARC_VALIDATE_UPDATE="warc-update.warc.gz"
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --warc-validate \
--rerun-args '--warc-file warc-update' \
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun --warc-validate \
--log-found 'no files updated' \
--found 'mini304/index.html' --found 'mini304/page.html' \
httrack 'BASEURL/mini304/index.html' --warc-file warc-out

View File

@@ -6,10 +6,9 @@
# the datapackage-digest chain (plus py-wacz/pywb when importable). The crawl
# skips cleanly on a build without OpenSSL (no conformant SHA-256 -> no package).
#
# The cacheless (-C0) second pass repackages over the .wacz the first one left:
# the clobber, not the happy rename, is what breaks when a platform refuses to
# rename onto an existing file (#726). A cache-served pass keeps the previous
# package instead, which is 97_local-warc-update-keep.
# The cacheless second pass repackages over the .wacz the first one left: the
# clobber, not the happy rename, is what breaks when a platform refuses to
# rename onto an existing file (#726).
set -eu

View File

@@ -1,19 +1,16 @@
#!/bin/bash
#
# A re-fetch cut mid-header must leave the mirrored file alone (#748): unfixed,
# the aborted read's leftover buffer lands on it. err.bin is the control, an
# HTTP 500 on the same resource already keeping its copy; stay.bin answers
# normally with a new body, so a fix that stopped overwriting anything fails.
#
# Purging is off: an update purge deletes both files for an unrelated reason
# (#746), which would hide what this asserts.
# A re-fetch cut mid-header must leave the mirrored file alone (#748) and keep
# it out of the update purge (#746). err.bin is the control, an HTTP 500 on the
# same resource already surviving both; stay.bin answers normally with a new
# body, so a fix that stopped overwriting anything fails.
set -euo pipefail
: "${top_srcdir:=..}"
bash "$top_srcdir/tests/local-crawl.sh" \
--rerun-args '--update --purge-old=0' \
--rerun-args '--update' \
--log-found 'after 2 retries at link .*keep/page\.html' \
--log-found 'after 2 retries at link .*keep/data\.bin' \
--file-matches 'keep/page.html' 'KEEP-PAGE-V1' \

View File

@@ -1,15 +0,0 @@
#!/bin/bash
#
# A cache-served second pass fetches no bodies, so it must leave the WARC, CDXJ
# and WACZ of the first pass untouched (#759). The kept package is re-validated
# to prove it still replays, not just that a same-named file is still there.
# The one Error: line is the keep notice; a second means the pass also failed.
set -eu
: "${top_srcdir:=..}"
bash "$top_srcdir/tests/local-crawl.sh" --errors 1 --archive-kept-on-rerun \
--rerun --wacz-validate --log-found 'WARC: this pass revisited' \
--found 'mini304/index.html' --found 'mini304/page.html' \
httrack 'BASEURL/mini304/index.html' --warc-file warc-out --wacz

View File

@@ -1,24 +0,0 @@
#!/bin/bash
#
# The same keep/replace rule under --warc-max-size, where the archive is several
# segments: a cache-served pass must leave every one alone, a cacheless pass must
# swap every one in. Half a swap is the corruption one rename loop can produce,
# and only a rotated archive exposes it (#759). --archive-min-files pins the two
# segments plus the .cdx, so a shrinking crawl cannot quietly stop rotating.
set -eu
: "${top_srcdir:=..}"
# The one Error: line is the keep notice; a second means the pass also failed.
bash "$top_srcdir/tests/local-crawl.sh" --errors 1 --archive-kept-on-rerun \
--archive-min-files 3 --rerun --log-found 'WARC: this pass revisited' \
--found 'mini304/index.html' --found 'mini304/page.html' \
httrack 'BASEURL/mini304/index.html' --warc-file warc-out --warc-cdx \
--warc-max-size 600
bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --archive-replaced-on-rerun \
--archive-min-files 3 --rerun-args '-C0' \
--found 'mini304/index.html' --found 'mini304/page.html' \
httrack 'BASEURL/mini304/index.html' --warc-file warc-out --warc-cdx \
--warc-max-size 600

View File

@@ -198,8 +198,6 @@ TESTS = \
93_local-changes.test \
94_local-single-file.test \
95_local-sitemap.test \
96_local-refetch-keep.test \
97_local-warc-update-keep.test \
98_local-warc-segments.test
96_local-refetch-keep.test
CLEANFILES = check-network_sh.cache

View File

@@ -34,12 +34,6 @@
# httrack via --cookies-file, to exercise preloaded cookies.
# --rerun-dead re-runs with the server stopped: the no-data rollback must
# restore the previous hts-cache generation byte-identical.
# --archive-kept-on-rerun: the second pass must leave the first pass's
# .warc[.gz]/.cdx/.wacz byte-identical, having no bodies to replace them (#759).
# --archive-replaced-on-rerun is its mirror: every one of them must have been
# rewritten. Both also require no *.tmp left behind, and take an optional
# --archive-min-files N guarding against a scenario that silently stopped
# producing the segments it means to check.
set -u
@@ -60,9 +54,6 @@ outdir_intl=
rerun=
rerun_args=
rerun_dead=
archive_kept=
archive_replaced=
archive_min_files=0
tmpdir=
serverpid=
crawlpid=
@@ -127,13 +118,6 @@ while test "$pos" -lt "$nargs"; do
--debug) verbose=1 ;;
--rerun) rerun=1 ;; # run httrack a second time (update pass) before auditing
--rerun-dead) rerun_dead=1 ;; # re-run with the server stopped (cache rollback)
# the second pass must leave the first pass's archive files untouched
--archive-kept-on-rerun) archive_kept=1 ;;
--archive-replaced-on-rerun) archive_replaced=1 ;; # ...or rewrite all of them
--archive-min-files)
pos=$((pos + 1))
archive_min_files="${args[$pos]}"
;;
# validate the produced .warc.gz (see the validation block near the end)
--warc-validate) warc_validate=1 ;;
# validate the produced .wacz package (stdlib, plus py-wacz/pywb if present)
@@ -290,26 +274,10 @@ if test -n "$warc_validate"; then
test -z "$w1" || cp "$w1" "${tmpdir}/warc-pass1.gz"
fi
# Snapshot the archive files the second pass must keep (or must replace).
declare -a kept_files=()
if test -n "${archive_kept}${archive_replaced}"; then
while read -r f; do
test -n "$f" || continue
cp "$f" "${tmpdir}/kept-${#kept_files[@]}" || die "could not snapshot $f"
kept_files+=("$f")
done < <(find "$mirrorroot" -maxdepth 2 \
\( -name '*.warc.gz' -o -name '*.warc' -o -name '*.cdx' -o -name '*.wacz' \) \
2>/dev/null | sort)
test "${#kept_files[@]}" -gt 0 ||
die "the first pass produced no archive to compare against"
test "${#kept_files[@]}" -ge "$archive_min_files" ||
die "only ${#kept_files[@]} archive file(s), wanted $archive_min_files: ${kept_files[*]}"
fi
# Poison the first-pass .wacz when a second pass follows: repackaging moves the
# new archive over it, so the marker must be gone afterwards (#726). Poisoning
# beats comparing the two packages, which can come out byte-identical.
if test -z "$archive_kept" && test -n "$wacz_validate" && test -n "${rerun}${rerun_args}"; then
if test -n "$wacz_validate" && test -n "${rerun}${rerun_args}"; then
wacz_poisoned=$(find "$mirrorroot" -maxdepth 2 -name '*.wacz' 2>/dev/null | sort | tail -n1)
test -z "$wacz_poisoned" || echo "$wacz_poison" >"$wacz_poisoned"
fi
@@ -358,29 +326,6 @@ if test -n "$rerun_args"; then
result "OK (second pass)"
fi
# --- optional: did the second pass keep, or replace, the whole archive? ------
if test "${#kept_files[@]}" -gt 0; then
i=0
for f in "${kept_files[@]}"; do
if test -n "$archive_kept"; then
info "checking the second pass kept $(basename "$f")"
cmp -s "${tmpdir}/kept-${i}" "$f" ||
die "$(basename "$f") was rewritten: the previous archive was destroyed"
else
info "checking the second pass replaced $(basename "$f")"
cmp -s "${tmpdir}/kept-${i}" "$f" &&
die "$(basename "$f") still holds the first pass's bytes"
fi
result "OK"
i=$((i + 1))
done
# A leftover in-progress file is as bad: the next pass would silently eat it.
info "checking no in-progress archive was left behind"
leftover=$(find "$mirrorroot" -maxdepth 2 \( -name '*.warc.gz.tmp' -o -name '*.warc.tmp' \) 2>/dev/null | head -n1)
test -z "$leftover" || die "left behind $leftover"
result "OK"
fi
# --- optional dead pass: server stopped, the cache must survive the rollback --
if test -n "$rerun_dead"; then
zip="${out}/hts-cache/new.zip"
@@ -452,18 +397,14 @@ if test -n "$warc_validate"; then
die "fresh WARC validation failed"
result "OK"
# After an update pass the unchanged assets must be revisits, in an archive
# of the pass's own (WARC_VALIDATE_UPDATE): a revisit-only pass never
# replaces the archive holding the bodies it would strand (#759).
if test -n "${WARC_VALIDATE_UPDATE:-}"; then
upd=$(find "$mirrorroot" -maxdepth 2 -name "$WARC_VALIDATE_UPDATE" 2>/dev/null | head -n1)
test -n "$upd" || die "no $WARC_VALIDATE_UPDATE produced under $mirrorroot"
# Final file: after an update pass the unchanged assets must be revisits.
if test -n "$rerun"; then
declare -a revargs=(--expect-revisit)
for sub in ${WARC_VALIDATE_NORESP:-}; do
revargs+=(--no-response-for "$sub")
done
info "validating update WARC (revisits)"
"$python" "$validator" "$(nativepath "$upd")" "${revargs[@]}" >&2 ||
"$python" "$validator" "$(nativepath "$warc")" "${revargs[@]}" >&2 ||
die "update WARC validation failed"
result "OK"
fi