Merge branch 'master' into fix/proxytrack-ndx-fields2-825

Fix the TESTS-tail union merge: restore numeric order and the
dropped continuation backslash.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -79,7 +79,10 @@ jobs:
       - name: Build
         run: make -j"$(nproc)"
 
+      # A backstop only: tests/test-timeout.sh bounds each test, and a healthy run
+      # is a minute here, two on macOS. Without it a stall ran to the job's 6h default.
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -119,6 +122,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test without python3
+        timeout-minutes: 20
         run: |
           set -euo pipefail
           # Hide every python3* so `command -v python3` fails like it does in the
@@ -173,6 +177,7 @@ jobs:
           sudo ifconfig lo0 alias 127.0.0.3 up
 
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -245,6 +250,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -302,6 +308,7 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -345,6 +352,7 @@ jobs:
       - name: Test (offline self-tests under MSan)
         env:
           MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
+        timeout-minutes: 20
         run: |
           set -euo pipefail
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
@@ -426,6 +434,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -618,7 +627,12 @@ jobs:
           set -euo pipefail
           git fetch --no-tags origin \
             "+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
-          base="origin/${{ github.base_ref }}"
+          # Merge base, not the branch tip: master moves during a run, and a
+          # tip-relative diff blames its new C on this PR.
+          if ! base="$(git merge-base "origin/${{ github.base_ref }}" HEAD)"; then
+            echo "::error::no merge base with origin/${{ github.base_ref }}; cannot scope the check."
+            exit 1
+          fi
           set +e
           diff="$(git clang-format --binary clang-format-19 --style=file \
                     --diff --extensions c,h "$base")"
@@ -660,6 +674,8 @@ jobs:
           git fetch --no-tags origin \
             "+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
           base="origin/${{ github.base_ref }}"
+          # Three dots: merge-base scoped, so commits landing on master mid-run
+          # are not counted as this PR's.
           changed="$(git diff --name-only "$base"...HEAD)"
           has() { printf '%s\n' "$changed" | grep -qx "$1"; }
           if has man/httrack.1 && ! has html/httrack.man.html; then

.github/workflows/windows-build.yml

@@ -189,25 +189,53 @@ jobs:
           # A wedged crawl must not eat the job's timeout budget. timeout(1)'s
           # signals can't reap a native httrack.exe (MSYS signals don't reach it),
           # so a hang orphaned processes that starved the runner; run_with_timeout
-          # TerminateProcess-es the whole tree. 600s clears the slowest multi-pass
-          # crawl (a few passes at --max-time=120 each).
+          # TerminateProcess-es the whole tree. 600s is unchanged: it clears the
+          # 540s a three-pass crawl may legitimately take under local-crawl.sh's
+          # own watchdogs, against a slowest healthy test here of 39s.
           . ./testlib.sh
+          per_test=600
 
-          pass=0 fail=0 skip=0 failed="" skipped=""
+          # The whole suite must give up before the step timeout above. A cancelled
+          # step keeps neither its log nor the artifacts the later if:always()
+          # steps would upload, so an overrun that ends in a cancel tells us
+          # nothing; failing on our own terms keeps both. Healthy runs take 8-9
+          # min. The check sits between tests, so the step can still reach 25 min
+          # plus one per-test budget, and that worst case stays inside the 45.
+          suite_deadline=1500
+          started=$SECONDS
+
+          # Survives into the artifact even if the tail of the step log does not.
+          progress=suite-progress.log
+          : >"$progress"
+
+          pass=0 fail=0 skip=0 failed="" skipped="" deadline=0
           for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
                    *_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
                    60_crawl-log-salvage.test; do
+            elapsed=$((SECONDS - started))
+            if [ "$elapsed" -ge "$suite_deadline" ]; then
+              echo "::error::suite deadline: ${elapsed}s elapsed, stopping before $t"
+              echo "DEADLINE before $t after ${elapsed}s" >>"$progress"
+              # Per-test start times, so the slow ones are named rather than guessed.
+              sed 's/^/      /' "$progress"
+              deadline=1
+              break
+            fi
+            echo "RUN $t at ${elapsed}s" >>"$progress"
             rc=0
-            run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
+            # Same guard "make check" uses on POSIX, so a wedge is diagnosed the
+            # same way on every platform. It dumps before it kills, which a bare
+            # run_with_timeout cannot: by the time that returns, the tree whose
+            # stack we wanted is already gone.
+            HTTRACK_TEST_TIMEOUT=$per_test bash ./test-timeout.sh "$t" >"$t.log" 2>&1 || rc=$?
             case "$rc" in
               0) pass=$((pass + 1)); echo "PASS $t" ;;
               77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
               124)
                 fail=$((fail + 1)) failed="$failed $t"
+                # test-timeout.sh has already written the process list, the stacks
+                # and the killed crawl's own logs into $t.log.
                 echo "FAIL $t (timed out, tree killed)"
-                # Re-running a wedge traced would just hang again: salvage the
-                # killed crawl's own logs into the artifact instead.
-                dump_crawl_logs >>"$t.log"
                 tail -n 25 "$t.log" | sed 's/^/      /'
                 ;;
               *)
@@ -215,10 +243,15 @@ jobs:
                 echo "FAIL $t (exit $rc)"
                 # These assert with `test "$(...)" == "..." || exit 1`, which
                 # says nothing at all on failure. Re-run traced, still bounded.
-                run_with_timeout 600 bash -x "$t" >>"$t.log" 2>&1 || true
+                run_with_timeout "$per_test" bash -x "$t" >>"$t.log" 2>&1 || true
                 tail -n 25 "$t.log" | sed 's/^/      /'
                 ;;
             esac
+            echo "$rc $t" >>"$progress"
+            # An orphaned native httrack.exe spins and starves the runner, which
+            # is how this job dies with "lost communication" rather than a plain
+            # timeout. Clear them between tests and name whoever leaked them.
+            reap_leftover_processes "$t" | tee -a "$progress"
           done
           echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
             tee -a "$GITHUB_STEP_SUMMARY"
@@ -229,8 +262,11 @@ jobs:
           # crange pending #581;
           # webdav-mime needs a reapable background listener, which MSYS cannot give it;
           # badmtime needs a filesystem that stores an mtime past gmtime's range;
-          # single-file ends on a GUI half needing htsserver, which this job does not build.
-          expected_skips=" 01_engine-footer-overflow.test 100_local-purge-longpath.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
+          # single-file ends on a GUI half needing htsserver, which this job does not build;
+          # update-304-leak needs a LeakSanitizer build, which MSVC has no equivalent of.
+          expected_skips=" 01_engine-footer-overflow.test 100_local-purge-longpath.test 114_local-update-304-leak.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
+          # First, or the deadline reads as an unexplained shortfall in the gates below.
+          [ "$deadline" -eq 0 ] || { echo "::error::suite did not finish within ${suite_deadline}s"; exit 1; }
           [ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
           [ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
           [ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }

AGENTS.md

@@ -26,12 +26,18 @@ the operational checklist: toolchain, invariants, and how to ship a change.
   check`, or `PATH="<bld>/src:$PATH"` for a manual run.
 - Give new `.test` scripts `set -e`: the older ones predate the rule, so several
   `local-crawl.sh` calls with no `set -e` report PASS on any non-last failure.
-- Never assert with `cmd | grep -q MARKER && fail`. Under `pipefail` the
-  pipeline is non-zero both when `cmd` fails and when `grep -q` matches early
-  and SIGPIPEs it, so the `&&` never fires and a probe that proved nothing reads
-  as "marker absent". Capture the reply, assert the status line it must carry
-  (an empty, truncated or redirected one is marker-free too), then match with a
-  here-string.
+- Run teardown with errexit off: `trap 'set +e; cleanup' EXIT`. Under `set -e` a
+  failing cleanup command becomes the test's exit status (#773). Keep the other
+  signals on their own `trap` line, or errexit stays off for the rest of the run.
+  The guard also resets `$?`, so save it first if teardown reads it.
+- Never pipe into `grep -q`: it exits on the first match, so whatever the
+  producer had left to write takes SIGPIPE, and under `pipefail` that becomes
+  the pipeline's status. `cmd | grep -q M && fail` then never fires and a probe
+  that proved nothing reads as "marker absent"; `cmd | grep -q M || fail` fails
+  a test whose marker was present. bash issues one `write()` per line, so any
+  match that is not on the last line is exposed. Capture the reply, assert the
+  status line it must carry (an empty, truncated or redirected one is
+  marker-free too), then match with a here-string: `grep -q M <<<"$reply"`.
 
 ## Hard invariants
 - **Generated autotools files are NOT in git.** `configure`, every

html/fcguide.html

@@ -237,7 +237,7 @@ Build options:
   x  replace external html links by error pages (--replace-external)
  %x  do not include any password for external password protected websites (%x0 include) (--no-passwords)
  %q *include query string for local files (useless, for information purpose only) (%q0 don't include) (--include-query-string)
-  o *generate output html file in case of error (404..) (o0 don't generate) (--generate-errors)
+  o *save the server's error pages (404..) (o0 discard them) (--generate-errors)
   X *purge old files after update (X0 keep delete) (--purge-old[=N])
 
 Spider options:
@@ -413,7 +413,7 @@ site. Specifically, the defauls are:
   NN  name conversion type (0 *original structure, 1+: see below)
   LN  long names (L1 *long names / L0 8-3 conversion)
   K   keep original links (e.g. http://www.adr/link) (K0 *relative link)
-  o *generate output html file in case of error (404..) (o0 don't generate)
+  o *save the server's error pages (404..) (o0 discard them)
   X *purge old files after update (X0 keep delete)
   bN  accept cookies in cookies.txt (0=do not accept,* 1=accept)
   u check document type if unknown (cgi,asp..) (u0 don't check, * u1 check but /, u2 check always)
@@ -473,11 +473,11 @@ store them with the same names used on the web site.
 URLs within this web site are adjusted to point to the files in the
 mirror.
 
-<pre><b><i>  o *generate output html file in case of error (404..) (o0 don't generate) </i></b></pre>
+<pre><b><i>  o *save the server's error pages (404..) (o0 discard them) </i></b></pre>
 
-<p align=justify> IF there are errors in downloading, create a file that
-indicates that the URL was not found.  This makes browsing go a lot
-smoother.
+<p align=justify> IF a page cannot be downloaded, the error page the
+server sent is saved in its place, so a broken link still lands on the
+site's own 'not found' page.  This makes browsing go a lot smoother.
 
 <pre><b><i>  X *purge old files after update (X0 keep delete) </i></b></pre>
 
@@ -1011,7 +1011,7 @@ Build options:
   LN  long names (L1 *long names / L0 8-3 conversion)
   K   keep original links (e.g. http://www.adr/link) (K0 *relative link)
   x   replace external html links by error pages
-  o *generate output html file in case of error (404..) (o0 don't generate)
+  o *save the server's error pages (404..) (o0 discard them)
   X *purge old files after update (X0 keep delete)
   %x  do not include any password for external password protected websites (%x0 include) (--no-passwords)
   %q *include query string for local files (information only) (%q0 don't include) (--include-query-string)
@@ -1118,9 +1118,9 @@ deactivated byt his process.
 httrack http://www.shoesizes.com -O /tmp/shoesizes -x
 </i></b></pre>
 
-<p align=justify> This option prevents the generation of '404' error
-files to replace files that were not found even though there were URLs
-pointing to them.  It is useful for saving space as well as eliminating
+<p align=justify> This option keeps the server's '404' error pages out
+of the mirror, even though there were URLs pointing to the missing
+files.  It is useful for saving space as well as eliminating
 unnecessary files in operations where a working web site is not the
 desired result. 
 

html/httrack.man.html

@@ -777,8 +777,8 @@ information purpose only) (%q0 don’t include)
 <td width="82%">
 
 
-<p>*generate output html file in case of error (404..) (o0
-don&rsquo;t generate) (--generate-errors)</p></td></tr>
+<p>*save the server&rsquo;s error pages (404..) (o0 discard
+them) (--generate-errors)</p></td></tr>
 <tr valign="top" align="left">
 <td width="9%"></td>
 <td width="4%">

html/step9_opt5.html

@@ -122,7 +122,7 @@ h4 { margin: 0;  font-weight: bold;  font-size: 1.18em; }
   </small><br><br>
 <!-- -->
   <li>No error pages</li>
-  <br><small>Do not generate error pages (if a 404 error occurred, for example)
+  <br><small>Do not save the error pages sent by the server (if a 404 error occurred, for example)
   <br>If a page is missing on the remote site, there will not be any warning on the local site
   </small><br><br>
 <!-- -->

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "26 July 2026" "httrack website copier"
+.TH httrack 1 "27 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -220,7 +220,7 @@ do not include any password for external password protected websites (%x0 includ
 .IP \-%g
 strip query keys for dedup ([host/pattern=]key1,key2,...) (\-\-strip\-query <param>)
 .IP \-o
-*generate output html file in case of error (404..) (o0 don't generate) (\-\-generate\-errors)
+*save the server's error pages (404..) (o0 discard them) (\-\-generate\-errors)
 .IP \-X
 *purge old files after update (X0 keep delete) (\-\-purge\-old[=N])
 .IP \-%p

src/htsback.c

@@ -340,12 +340,111 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr
 }
 
 static int slot_can_be_cached_on_disk(const lien_back * back) {
+  /* A pending backup or spool means the slot is not finalized, and the swap
+     would unlink it through back_clear_entry() (#771). */
+  if (back->tmpfile != NULL && back->tmpfile[0] != '\0')
+    return 0;
   return (back->status == STATUS_READY && back->locked == 0
           && back->url_sav[0] != '\0'
           && strcmp(back->url_sav, BACK_ADD_TEST) != 0);
   /* Note: not checking !IS_DELAYED_EXT(back->url_sav) or it will quickly cause the slots to be filled! */
 }
 
+int back_selftest_slot_swap(void) {
+  lien_back back;
+  int err = 0;
+
+#define CHECK(want, why)                                                       \
+  do {                                                                         \
+    if (slot_can_be_cached_on_disk(&back) != (want)) {                         \
+      fprintf(stderr, "backswap: expected %d for %s\n", (want), (why));        \
+      err = 1;                                                                 \
+    }                                                                          \
+  } while (0)
+
+  memset(&back, 0, sizeof(back));
+  back.status = STATUS_READY;
+  strcpybuff(back.url_sav, "/tmp/httrack-selftest.bin");
+  CHECK(1, "a plain ready slot");
+
+  back.tmpfile = back.tmpfile_buffer;
+  strcpybuff(back.tmpfile_buffer, "/tmp/httrack-selftest.bin.bak");
+  CHECK(0, "a slot still holding a re-fetch backup");
+
+  /* Callers clear a spent temporary by emptying the name, not the pointer. */
+  back.tmpfile_buffer[0] = '\0';
+  CHECK(1, "a slot whose temporary was already dropped");
+
+  back.tmpfile = NULL;
+  back.locked = 1;
+  CHECK(0, "a locked slot");
+  back.locked = 0;
+
+  back.status = STATUS_TRANSFER;
+  CHECK(0, "a slot still transferring");
+  back.status = STATUS_READY;
+
+  back.url_sav[0] = '\0';
+  CHECK(0, "a slot with no save name");
+
+  strcpybuff(back.url_sav, BACK_ADD_TEST);
+  CHECK(0, "the dummy test slot");
+#undef CHECK
+
+  /* The swap round-trip must not lose the size of a slot whose body is already
+     at url_sav, or the link writer blanks the file (#797). */
+  {
+    static const char body[] = "swapped body";
+    int c;
+
+    for (c = 0; c < 2; c++) {
+      const hts_boolean inmemory = c == 0 ? HTS_TRUE : HTS_FALSE;
+      FILE *const fp = tmpfile();
+      lien_back *copy = NULL;
+
+      memset(&back, 0, sizeof(back));
+      back.status = STATUS_READY;
+      strcpybuff(back.url_sav, "/tmp/httrack-selftest.bin");
+      back.r.size = (LLint) sizeof(body) - 1;
+      if (inmemory) {
+        back.r.adr = strdupt(body);
+      }
+      if (fp == NULL || back_serialize(fp, &back) != 0 ||
+          fseek(fp, 0, SEEK_SET) != 0 || back_unserialize(fp, &copy) != 0) {
+        fprintf(stderr, "backswap: round-trip failed for a %s slot\n",
+                inmemory ? "buffered" : "direct-to-disk");
+        err = 1;
+      } else {
+        if (copy->r.size != back.r.size) {
+          fprintf(stderr,
+                  "backswap: %s slot came back with size " LLintP
+                  ", expected " LLintP "\n",
+                  inmemory ? "buffered" : "direct-to-disk", copy->r.size,
+                  back.r.size);
+          err = 1;
+        }
+        if (inmemory && (copy->r.adr == NULL ||
+                         memcmp(copy->r.adr, body, sizeof(body) - 1) != 0)) {
+          fprintf(stderr, "backswap: buffered slot lost its body\n");
+          err = 1;
+        }
+        if (!inmemory && copy->r.adr != NULL) {
+          fprintf(stderr, "backswap: direct-to-disk slot gained a body\n");
+          err = 1;
+        }
+        back_clear_entry(copy);
+        freet(copy);
+      }
+      if (fp != NULL)
+        fclose(fp);
+      freet(back.r.adr);
+    }
+  }
+
+  printf("backswap self-test: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 /* Put all backing entries that are ready in the storage hashtable to spare space and CPU */
 int back_cleanup_background(httrackp * opt, cache_back * cache,
                             struct_back * sback) {
@@ -585,6 +684,29 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
   return 0;
 }
 
+/* Note: utf-8 */
+void back_refetch_backup(httrackp *opt, lien_back *const back) {
+  back->tmpfile = NULL;
+  if (fexist_utf8(back->url_sav)) {
+    hts_boolean saved = HTS_FALSE;
+
+    if (create_back_tmpfile(opt, back, "bak") == 0) {
+      /* clobber a .bak a killed run left behind, or the guard stays off for
+         good (#758) */
+      if (fexist_utf8(back->tmpfile))
+        hts_log_print(opt, LOG_WARNING, "replacing leftover backup %s",
+                      back->tmpfile);
+      saved = hts_rename_over(opt, back->url_sav, back->tmpfile);
+    }
+    if (!saved) {
+      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
+                    "could not back up %s; an aborted re-fetch will lose it",
+                    back->url_sav);
+      back->tmpfile = NULL;
+    }
+  }
+}
+
 /* Did the fetch fail to produce a response, as opposed to the engine
    deliberately passing the resource over? Only the latter may be purged. */
 static hts_boolean back_transfer_failed(const int statuscode) {
@@ -615,7 +737,7 @@ static void back_finalize_backup(httrackp *opt, lien_back *const back,
     }
     /* On failure keep the backup: an orphaned temp beats losing the good copy.
      */
-    if (!hts_rename_over(back->tmpfile, back->url_sav))
+    if (!hts_rename_over(opt, back->tmpfile, back->url_sav))
       hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
                     "could not restore %s; previous copy kept as %s",
                     back->url_sav, back->tmpfile);
@@ -746,7 +868,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                                  "Read error when decompressing");
                     }
                     UNLINK(unpacked);
-                  } else if (hts_rename_over(unpacked, back[p].url_sav)) {
+                  } else if (hts_rename_over(opt, unpacked, back[p].url_sav)) {
                     /* The temp bypassed filecreate(), which is what chmods. */
 #ifndef _WIN32
                     chmod(back[p].url_sav, HTS_ACCESS_FILE);
@@ -1101,6 +1223,13 @@ void back_connxfr(htsblk * src, htsblk * dst) {
   src->debugid = 0;
 }
 
+/* Release the buffers a response owns. The connection members are left alone:
+   back_connxfr() moves those, and the file handles are closed elsewhere. */
+static void back_free_response(htsblk *r) {
+  deleteaddr(r);
+  warc_free_request(r);
+}
+
 void back_move(lien_back * src, lien_back * dst) {
   memcpy(dst, src, sizeof(lien_back));
   memset(src, 0, sizeof(lien_back));
@@ -1194,7 +1323,10 @@ int back_unserialize(FILE * fp, lien_back ** dst) {
     (*dst)->r.ssl_con = NULL;
 #endif
     if (back_data_unserialize(fp, (void **) &(*dst)->r.adr, &size) == 0) {
-      (*dst)->r.size = size;
+      /* A bodyless slot already wrote its bytes to url_sav (FTP, direct to
+         disk); zeroing r.size makes the writer blank that file (#797). */
+      if ((*dst)->r.adr != NULL)
+        (*dst)->r.size = size;
       (*dst)->r.headers = NULL;
       if (back_string_unserialize(fp, &(*dst)->r.headers) == 0)
         return 0;               /* ok */
@@ -1633,10 +1765,7 @@ int back_clear_entry(lien_back * back) {
       back->r.soc = INVALID_SOCKET;
     }
 
-    if (back->r.adr != NULL) {  // reste un bloc à désallouer
-      freet(back->r.adr);
-      back->r.adr = NULL;
-    }
+    back_free_response(&back->r);
     if (back->chunk_adr != NULL) {      // reste un bloc à désallouer
       freet(back->chunk_adr);
       back->chunk_adr = NULL;
@@ -1649,12 +1778,6 @@ int back_clear_entry(lien_back * back) {
       (void) unlink(back->tmpfile);
       back->tmpfile = NULL;
     }
-    // headers
-    if (back->r.headers != NULL) {
-      freet(back->r.headers);
-      back->r.headers = NULL;
-    }
-    warc_free_request(&back->r);
     // Tout nettoyer
     memset(back, 0, sizeof(lien_back));
     back->r.soc = INVALID_SOCKET;
@@ -3164,36 +3287,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
-                              /* Re-fetch over an existing file (#77 follow-up):
-                                 move the good copy aside before truncating it
-                                 so an aborted transfer can restore it. url_sav
-                                 is still written normally (file list intact).
-                               */
-                              back[i].tmpfile = NULL;
-                              if (fexist_utf8(back[i].url_sav)) {
-                                hts_boolean saved = HTS_FALSE;
-
-                                if (create_back_tmpfile(opt, &back[i], "bak") ==
-                                    0) {
-                                  /* clobber a .bak a killed run left behind,
-                                     or the guard stays off for good (#758) */
-                                  if (fexist_utf8(back[i].tmpfile))
-                                    hts_log_print(
-                                        opt, LOG_WARNING,
-                                        "replacing leftover backup %s",
-                                        back[i].tmpfile);
-                                  saved = hts_rename_over(back[i].url_sav,
-                                                          back[i].tmpfile);
-                                }
-                                if (!saved) {
-                                  hts_log_print(
-                                      opt, LOG_WARNING | LOG_ERRNO,
-                                      "could not back up %s; an aborted "
-                                      "re-fetch will lose it",
-                                      back[i].url_sav);
-                                  back[i].tmpfile = NULL;
-                                }
-                              }
+                              back_refetch_backup(opt, &back[i]);
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {
@@ -3988,6 +4082,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
                       memset(&tmp, 0, sizeof(tmp));
                       back_connxfr(&back[i].r, &tmp);
+                      /* the cache entry overwrites the whole struct, so drop
+                         what the 304 response still owns first (#782) */
+                      back_free_response(&back[i].r);
                       back[i].r =
                         cache_read(opt, cache, back[i].url_adr, back[i].url_fil,
                                    back[i].url_sav, back[i].location_buffer);

src/htsback.h

@@ -139,6 +139,12 @@ int back_trylive(httrackp * opt, cache_back * cache, struct_back * sback,
                  const int p);
 int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                   const int p);
+/* Move the previous copy of back->url_sav to back->tmpfile so back_finalize()
+   can put it back when the re-fetch fails (#77 follow-up). Call right before
+   truncating url_sav; tmpfile stays NULL when there is nothing to save. */
+void back_refetch_backup(httrackp *opt, lien_back *const back);
+/* -#test=backswap: slots eligible for the on-disk ready table. */
+int back_selftest_slot_swap(void);
 void back_info(struct_back * sback, int i, int j, FILE * fp);
 void back_infostr(struct_back *sback, int i, int j, char *s, size_t size);
 LLint back_transferred(LLint add, struct_back * sback);

src/htscache.c

@@ -985,6 +985,7 @@ void cache_init(cache_back * cache, httrackp * opt) {
               StringBuff(opt->path_log),
               "hts-cache/new.zip")))) { // a previous cache exists.. rename it
         if (!hts_rename_over(
+                opt,
                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
                         StringBuff(opt->path_log), "hts-cache/new.zip"),
                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),

src/htsftp.c

@@ -483,16 +483,19 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
                     back->r.totalsize = size;
                   }
                 }
-                // REST?
-                if (fexist(back->url_sav) && (transfer_list == 0)) {
+                /* Only over a copy back_add() judged partial: on --update every
+                   mirrored file exists, and resuming a complete one splices the
+                   old body into the new (#798). */
+                if (back->range_req_size > 0 && (transfer_list == 0)) {
                   strcpybuff(back->info, "rest");
-                  snprintf(line, sizeof(line), "REST " LLintP, (LLint) fsize(back->url_sav));
+                  snprintf(line, sizeof(line), "REST " LLintP,
+                           (LLint) back->range_req_size);
                   send_line(soc_ctl, line);
                   get_ftp_line(soc_ctl, line, sizeof(line), timeout);
                   _CHECK_HALT_FTP;
                   if ((line[0] == '3') || (line[0] == '2')) {   // ok
                     rest_understood = 1;
-                  }             // sinon tant pis 
+                  } // else never mind
                 }
               }                 // sinon tant pis 
             }
@@ -617,13 +620,19 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
         // Ok, connexion initiée
         //
         if (soc_dat != INVALID_SOCKET) {
-          if (rest_understood) {        // REST envoyée et comprise
+          if (rest_understood) { // REST sent and understood
             file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 0, 1,
                         0);
+            /* The bytes already on disk count too, or the completeness check
+               below rejects every resumed transfer (#798). */
+            back->r.size = back->range_req_size;
             back->r.fp = fileappend(&opt->state.strc, back->url_sav);
           } else {
             file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 1, 1,
                         0);
+            /* Every failure exit below would else leave the mirror truncated
+               (#771); the resume branch appends and needs no backup. */
+            back_refetch_backup(opt, back);
             back->r.fp = filecreate(&opt->state.strc, back->url_sav);
           }
           strcpybuff(back->info, "receiving");

src/htshelp.c

@@ -558,8 +558,7 @@ void help(const char *app, int more) {
   infomsg
     (" %q *include query string for local files (useless, for information purpose only) (%q0 don't include)");
   infomsg(" %g  strip query keys for dedup ([host/pattern=]key1,key2,...)");
-  infomsg
-    ("  o *generate output html file in case of error (404..) (o0 don't generate)");
+  infomsg("  o *save the server's error pages (404..) (o0 discard them)");
   infomsg("  X *purge old files after update (X0 keep delete)");
   infomsg(" %p  preserve html files 'as is' (identical to '-K4 -%F \"\"')");
   infomsg(" %T  links conversion to UTF-8");

src/htslib.c

@@ -2703,17 +2703,8 @@ void hts_now_iso8601(char out[32]) {
   time_t t = time(NULL);
   struct tm tmv;
 
-#if defined(_WIN32)
-  struct tm *g = gmtime(&t);
-
-  if (g != NULL)
-    tmv = *g;
-  else
+  if (!hts_gmtime(t, &tmv))
     memset(&tmv, 0, sizeof(tmv));
-#else
-  if (gmtime_r(&t, &tmv) == NULL)
-    memset(&tmv, 0, sizeof(tmv));
-#endif
   strftime(out, 32, "%Y-%m-%dT%H:%M:%SZ", &tmv);
 }
 
@@ -5790,7 +5781,8 @@ HTSEXT_API void hts_log_vprint(httrackp * opt, int type, const char *format, va_
   if (hts_log_print_callback != NULL) {
     va_list args_copy;
     va_copy(args_copy, args);
-    hts_log_print_callback(opt, type, format, args);
+    /* the copy, so the vfprintf() below still has an unread list */
+    hts_log_print_callback(opt, type, format, args_copy);
     va_end(args_copy);
   }
   if (opt != NULL && opt->log != NULL) {

src/htslib.h

@@ -157,6 +157,19 @@ struct t_dnscache {
   char host_addr[HTS_MAXADDRNUM][HTS_MAXADDRLEN];
 };
 
+/* Break t down as UTC into the caller's buffer, HTS_FALSE if that failed.
+   gmtime()'s static is shared, and both the engine and ProxyTrack convert on
+   worker threads. */
+static HTS_INLINE HTS_UNUSED hts_boolean hts_gmtime(time_t t,
+                                                    struct tm *tmbuf) {
+#ifdef _WIN32
+  /* Microsoft's gmtime_s takes the destination first, unlike C11 Annex K. */
+  return gmtime_s(tmbuf, &t) == 0 ? HTS_TRUE : HTS_FALSE;
+#else
+  return gmtime_r(&t, tmbuf) != NULL ? HTS_TRUE : HTS_FALSE;
+#endif
+}
+
 /* Library internal definictions */
 #ifdef HTS_INTERNAL_BYTECODE
 

src/htsselftest.c

@@ -2622,6 +2622,74 @@ static int st_savename(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+static char st_log_callback_seen[256];
+
+static void st_log_callback(httrackp *opt, int type, const char *format,
+                            va_list args) {
+  (void) opt;
+  (void) type;
+  (void) vsnprintf(st_log_callback_seen, sizeof(st_log_callback_seen), format,
+                   args);
+}
+
+/* The callback must not consume the va_list the log file's vfprintf() needs. */
+static int st_logcallback(httrackp *opt, int argc, char **argv) {
+  static const char want[] = "42 sentinel";
+  static const char want_filtered[] = "7 filtered";
+  char BIGSTK seen[sizeof(st_log_callback_seen)];
+  char BIGSTK line[256];
+  FILE *fp;
+  int rc = 1;
+
+  (void) argc;
+  (void) argv;
+
+  fp = tmpfile();
+  if (fp == NULL) {
+    fprintf(stderr, "logcallback: tmpfile() failed\n");
+    return 1;
+  }
+  opt->log = fp;
+  opt->debug = LOG_NOTICE;
+  st_log_callback_seen[0] = '\0';
+  hts_set_log_vprint_callback(st_log_callback);
+  hts_log_print(opt, LOG_NOTICE, "%d %s", 42, "sentinel");
+  hts_set_log_vprint_callback(NULL);
+  opt->log = NULL;
+  strcpybuff(seen, st_log_callback_seen);
+
+  rewind(fp);
+  if (fgets(line, (int) sizeof(line), fp) == NULL) {
+    fprintf(stderr, "logcallback: log file is empty, nothing was written\n");
+    fclose(fp);
+    return 1;
+  }
+  fclose(fp);
+
+  /* The callback runs above the level filter and without a log file at all;
+     the front-ends that install one usually have no opt->log open. */
+  st_log_callback_seen[0] = '\0';
+  hts_set_log_vprint_callback(st_log_callback);
+  hts_log_print(opt, LOG_DEBUG, "%d %s", 7, "filtered");
+  hts_set_log_vprint_callback(NULL);
+
+  /* Same arguments both ways; the file line carries a level prefix. */
+  if (strcmp(seen, want) != 0)
+    fprintf(stderr, "logcallback: callback got '%s' want '%s'\n", seen, want);
+  else if (strstr(line, want) == NULL)
+    fprintf(stderr, "logcallback: log file got '%s' want it to carry '%s'\n",
+            line, want);
+  else if (strcmp(st_log_callback_seen, want_filtered) != 0)
+    fprintf(stderr, "logcallback: unfiltered callback got '%s' want '%s'\n",
+            st_log_callback_seen, want_filtered);
+  else
+    rc = 0;
+
+  if (rc == 0)
+    printf("logcallback self-test OK\n");
+  return rc;
+}
+
 /* an empty fil started htsAddLink's codebase walk before the buffer (#730) */
 static int st_addlink(httrackp *opt, int argc, char **argv) {
   htsmoduleStruct BIGSTK str;
@@ -4944,6 +5012,101 @@ static int st_warc_surt(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* A URL longer than the old 1024-byte header-format buffer must still reach the
+   archive: the record used to be abandoned whole, silently (#785). The sweep
+   straddles the boundary so both the stack-buffer and the grow path run. */
+static int st_warc_longurl(httrackp *opt, int argc, char **argv) {
+  /* "WARC-Target-URI: " + CRLF costs 19 bytes, so the old buffer failed at
+     1005; 9000 forces several reallocs within one record. */
+  static const size_t lengths[] = {100, 1003, 1004, 1005, 1006, 2000, 9000};
+  static const char resp_hdr[] =
+      "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n";
+  char path[HTS_URLMAXSIZE * 2];
+  char body[64];
+  warc_writer *w;
+  FILE *fp;
+  char *blob;
+  LLint fsz;
+  const char *at2;
+  size_t i, n, nrec = 0;
+  int err = 0;
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-longurl: need a writable directory\n");
+    return 1;
+  }
+  snprintf(path, sizeof(path), "%s/longurl.warc", argv[0]);
+
+  w = warc_open(opt, path);
+  if (w == NULL) {
+    fprintf(stderr, "warc-longurl: could not create %s\n", path);
+    return 1;
+  }
+  for (i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) {
+    const size_t len = lengths[i];
+    char *uri = malloct(len + 1);
+
+    if (uri == NULL) {
+      warc_close(w);
+      return 1;
+    }
+    /* A distinct tail per URI so a truncated one cannot match another. */
+    snprintf(uri, len + 1, "http://example.com/%04d/", (int) len);
+    memset(uri + strlen(uri), 'a', len - strlen(uri));
+    uri[len] = '\0';
+    /* Distinct payloads: identical ones dedupe into revisit records. */
+    snprintf(body, sizeof(body), "<html><body>%04d</body></html>\n", (int) len);
+    if (warc_write_transaction(w, uri, NULL, NULL, resp_hdr, body, strlen(body),
+                               NULL, 200, 0, 0) != 0) {
+      fprintf(stderr, "warc-longurl: write failed at length %d\n", (int) len);
+      err = 1;
+    }
+    freet(uri);
+  }
+  warc_close(w);
+
+  fsz = fsize_utf8(path);
+  blob = (fsz > 0) ? malloct((size_t) fsz + 1) : NULL;
+  if (blob == NULL) {
+    fprintf(stderr, "warc-longurl: no archive written\n");
+    return 1;
+  }
+  fp = FOPEN(path, "rb");
+  n = (fp != NULL) ? fread(blob, 1, (size_t) fsz, fp) : 0;
+  if (fp != NULL)
+    fclose(fp);
+  blob[n] = '\0';
+
+  for (i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) {
+    const size_t len = lengths[i];
+    char want[64];
+    const char *at;
+
+    snprintf(want, sizeof(want), "WARC-Target-URI: http://example.com/%04d/",
+             (int) len);
+    at = strstr(blob, want);
+    if (at == NULL) {
+      fprintf(stderr, "warc-longurl: length %d lost its record\n", (int) len);
+      err = 1;
+    } else if (strlen(at) < strlen("WARC-Target-URI: ") + len ||
+               at[strlen("WARC-Target-URI: ") + len] != '\r') {
+      fprintf(stderr, "warc-longurl: length %d truncated\n", (int) len);
+      err = 1;
+    }
+  }
+  for (at2 = blob; (at2 = strstr(at2, "WARC-Type: response")) != NULL; at2++)
+    nrec++;
+  if (nrec != sizeof(lengths) / sizeof(lengths[0])) {
+    fprintf(stderr, "warc-longurl: %d response records, want %d\n", (int) nrec,
+            (int) (sizeof(lengths) / sizeof(lengths[0])));
+    err = 1;
+  }
+
+  freet(blob);
+  printf("warc-longurl: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 /* End-to-end CDXJ: crawl a handful of records with --warc-cdx, then verify the
    .cdx is sorted, has exactly one line per response/revisit/resource (none for
    warcinfo/request), and each offset/length points at a gzip member that
@@ -5977,10 +6140,9 @@ static hts_boolean ro_is(const char *path, const char *data) {
 }
 
 // -#test=renameover <dir>: hts_rename_over() must replace an existing dst and
-// never delete one it did not replace (#779). Which half is live depends on
+// never lose one it did not replace (#779, #790). Which half is live depends on
 // what rename() does to an existing target, so probe that and name the regime.
 static int st_renameover(httrackp *opt, int argc, char **argv) {
-  (void) opt;
   if (argc < 1) {
     fprintf(stderr, "renameover: needs a writable base dir\n");
     return 1;
@@ -6009,7 +6171,7 @@ static int st_renameover(httrackp *opt, int argc, char **argv) {
   ro_put(dst, "old");
   if (replaceable) {
     /* An existing dst must still be replaced: the unlink is for this. */
-    if (!hts_rename_over(src, dst)) {
+    if (!hts_rename_over(opt, src, dst)) {
       fprintf(stderr, "renameover: replacing an existing dst failed: %s\n",
               strerror(errno));
       err++;
@@ -6019,7 +6181,7 @@ static int st_renameover(httrackp *opt, int argc, char **argv) {
     }
   } else {
     /* A failure the unlink cannot fix must leave dst as it was. */
-    if (hts_rename_over(src, dst)) {
+    if (hts_rename_over(opt, src, dst)) {
       fprintf(stderr, "renameover: an unfixable failure reported success\n");
       err++;
     }
@@ -6029,10 +6191,36 @@ static int st_renameover(httrackp *opt, int argc, char **argv) {
     }
   }
 
+  /* A directory in the way is not something the caller asked to replace: it
+     must be refused, never parked aside and orphaned. */
+  (void) UNLINK(dst);
+  ro_put(src, "new");
+  if (MKDIR(dst) == 0) {
+    char parked[sizeof(dst) + 16];
+
+    snprintf(parked, sizeof(parked), "%s.hts-old0", dst);
+    if (hts_rename_over(opt, src, dst)) {
+      fprintf(stderr, "renameover: a directory at dst reported success\n");
+      err++;
+    }
+    if (!ro_is(src, "new")) {
+      fprintf(stderr, "renameover: a directory at dst consumed src\n");
+      err++;
+    }
+    /* RMDIR only succeeds on a directory that is there, so it doubles as the
+       probe: the parked name must not exist at all. */
+    if (RMDIR(parked) == 0 || fexist_utf8(parked)) {
+      fprintf(stderr, "renameover: a directory at dst was parked aside\n");
+      err++;
+    }
+    (void) RMDIR(dst);
+  }
+  (void) UNLINK(src);
+
   /* A missing src must leave dst alone and report failure. */
   (void) UNLINK(src);
   ro_put(dst, "keep");
-  if (hts_rename_over(src, dst)) {
+  if (hts_rename_over(opt, src, dst)) {
     fprintf(stderr, "renameover: a missing src reported success\n");
     err++;
   }
@@ -6043,11 +6231,88 @@ static int st_renameover(httrackp *opt, int argc, char **argv) {
 
   /* Same, with dst absent too: nothing to lose, still a failure. */
   (void) UNLINK(dst);
-  if (hts_rename_over(src, dst)) {
+  if (hts_rename_over(opt, src, dst)) {
     fprintf(stderr, "renameover: a missing src and dst reported success\n");
     err++;
   }
 
+  /* The aside fallback, driven directly: a clobbering rename() never reaches
+     it. Skipped in the refused regime, where no rename at all succeeds. */
+  if (replaceable) {
+    char aside[sizeof(dst) + 16], keep[sizeof(dst) + 16];
+
+    snprintf(aside, sizeof(aside), "%s.hts-old0", dst);
+    snprintf(keep, sizeof(keep), "%s.hts-old1", dst);
+    (void) UNLINK(aside);
+    (void) UNLINK(keep);
+    ro_put(src, "new");
+    ro_put(dst, "old");
+    if (!hts_rename_over_aside_selftest(opt, src, dst)) {
+      fprintf(stderr, "renameover: the aside fallback failed: %s\n",
+              strerror(errno));
+      err++;
+    } else if (!ro_is(dst, "new") || fexist_utf8(src) || fexist_utf8(aside)) {
+      fprintf(stderr, "renameover: the aside fallback did not replace dst\n");
+      err++;
+    }
+
+    /* #790: the retry fails (no src). The old content must survive, back at dst
+       or, when the move back fails too, under the parked name it is logged as.
+       Name the outcome so a leg cannot pass having tested the other one. */
+    (void) UNLINK(src);
+    ro_put(dst, "old");
+    if (hts_rename_over_aside_selftest(opt, src, dst)) {
+      fprintf(stderr, "renameover: a failed aside retry reported success\n");
+      err++;
+    }
+    if (ro_is(dst, "old") && !fexist_utf8(aside)) {
+      printf("renameover: restore back\n");
+    } else if (ro_is(aside, "old") && !fexist_utf8(dst)) {
+      printf("renameover: restore parked\n");
+      (void) UNLINK(aside);
+      ro_put(dst, "old");
+    } else {
+      fprintf(stderr, "renameover: a failed aside retry lost the old copy\n");
+      err++;
+    }
+
+    /* An unrelated file already sitting on the aside name must survive. */
+    ro_put(src, "new");
+    ro_put(aside, "mine");
+    if (!hts_rename_over_aside_selftest(opt, src, dst)) {
+      fprintf(stderr, "renameover: a taken aside name failed the move: %s\n",
+              strerror(errno));
+      err++;
+    } else if (!ro_is(dst, "new") || !ro_is(aside, "mine") ||
+               fexist_utf8(keep)) {
+      fprintf(stderr, "renameover: a taken aside name was not skipped\n");
+      err++;
+    }
+    (void) UNLINK(aside);
+    (void) UNLINK(keep);
+
+    /* A directory there reads as free to the probe, so the park must skip it
+       on the refusal rather than give up. */
+    ro_put(src, "new");
+    ro_put(dst, "old");
+    if (MKDIR(aside) == 0) {
+      if (!hts_rename_over_aside_selftest(opt, src, dst)) {
+        fprintf(stderr,
+                "renameover: a directory on the aside name blocked the "
+                "move: %s\n",
+                strerror(errno));
+        err++;
+      } else if (!ro_is(dst, "new") || fexist_utf8(keep)) {
+        fprintf(stderr, "renameover: a directory on the aside name was not "
+                        "skipped\n");
+        err++;
+      }
+      (void) RMDIR(aside);
+    }
+    (void) UNLINK(keep);
+  }
+
+  (void) UNLINK(src);
   (void) UNLINK(dst);
   printf("renameover: %s\n", err ? "FAIL" : "OK");
   return err;
@@ -6366,6 +6631,13 @@ static void threadwait_gated_thread(void *arg) {
   hts_mutexrelease(&threadwait_lock);
 }
 
+static int st_backswap(httrackp *opt, int argc, char **argv) {
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  return back_selftest_slot_swap();
+}
+
 static int st_threadwait(httrackp *opt, int argc, char **argv) {
   int err = 0;
   int i, round;
@@ -6422,6 +6694,109 @@ static int st_threadwait(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* #794: hts_gmtime() must own its output. The table is an independent oracle;
+   the threaded phase is what corrupts if it ever goes back to gmtime()'s
+   shared static. */
+#define GMTIME_THREADS 8
+#define GMTIME_ROUNDS 50000
+
+static const struct {
+  time_t t;
+  int year, mon, mday, hour, min, sec, wday, yday;
+} gmtime_refs[] = {
+    {(time_t) 0, 70, 0, 1, 0, 0, 0, 4, 0},
+    {(time_t) 951782400, 100, 1, 29, 0, 0, 0, 2, 59}, /* a leap day */
+    {(time_t) 1000000000, 101, 8, 9, 1, 46, 40, 0, 251},
+    {(time_t) 2147483647, 138, 0, 19, 3, 14, 7, 2, 18}, /* 32-bit ceiling */
+};
+
+#define GMTIME_REFS ((int) (sizeof(gmtime_refs) / sizeof(gmtime_refs[0])))
+
+static hts_boolean gmtime_ref_matches(int i, const struct tm *tm) {
+  if (tm->tm_year != gmtime_refs[i].year || tm->tm_mon != gmtime_refs[i].mon ||
+      tm->tm_mday != gmtime_refs[i].mday ||
+      tm->tm_hour != gmtime_refs[i].hour || tm->tm_min != gmtime_refs[i].min ||
+      tm->tm_sec != gmtime_refs[i].sec || tm->tm_wday != gmtime_refs[i].wday ||
+      tm->tm_yday != gmtime_refs[i].yday)
+    return HTS_FALSE;
+  return HTS_TRUE;
+}
+
+static htsmutex gmtime_lock = HTSMUTEX_INIT;
+static int gmtime_bad = 0;
+
+static void gmtime_thread(void *arg) {
+  const int i = *(const int *) arg;
+  int bad = 0, round;
+
+  for (round = 0; round < GMTIME_ROUNDS; round++) {
+    struct tm tmv;
+
+    if (!hts_gmtime(gmtime_refs[i].t, &tmv) || !gmtime_ref_matches(i, &tmv))
+      bad++;
+  }
+  hts_mutexlock(&gmtime_lock);
+  gmtime_bad += bad;
+  hts_mutexrelease(&gmtime_lock);
+}
+
+static int st_gmtime(httrackp *opt, int argc, char **argv) {
+  static int idx[GMTIME_THREADS];
+  int err = 0, i;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+
+  for (i = 0; i < GMTIME_REFS; i++) {
+    struct tm tmv;
+
+    if (!hts_gmtime(gmtime_refs[i].t, &tmv)) {
+      fprintf(stderr, "gmtime: conversion #%d failed\n", i);
+      err = 1;
+    } else if (!gmtime_ref_matches(i, &tmv)) {
+      fprintf(stderr,
+              "gmtime: #%d gave %04d-%02d-%02d %02d:%02d:%02d (wday %d, "
+              "yday %d)\n",
+              i, tmv.tm_year + 1900, tmv.tm_mon + 1, tmv.tm_mday, tmv.tm_hour,
+              tmv.tm_min, tmv.tm_sec, tmv.tm_wday, tmv.tm_yday);
+      err = 1;
+    }
+  }
+
+  /* the return is the only failure signal the callers have, so a helper that
+     always claims success leaves them formatting an uninitialised struct tm.
+     Out of range for a 64-bit time_t: NULL from gmtime_r, EINVAL from
+     _gmtime64_s. */
+  if (sizeof(time_t) >= 8) {
+    const time_t beyond = (time_t) INT64_MAX;
+    struct tm tmv;
+
+    if (hts_gmtime(beyond, &tmv)) {
+      fprintf(stderr,
+              "gmtime: an out-of-range time_t was reported converted\n");
+      err = 1;
+    }
+  }
+
+  for (i = 0; i < GMTIME_THREADS; i++) {
+    idx[i] = i % GMTIME_REFS;
+    if (hts_newthread(gmtime_thread, &idx[i]) != 0) {
+      fprintf(stderr, "gmtime: cannot spawn\n");
+      return 1;
+    }
+  }
+  htsthread_wait();
+  if (gmtime_bad != 0) {
+    fprintf(stderr, "gmtime: %d/%d concurrent conversions were corrupt\n",
+            gmtime_bad, GMTIME_THREADS * GMTIME_ROUNDS);
+    err = 1;
+  }
+
+  printf("gmtime self-test: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 #define CHANGES_RACE_FILES 8
 #define CHANGES_RACE_ROUNDS 400
 
@@ -6688,6 +7063,10 @@ static const struct selftest_entry {
      st_changes_race},
     {"threadwait", "", "htsthread_wait() joins threads spawned just before it",
      st_threadwait},
+    {"gmtime", "",
+     "hts_gmtime() fills the caller's buffer, not a static (#794)", st_gmtime},
+    {"backswap", "", "which backlog slots may be swapped to the ready table",
+     st_backswap},
     {"pause", "", "randomized inter-file pause target self-test", st_pause},
     {"relative", "<link> <curr-file>", "relative link between two paths",
      st_relative},
@@ -6720,6 +7099,8 @@ static const struct selftest_entry {
      st_growsize},
     {"addlink", "", "htsAddLink codebase walk over an empty current path",
      st_addlink},
+    {"logcallback", "", "log callback must not consume the log file's va_list",
+     st_logcallback},
     {"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
     {"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
      st_cacheindex},
@@ -6778,6 +7159,9 @@ static const struct selftest_entry {
      st_warc_verbatim},
     {"warc-surt", "", "SURT canonicalization of the CDXJ sort key",
      st_warc_surt},
+    {"warc-longurl", "<dir>",
+     "a URL past the header-format buffer still reaches the archive",
+     st_warc_longurl},
     {"longpath", "<dir>",
      "round-trip a >MAX_PATH file through the _w* wrappers (\\\\?\\ on "
      "Windows)",

src/htssinglefile.c

@@ -1109,7 +1109,7 @@ hts_boolean singlefile_rewrite_file(httrackp *opt, const char *root,
                    HTS_ACCESS_FILE);
 #endif
     if (ok)
-      ok = hts_rename_over(StringBuff(tmp), page_path);
+      ok = hts_rename_over(opt, StringBuff(tmp), page_path);
     if (!ok) {
       hts_log_print(opt, LOG_ERROR, "single-file: could not rewrite %s",
                     page_path);

src/htstools.c

@@ -1438,21 +1438,75 @@ HTSEXT_API hts_boolean hts_findissystem(find_handle find) {
   return 0;
 }
 
-hts_boolean hts_rename_over(const char *src, const char *dst) {
+/* Park cdst under a free sibling name; caside receives it. */
+static hts_boolean rename_park_aside(char *caside, size_t size,
+                                     const char *cdst) {
+  int i;
+
+  for (i = 0; i < 16; i++) {
+    if (!slprintfbuff(caside, size, "%s.hts-old%d", cdst, i))
+      return HTS_FALSE;
+    /* Skip a name the mirror already holds: POSIX rename() would clobber it
+       (#774). A non-regular entry reads as free and the rename refuses it. */
+    if (fexist_utf8(caside))
+      continue;
+    if (RENAME(cdst, caside) == 0)
+      return HTS_TRUE;
+  }
+  return HTS_FALSE;
+}
+
+/* cdst is in the way of the move: park it, retry, and put it back if the retry
+   fails too. Unlinking it instead would leave nothing at all (#790). */
+static hts_boolean rename_over_aside(httrackp *opt, const char *csrc,
+                                     const char *cdst) {
+  char caside[CATBUFF_SIZE];
+  int err;
+
+  /* Only a regular file may be parked: a directory in the way is not what the
+     caller asked to replace, and parking it orphans it (UNLINK cannot drop). */
+  if (!fexist_utf8(cdst))
+    return HTS_FALSE;
+  if (!rename_park_aside(caside, sizeof(caside), cdst))
+    return HTS_FALSE;
+  if (RENAME(csrc, cdst) == 0) {
+    (void) UNLINK(caside);
+    return HTS_TRUE;
+  }
+  err = errno;
+  /* Retry once, then name the parked copy: nothing else on disk or in the log
+     points at it, and an --update purge would delete it unnoticed. */
+  if (RENAME(caside, cdst) != 0 && RENAME(caside, cdst) != 0)
+    hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
+                  "could not put %s back; its previous content is now %s", cdst,
+                  caside);
+  errno = err;
+  return HTS_FALSE;
+}
+
+hts_boolean hts_rename_over(httrackp *opt, const char *src, const char *dst) {
   char csrc[CATBUFF_SIZE], cdst[CATBUFF_SIZE];
 
   fconv(csrc, sizeof(csrc), src);
   fconv(cdst, sizeof(cdst), dst);
   if (RENAME(csrc, cdst) == 0)
     return HTS_TRUE;
-  /* Only a dst in the way is something the unlink can clear, and the CRT maps
-     that to EEXIST; it keeps EACCES for a src another process holds, where
-     removing dst would lose a file the retry cannot replace (#790). The src
-     check covers a CRT that reports neither. */
+  /* Only a dst in the way is something the fallback can clear, and the CRT maps
+     that to EEXIST; it keeps EACCES for a src another process holds, where the
+     retry would fail the same way. The src check covers a CRT that reports
+     neither. */
   const int err = errno;
 
   if (err != EEXIST || !fexist_utf8(src))
     return HTS_FALSE;
-  (void) UNLINK(cdst);
-  return RENAME(csrc, cdst) == 0 ? HTS_TRUE : HTS_FALSE;
+  return rename_over_aside(opt, csrc, cdst);
+}
+
+hts_boolean hts_rename_over_aside_selftest(httrackp *opt, const char *src,
+                                           const char *dst) {
+  char csrc[CATBUFF_SIZE], cdst[CATBUFF_SIZE];
+
+  fconv(csrc, sizeof(csrc), src);
+  fconv(cdst, sizeof(cdst), dst);
+  return rename_over_aside(opt, csrc, cdst);
 }

src/htstools.h

@@ -137,11 +137,17 @@ HTSEXT_API hts_boolean hts_findisdir(find_handle find);
 HTSEXT_API hts_boolean hts_findisfile(find_handle find);
 HTSEXT_API hts_boolean hts_findissystem(find_handle find);
 
-/* Move src onto dst, replacing an existing dst; HTS_TRUE on success. Both
-   paths are fconv()'d. dst is removed only to make room for a src that exists,
-   so a caller whose src was never written keeps its dst; a retry that still
-   fails does not (#790). */
-hts_boolean hts_rename_over(const char *src, const char *dst);
+/* Move src onto dst, replacing an existing dst; HTS_TRUE on success. Both paths
+   are fconv()'d. A dst in the way is parked under a sibling name rather than
+   removed, so the old content survives a failure: back at dst, or under that
+   sibling (named in the log) when the move back failed too. Not atomic: a crash
+   between the two renames leaves dst absent and its content beside it. */
+hts_boolean hts_rename_over(httrackp *opt, const char *src, const char *dst);
+
+/* Selftest hook: run the aside fallback directly, on a platform whose rename()
+   never reaches it. Both paths are fconv()'d. */
+hts_boolean hts_rename_over_aside_selftest(httrackp *opt, const char *src,
+                                           const char *dst);
 
 #endif
 

src/htswarc.c

@@ -127,24 +127,32 @@ static void wbuf_free(wbuf *b) {
   b->len = b->cap = 0;
 }
 
-/* Append n bytes; returns 0 on success, -1 on OOM/overflow. */
-static int wbuf_add(wbuf *b, const void *p, size_t n) {
+/* Make room for n more bytes; returns 0 on success, -1 on OOM/overflow. */
+static int wbuf_reserve(wbuf *b, size_t n) {
+  size_t ncap;
+  char *nd;
   if (n > (size_t) -1 - b->len)
     return -1;
-  if (b->len + n > b->cap) {
-    size_t ncap = b->cap ? b->cap : 256;
-    char *nd;
-    while (ncap < b->len + n) {
-      if (ncap > (size_t) -1 / 2)
-        return -1;
-      ncap *= 2;
-    }
-    nd = realloct(b->data, ncap);
-    if (nd == NULL)
+  if (b->len + n <= b->cap)
+    return 0;
+  ncap = b->cap ? b->cap : 256;
+  while (ncap < b->len + n) {
+    if (ncap > (size_t) -1 / 2)
       return -1;
-    b->data = nd;
-    b->cap = ncap;
+    ncap *= 2;
   }
+  nd = realloct(b->data, ncap);
+  if (nd == NULL)
+    return -1;
+  b->data = nd;
+  b->cap = ncap;
+  return 0;
+}
+
+/* Append n bytes; returns 0 on success, -1 on OOM/overflow. */
+static int wbuf_add(wbuf *b, const void *p, size_t n) {
+  if (wbuf_reserve(b, n) != 0)
+    return -1;
   memcpy(b->data + b->len, p, n);
   b->len += n;
   return 0;
@@ -156,16 +164,32 @@ static int wbuf_puts(wbuf *b, const char *s) {
 
 static int wbuf_printf(wbuf *b, const char *fmt, ...) HTS_PRINTF_FUN(2, 3);
 
+/* A header line carrying a URL has no useful bound, and giving up here loses
+   the whole record, so anything past the stack buffer is formatted into the
+   wbuf itself rather than rejected (#785). */
 static int wbuf_printf(wbuf *b, const char *fmt, ...) {
   char tmp[1024];
+  size_t need;
   int n;
   va_list ap;
   va_start(ap, fmt);
   n = vsnprintf(tmp, sizeof(tmp), fmt, ap);
   va_end(ap);
-  if (n < 0 || (size_t) n >= sizeof(tmp))
+  if (n < 0)
     return -1;
-  return wbuf_add(b, tmp, (size_t) n);
+  if ((size_t) n < sizeof(tmp))
+    return wbuf_add(b, tmp, (size_t) n);
+  need = (size_t) n + 1; /* +1: vsnprintf always writes the NUL */
+  if (wbuf_reserve(b, need) != 0)
+    return -1;
+  va_start(ap, fmt);
+  n = vsnprintf(b->data + b->len, need, fmt, ap);
+  va_end(ap);
+  /* Never advance past what was reserved, whatever the second pass returns. */
+  if (n < 0 || (size_t) n >= need)
+    return -1;
+  b->len += (size_t) n; /* the NUL is scratch, overwritten by the next append */
+  return 0;
 }
 
 /* ---- gzip-per-record member writer (mirrors ae_write_packed) ---- */
@@ -1090,7 +1114,7 @@ static void warc_wacz_package(warc_writer *w) {
     hts_log_print(w->opt, LOG_WARNING,
                   "WACZ: packaging failed, kept existing %s untouched",
                   waczpath);
-  } else if (!hts_rename_over(tmppath, waczpath)) {
+  } else if (!hts_rename_over(w->opt, tmppath, waczpath)) {
     (void) UNLINK(fconv(catbuff, sizeof(catbuff), tmppath));
     hts_log_print(w->opt, LOG_WARNING | LOG_ERRNO,
                   "WACZ: could not finalize %s", waczpath);
@@ -1368,16 +1392,8 @@ warc_writer *warc_open(httrackp *opt, const char *path) {
     char ts[32];
     time_t t = time(NULL);
     struct tm tmv;
-#if defined(_WIN32)
-    struct tm *g = gmtime(&t);
-    if (g != NULL)
-      tmv = *g;
-    else
+    if (!hts_gmtime(t, &tmv))
       memset(&tmv, 0, sizeof(tmv));
-#else
-    if (gmtime_r(&t, &tmv) == NULL)
-      memset(&tmv, 0, sizeof(tmv));
-#endif
     strftime(ts, sizeof(ts), "%Y%m%d%H%M%S", &tmv);
     snprintf(catbuff, sizeof(catbuff), "httrack-%s.warc.gz", ts);
     path =
@@ -1565,7 +1581,7 @@ static hts_boolean warc_commit(warc_writer *w) {
     for (s = 0; s < nseg; s++) {
       const char *final = warc_seg_path(w, s, finalbuf, sizeof(finalbuf));
       snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX, final);
-      if (!hts_rename_over(tmpbuf, final)) {
+      if (!hts_rename_over(w->opt, tmpbuf, final)) {
         hts_log_print(w->opt, LOG_ERROR | LOG_ERRNO,
                       "WARC: could not replace %s", final);
         return HTS_FALSE;
@@ -1684,6 +1700,10 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
   if (is_update_unchanged) {
     is_revisit = 1;
     profile = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
+    /* Replay resolves a revisit by this field alone, and a 304 stands in for
+       the same URL. No WARC-Refers-To-Date to go with it: the cache keeps the
+       document's Last-Modified, never the previous capture time. */
+    refers_uri = target_uri;
     /* Served from cache: the payload sits in the previous archive, not here. */
     w->unbacked_revisits++;
   } else if (have_pdig && w->seen != NULL) {

src/proxy/proxytrack.c

@@ -520,12 +520,13 @@ static void proxytrack_add_DAV_Item(String * item, String * buff,
                                     const char *filename, size_t size,
                                     time_t timestamp, const char *mime,
                                     int isDir, int isRoot, int isDefault) {
-  struct tm *timetm;
+  struct tm timetmbuf;
+  struct tm *timetm = &timetmbuf;
 
   if (timestamp == (time_t) 0 || timestamp == (time_t) - 1) {
     timestamp = time(NULL);
   }
-  if ((timetm = gmtime(&timestamp)) != NULL) {
+  if (hts_gmtime(timestamp, timetm)) {
     char tms[256 + 1];
     const char *name;
 

src/proxy/proxytrack.h

@@ -360,21 +360,14 @@ HTS_UNUSED static struct tm *convert_time_rfc822(struct tm *result, const char *
 HTS_UNUSED static struct tm PT_GetTime(time_t t) {
   struct tm tmbuf;
 
-#ifdef _WIN32
-  struct tm *tm = gmtime(&t);
-#else
-  struct tm *tm = gmtime_r(&t, &tmbuf);
-#endif
-  if (tm != NULL)
-    return *tm;
-  else {
+  if (!hts_gmtime(t, &tmbuf)) {
     /* an all-zero tm has tm_mday == 0, which the ARC date field prints as a
        day of "00"; the epoch is the conventional "date unknown" */
     memset(&tmbuf, 0, sizeof(tmbuf));
     tmbuf.tm_year = 70;
     tmbuf.tm_mday = 1;
-    return tmbuf;
   }
+  return tmbuf;
 }
 HTS_UNUSED static int set_filetime(const char *file, struct tm *tm_time) {
   struct utimbuf tim;

src/proxy/store.c

@@ -888,6 +888,31 @@ static PT_Element PT_ReadCache__New(PT_Index index, const char *url, int flags)
 	} \
 } while(0)
 
+/* Set path (capacity size) to filename's parent directory, separator included,
+   from an absolute filename. Empty when there is none, or when it would not
+   fit: a truncated prefix names a different directory. */
+static void index_base_path(char *path, size_t size, const char *filename) {
+  const char *abpath;
+  int slashes;
+
+  for (slashes = 2, abpath = filename + (int) strlen(filename) - 1;
+       abpath > filename &&
+       ((*abpath != '/' && *abpath != '\\') || --slashes > 0);
+       abpath--)
+    ;
+  path[0] = '\0';
+  if (slashes == 0 && *abpath != 0 && (size_t) (abpath - filename) < size - 1) {
+    int i;
+
+    strlncatbuff(path, filename, size, (size_t) (abpath - filename) + 1);
+    for (i = 0; path[i] != 0; i++) {
+      if (path[i] == '\\') {
+        path[i] = '/';
+      }
+    }
+  }
+}
+
 int PT_LoadCache__New(PT_Index index_, const char *filename) {
   if (index_ != NULL && filename != NULL) {
     PT_Index__New index = &index_->slots.formatNew;
@@ -898,26 +923,9 @@ int PT_LoadCache__New(PT_Index index_, const char *filename) {
 
     // Opened ?
     if (zFile != NULL) {
-      const char *abpath;
-      int slashes;
       coucal hashtable = index->hash;
 
-      /* Compute base path for this index - the filename MUST be absolute! */
-      for(slashes = 2, abpath = filename + (int) strlen(filename) - 1;
-          abpath > filename && ((*abpath != '/' && *abpath != '\\')
-                                || --slashes > 0);
-          abpath--) ;
-      index->path[0] = '\0';
-      if (slashes == 0 && *abpath != 0) {
-        int i;
-
-        strncat(index->path, filename, (int) (abpath - filename) + 1);
-        for(i = 0; index->path[i] != 0; i++) {
-          if (index->path[i] == '\\') {
-            index->path[i] = '/';
-          }
-        }
-      }
+      index_base_path(index->path, sizeof(index->path), filename);
 
       /* Ready directory entries */
       if (unzGoToFirstFile(zFile) == Z_OK) {
@@ -1490,41 +1498,23 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
 
     cache->filenameDat[0] = '\0';
     cache->filenameNdx[0] = '\0';
-    cache->path[0] = '\0';
 
-    {
-      PT_Index__Old index = cache;
-      const char *abpath;
-      int slashes;
-
-      /* -------------------- COPY OF THE __New() CODE -------------------- */
-      /* Compute base path for this index - the filename MUST be absolute! */
-      for(slashes = 2, abpath = filename + (int) strlen(filename) - 1;
-          abpath > filename && ((*abpath != '/' && *abpath != '\\')
-                                || --slashes > 0);
-          abpath--) ;
-      index->path[0] = '\0';
-      if (slashes == 0 && *abpath != 0) {
-        int i;
-
-        strncat(index->path, filename, (int) (abpath - filename) + 1);
-        for(i = 0; index->path[i] != 0; i++) {
-          if (index->path[i] == '\\') {
-            index->path[i] = '/';
-          }
-        }
-      }
-      /* -------------------- END OF COPY OF THE __New() CODE -------------------- */
-    }
+    index_base_path(cache->path, sizeof(cache->path), filename);
 
     /* Index/data filenames */
     if (pos != NULL) {
-      int nLen = (int) (pos - filename);
+      const size_t nLen = (size_t) (pos - filename);
 
-      strncat(cache->filenameDat, filename, nLen);
-      strncat(cache->filenameNdx, filename, nLen);
-      strcat(cache->filenameDat, ".dat");
-      strcat(cache->filenameNdx, ".ndx");
+      /* a base clipped to fit would name a different pair of files */
+      if (nLen > sizeof(cache->filenameDat) - sizeof(".dat")) {
+        return 0;
+      }
+      strlncatbuff(cache->filenameDat, filename, sizeof(cache->filenameDat),
+                   nLen);
+      strlncatbuff(cache->filenameNdx, filename, sizeof(cache->filenameNdx),
+                   nLen);
+      strcatbuff(cache->filenameDat, ".dat");
+      strcatbuff(cache->filenameNdx, ".ndx");
     }
     ndxSize = filesize(cache->filenameNdx);
     cache->timestamp = file_timestamp(cache->filenameDat);
@@ -1587,7 +1577,10 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
               }
               /* read position */
               a += binput(a, linepos, 200);
-              sscanf(linepos, "%d", &pos);
+              /* an unparseable field must not carry the previous entry's
+                 offset over, nor read the stack on the first one */
+              if (sscanf(linepos, "%d", &pos) != 1)
+                pos = 0;
 
               /* Add entry */
               coucal_add(cache->hash, line, pos);

tests/01_engine-addlink.test

@@ -4,4 +4,5 @@
 
 set -euo pipefail
 
-httrack -O /dev/null -#test=addlink | grep -q "addlink self-test OK"
+out=$(httrack -O /dev/null -#test=addlink)
+grep -q "addlink self-test OK" <<<"$out"

tests/01_engine-backswap.test

@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# A ready slot still owning a temporary must stay in memory: swapping it out
+# clears the entry, which unlinks the re-fetch backup (#771).
+out=$(httrack -O /dev/null -#test=backswap)
+grep -q "backswap self-test: OK" <<<"$out"

tests/01_engine-changes.test

@@ -3,25 +3,13 @@
 
 set -euo pipefail
 
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_changes_st.XXXXXX") || exit 1
-trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
 
-# No pipe into grep: SIGPIPE would mask a failing exit status.
-expect_ok() {
-    local label="$1" out
-    shift
-    out=$("$@" 2>&1) || {
-        echo "FAIL: ${label} exited non-zero: ${out}"
-        exit 1
-    }
-    case "$out" in
-    *"${label}: OK"*) ;;
-    *)
-        echo "FAIL: ${out}"
-        exit 1
-        ;;
-    esac
-}
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_changes_st.XXXXXX") || exit 1
+trap 'set +e; rm -rf "$tmpdir"' EXIT
+trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
 
 # --changes bucket accounting and JSON escaping (#714).
 expect_ok "changes self-test" httrack -O "${tmpdir}/o1" -#test=changes run

tests/01_engine-cmdline-split.test

@@ -5,4 +5,5 @@ set -euo pipefail
 
 # webhttrack posts its command line as one string: the argv split must grow past
 # 1024 arguments and keep a quote inside a value out of the option parser.
-httrack -O /dev/null -#test=cmdline-split run | grep -q "cmdline-split self-test OK"
+out=$(httrack -O /dev/null -#test=cmdline-split run)
+grep -q "cmdline-split self-test OK" <<<"$out"

tests/01_engine-cmdline.test

@@ -11,7 +11,8 @@
 set -euo pipefail
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_cmdline.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 echo '<html><body>hello</body></html>' >"$tmp/index.html"
 

tests/01_engine-cookieimport.test

@@ -7,6 +7,7 @@ set -euo pipefail
 # cookies *@*.txt) from a long, non-ASCII folder through the UTF-8/long-path
 # file wrappers (#133,#630).
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
-httrack -O /dev/null -#test=cookieimport "$dir" | grep -q "cookieimport:.*OK"
+out=$(httrack -O /dev/null -#test=cookieimport "$dir")
+grep -q "cookieimport:.*OK" <<<"$out"

tests/01_engine-direnum.test

@@ -6,6 +6,7 @@ set -euo pipefail
 # Drives -#test=direnum: enumerate a long+non-ASCII directory through the
 # opendir/readdir wrappers, checking each child round-trips as UTF-8 (#133,#630).
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
-httrack -O /dev/null -#test=direnum "$dir" | grep -q "direnum:.*OK"
+out=$(httrack -O /dev/null -#test=direnum "$dir")
+grep -q "direnum:.*OK" <<<"$out"

tests/01_engine-doitlog.test

@@ -28,7 +28,8 @@ case "$bin" in
 esac
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_doitlog.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 site="$tmp/site"
 out="$tmp/out"

tests/01_engine-escape-room.test

@@ -4,4 +4,5 @@
 set -euo pipefail
 
 # HT_ADD_HTMLESCAPED* must reserve the escaper's worst case (6 for _full).
-httrack -O /dev/null -#test=escape-room run | grep -q "escape-room self-test OK"
+out=$(httrack -O /dev/null -#test=escape-room run)
+grep -q "escape-room self-test OK" <<<"$out"

tests/01_engine-filelist.test

@@ -8,7 +8,8 @@
 set -euo pipefail
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_filelist.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 echo '<html><body>hi</body></html>' >"$tmp/index.html"
 

tests/01_engine-footer-overflow.test

@@ -18,7 +18,7 @@ MINGW* | MSYS* | CYGWIN*) exit 77 ;;
 esac
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # A few-hundred-char file path (kept well under the URL length limit) makes the
 # {path} field long.
@@ -50,7 +50,7 @@ httrack "file://$deep/index.html" -O "$mir" -%F "$footer" -q -s0 -%v0 \
 
 # The crawled page must exist (proves the URL wasn't rejected for length, so the
 # footer path ran). Look under file/, not $mir, to skip the makeindex top index.
-find "$mir/file" -name index.html | grep -q . || {
+test -n "$(find "$mir/file" -name index.html)" || {
     echo "page not mirrored; the oversized-footer path was not exercised" >&2
     exit 1
 }

tests/01_engine-fsize.test

@@ -6,7 +6,7 @@
 set -euo pipefail
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 rc=0
 out=$(httrack -#test=fsize "$dir") || rc=$?

tests/01_engine-ftp-line.test

@@ -4,4 +4,5 @@
 set -euo pipefail
 
 # get_ftp_line bounds a hostile CRLF-less FTP reply into its 1024-byte buffer.
-httrack -O /dev/null -#test=ftp-line run | grep -q "ftp-line self-test OK"
+out=$(httrack -O /dev/null -#test=ftp-line run)
+grep -q "ftp-line self-test OK" <<<"$out"

tests/01_engine-ftp-userpass.test

@@ -4,4 +4,5 @@
 set -euo pipefail
 
 # ftp_split_userpass bounds an over-long user:pass@ from a hostile ftp:// URL.
-httrack -O /dev/null -#test=ftp-userpass run | grep -q "ftp-userpass self-test OK"
+out=$(httrack -O /dev/null -#test=ftp-userpass run)
+grep -q "ftp-userpass self-test OK" <<<"$out"

tests/01_engine-gmtime.test

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+# A non-UTC zone, so a helper that reached for localtime instead of gmtime gives
+# a different answer: CI runners are UTC, where the two agree.
+TZ=XXX5
+export TZ
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_gmtime_st.XXXXXX") || exit 1
+trap 'set +e; rm -rf "$tmpdir"' EXIT
+trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
+
+# Concurrent conversions do not tread on each other the way gmtime()'s shared
+# static does (#794).
+expect_ok "gmtime self-test" httrack -O "${tmpdir}/o1" -#test=gmtime

tests/01_engine-growsize.test

@@ -10,7 +10,8 @@ echo "$out"
 test "$out" == "growsize self-test OK"
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_growsize.XXXXXX")
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 echo '<html><body>hi</body></html>' >"$tmp/index.html"
 printf -- '-*/zzmarker*\n' >"$tmp/rules.txt"
@@ -18,7 +19,7 @@ printf -- '-*/zzmarker*\n' >"$tmp/rules.txt"
 # the rules file lands in the URL/filter string, echoed back by the banner
 run=$(httrack -O "$tmp/out" --quiet -n "-%S" "$tmp/rules.txt" \
     "file://$tmp/index.html" 2>&1) || true
-printf '%s\n' "$run" | grep -q 'zzmarker' || {
+grep -q 'zzmarker' <<<"$run" || {
     echo "FAIL: -%S rules file was not loaded"
     printf '%s\n' "$run"
     exit 1

tests/01_engine-hashtable.test

@@ -6,4 +6,4 @@ set -euo pipefail
 # httrack internal hashtable autotest on 100K keys. Assert the success line (on
 # stderr) so a misrouted registry entry can't pass on exit code alone.
 out=$(httrack -#test=hashtable 100000 2>&1)
-printf '%s\n' "$out" | grep -q "all hashtable tests were successful!" || exit 1
+grep -q "all hashtable tests were successful!" <<<"$out" || exit 1

tests/01_engine-inplace-escape.test

@@ -4,4 +4,5 @@
 set -euo pipefail
 
 # inplace_escape_*() must match escape_*() on a copy: guards the shared helper.
-httrack -O /dev/null -#test=inplace-escape run | grep -q "inplace-escape self-test OK"
+out=$(httrack -O /dev/null -#test=inplace-escape run)
+grep -q "inplace-escape self-test OK" <<<"$out"

tests/01_engine-logcallback.test

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# a log callback consumed the va_list the log file's vfprintf() still needed (#801).
+
+set -euo pipefail
+
+out=$(httrack -O /dev/null -#test=logcallback)
+grep -q "logcallback self-test OK" <<<"$out"

tests/01_engine-longpath-io.test

@@ -6,6 +6,7 @@ set -euo pipefail
 # Drives -#test=longpath: a >MAX_PATH round trip exercising hts_pathToUCS2's
 # \\?\ prefixing on Windows (#133); a positive control on POSIX.
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
-httrack -O /dev/null -#test=longpath "$dir" | grep -q "longpath:.*OK"
+out=$(httrack -O /dev/null -#test=longpath "$dir")
+grep -q "longpath:.*OK" <<<"$out"

tests/01_engine-makeindex.test

@@ -6,7 +6,7 @@ set -euo pipefail
 # hts_finish_makeindex writes the footer and gates the refresh meta on a single
 # first link (guards the macro->function extraction).
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=makeindex "$dir" run |
     grep -q "makeindex self-test OK"

tests/01_engine-mirror-io.test

@@ -7,6 +7,7 @@ set -euo pipefail
 # (>MAX_PATH) and non-ASCII, exercising the mirror I/O wrappers the engine's
 # raw file ops now route to on Windows (#133, #630). Positive control on POSIX.
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
-httrack -O /dev/null -#test=mirrorio "$dir" | grep -q "mirrorio:.*OK"
+out=$(httrack -O /dev/null -#test=mirrorio "$dir")
+grep -q "mirrorio:.*OK" <<<"$out"

tests/01_engine-parse.test

@@ -7,7 +7,8 @@
 set -euo pipefail
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_parse.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 # a minimal valid 1x1 GIF, reused for every referenced asset
 gif() {

tests/01_engine-rcfile.test

@@ -30,7 +30,8 @@ case "$bin" in
 esac
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_rcfile.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 # HTS_HTTRACKRC is ".httrackrc" on POSIX but "httrackrc" on Windows: write both,
 # each platform reads the one it knows.

tests/01_engine-reconcile.test

@@ -7,7 +7,7 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 out=$(httrack -#test=reconcile "$dir")
 

tests/01_engine-redirect.test

@@ -6,4 +6,5 @@ set -euo pipefail
 # #159: a redirect to a same-file alias (http<->https, user@host, ..) must be
 # followed through, not turned into a self-pointing "moved" stub. The decision
 # helper is exercised by the engine self-test.
-httrack -O /dev/null -#test=redirect-samefile run | grep -q "redirect-samefile self-test OK"
+out=$(httrack -O /dev/null -#test=redirect-samefile run)
+grep -q "redirect-samefile self-test OK" <<<"$out"

tests/01_engine-renameover.test

@@ -4,11 +4,11 @@
 set -euo pipefail
 
 # Drives -#test=renameover: hts_rename_over() must replace an existing dst, and
-# must leave dst alone when the rename failed for a reason removing dst cannot
-# fix (#779). The selftest prints the regime it detected; pin it per platform so
-# a leg cannot pass having tested the other half.
+# must leave dst alone when the rename failed for a reason moving dst aside
+# cannot fix (#779, #790). The selftest prints the regime and the restore
+# outcome it took; pin both per leg so none can pass having tested another.
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 case "$(uname -s)" in
 MINGW* | MSYS_NT*) want=fallback ;; # native rename() refuses an existing target
@@ -16,8 +16,9 @@ MINGW* | MSYS_NT*) want=fallback ;; # native rename() refuses an existing target
 esac
 
 out=$(httrack -O /dev/null -#test=renameover "$dir")
-echo "$out" | grep -q "renameover: OK"
-echo "$out" | grep -q "renameover: regime $want"
+grep -q "renameover: OK" <<<"$out"
+grep -q "renameover: regime $want" <<<"$out"
+grep -q "renameover: restore back" <<<"$out"
 
 if [ "$(uname -s)" != "Linux" ]; then
     echo "renameover: LD_PRELOAD interposition is Linux-only here, skipping"
@@ -42,14 +43,29 @@ fi
 # refuses by default. The shim allocates nothing, so the ordering is harmless.
 export ASAN_OPTIONS="${ASAN_OPTIONS:+$ASAN_OPTIONS:}verify_asan_link_order=0"
 
-# The unlink fallback is dead code on POSIX, so borrow Windows' rename().
+# The aside fallback is dead code on POSIX, so borrow Windows' rename().
 out=$(LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
     -#test=renameover "$dir")
-echo "$out" | grep -q "renameover: OK"
-echo "$out" | grep -q "renameover: regime fallback"
+grep -q "renameover: OK" <<<"$out"
+grep -q "renameover: regime fallback" <<<"$out"
+grep -q "renameover: restore back" <<<"$out"
+
+# #790: the move back out of the parked name fails once. Without the retry the
+# old copy stays parked and dst is left absent.
+out=$(RENAMEFAIL_ASIDE_FAILS=1 LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
+    -#test=renameover "$dir")
+grep -q "renameover: OK" <<<"$out"
+grep -q "renameover: restore back" <<<"$out"
+
+# It keeps failing: the old copy must survive under the parked name, never be
+# deleted, and the call must still report failure.
+out=$(RENAMEFAIL_ASIDE_FAILS=9 LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
+    -#test=renameover "$dir")
+grep -q "renameover: OK" <<<"$out"
+grep -q "renameover: restore parked" <<<"$out"
 
 # A source another process holds fails with EACCES, which dst had no part in.
 out=$(RENAMEFAIL_MODE=locked LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
     -#test=renameover "$dir")
-echo "$out" | grep -q "renameover: OK"
-echo "$out" | grep -q "renameover: regime refused"
+grep -q "renameover: OK" <<<"$out"
+grep -q "renameover: regime refused" <<<"$out"

tests/01_engine-robots.test

@@ -4,4 +4,5 @@
 set -euo pipefail
 
 # robots.txt RFC 9309 Allow/Disallow precedence (#452): longest match wins.
-httrack -O /dev/null -#test=robots run | grep -q "robots self-test OK"
+out=$(httrack -O /dev/null -#test=robots run)
+grep -q "robots self-test OK" <<<"$out"

tests/01_engine-savename.test

@@ -12,7 +12,7 @@ httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
 # scratch dir: body= and cached= write temp files (st-savename-body.tmp, hts-cache/)
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 cd "$scratch"
 
 run() {

tests/01_engine-selftest-dispatch.test

@@ -7,11 +7,11 @@ set -eu
 
 # Bare -#test lists known tests (printed to stderr).
 list=$(httrack -#test 2>&1)
-printf '%s\n' "$list" | grep -q "filter" || exit 1
-printf '%s\n' "$list" | grep -q "cache-writefail" || exit 1
+grep -q "filter" <<<"$list" || exit 1
+grep -q "cache-writefail" <<<"$list" || exit 1
 
 # Unknown name: non-zero exit + diagnostic, and no test result line.
 rc=0
 err=$(httrack -#test=bogus 2>&1) || rc=$?
 test "$rc" -ne 0 || exit 1
-printf '%s\n' "$err" | grep -q "Unknown self-test" || exit 1
+grep -q "Unknown self-test" <<<"$err" || exit 1

tests/01_engine-socks5.test

@@ -6,4 +6,5 @@ set -euo pipefail
 # The SOCKS5 handshake framing and credential split, driven against scripted
 # server replies (frame draining, oversize rejects, RFC 1929 fields).
 
-httrack -O /dev/null '-#test=socks5' | grep -q "socks5 self-test OK"
+out=$(httrack -O /dev/null '-#test=socks5')
+grep -q "socks5 self-test OK" <<<"$out"

tests/01_engine-status.test

@@ -4,4 +4,5 @@
 set -euo pipefail
 
 # HTTP status -> reason phrase, including the modern 429/451 (#453).
-httrack -O /dev/null -#test=status run | grep -q "status self-test OK"
+out=$(httrack -O /dev/null -#test=status run)
+grep -q "status self-test OK" <<<"$out"

tests/01_engine-stripquery.test

@@ -5,4 +5,5 @@ set -euo pipefail
 
 # --strip-query: pattern-scoped query-key stripping for dedup. All assertions
 # live in the engine self-test (hts_query_strip_keys + fil_normalized_filtered).
-httrack -O /dev/null -#test=stripquery | grep -q "strip-query self-test OK"
+out=$(httrack -O /dev/null -#test=stripquery)
+grep -q "strip-query self-test OK" <<<"$out"

tests/01_engine-structcheck.test

@@ -6,7 +6,7 @@
 set -euo pipefail
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=structcheck "$dir" |
     grep -q "structcheck self-test OK"

tests/01_engine-threadwait.test

@@ -3,25 +3,13 @@
 
 set -euo pipefail
 
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_threadwait_st.XXXXXX") || exit 1
-trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
 
-# No pipe into grep: SIGPIPE would mask a failing exit status.
-expect_ok() {
-    local label="$1" out
-    shift
-    out=$("$@" 2>&1) || {
-        echo "FAIL: ${label} exited non-zero: ${out}"
-        exit 1
-    }
-    case "$out" in
-    *"${label}: OK"*) ;;
-    *)
-        echo "FAIL: ${out}"
-        exit 1
-        ;;
-    esac
-}
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_threadwait_st.XXXXXX") || exit 1
+trap 'set +e; rm -rf "$tmpdir"' EXIT
+trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
 
 # A thread is outstanding from the moment hts_newthread() returns, so a wait
 # that follows the spawn joins it, and wait_n(n) still leaves n behind (#747).

tests/01_engine-topindex.test

@@ -6,7 +6,7 @@ set -euo pipefail
 # hts_buildtopindex takes a system-charset path but verif_backblue below it
 # expects utf-8, mangling a non-ASCII project dir on Windows (#216, #217).
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=topindex "$dir" run |
     grep -q "topindex self-test OK"

tests/01_engine-unescape-bounds.test

@@ -4,4 +4,5 @@
 set -euo pipefail
 
 # Entity/URL unescapers reserve one byte for the trailing NUL (no 1-byte OOB).
-httrack -O /dev/null -#test=unescape-bounds run | grep -q "unescape-bounds self-test OK"
+out=$(httrack -O /dev/null -#test=unescape-bounds run)
+grep -q "unescape-bounds self-test OK" <<<"$out"

tests/01_engine-urlhack.test

@@ -5,4 +5,5 @@ set -euo pipefail
 
 # -%u url-hack split (#271): www / // / query-order dedup toggle independently.
 # All assertions live in the engine self-test (hash compare flag resolution).
-httrack -O /dev/null -#test=urlhack run | grep -q "urlhack self-test OK"
+out=$(httrack -O /dev/null -#test=urlhack run)
+grep -q "urlhack self-test OK" <<<"$out"

tests/01_engine-useragent.test

@@ -4,4 +4,5 @@
 set -euo pipefail
 
 # Default User-Agent (#449): honest HTTrack token, no Windows 98 relic.
-httrack -O /dev/null -#test=useragent run | grep -q "useragent self-test OK"
+out=$(httrack -O /dev/null -#test=useragent run)
+grep -q "useragent self-test OK" <<<"$out"

tests/01_engine-warc-surt.test

@@ -5,4 +5,5 @@ set -euo pipefail
 
 # SURT canonicalization of the CDXJ sort key (--warc-cdx). Pure string work,
 # so it runs under the MSan-instrumented 01_engine glob.
-httrack -O /dev/null -#test=warc-surt | grep -q "warc-surt: OK"
+out=$(httrack -O /dev/null -#test=warc-surt)
+grep -q "warc-surt: OK" <<<"$out"

tests/01_zlib-acceptencoding.test

@@ -5,7 +5,7 @@ set -euo pipefail
 
 # Accept-Encoding (#450): advertise gzip+deflate; decode gzip/zlib/raw-deflate.
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=acceptencoding "$dir" run |
     grep -q "acceptencoding self-test OK"

tests/01_zlib-cache-corrupt.test

@@ -7,7 +7,7 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # the smashed-header case logs expected "Corrupted cache entry" warnings on
 # stdout; the verdict is the last line

tests/01_zlib-cache-golden.test

@@ -27,7 +27,7 @@ test -e "$fixture/hts-cache/new.zip" || {
 }
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # Read against a private copy so the source tree is never touched (a read
 # session does not write, but copying keeps the test hermetic). Create the dir

tests/01_zlib-cache-legacy.test

@@ -6,7 +6,7 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # the refusal errors land on stdout (no log file); pin them and the verdict
 out=$(httrack -#test=cache-legacy "$dir" 2>/dev/null)

tests/01_zlib-cache-writefail.test

@@ -1,8 +1,6 @@
 #!/bin/bash
 #
-# Keep this POSIX-portable: the harness runs it via $(BASH), which is a plain
-# POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms and GNU-only
-# tool flags despite the #!/bin/bash above.
+# Stick to POSIX tool flags: macOS ships BSD grep/sed, not the GNU ones.
 
 # Cache write-failure policy (-#test=cache-writefail <dir>). #174/#219: disk
 # full or a failure streak aborts cleanly; an isolated failure or an oversized
@@ -11,19 +9,19 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 out=$(httrack -#test=cache-writefail "$dir")
 
 # Match the exact success line (error logs also go to stdout); a renamed/removed
 # test prints the registry to stderr, which exits non-zero but never prints this.
-printf '%s\n' "$out" | grep -qx "cache-writefail: OK" || {
+grep -qx "cache-writefail: OK" <<<"$out" || {
     echo "expected 'cache-writefail: OK', got: $out" >&2
     exit 1
 }
 
 # A skipped entry must be warned about with its URL.
-printf '%s\n' "$out" | grep -q "entry not cached: example.com/" || {
+grep -q "entry not cached: example.com/" <<<"$out" || {
     echo "expected a URL-bearing skip warning" >&2
     exit 1
 }

tests/01_zlib-cache.test

@@ -18,7 +18,7 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # The working directory is a required argument; without it the test prints a
 # usage line to stderr and returns non-zero.

tests/01_zlib-contentcodings.test

@@ -5,7 +5,7 @@ set -euo pipefail
 
 # brotli/zstd decode, unknown codings, and the decoded-size budget.
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=contentcodings "$dir" run |
     grep -q "contentcodings self-test OK"

tests/01_zlib-repair-shift.test

@@ -1,8 +1,6 @@
 #!/bin/bash
 #
-# Keep this POSIX-portable: the harness runs it via $(BASH), which is a plain
-# POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms and GNU-only
-# tool flags despite the #!/bin/bash above.
+# Stick to POSIX tool flags: macOS ships BSD grep/sed, not the GNU ones.
 
 # unzRepair header read must not overflow a signed shift (-#test=zip-repair-shift
 # <dir>). A damaged local file header whose CRC high word has bit 15 set made
@@ -11,11 +9,11 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 out=$(httrack -#test=zip-repair-shift "$dir")
 
-printf '%s\n' "$out" | grep -qx "zip-repair-shift: OK (recovered 1 entry)" || {
+grep -qx "zip-repair-shift: OK (recovered 1 entry)" <<<"$out" || {
     echo "expected 'zip-repair-shift: OK (recovered 1 entry)', got: $out" >&2
     exit 1
 }

tests/01_zlib-savename-cached.test

@@ -10,7 +10,7 @@ set -euo pipefail
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 cd "$scratch"
 
 name() {

tests/01_zlib-warc-cdx.test

@@ -9,7 +9,7 @@ set -euo pipefail
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 
 out=$("$httrack_bin" -O /dev/null -#test=warc-cdx "$scratch/")
 echo "$out"

tests/01_zlib-warc-wacz.test

@@ -10,13 +10,14 @@ set -euo pipefail
 
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
-if ! "$httrack_bin" -#test 2>&1 | grep -q '^  warc-wacz'; then
+registry=$("$httrack_bin" -#test 2>&1 || true)
+if ! grep -q '^  warc-wacz' <<<"$registry"; then
     echo "warc-wacz self-test unavailable (build without OpenSSL); skipping"
     exit 77
 fi
 
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 
 out=$("$httrack_bin" -O /dev/null -#test=warc-wacz "$scratch/")
 echo "$out"

tests/01_zlib-warc.test

@@ -10,7 +10,7 @@ set -euo pipefail
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 
 for t in warc warc-trunc warc-ftp warc-rotate warc-verbatim; do
     out=$("$httrack_bin" -O /dev/null "-#test=$t" "$scratch/")

tests/02_manpage-regen.test

@@ -29,7 +29,7 @@ command -v httrack >/dev/null 2>&1 || {
 tmp=$(mktemp) || exit 1
 committed_clean=$(mktemp) || exit 1
 generated_clean=$(mktemp) || exit 1
-trap 'rm -f "$tmp" "$committed_clean" "$generated_clean"' EXIT
+trap 'set +e; rm -f "$tmp" "$committed_clean" "$generated_clean"' EXIT
 
 README="$top_srcdir/README" bash "$gen" httrack >"$tmp" 2>/dev/null || {
     echo "makeman.sh failed" >&2

tests/02_update-cache.test

@@ -14,7 +14,7 @@ set -euo pipefail
 
 site=$(mktemp -d)
 out=$(mktemp -d)
-trap 'rm -rf "$site" "$out"' EXIT
+trap 'set +e; rm -rf "$site" "$out"' EXIT
 
 cat >"$site/index.html" <<EOF
 <a href="a.html">a</a> <a href="sub/b.html">b</a>

tests/100_local-purge-longpath.test

@@ -30,7 +30,7 @@ test -n "$httrack_bin" || {
 httrack_bin=$(cd "$(dirname "$httrack_bin")" && pwd)/$(basename "$httrack_bin")
 
 work=$(mktemp -d)
-trap 'kill -9 "${spid:-}" 2>/dev/null || true; rm -rf "$work"' EXIT
+trap 'set +e; kill -9 "${spid:-}" 2>/dev/null || true; rm -rf "$work"' EXIT
 
 mkdir -p "$work/root" "$work/proj"
 

tests/102_local-ftp-refetch.test

@@ -0,0 +1,166 @@
+#!/bin/bash
+#
+# An FTP re-fetch used to truncate the mirror before the transfer, so a failed
+# one destroyed the previous copy (#771). keep.bin is cut short and empty.bin
+# gets nothing; both must keep their old bytes. stay.bin takes the same path and
+# completes, so a fix that merely stopped writing would fail here.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+out="${tmpdir}/crawl"
+mode="${tmpdir}/mode"
+report="${out}/hts-changes.json"
+mkdir -p "$root" "$out"
+
+# The two generations differ in length, or a re-fetch that resumed at EOF
+# instead of truncating would leave the same bytes and pass for the wrong reason.
+write_bodies() {
+    local gen="$1" fill="$2" name
+    for name in keep empty stay; do
+        "$python" -c 'import sys; sys.stdout.buffer.write(
+            ("%s-FTP-%s " % (sys.argv[1].upper(), sys.argv[2])).encode()
+            + sys.argv[2][-1].encode() * int(sys.argv[3]))' "$name" "$gen" "$fill" \
+            >"${root}/${name}.bin"
+    done
+}
+write_bodies V1 4096
+: >"$mode"
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" \
+    --mode-file "$(nativepath "$mode")" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 300); do
+    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "ftp server exited early: $(cat "$serverlog")" >&2
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
+    exit 1
+}
+host="127.0.0.1_${port}"
+urls=()
+for name in keep empty stay; do
+    urls+=("ftp://127.0.0.1:${port}/${name}.bin")
+done
+# -c1: a parallel FTP crawl loses whole transfers to a separate, older bug in
+# the backlog slot swap (#797), which would flake this test on a loaded runner.
+common=(--quiet --disable-security-limits --robots=0 --timeout=20 --max-time=120
+    --retries=1 -c1 --changes)
+
+fail() {
+    echo "FAIL: $*" >&2
+    test ! -f "$report" || cat "$report" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+size_of() { wc -c <"$1" | tr -d '[:space:]'; }
+# Files listed under a report bucket, one "file:size" per line.
+listed() {
+    "$python" - "$report" "$1" <<'EOF' | tr -d '\r'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as fp:
+    report = json.load(fp)
+for entry in report[sys.argv[2]]:
+    print("%s:%s" % (entry["file"], entry.get("size", "none")))
+EOF
+}
+
+# --- pass 1: a healthy mirror ------------------------------------------------
+# Bounded like every local-crawl.sh pass: a wedged crawl must fail the test, not
+# hang the job until CI cancels it and discards the log (#796).
+run_with_timeout 300 httrack "${urls[@]}" -O "$out" "${common[@]}" \
+    >"${tmpdir}/log1" 2>&1
+for name in keep empty stay; do
+    test -s "${out}/${host}/${name}.bin" || fail "pass 1 did not mirror ${name}.bin"
+done
+mkdir "${tmpdir}/snap"
+cp "${out}/${host}"/*.bin "${tmpdir}/snap/"
+ok "pass 1 mirrored three files"
+
+# --- pass 2: the same three re-fetched, two of them failing -------------------
+# norest throughout: an accepted REST sends the client down the append path,
+# which is not the one that truncates.
+write_bodies V2 6000
+cat >"$mode" <<'EOF'
+/keep.bin truncate norest
+/empty.bin empty norest
+/stay.bin norest
+EOF
+run_with_timeout 300 httrack "${urls[@]}" -O "$out" "${common[@]}" --update \
+    >"${tmpdir}/log2" 2>&1
+
+# Both failures have to be the transfer dying mid-body: a connect or lookup
+# failure never opens the file, and everything below would pass vacuously.
+for name in keep empty; do
+    grep -aqE "Error:.*\"FTP file incomplete\".*${name}\.bin" "${out}/hts-log.txt" ||
+        fail "${name}.bin did not fail during the transfer"
+done
+ok "both re-fetches reached the body and failed there"
+
+for name in keep empty; do
+    cmp -s "${out}/${host}/${name}.bin" "${tmpdir}/snap/${name}.bin" ||
+        fail "${name}.bin differs from the copy pass 1 left ($(size_of \
+            "${out}/${host}/${name}.bin") bytes now)"
+done
+ok "a failed FTP transfer left the previously mirrored bytes alone"
+
+grep -aq "FTP-V2 " "${out}/${host}/stay.bin" ||
+    fail "stay.bin was not refreshed; the re-fetch writes nothing at all now"
+ok "a successful FTP re-fetch still replaces the file"
+
+leftover=$(find "${out}/${host}" -name '*.bak')
+test -z "$leftover" || fail "backup left behind: ${leftover}"
+ok "no backup temporary survived the pass"
+
+# --purge-old is on by default: a kept file the run never replaced must still be
+# in new.lst, or it is deleted as gone (#746).
+for name in keep empty stay; do
+    test -f "${out}/${host}/${name}.bin" || fail "${name}.bin was purged"
+done
+ok "the kept copies survived the update purge"
+
+# --- the change report is the mirror's own account of the pass ----------------
+test -s "$report" || fail "pass 2 wrote no ${report}"
+test -z "$(listed gone)" || fail "the report calls something gone: $(listed gone)"
+unchanged=$(listed unchanged)
+changed=$(listed changed)
+for name in keep empty; do
+    grep -qx "${host}/${name}.bin:$(size_of "${tmpdir}/snap/${name}.bin")" <<<"$unchanged" ||
+        fail "${name}.bin is not reported unchanged at its previous size"
+    if grep -q "^${host}/${name}.bin:" <<<"$changed"; then
+        fail "${name}.bin is reported changed as well"
+    fi
+done
+grep -qx "${host}/stay.bin:$(size_of "${out}/${host}/stay.bin")" <<<"$changed" ||
+    fail "stay.bin is not reported changed"
+ok "the change report calls the kept files unchanged and the refreshed one changed"

tests/103_teardown-status.test

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Issue #773: a test must fail only when an assertion fails. Under "set -e" a
+# failing command in an EXIT trap becomes the script's exit status, so a
+# teardown hiccup turned a test whose assertions all passed into a red CI leg.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+tmp=$(mktemp -d)
+trap 'set +e; rm -rf "$tmp"' EXIT
+
+fail() {
+    echo "FAIL: $1" >&2
+    exit 1
+}
+
+# run <trap-body> <script-body>: exit status and stderr of a subject script.
+run() {
+    {
+        echo "set -euo pipefail"
+        # printf, so the scanner below does not read this as a trap of our own
+        printf 'trap %s EXIT\n' "$1"
+        echo "$2"
+    } >"$tmp/subject"
+    rc=0
+    err=$(bash "$tmp/subject" 2>&1 >/dev/null) || rc=$?
+}
+
+# 1. the shape in the tree: teardown fails, every assertion passed -> pass
+run "'set +e; false'" "true"
+test "$rc" -eq 0 || fail "guarded teardown failure exited $rc, want 0"
+
+# 2. teeth for case 1: unguarded, the same teardown is what went red
+run "'false'" "true"
+test "$rc" -eq 1 || fail "unguarded teardown failure exited $rc, want 1"
+
+# 3. the guard must not swallow a real failure, nor a skip
+run "'set +e; false'" "exit 1"
+test "$rc" -eq 1 || fail "assertion failure exited $rc, want 1"
+run "'set +e; false'" "exit 77"
+test "$rc" -eq 77 || fail "skip exited $rc, want 77"
+
+# 4. not failing must not mean going quiet: teardown still reports why
+run "'set +e; rm /nonexistent-teardown-probe'" "true"
+test "$rc" -eq 0 || fail "guarded teardown failure exited $rc, want 0"
+case "$err" in
+*nonexistent-teardown-probe*) ;;
+*) fail "teardown error was silenced (stderr: '$err')" ;;
+esac
+echo "OK: teardown cannot decide the verdict, and still reports"
+
+# 5. no test may reintroduce the shape case 2 pins. Two spellings neutralise
+# errexit for teardown: "set +e" first, or "|| true" over the whole body.
+scanned=0
+for f in "$top_srcdir"/tests/*.test "$top_srcdir"/tests/*.sh; do
+    test -r "$f" || continue
+    scanned=$((scanned + 1))
+done
+# an unexpanded glob would report no offender having read nothing
+test "$scanned" -ge 50 || fail "scanned $scanned scripts in $top_srcdir/tests, want the whole suite"
+
+bad=$(awk '
+    FNR == 1 { errexit = 0 }
+    /^[[:space:]]*set[[:space:]]+(-[a-z]*e|-o[[:space:]]+errexit)/ { errexit = 1 }
+    !errexit { next }
+    !/^[[:space:]]*trap[[:space:]]+.*[[:space:]]EXIT([[:space:]]|;|$)/ { next }
+    {
+        guarded = /^[[:space:]]*trap[[:space:]]+.set \+e; / || /\|\|[[:space:]]+true.[[:space:]]+EXIT/
+        if (!guarded)
+            print FILENAME ":" FNR " unguarded"
+        # "set +e" on a trap shared with a signal leaves errexit off for the
+        # rest of the run, so a torn-down test could still report success
+        else if (/^[[:space:]]*trap[[:space:]]+.set \+e; / && /[[:space:]]EXIT[[:space:]]+[A-Z]/)
+            print FILENAME ":" FNR " set +e shared with a signal trap"
+    }
+' "$top_srcdir"/tests/*.test "$top_srcdir"/tests/*.sh)
+test -z "$bad" || fail "EXIT trap can decide the verdict:
+$bad"
+echo "OK: every EXIT trap under set -e runs teardown with errexit off"

tests/104_engine-warc-longurl.test

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# A URL past the header-format buffer must still produce a record: the archive
+# used to lose it whole, with no error and an exit status of 0 (#785).
+
+set -e
+
+: "${top_srcdir:=..}"
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_longurl.XXXXXX")
+trap 'set +e; rm -rf "$tmp"' EXIT
+
+httrack -O "$tmp/out" "-#test=warc-longurl" "$tmp"

tests/105_suite-timeout.test

@@ -0,0 +1,141 @@
+#!/bin/bash
+#
+# The harness timeout (tests/test-timeout.sh) must name the test it killed, say
+# what was still running, and exit 124 -- otherwise a wedge only ever shows up as
+# a cancelled CI step, which keeps no log and no artifact.
+
+set -eu
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+driver="${testdir}/test-timeout.sh"
+test -r "$driver" || {
+    echo "no test-timeout.sh next to $0" >&2
+    exit 1
+}
+
+fail() {
+    echo "FAIL: $*" >&2
+    test -z "${out:-}" || sed 's/^/  | /' <"$out" >&2
+    exit 1
+}
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_sutmo.XXXXXX") || exit 1
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
+out="$tmp/out"
+
+# --- a hanging test is killed, named, and reported 124 ----------------------
+cat >"$tmp/90_wedged.test" <<'EOF'
+echo "marker: the wedged test started"
+sleep 300
+EOF
+
+start=$SECONDS
+rc=0
+HTTRACK_TEST_TIMEOUT=5 bash "$driver" "$tmp/90_wedged.test" >"$out" 2>&1 || rc=$?
+elapsed=$((SECONDS - start))
+
+test "$rc" -eq 124 || fail "wedged test reported $rc, want 124"
+# Never before the budget, or a slow-but-healthy test would be killed too.
+test "$elapsed" -ge 5 || fail "the guard fired early (${elapsed}s of a 5s budget)"
+test "$elapsed" -lt 30 || fail "the guard fired late (${elapsed}s)"
+grep -q 'marker: the wedged test started' "$out" || fail "the test's own output was lost"
+# The header, not a bare name: the process list quotes the test's path too, so a
+# wrapper that named nothing would still match that.
+grep -q '^===== TIMEOUT: 90_wedged.test exceeded' "$out" ||
+    fail "the diagnostics do not name the test"
+grep -q "own process tree" "$out" || fail "no process list in the diagnostics"
+
+# The budget is read, not hard-coded: well under it, the same shape survives.
+printf 'sleep 3\necho "slow but healthy"\n' >"$tmp/92_slow.test"
+rc=0
+HTTRACK_TEST_TIMEOUT=45 bash "$driver" "$tmp/92_slow.test" >"$out" 2>&1 || rc=$?
+test "$rc" -eq 0 || fail "a healthy 3s test under a 45s budget reported $rc"
+grep -q 'slow but healthy' "$out" || fail "healthy test output lost"
+! grep -q 'TIMEOUT' "$out" || fail "the guard fired on a healthy test"
+
+# The killed tree must really be gone, or the next test inherits its ports.
+sleep 1
+! grep -q "$tmp/90_wedged.test" <<<"$(ps -A -o args 2>/dev/null)" ||
+    fail "the wedged test survived the guard"
+
+# --- exit status and output of a healthy test pass straight through ----------
+# 77 is automake's SKIP and 99 its hard error, so both have to survive the wrapper.
+for want in 0 1 77 99; do
+    printf 'echo "ran with %s"\nexit %s\n' "$want" "$want" >"$tmp/91_plain.test"
+    rc=0
+    HTTRACK_TEST_TIMEOUT=60 bash "$driver" "$tmp/91_plain.test" >"$out" 2>&1 || rc=$?
+    test "$rc" -eq "$want" || fail "exit $want came back as $rc"
+    grep -q "ran with $want" "$out" || fail "output lost for exit $want"
+    ! grep -q 'TIMEOUT' "$out" || fail "the guard fired on a test that exited at once"
+done
+
+# A budget of 0 disables the guard: the test still runs, and is not killed. Its
+# own file, so the assertion cannot ride on whatever the loop above left behind.
+printf 'echo "guard off"\nexit 3\n' >"$tmp/93_off.test"
+rc=0
+HTTRACK_TEST_TIMEOUT=0 bash "$driver" "$tmp/93_off.test" >"$out" 2>&1 || rc=$?
+test "$rc" -eq 3 || fail "disabled guard changed the exit status ($rc)"
+grep -q 'guard off' "$out" || fail "output lost with the guard disabled"
+
+# --- a wedged crawl yields a symbolized engine stack ------------------------
+# The whole point of the dump: name the frame the engine is stuck in. Windows has
+# neither half (MSYS signals do not reach a native httrack.exe, and that build has
+# no backtrace()), so only assert it where the machinery exists.
+if is_windows; then
+    echo "suite timeout OK (no stack capture on Windows)"
+    exit 0
+fi
+command -v httrack >/dev/null || {
+    echo "SKIP: no httrack on PATH for the stack half" >&2
+    exit 77
+}
+
+# /watchdog/stall never answers, so the crawl wedges for real. CRAWL_DEADLINE is
+# raised past our budget so the guard under test is what fires, not local-crawl's.
+cat >"$tmp/92_stalled.test" <<EOF
+CRAWL_DEADLINE=600 bash "${testdir}/local-crawl.sh" --errors 0 \\
+    httrack 'BASEURL/watchdog/stall'
+EOF
+
+rc=0
+HTTRACK_TEST_TIMEOUT=20 bash "$driver" "$tmp/92_stalled.test" >"$out" 2>&1 || rc=$?
+
+if grep -qE 'python3 not found|could not discover server port' "$out"; then
+    echo "SKIP: local crawl prerequisites missing" >&2
+    exit 77
+fi
+test "$rc" -eq 124 || fail "stalled crawl reported $rc, want 124"
+
+if grep -q 'No stack trace available on this OS' "$out"; then
+    echo "suite timeout OK (no backtrace() on this platform)"
+    exit 0
+fi
+
+# Each platform's mechanism has its own marker, and the marker is what proves the
+# whole chain connected up rather than the dump merely running.
+case "$(uname -s)" in
+Linux)
+    # sig_fatal writes to the engine's own stderr, so this reaching our log also
+    # proves the crawl-log salvage worked.
+    grep -q '^Caught signal ' "$out" || fail "no engine backtrace was salvaged"
+    ;;
+Darwin)
+    grep -q '^Call graph:' "$out" || fail "sample(1) produced no call graph"
+    ;;
+*)
+    grep -q 'no stack mechanism known' "$out" ||
+        fail "unknown platform reported neither a stack nor its absence"
+    echo "suite timeout OK (no stack mechanism on $(uname -s))"
+    exit 0
+    ;;
+esac
+# A stalled crawl is necessarily inside httpmirror, so these frames must appear.
+# Matching "httrack" alone would pass on the process list above.
+grep -qE '\b(back_wait|httpmirror|hts_main_internal)\b' "$out" ||
+    fail "the stack names no engine frame"
+
+echo "suite timeout OK (named the test, dumped a stack, exited 124)"

tests/110_local-ftp-parallel.test

@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+# A completed FTP transfer used to end as a 0-byte file when its ready backlog
+# slot was swapped to the on-disk table (#797): the FTP worker writes url_sav
+# itself and leaves no in-memory body, and the swap round-trip zeroed the slot's
+# size, so the link writer then created an empty file over the bytes.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+out="${tmpdir}/crawl"
+mkdir -p "$root" "$out"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+size_of() { wc -c <"$1" | tr -d '[:space:]'; }
+
+# Distinct bodies and lengths, so a mirror that mixed two of them up cannot
+# compare equal.
+names=""
+for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
+    names="${names} f${i}"
+    "$python" -c 'import sys; sys.stdout.buffer.write(
+        ("BODY-%s " % sys.argv[1]).encode() + sys.argv[1].encode() * (400 * int(sys.argv[1])))' \
+        "$i" >"${root}/f${i}.bin"
+done
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 300); do
+    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "ftp server exited early: $(cat "$serverlog")" >&2
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
+    exit 1
+}
+host="127.0.0.1_${port}"
+
+urls=()
+for name in $names; do
+    urls+=("ftp://127.0.0.1:${port}/${name}.bin")
+done
+
+# -c8, never -c1: a slot is only swapped out while other transfers keep the
+# parser from picking it up. -Z because the swap only logs at debug level, and
+# the run below proves nothing unless it happened. Bounded like every crawl
+# pass: a wedge must fail the test, not hang the job until CI discards the log.
+run_with_timeout 300 httrack "${urls[@]}" -O "$out" --quiet -Z -c8 \
+    --disable-security-limits --robots=0 --timeout=20 --max-time=120 --retries=1 \
+    >"${tmpdir}/log" 2>&1
+
+swapped=$(grep -ac 'slots ready moved to background' "${out}/hts-log.txt" || true)
+test "${swapped:-0}" -gt 0 ||
+    fail "no ready slot was swapped to disk; this run never reached the bug"
+ok "the crawl swapped ready slots to the on-disk table ${swapped} times"
+
+for name in $names; do
+    test -f "${out}/${host}/${name}.bin" || fail "${name}.bin was not mirrored"
+    cmp -s "${out}/${host}/${name}.bin" "${root}/${name}.bin" ||
+        fail "${name}.bin has $(size_of "${out}/${host}/${name}.bin") of the" \
+            "$(size_of "${root}/${name}.bin") bytes served"
+done
+ok "every FTP transfer landed on disk intact"
+
+errors=$(grep -a 'Error:' "${out}/hts-log.txt" || true)
+test -z "$errors" || fail "the crawl reported errors: ${errors}"
+ok "no transfer was reported failed"

tests/111_local-ftp-update-rest.test

@@ -0,0 +1,125 @@
+#!/bin/bash
+#
+# FTP resumed with REST whenever the mirrored file merely existed, so an
+# --update over a complete copy spliced the old body into the new one and the
+# result matched the remote length (#798). Pass 1 is cut short to leave a real
+# partial, pass 2 must still resume it, pass 3 must not.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+out="${tmpdir}/crawl"
+mode="${tmpdir}/mode"
+cmds="${tmpdir}/cmds"
+mkdir -p "$root" "$out"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+size_of() { wc -c <"$1" | tr -d '[:space:]'; }
+# hts-log.txt is rewritten per pass, so this only ever sees the pass just run.
+no_errors() {
+    local errors
+    errors=$(grep -a 'Error:' "${out}/hts-log.txt" || true)
+    test -z "$errors" || fail "$1 reported errors: ${errors}"
+}
+# Commands the server saw since the last pass; each pass starts by emptying it.
+sent() { cat "$cmds"; }
+
+# The two generations differ in length and in their first bytes, so a splice
+# shows up whichever end of the file it lands in.
+write_body() {
+    "$python" -c 'import sys; sys.stdout.buffer.write(
+        ("%s-BODY " % sys.argv[1]).encode() + sys.argv[1][:1].encode() * int(sys.argv[2]))' \
+        "$1" "$2" >"${root}/a.bin"
+}
+write_body OLD 8000
+echo '/a.bin truncate' >"$mode"
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" \
+    --mode-file "$(nativepath "$mode")" --log "$(nativepath "$cmds")" \
+    >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 300); do
+    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "ftp server exited early: $(cat "$serverlog")" >&2
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
+    exit 1
+}
+host="127.0.0.1_${port}"
+url="ftp://127.0.0.1:${port}/a.bin"
+mirror="${out}/${host}/a.bin"
+# Bounded like every crawl pass: a wedge must fail the test, not hang the job.
+crawl() { run_with_timeout 300 httrack "$url" -O "$out" --quiet \
+    --disable-security-limits --robots=0 --timeout=20 --max-time=120 --retries=1 \
+    -c1 "$@"; }
+
+# --- pass 1: the transfer dies mid-body and leaves a partial -----------------
+crawl >"${tmpdir}/log1" 2>&1 || true
+test -f "$mirror" || fail "pass 1 mirrored nothing to resume"
+partial=$(size_of "$mirror")
+whole=$(size_of "${root}/a.bin")
+if test "$partial" -le 0 || test "$partial" -ge "$whole"; then
+    fail "pass 1 left ${partial} bytes, wanted a partial of ${whole}"
+fi
+ok "pass 1 left a ${partial}-byte partial copy"
+
+# --- pass 2: the partial is resumed ------------------------------------------
+: >"$mode"
+: >"$cmds"
+crawl >"${tmpdir}/log2" 2>&1
+case "$(sent)" in
+*"REST ${partial}"*) ok "pass 2 resumed at the partial's ${partial} bytes" ;;
+*) fail "pass 2 sent no REST ${partial}; commands were: $(sent)" ;;
+esac
+cmp -s "$mirror" "${root}/a.bin" ||
+    fail "the resumed copy is $(size_of "$mirror") bytes, served $(size_of "${root}/a.bin")"
+# A resume used to count only the appended tail against the remote size, so it
+# was reported "FTP file incomplete" even when it produced the whole file.
+no_errors "pass 2"
+ok "the resumed transfer produced the served body and no error"
+
+# --- pass 3: --update over that complete copy --------------------------------
+write_body NEW 12000
+: >"$cmds"
+crawl --update >"${tmpdir}/log3" 2>&1
+case "$(sent)" in
+*REST*) fail "the update resumed a complete mirror: $(sent)" ;;
+*RETR*) ok "the update re-fetched without REST" ;;
+*) fail "pass 3 never reached RETR; commands were: $(sent)" ;;
+esac
+cmp -s "$mirror" "${root}/a.bin" ||
+    fail "the updated copy is $(size_of "$mirror") bytes, served $(size_of "${root}/a.bin")"
+no_errors "pass 3"
+ok "the update replaced the mirror with the new body"

tests/114_local-update-304-leak.test

@@ -0,0 +1,108 @@
+#!/bin/bash
+#
+# Issue #782: a 304 during --update overwrote the whole response struct with the
+# cache entry, dropping the 8 KB header buffer it owned. Only a leak checker
+# sees it and the sanitized CI leg runs with detect_leaks off, so this one turns
+# it back on around a two-pass mini304 crawl; the fresh pass is the control.
+
+set -eu
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+server=$(nativepath "${testdir}/local-server.py")
+root=$(nativepath "${testdir}/server-root")
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+
+probe=$(ASAN_OPTIONS=help=1 httrack -O /dev/null --version 2>&1 || true)
+case "$probe" in
+*AddressSanitizer*) ;;
+*)
+    echo "not a sanitized build, skipping" >&2
+    exit 77
+    ;;
+esac
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_304leak.XXXXXX") || exit 1
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+serverlog="${tmpdir}/server.log"
+: >"$serverlog"
+"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$serverlog" 2>/dev/null)
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$serverlog")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+
+out="${tmpdir}/crawl"
+mkdir "$out"
+url="http://127.0.0.1:${port}/mini304/index.html"
+
+# Last value wins in the sanitizer flag parser, so this overrides the job's
+# settings; the report file, not the exit status, is the verdict.
+asan_base="${ASAN_OPTIONS:+$ASAN_OPTIONS:}detect_leaks=1:abort_on_error=0:exitcode=0"
+
+# run_pass <label> <extra httrack args...>
+run_pass() {
+    local label="$1"
+    shift
+    rm -f "${tmpdir}/asan".*
+    printf '[%s] ..\t' "$label"
+    ASAN_OPTIONS="${asan_base}:log_path=${tmpdir}/asan" \
+        httrack -O "$out" --quiet --disable-security-limits --robots=0 \
+        --timeout=30 --retries=1 -c1 "$@" "$url" >"${tmpdir}/log_${label}" 2>&1 || true
+    report=$(cat "${tmpdir}/asan".* 2>/dev/null || true)
+    test -z "$report" || {
+        echo "FAIL: sanitizer report on the ${label} pass"
+        echo "$report"
+        exit 1
+    }
+    echo "OK (clean)"
+}
+
+run_pass fresh
+run_pass update --update
+# only --warc-file stashes the two header blocks, so the archive limb of the fix
+# runs on this pass alone
+run_pass warc --update --warc-file leak
+
+# Teeth: the update passes must have gone through the 304 path, or a run that
+# re-downloaded everything would report clean having tested nothing.
+printf '[update used the cache] ..\t'
+if grep -aqE "no files updated" "${out}/hts-log.txt"; then
+    echo "OK"
+else
+    echo "FAIL: the update pass did not report unchanged files"
+    exit 1
+fi
+
+# ... and the WARC pass must have written an archive, or it stashed nothing.
+printf '[warc pass archived] ..\t'
+if [ -n "$(find "$out" -name 'leak*.warc*' -size +0c 2>/dev/null)" ]; then
+    echo "OK"
+else
+    echo "FAIL: the --warc-file pass produced no archive"
+    exit 1
+fi

tests/119_local-proxytrack-ndx-fields2.test

@@ -0,0 +1,155 @@
+#!/bin/bash
+# Two bounds in the .ndx/.dat reader: a position field that did not parse left
+# "pos" holding the previous entry's offset, or the stack on the first entry,
+# and the base path and index filenames were copied with strncat calls bounded
+# by the source path instead of by their 1024-byte destinations.
+
+set -euo pipefail
+export LC_ALL=C
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+dir=$(mktemp -d)
+cleanup() { rm -rf "$dir"; }
+trap 'set +e; cleanup' EXIT
+trap 'set +e; cleanup; exit 1' HUP INT TERM
+
+fail() {
+    echo "FAIL: $1" >&2
+    exit 1
+}
+
+enc() { printf '%d\n%s' "${#1}" "$1"; }
+
+# One version-1.4 .dat record in cache_rstr() framing, field by field:
+# $1 content-type, $2 status message, $3 body.
+record() {
+    enc 200
+    enc "${#3}"
+    enc "$2"
+    enc "$1"
+    enc ""
+    enc ""
+    enc ""
+    enc ""
+    enc ""
+    enc ""
+    enc ""
+    enc ""
+    enc HTS
+    enc "${#3}"
+    printf '%s' "$3"
+}
+
+# Two records in $1.dat, and in $1.ndx two entries with positions $2 and $3.
+# The parser walks the index a line at a time and skips one line per entry, so
+# each block leads with a filler line.
+build_cache() {
+    record audio/midi ZERO ZEROBODY >"$dir/r0"
+    record image/gif ALPHA ALPHABODY >"$dir/ra"
+    cat "$dir/r0" "$dir/ra" >"$1.dat"
+    {
+        enc CACHE-1.4
+        enc "Wed, 01 Jan 2025 00:00:00 GMT"
+        printf 'X\nexample.com/a.html\n\n%s\n' "$2"
+        printf 'X\nexample.com/b.html\n\n%s\n' "$3"
+    } >"$1.ndx"
+}
+
+# Content-type proxytrack wrote for URL $2 of the .arc $1, or "none".
+mime_of() {
+    local rec
+    rec=$(grep -a -m1 "^http://$2 " "$1") || {
+        echo none
+        return
+    }
+    awk '{ print $4 }' <<<"$rec"
+}
+
+# Offset of the second record: the position an entry must not inherit.
+record audio/midi ZERO ZEROBODY >"$dir/r0"
+posA=$(wc -c <"$dir/r0")
+
+convert() {
+    proxytrack --convert "$dir/out.arc" "$1" >"$dir/log" 2>&1 ||
+        fail "$2: proxytrack died on the cache"
+}
+
+# Control: both positions parse, so each URL resolves to its own record and a
+# broken probe cannot pass as a fixed reader.
+build_cache "$dir/ctl" 0 "$posA"
+convert "$dir/ctl.ndx" "control"
+test "$(mime_of "$dir/out.arc" example.com/a.html)" = audio/midi ||
+    fail "control: a.html did not resolve to the record at offset 0"
+test "$(mime_of "$dir/out.arc" example.com/b.html)" = image/gif ||
+    fail "control: b.html did not resolve to the record at offset $posA"
+
+# A field that does not parse must read as offset 0, not as the offset the
+# entry before it left behind.
+build_cache "$dir/carry" "$posA" Z
+convert "$dir/carry.ndx" "carry-over"
+test "$(mime_of "$dir/out.arc" example.com/b.html)" = audio/midi ||
+    fail "an unparseable position inherited the previous entry's offset"
+
+# Same on the first entry, where there is no previous value and the read came
+# off the stack.
+build_cache "$dir/first" Z "$posA"
+convert "$dir/first.ndx" "first entry"
+test "$(mime_of "$dir/out.arc" example.com/a.html)" = audio/midi ||
+    fail "an unparseable position on the first entry did not read as 0"
+
+# The .ndx name reaches the copy before anything opens it, so no file needed.
+long=$(printf '%3000s' '' | tr ' ' 'x')
+convert "$dir/$long.ndx" "over-long .ndx name"
+
+if is_windows; then
+    echo "deep paths need a POSIX filesystem; index name half only" >&2
+    exit 0
+fi
+
+comp=$(printf '%200s' '' | tr ' ' 'd')
+
+# A path just short of the 1024-byte destinations must still load: refusing
+# these is as wrong as overflowing them.
+fits="$dir/$comp/$comp/$comp"
+mkdir -p "$fits"
+build_cache "$fits/c" 0 "$posA"
+convert "$fits/c.ndx" "path that fits"
+test "$(mime_of "$dir/out.arc" example.com/b.html)" = image/gif ||
+    fail "a cache at a $((${#fits} + 6))-byte path was dropped"
+
+# Past them, the base path and both index filenames each overshoot.
+deep="$fits/$comp/$comp/$comp"
+mkdir -p "$deep" 2>/dev/null || {
+    echo "filesystem will not hold a ${#deep}-byte path; deep half skipped" >&2
+    exit 0
+}
+build_cache "$deep/c" 0 "$posA"
+convert "$deep/c.ndx" "path past the destinations"
+
+# The zip reader keeps its own copy of that base-path code, and only reaches it
+# once the archive opens, so this one needs a real zip at the deep path.
+python=$(find_python) || {
+    echo "python3 not found; zip half skipped" >&2
+    exit 0
+}
+"$python" - "$deep/c.zip" <<'EOF'
+import sys, zipfile
+
+zi = zipfile.ZipInfo("example.com/page.html")
+zi.compress_type = zipfile.ZIP_STORED
+zi.extra = (
+    "X-In-Cache: 1\r\nX-StatusCode: 200\r\nX-StatusMessage: OK\r\n"
+    "X-Size: 5\r\nContent-Type: text/html\r\n"
+    "Last-Modified: Wed, 01 Jan 2025 00:00:00 GMT\r\nX-Save: out/page.html\r\n"
+).encode()
+with zipfile.ZipFile(sys.argv[1], "w") as z:
+    z.writestr(zi, b"hello")
+EOF
+convert "$deep/c.zip" "zip at a path past the base buffer"
+test "$(mime_of "$dir/out.arc" example.com/page.html)" = text/html ||
+    fail "the zip entry was dropped instead of keeping an empty base path"
+
+echo "OK: an unparseable .ndx position reads as 0, and an over-long cache path is refused"

tests/13_crawl_proxy_https.test

@@ -31,7 +31,7 @@ cleanup() {
     done
     rm -rf "$tmpdir"
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # self-signed cert for the local TLS origin (httrack does not verify certs)
 openssl req -x509 -newkey rsa:2048 -keyout "$tmpdir/key.pem" \

tests/19_local-connect-fallback.test

@@ -40,9 +40,8 @@ serverpid=
 cleanup() {
     stop_server "$serverpid"
     rm -rf "$tmpdir"
-    return 0
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # bind the live server to 127.0.0.1 only, so 127.0.0.2 refuses the connect
 "$python" "$server" --root "$root" --bind 127.0.0.1 >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &

tests/23_local-errpage.test

@@ -13,7 +13,10 @@ bash "$top_srcdir/tests/local-crawl.sh" --errors 1 \
     --not-found 'errpage/missing.html' \
     httrack 'BASEURL/errpage/index.html' '-o0'
 
-# Control -o1 (default): the 404 error page is written.
+# Control -o1 (default): the 404 body is the server's verbatim, no HTTrack marker (#787); good.html proves the marker check can fire.
 bash "$top_srcdir/tests/local-crawl.sh" --errors 1 \
     --found 'errpage/missing.html' \
+    --file-matches 'errpage/missing.html' '404 error body' \
+    --file-not-matches 'errpage/missing.html' 'HTTrack' \
+    --file-matches 'errpage/good.html' 'HTTrack' \
     httrack 'BASEURL/errpage/index.html' '-o1'

tests/24_local-resume-overlap.test

@@ -24,7 +24,8 @@ cleanup() {
     stop_server "$serverpid"
     rm -rf "$tmpdir"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 # OVERLAP_COUNTER gets a byte per flaky.bin request so pass 1 knows when to interrupt.
 serverlog="${tmpdir}/server.log"

tests/40_local-why.test

@@ -6,7 +6,7 @@ set -euo pipefail
 # --why: report which +/- filter rule decides for a URL, without crawling.
 
 tmpdir="$(mktemp -d)"
-trap 'rm -rf "$tmpdir"' EXIT
+trap 'set +e; rm -rf "$tmpdir"' EXIT
 
 why() {
     local want="$1" out

tests/52_local-socks5.test

@@ -34,7 +34,7 @@ cleanup() {
     done
     rm -rf "$tmpdir"
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 openssl req -x509 -newkey rsa:2048 -keyout "$tmpdir/key.pem" \
     -out "$tmpdir/cert.pem" -days 2 -nodes -subj "/CN=127.0.0.1" \

tests/53_local-proxytrack-arc-reason.test

@@ -5,7 +5,7 @@
 set -euo pipefail
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 printf 'HTTP/1.1 404 Not Found Here At All\r\nContent-Type: text/html\r\nLast-Modified: Wed, 01 Jan 2025 00:00:00 GMT\r\nContent-Length: 5\r\n\r\n' >"$dir/hdr"
 printf 'hello' >"$dir/body"

tests/53_local-proxytrack-cache-corrupt.test

@@ -5,7 +5,7 @@
 set -euo pipefail
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # The first length-prefixed field declares far more than firstline[256] holds;
 # cache_brstr must clamp the copy to the destination, not the declared length.

tests/56_local-proxy-noleak.test

@@ -29,9 +29,8 @@ serverpid=
 cleanup() {
     stop_server "$serverpid"
     rm -rf "$tmpdir"
-    return 0
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # decoy origin: it must stay silent. LOCAL_SERVER_VERBOSE logs any request it gets.
 LOCAL_SERVER_VERBOSE=1 "$python" "$server" --root "$root" --bind 127.0.0.1 \

tests/57_local-proxy-connect.test

@@ -29,7 +29,7 @@ cleanup() {
     done
     rm -rf "$tmpdir"
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # start_server <logdir> <mode>: launches a proxy+origin pair, sets $origin_port
 # and $proxy_port from its announced ephemeral ports.

tests/58_watchdog.test

@@ -12,7 +12,7 @@ testdir=$(cd "$(dirname "$0")" && pwd)
 . "${testdir}/testlib.sh"
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_wd.XXXXXX")
-trap 'rm -rf "$tmp"' EXIT
+trap 'set +e; rm -rf "$tmp"' EXIT
 fail() {
     echo "FAIL: $*" >&2
     exit 1

tests/59_local-tls-stall.test

@@ -27,9 +27,8 @@ serverpid=
 cleanup() {
     stop_server "$serverpid"
     rm -rf "$tmpdir"
-    return 0
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # start_stall_server <tag> <mode-args...>: sets $port from the announced one.
 start_stall_server() {

tests/60_crawl-log-salvage.test

@@ -12,7 +12,7 @@ testdir=$(cd "$(dirname "$0")" && pwd)
 
 TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/httrack_salv.XXXXXX")
 export TMPDIR
-trap 'rm -rf "$TMPDIR"' EXIT
+trap 'set +e; rm -rf "$TMPDIR"' EXIT
 fail() {
     echo "FAIL: $*" >&2
     exit 1

tests/61_webhttrack-locale.test

@@ -23,7 +23,7 @@ funcs=$(sed -n '/^function lang_index/,/^}/p' "${script}")
 # would truncate the lift and silently stop testing everything below it.
 grep -q '^# Find the browser' "${script}" || fail "locale block terminator moved in ${script}"
 block=$(sed -n '/^# Locale/,/^# Find the browser/p' "${script}" | sed '$d')
-echo "${block}" | grep -q 'LANGN=.*lang_index' || fail "could not lift the whole locale block from ${script}"
+grep -q 'LANGN=.*lang_index' <<<"${block}" || fail "could not lift the whole locale block from ${script}"
 
 # Run the lifted block against whatever locale vars the caller exported.
 run_block() {

tests/62_lang-integrity.test

@@ -19,7 +19,7 @@ for f in "$eng" "$def"; do
 done
 
 tmp=$(mktemp -d)
-trap 'rm -rf "$tmp"' EXIT
+trap 'set +e; rm -rf "$tmp"' EXIT
 keys="$tmp/english.keys"
 
 # Byte-wise: each file is in its own declared legacy charset, lang.def is CRLF.