Drop the historical narration from hts_lastcharptr's contract

The declaration is the API surface, so it keeps the double-evaluation
warning a caller cannot derive from the signature; what the macro
replaced is git's job. Its grep exclusion in the test goes with it,
since that comment was the only htssafe.h line the scan matched.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -79,7 +79,10 @@ jobs:
       - name: Build
         run: make -j"$(nproc)"
 
+      # A backstop only: tests/test-timeout.sh bounds each test, and a healthy run
+      # is a minute here, two on macOS. Without it a stall ran to the job's 6h default.
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -119,6 +122,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test without python3
+        timeout-minutes: 20
         run: |
           set -euo pipefail
           # Hide every python3* so `command -v python3` fails like it does in the
@@ -173,6 +177,7 @@ jobs:
           sudo ifconfig lo0 alias 127.0.0.3 up
 
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -245,6 +250,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -302,6 +308,7 @@ jobs:
         env:
           ASAN_OPTIONS: detect_leaks=0:abort_on_error=1:halt_on_error=1:strict_string_checks=1:malloc_fill_byte=202:max_malloc_fill_size=2147483647:free_fill_byte=203:max_free_fill_size=2147483647
           UBSAN_OPTIONS: print_stacktrace=1:halt_on_error=1
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -345,6 +352,7 @@ jobs:
       - name: Test (offline self-tests under MSan)
         env:
           MSAN_OPTIONS: abort_on_error=1:halt_on_error=1
+        timeout-minutes: 20
         run: |
           set -euo pipefail
           # 01_engine-* only; zlib-dependent self-tests are named 01_zlib-* and
@@ -426,6 +434,7 @@ jobs:
         run: make -j"$(nproc)"
 
       - name: Test
+        timeout-minutes: 20
         run: |
           jobs=$(( $(nproc) * 2 )); [ "$jobs" -le 16 ] || jobs=16
           make check -j"$jobs"
@@ -618,7 +627,12 @@ jobs:
           set -euo pipefail
           git fetch --no-tags origin \
             "+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
-          base="origin/${{ github.base_ref }}"
+          # Merge base, not the branch tip: master moves during a run, and a
+          # tip-relative diff blames its new C on this PR.
+          if ! base="$(git merge-base "origin/${{ github.base_ref }}" HEAD)"; then
+            echo "::error::no merge base with origin/${{ github.base_ref }}; cannot scope the check."
+            exit 1
+          fi
           set +e
           diff="$(git clang-format --binary clang-format-19 --style=file \
                     --diff --extensions c,h "$base")"
@@ -660,6 +674,8 @@ jobs:
           git fetch --no-tags origin \
             "+refs/heads/${{ github.base_ref }}:refs/remotes/origin/${{ github.base_ref }}"
           base="origin/${{ github.base_ref }}"
+          # Three dots: merge-base scoped, so commits landing on master mid-run
+          # are not counted as this PR's.
           changed="$(git diff --name-only "$base"...HEAD)"
           has() { printf '%s\n' "$changed" | grep -qx "$1"; }
           if has man/httrack.1 && ! has html/httrack.man.html; then

.github/workflows/windows-build.yml

@@ -189,25 +189,53 @@ jobs:
           # A wedged crawl must not eat the job's timeout budget. timeout(1)'s
           # signals can't reap a native httrack.exe (MSYS signals don't reach it),
           # so a hang orphaned processes that starved the runner; run_with_timeout
-          # TerminateProcess-es the whole tree. 600s clears the slowest multi-pass
-          # crawl (a few passes at --max-time=120 each).
+          # TerminateProcess-es the whole tree. 600s is unchanged: it clears the
+          # 540s a three-pass crawl may legitimately take under local-crawl.sh's
+          # own watchdogs, against a slowest healthy test here of 39s.
           . ./testlib.sh
+          per_test=600
 
-          pass=0 fail=0 skip=0 failed="" skipped=""
+          # The whole suite must give up before the step timeout above. A cancelled
+          # step keeps neither its log nor the artifacts the later if:always()
+          # steps would upload, so an overrun that ends in a cancel tells us
+          # nothing; failing on our own terms keeps both. Healthy runs take 8-9
+          # min. The check sits between tests, so the step can still reach 25 min
+          # plus one per-test budget, and that worst case stays inside the 45.
+          suite_deadline=1500
+          started=$SECONDS
+
+          # Survives into the artifact even if the tail of the step log does not.
+          progress=suite-progress.log
+          : >"$progress"
+
+          pass=0 fail=0 skip=0 failed="" skipped="" deadline=0
           for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
                    *_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
-                   60_crawl-log-salvage.test; do
+                   60_crawl-log-salvage.test 116_engine-rtrim.test; do
+            elapsed=$((SECONDS - started))
+            if [ "$elapsed" -ge "$suite_deadline" ]; then
+              echo "::error::suite deadline: ${elapsed}s elapsed, stopping before $t"
+              echo "DEADLINE before $t after ${elapsed}s" >>"$progress"
+              # Per-test start times, so the slow ones are named rather than guessed.
+              sed 's/^/      /' "$progress"
+              deadline=1
+              break
+            fi
+            echo "RUN $t at ${elapsed}s" >>"$progress"
             rc=0
-            run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
+            # Same guard "make check" uses on POSIX, so a wedge is diagnosed the
+            # same way on every platform. It dumps before it kills, which a bare
+            # run_with_timeout cannot: by the time that returns, the tree whose
+            # stack we wanted is already gone.
+            HTTRACK_TEST_TIMEOUT=$per_test bash ./test-timeout.sh "$t" >"$t.log" 2>&1 || rc=$?
             case "$rc" in
               0) pass=$((pass + 1)); echo "PASS $t" ;;
               77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
               124)
                 fail=$((fail + 1)) failed="$failed $t"
+                # test-timeout.sh has already written the process list, the stacks
+                # and the killed crawl's own logs into $t.log.
                 echo "FAIL $t (timed out, tree killed)"
-                # Re-running a wedge traced would just hang again: salvage the
-                # killed crawl's own logs into the artifact instead.
-                dump_crawl_logs >>"$t.log"
                 tail -n 25 "$t.log" | sed 's/^/      /'
                 ;;
               *)
@@ -215,21 +243,29 @@ jobs:
                 echo "FAIL $t (exit $rc)"
                 # These assert with `test "$(...)" == "..." || exit 1`, which
                 # says nothing at all on failure. Re-run traced, still bounded.
-                run_with_timeout 600 bash -x "$t" >>"$t.log" 2>&1 || true
+                run_with_timeout "$per_test" bash -x "$t" >>"$t.log" 2>&1 || true
                 tail -n 25 "$t.log" | sed 's/^/      /'
                 ;;
             esac
+            echo "$rc $t" >>"$progress"
+            # An orphaned native httrack.exe spins and starves the runner, which
+            # is how this job dies with "lost communication" rather than a plain
+            # timeout. Clear them between tests and name whoever leaked them.
+            reap_leftover_processes "$t" | tee -a "$progress"
           done
           echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
             tee -a "$GITHUB_STEP_SUMMARY"
 
           # Every gate here exits 77, so an all-skipped suite would report green having
           # tested nothing: pin the skips, and floor the passes in case the glob empties.
-          # footer-overflow skips on Windows (needs a path past MAX_PATH); crange pending #581;
+          # footer-overflow and purge-longpath skip on Windows (need a path past MAX_PATH);
+          # crange pending #581;
           # webdav-mime needs a reapable background listener, which MSYS cannot give it;
           # badmtime needs a filesystem that stores an mtime past gmtime's range;
           # single-file ends on a GUI half needing htsserver, which this job does not build.
-          expected_skips=" 01_engine-footer-overflow.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
+          expected_skips=" 01_engine-footer-overflow.test 100_local-purge-longpath.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
+          # First, or the deadline reads as an unexplained shortfall in the gates below.
+          [ "$deadline" -eq 0 ] || { echo "::error::suite did not finish within ${suite_deadline}s"; exit 1; }
           [ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
           [ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
           [ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }

AGENTS.md

@@ -26,6 +26,10 @@ the operational checklist: toolchain, invariants, and how to ship a change.
   check`, or `PATH="<bld>/src:$PATH"` for a manual run.
 - Give new `.test` scripts `set -e`: the older ones predate the rule, so several
   `local-crawl.sh` calls with no `set -e` report PASS on any non-last failure.
+- Run teardown with errexit off: `trap 'set +e; cleanup' EXIT`. Under `set -e` a
+  failing cleanup command becomes the test's exit status (#773). Keep the other
+  signals on their own `trap` line, or errexit stays off for the rest of the run.
+  The guard also resets `$?`, so save it first if teardown reads it.
 - Never assert with `cmd | grep -q MARKER && fail`. Under `pipefail` the
   pipeline is non-zero both when `cmd` fails and when `grep -q` matches early
   and SIGPIPEs it, so the `&&` never fires and a probe that proved nothing reads

src/htsalias.c

@@ -501,13 +501,11 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
       /* read line */
       linput(fp, line, 250);
       hts_lowcase(line);
+      /* trim first: a blank line is skipped, not parsed as an option */
+      hts_rtrim(line, HTS_REALSPACES);
       if (strnotempty(line)) {
         /* no comment line: # // ; */
         if (strchr("#/;", line[0]) == NULL) {
-          /* right trim */
-          a = line + strlen(line) - 1;
-          while(is_realspace(*a))
-            *(a--) = '\0';
           /* jump "set " and spaces */
           a = line;
           while(is_realspace(*a))
@@ -577,7 +575,6 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
             }
           }
         }
-
       }
     }
     fclose(fp);

src/htsback.c

@@ -340,12 +340,61 @@ static int back_index_ready(httrackp * opt, struct_back * sback, const char *adr
 }
 
 static int slot_can_be_cached_on_disk(const lien_back * back) {
+  /* A pending backup or spool means the slot is not finalized, and the swap
+     would unlink it through back_clear_entry() (#771). */
+  if (back->tmpfile != NULL && back->tmpfile[0] != '\0')
+    return 0;
   return (back->status == STATUS_READY && back->locked == 0
           && back->url_sav[0] != '\0'
           && strcmp(back->url_sav, BACK_ADD_TEST) != 0);
   /* Note: not checking !IS_DELAYED_EXT(back->url_sav) or it will quickly cause the slots to be filled! */
 }
 
+int back_selftest_slot_swap(void) {
+  lien_back back;
+  int err = 0;
+
+#define CHECK(want, why)                                                       \
+  do {                                                                         \
+    if (slot_can_be_cached_on_disk(&back) != (want)) {                         \
+      fprintf(stderr, "backswap: expected %d for %s\n", (want), (why));        \
+      err = 1;                                                                 \
+    }                                                                          \
+  } while (0)
+
+  memset(&back, 0, sizeof(back));
+  back.status = STATUS_READY;
+  strcpybuff(back.url_sav, "/tmp/httrack-selftest.bin");
+  CHECK(1, "a plain ready slot");
+
+  back.tmpfile = back.tmpfile_buffer;
+  strcpybuff(back.tmpfile_buffer, "/tmp/httrack-selftest.bin.bak");
+  CHECK(0, "a slot still holding a re-fetch backup");
+
+  /* Callers clear a spent temporary by emptying the name, not the pointer. */
+  back.tmpfile_buffer[0] = '\0';
+  CHECK(1, "a slot whose temporary was already dropped");
+
+  back.tmpfile = NULL;
+  back.locked = 1;
+  CHECK(0, "a locked slot");
+  back.locked = 0;
+
+  back.status = STATUS_TRANSFER;
+  CHECK(0, "a slot still transferring");
+  back.status = STATUS_READY;
+
+  back.url_sav[0] = '\0';
+  CHECK(0, "a slot with no save name");
+
+  strcpybuff(back.url_sav, BACK_ADD_TEST);
+  CHECK(0, "the dummy test slot");
+#undef CHECK
+
+  printf("backswap self-test: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 /* Put all backing entries that are ready in the storage hashtable to spare space and CPU */
 int back_cleanup_background(httrackp * opt, cache_back * cache,
                             struct_back * sback) {
@@ -585,6 +634,29 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
   return 0;
 }
 
+/* Note: utf-8 */
+void back_refetch_backup(httrackp *opt, lien_back *const back) {
+  back->tmpfile = NULL;
+  if (fexist_utf8(back->url_sav)) {
+    hts_boolean saved = HTS_FALSE;
+
+    if (create_back_tmpfile(opt, back, "bak") == 0) {
+      /* clobber a .bak a killed run left behind, or the guard stays off for
+         good (#758) */
+      if (fexist_utf8(back->tmpfile))
+        hts_log_print(opt, LOG_WARNING, "replacing leftover backup %s",
+                      back->tmpfile);
+      saved = hts_rename_over(back->url_sav, back->tmpfile);
+    }
+    if (!saved) {
+      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
+                    "could not back up %s; an aborted re-fetch will lose it",
+                    back->url_sav);
+      back->tmpfile = NULL;
+    }
+  }
+}
+
 /* Did the fetch fail to produce a response, as opposed to the engine
    deliberately passing the resource over? Only the latter may be purged. */
 static hts_boolean back_transfer_failed(const int statuscode) {
@@ -3164,36 +3236,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
-                              /* Re-fetch over an existing file (#77 follow-up):
-                                 move the good copy aside before truncating it
-                                 so an aborted transfer can restore it. url_sav
-                                 is still written normally (file list intact).
-                               */
-                              back[i].tmpfile = NULL;
-                              if (fexist_utf8(back[i].url_sav)) {
-                                hts_boolean saved = HTS_FALSE;
-
-                                if (create_back_tmpfile(opt, &back[i], "bak") ==
-                                    0) {
-                                  /* clobber a .bak a killed run left behind,
-                                     or the guard stays off for good (#758) */
-                                  if (fexist_utf8(back[i].tmpfile))
-                                    hts_log_print(
-                                        opt, LOG_WARNING,
-                                        "replacing leftover backup %s",
-                                        back[i].tmpfile);
-                                  saved = hts_rename_over(back[i].url_sav,
-                                                          back[i].tmpfile);
-                                }
-                                if (!saved) {
-                                  hts_log_print(
-                                      opt, LOG_WARNING | LOG_ERRNO,
-                                      "could not back up %s; an aborted "
-                                      "re-fetch will lose it",
-                                      back[i].url_sav);
-                                  back[i].tmpfile = NULL;
-                                }
-                              }
+                              back_refetch_backup(opt, &back[i]);
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {

src/htsback.h

@@ -139,6 +139,12 @@ int back_trylive(httrackp * opt, cache_back * cache, struct_back * sback,
                  const int p);
 int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                   const int p);
+/* Move the previous copy of back->url_sav to back->tmpfile so back_finalize()
+   can put it back when the re-fetch fails (#77 follow-up). Call right before
+   truncating url_sav; tmpfile stays NULL when there is nothing to save. */
+void back_refetch_backup(httrackp *opt, lien_back *const back);
+/* -#test=backswap: slots eligible for the on-disk ready table. */
+int back_selftest_slot_swap(void);
 void back_info(struct_back * sback, int i, int j, FILE * fp);
 void back_infostr(struct_back *sback, int i, int j, char *s, size_t size);
 LLint back_transferred(LLint add, struct_back * sback);

src/htsbauth.c

@@ -491,7 +491,7 @@ char *bauth_prefix(char *prefix, const char *adr, const char *fil) {
   if (a)
     *a = '\0';
   if (strchr(prefix, '/')) {
-    a = prefix + strlen(prefix) - 1;
+    a = hts_lastcharptr(prefix);
     while(*a != '/')
       a--;
     *(a + 1) = '\0';

src/htscache_selftest.c

@@ -676,7 +676,7 @@ int cache_selftests(httrackp *opt, const char *dir) {
     char base[HTS_URLMAXSIZE];
 
     strcpybuff(base, dir);
-    if (base[0] != '\0' && base[strlen(base) - 1] != '/') {
+    if (base[0] != '\0' && hts_lastchar(base) != '/') {
       strcatbuff(base, "/");
     }
     StringCopy(opt->path_log, base);
@@ -856,7 +856,7 @@ static void golden_setup(httrackp *opt, const char *dir) {
   char base[HTS_URLMAXSIZE];
 
   strcpybuff(base, dir);
-  if (base[0] != '\0' && base[strlen(base) - 1] != '/') {
+  if (base[0] != '\0' && hts_lastchar(base) != '/') {
     strcatbuff(base, "/");
   }
   StringCopy(opt->path_log, base);

src/htscore.c

@@ -773,7 +773,7 @@ int httpmirror(char *url1, httrackp * opt) {
         // sauter les + sans rien après..
         if (strnotempty(tempo)) {
           if ((plus == 0) && (type == 1)) {     // implicite: *www.edf.fr par exemple
-            if (tempo[strlen(tempo) - 1] != '*') {
+            if (hts_lastchar(tempo) != '*') {
               strcatbuff(tempo, "*");   // ajouter un *
             }
           }
@@ -2136,7 +2136,9 @@ int httpmirror(char *url1, httrackp * opt) {
                   continue;
                 strcpybuff(file, StringBuff(opt->path_html));
                 strcatbuff(file, line + 1);
-                file[strlen(file) - 1] = '\0';
+                /* strip filenote()'s ']', absent when linput() truncated the
+                   line */
+                hts_striplastchar(file, ']');
                 hts_changes_previous(opt, file + StringLength(opt->path_html));
                 if (!strstr(adr, line)) { // not found in the new list?
                   if (fexist_utf8(file)) { // still on disk
@@ -2163,12 +2165,11 @@ int httpmirror(char *url1, httrackp * opt) {
                 fseek(old_lst, 0, SEEK_SET);
                 while(!feof(old_lst)) {
                   linput(old_lst, line, 1000);
-                  while(strnotempty(line) && (line[strlen(line) - 1] != '/')
-                        && (line[strlen(line) - 1] != '\\')) {
-                    line[strlen(line) - 1] = '\0';
+                  while (strnotempty(line) && (hts_lastchar(line) != '/') &&
+                         (hts_lastchar(line) != '\\')) {
+                    hts_choplastchar(line);
                   }
-                  if (strnotempty(line))
-                    line[strlen(line) - 1] = '\0';
+                  hts_choplastchar(line);
                   if (strnotempty(line))
                     if (!strstr(adr, line)) {   // non trouvé?
                       char BIGSTK file[HTS_URLMAXSIZE * 2];
@@ -2181,13 +2182,12 @@ int httpmirror(char *url1, httrackp * opt) {
                         if (opt->log) {
                           hts_log_print(opt, LOG_INFO, "Purging directory %s/",
                                         file);
-                          while(strnotempty(file)
-                                && (file[strlen(file) - 1] != '/')
-                                && (file[strlen(file) - 1] != '\\')) {
-                            file[strlen(file) - 1] = '\0';
+                          while (strnotempty(file) &&
+                                 (hts_lastchar(file) != '/') &&
+                                 (hts_lastchar(file) != '\\')) {
+                            hts_choplastchar(file);
                           }
-                          if (strnotempty(file))
-                            file[strlen(file) - 1] = '\0';
+                          hts_choplastchar(file);
                         }
                       }
                     }
@@ -3774,8 +3774,7 @@ int htsAddLink(htsmoduleStruct * str, char *link) {
         strcpybuff(codebase, heap(ptr)->fil);
       else
         strcpybuff(codebase, heap(heap(ptr)->precedent)->fil);
-      // empty codebase has no last char; codebase-1 would underflow
-      a = codebase[0] != '\0' ? codebase + strlen(codebase) - 1 : codebase;
+      a = hts_lastcharptr(codebase);
       while((*a) && (*a != '/') && (a > codebase))
         a--;
       if (*a == '/')

src/htscoremain.c

@@ -357,7 +357,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           char BIGSTK tempo[HTS_CDLMAXSIZE];
 
           strcpybuff(tempo, argv[na] + 1);
-          if (tempo[0] == '\0' || tempo[strlen(tempo) - 1] != '"') {
+          if (hts_lastchar(tempo) != '"') {
             char BIGSTK s[HTS_CDLMAXSIZE];
 
             sprintf(s, "Missing quote in %s", argv[na]);
@@ -365,7 +365,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
             htsmain_free();
             return -1;
           }
-          tempo[strlen(tempo) - 1] = '\0';
+          hts_choplastchar(tempo);
           /* tempo is argv[na] minus its surrounding quotes, so it fits in place
            */
           strlcpybuff(argv[na], tempo, strlen(argv[na]) + 1);
@@ -863,7 +863,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
         char BIGSTK tempo[HTS_CDLMAXSIZE + 256];
 
         strcpybuff(tempo, argv[na] + 1);
-        if (tempo[0] == '\0' || tempo[strlen(tempo) - 1] != '"') {
+        if (hts_lastchar(tempo) != '"') {
           char s[HTS_CDLMAXSIZE + 256];
 
           sprintf(s, "Missing quote in %s", argv[na]);
@@ -871,7 +871,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
           htsmain_free();
           return -1;
         }
-        tempo[strlen(tempo) - 1] = '\0';
+        hts_choplastchar(tempo);
         /* tempo is argv[na] minus its surrounding quotes, so it fits in place
          */
         strlcpybuff(argv[na], tempo, strlen(argv[na]) + 1);
@@ -2724,10 +2724,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       char *a;
 
       strcpybuff(rpath, StringBuff(opt->path_html));
-      if (rpath[0]) {
-        if (rpath[strlen(rpath) - 1] == '/')
-          rpath[strlen(rpath) - 1] = '\0';
-      }
+      hts_striplastchar(rpath, '/');
       a = strrchr(rpath, '/');
       if (a) {
         *a = '\0';

src/htsftp.c

@@ -624,6 +624,9 @@ int run_launch_ftp(FTPDownloadStruct * pStruct) {
           } else {
             file_notify(opt, back->url_adr, back->url_fil, back->url_sav, 1, 1,
                         0);
+            /* Every failure exit below would else leave the mirror truncated
+               (#771); the resume branch appends and needs no backup. */
+            back_refetch_backup(opt, back);
             back->r.fp = filecreate(&opt->state.strc, back->url_sav);
           }
           strcpybuff(back->info, "receiving");

src/htshelp.c

@@ -83,9 +83,8 @@ void infomsg(const char *msg) {
               /* try the flag as-is, then strip a trailing N as the numeric-arg
                  placeholder (cN -> c); this order keeps -%N from becoming -% */
               p = optreal_find(cmd);
-              if (p < 0 && (int) strlen(cmd) > 2 &&
-                  cmd[strlen(cmd) - 1] == 'N') {
-                cmd[strlen(cmd) - 1] = '\0';
+              if (p < 0 && (int) strlen(cmd) > 2 && hts_lastchar(cmd) == 'N') {
+                hts_striplastchar(cmd, 'N');
                 p = optreal_find(cmd);
               }
               if (p >= 0) {
@@ -211,7 +210,7 @@ void help_wizard(httrackp * opt) {
     strcatbuff(str, "/websites/");
   }
   if (strnotempty(str))
-    if ((str[strlen(str) - 1] != '/') && (str[strlen(str) - 1] != '\\'))
+    if ((hts_lastchar(str) != '/') && (hts_lastchar(str) != '\\'))
       strcatbuff(str, "/");
   strcatbuff(stropt2, "-O \"");
   strcatbuff(stropt2, str);

src/htslib.c

@@ -1322,7 +1322,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
       p++;                      // sauter espaces
     if ((int) strlen(rcvd + p) < 250) { // pas trop long?
       char tmp[256];
-      char *a = NULL, *b = NULL;
+      char *a = NULL;
 
       strcpybuff(tmp, rcvd + p);
       a = strstr(tmp, "filename=");
@@ -1335,15 +1335,9 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
 
           while((c = strchr(a, '/')))   /* skip all / (see RFC2616) */
             a = c + 1;
-          b = a + strlen(a) - 1;
-          while(is_space(*b))
-            b--;
-          b++;
-          if (b) {
-            *b = '\0';
-            if ((int) strlen(a) < 200) {        // pas trop long?
-              strcpybuff(retour->cdispo, a);
-            }
+          hts_rtrim(a, HTS_SPACES);
+          if ((int) strlen(a) < 200) { // pas trop long?
+            strcpybuff(retour->cdispo, a);
           }
         }
       }
@@ -3315,8 +3309,9 @@ int ishtml(httrackp * opt, const char *fil) {
   }
 
   /* Search for known ext */
-  for(a = fil_noquery + strlen(fil_noquery) - 1;
-      *a != '.' && *a != '/' && a > fil_noquery; a--) ;
+  for (a = hts_lastcharptr(fil_noquery);
+       *a != '.' && *a != '/' && a > fil_noquery; a--)
+    ;
   if (*a == '.') {              // a une extension
     char BIGSTK fil_noquery[HTS_URLMAXSIZE * 2];
     char *b;
@@ -4253,9 +4248,8 @@ HTSEXT_API hts_boolean get_httptype_sized(httrackp *opt, char *s, size_t ssize,
     return 1;
   } else {
     /* Check html -> text/html */
-    const char *a = fil + strlen(fil) - 1;
+    const char *a = hts_lastcharptr(fil);
 
-    /* a < fil when fil is empty: bound before dereferencing */
     while ((a > fil) && (*a != '.') && (*a != '/'))
       a--;
     if (a >= fil && *a == '.' && strlen(a) < 32) {
@@ -5078,7 +5072,7 @@ static int hts_dns_resolve_nocache_list(const char *const hostname,
   if (!strnotempty(hostname) || max <= 0) {
     return 0;
   }
-  if ((hostname[0] == '[') && (hostname[strlen(hostname) - 1] == ']')) {
+  if ((hostname[0] == '[') && (hts_lastchar(hostname) == ']')) {
     size_t size = strlen(hostname);
     char *copy = malloct(size + 1);
     int count;
@@ -5491,9 +5485,8 @@ void cut_path(char *fullpath, char *path, size_t path_size, char *pname,
               size_t pname_size) {
   path[0] = pname[0] = '\0';
   if (strnotempty(fullpath)) {
-    if ((fullpath[strlen(fullpath) - 1] == '/')
-        || (fullpath[strlen(fullpath) - 1] == '\\'))
-      fullpath[strlen(fullpath) - 1] = '\0';
+    if (!hts_striplastchar(fullpath, '/'))
+      hts_striplastchar(fullpath, '\\');
     if (strlen(fullpath) > 1) {
       char *a;
 
@@ -5791,7 +5784,8 @@ HTSEXT_API void hts_log_vprint(httrackp * opt, int type, const char *format, va_
   if (hts_log_print_callback != NULL) {
     va_list args_copy;
     va_copy(args_copy, args);
-    hts_log_print_callback(opt, type, format, args);
+    /* the copy, so the vfprintf() below still has an unread list */
+    hts_log_print_callback(opt, type, format, args_copy);
     va_end(args_copy);
   }
   if (opt != NULL && opt->log != NULL) {
@@ -6648,9 +6642,12 @@ int hts_rename_utf8(const char *oldpath, const char *newpath) {
   LPWSTR wnewpath = hts_pathToUCS2(newpath);
   if (woldpath != NULL && wnewpath != NULL) {
     const int result = _wrename(woldpath, wnewpath);
+    /* Save errno: callers key off it (#779) and free() may clobber it. */
+    const int err = errno;
 
     free(woldpath);
     free(wnewpath);
+    errno = err;
     return result;
   } else {
     if (woldpath != NULL)

src/htsname.c

@@ -437,9 +437,7 @@ int url_savename(lien_adrfilsave *const afs,
 
       strcpybuff(fil_complete_patche, normfil);
       // Version avec ou sans /
-      if (fil_complete_patche[strlen(fil_complete_patche) - 1] == '/')
-        fil_complete_patche[strlen(fil_complete_patche) - 1] = '\0';
-      else
+      if (!hts_striplastchar(fil_complete_patche, '/'))
         strcatbuff(fil_complete_patche, "/");
       i = hash_read(hash, normadr, fil_complete_patche, HASH_STRUCT_ORIGINAL_ADR_PATH);       // recherche table 2 (former->adr+former->fil)
       if (i >= 0) {
@@ -515,7 +513,8 @@ int url_savename(lien_adrfilsave *const afs,
         && protocol != PROTOCOL_FTP
       ) {
       // tester type avec requète HEAD si on ne connait pas le type du fichier
-      if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/')))   // slash doit être html?
+      if (!((opt->check_type == 1) &&
+            (hts_lastchar(fil) == '/'))) // slash doit être html?
         if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
             ishtml(opt, fil) < 0) { // unsure whether it's html or a file
           // lire dans le cache
@@ -810,7 +809,7 @@ int url_savename(lien_adrfilsave *const afs,
   // - - - DEBUT NOMMAGE - - -
 
   // Donner nom par défaut?
-  if (fil[strlen(fil) - 1] == '/') {
+  if (hts_lastchar(fil) == '/') {
     if (!strfield(adr_complete, "ftp://")
       ) {
       strcatbuff(fil, DEFAULT_HTML);    // nommer page par défaut!!
@@ -824,7 +823,7 @@ int url_savename(lien_adrfilsave *const afs,
   // Change the extension? e.g. php3 saved as html, cgi as html or gif/xbm
   // depending on the resolved type.
   if (ext_chg && !opt->no_type_change) {
-    char *a = fil + strlen(fil) - 1;
+    char *a = hts_lastcharptr(fil);
 
     if ((opt->debug > 1) && (opt->log != NULL)) {
       if (ext_chg == 1)
@@ -859,7 +858,7 @@ int url_savename(lien_adrfilsave *const afs,
   }
   // Rechercher premier / et dernier .
   {
-    const char *a = fil + strlen(fil) - 1;
+    const char *a = hts_lastcharptr(fil);
 
     // passer structures
     start_pos = fil;
@@ -1204,29 +1203,29 @@ int url_savename(lien_adrfilsave *const afs,
     switch (opt->savename_type % 100) {
     case 4:
     case 5:{                   // séparer par types
-        const char *a = fil + strlen(fil) - 1;
+      const char *a = hts_lastcharptr(fil);
 
-        // passer structures
-        while((a > fil) && (*a != '/') && (*a != '\\'))
+      // passer structures
+      while ((a > fil) && (*a != '/') && (*a != '\\'))
+        a--;
+      if ((*a == '/') || (*a == '\\'))
+        a++;
+
+      // html?
+      if ((ext_chg != 0) ? (ishtml_ext(ext) == 1) : (ishtml(opt, fil) == 1)) {
+        if (opt->savename_type % 100 == 5)
+          strcatbuff(afs->save, "html/");
+      } else {
+        const char *a = hts_lastcharptr(fil);
+
+        while ((a > fil) && (*a != '/') && (*a != '.'))
           a--;
-        if ((*a == '/') || (*a == '\\'))
-          a++;
-
-        // html?
-        if ((ext_chg != 0) ? (ishtml_ext(ext) == 1) : (ishtml(opt, fil) == 1)) {
-          if (opt->savename_type % 100 == 5)
-            strcatbuff(afs->save, "html/");
-        } else {
-          const char *a = fil + strlen(fil) - 1;
-
-          while((a > fil) && (*a != '/') && (*a != '.'))
-            a--;
-          if (*a != '.')
-            strcatbuff(afs->save, "other");
-          else
-            strcatbuff(afs->save, a + 1);
-          strcatbuff(afs->save, "/");
-        }
+        if (*a != '.')
+          strcatbuff(afs->save, "other");
+        else
+          strcatbuff(afs->save, a + 1);
+        strcatbuff(afs->save, "/");
+      }
         /*strcatbuff(save,a); */
         /* add name */
         ADD_STANDARD_NAME(0);
@@ -1259,7 +1258,7 @@ int url_savename(lien_adrfilsave *const afs,
         }
         afs->save[i + j] = '\0';
         // ajouter extension
-        a = fil + strlen(fil) - 1;
+        a = hts_lastcharptr(fil);
         while((a > fil) && (*a != '/') && (*a != '.'))
           a--;
         if (*a == '.') {
@@ -1284,7 +1283,7 @@ int url_savename(lien_adrfilsave *const afs,
 
     hts_lowcase(afs->save);
 
-    if (afs->save[strlen(afs->save) - 1] == '/')
+    if (hts_lastchar(afs->save) == '/')
       strcatbuff(afs->save, DEFAULT_HTML);   // nommer page par défaut!!
   }
 
@@ -1304,7 +1303,7 @@ int url_savename(lien_adrfilsave *const afs,
   // cela évite les /chez/toto et les /chez/toto/index.html incompatibles
   if (opt->savename_type != -1 &&
       opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD) {
-    char *a = afs->save + strlen(afs->save) - 1;
+    char *a = hts_lastcharptr(afs->save);
 
     while((a > afs->save) && (*a != '.') && (*a != '/'))
       a--;
@@ -1421,8 +1420,10 @@ int url_savename(lien_adrfilsave *const afs,
   if (opt->savename_83 > 0) {
     char *a, *last;
 
-    for(last = afs->save + strlen(afs->save) - 1;
-        last != afs->save && *last != '/' && *last != '\\' && *last != '.'; last--) ;
+    for (last = hts_lastcharptr(afs->save);
+         last != afs->save && *last != '/' && *last != '\\' && *last != '.';
+         last--)
+      ;
     if (*last != '.') {
       last = NULL;
     }
@@ -1677,8 +1678,8 @@ int url_savename(lien_adrfilsave *const afs,
 #endif
         } else {                // utilisé par un AUTRE, changer de nom
           char BIGSTK tempo[HTS_URLMAXSIZE * 2];
-          char *a = afs->save + strlen(afs->save) - 1;
-          char *b;
+          char *a = hts_lastcharptr(afs->save);
+          size_t stem;
           int n = 2;
           char collisionSeparator =
               ((opt->savename_83 != HTS_SAVENAME_83_ISO9660) ? '-' : '_');
@@ -1700,18 +1701,16 @@ int url_savename(lien_adrfilsave *const afs,
             strcatbuff(tempo, afs->save);
 
           // tester la présence d'un -xx (ex: index-2.html -> index-3.html)
-          b = tempo + strlen(tempo) - 1;
-          while(isdigit((unsigned char) *b))
-            b--;
-          if (*b == collisionSeparator) {
-            sscanf(b + 1, "%d", &n);
-            *b = '\0';          // couper
+          stem = hts_rtrimlen(tempo, "0123456789");
+          if (stem != 0 && tempo[stem - 1] == collisionSeparator) {
+            sscanf(tempo + stem, "%d", &n);
+            tempo[stem - 1] = '\0'; // couper
             n++;                // plus un
           }
           // en plus il faut gérer le 8-3 .. pas facile le client
           if (opt->savename_83) {
             int max;
-            char *a = tempo + strlen(tempo) - 1;
+            char *a = hts_lastcharptr(tempo);
 
             while((a > tempo) && (*a != '/'))
               a--;

src/htsparse.c

@@ -1701,7 +1701,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 #endif
                                     )   // ok pas de problème
                                     url_ok = 1;
-                                  else if (tempo[strlen(tempo) - 1] == '/') {   // un slash: ok..
+                                  else if (hts_lastchar(tempo) ==
+                                           '/') {       // un slash: ok..
                                     if (inscript)       // sinon si pas javascript, méfiance (répertoire style base?)
                                       url_ok = 1;
                                   }
@@ -2005,9 +2006,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               if (eadr - html - 1 < HTS_URLMAXSIZE) {        // pas trop long?
                 strncpy(lien, html, eadr - html - 1);
                 lien[eadr - html - 1] = '\0';
-                // supprimer les espaces
-                while((lien[strlen(lien) - 1] == ' ') && (strnotempty(lien)))
-                  lien[strlen(lien) - 1] = '\0';
+                while (hts_striplastchar(lien, ' ')) {
+                }
 
               } else
                 lien[0] = '\0'; // erreur
@@ -2057,7 +2057,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   }
                   q = strchr(a, '?');   // ne pas traiter après '?'
                   if (!q)
-                    q = a + strlen(a) - 1;
+                    q = hts_lastcharptr(a);
                   while((p = strstr(a, "//")) && (!done)) {     // remplacer // par /
                     if (p > q) {    // après le ? (toto.cgi?param=1//2.3)
                       done = 1; // stopper
@@ -2207,8 +2207,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 // supposition dangereuse?
                 // OUI!!
 #if HTS_TILDE_SLASH
-                if (lien[strlen(lien) - 1] != '/') {
-                  char *a = lien + strlen(lien) - 1;
+                if (hts_lastchar(lien) != '/') {
+                  char *a = hts_lastcharptr(lien);
 
                   // éviter aussi index~1.html
                   while(a > lien && (*a != '~') && (*a != '/')
@@ -2254,7 +2254,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 // vérifier que l'on ne doit pas ajouter de .class
                 if (!error) {
                   if (add_class) {
-                    char *a = lien + strlen(lien) - 1;
+                    char *a = hts_lastcharptr(lien);
 
                     while((a > lien) && (*a != '/') && (*a != '.'))
                       a--;
@@ -2272,7 +2272,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     // Vérifier les codebase=applet (au lieu de applet/)
                     if (p_type == -2) { // codebase
                       if (strnotempty(lien)) {
-                        if (lien[strlen(lien) - 1] != '/') {     // pas répertoire
+                        if (hts_lastchar(lien) != '/') { // pas répertoire
                           strcatbuff(lien, "/");
                         }
                       }
@@ -2309,7 +2309,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                         {
                           char *a;
 
-                          a = lien + strlen(lien) - 1;
+                          a = hts_lastcharptr(lien);
                           while((*a) && (*a != '/') && (a > lien))
                             a--;
                           if (*a == '/') {
@@ -2320,8 +2320,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     }
 
                     if (!error) {       // pas d'erreur?
-                      if (p_type == 2) {        // code ET PAS codebase      
-                        char *a = lien + strlen(lien) - 1;
+                      if (p_type == 2) { // code ET PAS codebase
+                        char *a = hts_lastcharptr(lien);
                         char *start_of_filename = jump_identification(lien);
 
                         if (start_of_filename != NULL
@@ -2688,9 +2688,11 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                             int cat_data_len = 0;
 
                             // ajouter lien external
-                            switch ((link_has_authority(afs.af.adr)) ? 1
-                                    : ((afs.af.fil[strlen(afs.af.fil) - 1] ==
-                                        '/') ? 1 : (ishtml(opt, afs.af.fil)))) {
+                            switch ((link_has_authority(afs.af.adr))
+                                        ? 1
+                                        : ((hts_lastchar(afs.af.fil) == '/')
+                                               ? 1
+                                               : (ishtml(opt, afs.af.fil)))) {
                             case 1:
                             case -2:   // html ou répertoire
                               if (opt->getmode & HTS_GETMODE_HTML) {
@@ -2733,7 +2735,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                                 cat_data_len = HTS_DATA_UNKNOWN_HTML_LEN;
                               }
                               break;
-                            }   // html,gif
+                            } // html,gif
 
                             if (patch_it) {
                               char BIGSTK save[HTS_URLMAXSIZE * 2];

src/htssafe.h

@@ -539,6 +539,68 @@ static HTS_INLINE HTS_UNUSED HTS_PRINTF_FUN(3, 4) void slprintfbuff_clip(
 #define sprintfbuff(ARR, ...) slprintfbuff((ARR), sizeof(ARR), __VA_ARGS__)
 #endif
 
+/* Last character of s, or '\0' when s is empty. Replaces s[strlen(s) - 1],
+   which indexes one byte before the buffer on an empty string. */
+static HTS_INLINE HTS_UNUSED char hts_lastchar(const char *s) {
+  const size_t len = strlen(s);
+
+  return len != 0 ? s[len - 1] : '\0';
+}
+
+/* Drop a trailing c from s if present; HTS_TRUE if one was dropped. */
+static HTS_INLINE HTS_UNUSED hts_boolean hts_striplastchar(char *s, char c) {
+  const size_t len = strlen(s);
+
+  if (len != 0 && s[len - 1] == c) {
+    s[len - 1] = '\0';
+    return HTS_TRUE;
+  }
+  return HTS_FALSE;
+}
+
+/* Drop the last character of s whatever it is; HTS_TRUE if s was not empty. */
+static HTS_INLINE HTS_UNUSED hts_boolean hts_choplastchar(char *s) {
+  const size_t len = strlen(s);
+
+  if (len != 0) {
+    s[len - 1] = '\0';
+    return HTS_TRUE;
+  }
+  return HTS_FALSE;
+}
+
+/* htslib.h's is_space() and is_realspace() as hts_rtrim() sets; -#test=rtrim
+   keeps them in sync. */
+#define HTS_SPACES " \"\n\r\t\f\v'"
+#define HTS_REALSPACES " \n\r\t\f\v"
+
+/* Length of s once its trailing bytes from set are dropped; 0 if they all are.
+   Counts down from the end, so it stops at s rather than below the buffer. */
+static HTS_INLINE HTS_UNUSED size_t hts_rtrimlen(const char *s,
+                                                 const char *set) {
+  size_t len = strlen(s);
+
+  while (len != 0 && strchr(set, s[len - 1]) != NULL)
+    len--;
+  return len;
+}
+
+/* Drop the trailing bytes of s that occur in set. */
+static HTS_INLINE HTS_UNUSED void hts_rtrim(char *s, const char *set) {
+  s[hts_rtrimlen(s, set)] = '\0';
+}
+
+/* Offset of the last character of s, or 0 when s is empty. */
+static HTS_INLINE HTS_UNUSED size_t hts_lastcharoffset(const char *s) {
+  const size_t len = strlen(s);
+
+  return len != 0 ? len - 1 : 0;
+}
+
+/* Address of the last character of S, or of its terminating NUL when S is
+   empty. S is evaluated twice, so pass an lvalue. */
+#define hts_lastcharptr(S) ((S) + hts_lastcharoffset(S))
+
 /* Thin aliases over the libc allocator/memcpy (historical "t" suffix); no
    added bounds checking. freet() also NULLs the freed pointer and tolerates
    NULL. memcpybuff() despite the name is a raw memcpy: the caller owns the

src/htsselftest.c

@@ -2622,6 +2622,71 @@ static int st_savename(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+static char st_log_callback_seen[256];
+
+static void st_log_callback(httrackp *opt, int type, const char *format,
+                            va_list args) {
+  (void) opt;
+  (void) type;
+  (void) vsnprintf(st_log_callback_seen, sizeof(st_log_callback_seen), format,
+                   args);
+}
+
+/* The callback must not consume the va_list the log file's vfprintf() needs. */
+static int st_logcallback(httrackp *opt, int argc, char **argv) {
+  static const char want[] = "42 sentinel";
+  static const char want_filtered[] = "7 filtered";
+  char BIGSTK seen[sizeof(st_log_callback_seen)];
+  char BIGSTK line[256];
+  FILE *fp;
+  int rc = 1;
+
+  (void) argc;
+  (void) argv;
+
+  fp = tmpfile();
+  if (fp == NULL) {
+    fprintf(stderr, "logcallback: tmpfile() failed\n");
+    return 1;
+  }
+  opt->log = fp;
+  opt->debug = LOG_NOTICE;
+  st_log_callback_seen[0] = '\0';
+  hts_set_log_vprint_callback(st_log_callback);
+  hts_log_print(opt, LOG_NOTICE, "%d %s", 42, "sentinel");
+  hts_set_log_vprint_callback(NULL);
+  opt->log = NULL;
+  strcpybuff(seen, st_log_callback_seen);
+
+  rewind(fp);
+  line[0] = '\0';
+  (void) fgets(line, (int) sizeof(line), fp);
+  fclose(fp);
+
+  /* The callback runs above the level filter and without a log file at all;
+     the front-ends that install one usually have no opt->log open. */
+  st_log_callback_seen[0] = '\0';
+  hts_set_log_vprint_callback(st_log_callback);
+  hts_log_print(opt, LOG_DEBUG, "%d %s", 7, "filtered");
+  hts_set_log_vprint_callback(NULL);
+
+  /* Same arguments both ways; the file line carries a level prefix. */
+  if (strcmp(seen, want) != 0)
+    fprintf(stderr, "logcallback: callback got '%s' want '%s'\n", seen, want);
+  else if (strstr(line, want) == NULL)
+    fprintf(stderr, "logcallback: log file got '%s' want it to carry '%s'\n",
+            line, want);
+  else if (strcmp(st_log_callback_seen, want_filtered) != 0)
+    fprintf(stderr, "logcallback: unfiltered callback got '%s' want '%s'\n",
+            st_log_callback_seen, want_filtered);
+  else
+    rc = 0;
+
+  if (rc == 0)
+    printf("logcallback self-test OK\n");
+  return rc;
+}
+
 /* an empty fil started htsAddLink's codebase walk before the buffer (#730) */
 static int st_addlink(httrackp *opt, int argc, char **argv) {
   htsmoduleStruct BIGSTK str;
@@ -4944,6 +5009,101 @@ static int st_warc_surt(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* A URL longer than the old 1024-byte header-format buffer must still reach the
+   archive: the record used to be abandoned whole, silently (#785). The sweep
+   straddles the boundary so both the stack-buffer and the grow path run. */
+static int st_warc_longurl(httrackp *opt, int argc, char **argv) {
+  /* "WARC-Target-URI: " + CRLF costs 19 bytes, so the old buffer failed at
+     1005; 9000 forces several reallocs within one record. */
+  static const size_t lengths[] = {100, 1003, 1004, 1005, 1006, 2000, 9000};
+  static const char resp_hdr[] =
+      "HTTP/1.1 200 OK\r\nContent-Type: text/html\r\n\r\n";
+  char path[HTS_URLMAXSIZE * 2];
+  char body[64];
+  warc_writer *w;
+  FILE *fp;
+  char *blob;
+  LLint fsz;
+  const char *at2;
+  size_t i, n, nrec = 0;
+  int err = 0;
+
+  if (argc < 1) {
+    fprintf(stderr, "warc-longurl: need a writable directory\n");
+    return 1;
+  }
+  snprintf(path, sizeof(path), "%s/longurl.warc", argv[0]);
+
+  w = warc_open(opt, path);
+  if (w == NULL) {
+    fprintf(stderr, "warc-longurl: could not create %s\n", path);
+    return 1;
+  }
+  for (i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) {
+    const size_t len = lengths[i];
+    char *uri = malloct(len + 1);
+
+    if (uri == NULL) {
+      warc_close(w);
+      return 1;
+    }
+    /* A distinct tail per URI so a truncated one cannot match another. */
+    snprintf(uri, len + 1, "http://example.com/%04d/", (int) len);
+    memset(uri + strlen(uri), 'a', len - strlen(uri));
+    uri[len] = '\0';
+    /* Distinct payloads: identical ones dedupe into revisit records. */
+    snprintf(body, sizeof(body), "<html><body>%04d</body></html>\n", (int) len);
+    if (warc_write_transaction(w, uri, NULL, NULL, resp_hdr, body, strlen(body),
+                               NULL, 200, 0, 0) != 0) {
+      fprintf(stderr, "warc-longurl: write failed at length %d\n", (int) len);
+      err = 1;
+    }
+    freet(uri);
+  }
+  warc_close(w);
+
+  fsz = fsize_utf8(path);
+  blob = (fsz > 0) ? malloct((size_t) fsz + 1) : NULL;
+  if (blob == NULL) {
+    fprintf(stderr, "warc-longurl: no archive written\n");
+    return 1;
+  }
+  fp = FOPEN(path, "rb");
+  n = (fp != NULL) ? fread(blob, 1, (size_t) fsz, fp) : 0;
+  if (fp != NULL)
+    fclose(fp);
+  blob[n] = '\0';
+
+  for (i = 0; i < sizeof(lengths) / sizeof(lengths[0]); i++) {
+    const size_t len = lengths[i];
+    char want[64];
+    const char *at;
+
+    snprintf(want, sizeof(want), "WARC-Target-URI: http://example.com/%04d/",
+             (int) len);
+    at = strstr(blob, want);
+    if (at == NULL) {
+      fprintf(stderr, "warc-longurl: length %d lost its record\n", (int) len);
+      err = 1;
+    } else if (strlen(at) < strlen("WARC-Target-URI: ") + len ||
+               at[strlen("WARC-Target-URI: ") + len] != '\r') {
+      fprintf(stderr, "warc-longurl: length %d truncated\n", (int) len);
+      err = 1;
+    }
+  }
+  for (at2 = blob; (at2 = strstr(at2, "WARC-Type: response")) != NULL; at2++)
+    nrec++;
+  if (nrec != sizeof(lengths) / sizeof(lengths[0])) {
+    fprintf(stderr, "warc-longurl: %d response records, want %d\n", (int) nrec,
+            (int) (sizeof(lengths) / sizeof(lengths[0])));
+    err = 1;
+  }
+
+  freet(blob);
+  printf("warc-longurl: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 /* End-to-end CDXJ: crawl a handful of records with --warc-cdx, then verify the
    .cdx is sorted, has exactly one line per response/revisit/resource (none for
    warcinfo/request), and each offset/length points at a gzip member that
@@ -5955,6 +6115,104 @@ static int st_mirrorio(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+static void ro_put(const char *path, const char *data) {
+  FILE *const fp = FOPEN(path, "wb");
+
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, strlen(data), fp) == strlen(data));
+  fclose(fp);
+}
+
+/* HTS_TRUE if path holds exactly data. */
+static hts_boolean ro_is(const char *path, const char *data) {
+  char buf[64];
+  FILE *const fp = FOPEN(path, "rb");
+  size_t n;
+
+  if (fp == NULL)
+    return HTS_FALSE;
+  n = fread(buf, 1, sizeof(buf), fp);
+  fclose(fp);
+  return n == strlen(data) && memcmp(buf, data, n) == 0 ? HTS_TRUE : HTS_FALSE;
+}
+
+// -#test=renameover <dir>: hts_rename_over() must replace an existing dst and
+// never delete one it did not replace (#779). Which half is live depends on
+// what rename() does to an existing target, so probe that and name the regime.
+static int st_renameover(httrackp *opt, int argc, char **argv) {
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "renameover: needs a writable base dir\n");
+    return 1;
+  }
+  char src[HTS_URLMAXSIZE * 2], dst[HTS_URLMAXSIZE * 2];
+  int err = 0;
+
+  fconcat(src, sizeof(src), argv[0], "renameover-src.bin");
+  fconcat(dst, sizeof(dst), argv[0], "renameover-dst.bin");
+
+  (void) UNLINK(src);
+  (void) UNLINK(dst);
+  ro_put(src, "probe");
+  ro_put(dst, "probe");
+
+  const int probe = RENAME(src, dst) == 0 ? 0 : errno;
+  /* Only a target in the way is something the unlink can clear. */
+  const hts_boolean replaceable = probe == 0 || probe == EEXIST;
+
+  printf("renameover: regime %s\n",
+         probe == 0 ? "clobber" : (probe == EEXIST ? "fallback" : "refused"));
+
+  (void) UNLINK(src);
+  (void) UNLINK(dst);
+  ro_put(src, "new");
+  ro_put(dst, "old");
+  if (replaceable) {
+    /* An existing dst must still be replaced: the unlink is for this. */
+    if (!hts_rename_over(src, dst)) {
+      fprintf(stderr, "renameover: replacing an existing dst failed: %s\n",
+              strerror(errno));
+      err++;
+    } else if (!ro_is(dst, "new") || fexist_utf8(src)) {
+      fprintf(stderr, "renameover: dst was not replaced by src\n");
+      err++;
+    }
+  } else {
+    /* A failure the unlink cannot fix must leave dst as it was. */
+    if (hts_rename_over(src, dst)) {
+      fprintf(stderr, "renameover: an unfixable failure reported success\n");
+      err++;
+    }
+    if (!ro_is(dst, "old")) {
+      fprintf(stderr, "renameover: an unfixable failure destroyed dst\n");
+      err++;
+    }
+  }
+
+  /* A missing src must leave dst alone and report failure. */
+  (void) UNLINK(src);
+  ro_put(dst, "keep");
+  if (hts_rename_over(src, dst)) {
+    fprintf(stderr, "renameover: a missing src reported success\n");
+    err++;
+  }
+  if (!ro_is(dst, "keep")) {
+    fprintf(stderr, "renameover: a missing src destroyed dst\n");
+    err++;
+  }
+
+  /* Same, with dst absent too: nothing to lose, still a failure. */
+  (void) UNLINK(dst);
+  if (hts_rename_over(src, dst)) {
+    fprintf(stderr, "renameover: a missing src and dst reported success\n");
+    err++;
+  }
+
+  (void) UNLINK(dst);
+  printf("renameover: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 // -#test=direnum <dir>: enumerate a long+non-ASCII directory via the
 // opendir/readdir wrappers; children must round-trip as UTF-8 (#133,#630).
 static int st_direnum(httrackp *opt, int argc, char **argv) {
@@ -6268,6 +6526,13 @@ static void threadwait_gated_thread(void *arg) {
   hts_mutexrelease(&threadwait_lock);
 }
 
+static int st_backswap(httrackp *opt, int argc, char **argv) {
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  return back_selftest_slot_swap();
+}
+
 static int st_threadwait(httrackp *opt, int argc, char **argv) {
   int err = 0;
   int i, round;
@@ -6374,7 +6639,7 @@ static int st_changes_race(httrackp *opt, int argc, char **argv) {
     return 1;
   }
   strcpybuff(base, argv[0]);
-  if (base[0] != '\0' && base[strlen(base) - 1] != '/')
+  if (base[0] != '\0' && hts_lastchar(base) != '/')
     strcatbuff(base, "/");
   StringCopy(opt->path_html, base);
   StringCopy(opt->path_html_utf8, base);
@@ -6445,6 +6710,215 @@ static int st_changes_race(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* The x[strlen(x) - 1] class (#770) and its pointer spelling x + strlen(x) - 1
+   (#781). The string starts mid-arena so the byte it must not touch is a real
+   neighbour; poisoned with '#', not 0, or a stray NUL terminator would read as
+   untouched. */
+static int st_lastchar(httrackp *opt, int argc, char **argv) {
+  enum { off = 8 };
+
+  char arena[16];
+  char *const s = &arena[off];
+  const int guard = off - 1; /* what the old idiom clobbers */
+  int err = 0;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+
+#define REPOISON(str)                                                          \
+  do {                                                                         \
+    memset(arena, '#', sizeof(arena));                                         \
+    strlcpybuff(s, (str), sizeof(arena) - off);                                \
+  } while (0)
+#define CHECK(cond)                                                            \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      printf("  FAIL line %d: %s\n", __LINE__, #cond);                         \
+      err = 1;                                                                 \
+    }                                                                          \
+  } while (0)
+
+  /* the empty string: every helper must report "nothing" and touch nothing */
+  REPOISON("");
+  CHECK(hts_lastchar(s) == '\0');
+  CHECK(arena[guard] == '#');
+  REPOISON("");
+  CHECK(hts_striplastchar(s, '/') == HTS_FALSE);
+  CHECK(arena[guard] == '#');
+  CHECK(s[0] == '\0');
+  REPOISON("");
+  CHECK(hts_choplastchar(s) == HTS_FALSE);
+  CHECK(arena[guard] == '#');
+  CHECK(s[0] == '\0');
+
+  /* a '/' sitting where the underflow would land must not be mistaken for the
+     string's own last byte -- this is the #768 shape */
+  REPOISON("");
+  arena[guard] = '/';
+  CHECK(hts_lastchar(s) == '\0');
+  CHECK(hts_striplastchar(s, '/') == HTS_FALSE);
+  CHECK(arena[guard] == '/');
+
+  /* non-empty: ordinary behaviour */
+  REPOISON("ab/");
+  CHECK(hts_lastchar(s) == '/');
+  CHECK(hts_striplastchar(s, '/') == HTS_TRUE);
+  CHECK(strcmp(s, "ab") == 0);
+  CHECK(hts_striplastchar(s, '/') == HTS_FALSE);
+  CHECK(strcmp(s, "ab") == 0);
+  CHECK(hts_choplastchar(s) == HTS_TRUE);
+  CHECK(strcmp(s, "a") == 0);
+  CHECK(hts_choplastchar(s) == HTS_TRUE);
+  CHECK(s[0] == '\0');
+  CHECK(arena[guard] == '#');
+
+  /* one-character string: the boundary the guards get wrong */
+  REPOISON("/");
+  CHECK(hts_lastchar(s) == '/');
+  CHECK(hts_striplastchar(s, '/') == HTS_TRUE);
+  CHECK(s[0] == '\0');
+  CHECK(arena[guard] == '#');
+
+  /* the pointer spelling (#781): on an empty string the address must be the
+     terminating NUL, never the byte before it */
+  REPOISON("");
+  CHECK(hts_lastcharoffset(s) == 0);
+  CHECK(hts_lastcharptr(s) == s);
+  CHECK(*hts_lastcharptr(s) == '\0');
+  *hts_lastcharptr(s) = 'Z'; /* a write through it must stay inside s */
+  CHECK(arena[guard] == '#');
+  CHECK(s[0] == 'Z');
+
+  /* the neighbour must not be mistaken for the string's own last byte */
+  REPOISON("");
+  arena[guard] = '/';
+  CHECK(hts_lastcharptr(s) == s);
+  CHECK(*hts_lastcharptr(s) != '/');
+  CHECK(arena[guard] == '/');
+
+  /* the walk-back loops the sites use must stop at once on an empty string */
+  REPOISON("");
+  {
+    const char *p = hts_lastcharptr(s);
+    int steps = 0;
+
+    while (p > s && *p != '/')
+      p--, steps++;
+    CHECK(steps == 0);
+    CHECK(p == s);
+  }
+
+  REPOISON("ab/");
+  CHECK(hts_lastcharoffset(s) == 2);
+  CHECK(hts_lastcharptr(s) == s + 2);
+  CHECK(*hts_lastcharptr(s) == '/');
+  REPOISON("/");
+  CHECK(hts_lastcharptr(s) == s);
+  CHECK(*hts_lastcharptr(s) == '/');
+  CHECK(arena[guard] == '#');
+
+  /* control: the canary must be able to fail, or the checks above prove
+     nothing. Clobber it exactly as the unguarded idiom would. */
+  REPOISON("");
+  s[-1] = '\0';
+  CHECK(arena[guard] != '#');
+  REPOISON("");
+  *(s + strlen(s) - 1) = 'X';
+  CHECK(arena[guard] != '#');
+
+#undef REPOISON
+#undef CHECK
+
+  printf("lastchar self-test: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
+/* hts_rtrim() and the sets it is called with. The string starts mid-arena, and
+   the byte below it is poisoned with '#' rather than 0, or the stray NUL the
+   old loop wrote there would read as untouched. */
+static int st_rtrim(httrackp *opt, int argc, char **argv) {
+  enum { off = 8 };
+
+  char arena[24];
+  char *const s = &arena[off];
+  const int guard = off - 1;
+  int err = 0;
+  int c;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+
+#define REPOISON(str)                                                          \
+  do {                                                                         \
+    memset(arena, '#', sizeof(arena));                                         \
+    strlcpybuff(s, (str), sizeof(arena) - off);                                \
+  } while (0)
+#define CHECK(cond)                                                            \
+  do {                                                                         \
+    if (!(cond)) {                                                             \
+      printf("  FAIL line %d: %s\n", __LINE__, #cond);                         \
+      err = 1;                                                                 \
+    }                                                                          \
+  } while (0)
+
+  /* nothing but spaces: the case that ran the old loop off the front */
+  REPOISON("   ");
+  hts_rtrim(s, HTS_REALSPACES);
+  CHECK(s[0] == '\0');
+  CHECK(arena[guard] == '#');
+
+  /* a space sitting below the string must not be eaten as if it were part of
+     it, which is exactly what the old loop did */
+  REPOISON("   ");
+  arena[guard] = ' ';
+  hts_rtrim(s, HTS_REALSPACES);
+  CHECK(s[0] == '\0');
+  CHECK(arena[guard] == ' ');
+
+  REPOISON("");
+  hts_rtrim(s, HTS_REALSPACES);
+  CHECK(s[0] == '\0');
+  CHECK(arena[guard] == '#');
+
+  REPOISON("a b \t\r\n");
+  hts_rtrim(s, HTS_REALSPACES);
+  CHECK(strcmp(s, "a b") == 0);
+  CHECK(arena[guard] == '#');
+
+  REPOISON("ab");
+  hts_rtrim(s, HTS_REALSPACES);
+  CHECK(strcmp(s, "ab") == 0);
+
+  /* quotes count as space for is_space() but not for is_realspace() */
+  REPOISON("v\" ");
+  hts_rtrim(s, HTS_REALSPACES);
+  CHECK(strcmp(s, "v\"") == 0);
+  REPOISON("v\" ");
+  hts_rtrim(s, HTS_SPACES);
+  CHECK(strcmp(s, "v") == 0);
+
+  /* the sets must stay the macros they stand for */
+  for (c = 1; c < 256; c++) {
+    const char b = (char) c;
+
+    CHECK((strchr(HTS_SPACES, b) != NULL) == (is_space(b) != 0));
+    CHECK((strchr(HTS_REALSPACES, b) != NULL) == (is_realspace(b) != 0));
+  }
+
+  /* control: the canary must be able to fail */
+  REPOISON("   ");
+  s[-1] = '\0';
+  CHECK(arena[guard] != '#');
+
+#undef REPOISON
+#undef CHECK
+
+  printf("rtrim self-test: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 /* ------------------------------------------------------------ */
 /* Registry: name -> handler, with a usage hint and a one-line description. */
 /* ------------------------------------------------------------ */
@@ -6500,12 +6974,18 @@ static const struct selftest_entry {
     {"strsafe", "[overflow|overflow-buff|overflow-src [str]]",
      "bounded string-op self-test", st_strsafe},
     {"copyopt", "", "copy_htsopt option-copy self-test", st_copyopt},
+    {"lastchar", "",
+     "last-char helpers never index before the buffer (#770, #781)",
+     st_lastchar},
+    {"rtrim", "", "hts_rtrim never walks below the buffer", st_rtrim},
     {"changes", "", "--changes bucket accounting and JSON escaping (#714)",
      st_changes},
     {"changes-race", "<dir>", "--changes under a late transfer thread (#714)",
      st_changes_race},
     {"threadwait", "", "htsthread_wait() joins threads spawned just before it",
      st_threadwait},
+    {"backswap", "", "which backlog slots may be swapped to the ready table",
+     st_backswap},
     {"pause", "", "randomized inter-file pause target self-test", st_pause},
     {"relative", "<link> <curr-file>", "relative link between two paths",
      st_relative},
@@ -6538,6 +7018,8 @@ static const struct selftest_entry {
      st_growsize},
     {"addlink", "", "htsAddLink codebase walk over an empty current path",
      st_addlink},
+    {"logcallback", "", "log callback must not consume the log file's va_list",
+     st_logcallback},
     {"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
     {"cacheindex", "", "cache-index (.ndx) parse must stay in bounds",
      st_cacheindex},
@@ -6596,6 +7078,9 @@ static const struct selftest_entry {
      st_warc_verbatim},
     {"warc-surt", "", "SURT canonicalization of the CDXJ sort key",
      st_warc_surt},
+    {"warc-longurl", "<dir>",
+     "a URL past the header-format buffer still reaches the archive",
+     st_warc_longurl},
     {"longpath", "<dir>",
      "round-trip a >MAX_PATH file through the _w* wrappers (\\\\?\\ on "
      "Windows)",
@@ -6603,6 +7088,10 @@ static const struct selftest_entry {
     {"mirrorio", "<dir>",
      "round-trip a long+non-ASCII path through the mirror I/O wrappers",
      st_mirrorio},
+    {"renameover", "<dir>",
+     "hts_rename_over(): replace dst, but never delete a dst it did not "
+     "replace",
+     st_renameover},
     {"direnum", "<dir>",
      "enumerate a long+non-ASCII directory through opendir/readdir",
      st_direnum},

src/htsserver.c

@@ -1050,9 +1050,9 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                 if (!linput(fp, line, sizeof(line) - 2)) {
                   *str = '\0';
                 }
-                if (*str && str[strlen(str) - 1] == '\\') {
+                if (hts_lastchar(str) == '\\') {
                   nocr = 1;
-                  str[strlen(str) - 1] = '\0';
+                  hts_striplastchar(str, '\\');
                 }
                 while(*str) {
                   char *pos;
@@ -1169,11 +1169,8 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                           char *rpath = (char *) adr;
 
                           //find_handle h;
-                          if (rpath[0]) {
-                            if (rpath[strlen(rpath) - 1] == '/') {
-                              rpath[strlen(rpath) - 1] = '\0';  /* note: patching stored (inhash) value */
-                            }
-                          }
+                          /* note: patching stored (inhash) value */
+                          hts_striplastchar(rpath, '/');
                           {
                             const char *profiles = hts_getcategories(rpath, 0);
                             const char *categ = hts_getcategories(rpath, 1);

src/htstools.c

@@ -324,8 +324,7 @@ int lienrelatif(char *s, size_t ssize, const char *link, const char *curr_fil) {
   curr = _curr;
   strlcpybuff(curr, curr_fil, sizeof(_curr));
   if ((a = strchr(curr, '?')) == NULL) { // cut at the ? (query parameters)
-    // an empty path has no last character: curr-1 would read before the buffer
-    a = curr[0] != '\0' ? curr + strlen(curr) - 1 : curr;
+    a = hts_lastcharptr(curr);
   }
   while((*a != '/') && (a > curr))
     a--;                        // chercher dernier / du chemin courant
@@ -985,10 +984,7 @@ HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
       && toptemplate_bodycat) {
 
     strcpybuff(rpath, path);
-    if (rpath[0]) {
-      if (rpath[strlen(rpath) - 1] == '/')
-        rpath[strlen(rpath) - 1] = '\0';
-    }
+    hts_striplastchar(rpath, '/');
 
     fpo = fopen(fconcat(catbuff, sizeof(catbuff), rpath, "/index.html"), "wb");
     if (fpo) {
@@ -1201,11 +1197,8 @@ HTSEXT_API char *hts_getcategories(char *path, int type) {
   find_handle h;
   coucal hashCateg = NULL;
 
-  if (rpath[0]) {
-    if (rpath[strlen(rpath) - 1] == '/') {
-      rpath[strlen(rpath) - 1] = '\0';  /* note: patching stored (inhash) value */
-    }
-  }
+  /* note: patching stored (inhash) value */
+  hts_striplastchar(rpath, '/');
   h = hts_findfirst(rpath);
   if (h) {
     String iname = STRING_EMPTY;
@@ -1302,7 +1295,7 @@ HTSEXT_API find_handle hts_findfirst(char *path) {
 
           strcpybuff(rpath, path);
           if (rpath[0]) {
-            if (rpath[strlen(rpath) - 1] != '\\')
+            if (hts_lastchar(rpath) != '\\')
               strcatbuff(rpath, "\\");
           }
           strcatbuff(rpath, "*.*");
@@ -1314,7 +1307,7 @@ HTSEXT_API find_handle hts_findfirst(char *path) {
         strcpybuff(find->path, path);
         {
           if (find->path[0]) {
-            if (find->path[strlen(find->path) - 1] != '/')
+            if (hts_lastchar(find->path) != '/')
               strcatbuff(find->path, "/");
           }
         }
@@ -1451,7 +1444,14 @@ hts_boolean hts_rename_over(const char *src, const char *dst) {
   fconv(cdst, sizeof(cdst), dst);
   if (RENAME(csrc, cdst) == 0)
     return HTS_TRUE;
-  /* RENAME does not clobber an existing target on Windows. */
+  /* Only a dst in the way is something the unlink can clear, and the CRT maps
+     that to EEXIST; it keeps EACCES for a src another process holds, where
+     removing dst would lose a file the retry cannot replace (#790). The src
+     check covers a CRT that reports neither. */
+  const int err = errno;
+
+  if (err != EEXIST || !fexist_utf8(src))
+    return HTS_FALSE;
   (void) UNLINK(cdst);
   return RENAME(csrc, cdst) == 0 ? HTS_TRUE : HTS_FALSE;
 }

src/htstools.h

@@ -138,7 +138,9 @@ HTSEXT_API hts_boolean hts_findisfile(find_handle find);
 HTSEXT_API hts_boolean hts_findissystem(find_handle find);
 
 /* Move src onto dst, replacing an existing dst; HTS_TRUE on success. Both
-   paths are fconv()'d. */
+   paths are fconv()'d. dst is removed only to make room for a src that exists,
+   so a caller whose src was never written keeps its dst; a retry that still
+   fails does not (#790). */
 hts_boolean hts_rename_over(const char *src, const char *dst);
 
 #endif

src/htswarc.c

@@ -127,24 +127,32 @@ static void wbuf_free(wbuf *b) {
   b->len = b->cap = 0;
 }
 
-/* Append n bytes; returns 0 on success, -1 on OOM/overflow. */
-static int wbuf_add(wbuf *b, const void *p, size_t n) {
+/* Make room for n more bytes; returns 0 on success, -1 on OOM/overflow. */
+static int wbuf_reserve(wbuf *b, size_t n) {
+  size_t ncap;
+  char *nd;
   if (n > (size_t) -1 - b->len)
     return -1;
-  if (b->len + n > b->cap) {
-    size_t ncap = b->cap ? b->cap : 256;
-    char *nd;
-    while (ncap < b->len + n) {
-      if (ncap > (size_t) -1 / 2)
-        return -1;
-      ncap *= 2;
-    }
-    nd = realloct(b->data, ncap);
-    if (nd == NULL)
+  if (b->len + n <= b->cap)
+    return 0;
+  ncap = b->cap ? b->cap : 256;
+  while (ncap < b->len + n) {
+    if (ncap > (size_t) -1 / 2)
       return -1;
-    b->data = nd;
-    b->cap = ncap;
+    ncap *= 2;
   }
+  nd = realloct(b->data, ncap);
+  if (nd == NULL)
+    return -1;
+  b->data = nd;
+  b->cap = ncap;
+  return 0;
+}
+
+/* Append n bytes; returns 0 on success, -1 on OOM/overflow. */
+static int wbuf_add(wbuf *b, const void *p, size_t n) {
+  if (wbuf_reserve(b, n) != 0)
+    return -1;
   memcpy(b->data + b->len, p, n);
   b->len += n;
   return 0;
@@ -156,16 +164,32 @@ static int wbuf_puts(wbuf *b, const char *s) {
 
 static int wbuf_printf(wbuf *b, const char *fmt, ...) HTS_PRINTF_FUN(2, 3);
 
+/* A header line carrying a URL has no useful bound, and giving up here loses
+   the whole record, so anything past the stack buffer is formatted into the
+   wbuf itself rather than rejected (#785). */
 static int wbuf_printf(wbuf *b, const char *fmt, ...) {
   char tmp[1024];
+  size_t need;
   int n;
   va_list ap;
   va_start(ap, fmt);
   n = vsnprintf(tmp, sizeof(tmp), fmt, ap);
   va_end(ap);
-  if (n < 0 || (size_t) n >= sizeof(tmp))
+  if (n < 0)
     return -1;
-  return wbuf_add(b, tmp, (size_t) n);
+  if ((size_t) n < sizeof(tmp))
+    return wbuf_add(b, tmp, (size_t) n);
+  need = (size_t) n + 1; /* +1: vsnprintf always writes the NUL */
+  if (wbuf_reserve(b, need) != 0)
+    return -1;
+  va_start(ap, fmt);
+  n = vsnprintf(b->data + b->len, need, fmt, ap);
+  va_end(ap);
+  /* Never advance past what was reserved, whatever the second pass returns. */
+  if (n < 0 || (size_t) n >= need)
+    return -1;
+  b->len += (size_t) n; /* the NUL is scratch, overwritten by the next append */
+  return 0;
 }
 
 /* ---- gzip-per-record member writer (mirrors ae_write_packed) ---- */
@@ -1552,8 +1576,8 @@ static hts_boolean warc_commit(warc_writer *w) {
   if (!w->protect_prev)
     return HTS_TRUE; /* nothing was there to lose: written in place */
 
-  /* hts_rename_over unlinks its destination when the source is missing, so
-     every segment has to be on disk before the first rename. */
+  /* All or nothing: a swap stopping halfway would mix this run's segments with
+     the previous one's, so require every temp before renaming any. */
   swap = w->opened && !w->failed && w->unbacked_revisits == 0;
   for (s = 0; s < nseg && swap; s++) {
     snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX,
@@ -1684,6 +1708,10 @@ int warc_write_transaction(warc_writer *w, const char *target_uri,
   if (is_update_unchanged) {
     is_revisit = 1;
     profile = "http://netpreserve.org/warc/1.1/revisit/server-not-modified";
+    /* Replay resolves a revisit by this field alone, and a 304 stands in for
+       the same URL. No WARC-Refers-To-Date to go with it: the cache keeps the
+       document's Last-Modified, never the previous capture time. */
+    refers_uri = target_uri;
     /* Served from cache: the payload sits in the previous archive, not here. */
     w->unbacked_revisits++;
   } else if (have_pdig && w->seen != NULL) {

src/proxy/store.c

@@ -903,10 +903,11 @@ int PT_LoadCache__New(PT_Index index_, const char *filename) {
       coucal hashtable = index->hash;
 
       /* Compute base path for this index - the filename MUST be absolute! */
-      for(slashes = 2, abpath = filename + (int) strlen(filename) - 1;
-          abpath > filename && ((*abpath != '/' && *abpath != '\\')
-                                || --slashes > 0);
-          abpath--) ;
+      for (slashes = 2, abpath = hts_lastcharptr(filename);
+           abpath > filename &&
+           ((*abpath != '/' && *abpath != '\\') || --slashes > 0);
+           abpath--)
+        ;
       index->path[0] = '\0';
       if (slashes == 0 && *abpath != 0) {
         int i;
@@ -1499,10 +1500,11 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
 
       /* -------------------- COPY OF THE __New() CODE -------------------- */
       /* Compute base path for this index - the filename MUST be absolute! */
-      for(slashes = 2, abpath = filename + (int) strlen(filename) - 1;
-          abpath > filename && ((*abpath != '/' && *abpath != '\\')
-                                || --slashes > 0);
-          abpath--) ;
+      for (slashes = 2, abpath = hts_lastcharptr(filename);
+           abpath > filename &&
+           ((*abpath != '/' && *abpath != '\\') || --slashes > 0);
+           abpath--)
+        ;
       index->path[0] = '\0';
       if (slashes == 0 && *abpath != 0) {
         int i;

tests/01_engine-backswap.test

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# A ready slot still owning a temporary must stay in memory: swapping it out
+# clears the entry, which unlinks the re-fetch backup (#771).
+httrack -O /dev/null -#test=backswap | grep -q "backswap self-test: OK"

tests/01_engine-changes.test

@@ -4,7 +4,8 @@
 set -euo pipefail
 
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_changes_st.XXXXXX") || exit 1
-trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmpdir"' EXIT
+trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
 
 # No pipe into grep: SIGPIPE would mask a failing exit status.
 expect_ok() {

tests/01_engine-cmdline.test

@@ -11,7 +11,8 @@
 set -euo pipefail
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_cmdline.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 echo '<html><body>hello</body></html>' >"$tmp/index.html"
 

tests/01_engine-cookieimport.test

@@ -7,6 +7,6 @@ set -euo pipefail
 # cookies *@*.txt) from a long, non-ASCII folder through the UTF-8/long-path
 # file wrappers (#133,#630).
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=cookieimport "$dir" | grep -q "cookieimport:.*OK"

tests/01_engine-direnum.test

@@ -6,6 +6,6 @@ set -euo pipefail
 # Drives -#test=direnum: enumerate a long+non-ASCII directory through the
 # opendir/readdir wrappers, checking each child round-trips as UTF-8 (#133,#630).
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=direnum "$dir" | grep -q "direnum:.*OK"

tests/01_engine-doitlog.test

@@ -28,7 +28,8 @@ case "$bin" in
 esac
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_doitlog.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 site="$tmp/site"
 out="$tmp/out"

tests/01_engine-filelist.test

@@ -8,7 +8,8 @@
 set -euo pipefail
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_filelist.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 echo '<html><body>hi</body></html>' >"$tmp/index.html"
 

tests/01_engine-footer-overflow.test

@@ -18,7 +18,7 @@ MINGW* | MSYS* | CYGWIN*) exit 77 ;;
 esac
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # A few-hundred-char file path (kept well under the URL length limit) makes the
 # {path} field long.

tests/01_engine-fsize.test

@@ -6,7 +6,7 @@
 set -euo pipefail
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 rc=0
 out=$(httrack -#test=fsize "$dir") || rc=$?

tests/01_engine-growsize.test

@@ -10,7 +10,8 @@ echo "$out"
 test "$out" == "growsize self-test OK"
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_growsize.XXXXXX")
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 echo '<html><body>hi</body></html>' >"$tmp/index.html"
 printf -- '-*/zzmarker*\n' >"$tmp/rules.txt"

tests/01_engine-lastchar.test

@@ -0,0 +1,68 @@
+#!/bin/bash
+#
+# Runs the lastchar self-test (#770, #781), then keeps the idiom it replaced
+# from coming back: x[strlen(x) - 1] and x + strlen(x) - 1 both land one byte
+# before the buffer when x is empty, and the guard is never where they are.
+
+set -eu
+
+: "${top_srcdir:=..}"
+
+out=$(httrack -#test=lastchar) || {
+    echo "httrack -#test=lastchar exited non-zero: $out" >&2
+    exit 1
+}
+
+# Exact-match the success line so a renamed/removed test can't pass silently.
+test "$out" = "lastchar self-test: OK" || {
+    echo "expected 'lastchar self-test: OK', got: $out" >&2
+    exit 1
+}
+
+# The self-test only covers the helpers, so guard the class at the source: the
+# lone survivor is inside a comment block in htsname.c. coucal is vendored, so
+# its idioms are not ours to police.
+hits=$(command grep -rn 'strlen([^)]*) *- *1\]' "$top_srcdir/src" \
+    --include='*.c' --include='*.h' --exclude-dir=coucal |
+    grep -v 'htsname\.c:.*save\[strlen(save)-1\]' |
+    grep -v 'Replaces s\[strlen(s) - 1\]' |
+    grep -v 'The x\[strlen(x) - 1\] class' || true)
+
+test -z "$hits" || {
+    echo "x[strlen(x) - 1] reintroduced; use hts_lastchar/hts_striplastchar/hts_choplastchar:" >&2
+    echo "$hits" >&2
+    exit 1
+}
+
+# Same for the pointer spelling. The three survivors sit in commented-out
+# blocks in htsname.c; the rest are this class being named, plus the self-test's
+# own deliberate underflow.
+hits=$(command grep -rnE '\+ *\(?(int|size_t)?\)? *strlen *\([^)]*\) *- *1' \
+    "$top_srcdir/src" --include='*.c' --include='*.h' --exclude-dir=coucal |
+    grep -v 'htsname\.c:.*char\* a=\(fil+strlen(fil)\|save+strlen(save)\)-1;' |
+    grep -v 'htsselftest\.c:.*pointer spelling x + strlen(x) - 1' |
+    grep -v 'htsselftest\.c:.*\*(s + strlen(s) - 1) =' || true)
+
+test -z "$hits" || {
+    echo "x + strlen(x) - 1 reintroduced; use hts_lastcharptr:" >&2
+    echo "$hits" >&2
+    exit 1
+}
+
+# url_savename() has nine of these sites and reaches all of them with an empty
+# "fil", which "?x=1" produces. Names are asserted loosely, since the point is
+# the run: on unfixed code the sanitized CI leg aborts inside htsname.c.
+for args in "?x=1 text/plain" "?x=1 text/html" "?x=1 text/html type=4"; do
+    # shellcheck disable=SC2086
+    name=$(httrack -#test=savename $args) || {
+        echo "httrack -#test=savename $args exited non-zero: $name" >&2
+        exit 1
+    }
+    case "$name" in
+    "savename: "?*) ;;
+    *)
+        echo "-#test=savename $args produced no name: $name" >&2
+        exit 1
+        ;;
+    esac
+done

tests/01_engine-logcallback.test

@@ -0,0 +1,7 @@
+#!/bin/bash
+#
+# a log callback consumed the va_list the log file's vfprintf() still needed (#801).
+
+set -euo pipefail
+
+httrack -O /dev/null -#test=logcallback | grep -q "logcallback self-test OK"

tests/01_engine-longpath-io.test

@@ -6,6 +6,6 @@ set -euo pipefail
 # Drives -#test=longpath: a >MAX_PATH round trip exercising hts_pathToUCS2's
 # \\?\ prefixing on Windows (#133); a positive control on POSIX.
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=longpath "$dir" | grep -q "longpath:.*OK"

tests/01_engine-makeindex.test

@@ -6,7 +6,7 @@ set -euo pipefail
 # hts_finish_makeindex writes the footer and gates the refresh meta on a single
 # first link (guards the macro->function extraction).
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=makeindex "$dir" run |
     grep -q "makeindex self-test OK"

tests/01_engine-mirror-io.test

@@ -7,6 +7,6 @@ set -euo pipefail
 # (>MAX_PATH) and non-ASCII, exercising the mirror I/O wrappers the engine's
 # raw file ops now route to on Windows (#133, #630). Positive control on POSIX.
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=mirrorio "$dir" | grep -q "mirrorio:.*OK"

tests/01_engine-parse.test

@@ -7,7 +7,8 @@
 set -euo pipefail
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_parse.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 # a minimal valid 1x1 GIF, reused for every referenced asset
 gif() {

tests/01_engine-rcfile.test

@@ -30,7 +30,8 @@ case "$bin" in
 esac
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_rcfile.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 # HTS_HTTRACKRC is ".httrackrc" on POSIX but "httrackrc" on Windows: write both,
 # each platform reads the one it knows.

tests/01_engine-reconcile.test

@@ -7,7 +7,7 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 out=$(httrack -#test=reconcile "$dir")
 

tests/01_engine-renameover.test

@@ -0,0 +1,55 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Drives -#test=renameover: hts_rename_over() must replace an existing dst, and
+# must leave dst alone when the rename failed for a reason removing dst cannot
+# fix (#779). The selftest prints the regime it detected; pin it per platform so
+# a leg cannot pass having tested the other half.
+dir=$(mktemp -d)
+trap 'set +e; rm -rf "$dir"' EXIT
+
+case "$(uname -s)" in
+MINGW* | MSYS_NT*) want=fallback ;; # native rename() refuses an existing target
+*) want=clobber ;;
+esac
+
+out=$(httrack -O /dev/null -#test=renameover "$dir")
+echo "$out" | grep -q "renameover: OK"
+echo "$out" | grep -q "renameover: regime $want"
+
+if [ "$(uname -s)" != "Linux" ]; then
+    echo "renameover: LD_PRELOAD interposition is Linux-only here, skipping"
+    exit 0
+fi
+# A --disable-shared build has nothing to preload; anything else missing is a
+# build problem, not a skip, or the interposed legs would pass vacuously.
+if [ ! -r "${RENAMEFAIL_LA:-}" ]; then
+    echo "renameover: ${RENAMEFAIL_LA:-\$RENAMEFAIL_LA} was not built" >&2
+    exit 1
+fi
+if grep -q "^dlname=''" "$RENAMEFAIL_LA"; then
+    echo "renameover: static-only build, skipping the interposed legs"
+    exit 0
+fi
+if [ ! -r "${RENAMEFAIL_LIB:-}" ]; then
+    echo "renameover: ${RENAMEFAIL_LIB:-\$RENAMEFAIL_LIB} was not built" >&2
+    exit 1
+fi
+
+# An LD_PRELOAD library loads ahead of the executable's own libasan, which ASan
+# refuses by default. The shim allocates nothing, so the ordering is harmless.
+export ASAN_OPTIONS="${ASAN_OPTIONS:+$ASAN_OPTIONS:}verify_asan_link_order=0"
+
+# The unlink fallback is dead code on POSIX, so borrow Windows' rename().
+out=$(LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
+    -#test=renameover "$dir")
+echo "$out" | grep -q "renameover: OK"
+echo "$out" | grep -q "renameover: regime fallback"
+
+# A source another process holds fails with EACCES, which dst had no part in.
+out=$(RENAMEFAIL_MODE=locked LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
+    -#test=renameover "$dir")
+echo "$out" | grep -q "renameover: OK"
+echo "$out" | grep -q "renameover: regime refused"

tests/01_engine-savename.test

@@ -12,7 +12,7 @@ httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
 # scratch dir: body= and cached= write temp files (st-savename-body.tmp, hts-cache/)
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 cd "$scratch"
 
 run() {

tests/01_engine-structcheck.test

@@ -6,7 +6,7 @@
 set -euo pipefail
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=structcheck "$dir" |
     grep -q "structcheck self-test OK"

tests/01_engine-threadwait.test

@@ -4,7 +4,8 @@
 set -euo pipefail
 
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_threadwait_st.XXXXXX") || exit 1
-trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmpdir"' EXIT
+trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
 
 # No pipe into grep: SIGPIPE would mask a failing exit status.
 expect_ok() {

tests/01_engine-topindex.test

@@ -6,7 +6,7 @@ set -euo pipefail
 # hts_buildtopindex takes a system-charset path but verif_backblue below it
 # expects utf-8, mangling a non-ASCII project dir on Windows (#216, #217).
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=topindex "$dir" run |
     grep -q "topindex self-test OK"

tests/01_zlib-acceptencoding.test

@@ -5,7 +5,7 @@ set -euo pipefail
 
 # Accept-Encoding (#450): advertise gzip+deflate; decode gzip/zlib/raw-deflate.
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=acceptencoding "$dir" run |
     grep -q "acceptencoding self-test OK"

tests/01_zlib-cache-corrupt.test

@@ -7,7 +7,7 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # the smashed-header case logs expected "Corrupted cache entry" warnings on
 # stdout; the verdict is the last line

tests/01_zlib-cache-golden.test

@@ -27,7 +27,7 @@ test -e "$fixture/hts-cache/new.zip" || {
 }
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # Read against a private copy so the source tree is never touched (a read
 # session does not write, but copying keeps the test hermetic). Create the dir

tests/01_zlib-cache-legacy.test

@@ -6,7 +6,7 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # the refusal errors land on stdout (no log file); pin them and the verdict
 out=$(httrack -#test=cache-legacy "$dir" 2>/dev/null)

tests/01_zlib-cache-writefail.test

@@ -11,7 +11,7 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 out=$(httrack -#test=cache-writefail "$dir")
 

tests/01_zlib-cache.test

@@ -18,7 +18,7 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # The working directory is a required argument; without it the test prints a
 # usage line to stderr and returns non-zero.

tests/01_zlib-contentcodings.test

@@ -5,7 +5,7 @@ set -euo pipefail
 
 # brotli/zstd decode, unknown codings, and the decoded-size budget.
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 httrack -O /dev/null -#test=contentcodings "$dir" run |
     grep -q "contentcodings self-test OK"

tests/01_zlib-repair-shift.test

@@ -11,7 +11,7 @@
 set -eu
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 out=$(httrack -#test=zip-repair-shift "$dir")
 

tests/01_zlib-savename-cached.test

@@ -10,7 +10,7 @@ set -euo pipefail
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 cd "$scratch"
 
 name() {

tests/01_zlib-warc-cdx.test

@@ -9,7 +9,7 @@ set -euo pipefail
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 
 out=$("$httrack_bin" -O /dev/null -#test=warc-cdx "$scratch/")
 echo "$out"

tests/01_zlib-warc-wacz.test

@@ -16,7 +16,7 @@ if ! "$httrack_bin" -#test 2>&1 | grep -q '^  warc-wacz'; then
 fi
 
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 
 out=$("$httrack_bin" -O /dev/null -#test=warc-wacz "$scratch/")
 echo "$out"

tests/01_zlib-warc.test

@@ -10,7 +10,7 @@ set -euo pipefail
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 
 for t in warc warc-trunc warc-ftp warc-rotate warc-verbatim; do
     out=$("$httrack_bin" -O /dev/null "-#test=$t" "$scratch/")

tests/02_manpage-regen.test

@@ -29,7 +29,7 @@ command -v httrack >/dev/null 2>&1 || {
 tmp=$(mktemp) || exit 1
 committed_clean=$(mktemp) || exit 1
 generated_clean=$(mktemp) || exit 1
-trap 'rm -f "$tmp" "$committed_clean" "$generated_clean"' EXIT
+trap 'set +e; rm -f "$tmp" "$committed_clean" "$generated_clean"' EXIT
 
 README="$top_srcdir/README" bash "$gen" httrack >"$tmp" 2>/dev/null || {
     echo "makeman.sh failed" >&2

tests/02_update-cache.test

@@ -14,7 +14,7 @@ set -euo pipefail
 
 site=$(mktemp -d)
 out=$(mktemp -d)
-trap 'rm -rf "$site" "$out"' EXIT
+trap 'set +e; rm -rf "$site" "$out"' EXIT
 
 cat >"$site/index.html" <<EOF
 <a href="a.html">a</a> <a href="sub/b.html">b</a>

tests/100_local-purge-longpath.test

@@ -0,0 +1,111 @@
+#!/bin/bash
+#
+# The end-of-mirror purge read old.lst with linput(fp, line, 1000), which caps a
+# chunk at 999 bytes. filenote() writes "[<save path>]", so a 998-byte save path
+# makes a 1000-byte line that comes back as 999 bytes plus a one-byte "]" tail.
+# For that tail `line + 1` is empty, and with no -O so is path_html, so the purge
+# indexed file[-1] (#770). Needs no -O, hence not local-crawl.sh.
+#
+# The OOB write lands in stack padding, so only the sanitized CI leg fails on
+# unfixed code; the length assertion below is what keeps this from passing
+# vacuously if the save path ever stops landing on the chunk boundary.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
+# python3 is required; mirror check-network.sh's skip-with-77 convention.
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+
+# Resolve httrack now: the harness prepends a RELATIVE dir to PATH, so the cd
+# below would otherwise silently fall through to an installed copy.
+httrack_bin=$(command -v httrack)
+test -n "$httrack_bin" || {
+    echo "httrack not on PATH" >&2
+    exit 1
+}
+httrack_bin=$(cd "$(dirname "$httrack_bin")" && pwd)/$(basename "$httrack_bin")
+
+work=$(mktemp -d)
+trap 'set +e; kill -9 "${spid:-}" 2>/dev/null || true; rm -rf "$work"' EXIT
+
+mkdir -p "$work/root" "$work/proj"
+
+"$python" "$top_srcdir/tests/local-server.py" --root "$work/root" \
+    >"$work/server.out" 2>"$work/server.err" &
+spid=$!
+
+port=
+for _ in $(seq 1 100); do
+    port=$(sed -n 's/^PORT \([0-9]*\)$/\1/p' "$work/server.out" 2>/dev/null || true)
+    test -n "$port" && break
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "local-server.py did not announce a port" >&2
+    exit 1
+}
+
+# Save path is "127.0.0.1_<port>/" + the URL path; target 998 bytes total.
+prefix="127.0.0.1_${port}/"
+need=$((998 - ${#prefix}))
+
+# macOS caps PATH_MAX at 1024, so neither the docroot nor the mirror can hold a
+# 998-byte path, and httrack's own HTS_MAX_PATH_LEN clamps below it too. The
+# trigger is simply not reachable there.
+pathmax=$(getconf PATH_MAX / 2>/dev/null || echo 4096)
+test "$pathmax" -ge $((need + ${#work} + 128)) || {
+    echo "PATH_MAX $pathmax too small for a $need-byte path; skipping" >&2
+    exit 77
+}
+
+# Same on Win32, where MAX_PATH clamps the save path; getconf answers for the
+# MSYS emulation, not the Win32 API httrack calls, so it cannot see this.
+case "$(uname -s)" in
+MINGW* | MSYS* | CYGWIN*)
+    echo "Win32 MAX_PATH cannot hold a $need-byte save path; skipping" >&2
+    exit 77
+    ;;
+esac
+
+urlpath=""
+while test $((need - ${#urlpath})) -gt 60; do
+    urlpath="${urlpath}$(printf 'd%02d' ${#urlpath})$(printf 'a%.0s' $(seq 1 46))/"
+done
+urlpath="${urlpath}$(printf 'f%.0s' $(seq 1 $((need - ${#urlpath} - 5)))).html"
+test ${#urlpath} -eq "$need"
+
+mkdir -p "$work/root/$(dirname "$urlpath")"
+printf '<html><body>leaf</body></html>\n' >"$work/root/$urlpath"
+printf '<html><body><a href="/%s">x</a></body></html>\n' "$urlpath" \
+    >"$work/root/index.html"
+
+cd "$work/proj" # no -O: opt->path_html stays empty
+
+"$httrack_bin" "http://127.0.0.1:$port/" -q -%v0 -s0 -r3 >"$work/crawl1.log" 2>&1 || true
+
+lst=$work/proj/hts-cache/new.lst
+test -f "$lst" || {
+    echo "pass 1 produced no new.lst" >&2
+    exit 1
+}
+# Arms the test: without a line of exactly 1000 bytes linput() never splits and
+# the purge never sees a one-byte tail.
+awk 'length == 1000 { found = 1 } END { exit !found }' "$lst" || {
+    echo "no 1000-byte line in new.lst; the chunk split is not exercised" >&2
+    awk '{ print length }' "$lst" | sort -n | tail -3 >&2
+    echo "--- urlpath len ${#urlpath}, prefix $prefix" >&2
+    tail -20 "$work/crawl1.log" >&2
+    exit 1
+}
+
+# Pass 2 renames new.lst to old.lst and runs it through the purge loop.
+"$httrack_bin" "http://127.0.0.1:$port/" -q -%v0 -s0 -r3 >"$work/crawl2.log" 2>&1
+
+test -s "$work/proj/$prefix$urlpath" || {
+    echo "the mirrored leaf did not survive the purge" >&2
+    exit 1
+}

tests/102_local-ftp-refetch.test

@@ -0,0 +1,164 @@
+#!/bin/bash
+#
+# An FTP re-fetch used to truncate the mirror before the transfer, so a failed
+# one destroyed the previous copy (#771). keep.bin is cut short and empty.bin
+# gets nothing; both must keep their old bytes. stay.bin takes the same path and
+# completes, so a fix that merely stopped writing would fail here.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+out="${tmpdir}/crawl"
+mode="${tmpdir}/mode"
+report="${out}/hts-changes.json"
+mkdir -p "$root" "$out"
+
+# The two generations differ in length, or a re-fetch that resumed at EOF
+# instead of truncating would leave the same bytes and pass for the wrong reason.
+write_bodies() {
+    local gen="$1" fill="$2" name
+    for name in keep empty stay; do
+        "$python" -c 'import sys; sys.stdout.buffer.write(
+            ("%s-FTP-%s " % (sys.argv[1].upper(), sys.argv[2])).encode()
+            + sys.argv[2][-1].encode() * int(sys.argv[3]))' "$name" "$gen" "$fill" \
+            >"${root}/${name}.bin"
+    done
+}
+write_bodies V1 4096
+: >"$mode"
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" \
+    --mode-file "$(nativepath "$mode")" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 300); do
+    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "ftp server exited early: $(cat "$serverlog")" >&2
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
+    exit 1
+}
+host="127.0.0.1_${port}"
+urls=()
+for name in keep empty stay; do
+    urls+=("ftp://127.0.0.1:${port}/${name}.bin")
+done
+# -c1: a parallel FTP crawl loses whole transfers to a separate, older bug in
+# the backlog slot swap (#797), which would flake this test on a loaded runner.
+common=(--quiet --disable-security-limits --robots=0 --timeout=20 --max-time=120
+    --retries=1 -c1 --changes)
+
+fail() {
+    echo "FAIL: $*" >&2
+    test ! -f "$report" || cat "$report" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+size_of() { wc -c <"$1" | tr -d '[:space:]'; }
+# Files listed under a report bucket, one "file:size" per line.
+listed() {
+    "$python" - "$report" "$1" <<'EOF' | tr -d '\r'
+import json
+import sys
+
+with open(sys.argv[1], encoding="utf-8") as fp:
+    report = json.load(fp)
+for entry in report[sys.argv[2]]:
+    print("%s:%s" % (entry["file"], entry.get("size", "none")))
+EOF
+}
+
+# --- pass 1: a healthy mirror ------------------------------------------------
+# Bounded like every local-crawl.sh pass: a wedged crawl must fail the test, not
+# hang the job until CI cancels it and discards the log (#796).
+run_with_timeout 300 httrack "${urls[@]}" -O "$out" "${common[@]}" \
+    >"${tmpdir}/log1" 2>&1
+for name in keep empty stay; do
+    test -s "${out}/${host}/${name}.bin" || fail "pass 1 did not mirror ${name}.bin"
+done
+mkdir "${tmpdir}/snap"
+cp "${out}/${host}"/*.bin "${tmpdir}/snap/"
+ok "pass 1 mirrored three files"
+
+# --- pass 2: the same three re-fetched, two of them failing -------------------
+# norest throughout: an accepted REST sends the client down the append path,
+# which is not the one that truncates.
+write_bodies V2 6000
+cat >"$mode" <<'EOF'
+/keep.bin truncate norest
+/empty.bin empty norest
+/stay.bin norest
+EOF
+run_with_timeout 300 httrack "${urls[@]}" -O "$out" "${common[@]}" --update \
+    >"${tmpdir}/log2" 2>&1
+
+# Both failures have to be the transfer dying mid-body: a connect or lookup
+# failure never opens the file, and everything below would pass vacuously.
+for name in keep empty; do
+    grep -aqE "Error:.*\"FTP file incomplete\".*${name}\.bin" "${out}/hts-log.txt" ||
+        fail "${name}.bin did not fail during the transfer"
+done
+ok "both re-fetches reached the body and failed there"
+
+for name in keep empty; do
+    cmp -s "${out}/${host}/${name}.bin" "${tmpdir}/snap/${name}.bin" ||
+        fail "${name}.bin differs from the copy pass 1 left ($(size_of \
+            "${out}/${host}/${name}.bin") bytes now)"
+done
+ok "a failed FTP transfer left the previously mirrored bytes alone"
+
+grep -aq "FTP-V2 " "${out}/${host}/stay.bin" ||
+    fail "stay.bin was not refreshed; the re-fetch writes nothing at all now"
+ok "a successful FTP re-fetch still replaces the file"
+
+leftover=$(find "${out}/${host}" -name '*.bak')
+test -z "$leftover" || fail "backup left behind: ${leftover}"
+ok "no backup temporary survived the pass"
+
+# --purge-old is on by default: a kept file the run never replaced must still be
+# in new.lst, or it is deleted as gone (#746).
+for name in keep empty stay; do
+    test -f "${out}/${host}/${name}.bin" || fail "${name}.bin was purged"
+done
+ok "the kept copies survived the update purge"
+
+# --- the change report is the mirror's own account of the pass ----------------
+test -s "$report" || fail "pass 2 wrote no ${report}"
+test -z "$(listed gone)" || fail "the report calls something gone: $(listed gone)"
+for name in keep empty; do
+    listed unchanged | grep -qx "${host}/${name}.bin:$(size_of "${tmpdir}/snap/${name}.bin")" ||
+        fail "${name}.bin is not reported unchanged at its previous size"
+    if listed changed | grep -q "^${host}/${name}.bin:"; then
+        fail "${name}.bin is reported changed as well"
+    fi
+done
+listed changed | grep -qx "${host}/stay.bin:$(size_of "${out}/${host}/stay.bin")" ||
+    fail "stay.bin is not reported changed"
+ok "the change report calls the kept files unchanged and the refreshed one changed"

tests/103_teardown-status.test

@@ -0,0 +1,81 @@
+#!/bin/bash
+#
+# Issue #773: a test must fail only when an assertion fails. Under "set -e" a
+# failing command in an EXIT trap becomes the script's exit status, so a
+# teardown hiccup turned a test whose assertions all passed into a red CI leg.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+tmp=$(mktemp -d)
+trap 'set +e; rm -rf "$tmp"' EXIT
+
+fail() {
+    echo "FAIL: $1" >&2
+    exit 1
+}
+
+# run <trap-body> <script-body>: exit status and stderr of a subject script.
+run() {
+    {
+        echo "set -euo pipefail"
+        # printf, so the scanner below does not read this as a trap of our own
+        printf 'trap %s EXIT\n' "$1"
+        echo "$2"
+    } >"$tmp/subject"
+    rc=0
+    err=$(bash "$tmp/subject" 2>&1 >/dev/null) || rc=$?
+}
+
+# 1. the shape in the tree: teardown fails, every assertion passed -> pass
+run "'set +e; false'" "true"
+test "$rc" -eq 0 || fail "guarded teardown failure exited $rc, want 0"
+
+# 2. teeth for case 1: unguarded, the same teardown is what went red
+run "'false'" "true"
+test "$rc" -eq 1 || fail "unguarded teardown failure exited $rc, want 1"
+
+# 3. the guard must not swallow a real failure, nor a skip
+run "'set +e; false'" "exit 1"
+test "$rc" -eq 1 || fail "assertion failure exited $rc, want 1"
+run "'set +e; false'" "exit 77"
+test "$rc" -eq 77 || fail "skip exited $rc, want 77"
+
+# 4. not failing must not mean going quiet: teardown still reports why
+run "'set +e; rm /nonexistent-teardown-probe'" "true"
+test "$rc" -eq 0 || fail "guarded teardown failure exited $rc, want 0"
+case "$err" in
+*nonexistent-teardown-probe*) ;;
+*) fail "teardown error was silenced (stderr: '$err')" ;;
+esac
+echo "OK: teardown cannot decide the verdict, and still reports"
+
+# 5. no test may reintroduce the shape case 2 pins. Two spellings neutralise
+# errexit for teardown: "set +e" first, or "|| true" over the whole body.
+scanned=0
+for f in "$top_srcdir"/tests/*.test "$top_srcdir"/tests/*.sh; do
+    test -r "$f" || continue
+    scanned=$((scanned + 1))
+done
+# an unexpanded glob would report no offender having read nothing
+test "$scanned" -ge 50 || fail "scanned $scanned scripts in $top_srcdir/tests, want the whole suite"
+
+bad=$(awk '
+    FNR == 1 { errexit = 0 }
+    /^[[:space:]]*set[[:space:]]+(-[a-z]*e|-o[[:space:]]+errexit)/ { errexit = 1 }
+    !errexit { next }
+    !/^[[:space:]]*trap[[:space:]]+.*[[:space:]]EXIT([[:space:]]|;|$)/ { next }
+    {
+        guarded = /^[[:space:]]*trap[[:space:]]+.set \+e; / || /\|\|[[:space:]]+true.[[:space:]]+EXIT/
+        if (!guarded)
+            print FILENAME ":" FNR " unguarded"
+        # "set +e" on a trap shared with a signal leaves errexit off for the
+        # rest of the run, so a torn-down test could still report success
+        else if (/^[[:space:]]*trap[[:space:]]+.set \+e; / && /[[:space:]]EXIT[[:space:]]+[A-Z]/)
+            print FILENAME ":" FNR " set +e shared with a signal trap"
+    }
+' "$top_srcdir"/tests/*.test "$top_srcdir"/tests/*.sh)
+test -z "$bad" || fail "EXIT trap can decide the verdict:
+$bad"
+echo "OK: every EXIT trap under set -e runs teardown with errexit off"

tests/104_engine-warc-longurl.test

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# A URL past the header-format buffer must still produce a record: the archive
+# used to lose it whole, with no error and an exit status of 0 (#785).
+
+set -e
+
+: "${top_srcdir:=..}"
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_longurl.XXXXXX")
+trap 'set +e; rm -rf "$tmp"' EXIT
+
+httrack -O "$tmp/out" "-#test=warc-longurl" "$tmp"

tests/105_suite-timeout.test

@@ -0,0 +1,141 @@
+#!/bin/bash
+#
+# The harness timeout (tests/test-timeout.sh) must name the test it killed, say
+# what was still running, and exit 124 -- otherwise a wedge only ever shows up as
+# a cancelled CI step, which keeps no log and no artifact.
+
+set -eu
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+driver="${testdir}/test-timeout.sh"
+test -r "$driver" || {
+    echo "no test-timeout.sh next to $0" >&2
+    exit 1
+}
+
+fail() {
+    echo "FAIL: $*" >&2
+    test -z "${out:-}" || sed 's/^/  | /' <"$out" >&2
+    exit 1
+}
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_sutmo.XXXXXX") || exit 1
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
+out="$tmp/out"
+
+# --- a hanging test is killed, named, and reported 124 ----------------------
+cat >"$tmp/90_wedged.test" <<'EOF'
+echo "marker: the wedged test started"
+sleep 300
+EOF
+
+start=$SECONDS
+rc=0
+HTTRACK_TEST_TIMEOUT=5 bash "$driver" "$tmp/90_wedged.test" >"$out" 2>&1 || rc=$?
+elapsed=$((SECONDS - start))
+
+test "$rc" -eq 124 || fail "wedged test reported $rc, want 124"
+# Never before the budget, or a slow-but-healthy test would be killed too.
+test "$elapsed" -ge 5 || fail "the guard fired early (${elapsed}s of a 5s budget)"
+test "$elapsed" -lt 30 || fail "the guard fired late (${elapsed}s)"
+grep -q 'marker: the wedged test started' "$out" || fail "the test's own output was lost"
+# The header, not a bare name: the process list quotes the test's path too, so a
+# wrapper that named nothing would still match that.
+grep -q '^===== TIMEOUT: 90_wedged.test exceeded' "$out" ||
+    fail "the diagnostics do not name the test"
+grep -q "own process tree" "$out" || fail "no process list in the diagnostics"
+
+# The budget is read, not hard-coded: well under it, the same shape survives.
+printf 'sleep 3\necho "slow but healthy"\n' >"$tmp/92_slow.test"
+rc=0
+HTTRACK_TEST_TIMEOUT=45 bash "$driver" "$tmp/92_slow.test" >"$out" 2>&1 || rc=$?
+test "$rc" -eq 0 || fail "a healthy 3s test under a 45s budget reported $rc"
+grep -q 'slow but healthy' "$out" || fail "healthy test output lost"
+! grep -q 'TIMEOUT' "$out" || fail "the guard fired on a healthy test"
+
+# The killed tree must really be gone, or the next test inherits its ports.
+sleep 1
+! grep -q "$tmp/90_wedged.test" <<<"$(ps -A -o args 2>/dev/null)" ||
+    fail "the wedged test survived the guard"
+
+# --- exit status and output of a healthy test pass straight through ----------
+# 77 is automake's SKIP and 99 its hard error, so both have to survive the wrapper.
+for want in 0 1 77 99; do
+    printf 'echo "ran with %s"\nexit %s\n' "$want" "$want" >"$tmp/91_plain.test"
+    rc=0
+    HTTRACK_TEST_TIMEOUT=60 bash "$driver" "$tmp/91_plain.test" >"$out" 2>&1 || rc=$?
+    test "$rc" -eq "$want" || fail "exit $want came back as $rc"
+    grep -q "ran with $want" "$out" || fail "output lost for exit $want"
+    ! grep -q 'TIMEOUT' "$out" || fail "the guard fired on a test that exited at once"
+done
+
+# A budget of 0 disables the guard: the test still runs, and is not killed. Its
+# own file, so the assertion cannot ride on whatever the loop above left behind.
+printf 'echo "guard off"\nexit 3\n' >"$tmp/93_off.test"
+rc=0
+HTTRACK_TEST_TIMEOUT=0 bash "$driver" "$tmp/93_off.test" >"$out" 2>&1 || rc=$?
+test "$rc" -eq 3 || fail "disabled guard changed the exit status ($rc)"
+grep -q 'guard off' "$out" || fail "output lost with the guard disabled"
+
+# --- a wedged crawl yields a symbolized engine stack ------------------------
+# The whole point of the dump: name the frame the engine is stuck in. Windows has
+# neither half (MSYS signals do not reach a native httrack.exe, and that build has
+# no backtrace()), so only assert it where the machinery exists.
+if is_windows; then
+    echo "suite timeout OK (no stack capture on Windows)"
+    exit 0
+fi
+command -v httrack >/dev/null || {
+    echo "SKIP: no httrack on PATH for the stack half" >&2
+    exit 77
+}
+
+# /watchdog/stall never answers, so the crawl wedges for real. CRAWL_DEADLINE is
+# raised past our budget so the guard under test is what fires, not local-crawl's.
+cat >"$tmp/92_stalled.test" <<EOF
+CRAWL_DEADLINE=600 bash "${testdir}/local-crawl.sh" --errors 0 \\
+    httrack 'BASEURL/watchdog/stall'
+EOF
+
+rc=0
+HTTRACK_TEST_TIMEOUT=20 bash "$driver" "$tmp/92_stalled.test" >"$out" 2>&1 || rc=$?
+
+if grep -qE 'python3 not found|could not discover server port' "$out"; then
+    echo "SKIP: local crawl prerequisites missing" >&2
+    exit 77
+fi
+test "$rc" -eq 124 || fail "stalled crawl reported $rc, want 124"
+
+if grep -q 'No stack trace available on this OS' "$out"; then
+    echo "suite timeout OK (no backtrace() on this platform)"
+    exit 0
+fi
+
+# Each platform's mechanism has its own marker, and the marker is what proves the
+# whole chain connected up rather than the dump merely running.
+case "$(uname -s)" in
+Linux)
+    # sig_fatal writes to the engine's own stderr, so this reaching our log also
+    # proves the crawl-log salvage worked.
+    grep -q '^Caught signal ' "$out" || fail "no engine backtrace was salvaged"
+    ;;
+Darwin)
+    grep -q '^Call graph:' "$out" || fail "sample(1) produced no call graph"
+    ;;
+*)
+    grep -q 'no stack mechanism known' "$out" ||
+        fail "unknown platform reported neither a stack nor its absence"
+    echo "suite timeout OK (no stack mechanism on $(uname -s))"
+    exit 0
+    ;;
+esac
+# A stalled crawl is necessarily inside httpmirror, so these frames must appear.
+# Matching "httrack" alone would pass on the process list above.
+grep -qE '\b(back_wait|httpmirror|hts_main_internal)\b' "$out" ||
+    fail "the stack names no engine frame"
+
+echo "suite timeout OK (named the test, dumped a stack, exited 124)"

tests/116_engine-rtrim.test

@@ -0,0 +1,107 @@
+#!/bin/bash
+#
+# Backward scans that start at s + strlen(s) - 1 and decrement with no lower
+# bound: the config-file right trim and the collision-rename digit scan both
+# walked off the front of a stack buffer. Neither ASan nor _FORTIFY_SOURCE sees
+# the self-test's overrun (it lands in the same frame), hence the poisoned byte.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# Resolve httrack before any cd: PATH may carry a build-relative entry.
+bin=$(command -v httrack) || {
+    echo "FAIL: httrack not found on PATH" >&2
+    exit 1
+}
+case "$bin" in
+/*) ;;
+*) bin="$(cd "$(dirname "$bin")" && pwd)/$(basename "$bin")" ;;
+esac
+
+tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_rtrim.XXXXXX") || exit 1
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
+
+fail() {
+    echo "FAIL: $1" >&2
+    exit 1
+}
+
+# --- 1. the helper itself ---------------------------------------------------
+out=$("$bin" -#test=rtrim) || fail "httrack -#test=rtrim exited non-zero: $out"
+test "$out" = "rtrim self-test: OK" || fail "expected 'rtrim self-test: OK', got: $out"
+
+# --- 2. collision rename over an all-digit name -----------------------------
+# Two pages with the same all-digit name save to 2024.html, so the second goes
+# through the -<n> rename and the digit scan has nothing but digits to walk
+# over. Driven twice, because the two routes differ in who picks the URLs.
+site="$tmp/site"
+mkdir -p "$site/a" "$site/b"
+printf '%s\n' '<html><body><a href="a/2024.html">a</a> <a href="b/2024.html">b</a></body></html>' \
+    >"$site/index.html"
+echo '<html><body>A</body></html>' >"$site/a/2024.html"
+echo '<html><body>B</body></html>' >"$site/b/2024.html"
+
+# -g flattens the tree, and pins depth to 0: the operator supplies both URLs.
+out1="$tmp/out1"
+mkdir -p "$out1"
+(cd "$out1" && "$bin" -g "file://$site/a/2024.html" "file://$site/b/2024.html" \
+    --quiet -n >.log 2>&1) || fail "collision crawl exited non-zero: $(tail -3 "$out1/.log")"
+
+test -f "$out1/2024.html" || fail "2024.html not mirrored"
+test -f "$out1/2024-2.html" || fail "colliding name was not renamed to 2024-2.html"
+
+# -N "%n.%t" collides the same way with depth unrestricted, so one starting URL
+# is enough and the crawled page picks both colliding names itself.
+out3="$tmp/out3"
+mkdir -p "$out3"
+(cd "$out3" && "$bin" "file://$site/index.html" -N "%n.%t" --quiet -n >.log 2>&1) ||
+    fail "userdef-savename crawl exited non-zero: $(tail -3 "$out3/.log")"
+test -f "$out3/2024.html" || fail "2024.html not mirrored under -N '%n.%t'"
+test -f "$out3/2024-2.html" ||
+    fail "colliding name was not renamed under -N '%n.%t' (links followed, not argv)"
+
+# The rename must keep counting, not restart: a third collision is -3.
+mkdir -p "$site/c"
+echo '<html><body>C</body></html>' >"$site/c/2024.html"
+out2="$tmp/out2"
+mkdir -p "$out2"
+(cd "$out2" && "$bin" -g "file://$site/a/2024.html" "file://$site/b/2024.html" \
+    "file://$site/c/2024.html" --quiet -n >.log 2>&1) ||
+    fail "three-way collision crawl exited non-zero: $(tail -3 "$out2/.log")"
+test -f "$out2/2024-3.html" || fail "third colliding name was not renamed to 2024-3.html"
+
+# --- 3. a blank config line is skipped, not parsed ---------------------------
+# HTS_HTTRACKRC is ".httrackrc" on POSIX and "httrackrc" on Windows.
+rcdir="$tmp/rc"
+mkdir -p "$rcdir"
+echo '<html><body>hi</body></html>' >"$rcdir/index.html"
+printf '   \nzzz-not-an-option\n' >"$rcdir/.httrackrc"
+cp "$rcdir/.httrackrc" "$rcdir/httrackrc"
+
+rc=0
+log=$( (cd "$rcdir" && "$bin" "file://$rcdir/index.html" --quiet -n 2>&1)) || rc=$?
+test "$rc" -eq 0 || fail "rc-file crawl exited $rc: $log"
+log=${log//$'\r'/} # the empty-option check below must not miss a CRLF line
+
+# The line after the blank one must still be read, or this proves nothing.
+case "$log" in
+*"Unknown option: zzz-not-an-option"*) ;;
+*) fail "config file was not parsed past the blank line: $log" ;;
+esac
+# The blank line itself must not become an empty option.
+case "$log" in
+*"Unknown option: "$'\n'*) fail "blank config line was parsed as an empty option: $log" ;;
+esac
+
+# --- 4. the call sites stay bounded -----------------------------------------
+# Sections 2 and 3 only go red under a sanitizer: on a plain build the byte
+# below the buffer is usually not a digit, so a reverted scan stops anyway and
+# produces the same names. Pin the two sites at the source instead.
+command grep -q 'hts_rtrim(line, HTS_REALSPACES)' "$top_srcdir/src/htsalias.c" ||
+    fail "htsalias.c right trim no longer goes through hts_rtrim"
+command grep -q 'hts_rtrimlen(tempo, "0123456789")' "$top_srcdir/src/htsname.c" ||
+    fail "htsname.c collision-suffix scan no longer goes through hts_rtrimlen"
+
+exit 0

tests/13_crawl_proxy_https.test

@@ -31,7 +31,7 @@ cleanup() {
     done
     rm -rf "$tmpdir"
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # self-signed cert for the local TLS origin (httrack does not verify certs)
 openssl req -x509 -newkey rsa:2048 -keyout "$tmpdir/key.pem" \

tests/19_local-connect-fallback.test

@@ -40,9 +40,8 @@ serverpid=
 cleanup() {
     stop_server "$serverpid"
     rm -rf "$tmpdir"
-    return 0
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # bind the live server to 127.0.0.1 only, so 127.0.0.2 refuses the connect
 "$python" "$server" --root "$root" --bind 127.0.0.1 >"$tmpdir/srv.out" 2>"$tmpdir/srv.err" &

tests/24_local-resume-overlap.test

@@ -24,7 +24,8 @@ cleanup() {
     stop_server "$serverpid"
     rm -rf "$tmpdir"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 # OVERLAP_COUNTER gets a byte per flaky.bin request so pass 1 knows when to interrupt.
 serverlog="${tmpdir}/server.log"

tests/40_local-why.test

@@ -6,7 +6,7 @@ set -euo pipefail
 # --why: report which +/- filter rule decides for a URL, without crawling.
 
 tmpdir="$(mktemp -d)"
-trap 'rm -rf "$tmpdir"' EXIT
+trap 'set +e; rm -rf "$tmpdir"' EXIT
 
 why() {
     local want="$1" out

tests/52_local-socks5.test

@@ -34,7 +34,7 @@ cleanup() {
     done
     rm -rf "$tmpdir"
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 openssl req -x509 -newkey rsa:2048 -keyout "$tmpdir/key.pem" \
     -out "$tmpdir/cert.pem" -days 2 -nodes -subj "/CN=127.0.0.1" \

tests/53_local-proxytrack-arc-reason.test

@@ -5,7 +5,7 @@
 set -euo pipefail
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 printf 'HTTP/1.1 404 Not Found Here At All\r\nContent-Type: text/html\r\nLast-Modified: Wed, 01 Jan 2025 00:00:00 GMT\r\nContent-Length: 5\r\n\r\n' >"$dir/hdr"
 printf 'hello' >"$dir/body"

tests/53_local-proxytrack-cache-corrupt.test

@@ -5,7 +5,7 @@
 set -euo pipefail
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # The first length-prefixed field declares far more than firstline[256] holds;
 # cache_brstr must clamp the copy to the destination, not the declared length.

tests/56_local-proxy-noleak.test

@@ -29,9 +29,8 @@ serverpid=
 cleanup() {
     stop_server "$serverpid"
     rm -rf "$tmpdir"
-    return 0
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # decoy origin: it must stay silent. LOCAL_SERVER_VERBOSE logs any request it gets.
 LOCAL_SERVER_VERBOSE=1 "$python" "$server" --root "$root" --bind 127.0.0.1 \

tests/57_local-proxy-connect.test

@@ -29,7 +29,7 @@ cleanup() {
     done
     rm -rf "$tmpdir"
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # start_server <logdir> <mode>: launches a proxy+origin pair, sets $origin_port
 # and $proxy_port from its announced ephemeral ports.

tests/58_watchdog.test

@@ -12,7 +12,7 @@ testdir=$(cd "$(dirname "$0")" && pwd)
 . "${testdir}/testlib.sh"
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_wd.XXXXXX")
-trap 'rm -rf "$tmp"' EXIT
+trap 'set +e; rm -rf "$tmp"' EXIT
 fail() {
     echo "FAIL: $*" >&2
     exit 1

tests/59_local-tls-stall.test

@@ -27,9 +27,8 @@ serverpid=
 cleanup() {
     stop_server "$serverpid"
     rm -rf "$tmpdir"
-    return 0
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # start_stall_server <tag> <mode-args...>: sets $port from the announced one.
 start_stall_server() {

tests/60_crawl-log-salvage.test

@@ -12,7 +12,7 @@ testdir=$(cd "$(dirname "$0")" && pwd)
 
 TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/httrack_salv.XXXXXX")
 export TMPDIR
-trap 'rm -rf "$TMPDIR"' EXIT
+trap 'set +e; rm -rf "$TMPDIR"' EXIT
 fail() {
     echo "FAIL: $*" >&2
     exit 1

tests/62_lang-integrity.test

@@ -19,7 +19,7 @@ for f in "$eng" "$def"; do
 done
 
 tmp=$(mktemp -d)
-trap 'rm -rf "$tmp"' EXIT
+trap 'set +e; rm -rf "$tmp"' EXIT
 keys="$tmp/english.keys"
 
 # Byte-wise: each file is in its own declared legacy charset, lang.def is CRLF.

tests/63_webhttrack-home.test

@@ -25,10 +25,11 @@ srv=
 cleanup() {
     test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 log=$(mktemp)
-trap 'cleanup; rm -f "${log}"' EXIT
+trap 'set +e; cleanup; rm -f "${log}"' EXIT
 
 # Ask htsserver for its rendered default base path, under the given $HOME
 # ("-" = leave $HOME unset).

tests/65_port-siblings.test

@@ -14,7 +14,8 @@ testdir=$(cd "$(dirname "$0")" && pwd)
 . "${testdir}/testlib.sh"
 
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_portsib.XXXXXX") || exit 1
-trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmp"' EXIT
+trap 'rm -rf "$tmp"' HUP INT QUIT PIPE TERM
 
 # A port the old code accepted only by wrapping: 4295009395 = 2^32 + 42099 and
 # 4294967376 = 2^32 + 80 both fold to a plausible listen port. They are the

tests/67_engine-delayed-truncate.test

@@ -14,7 +14,7 @@ set -euo pipefail
 
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 cd "$scratch"
 
 hexseg=$(printf 'a1b2c3d4e5f60718%.0s' {1..8}) # 128 hex chars

tests/68_webhttrack-outdir-charset.test

@@ -36,7 +36,8 @@ cleanup() {
     wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
     rm -rf "${work}" "${srvlog}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 # webhttrack server on a pre-picked port; an isolated HOME keeps a stray
 # ~/.httrack.ini out of it.

tests/75_engine-longpath-posix.test

@@ -9,7 +9,7 @@ set -euo pipefail
 
 httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
 scratch=$(mktemp -d)
-trap 'rm -rf "$scratch"' EXIT
+trap 'set +e; rm -rf "$scratch"' EXIT
 cd "$scratch"
 
 # ~300 chars: over the old 236 ceiling, well under any POSIX PATH_MAX. The

tests/76_cli-resize.test

@@ -23,7 +23,8 @@ cleanup() {
     stop_server "$serverpid"
     rm -rf "$tmpdir"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 serverlog="${tmpdir}/server.log"
 : >"$serverlog"

tests/77_webhttrack-redirect.test

@@ -32,7 +32,8 @@ cleanup() {
     stop
     rm -f "${log}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 freeport() {
     python3 -c 'import socket

tests/78_webhttrack-sid.test

@@ -26,7 +26,8 @@ cleanup() {
     test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
     rm -f "${log}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 freeport() {
     python3 -c 'import socket

tests/79_local-proxytrack-webdav-mime.test

@@ -30,7 +30,7 @@ cleanup() {
     stop_server "$ptpid"
     rm -rf "$dir"
 }
-trap cleanup EXIT
+trap 'set +e; cleanup' EXIT
 
 # Neither header is present, so file->contenttype and ->lastmodified stay "".
 printf 'HTTP/1.1 200 OK\r\nContent-Length: 5\r\n\r\n' >"$dir/hdr"

tests/80_engine-crash-symbolize.test

@@ -17,7 +17,8 @@ command -v httrack >/dev/null || {
 }
 
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_symbolize.XXXXXX") || exit 1
-trap 'rm -rf "$tmpdir"' EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; rm -rf "$tmpdir"' EXIT
+trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
 out="${tmpdir}/trace"
 raw="${tmpdir}/trace-optout"
 overflow="this string is far too long for the buffer"

tests/81_webhttrack-maxsize.test

@@ -32,7 +32,8 @@ cleanup() {
     srv=
     rm -rf "${work}" "${srvlog}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 # webhttrack server on a pre-picked port; an isolated HOME keeps a stray
 # ~/.httrack.ini out of it.

tests/82_webhttrack-browse-links.test

@@ -38,7 +38,8 @@ cleanup() {
     wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
     rm -rf "${work}" "${srvlog}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 # Stand in for a finished mirror: the crawl itself is not under test.
 proj="${work}/websites/proj"

tests/83_webhttrack-argescape.test

@@ -27,7 +27,8 @@ cleanup() {
     test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
     rm -f "${log}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 freeport() {
     python3 -c 'import socket

tests/84_webhttrack-mirror-verbatim.test

@@ -34,7 +34,8 @@ cleanup() {
     wait "${csrv}" 2>/dev/null || true # absorb bash's async "Killed" notice
     rm -rf "${log}" "${work}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 freeport() {
     python3 -c 'import socket

tests/85_webhttrack-projpath.test

@@ -28,7 +28,8 @@ cleanup() {
     rm -f "${log}"
     rm -rf "${base}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 freeport() {
     python3 -c 'import socket

tests/86_local-proxytrack-cache-longfields.test

@@ -9,7 +9,7 @@ testdir=$(cd "$(dirname "$0")" && pwd)
 . "${testdir}/testlib.sh"
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 pad() { printf "%${1}s" '' | tr ' ' "$2"; }
 

tests/87_local-proxytrack-nodate.test

@@ -5,7 +5,7 @@
 set -euo pipefail
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # $1 Last-Modified value (empty = omit the header), $2 expected archive date,
 # $3 label. The date is asserted exactly: a guard that fires unconditionally

tests/88_local-proxytrack-badmtime.test

@@ -14,7 +14,7 @@ python=$(find_python) || {
 }
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 rc=0
 "$python" - "$dir/in.zip" <<'EOF' || rc=$?

tests/89_webhttrack-error-overflow.test

@@ -33,7 +33,8 @@ cleanup() {
     rm -f "${log}"
     rm -rf "${base}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 freeport() {
     python3 -c 'import socket

tests/90_webhttrack-checkbox-clear.test

@@ -33,7 +33,8 @@ cleanup() {
     wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
     rm -rf "${work}" "${srvlog}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 # An isolated HOME keeps a stray ~/.httrack.ini out of the served settings.
 sport=$("${python}" -c 'import socket

tests/91_webhttrack-directory.test

@@ -28,7 +28,8 @@ cleanup() {
     rm -f "${log}"
     rm -rf "${base}"
 }
-trap cleanup EXIT HUP INT QUIT PIPE TERM
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
 
 # First line only. A "| head -1" would close the pipe early and, under pipefail,
 # SIGPIPE the producer into a spurious failure.

tests/92_local-proxytrack-ndx-fields.test

@@ -15,7 +15,7 @@ python=$(find_python) || {
 }
 
 dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"' EXIT
 
 # each field fills binput's own bound, so together they run one past line[]
 "$python" - "$dir/foo.ndx" <<'EOF'