Merge branch 'master' into fix/proxytrack-dav-default-doc

Union both sides' expected-skip entries and TESTS-tail additions,
deduplicating and restoring numeric order.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/windows-build.yml

@@ -260,10 +260,11 @@ jobs:
           # tested nothing: pin the skips, and floor the passes in case the glob empties.
           # footer-overflow and purge-longpath skip on Windows (need a path past MAX_PATH);
           # crange pending #581;
-          # webdav-mime needs a reapable background listener, which MSYS cannot give it;
+          # webdav-default and webdav-mime need a reapable background listener, which MSYS cannot give them;
           # badmtime needs a filesystem that stores an mtime past gmtime's range;
-          # single-file ends on a GUI half needing htsserver, which this job does not build.
-          expected_skips=" 01_engine-footer-overflow.test 100_local-purge-longpath.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
+          # single-file ends on a GUI half needing htsserver, which this job does not build;
+          # update-304-leak needs a LeakSanitizer build, which MSVC has no equivalent of.
+          expected_skips=" 01_engine-footer-overflow.test 100_local-purge-longpath.test 114_local-update-304-leak.test 120_local-proxytrack-webdav-default.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
           # First, or the deadline reads as an unexplained shortfall in the gates below.
           [ "$deadline" -eq 0 ] || { echo "::error::suite did not finish within ${suite_deadline}s"; exit 1; }
           [ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }

src/htsback.c

@@ -391,6 +391,56 @@ int back_selftest_slot_swap(void) {
   CHECK(0, "the dummy test slot");
 #undef CHECK
 
+  /* The swap round-trip must not lose the size of a slot whose body is already
+     at url_sav, or the link writer blanks the file (#797). */
+  {
+    static const char body[] = "swapped body";
+    int c;
+
+    for (c = 0; c < 2; c++) {
+      const hts_boolean inmemory = c == 0 ? HTS_TRUE : HTS_FALSE;
+      FILE *const fp = tmpfile();
+      lien_back *copy = NULL;
+
+      memset(&back, 0, sizeof(back));
+      back.status = STATUS_READY;
+      strcpybuff(back.url_sav, "/tmp/httrack-selftest.bin");
+      back.r.size = (LLint) sizeof(body) - 1;
+      if (inmemory) {
+        back.r.adr = strdupt(body);
+      }
+      if (fp == NULL || back_serialize(fp, &back) != 0 ||
+          fseek(fp, 0, SEEK_SET) != 0 || back_unserialize(fp, &copy) != 0) {
+        fprintf(stderr, "backswap: round-trip failed for a %s slot\n",
+                inmemory ? "buffered" : "direct-to-disk");
+        err = 1;
+      } else {
+        if (copy->r.size != back.r.size) {
+          fprintf(stderr,
+                  "backswap: %s slot came back with size " LLintP
+                  ", expected " LLintP "\n",
+                  inmemory ? "buffered" : "direct-to-disk", copy->r.size,
+                  back.r.size);
+          err = 1;
+        }
+        if (inmemory && (copy->r.adr == NULL ||
+                         memcmp(copy->r.adr, body, sizeof(body) - 1) != 0)) {
+          fprintf(stderr, "backswap: buffered slot lost its body\n");
+          err = 1;
+        }
+        if (!inmemory && copy->r.adr != NULL) {
+          fprintf(stderr, "backswap: direct-to-disk slot gained a body\n");
+          err = 1;
+        }
+        back_clear_entry(copy);
+        freet(copy);
+      }
+      if (fp != NULL)
+        fclose(fp);
+      freet(back.r.adr);
+    }
+  }
+
   printf("backswap self-test: %s\n", err ? "FAIL" : "OK");
   return err;
 }
@@ -646,7 +696,7 @@ void back_refetch_backup(httrackp *opt, lien_back *const back) {
       if (fexist_utf8(back->tmpfile))
         hts_log_print(opt, LOG_WARNING, "replacing leftover backup %s",
                       back->tmpfile);
-      saved = hts_rename_over(back->url_sav, back->tmpfile);
+      saved = hts_rename_over(opt, back->url_sav, back->tmpfile);
     }
     if (!saved) {
       hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
@@ -687,7 +737,7 @@ static void back_finalize_backup(httrackp *opt, lien_back *const back,
     }
     /* On failure keep the backup: an orphaned temp beats losing the good copy.
      */
-    if (!hts_rename_over(back->tmpfile, back->url_sav))
+    if (!hts_rename_over(opt, back->tmpfile, back->url_sav))
       hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
                     "could not restore %s; previous copy kept as %s",
                     back->url_sav, back->tmpfile);
@@ -818,7 +868,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                                  "Read error when decompressing");
                     }
                     UNLINK(unpacked);
-                  } else if (hts_rename_over(unpacked, back[p].url_sav)) {
+                  } else if (hts_rename_over(opt, unpacked, back[p].url_sav)) {
                     /* The temp bypassed filecreate(), which is what chmods. */
 #ifndef _WIN32
                     chmod(back[p].url_sav, HTS_ACCESS_FILE);
@@ -1173,6 +1223,13 @@ void back_connxfr(htsblk * src, htsblk * dst) {
   src->debugid = 0;
 }
 
+/* Release the buffers a response owns. The connection members are left alone:
+   back_connxfr() moves those, and the file handles are closed elsewhere. */
+static void back_free_response(htsblk *r) {
+  deleteaddr(r);
+  warc_free_request(r);
+}
+
 void back_move(lien_back * src, lien_back * dst) {
   memcpy(dst, src, sizeof(lien_back));
   memset(src, 0, sizeof(lien_back));
@@ -1266,7 +1323,10 @@ int back_unserialize(FILE * fp, lien_back ** dst) {
     (*dst)->r.ssl_con = NULL;
 #endif
     if (back_data_unserialize(fp, (void **) &(*dst)->r.adr, &size) == 0) {
-      (*dst)->r.size = size;
+      /* A bodyless slot already wrote its bytes to url_sav (FTP, direct to
+         disk); zeroing r.size makes the writer blank that file (#797). */
+      if ((*dst)->r.adr != NULL)
+        (*dst)->r.size = size;
       (*dst)->r.headers = NULL;
       if (back_string_unserialize(fp, &(*dst)->r.headers) == 0)
         return 0;               /* ok */
@@ -1705,10 +1765,7 @@ int back_clear_entry(lien_back * back) {
       back->r.soc = INVALID_SOCKET;
     }
 
-    if (back->r.adr != NULL) {  // reste un bloc à désallouer
-      freet(back->r.adr);
-      back->r.adr = NULL;
-    }
+    back_free_response(&back->r);
     if (back->chunk_adr != NULL) {      // reste un bloc à désallouer
       freet(back->chunk_adr);
       back->chunk_adr = NULL;
@@ -1721,12 +1778,6 @@ int back_clear_entry(lien_back * back) {
       (void) unlink(back->tmpfile);
       back->tmpfile = NULL;
     }
-    // headers
-    if (back->r.headers != NULL) {
-      freet(back->r.headers);
-      back->r.headers = NULL;
-    }
-    warc_free_request(&back->r);
     // Tout nettoyer
     memset(back, 0, sizeof(lien_back));
     back->r.soc = INVALID_SOCKET;
@@ -4031,6 +4082,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
                       memset(&tmp, 0, sizeof(tmp));
                       back_connxfr(&back[i].r, &tmp);
+                      /* the cache entry overwrites the whole struct, so drop
+                         what the 304 response still owns first (#782) */
+                      back_free_response(&back[i].r);
                       back[i].r =
                         cache_read(opt, cache, back[i].url_adr, back[i].url_fil,
                                    back[i].url_sav, back[i].location_buffer);

src/htscache.c

@@ -985,6 +985,7 @@ void cache_init(cache_back * cache, httrackp * opt) {
               StringBuff(opt->path_log),
               "hts-cache/new.zip")))) { // a previous cache exists.. rename it
         if (!hts_rename_over(
+                opt,
                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
                         StringBuff(opt->path_log), "hts-cache/new.zip"),
                 fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),

src/htslib.c

@@ -2703,17 +2703,8 @@ void hts_now_iso8601(char out[32]) {
   time_t t = time(NULL);
   struct tm tmv;
 
-#if defined(_WIN32)
-  struct tm *g = gmtime(&t);
-
-  if (g != NULL)
-    tmv = *g;
-  else
+  if (!hts_gmtime(t, &tmv))
     memset(&tmv, 0, sizeof(tmv));
-#else
-  if (gmtime_r(&t, &tmv) == NULL)
-    memset(&tmv, 0, sizeof(tmv));
-#endif
   strftime(out, 32, "%Y-%m-%dT%H:%M:%SZ", &tmv);
 }
 

src/htslib.h

@@ -157,6 +157,19 @@ struct t_dnscache {
   char host_addr[HTS_MAXADDRNUM][HTS_MAXADDRLEN];
 };
 
+/* Break t down as UTC into the caller's buffer, HTS_FALSE if that failed.
+   gmtime()'s static is shared, and both the engine and ProxyTrack convert on
+   worker threads. */
+static HTS_INLINE HTS_UNUSED hts_boolean hts_gmtime(time_t t,
+                                                    struct tm *tmbuf) {
+#ifdef _WIN32
+  /* Microsoft's gmtime_s takes the destination first, unlike C11 Annex K. */
+  return gmtime_s(tmbuf, &t) == 0 ? HTS_TRUE : HTS_FALSE;
+#else
+  return gmtime_r(&t, tmbuf) != NULL ? HTS_TRUE : HTS_FALSE;
+#endif
+}
+
 /* Library internal definictions */
 #ifdef HTS_INTERNAL_BYTECODE
 

src/htsselftest.c

@@ -2659,8 +2659,11 @@ static int st_logcallback(httrackp *opt, int argc, char **argv) {
   strcpybuff(seen, st_log_callback_seen);
 
   rewind(fp);
-  line[0] = '\0';
-  (void) fgets(line, (int) sizeof(line), fp);
+  if (fgets(line, (int) sizeof(line), fp) == NULL) {
+    fprintf(stderr, "logcallback: log file is empty, nothing was written\n");
+    fclose(fp);
+    return 1;
+  }
   fclose(fp);
 
   /* The callback runs above the level filter and without a log file at all;
@@ -6137,10 +6140,9 @@ static hts_boolean ro_is(const char *path, const char *data) {
 }
 
 // -#test=renameover <dir>: hts_rename_over() must replace an existing dst and
-// never delete one it did not replace (#779). Which half is live depends on
+// never lose one it did not replace (#779, #790). Which half is live depends on
 // what rename() does to an existing target, so probe that and name the regime.
 static int st_renameover(httrackp *opt, int argc, char **argv) {
-  (void) opt;
   if (argc < 1) {
     fprintf(stderr, "renameover: needs a writable base dir\n");
     return 1;
@@ -6169,7 +6171,7 @@ static int st_renameover(httrackp *opt, int argc, char **argv) {
   ro_put(dst, "old");
   if (replaceable) {
     /* An existing dst must still be replaced: the unlink is for this. */
-    if (!hts_rename_over(src, dst)) {
+    if (!hts_rename_over(opt, src, dst)) {
       fprintf(stderr, "renameover: replacing an existing dst failed: %s\n",
               strerror(errno));
       err++;
@@ -6179,7 +6181,7 @@ static int st_renameover(httrackp *opt, int argc, char **argv) {
     }
   } else {
     /* A failure the unlink cannot fix must leave dst as it was. */
-    if (hts_rename_over(src, dst)) {
+    if (hts_rename_over(opt, src, dst)) {
       fprintf(stderr, "renameover: an unfixable failure reported success\n");
       err++;
     }
@@ -6189,10 +6191,36 @@ static int st_renameover(httrackp *opt, int argc, char **argv) {
     }
   }
 
+  /* A directory in the way is not something the caller asked to replace: it
+     must be refused, never parked aside and orphaned. */
+  (void) UNLINK(dst);
+  ro_put(src, "new");
+  if (MKDIR(dst) == 0) {
+    char parked[sizeof(dst) + 16];
+
+    snprintf(parked, sizeof(parked), "%s.hts-old0", dst);
+    if (hts_rename_over(opt, src, dst)) {
+      fprintf(stderr, "renameover: a directory at dst reported success\n");
+      err++;
+    }
+    if (!ro_is(src, "new")) {
+      fprintf(stderr, "renameover: a directory at dst consumed src\n");
+      err++;
+    }
+    /* RMDIR only succeeds on a directory that is there, so it doubles as the
+       probe: the parked name must not exist at all. */
+    if (RMDIR(parked) == 0 || fexist_utf8(parked)) {
+      fprintf(stderr, "renameover: a directory at dst was parked aside\n");
+      err++;
+    }
+    (void) RMDIR(dst);
+  }
+  (void) UNLINK(src);
+
   /* A missing src must leave dst alone and report failure. */
   (void) UNLINK(src);
   ro_put(dst, "keep");
-  if (hts_rename_over(src, dst)) {
+  if (hts_rename_over(opt, src, dst)) {
     fprintf(stderr, "renameover: a missing src reported success\n");
     err++;
   }
@@ -6203,11 +6231,88 @@ static int st_renameover(httrackp *opt, int argc, char **argv) {
 
   /* Same, with dst absent too: nothing to lose, still a failure. */
   (void) UNLINK(dst);
-  if (hts_rename_over(src, dst)) {
+  if (hts_rename_over(opt, src, dst)) {
     fprintf(stderr, "renameover: a missing src and dst reported success\n");
     err++;
   }
 
+  /* The aside fallback, driven directly: a clobbering rename() never reaches
+     it. Skipped in the refused regime, where no rename at all succeeds. */
+  if (replaceable) {
+    char aside[sizeof(dst) + 16], keep[sizeof(dst) + 16];
+
+    snprintf(aside, sizeof(aside), "%s.hts-old0", dst);
+    snprintf(keep, sizeof(keep), "%s.hts-old1", dst);
+    (void) UNLINK(aside);
+    (void) UNLINK(keep);
+    ro_put(src, "new");
+    ro_put(dst, "old");
+    if (!hts_rename_over_aside_selftest(opt, src, dst)) {
+      fprintf(stderr, "renameover: the aside fallback failed: %s\n",
+              strerror(errno));
+      err++;
+    } else if (!ro_is(dst, "new") || fexist_utf8(src) || fexist_utf8(aside)) {
+      fprintf(stderr, "renameover: the aside fallback did not replace dst\n");
+      err++;
+    }
+
+    /* #790: the retry fails (no src). The old content must survive, back at dst
+       or, when the move back fails too, under the parked name it is logged as.
+       Name the outcome so a leg cannot pass having tested the other one. */
+    (void) UNLINK(src);
+    ro_put(dst, "old");
+    if (hts_rename_over_aside_selftest(opt, src, dst)) {
+      fprintf(stderr, "renameover: a failed aside retry reported success\n");
+      err++;
+    }
+    if (ro_is(dst, "old") && !fexist_utf8(aside)) {
+      printf("renameover: restore back\n");
+    } else if (ro_is(aside, "old") && !fexist_utf8(dst)) {
+      printf("renameover: restore parked\n");
+      (void) UNLINK(aside);
+      ro_put(dst, "old");
+    } else {
+      fprintf(stderr, "renameover: a failed aside retry lost the old copy\n");
+      err++;
+    }
+
+    /* An unrelated file already sitting on the aside name must survive. */
+    ro_put(src, "new");
+    ro_put(aside, "mine");
+    if (!hts_rename_over_aside_selftest(opt, src, dst)) {
+      fprintf(stderr, "renameover: a taken aside name failed the move: %s\n",
+              strerror(errno));
+      err++;
+    } else if (!ro_is(dst, "new") || !ro_is(aside, "mine") ||
+               fexist_utf8(keep)) {
+      fprintf(stderr, "renameover: a taken aside name was not skipped\n");
+      err++;
+    }
+    (void) UNLINK(aside);
+    (void) UNLINK(keep);
+
+    /* A directory there reads as free to the probe, so the park must skip it
+       on the refusal rather than give up. */
+    ro_put(src, "new");
+    ro_put(dst, "old");
+    if (MKDIR(aside) == 0) {
+      if (!hts_rename_over_aside_selftest(opt, src, dst)) {
+        fprintf(stderr,
+                "renameover: a directory on the aside name blocked the "
+                "move: %s\n",
+                strerror(errno));
+        err++;
+      } else if (!ro_is(dst, "new") || fexist_utf8(keep)) {
+        fprintf(stderr, "renameover: a directory on the aside name was not "
+                        "skipped\n");
+        err++;
+      }
+      (void) RMDIR(aside);
+    }
+    (void) UNLINK(keep);
+  }
+
+  (void) UNLINK(src);
   (void) UNLINK(dst);
   printf("renameover: %s\n", err ? "FAIL" : "OK");
   return err;
@@ -6589,6 +6694,109 @@ static int st_threadwait(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* #794: hts_gmtime() must own its output. The table is an independent oracle;
+   the threaded phase is what corrupts if it ever goes back to gmtime()'s
+   shared static. */
+#define GMTIME_THREADS 8
+#define GMTIME_ROUNDS 50000
+
+static const struct {
+  time_t t;
+  int year, mon, mday, hour, min, sec, wday, yday;
+} gmtime_refs[] = {
+    {(time_t) 0, 70, 0, 1, 0, 0, 0, 4, 0},
+    {(time_t) 951782400, 100, 1, 29, 0, 0, 0, 2, 59}, /* a leap day */
+    {(time_t) 1000000000, 101, 8, 9, 1, 46, 40, 0, 251},
+    {(time_t) 2147483647, 138, 0, 19, 3, 14, 7, 2, 18}, /* 32-bit ceiling */
+};
+
+#define GMTIME_REFS ((int) (sizeof(gmtime_refs) / sizeof(gmtime_refs[0])))
+
+static hts_boolean gmtime_ref_matches(int i, const struct tm *tm) {
+  if (tm->tm_year != gmtime_refs[i].year || tm->tm_mon != gmtime_refs[i].mon ||
+      tm->tm_mday != gmtime_refs[i].mday ||
+      tm->tm_hour != gmtime_refs[i].hour || tm->tm_min != gmtime_refs[i].min ||
+      tm->tm_sec != gmtime_refs[i].sec || tm->tm_wday != gmtime_refs[i].wday ||
+      tm->tm_yday != gmtime_refs[i].yday)
+    return HTS_FALSE;
+  return HTS_TRUE;
+}
+
+static htsmutex gmtime_lock = HTSMUTEX_INIT;
+static int gmtime_bad = 0;
+
+static void gmtime_thread(void *arg) {
+  const int i = *(const int *) arg;
+  int bad = 0, round;
+
+  for (round = 0; round < GMTIME_ROUNDS; round++) {
+    struct tm tmv;
+
+    if (!hts_gmtime(gmtime_refs[i].t, &tmv) || !gmtime_ref_matches(i, &tmv))
+      bad++;
+  }
+  hts_mutexlock(&gmtime_lock);
+  gmtime_bad += bad;
+  hts_mutexrelease(&gmtime_lock);
+}
+
+static int st_gmtime(httrackp *opt, int argc, char **argv) {
+  static int idx[GMTIME_THREADS];
+  int err = 0, i;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+
+  for (i = 0; i < GMTIME_REFS; i++) {
+    struct tm tmv;
+
+    if (!hts_gmtime(gmtime_refs[i].t, &tmv)) {
+      fprintf(stderr, "gmtime: conversion #%d failed\n", i);
+      err = 1;
+    } else if (!gmtime_ref_matches(i, &tmv)) {
+      fprintf(stderr,
+              "gmtime: #%d gave %04d-%02d-%02d %02d:%02d:%02d (wday %d, "
+              "yday %d)\n",
+              i, tmv.tm_year + 1900, tmv.tm_mon + 1, tmv.tm_mday, tmv.tm_hour,
+              tmv.tm_min, tmv.tm_sec, tmv.tm_wday, tmv.tm_yday);
+      err = 1;
+    }
+  }
+
+  /* the return is the only failure signal the callers have, so a helper that
+     always claims success leaves them formatting an uninitialised struct tm.
+     Out of range for a 64-bit time_t: NULL from gmtime_r, EINVAL from
+     _gmtime64_s. */
+  if (sizeof(time_t) >= 8) {
+    const time_t beyond = (time_t) INT64_MAX;
+    struct tm tmv;
+
+    if (hts_gmtime(beyond, &tmv)) {
+      fprintf(stderr,
+              "gmtime: an out-of-range time_t was reported converted\n");
+      err = 1;
+    }
+  }
+
+  for (i = 0; i < GMTIME_THREADS; i++) {
+    idx[i] = i % GMTIME_REFS;
+    if (hts_newthread(gmtime_thread, &idx[i]) != 0) {
+      fprintf(stderr, "gmtime: cannot spawn\n");
+      return 1;
+    }
+  }
+  htsthread_wait();
+  if (gmtime_bad != 0) {
+    fprintf(stderr, "gmtime: %d/%d concurrent conversions were corrupt\n",
+            gmtime_bad, GMTIME_THREADS * GMTIME_ROUNDS);
+    err = 1;
+  }
+
+  printf("gmtime self-test: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 #define CHANGES_RACE_FILES 8
 #define CHANGES_RACE_ROUNDS 400
 
@@ -6855,6 +7063,8 @@ static const struct selftest_entry {
      st_changes_race},
     {"threadwait", "", "htsthread_wait() joins threads spawned just before it",
      st_threadwait},
+    {"gmtime", "",
+     "hts_gmtime() fills the caller's buffer, not a static (#794)", st_gmtime},
     {"backswap", "", "which backlog slots may be swapped to the ready table",
      st_backswap},
     {"pause", "", "randomized inter-file pause target self-test", st_pause},

src/htssinglefile.c

@@ -1109,7 +1109,7 @@ hts_boolean singlefile_rewrite_file(httrackp *opt, const char *root,
                    HTS_ACCESS_FILE);
 #endif
     if (ok)
-      ok = hts_rename_over(StringBuff(tmp), page_path);
+      ok = hts_rename_over(opt, StringBuff(tmp), page_path);
     if (!ok) {
       hts_log_print(opt, LOG_ERROR, "single-file: could not rewrite %s",
                     page_path);

src/htstools.c

@@ -1438,21 +1438,75 @@ HTSEXT_API hts_boolean hts_findissystem(find_handle find) {
   return 0;
 }
 
-hts_boolean hts_rename_over(const char *src, const char *dst) {
+/* Park cdst under a free sibling name; caside receives it. */
+static hts_boolean rename_park_aside(char *caside, size_t size,
+                                     const char *cdst) {
+  int i;
+
+  for (i = 0; i < 16; i++) {
+    if (!slprintfbuff(caside, size, "%s.hts-old%d", cdst, i))
+      return HTS_FALSE;
+    /* Skip a name the mirror already holds: POSIX rename() would clobber it
+       (#774). A non-regular entry reads as free and the rename refuses it. */
+    if (fexist_utf8(caside))
+      continue;
+    if (RENAME(cdst, caside) == 0)
+      return HTS_TRUE;
+  }
+  return HTS_FALSE;
+}
+
+/* cdst is in the way of the move: park it, retry, and put it back if the retry
+   fails too. Unlinking it instead would leave nothing at all (#790). */
+static hts_boolean rename_over_aside(httrackp *opt, const char *csrc,
+                                     const char *cdst) {
+  char caside[CATBUFF_SIZE];
+  int err;
+
+  /* Only a regular file may be parked: a directory in the way is not what the
+     caller asked to replace, and parking it orphans it (UNLINK cannot drop). */
+  if (!fexist_utf8(cdst))
+    return HTS_FALSE;
+  if (!rename_park_aside(caside, sizeof(caside), cdst))
+    return HTS_FALSE;
+  if (RENAME(csrc, cdst) == 0) {
+    (void) UNLINK(caside);
+    return HTS_TRUE;
+  }
+  err = errno;
+  /* Retry once, then name the parked copy: nothing else on disk or in the log
+     points at it, and an --update purge would delete it unnoticed. */
+  if (RENAME(caside, cdst) != 0 && RENAME(caside, cdst) != 0)
+    hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
+                  "could not put %s back; its previous content is now %s", cdst,
+                  caside);
+  errno = err;
+  return HTS_FALSE;
+}
+
+hts_boolean hts_rename_over(httrackp *opt, const char *src, const char *dst) {
   char csrc[CATBUFF_SIZE], cdst[CATBUFF_SIZE];
 
   fconv(csrc, sizeof(csrc), src);
   fconv(cdst, sizeof(cdst), dst);
   if (RENAME(csrc, cdst) == 0)
     return HTS_TRUE;
-  /* Only a dst in the way is something the unlink can clear, and the CRT maps
-     that to EEXIST; it keeps EACCES for a src another process holds, where
-     removing dst would lose a file the retry cannot replace (#790). The src
-     check covers a CRT that reports neither. */
+  /* Only a dst in the way is something the fallback can clear, and the CRT maps
+     that to EEXIST; it keeps EACCES for a src another process holds, where the
+     retry would fail the same way. The src check covers a CRT that reports
+     neither. */
   const int err = errno;
 
   if (err != EEXIST || !fexist_utf8(src))
     return HTS_FALSE;
-  (void) UNLINK(cdst);
-  return RENAME(csrc, cdst) == 0 ? HTS_TRUE : HTS_FALSE;
+  return rename_over_aside(opt, csrc, cdst);
+}
+
+hts_boolean hts_rename_over_aside_selftest(httrackp *opt, const char *src,
+                                           const char *dst) {
+  char csrc[CATBUFF_SIZE], cdst[CATBUFF_SIZE];
+
+  fconv(csrc, sizeof(csrc), src);
+  fconv(cdst, sizeof(cdst), dst);
+  return rename_over_aside(opt, csrc, cdst);
 }

src/htstools.h

@@ -137,11 +137,17 @@ HTSEXT_API hts_boolean hts_findisdir(find_handle find);
 HTSEXT_API hts_boolean hts_findisfile(find_handle find);
 HTSEXT_API hts_boolean hts_findissystem(find_handle find);
 
-/* Move src onto dst, replacing an existing dst; HTS_TRUE on success. Both
-   paths are fconv()'d. dst is removed only to make room for a src that exists,
-   so a caller whose src was never written keeps its dst; a retry that still
-   fails does not (#790). */
-hts_boolean hts_rename_over(const char *src, const char *dst);
+/* Move src onto dst, replacing an existing dst; HTS_TRUE on success. Both paths
+   are fconv()'d. A dst in the way is parked under a sibling name rather than
+   removed, so the old content survives a failure: back at dst, or under that
+   sibling (named in the log) when the move back failed too. Not atomic: a crash
+   between the two renames leaves dst absent and its content beside it. */
+hts_boolean hts_rename_over(httrackp *opt, const char *src, const char *dst);
+
+/* Selftest hook: run the aside fallback directly, on a platform whose rename()
+   never reaches it. Both paths are fconv()'d. */
+hts_boolean hts_rename_over_aside_selftest(httrackp *opt, const char *src,
+                                           const char *dst);
 
 #endif
 

src/htswarc.c

@@ -1114,7 +1114,7 @@ static void warc_wacz_package(warc_writer *w) {
     hts_log_print(w->opt, LOG_WARNING,
                   "WACZ: packaging failed, kept existing %s untouched",
                   waczpath);
-  } else if (!hts_rename_over(tmppath, waczpath)) {
+  } else if (!hts_rename_over(w->opt, tmppath, waczpath)) {
     (void) UNLINK(fconv(catbuff, sizeof(catbuff), tmppath));
     hts_log_print(w->opt, LOG_WARNING | LOG_ERRNO,
                   "WACZ: could not finalize %s", waczpath);
@@ -1392,16 +1392,8 @@ warc_writer *warc_open(httrackp *opt, const char *path) {
     char ts[32];
     time_t t = time(NULL);
     struct tm tmv;
-#if defined(_WIN32)
-    struct tm *g = gmtime(&t);
-    if (g != NULL)
-      tmv = *g;
-    else
+    if (!hts_gmtime(t, &tmv))
       memset(&tmv, 0, sizeof(tmv));
-#else
-    if (gmtime_r(&t, &tmv) == NULL)
-      memset(&tmv, 0, sizeof(tmv));
-#endif
     strftime(ts, sizeof(ts), "%Y%m%d%H%M%S", &tmv);
     snprintf(catbuff, sizeof(catbuff), "httrack-%s.warc.gz", ts);
     path =
@@ -1589,7 +1581,7 @@ static hts_boolean warc_commit(warc_writer *w) {
     for (s = 0; s < nseg; s++) {
       const char *final = warc_seg_path(w, s, finalbuf, sizeof(finalbuf));
       snprintf(tmpbuf, sizeof(tmpbuf), "%s" WARC_TMP_SUFFIX, final);
-      if (!hts_rename_over(tmpbuf, final)) {
+      if (!hts_rename_over(w->opt, tmpbuf, final)) {
         hts_log_print(w->opt, LOG_ERROR | LOG_ERRNO,
                       "WARC: could not replace %s", final);
         return HTS_FALSE;

src/proxy/proxytrack.c

@@ -520,12 +520,13 @@ static void proxytrack_add_DAV_Item(String * item, String * buff,
                                     const char *filename, size_t size,
                                     time_t timestamp, const char *mime,
                                     int isDir, int isRoot, int isDefault) {
-  struct tm *timetm;
+  struct tm timetmbuf;
+  struct tm *timetm = &timetmbuf;
 
   if (timestamp == (time_t) 0 || timestamp == (time_t) - 1) {
     timestamp = time(NULL);
   }
-  if ((timetm = gmtime(&timestamp)) != NULL) {
+  if (hts_gmtime(timestamp, timetm)) {
     char tms[256 + 1];
     const char *name;
 
@@ -747,7 +748,8 @@ static PT_Element proxytrack_process_DAV_Request(PT_Indexes indexes,
             const char *thisUrl = list[i];
             const char *mimeType = "application/octet-stream";
             unsigned int thisUrlLen = (unsigned int) strlen(thisUrl);
-            int thisIsDir = (thisUrl[thisUrlLen - 1] == '/') ? 1 : 0;
+            /* the folder's default document is enumerated as an empty name */
+            int thisIsDir = (hts_lastchar(thisUrl) == '/') ? 1 : 0;
 
             /* Item URL */
             StringRoom(itemUrl,
@@ -852,8 +854,7 @@ static PT_Element proxytrack_process_HTTP_List(PT_Indexes indexes,
     for(isDir = 1; isDir >= 0; isDir--) {
       for(i = 0; list[i] != NULL; i++) {
         char *thisUrl = list[i];
-        unsigned int thisUrlLen = (unsigned int) strlen(thisUrl);
-        int thisIsDir = (thisUrl[thisUrlLen - 1] == '/') ? 1 : 0;
+        int thisIsDir = (hts_lastchar(thisUrl) == '/') ? 1 : 0;
 
         if (thisIsDir == isDir) {
           if (isDir)

src/proxy/proxytrack.h

@@ -360,21 +360,14 @@ HTS_UNUSED static struct tm *convert_time_rfc822(struct tm *result, const char *
 HTS_UNUSED static struct tm PT_GetTime(time_t t) {
   struct tm tmbuf;
 
-#ifdef _WIN32
-  struct tm *tm = gmtime(&t);
-#else
-  struct tm *tm = gmtime_r(&t, &tmbuf);
-#endif
-  if (tm != NULL)
-    return *tm;
-  else {
+  if (!hts_gmtime(t, &tmbuf)) {
     /* an all-zero tm has tm_mday == 0, which the ARC date field prints as a
        day of "00"; the epoch is the conventional "date unknown" */
     memset(&tmbuf, 0, sizeof(tmbuf));
     tmbuf.tm_year = 70;
     tmbuf.tm_mday = 1;
-    return tmbuf;
   }
+  return tmbuf;
 }
 HTS_UNUSED static int set_filetime(const char *file, struct tm *tm_time) {
   struct utimbuf tim;

tests/01_engine-changes.test

@@ -3,27 +3,14 @@
 
 set -euo pipefail
 
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_changes_st.XXXXXX") || exit 1
 trap 'set +e; rm -rf "$tmpdir"' EXIT
 trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
 
-# No pipe into grep: SIGPIPE would mask a failing exit status.
-expect_ok() {
-    local label="$1" out
-    shift
-    out=$("$@" 2>&1) || {
-        echo "FAIL: ${label} exited non-zero: ${out}"
-        exit 1
-    }
-    case "$out" in
-    *"${label}: OK"*) ;;
-    *)
-        echo "FAIL: ${out}"
-        exit 1
-        ;;
-    esac
-}
-
 # --changes bucket accounting and JSON escaping (#714).
 expect_ok "changes self-test" httrack -O "${tmpdir}/o1" -#test=changes run
 # The report must hold against a transfer thread the crawl never joins.

tests/01_engine-gmtime.test

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+# A non-UTC zone, so a helper that reached for localtime instead of gmtime gives
+# a different answer: CI runners are UTC, where the two agree.
+TZ=XXX5
+export TZ
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_gmtime_st.XXXXXX") || exit 1
+trap 'set +e; rm -rf "$tmpdir"' EXIT
+trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
+
+# Concurrent conversions do not tread on each other the way gmtime()'s shared
+# static does (#794).
+expect_ok "gmtime self-test" httrack -O "${tmpdir}/o1" -#test=gmtime

tests/01_engine-renameover.test

@@ -4,9 +4,9 @@
 set -euo pipefail
 
 # Drives -#test=renameover: hts_rename_over() must replace an existing dst, and
-# must leave dst alone when the rename failed for a reason removing dst cannot
-# fix (#779). The selftest prints the regime it detected; pin it per platform so
-# a leg cannot pass having tested the other half.
+# must leave dst alone when the rename failed for a reason moving dst aside
+# cannot fix (#779, #790). The selftest prints the regime and the restore
+# outcome it took; pin both per leg so none can pass having tested another.
 dir=$(mktemp -d)
 trap 'set +e; rm -rf "$dir"' EXIT
 
@@ -18,6 +18,7 @@ esac
 out=$(httrack -O /dev/null -#test=renameover "$dir")
 grep -q "renameover: OK" <<<"$out"
 grep -q "renameover: regime $want" <<<"$out"
+grep -q "renameover: restore back" <<<"$out"
 
 if [ "$(uname -s)" != "Linux" ]; then
     echo "renameover: LD_PRELOAD interposition is Linux-only here, skipping"
@@ -42,11 +43,26 @@ fi
 # refuses by default. The shim allocates nothing, so the ordering is harmless.
 export ASAN_OPTIONS="${ASAN_OPTIONS:+$ASAN_OPTIONS:}verify_asan_link_order=0"
 
-# The unlink fallback is dead code on POSIX, so borrow Windows' rename().
+# The aside fallback is dead code on POSIX, so borrow Windows' rename().
 out=$(LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
     -#test=renameover "$dir")
 grep -q "renameover: OK" <<<"$out"
 grep -q "renameover: regime fallback" <<<"$out"
+grep -q "renameover: restore back" <<<"$out"
+
+# #790: the move back out of the parked name fails once. Without the retry the
+# old copy stays parked and dst is left absent.
+out=$(RENAMEFAIL_ASIDE_FAILS=1 LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
+    -#test=renameover "$dir")
+grep -q "renameover: OK" <<<"$out"
+grep -q "renameover: restore back" <<<"$out"
+
+# It keeps failing: the old copy must survive under the parked name, never be
+# deleted, and the call must still report failure.
+out=$(RENAMEFAIL_ASIDE_FAILS=9 LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \
+    -#test=renameover "$dir")
+grep -q "renameover: OK" <<<"$out"
+grep -q "renameover: restore parked" <<<"$out"
 
 # A source another process holds fails with EACCES, which dst had no part in.
 out=$(RENAMEFAIL_MODE=locked LD_PRELOAD="$RENAMEFAIL_LIB" httrack -O /dev/null \

tests/01_engine-threadwait.test

@@ -3,27 +3,14 @@
 
 set -euo pipefail
 
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_threadwait_st.XXXXXX") || exit 1
 trap 'set +e; rm -rf "$tmpdir"' EXIT
 trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
 
-# No pipe into grep: SIGPIPE would mask a failing exit status.
-expect_ok() {
-    local label="$1" out
-    shift
-    out=$("$@" 2>&1) || {
-        echo "FAIL: ${label} exited non-zero: ${out}"
-        exit 1
-    }
-    case "$out" in
-    *"${label}: OK"*) ;;
-    *)
-        echo "FAIL: ${out}"
-        exit 1
-        ;;
-    esac
-}
-
 # A thread is outstanding from the moment hts_newthread() returns, so a wait
 # that follows the spawn joins it, and wait_n(n) still leaves n behind (#747).
 expect_ok "threadwait self-test" httrack -O "${tmpdir}/o1" -#test=threadwait

tests/110_local-ftp-parallel.test

@@ -0,0 +1,98 @@
+#!/bin/bash
+#
+# A completed FTP transfer used to end as a 0-byte file when its ready backlog
+# slot was swapped to the on-disk table (#797): the FTP worker writes url_sav
+# itself and leaves no in-memory body, and the swap round-trip zeroed the slot's
+# size, so the link writer then created an empty file over the bytes.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+command -v httrack >/dev/null || {
+    echo "could not find httrack" >&2
+    exit 1
+}
+
+server=$(nativepath "${testdir}/ftp-server.py")
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+root="${tmpdir}/root"
+out="${tmpdir}/crawl"
+mkdir -p "$root" "$out"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+ok() { echo "OK: $*"; }
+size_of() { wc -c <"$1" | tr -d '[:space:]'; }
+
+# Distinct bodies and lengths, so a mirror that mixed two of them up cannot
+# compare equal.
+names=""
+for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
+    names="${names} f${i}"
+    "$python" -c 'import sys; sys.stdout.buffer.write(
+        ("BODY-%s " % sys.argv[1]).encode() + sys.argv[1].encode() * (400 * int(sys.argv[1])))' \
+        "$i" >"${root}/f${i}.bin"
+done
+
+serverlog="${tmpdir}/server.out"
+"$python" "$server" --root "$(nativepath "$root")" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 300); do
+    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "ftp server exited early: $(cat "$serverlog")" >&2
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
+    exit 1
+}
+host="127.0.0.1_${port}"
+
+urls=()
+for name in $names; do
+    urls+=("ftp://127.0.0.1:${port}/${name}.bin")
+done
+
+# -c8, never -c1: a slot is only swapped out while other transfers keep the
+# parser from picking it up. -Z because the swap only logs at debug level, and
+# the run below proves nothing unless it happened. Bounded like every crawl
+# pass: a wedge must fail the test, not hang the job until CI discards the log.
+run_with_timeout 300 httrack "${urls[@]}" -O "$out" --quiet -Z -c8 \
+    --disable-security-limits --robots=0 --timeout=20 --max-time=120 --retries=1 \
+    >"${tmpdir}/log" 2>&1
+
+swapped=$(grep -ac 'slots ready moved to background' "${out}/hts-log.txt" || true)
+test "${swapped:-0}" -gt 0 ||
+    fail "no ready slot was swapped to disk; this run never reached the bug"
+ok "the crawl swapped ready slots to the on-disk table ${swapped} times"
+
+for name in $names; do
+    test -f "${out}/${host}/${name}.bin" || fail "${name}.bin was not mirrored"
+    cmp -s "${out}/${host}/${name}.bin" "${root}/${name}.bin" ||
+        fail "${name}.bin has $(size_of "${out}/${host}/${name}.bin") of the" \
+            "$(size_of "${root}/${name}.bin") bytes served"
+done
+ok "every FTP transfer landed on disk intact"
+
+errors=$(grep -a 'Error:' "${out}/hts-log.txt" || true)
+test -z "$errors" || fail "the crawl reported errors: ${errors}"
+ok "no transfer was reported failed"

tests/114_local-update-304-leak.test

@@ -0,0 +1,108 @@
+#!/bin/bash
+#
+# Issue #782: a 304 during --update overwrote the whole response struct with the
+# cache entry, dropping the 8 KB header buffer it owned. Only a leak checker
+# sees it and the sanitized CI leg runs with detect_leaks off, so this one turns
+# it back on around a two-pass mini304 crawl; the fresh pass is the control.
+
+set -eu
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+server=$(nativepath "${testdir}/local-server.py")
+root=$(nativepath "${testdir}/server-root")
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+
+probe=$(ASAN_OPTIONS=help=1 httrack -O /dev/null --version 2>&1 || true)
+case "$probe" in
+*AddressSanitizer*) ;;
+*)
+    echo "not a sanitized build, skipping" >&2
+    exit 77
+    ;;
+esac
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_304leak.XXXXXX") || exit 1
+serverpid=
+cleanup() {
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap 'set +e; cleanup' EXIT
+trap cleanup HUP INT QUIT PIPE TERM
+
+serverlog="${tmpdir}/server.log"
+: >"$serverlog"
+"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$serverlog" 2>/dev/null)
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$serverlog")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+
+out="${tmpdir}/crawl"
+mkdir "$out"
+url="http://127.0.0.1:${port}/mini304/index.html"
+
+# Last value wins in the sanitizer flag parser, so this overrides the job's
+# settings; the report file, not the exit status, is the verdict.
+asan_base="${ASAN_OPTIONS:+$ASAN_OPTIONS:}detect_leaks=1:abort_on_error=0:exitcode=0"
+
+# run_pass <label> <extra httrack args...>
+run_pass() {
+    local label="$1"
+    shift
+    rm -f "${tmpdir}/asan".*
+    printf '[%s] ..\t' "$label"
+    ASAN_OPTIONS="${asan_base}:log_path=${tmpdir}/asan" \
+        httrack -O "$out" --quiet --disable-security-limits --robots=0 \
+        --timeout=30 --retries=1 -c1 "$@" "$url" >"${tmpdir}/log_${label}" 2>&1 || true
+    report=$(cat "${tmpdir}/asan".* 2>/dev/null || true)
+    test -z "$report" || {
+        echo "FAIL: sanitizer report on the ${label} pass"
+        echo "$report"
+        exit 1
+    }
+    echo "OK (clean)"
+}
+
+run_pass fresh
+run_pass update --update
+# only --warc-file stashes the two header blocks, so the archive limb of the fix
+# runs on this pass alone
+run_pass warc --update --warc-file leak
+
+# Teeth: the update passes must have gone through the 304 path, or a run that
+# re-downloaded everything would report clean having tested nothing.
+printf '[update used the cache] ..\t'
+if grep -aqE "no files updated" "${out}/hts-log.txt"; then
+    echo "OK"
+else
+    echo "FAIL: the update pass did not report unchanged files"
+    exit 1
+fi
+
+# ... and the WARC pass must have written an archive, or it stashed nothing.
+printf '[warc pass archived] ..\t'
+if [ -n "$(find "$out" -name 'leak*.warc*' -size +0c 2>/dev/null)" ]; then
+    echo "OK"
+else
+    echo "FAIL: the --warc-file pass produced no archive"
+    exit 1
+fi

tests/120_local-proxytrack-webdav-default.test

@@ -0,0 +1,121 @@
+#!/bin/bash
+#
+# A PROPFIND whose path is an exact cache entry enumerates that entry's default
+# document under an empty name; proxytrack used to index one byte before it and
+# die on the whole listener.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+# shellcheck source=tests/testlib.sh
+. "$top_srcdir/tests/testlib.sh"
+
+python=$(find_python) || {
+    echo "python3 missing, skipping"
+    exit 77
+}
+command -v curl >/dev/null 2>&1 || {
+    echo "curl missing, skipping"
+    exit 77
+}
+# MSYS cannot reap a native listener, and the orphan wedges the suite (#595).
+if is_windows; then
+    echo "windows: cannot reap a backgrounded proxytrack, skipping"
+    exit 77
+fi
+
+dir=$(mktemp -d)
+ptpid=
+cleanup() {
+    stop_server "$ptpid"
+    rm -rf "$dir"
+}
+trap 'set +e; cleanup' EXIT
+trap 'set +e; cleanup; exit 1' INT TERM HUP
+
+printf 'HTTP/1.1 200 OK\r\nContent-Type: text/html\r\nContent-Length: 5\r\n\r\n' >"$dir/hdr"
+printf 'hello' >"$dir/body"
+alen=$(($(wc -c <"$dir/hdr") + $(wc -c <"$dir/body")))
+{
+    printf 'filedesc://t.arc 0.0.0.0 20250101000000 text/plain 200 - - 0 t.arc 9\n'
+    printf '2 0 test\n'
+    printf '\n\n'
+    printf 'http://example.com/page.html 0.0.0.0 20250101000000 text/html 200 - - 0 t.arc %d\n' "$alen"
+    cat "$dir/hdr" "$dir/body"
+} >"$dir/in.arc"
+
+freeport() {
+    "$python" -c 'import socket; s=socket.socket(); s.bind(("127.0.0.1",0)); print(s.getsockname()[1]); s.close()'
+}
+proxyport=$(freeport)
+icpport=$(freeport)
+
+proxytrack "127.0.0.1:$proxyport" "127.0.0.1:$icpport" "$dir/in.arc" >"$dir/pt.log" 2>&1 &
+ptpid=$!
+waited=0
+until grep -qE "HTTP Proxy installed on|Unable to (initialize a temporary server|create the server)" "$dir/pt.log"; do
+    kill -0 "$ptpid" 2>/dev/null || {
+        echo "FAIL: proxytrack exited before listening"
+        cat "$dir/pt.log"
+        exit 1
+    }
+    test "$waited" -lt 50 || {
+        echo "FAIL: proxytrack never announced its listen port"
+        exit 1
+    }
+    sleep 0.1
+    waited=$((waited + 1))
+done
+grep -q "HTTP Proxy installed on" "$dir/pt.log" || {
+    echo "FAIL: proxytrack failed to bind"
+    cat "$dir/pt.log"
+    exit 1
+}
+
+propfind() {
+    curl -s -i --max-time 30 -X PROPFIND -H "Depth: 1" \
+        "http://127.0.0.1:$proxyport/webdav/example.com/$1"
+}
+
+resp=$(propfind page.html) || {
+    echo "FAIL: PROPFIND on an exact cache entry did not answer"
+    cat "$dir/pt.log"
+    exit 1
+}
+grep -q '^HTTP/1\.[01] 207 ' <<<"$resp" || {
+    echo "FAIL: expected 207 Multi-Status"
+    printf '%s\n' "$resp"
+    exit 1
+}
+grep -q '<displayname>Default Document for the Folder</displayname>' <<<"$resp" || {
+    echo "FAIL: the default document of the entry was not listed"
+    printf '%s\n' "$resp"
+    exit 1
+}
+# The name alone would still pass on a truncated or garbage URL, so pin the
+# href: the default document is the collection path plus an empty name.
+grep -q '<href>/webdav/example\.com/page\.html/</href>' <<<"$resp" || {
+    echo "FAIL: the default document was listed under the wrong href"
+    printf '%s\n' "$resp"
+    exit 1
+}
+responses=$(grep -c '<response ' <<<"$resp" || true)
+test "$responses" -eq 2 || {
+    echo "FAIL: expected the root and its default document, got $responses responses"
+    printf '%s\n' "$resp"
+    exit 1
+}
+
+# The crash killed the process, so a second request is the liveness proof.
+resp=$(propfind "") || {
+    echo "FAIL: proxytrack died serving the previous PROPFIND"
+    cat "$dir/pt.log"
+    exit 1
+}
+grep -q '^HTTP/1\.[01] 207 ' <<<"$resp" || {
+    echo "FAIL: expected 207 Multi-Status on the follow-up listing"
+    printf '%s\n' "$resp"
+    exit 1
+}
+
+echo "OK: PROPFIND on an exact cache entry lists its default document and survives"

tests/Makefile.am

@@ -64,6 +64,7 @@ TESTS = \
 	01_engine-filterdual.test \
 	01_engine-ftp-line.test \
 	01_engine-ftp-userpass.test \
+	01_engine-gmtime.test \
 	01_engine-backswap.test \
 	01_engine-cacheindex.test \
 	01_engine-hashtable.test \
@@ -227,6 +228,9 @@ TESTS = \
 	103_teardown-status.test \
 	104_engine-warc-longurl.test \
 	105_suite-timeout.test \
-	111_local-ftp-update-rest.test
+	110_local-ftp-parallel.test \
+	111_local-ftp-update-rest.test \
+	114_local-update-304-leak.test \
+	120_local-proxytrack-webdav-default.test
 
 CLEANFILES = check-network_sh.cache

tests/renamefail.c

@@ -1,7 +1,9 @@
 /* Borrows Windows' rename() for 01_engine-renameover.test, since POSIX cannot
-   reach hts_rename_over()'s unlink fallback: an existing target is refused with
+   reach hts_rename_over()'s aside fallback: an existing target is refused with
    EEXIST, and RENAMEFAIL_MODE=locked reports EACCES instead, as the CRT does
-   for a source another process holds. */
+   for a source another process holds. RENAMEFAIL_ASIDE_FAILS=N refuses the
+   first N moves back out of the parked ".hts-old" name, which is the only way
+   to reach hts_rename_over()'s restore retry (#790). */
 
 #define _GNU_SOURCE
 #include <dlfcn.h>
@@ -18,7 +20,9 @@ SHIM_EXPORT int rename(const char *oldpath, const char *newpath);
 
 SHIM_EXPORT int rename(const char *oldpath, const char *newpath) {
   static int (*real_rename)(const char *, const char *) = NULL;
+  static int aside_failures = 0;
   const char *const mode = getenv("RENAMEFAIL_MODE");
+  const char *const aside_fails = getenv("RENAMEFAIL_ASIDE_FAILS");
   const int locked = mode != NULL && strcmp(mode, "locked") == 0;
   struct stat st;
 
@@ -26,7 +30,13 @@ SHIM_EXPORT int rename(const char *oldpath, const char *newpath) {
     errno = EACCES;
     return -1;
   }
-  if (stat(newpath, &st) == 0) {
+  if (aside_fails != NULL && strstr(oldpath, ".hts-old") != NULL &&
+      aside_failures < atoi(aside_fails)) {
+    aside_failures++;
+    errno = EACCES;
+    return -1;
+  }
+  if (aside_fails == NULL && stat(newpath, &st) == 0) {
     errno = EEXIST;
     return -1;
   }

tests/testlib.sh

@@ -24,6 +24,24 @@ nativepath() {
     fi
 }
 
+# Run an engine self-test and require its "<label>: OK" line. No pipe into grep:
+# SIGPIPE would mask a failing exit status.
+expect_ok() {
+    local label="$1" out
+    shift
+    out=$("$@" 2>&1) || {
+        echo "FAIL: ${label} exited non-zero: ${out}"
+        exit 1
+    }
+    case "$out" in
+    *"${label}: OK"*) ;;
+    *)
+        echo "FAIL: ${out}"
+        exit 1
+        ;;
+    esac
+}
+
 is_windows() {
     case "$(uname -s)" in
     MINGW* | MSYS* | CYGWIN*) return 0 ;;