Merge fix/cache-repair-rename into fix/refetch-backup-name

Carries the here-string reconciliation with #822 and the aside
directory-refusal fix. TESTS tail keeps all four new entries in order.

Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/windows-build.yml

@@ -211,7 +211,8 @@ jobs:
           pass=0 fail=0 skip=0 failed="" skipped="" deadline=0
           for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
                    *_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
-                   60_crawl-log-salvage.test; do
+                   60_crawl-log-salvage.test 106_engine-repair-rename.test \
+                   108_engine-refetch-backup.test; do
             elapsed=$((SECONDS - started))
             if [ "$elapsed" -ge "$suite_deadline" ]; then
               echo "::error::suite deadline: ${elapsed}s elapsed, stopping before $t"
@@ -262,9 +263,8 @@ jobs:
           # crange pending #581;
           # webdav-mime needs a reapable background listener, which MSYS cannot give it;
           # badmtime needs a filesystem that stores an mtime past gmtime's range;
-          # single-file ends on a GUI half needing htsserver, which this job does not build;
-          # update-304-leak needs a LeakSanitizer build, which MSVC has no equivalent of.
-          expected_skips=" 01_engine-footer-overflow.test 100_local-purge-longpath.test 114_local-update-304-leak.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
+          # single-file ends on a GUI half needing htsserver, which this job does not build.
+          expected_skips=" 01_engine-footer-overflow.test 100_local-purge-longpath.test 48_local-crange-memresume.test 71_local-crange-repaircache.test 79_local-proxytrack-webdav-mime.test 88_local-proxytrack-badmtime.test 94_local-single-file.test"
           # First, or the deadline reads as an unexplained shortfall in the gates below.
           [ "$deadline" -eq 0 ] || { echo "::error::suite did not finish within ${suite_deadline}s"; exit 1; }
           [ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }

src/htsback.c

@@ -391,56 +391,6 @@ int back_selftest_slot_swap(void) {
   CHECK(0, "the dummy test slot");
 #undef CHECK
 
-  /* The swap round-trip must not lose the size of a slot whose body is already
-     at url_sav, or the link writer blanks the file (#797). */
-  {
-    static const char body[] = "swapped body";
-    int c;
-
-    for (c = 0; c < 2; c++) {
-      const hts_boolean inmemory = c == 0 ? HTS_TRUE : HTS_FALSE;
-      FILE *const fp = tmpfile();
-      lien_back *copy = NULL;
-
-      memset(&back, 0, sizeof(back));
-      back.status = STATUS_READY;
-      strcpybuff(back.url_sav, "/tmp/httrack-selftest.bin");
-      back.r.size = (LLint) sizeof(body) - 1;
-      if (inmemory) {
-        back.r.adr = strdupt(body);
-      }
-      if (fp == NULL || back_serialize(fp, &back) != 0 ||
-          fseek(fp, 0, SEEK_SET) != 0 || back_unserialize(fp, &copy) != 0) {
-        fprintf(stderr, "backswap: round-trip failed for a %s slot\n",
-                inmemory ? "buffered" : "direct-to-disk");
-        err = 1;
-      } else {
-        if (copy->r.size != back.r.size) {
-          fprintf(stderr,
-                  "backswap: %s slot came back with size " LLintP
-                  ", expected " LLintP "\n",
-                  inmemory ? "buffered" : "direct-to-disk", copy->r.size,
-                  back.r.size);
-          err = 1;
-        }
-        if (inmemory && (copy->r.adr == NULL ||
-                         memcmp(copy->r.adr, body, sizeof(body) - 1) != 0)) {
-          fprintf(stderr, "backswap: buffered slot lost its body\n");
-          err = 1;
-        }
-        if (!inmemory && copy->r.adr != NULL) {
-          fprintf(stderr, "backswap: direct-to-disk slot gained a body\n");
-          err = 1;
-        }
-        back_clear_entry(copy);
-        freet(copy);
-      }
-      if (fp != NULL)
-        fclose(fp);
-      freet(back.r.adr);
-    }
-  }
-
   printf("backswap self-test: %s\n", err ? "FAIL" : "OK");
   return err;
 }
@@ -646,6 +596,41 @@ int back_nsoc_overall(const struct_back * sback) {
   return n;
 }
 
+/* Reserved subdirectory holding a mirrored file's temporaries, beside it. */
+#define HTS_TMPDIR "hts-tmp"
+
+/* Build save's temporary as <dir>/hts-tmp/<name>.<ext>. Appending the extension
+   to save instead put it in the mirror namespace, so a site serving <path>.bak
+   had its copy taken as the backup and then unlinked (#774). HTS_FALSE (dest
+   emptied) if it would not fit. Note: utf-8. */
+static hts_boolean back_tmpname(char *dest, size_t size, const char *save,
+                                const char *ext) {
+  const char *const slash = strrchr(save, '/');
+  const int dirlen = slash != NULL ? (int) (slash - save) + 1 : 0;
+
+  if (!slprintfbuff(dest, size, "%.*s" HTS_TMPDIR "/%s.%s", dirlen, save,
+                    slash != NULL ? slash + 1 : save, ext)) {
+    dest[0] = '\0';
+    return HTS_FALSE;
+  }
+  return HTS_TRUE;
+}
+
+/* Note: utf-8 */
+void back_tmpdir_drop(const char *tmp) {
+  char BIGSTK dir[HTS_URLMAXSIZE * 2];
+  const char *slash;
+
+  if (tmp == NULL || (slash = strrchr(tmp, '/')) == NULL)
+    return;
+  if (!strclipbuff(dir, sizeof(dir), tmp))
+    return;
+  dir[slash - tmp] = '\0';
+  slash = strrchr(dir, '/');
+  if (strcmp(slash != NULL ? slash + 1 : dir, HTS_TMPDIR) == 0)
+    (void) RMDIR(dir);
+}
+
 /* generate temporary file on lien_back */
 /* Note: utf-8 */
 static int create_back_tmpfile(httrackp *opt, lien_back *const back,
@@ -653,12 +638,10 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
   // do not use tempnam() but a regular filename
   back->tmpfile_buffer[0] = '\0';
   if (back->url_sav[0] != '\0') {
-    /* same capacity as url_sav, so truncation drops the extension and aliases
-       the temp name onto the live file that back_finalize_backup() UNLINKs */
-    if (!sprintfbuff(back->tmpfile_buffer, "%s.%s", back->url_sav, ext)) {
+    if (!back_tmpname(back->tmpfile_buffer, sizeof(back->tmpfile_buffer),
+                      back->url_sav, ext)) {
       hts_log_print(opt, LOG_WARNING, "temporary filename too long for %s",
                     back->url_sav);
-      back->tmpfile_buffer[0] = '\0';
       return -1;
     }
     back->tmpfile = back->tmpfile_buffer;
@@ -697,6 +680,10 @@ void back_refetch_backup(httrackp *opt, lien_back *const back) {
         hts_log_print(opt, LOG_WARNING, "replacing leftover backup %s",
                       back->tmpfile);
       saved = hts_rename_over(opt, back->url_sav, back->tmpfile);
+      /* Another slot sharing the directory may have removed it between the
+         structcheck above and the rename: recreate it and try once more. */
+      if (!saved && structcheck(back->tmpfile) == 0)
+        saved = hts_rename_over(opt, back->url_sav, back->tmpfile);
     }
     if (!saved) {
       hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
@@ -720,16 +707,22 @@ static hts_boolean back_transfer_failed(const int statuscode) {
   }
 }
 
-/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
-   existing file moved the good copy to back->tmpfile before truncating url_sav.
-   commit keeps the new file and drops the backup; else restore it so an aborted
-   transfer leaves the previous copy intact. Skips the zlib .z temp. */
-static void back_finalize_backup(httrackp *opt, lien_back *const back,
-                                 const hts_boolean commit) {
+hts_boolean back_finalize_backup(httrackp *opt, lien_back *const back,
+                                 hts_boolean commit) {
+  const hts_boolean wanted = commit;
+
   if (back->tmpfile == NULL || back->r.compressed)
-    return;
+    return HTS_TRUE;
+  /* Nothing to commit to: filecreate() can fail after the backup was taken,
+     and dropping it then loses both copies (#775). */
+  if (commit && !fexist_utf8(back->url_sav)) {
+    hts_log_print(opt, LOG_WARNING, "%s was never created; restoring %s",
+                  back->url_sav, back->tmpfile);
+    commit = HTS_FALSE;
+  }
   if (commit) {
     (void) UNLINK(back->tmpfile); /* new copy is good; drop the backup */
+    back_tmpdir_drop(back->tmpfile);
   } else {
     if (back->r.out != NULL) {
       fclose(back->r.out);
@@ -741,8 +734,11 @@ static void back_finalize_backup(httrackp *opt, lien_back *const back,
       hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
                     "could not restore %s; previous copy kept as %s",
                     back->url_sav, back->tmpfile);
+    else
+      back_tmpdir_drop(back->tmpfile);
   }
   back->tmpfile = NULL;
+  return commit == wanted ? HTS_TRUE : HTS_FALSE;
 }
 
 // objet (lien) téléchargé ou transféré depuis le cache
@@ -844,12 +840,21 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                     hts_codec_parse(back[p].r.contentencoding);
                 /* Never decode over url_sav: a failed decode would destroy the
                    copy an --update re-fetch is supposed to refresh (#557). */
-                char BIGSTK unpacked[HTS_URLMAXSIZE * 2 + 4]; // room for ".u"
+                char BIGSTK unpacked[HTS_URLMAXSIZE * 2];
                 LLint size;
 
-                snprintf(unpacked, sizeof(unpacked), "%s.u", back[p].url_sav);
-                if ((size = hts_codec_unpack(codec, back[p].tmpfile,
-                                             unpacked)) >= 0) {
+                /* fits whenever the .z temp it decodes from did */
+                if (!back_tmpname(unpacked, sizeof(unpacked), back[p].url_sav,
+                                  "u")) {
+                  back[p].r.statuscode = STATUSCODE_INVALID;
+                  strcpybuff(back[p].r.msg, "Error when decompressing (the "
+                                            "temporary filename is too long)");
+                  /* as the decode-failure branch below: never let the coded
+                     bytes be committed as the page */
+                  if (!back[p].r.is_write)
+                    deleteaddr(&back[p].r);
+                } else if ((size = hts_codec_unpack(codec, back[p].tmpfile,
+                                                    unpacked)) >= 0) {
                   back[p].r.size = back[p].r.totalsize = size;
                   if (back[p].r.is_write) {
                     /* Sample the previous copy now: the rename below replaces
@@ -921,6 +926,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
               /* ensure that no remaining temporary file exists */
               if (back[p].tmpfile != NULL) {
                 unlink(back[p].tmpfile);
+                back_tmpdir_drop(back[p].tmpfile); /* the .u went with it */
                 back[p].tmpfile = NULL;
               }
             }
@@ -933,7 +939,14 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
         }
         /* Body fully received: keep the freshly written url_sav, drop the
            backup of the previous copy. */
-        back_finalize_backup(opt, &back[p], HTS_TRUE);
+        if (!back_finalize_backup(opt, &back[p], HTS_TRUE)) {
+          /* The previous copy is back because the new one was never created;
+             caching this response's validators against it would pin the stale
+             body on every later --update. */
+          if (fexist_utf8(back[p].url_sav))
+            filenote(&opt->state.strc, back[p].url_sav, NULL);
+          return -1;
+        }
         /* Write mode to disk */
         if (back[p].r.is_write && back[p].r.adr != NULL) {
           freet(back[p].r.adr);
@@ -1223,13 +1236,6 @@ void back_connxfr(htsblk * src, htsblk * dst) {
   src->debugid = 0;
 }
 
-/* Release the buffers a response owns. The connection members are left alone:
-   back_connxfr() moves those, and the file handles are closed elsewhere. */
-static void back_free_response(htsblk *r) {
-  deleteaddr(r);
-  warc_free_request(r);
-}
-
 void back_move(lien_back * src, lien_back * dst) {
   memcpy(dst, src, sizeof(lien_back));
   memset(src, 0, sizeof(lien_back));
@@ -1323,10 +1329,7 @@ int back_unserialize(FILE * fp, lien_back ** dst) {
     (*dst)->r.ssl_con = NULL;
 #endif
     if (back_data_unserialize(fp, (void **) &(*dst)->r.adr, &size) == 0) {
-      /* A bodyless slot already wrote its bytes to url_sav (FTP, direct to
-         disk); zeroing r.size makes the writer blank that file (#797). */
-      if ((*dst)->r.adr != NULL)
-        (*dst)->r.size = size;
+      (*dst)->r.size = size;
       (*dst)->r.headers = NULL;
       if (back_string_unserialize(fp, &(*dst)->r.headers) == 0)
         return 0;               /* ok */
@@ -1765,7 +1768,10 @@ int back_clear_entry(lien_back * back) {
       back->r.soc = INVALID_SOCKET;
     }
 
-    back_free_response(&back->r);
+    if (back->r.adr != NULL) {  // reste un bloc à désallouer
+      freet(back->r.adr);
+      back->r.adr = NULL;
+    }
     if (back->chunk_adr != NULL) {      // reste un bloc à désallouer
       freet(back->chunk_adr);
       back->chunk_adr = NULL;
@@ -1776,8 +1782,15 @@ int back_clear_entry(lien_back * back) {
     // only for security
     if (back->tmpfile && back->tmpfile[0] != '\0') {
       (void) unlink(back->tmpfile);
+      back_tmpdir_drop(back->tmpfile);
       back->tmpfile = NULL;
     }
+    // headers
+    if (back->r.headers != NULL) {
+      freet(back->r.headers);
+      back->r.headers = NULL;
+    }
+    warc_free_request(&back->r);
     // Tout nettoyer
     memset(back, 0, sizeof(lien_back));
     back->r.soc = INVALID_SOCKET;
@@ -4082,9 +4095,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
 
                       memset(&tmp, 0, sizeof(tmp));
                       back_connxfr(&back[i].r, &tmp);
-                      /* the cache entry overwrites the whole struct, so drop
-                         what the 304 response still owns first (#782) */
-                      back_free_response(&back[i].r);
                       back[i].r =
                         cache_read(opt, cache, back[i].url_adr, back[i].url_fil,
                                    back[i].url_sav, back[i].location_buffer);

src/htsback.h

@@ -143,6 +143,18 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
    can put it back when the re-fetch fails (#77 follow-up). Call right before
    truncating url_sav; tmpfile stays NULL when there is nothing to save. */
 void back_refetch_backup(httrackp *opt, lien_back *const back);
+/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
+   existing file moved the good copy to back->tmpfile before truncating url_sav.
+   commit keeps the new file and drops the backup, unless url_sav was never
+   created; else restore it so an aborted transfer leaves the previous copy
+   intact. Skips the zlib .z temp. HTS_FALSE when a requested commit had to
+   restore instead: the caller then holds the OLD body and must not cache this
+   response's validators against it. */
+hts_boolean back_finalize_backup(httrackp *opt, lien_back *const back,
+                                 hts_boolean commit);
+/* Remove the reserved directory a temporary sat in, once the last slot sharing
+   it is done; a non-empty one just refuses. No-op outside that directory. */
+void back_tmpdir_drop(const char *tmp);
 /* -#test=backswap: slots eligible for the on-disk ready table. */
 int back_selftest_slot_swap(void);
 void back_info(struct_back * sback, int i, int j, FILE * fp);

src/htscache.c

@@ -957,6 +957,35 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
     return NULL;
 }
 
+const char *cache_repair(httrackp *opt, const char *name,
+                         unsigned long *entries, unsigned long *bytes) {
+  char BIGSTK repairname[HTS_URLMAXSIZE * 2];
+  unzFile zip;
+
+  *entries = 0;
+  *bytes = 0;
+  if (!slprintfbuff(repairname, sizeof(repairname), "%s%s",
+                    StringBuff(opt->path_log), "hts-cache/repair.zip"))
+    return "the repair path is too long";
+  if (unzRepair(name, repairname,
+                fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                        StringBuff(opt->path_log), "hts-cache/repair.tmp"),
+                entries, bytes) != Z_OK)
+    return "could not repair the cache";
+  /* unzRepair writes an end-of-central-directory record whatever it found, so
+     an input holding no local file header at all yields a valid empty archive
+     and a short write yields a truncated one. Only an archive that holds
+     something and opens may replace the cache (#824). */
+  if (*entries == 0)
+    return "the repaired cache holds no entry, keeping the damaged one";
+  if ((zip = hts_unzOpen_utf8(repairname)) == NULL)
+    return "the repaired cache does not open, keeping the damaged one";
+  unzClose(zip);
+  if (!hts_rename_over(opt, repairname, name))
+    return "could not put the repaired cache in place";
+  return NULL;
+}
+
 // Initialisation du cache: créer nouveau, renomer ancien, charger..
 void cache_init(cache_back * cache, httrackp * opt) {
   // ---
@@ -1026,8 +1055,9 @@ void cache_init(cache_back * cache, httrackp * opt) {
       // Corrupted ZIP file ? Try to repair!
       if (cache->zipInput == NULL && !cache->ro) {
         char *name;
-        uLong repaired = 0;
-        uLong repairedBytes = 0;
+        const char *why;
+        unsigned long repaired = 0;
+        unsigned long repairedBytes = 0;
 
         if (!cache->ro) {
           name =
@@ -1040,25 +1070,16 @@ void cache_init(cache_back * cache, httrackp * opt) {
         }
         hts_log_print(opt, LOG_WARNING,
                       "Cache: damaged cache, trying to repair");
-        /* mztools has no UTF-8 hook, so repairing a corrupt cache under a
-           non-ASCII path_log fails cleanly (re-crawl), never forks a twin. */
-        if (unzRepair
-            (name,
-             fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                     "hts-cache/repair.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                      StringBuff(opt->path_log),
-                                                      "hts-cache/repair.tmp"),
-             &repaired, &repairedBytes) == Z_OK) {
-          UNLINK(name);
-          RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                         StringBuff(opt->path_log), "hts-cache/repair.zip"),
-                 name);
-          cache->zipInput = hts_unzOpen_utf8(name);
+        why = cache_repair(opt, name, &repaired, &repairedBytes);
+        if (why != NULL) {
+          hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "Cache: %s", why);
+        } else if ((cache->zipInput = hts_unzOpen_utf8(name)) != NULL) {
           hts_log_print(opt, LOG_WARNING,
                         "Cache: %d bytes successfully recovered in %d entries",
                         (int) repairedBytes, (int) repaired);
         } else {
-          hts_log_print(opt, LOG_WARNING, "Cache: could not repair the cache");
+          hts_log_print(opt, LOG_WARNING,
+                        "Cache: the repaired cache could not be reopened");
         }
       }
       // Opened ?

src/htscache.h

@@ -78,6 +78,14 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, htsblk * r);
 void cache_init(cache_back * cache, httrackp * opt);
 
+/* Recover the damaged cache at name into hts-cache/repair.zip and move it over
+   name, storing what was recovered in *entries and *bytes. Returns NULL on
+   success, else a reason the caller reports: a recovery that is empty or does
+   not open never replaces the cache, and neither does one that cannot be moved
+   into place (#786, #824). Note: utf-8. */
+const char *cache_repair(httrackp *opt, const char *name,
+                         unsigned long *entries, unsigned long *bytes);
+
 /* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
 typedef enum {
   CACHE_RECONCILE_PROMOTE,     /* no new cache: promote the old generation */

src/htscoremain.c

@@ -2160,8 +2160,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               case 'R':
                 {
                   char *name;
-                  uLong repaired = 0;
-                  uLong repairedBytes = 0;
+                  const char *why;
+                  unsigned long repaired = 0;
+                  unsigned long repairedBytes = 0;
 
                   if (fexist_utf8(fconcat(
                           OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
@@ -2184,24 +2185,15 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                     return 1;
                   }
                   fprintf(stderr, "Cache: trying to repair %s\n", name);
-                  if (unzRepair
-                      (name,
-                       fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                               "hts-cache/repair.zip"),
-                       fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                               "hts-cache/repair.tmp"), &repaired,
-                       &repairedBytes) == Z_OK) {
-                    UNLINK(name);
-                    RENAME(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                   StringBuff(opt->path_log),
-                                   "hts-cache/repair.zip"),
-                           name);
-                    fprintf(stderr,
-                            "Cache: %d bytes successfully recovered in %d entries\n",
-                            (int) repairedBytes, (int) repaired);
-                  } else {
-                    fprintf(stderr, "Cache: could not repair the cache\n");
+                  why = cache_repair(opt, name, &repaired, &repairedBytes);
+                  if (why != NULL) {
+                    fprintf(stderr, "Cache: %s\n", why);
+                    return 1;
                   }
+                  fprintf(
+                      stderr,
+                      "Cache: %d bytes successfully recovered in %d entries\n",
+                      (int) repairedBytes, (int) repaired);
                 }
                 return 0;
                 break;

src/htslib.c

@@ -2703,8 +2703,17 @@ void hts_now_iso8601(char out[32]) {
   time_t t = time(NULL);
   struct tm tmv;
 
-  if (!hts_gmtime(t, &tmv))
+#if defined(_WIN32)
+  struct tm *g = gmtime(&t);
+
+  if (g != NULL)
+    tmv = *g;
+  else
     memset(&tmv, 0, sizeof(tmv));
+#else
+  if (gmtime_r(&t, &tmv) == NULL)
+    memset(&tmv, 0, sizeof(tmv));
+#endif
   strftime(out, 32, "%Y-%m-%dT%H:%M:%SZ", &tmv);
 }
 

src/htslib.h

@@ -157,19 +157,6 @@ struct t_dnscache {
   char host_addr[HTS_MAXADDRNUM][HTS_MAXADDRLEN];
 };
 
-/* Break t down as UTC into the caller's buffer, HTS_FALSE if that failed.
-   gmtime()'s static is shared, and both the engine and ProxyTrack convert on
-   worker threads. */
-static HTS_INLINE HTS_UNUSED hts_boolean hts_gmtime(time_t t,
-                                                    struct tm *tmbuf) {
-#ifdef _WIN32
-  /* Microsoft's gmtime_s takes the destination first, unlike C11 Annex K. */
-  return gmtime_s(tmbuf, &t) == 0 ? HTS_TRUE : HTS_FALSE;
-#else
-  return gmtime_r(&t, tmbuf) != NULL ? HTS_TRUE : HTS_FALSE;
-#endif
-}
-
 /* Library internal definictions */
 #ifdef HTS_INTERNAL_BYTECODE
 

src/htsname.c

@@ -74,6 +74,41 @@ static const char *hts_tbdev[] = {
   ""
 };
 
+/* Directories the engine owns inside the mirror. A URL able to name one lands
+   on the cache or on another slot's temporary and destroys it (#774), so they
+   are escaped like the DOS devices are. */
+static const char *hts_tbreserved[] = {"/hts-cache", "/hts-tmp", ""};
+
+/* Replace /foo/<reserved>/bar by /foo/<reserved>_/bar, matching a whole path
+   component only (case-insensitively: the filesystem may be too). */
+static void escapeReservedNames(char *save, size_t size,
+                                const char *const *names) {
+  int i;
+
+  for (i = 0; names[i][0] != '\0'; i++) {
+    const char *a = save;
+    const size_t len = strlen(names[i]);
+
+    while ((a = strstrcase(a, names[i]))) {
+      switch ((int) a[len]) {
+      case '\0':
+      case '/':
+      case '.': {
+        char BIGSTK tempo[HTS_URLMAXSIZE * 2];
+
+        tempo[0] = '\0';
+        strncatbuff(tempo, save, (int) (a - save) + (int) len);
+        strcatbuff(tempo, "_");
+        strcatbuff(tempo, a + len);
+        /* clip rather than abort: the name comes from the wire */
+        (void) strclipbuff(save, size, tempo);
+      } break;
+      }
+      a += len;
+    }
+  }
+}
+
 /* Strip all // */
 static void cleanDoubleSlash(char *s) {
   int i, j;
@@ -1369,35 +1404,12 @@ int url_savename(lien_adrfilsave *const afs,
   // éliminer les // (comme ftp://)
   cleanDoubleSlash(afs->save);
 
+  /* Runs on every platform, and before path_html is prepended below, so the
+     user's own output directory is never renamed. */
+  escapeReservedNames(afs->save, sizeof(afs->save), hts_tbreserved);
+
 #if HTS_OVERRIDE_DOS_FOLDERS
-  /* Replace /foo/nul/bar by /foo/nul_/bar */
-  {
-    int i = 0;
-
-    while(hts_tbdev[i][0]) {
-      const char *a = afs->save;
-
-      while((a = strstrcase(a, hts_tbdev[i]))) {
-        switch ((int) a[strlen(hts_tbdev[i])]) {
-        case '\0':
-        case '/':
-        case '.':
-          {
-            char BIGSTK tempo[HTS_URLMAXSIZE * 2];
-
-            tempo[0] = '\0';
-            strncatbuff(tempo, afs->save, (int) (a - afs->save) + strlen(hts_tbdev[i]));
-            strcatbuff(tempo, "_");
-            strcatbuff(tempo, a + strlen(hts_tbdev[i]));
-            strcpybuff(afs->save, tempo);
-          }
-          break;
-        }
-        a += strlen(hts_tbdev[i]);
-      }
-      i++;
-    }
-  }
+  escapeReservedNames(afs->save, sizeof(afs->save), hts_tbdev);
 
   /* Strip ending . or ' ' forbidden on windoz */
   cleanEndingSpaceOrDot(afs->save);

src/htsselftest.c

@@ -2659,11 +2659,8 @@ static int st_logcallback(httrackp *opt, int argc, char **argv) {
   strcpybuff(seen, st_log_callback_seen);
 
   rewind(fp);
-  if (fgets(line, (int) sizeof(line), fp) == NULL) {
-    fprintf(stderr, "logcallback: log file is empty, nothing was written\n");
-    fclose(fp);
-    return 1;
-  }
+  line[0] = '\0';
+  (void) fgets(line, (int) sizeof(line), fp);
   fclose(fp);
 
   /* The callback runs above the level filter and without a log file at all;
@@ -6318,6 +6315,90 @@ static int st_renameover(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+// -#test=refetchbackup <dir>: the #77 re-fetch backup must build its temporary
+// inside the reserved hts-tmp directory, whose segment url_savename escapes so
+// no mirrored file can ever sit there (#774), and must never leave the resource
+// without a copy (#775).
+static int st_refetchbackup(httrackp *opt, int argc, char **argv) {
+  lien_back *back;
+  char want[HTS_URLMAXSIZE * 2 + 32];
+  int err = 0;
+
+  if (argc < 1) {
+    fprintf(stderr, "refetchbackup: needs a writable base dir\n");
+    return 1;
+  }
+  back = calloct(1, sizeof(lien_back));
+  if (back == NULL) {
+    fprintf(stderr, "refetchbackup: out of memory\n");
+    return 1;
+  }
+  /* explicit separator: fconcat() joins without one, which would put the
+     temporary in the parent of the directory under test */
+  snprintf(back->url_sav, sizeof(back->url_sav), "%s/refetch.bin", argv[0]);
+  snprintf(want, sizeof(want), "%s/hts-tmp/refetch.bin.bak", argv[0]);
+
+  /* #774: pin the name, so moving the temporary out of the reserved directory
+     cannot pass without url_savename reserving wherever it went instead. */
+  ro_put(back->url_sav, "old");
+  back_refetch_backup(opt, back);
+  if (back->tmpfile == NULL || fexist_utf8(back->url_sav)) {
+    fprintf(stderr, "refetchbackup: the previous copy was not moved aside\n");
+    err++;
+  } else if (strcmp(back->tmpfile, want) != 0) {
+    fprintf(stderr, "refetchbackup: temporary is %s, want %s\n", back->tmpfile,
+            want);
+    err++;
+  }
+  ro_put(back->url_sav, "new"); /* what filecreate() + the transfer produce */
+  back_finalize_backup(opt, back, HTS_TRUE);
+  if (!ro_is(back->url_sav, "new")) {
+    fprintf(stderr, "refetchbackup: the committed copy is not the new one\n");
+    err++;
+  }
+
+  /* #758: only a killed run can leave something there, and it must be replaced
+     rather than disable the backup for good. */
+  if (structcheck(want) != 0) {
+    fprintf(stderr, "refetchbackup: cannot create %s\n", want);
+    freet(back);
+    return 1;
+  }
+  ro_put(want, "leftover");
+  back_refetch_backup(opt, back);
+  if (back->tmpfile == NULL || !ro_is(want, "new")) {
+    fprintf(stderr, "refetchbackup: a leftover temporary blocked the backup\n");
+    err++;
+  }
+
+  /* #775: filecreate() failed, so there is nothing to commit to. Saying so is
+     load-bearing: the caller must not cache this response against the old
+     body, or the next --update gets a 304 pinning it. */
+  (void) UNLINK(back->url_sav);
+  if (back_finalize_backup(opt, back, HTS_TRUE)) {
+    fprintf(stderr, "refetchbackup: a commit that restored reported success\n");
+    err++;
+  }
+  if (!ro_is(back->url_sav, "new")) {
+    fprintf(stderr, "refetchbackup: a commit with no new copy lost both\n");
+    err++;
+  }
+
+  /* An aborted transfer restores, as before. */
+  back_refetch_backup(opt, back);
+  ro_put(back->url_sav, "partial");
+  back_finalize_backup(opt, back, HTS_FALSE);
+  if (!ro_is(back->url_sav, "new")) {
+    fprintf(stderr, "refetchbackup: an aborted re-fetch kept the partial\n");
+    err++;
+  }
+
+  (void) UNLINK(back->url_sav);
+  freet(back);
+  printf("refetchbackup: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 // -#test=direnum <dir>: enumerate a long+non-ASCII directory via the
 // opendir/readdir wrappers; children must round-trip as UTF-8 (#133,#630).
 static int st_direnum(httrackp *opt, int argc, char **argv) {
@@ -6694,109 +6775,6 @@ static int st_threadwait(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
-/* #794: hts_gmtime() must own its output. The table is an independent oracle;
-   the threaded phase is what corrupts if it ever goes back to gmtime()'s
-   shared static. */
-#define GMTIME_THREADS 8
-#define GMTIME_ROUNDS 50000
-
-static const struct {
-  time_t t;
-  int year, mon, mday, hour, min, sec, wday, yday;
-} gmtime_refs[] = {
-    {(time_t) 0, 70, 0, 1, 0, 0, 0, 4, 0},
-    {(time_t) 951782400, 100, 1, 29, 0, 0, 0, 2, 59}, /* a leap day */
-    {(time_t) 1000000000, 101, 8, 9, 1, 46, 40, 0, 251},
-    {(time_t) 2147483647, 138, 0, 19, 3, 14, 7, 2, 18}, /* 32-bit ceiling */
-};
-
-#define GMTIME_REFS ((int) (sizeof(gmtime_refs) / sizeof(gmtime_refs[0])))
-
-static hts_boolean gmtime_ref_matches(int i, const struct tm *tm) {
-  if (tm->tm_year != gmtime_refs[i].year || tm->tm_mon != gmtime_refs[i].mon ||
-      tm->tm_mday != gmtime_refs[i].mday ||
-      tm->tm_hour != gmtime_refs[i].hour || tm->tm_min != gmtime_refs[i].min ||
-      tm->tm_sec != gmtime_refs[i].sec || tm->tm_wday != gmtime_refs[i].wday ||
-      tm->tm_yday != gmtime_refs[i].yday)
-    return HTS_FALSE;
-  return HTS_TRUE;
-}
-
-static htsmutex gmtime_lock = HTSMUTEX_INIT;
-static int gmtime_bad = 0;
-
-static void gmtime_thread(void *arg) {
-  const int i = *(const int *) arg;
-  int bad = 0, round;
-
-  for (round = 0; round < GMTIME_ROUNDS; round++) {
-    struct tm tmv;
-
-    if (!hts_gmtime(gmtime_refs[i].t, &tmv) || !gmtime_ref_matches(i, &tmv))
-      bad++;
-  }
-  hts_mutexlock(&gmtime_lock);
-  gmtime_bad += bad;
-  hts_mutexrelease(&gmtime_lock);
-}
-
-static int st_gmtime(httrackp *opt, int argc, char **argv) {
-  static int idx[GMTIME_THREADS];
-  int err = 0, i;
-
-  (void) opt;
-  (void) argc;
-  (void) argv;
-
-  for (i = 0; i < GMTIME_REFS; i++) {
-    struct tm tmv;
-
-    if (!hts_gmtime(gmtime_refs[i].t, &tmv)) {
-      fprintf(stderr, "gmtime: conversion #%d failed\n", i);
-      err = 1;
-    } else if (!gmtime_ref_matches(i, &tmv)) {
-      fprintf(stderr,
-              "gmtime: #%d gave %04d-%02d-%02d %02d:%02d:%02d (wday %d, "
-              "yday %d)\n",
-              i, tmv.tm_year + 1900, tmv.tm_mon + 1, tmv.tm_mday, tmv.tm_hour,
-              tmv.tm_min, tmv.tm_sec, tmv.tm_wday, tmv.tm_yday);
-      err = 1;
-    }
-  }
-
-  /* the return is the only failure signal the callers have, so a helper that
-     always claims success leaves them formatting an uninitialised struct tm.
-     Out of range for a 64-bit time_t: NULL from gmtime_r, EINVAL from
-     _gmtime64_s. */
-  if (sizeof(time_t) >= 8) {
-    const time_t beyond = (time_t) INT64_MAX;
-    struct tm tmv;
-
-    if (hts_gmtime(beyond, &tmv)) {
-      fprintf(stderr,
-              "gmtime: an out-of-range time_t was reported converted\n");
-      err = 1;
-    }
-  }
-
-  for (i = 0; i < GMTIME_THREADS; i++) {
-    idx[i] = i % GMTIME_REFS;
-    if (hts_newthread(gmtime_thread, &idx[i]) != 0) {
-      fprintf(stderr, "gmtime: cannot spawn\n");
-      return 1;
-    }
-  }
-  htsthread_wait();
-  if (gmtime_bad != 0) {
-    fprintf(stderr, "gmtime: %d/%d concurrent conversions were corrupt\n",
-            gmtime_bad, GMTIME_THREADS * GMTIME_ROUNDS);
-    err = 1;
-  }
-
-  printf("gmtime self-test: %s\n", err ? "FAIL" : "OK");
-  return err;
-}
-
 #define CHANGES_RACE_FILES 8
 #define CHANGES_RACE_ROUNDS 400
 
@@ -7063,8 +7041,6 @@ static const struct selftest_entry {
      st_changes_race},
     {"threadwait", "", "htsthread_wait() joins threads spawned just before it",
      st_threadwait},
-    {"gmtime", "",
-     "hts_gmtime() fills the caller's buffer, not a static (#794)", st_gmtime},
     {"backswap", "", "which backlog slots may be swapped to the ready table",
      st_backswap},
     {"pause", "", "randomized inter-file pause target self-test", st_pause},
@@ -7173,6 +7149,9 @@ static const struct selftest_entry {
      "hts_rename_over(): replace dst, but never delete a dst it did not "
      "replace",
      st_renameover},
+    {"refetchbackup", "<dir>",
+     "the re-fetch backup always leaves a copy, and stays out of the mirror",
+     st_refetchbackup},
     {"direnum", "<dir>",
      "enumerate a long+non-ASCII directory through opendir/readdir",
      st_direnum},

src/htswarc.c

@@ -38,6 +38,7 @@ Please visit our Website: http://www.httrack.com
 
 #include "htscore.h"
 #include "htslib.h"
+#include "htsback.h"
 #include "htstools.h"
 #include "htssafe.h"
 #include "htszlib.h"
@@ -1354,6 +1355,7 @@ void warc_free_request(htsblk *r) {
     freet(r->warc_resphdr);
     if (r->warc_rawpath != NULL) {
       (void) UNLINK(r->warc_rawpath); /* owns the verbatim spool file */
+      back_tmpdir_drop(r->warc_rawpath);
       freet(r->warc_rawpath);
       r->warc_rawpath = NULL;
     }
@@ -1392,8 +1394,16 @@ warc_writer *warc_open(httrackp *opt, const char *path) {
     char ts[32];
     time_t t = time(NULL);
     struct tm tmv;
-    if (!hts_gmtime(t, &tmv))
+#if defined(_WIN32)
+    struct tm *g = gmtime(&t);
+    if (g != NULL)
+      tmv = *g;
+    else
       memset(&tmv, 0, sizeof(tmv));
+#else
+    if (gmtime_r(&t, &tmv) == NULL)
+      memset(&tmv, 0, sizeof(tmv));
+#endif
     strftime(ts, sizeof(ts), "%Y%m%d%H%M%S", &tmv);
     snprintf(catbuff, sizeof(catbuff), "httrack-%s.warc.gz", ts);
     path =

src/proxy/proxytrack.c

@@ -520,13 +520,12 @@ static void proxytrack_add_DAV_Item(String * item, String * buff,
                                     const char *filename, size_t size,
                                     time_t timestamp, const char *mime,
                                     int isDir, int isRoot, int isDefault) {
-  struct tm timetmbuf;
-  struct tm *timetm = &timetmbuf;
+  struct tm *timetm;
 
   if (timestamp == (time_t) 0 || timestamp == (time_t) - 1) {
     timestamp = time(NULL);
   }
-  if (hts_gmtime(timestamp, timetm)) {
+  if ((timetm = gmtime(&timestamp)) != NULL) {
     char tms[256 + 1];
     const char *name;
 

src/proxy/proxytrack.h

@@ -360,14 +360,21 @@ HTS_UNUSED static struct tm *convert_time_rfc822(struct tm *result, const char *
 HTS_UNUSED static struct tm PT_GetTime(time_t t) {
   struct tm tmbuf;
 
-  if (!hts_gmtime(t, &tmbuf)) {
+#ifdef _WIN32
+  struct tm *tm = gmtime(&t);
+#else
+  struct tm *tm = gmtime_r(&t, &tmbuf);
+#endif
+  if (tm != NULL)
+    return *tm;
+  else {
     /* an all-zero tm has tm_mday == 0, which the ARC date field prints as a
        day of "00"; the epoch is the conventional "date unknown" */
     memset(&tmbuf, 0, sizeof(tmbuf));
     tmbuf.tm_year = 70;
     tmbuf.tm_mday = 1;
+    return tmbuf;
   }
-  return tmbuf;
 }
 HTS_UNUSED static int set_filetime(const char *file, struct tm *tm_time) {
   struct utimbuf tim;

src/proxy/store.c

@@ -888,31 +888,6 @@ static PT_Element PT_ReadCache__New(PT_Index index, const char *url, int flags)
 	} \
 } while(0)
 
-/* Set path (capacity size) to filename's parent directory, separator included,
-   from an absolute filename. Empty when there is none, or when it would not
-   fit: a truncated prefix names a different directory. */
-static void index_base_path(char *path, size_t size, const char *filename) {
-  const char *abpath;
-  int slashes;
-
-  for (slashes = 2, abpath = filename + (int) strlen(filename) - 1;
-       abpath > filename &&
-       ((*abpath != '/' && *abpath != '\\') || --slashes > 0);
-       abpath--)
-    ;
-  path[0] = '\0';
-  if (slashes == 0 && *abpath != 0 && (size_t) (abpath - filename) < size - 1) {
-    int i;
-
-    strlncatbuff(path, filename, size, (size_t) (abpath - filename) + 1);
-    for (i = 0; path[i] != 0; i++) {
-      if (path[i] == '\\') {
-        path[i] = '/';
-      }
-    }
-  }
-}
-
 int PT_LoadCache__New(PT_Index index_, const char *filename) {
   if (index_ != NULL && filename != NULL) {
     PT_Index__New index = &index_->slots.formatNew;
@@ -923,9 +898,26 @@ int PT_LoadCache__New(PT_Index index_, const char *filename) {
 
     // Opened ?
     if (zFile != NULL) {
+      const char *abpath;
+      int slashes;
       coucal hashtable = index->hash;
 
-      index_base_path(index->path, sizeof(index->path), filename);
+      /* Compute base path for this index - the filename MUST be absolute! */
+      for(slashes = 2, abpath = filename + (int) strlen(filename) - 1;
+          abpath > filename && ((*abpath != '/' && *abpath != '\\')
+                                || --slashes > 0);
+          abpath--) ;
+      index->path[0] = '\0';
+      if (slashes == 0 && *abpath != 0) {
+        int i;
+
+        strncat(index->path, filename, (int) (abpath - filename) + 1);
+        for(i = 0; index->path[i] != 0; i++) {
+          if (index->path[i] == '\\') {
+            index->path[i] = '/';
+          }
+        }
+      }
 
       /* Ready directory entries */
       if (unzGoToFirstFile(zFile) == Z_OK) {
@@ -1498,23 +1490,41 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
 
     cache->filenameDat[0] = '\0';
     cache->filenameNdx[0] = '\0';
+    cache->path[0] = '\0';
 
-    index_base_path(cache->path, sizeof(cache->path), filename);
+    {
+      PT_Index__Old index = cache;
+      const char *abpath;
+      int slashes;
+
+      /* -------------------- COPY OF THE __New() CODE -------------------- */
+      /* Compute base path for this index - the filename MUST be absolute! */
+      for(slashes = 2, abpath = filename + (int) strlen(filename) - 1;
+          abpath > filename && ((*abpath != '/' && *abpath != '\\')
+                                || --slashes > 0);
+          abpath--) ;
+      index->path[0] = '\0';
+      if (slashes == 0 && *abpath != 0) {
+        int i;
+
+        strncat(index->path, filename, (int) (abpath - filename) + 1);
+        for(i = 0; index->path[i] != 0; i++) {
+          if (index->path[i] == '\\') {
+            index->path[i] = '/';
+          }
+        }
+      }
+      /* -------------------- END OF COPY OF THE __New() CODE -------------------- */
+    }
 
     /* Index/data filenames */
     if (pos != NULL) {
-      const size_t nLen = (size_t) (pos - filename);
+      int nLen = (int) (pos - filename);
 
-      /* a base clipped to fit would name a different pair of files */
-      if (nLen > sizeof(cache->filenameDat) - sizeof(".dat")) {
-        return 0;
-      }
-      strlncatbuff(cache->filenameDat, filename, sizeof(cache->filenameDat),
-                   nLen);
-      strlncatbuff(cache->filenameNdx, filename, sizeof(cache->filenameNdx),
-                   nLen);
-      strcatbuff(cache->filenameDat, ".dat");
-      strcatbuff(cache->filenameNdx, ".ndx");
+      strncat(cache->filenameDat, filename, nLen);
+      strncat(cache->filenameNdx, filename, nLen);
+      strcat(cache->filenameDat, ".dat");
+      strcat(cache->filenameNdx, ".ndx");
     }
     ndxSize = filesize(cache->filenameNdx);
     cache->timestamp = file_timestamp(cache->filenameDat);
@@ -1577,10 +1587,7 @@ static int PT_LoadCache__Old(PT_Index index_, const char *filename) {
               }
               /* read position */
               a += binput(a, linepos, 200);
-              /* an unparseable field must not carry the previous entry's
-                 offset over, nor read the stack on the first one */
-              if (sscanf(linepos, "%d", &pos) != 1)
-                pos = 0;
+              sscanf(linepos, "%d", &pos);
 
               /* Add entry */
               coucal_add(cache->hash, line, pos);

tests/01_engine-changes.test

@@ -3,14 +3,27 @@
 
 set -euo pipefail
 
-testdir=$(cd "$(dirname "$0")" && pwd)
-# shellcheck source=tests/testlib.sh
-. "${testdir}/testlib.sh"
-
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_changes_st.XXXXXX") || exit 1
 trap 'set +e; rm -rf "$tmpdir"' EXIT
 trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
 
+# No pipe into grep: SIGPIPE would mask a failing exit status.
+expect_ok() {
+    local label="$1" out
+    shift
+    out=$("$@" 2>&1) || {
+        echo "FAIL: ${label} exited non-zero: ${out}"
+        exit 1
+    }
+    case "$out" in
+    *"${label}: OK"*) ;;
+    *)
+        echo "FAIL: ${out}"
+        exit 1
+        ;;
+    esac
+}
+
 # --changes bucket accounting and JSON escaping (#714).
 expect_ok "changes self-test" httrack -O "${tmpdir}/o1" -#test=changes run
 # The report must hold against a transfer thread the crawl never joins.

tests/01_engine-gmtime.test

@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-testdir=$(cd "$(dirname "$0")" && pwd)
-# shellcheck source=tests/testlib.sh
-. "${testdir}/testlib.sh"
-
-# A non-UTC zone, so a helper that reached for localtime instead of gmtime gives
-# a different answer: CI runners are UTC, where the two agree.
-TZ=XXX5
-export TZ
-
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_gmtime_st.XXXXXX") || exit 1
-trap 'set +e; rm -rf "$tmpdir"' EXIT
-trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
-
-# Concurrent conversions do not tread on each other the way gmtime()'s shared
-# static does (#794).
-expect_ok "gmtime self-test" httrack -O "${tmpdir}/o1" -#test=gmtime

tests/01_engine-savename.test

@@ -165,3 +165,20 @@ MINGW* | MSYS* | CYGWIN*)
     name "/$(printf 'a%.0s' {1..300}).php" 'text/html' "$(printf 'a%.0s' {1..300}).html"
     ;;
 esac
+
+# #774: the engine owns hts-cache/ and hts-tmp/ inside the mirror, so a URL must
+# never be able to name one; a site serving them gets the /nul -> nul_ treatment.
+full '/hts-tmp/a.bin' 'application/octet-stream' \
+    '/dev/null/www.example.com/hts-tmp_/a.bin'
+full '/hts-cache/new.zip' 'application/octet-stream' \
+    '/dev/null/www.example.com/hts-cache_/new.zip'
+full '/d/hts-tmp/a.bin.bak' 'application/octet-stream' \
+    '/dev/null/www.example.com/d/hts-tmp_/a.bin.bak'
+# A whole component only, and case-insensitively: the filesystem may be too.
+full '/hts-tmpfoo/a.bin' 'application/octet-stream' \
+    '/dev/null/www.example.com/hts-tmpfoo/a.bin'
+full '/HTS-TMP/a.bin' 'application/octet-stream' \
+    '/dev/null/www.example.com/HTS-TMP_/a.bin'
+# Control: the DOS device names this reuses must still be escaped.
+full '/nul/x.bin' 'application/octet-stream' \
+    '/dev/null/www.example.com/nul_/x.bin'

tests/01_engine-threadwait.test

@@ -3,14 +3,27 @@
 
 set -euo pipefail
 
-testdir=$(cd "$(dirname "$0")" && pwd)
-# shellcheck source=tests/testlib.sh
-. "${testdir}/testlib.sh"
-
 tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_threadwait_st.XXXXXX") || exit 1
 trap 'set +e; rm -rf "$tmpdir"' EXIT
 trap 'rm -rf "$tmpdir"' HUP INT QUIT PIPE TERM
 
+# No pipe into grep: SIGPIPE would mask a failing exit status.
+expect_ok() {
+    local label="$1" out
+    shift
+    out=$("$@" 2>&1) || {
+        echo "FAIL: ${label} exited non-zero: ${out}"
+        exit 1
+    }
+    case "$out" in
+    *"${label}: OK"*) ;;
+    *)
+        echo "FAIL: ${out}"
+        exit 1
+        ;;
+    esac
+}
+
 # A thread is outstanding from the moment hts_newthread() returns, so a wait
 # that follows the spawn joins it, and wait_n(n) still leaves n behind (#747).
 expect_ok "threadwait self-test" httrack -O "${tmpdir}/o1" -#test=threadwait

tests/101_local-update-stale-bak.test

@@ -12,8 +12,8 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" \
-    --plant-file bigtrunc/slow.bin.bak \
-    --plant-dir bigtrunc/fast.bin.bak \
+    --plant-file bigtrunc/hts-tmp/slow.bin.bak \
+    --plant-dir bigtrunc/hts-tmp/fast.bin.bak \
     --rerun-args '--update -M400000' \
     --log-found 'More than 400000 bytes have been transferred.. giving up' \
     --log-found 'replacing leftover backup .*slow\.bin\.bak' \

tests/106_engine-repair-rename.test

@@ -0,0 +1,173 @@
+#!/bin/bash
+#
+# #786: both cache-repair paths moved the recovered zip onto the cache without
+# checking the move, and announced success either way. #824: they also committed
+# a recovery holding no entry at all. Neither may replace the cache, and both
+# must be reported.
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+tmpdir=$(mktemp -d) # honors the Windows-shaped TMPDIR the MSYS suite exports
+cleanup() { rm -rf "$tmpdir"; }
+trap 'set +e; cleanup' EXIT
+trap 'set +e; cleanup; exit 1' HUP INT QUIT PIPE TERM
+
+# One local file header, no central directory: unzOpen fails, unzRepair
+# recovers the single entry. Same vector as -#test=zip-repair-shift.
+plant_damaged_cache() {
+    rm -rf "$1"
+    mkdir -p "$1/hts-cache"
+    printf '\120\113\003\004\024\000\000\000\000\000\000\000\000\000\000\000\350\212\000\000\000\000\000\000\000\000\001\000\000\000\141' \
+        >"$1/hts-cache/new.zip"
+}
+
+dead=http://127.0.0.1:1/
+
+# --- the CLI path (-#R), repair succeeding -----------------------------------
+printf '[-#R repairs in place] ..\t'
+proj="${tmpdir}/cli"
+plant_damaged_cache "$proj"
+out=$(httrack -O "$proj" -#R "$dead" 2>&1)
+grep -q 'successfully recovered' <<<"$out" || {
+    echo "FAIL: no recovery reported: $out"
+    exit 1
+}
+test ! -e "${proj}/hts-cache/repair.zip" || {
+    echo "FAIL: the recovery was left in repair.zip"
+    exit 1
+}
+test "$(wc -c <"${proj}/hts-cache/new.zip")" -gt 31 || {
+    echo "FAIL: new.zip is still the damaged one"
+    exit 1
+}
+echo "OK"
+
+# --- #824: nothing was recoverable, so nothing may be committed --------------
+printf '[-#R refuses an empty recovery] ..\t'
+proj="${tmpdir}/empty"
+rm -rf "$proj"
+mkdir -p "$proj/hts-cache"
+printf 'not a zip at all, some bytes' >"$proj/hts-cache/new.zip"
+rc=0
+out=$(httrack -O "$proj" -#R "$dead" 2>&1) || rc=$?
+test "$rc" -ne 0 || {
+    echo "FAIL: an empty recovery exited 0: $out"
+    exit 1
+}
+if grep -q 'successfully recovered' <<<"$out"; then
+    echo "FAIL: an empty recovery claimed success: $out"
+    exit 1
+fi
+test "$(wc -c <"${proj}/hts-cache/new.zip")" -eq 28 || {
+    echo "FAIL: the damaged cache was replaced by the empty recovery"
+    exit 1
+}
+echo "OK"
+
+if [ "$(uname -s)" != "Linux" ]; then
+    echo "repair-rename: LD_PRELOAD interposition is Linux-only here, skipping"
+    exit 0
+fi
+# A --disable-shared build has nothing to preload; anything else missing is a
+# build problem, not a skip, or the failure legs would pass vacuously.
+if [ ! -r "${RENAMEFAIL_LA:-}" ]; then
+    echo "repair-rename: ${RENAMEFAIL_LA:-\$RENAMEFAIL_LA} was not built" >&2
+    exit 1
+fi
+if grep -q "^dlname=''" "$RENAMEFAIL_LA"; then
+    echo "repair-rename: static-only build, skipping the interposed legs"
+    exit 0
+fi
+if [ ! -r "${RENAMEFAIL_LIB:-}" ]; then
+    echo "repair-rename: ${RENAMEFAIL_LIB:-\$RENAMEFAIL_LIB} was not built" >&2
+    exit 1
+fi
+
+# An LD_PRELOAD library loads ahead of the executable's own libasan, which ASan
+# refuses by default. The shim allocates nothing, so the ordering is harmless.
+export ASAN_OPTIONS="${ASAN_OPTIONS:+$ASAN_OPTIONS:}verify_asan_link_order=0"
+export RENAMEFAIL_MODE=repair
+export LD_PRELOAD="$RENAMEFAIL_LIB"
+
+# --- the CLI path, move refused ----------------------------------------------
+printf '[-#R reports a refused move] ..\t'
+proj="${tmpdir}/cli-fail"
+plant_damaged_cache "$proj"
+rc=0
+out=$(httrack -O "$proj" -#R "$dead" 2>&1) || rc=$?
+test "$rc" -ne 0 || {
+    echo "FAIL: a refused move exited 0: $out"
+    exit 1
+}
+if grep -q 'successfully recovered' <<<"$out"; then
+    echo "FAIL: a refused move still claimed success: $out"
+    exit 1
+fi
+test -s "${proj}/hts-cache/repair.zip" || {
+    echo "FAIL: the recovery was not kept for a retry"
+    exit 1
+}
+test "$(wc -c <"${proj}/hts-cache/new.zip")" -eq 31 || {
+    echo "FAIL: the damaged cache was destroyed anyway"
+    exit 1
+}
+echo "OK"
+
+# --- the engine path (cache_init), move refused ------------------------------
+printf '[cache_init reports a refused move] ..\t'
+proj="${tmpdir}/engine"
+plant_damaged_cache "$proj"
+httrack -O "$proj" "$dead" --timeout=2 --retries=0 -q >/dev/null 2>&1 || true
+log="${proj}/hts-log.txt"
+test -r "$log" || {
+    echo "FAIL: no crawl log"
+    exit 1
+}
+grep -q 'damaged cache' "$log" || {
+    echo "FAIL: the repair path never ran"
+    exit 1
+}
+grep -q 'could not put the repaired cache in place' "$log" || {
+    echo "FAIL: a refused move was not reported: $(grep -i cache "$log")"
+    exit 1
+}
+if grep -q 'successfully recovered' "$log"; then
+    echo "FAIL: a refused move still claimed success"
+    exit 1
+fi
+test -s "${proj}/hts-cache/repair.zip" || {
+    echo "FAIL: the recovery was not kept for a retry"
+    exit 1
+}
+echo "OK"
+
+# --- the move is refused the way Windows refuses it --------------------------
+# The destination always exists there, so every repair takes hts_rename_over()'s
+# fallback. Before #790 that fallback removed the cache and then failed.
+printf '[a refused fallback keeps the cache] ..\t'
+proj="${tmpdir}/eexist"
+plant_damaged_cache "$proj"
+rc=0
+out=$(RENAMEFAIL_MODE=repair-eexist httrack -O "$proj" -#R "$dead" 2>&1) || rc=$?
+test "$rc" -ne 0 || {
+    echo "FAIL: a refused fallback exited 0: $out"
+    exit 1
+}
+test "$(wc -c <"${proj}/hts-cache/new.zip")" -eq 31 || {
+    echo "FAIL: the fallback destroyed the cache"
+    exit 1
+}
+test -s "${proj}/hts-cache/repair.zip" || {
+    echo "FAIL: the recovery was not kept for a retry"
+    exit 1
+}
+leftover=$(find "${proj}/hts-cache" -name '*.hts-old*' | head -1)
+test -z "$leftover" || {
+    echo "FAIL: the parked copy was left behind: $leftover"
+    exit 1
+}
+echo "OK"

tests/107_local-bak-collision.test

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# #774: the re-fetch backup lives in an hts-tmp directory beside the mirrored
+# file, so the site here serves that exact path. url_savename must escape the
+# reserved segment, landing the sibling in hts-tmp_ where the backup of a.bin
+# cannot consume it. Asserting the unescaped path instead would pass with the
+# bug live, which is how the first attempt at this test missed it.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update' \
+    --file-matches 'bakname/a.bin' 'BAKNAME-MAIN-V2' \
+    --file-matches 'bakname/hts-tmp_/a.bin.bak' 'BAKNAME-SIBLING' \
+    --file-min-bytes 'bakname/hts-tmp_/a.bin.bak' 2048 \
+    --not-found 'bakname/hts-tmp/a.bin.bak' \
+    --log-not-found 'could not back up' \
+    --log-not-found 'could not restore' \
+    httrack 'BASEURL/bakname/index.html'

tests/108_engine-refetch-backup.test

@@ -0,0 +1,25 @@
+#!/bin/bash
+#
+# Drives -#test=refetchbackup: the #77 re-fetch backup must leave a copy of the
+# resource whatever happens, and must not build its temporary name inside the
+# mirror namespace (#774, #775).
+
+set -euo pipefail
+
+dir=$(mktemp -d)
+trap 'set +e; rm -rf "$dir"' EXIT
+trap 'set +e; rm -rf "$dir"; exit 1' HUP INT QUIT PIPE TERM
+
+rc=0
+out=$(httrack -O /dev/null -#test=refetchbackup "$dir" 2>&1) || rc=$?
+if test "$rc" -ne 0 || ! grep -q "refetchbackup: OK" <<<"$out"; then
+    echo "FAIL (exit $rc): $out" >&2
+    exit 1
+fi
+
+# The temporaries live in hts-tmp and the last user removes it.
+leftovers=$(find "$dir" -mindepth 1 | head -5)
+test -z "$leftovers" || {
+    echo "FAIL: leftover temporaries: $leftovers" >&2
+    exit 1
+}

tests/110_local-ftp-parallel.test

@@ -1,98 +0,0 @@
-#!/bin/bash
-#
-# A completed FTP transfer used to end as a 0-byte file when its ready backlog
-# slot was swapped to the on-disk table (#797): the FTP worker writes url_sav
-# itself and leaves no in-memory body, and the swap round-trip zeroed the slot's
-# size, so the link writer then created an empty file over the bytes.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-testdir=$(cd "$(dirname "$0")" && pwd)
-# shellcheck source=tests/testlib.sh
-. "${testdir}/testlib.sh"
-
-python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
-command -v httrack >/dev/null || {
-    echo "could not find httrack" >&2
-    exit 1
-}
-
-server=$(nativepath "${testdir}/ftp-server.py")
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_ftp.XXXXXX")
-serverpid=
-cleanup() {
-    stop_server "$serverpid"
-    rm -rf "$tmpdir"
-}
-trap 'set +e; cleanup' EXIT
-trap cleanup HUP INT QUIT PIPE TERM
-
-root="${tmpdir}/root"
-out="${tmpdir}/crawl"
-mkdir -p "$root" "$out"
-
-fail() {
-    echo "FAIL: $*" >&2
-    exit 1
-}
-ok() { echo "OK: $*"; }
-size_of() { wc -c <"$1" | tr -d '[:space:]'; }
-
-# Distinct bodies and lengths, so a mirror that mixed two of them up cannot
-# compare equal.
-names=""
-for i in 1 2 3 4 5 6 7 8 9 10 11 12; do
-    names="${names} f${i}"
-    "$python" -c 'import sys; sys.stdout.buffer.write(
-        ("BODY-%s " % sys.argv[1]).encode() + sys.argv[1].encode() * (400 * int(sys.argv[1])))' \
-        "$i" >"${root}/f${i}.bin"
-done
-
-serverlog="${tmpdir}/server.out"
-"$python" "$server" --root "$(nativepath "$root")" >"$serverlog" 2>&1 &
-serverpid=$!
-port=
-for _ in $(seq 1 300); do
-    line=$(grep -m1 '^PORT ' "$serverlog" 2>/dev/null) && port="${line#PORT }" && break
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "ftp server exited early: $(cat "$serverlog")" >&2
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover ftp server port: $(cat "$serverlog")" >&2
-    exit 1
-}
-host="127.0.0.1_${port}"
-
-urls=()
-for name in $names; do
-    urls+=("ftp://127.0.0.1:${port}/${name}.bin")
-done
-
-# -c8, never -c1: a slot is only swapped out while other transfers keep the
-# parser from picking it up. -Z because the swap only logs at debug level, and
-# the run below proves nothing unless it happened. Bounded like every crawl
-# pass: a wedge must fail the test, not hang the job until CI discards the log.
-run_with_timeout 300 httrack "${urls[@]}" -O "$out" --quiet -Z -c8 \
-    --disable-security-limits --robots=0 --timeout=20 --max-time=120 --retries=1 \
-    >"${tmpdir}/log" 2>&1
-
-swapped=$(grep -ac 'slots ready moved to background' "${out}/hts-log.txt" || true)
-test "${swapped:-0}" -gt 0 ||
-    fail "no ready slot was swapped to disk; this run never reached the bug"
-ok "the crawl swapped ready slots to the on-disk table ${swapped} times"
-
-for name in $names; do
-    test -f "${out}/${host}/${name}.bin" || fail "${name}.bin was not mirrored"
-    cmp -s "${out}/${host}/${name}.bin" "${root}/${name}.bin" ||
-        fail "${name}.bin has $(size_of "${out}/${host}/${name}.bin") of the" \
-            "$(size_of "${root}/${name}.bin") bytes served"
-done
-ok "every FTP transfer landed on disk intact"
-
-errors=$(grep -a 'Error:' "${out}/hts-log.txt" || true)
-test -z "$errors" || fail "the crawl reported errors: ${errors}"
-ok "no transfer was reported failed"

tests/114_local-update-304-leak.test

@@ -1,108 +0,0 @@
-#!/bin/bash
-#
-# Issue #782: a 304 during --update overwrote the whole response struct with the
-# cache entry, dropping the 8 KB header buffer it owned. Only a leak checker
-# sees it and the sanitized CI leg runs with detect_leaks off, so this one turns
-# it back on around a two-pass mini304 crawl; the fresh pass is the control.
-
-set -eu
-
-: "${top_srcdir:=..}"
-testdir=$(cd "$(dirname "$0")" && pwd)
-# shellcheck source=tests/testlib.sh
-. "${testdir}/testlib.sh"
-server=$(nativepath "${testdir}/local-server.py")
-root=$(nativepath "${testdir}/server-root")
-
-python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
-
-probe=$(ASAN_OPTIONS=help=1 httrack -O /dev/null --version 2>&1 || true)
-case "$probe" in
-*AddressSanitizer*) ;;
-*)
-    echo "not a sanitized build, skipping" >&2
-    exit 77
-    ;;
-esac
-
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_304leak.XXXXXX") || exit 1
-serverpid=
-cleanup() {
-    stop_server "$serverpid"
-    rm -rf "$tmpdir"
-}
-trap 'set +e; cleanup' EXIT
-trap cleanup HUP INT QUIT PIPE TERM
-
-serverlog="${tmpdir}/server.log"
-: >"$serverlog"
-"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
-serverpid=$!
-port=
-for _ in $(seq 1 50); do
-    line=$(head -n1 "$serverlog" 2>/dev/null)
-    if test "${line%% *}" == "PORT"; then
-        port="${line#PORT }"
-        break
-    fi
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "server exited early: $(cat "$serverlog")"
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover server port"
-    exit 1
-}
-
-out="${tmpdir}/crawl"
-mkdir "$out"
-url="http://127.0.0.1:${port}/mini304/index.html"
-
-# Last value wins in the sanitizer flag parser, so this overrides the job's
-# settings; the report file, not the exit status, is the verdict.
-asan_base="${ASAN_OPTIONS:+$ASAN_OPTIONS:}detect_leaks=1:abort_on_error=0:exitcode=0"
-
-# run_pass <label> <extra httrack args...>
-run_pass() {
-    local label="$1"
-    shift
-    rm -f "${tmpdir}/asan".*
-    printf '[%s] ..\t' "$label"
-    ASAN_OPTIONS="${asan_base}:log_path=${tmpdir}/asan" \
-        httrack -O "$out" --quiet --disable-security-limits --robots=0 \
-        --timeout=30 --retries=1 -c1 "$@" "$url" >"${tmpdir}/log_${label}" 2>&1 || true
-    report=$(cat "${tmpdir}/asan".* 2>/dev/null || true)
-    test -z "$report" || {
-        echo "FAIL: sanitizer report on the ${label} pass"
-        echo "$report"
-        exit 1
-    }
-    echo "OK (clean)"
-}
-
-run_pass fresh
-run_pass update --update
-# only --warc-file stashes the two header blocks, so the archive limb of the fix
-# runs on this pass alone
-run_pass warc --update --warc-file leak
-
-# Teeth: the update passes must have gone through the 304 path, or a run that
-# re-downloaded everything would report clean having tested nothing.
-printf '[update used the cache] ..\t'
-if grep -aqE "no files updated" "${out}/hts-log.txt"; then
-    echo "OK"
-else
-    echo "FAIL: the update pass did not report unchanged files"
-    exit 1
-fi
-
-# ... and the WARC pass must have written an archive, or it stashed nothing.
-printf '[warc pass archived] ..\t'
-if [ -n "$(find "$out" -name 'leak*.warc*' -size +0c 2>/dev/null)" ]; then
-    echo "OK"
-else
-    echo "FAIL: the --warc-file pass produced no archive"
-    exit 1
-fi

tests/119_local-proxytrack-ndx-fields2.test

@@ -1,155 +0,0 @@
-#!/bin/bash
-# Two bounds in the .ndx/.dat reader: a position field that did not parse left
-# "pos" holding the previous entry's offset, or the stack on the first entry,
-# and the base path and index filenames were copied with strncat calls bounded
-# by the source path instead of by their 1024-byte destinations.
-
-set -euo pipefail
-export LC_ALL=C
-
-testdir=$(cd "$(dirname "$0")" && pwd)
-# shellcheck source=tests/testlib.sh
-. "${testdir}/testlib.sh"
-
-dir=$(mktemp -d)
-cleanup() { rm -rf "$dir"; }
-trap 'set +e; cleanup' EXIT
-trap 'set +e; cleanup; exit 1' HUP INT TERM
-
-fail() {
-    echo "FAIL: $1" >&2
-    exit 1
-}
-
-enc() { printf '%d\n%s' "${#1}" "$1"; }
-
-# One version-1.4 .dat record in cache_rstr() framing, field by field:
-# $1 content-type, $2 status message, $3 body.
-record() {
-    enc 200
-    enc "${#3}"
-    enc "$2"
-    enc "$1"
-    enc ""
-    enc ""
-    enc ""
-    enc ""
-    enc ""
-    enc ""
-    enc ""
-    enc ""
-    enc HTS
-    enc "${#3}"
-    printf '%s' "$3"
-}
-
-# Two records in $1.dat, and in $1.ndx two entries with positions $2 and $3.
-# The parser walks the index a line at a time and skips one line per entry, so
-# each block leads with a filler line.
-build_cache() {
-    record audio/midi ZERO ZEROBODY >"$dir/r0"
-    record image/gif ALPHA ALPHABODY >"$dir/ra"
-    cat "$dir/r0" "$dir/ra" >"$1.dat"
-    {
-        enc CACHE-1.4
-        enc "Wed, 01 Jan 2025 00:00:00 GMT"
-        printf 'X\nexample.com/a.html\n\n%s\n' "$2"
-        printf 'X\nexample.com/b.html\n\n%s\n' "$3"
-    } >"$1.ndx"
-}
-
-# Content-type proxytrack wrote for URL $2 of the .arc $1, or "none".
-mime_of() {
-    local rec
-    rec=$(grep -a -m1 "^http://$2 " "$1") || {
-        echo none
-        return
-    }
-    awk '{ print $4 }' <<<"$rec"
-}
-
-# Offset of the second record: the position an entry must not inherit.
-record audio/midi ZERO ZEROBODY >"$dir/r0"
-posA=$(wc -c <"$dir/r0")
-
-convert() {
-    proxytrack --convert "$dir/out.arc" "$1" >"$dir/log" 2>&1 ||
-        fail "$2: proxytrack died on the cache"
-}
-
-# Control: both positions parse, so each URL resolves to its own record and a
-# broken probe cannot pass as a fixed reader.
-build_cache "$dir/ctl" 0 "$posA"
-convert "$dir/ctl.ndx" "control"
-test "$(mime_of "$dir/out.arc" example.com/a.html)" = audio/midi ||
-    fail "control: a.html did not resolve to the record at offset 0"
-test "$(mime_of "$dir/out.arc" example.com/b.html)" = image/gif ||
-    fail "control: b.html did not resolve to the record at offset $posA"
-
-# A field that does not parse must read as offset 0, not as the offset the
-# entry before it left behind.
-build_cache "$dir/carry" "$posA" Z
-convert "$dir/carry.ndx" "carry-over"
-test "$(mime_of "$dir/out.arc" example.com/b.html)" = audio/midi ||
-    fail "an unparseable position inherited the previous entry's offset"
-
-# Same on the first entry, where there is no previous value and the read came
-# off the stack.
-build_cache "$dir/first" Z "$posA"
-convert "$dir/first.ndx" "first entry"
-test "$(mime_of "$dir/out.arc" example.com/a.html)" = audio/midi ||
-    fail "an unparseable position on the first entry did not read as 0"
-
-# The .ndx name reaches the copy before anything opens it, so no file needed.
-long=$(printf '%3000s' '' | tr ' ' 'x')
-convert "$dir/$long.ndx" "over-long .ndx name"
-
-if is_windows; then
-    echo "deep paths need a POSIX filesystem; index name half only" >&2
-    exit 0
-fi
-
-comp=$(printf '%200s' '' | tr ' ' 'd')
-
-# A path just short of the 1024-byte destinations must still load: refusing
-# these is as wrong as overflowing them.
-fits="$dir/$comp/$comp/$comp"
-mkdir -p "$fits"
-build_cache "$fits/c" 0 "$posA"
-convert "$fits/c.ndx" "path that fits"
-test "$(mime_of "$dir/out.arc" example.com/b.html)" = image/gif ||
-    fail "a cache at a $((${#fits} + 6))-byte path was dropped"
-
-# Past them, the base path and both index filenames each overshoot.
-deep="$fits/$comp/$comp/$comp"
-mkdir -p "$deep" 2>/dev/null || {
-    echo "filesystem will not hold a ${#deep}-byte path; deep half skipped" >&2
-    exit 0
-}
-build_cache "$deep/c" 0 "$posA"
-convert "$deep/c.ndx" "path past the destinations"
-
-# The zip reader keeps its own copy of that base-path code, and only reaches it
-# once the archive opens, so this one needs a real zip at the deep path.
-python=$(find_python) || {
-    echo "python3 not found; zip half skipped" >&2
-    exit 0
-}
-"$python" - "$deep/c.zip" <<'EOF'
-import sys, zipfile
-
-zi = zipfile.ZipInfo("example.com/page.html")
-zi.compress_type = zipfile.ZIP_STORED
-zi.extra = (
-    "X-In-Cache: 1\r\nX-StatusCode: 200\r\nX-StatusMessage: OK\r\n"
-    "X-Size: 5\r\nContent-Type: text/html\r\n"
-    "Last-Modified: Wed, 01 Jan 2025 00:00:00 GMT\r\nX-Save: out/page.html\r\n"
-).encode()
-with zipfile.ZipFile(sys.argv[1], "w") as z:
-    z.writestr(zi, b"hello")
-EOF
-convert "$deep/c.zip" "zip at a path past the base buffer"
-test "$(mime_of "$dir/out.arc" example.com/page.html)" = text/html ||
-    fail "the zip entry was dropped instead of keeping an empty base path"
-
-echo "OK: an unparseable .ndx position reads as 0, and an over-long cache path is refused"

tests/Makefile.am

@@ -64,7 +64,6 @@ TESTS = \
 	01_engine-filterdual.test \
 	01_engine-ftp-line.test \
 	01_engine-ftp-userpass.test \
-	01_engine-gmtime.test \
 	01_engine-backswap.test \
 	01_engine-cacheindex.test \
 	01_engine-hashtable.test \
@@ -228,9 +227,9 @@ TESTS = \
 	103_teardown-status.test \
 	104_engine-warc-longurl.test \
 	105_suite-timeout.test \
-	110_local-ftp-parallel.test \
-	111_local-ftp-update-rest.test \
-	114_local-update-304-leak.test \
-	119_local-proxytrack-ndx-fields2.test
+	106_engine-repair-rename.test \
+	107_local-bak-collision.test \
+	108_engine-refetch-backup.test \
+	111_local-ftp-update-rest.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -345,6 +345,7 @@ if test "${#plants[@]}" -gt 0; then
         if test "${plants[$i]}" = "--plant-dir"; then
             mkdir -p "$path" || die "could not create $path"
         else
+            mkdir -p "$(dirname "$path")" || die "could not create ${path%/*}"
             echo "$plant_poison" >"$path" || die "could not write $path"
         fi
         result "OK"
@@ -529,9 +530,25 @@ if test -n "$wacz_validate"; then
 fi
 
 # No crawl, even a cancelled one, may leave engine temporaries: .delayed (#107,
-# #483), or the .z/.u content-coding temps (#557).
+# #483), the .z/.u content-coding temps (#557), or the hts-tmp directory those
+# and the re-fetch backup live in (#774). Only a test that planted something in
+# hts-tmp itself owns what is left there, so only that case skips the scan.
 info "checking for leftover engine temporaries"
-leftovers=$(find "$out" \( -name '*.delayed' -o -name '*.z' -o -name '*.u' \) 2>/dev/null | head -5)
+scan_tmpdir=1
+i=0
+while test "$i" -lt "${#plants[@]}"; do
+    case "${plants[$((i + 1))]}" in
+    */hts-tmp/*) scan_tmpdir=0 ;;
+    esac
+    i=$((i + 2))
+done
+if test "$scan_tmpdir" -eq 1; then
+    leftovers=$(find "$out" \( -name '*.delayed' -o -name '*.z' -o -name '*.u' \
+        -o -name 'hts-tmp' \) 2>/dev/null | head -5)
+else
+    leftovers=$(find "$out" \( -name '*.delayed' -o -name '*.z' -o -name '*.u' \) \
+        2>/dev/null | head -5)
+fi
 if test -z "$leftovers"; then result "OK"; else
     result "leftover: $leftovers"
     exit 1

tests/local-server.py

@@ -1553,6 +1553,40 @@ class Handler(SimpleHTTPRequestHandler):
     def route_mini304_page(self):
         self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
 
+    # --- /bakname/: #774 — a mirrored file named like a re-fetch backup ----
+    # a.bin gets a new body every pass, so the update re-fetches it and takes a
+    # backup. The sibling carries a validator, so it only revalidates and must
+    # still be there afterwards. It is served under hts-tmp/, the directory the
+    # backup lives in, so the crawl asks for the exact path the engine writes.
+    BAKNAME_SIB = b"BAKNAME-SIBLING\n" + b"\x41\x42\x43\x44" * 512
+
+    def route_bakname_index(self):
+        self.send_html(
+            '\t<a href="a.bin">a</a>\n'
+            '\t<a href="hts-tmp/a.bin.bak">bak</a>\n'
+        )
+
+    def route_bakname_main(self):
+        v = 1 if self.refetch_pass() == 1 else 2
+        self.send_raw(
+            b"BAKNAME-MAIN-V%d\n" % v + b"\x31\x32\x33\x34" * 512,
+            "application/octet-stream",
+        )
+
+    def route_bakname_sibling(self):
+        if self.headers.get("If-Modified-Since") or self.headers.get("If-None-Match"):
+            self.send_response(304)
+            self.send_header("Content-Length", "0")
+            self.end_headers()
+            return
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Last-Modified", BIG_LASTMOD)
+        self.send_header("Content-Length", str(len(self.BAKNAME_SIB)))
+        self.end_headers()
+        if self.command != "HEAD":
+            self.wfile.write(self.BAKNAME_SIB)
+
     # --- /errmask/: issue #176 — a page that 200'd on the first crawl but 403s
     # on the update fetch must keep its good copy, not be overwritten nor purged.
     ERRMASK_GOOD = b"KEEP" + b"." * 1020  # 1024 B distinctive non-HTML body
@@ -2134,6 +2168,9 @@ class Handler(SimpleHTTPRequestHandler):
         "/redir/index.html": route_redir_index,
         "/redir/go.php": route_redir_go,
         "/redir/target.html": route_redir_target,
+        "/bakname/index.html": route_bakname_index,
+        "/bakname/a.bin": route_bakname_main,
+        "/bakname/hts-tmp/a.bin.bak": route_bakname_sibling,
         "/mini304/index.html": route_mini304_index,
         "/mini304/page.html": route_mini304_page,
         "/errmask/index.html": route_errmask_index,

tests/renamefail.c

@@ -1,9 +1,10 @@
-/* Borrows Windows' rename() for 01_engine-renameover.test, since POSIX cannot
-   reach hts_rename_over()'s aside fallback: an existing target is refused with
+/* Borrows Windows' rename() for the tests POSIX cannot otherwise reach:
+   hts_rename_over()'s aside fallback needs an existing target refused with
    EEXIST, and RENAMEFAIL_MODE=locked reports EACCES instead, as the CRT does
-   for a source another process holds. RENAMEFAIL_ASIDE_FAILS=N refuses the
-   first N moves back out of the parked ".hts-old" name, which is the only way
-   to reach hts_rename_over()'s restore retry (#790). */
+   for a source another process holds. =repair and =repair-eexist narrow the
+   refusal to the cache-repair move (#786), the latter through the fallback so
+   the retry inside it is the failing one. RENAMEFAIL_ASIDE_FAILS=N refuses the
+   first N moves back out of a parked ".hts-old" name (#790). */
 
 #define _GNU_SOURCE
 #include <dlfcn.h>
@@ -23,20 +24,29 @@ SHIM_EXPORT int rename(const char *oldpath, const char *newpath) {
   static int aside_failures = 0;
   const char *const mode = getenv("RENAMEFAIL_MODE");
   const char *const aside_fails = getenv("RENAMEFAIL_ASIDE_FAILS");
-  const int locked = mode != NULL && strcmp(mode, "locked") == 0;
+  const char *const slash = strrchr(oldpath, '/');
+  const char *const base = slash != NULL ? slash + 1 : oldpath;
+  const int repaired = strcmp(base, "repair.zip") == 0;
   struct stat st;
 
-  if (locked) {
+  if (mode != NULL && (strcmp(mode, "locked") == 0 ||
+                       (repaired && strcmp(mode, "repair") == 0))) {
     errno = EACCES;
     return -1;
   }
-  if (aside_fails != NULL && strstr(oldpath, ".hts-old") != NULL &&
+  if (repaired && mode != NULL && strcmp(mode, "repair-eexist") == 0) {
+    errno = stat(newpath, &st) == 0 ? EEXIST : EACCES;
+    return -1;
+  }
+  if (aside_fails != NULL && strstr(base, ".hts-old") != NULL &&
       aside_failures < atoi(aside_fails)) {
     aside_failures++;
     errno = EACCES;
     return -1;
   }
-  if (aside_fails == NULL && stat(newpath, &st) == 0) {
+  /* Only the bare shim emulates Windows for every rename; a narrowed mode
+     leaves the rest of the run on plain POSIX semantics. */
+  if (mode == NULL && aside_fails == NULL && stat(newpath, &st) == 0) {
     errno = EEXIST;
     return -1;
   }

tests/testlib.sh

@@ -24,24 +24,6 @@ nativepath() {
     fi
 }
 
-# Run an engine self-test and require its "<label>: OK" line. No pipe into grep:
-# SIGPIPE would mask a failing exit status.
-expect_ok() {
-    local label="$1" out
-    shift
-    out=$("$@" 2>&1) || {
-        echo "FAIL: ${label} exited non-zero: ${out}"
-        exit 1
-    }
-    case "$out" in
-    *"${label}: OK"*) ;;
-    *)
-        echo "FAIL: ${out}"
-        exit 1
-        ;;
-    esac
-}
-
 is_windows() {
     case "$(uname -s)" in
     MINGW* | MSYS* | CYGWIN*) return 0 ;;