Cancelling a crawl mid delayed-type-check orphans .delayed placeholders (#496)

* Cancelling a crawl mid delayed-type-check orphaned .delayed placeholders

hts_wait_delayed() patches url_sav to the final name while a
direct-to-disk transfer may still be writing to the .delayed placeholder
path (bound when a back_wait tick lands between the slot's unlock and
the patch, e.g. url_savename's post-stop test branch pumping sockets).
From then on every cleanup keyed on IS_DELAYED_EXT(url_sav) is blind to
the on-disk name and a cancelled mirror leaves X.N.delayed partials
behind (#483).

Move the placeholder to the final name right before the patch
(back_delayed_rename, reopening a mid-transfer stream in append mode),
and drop it in the cache-miss re-add path where a cancel between
back_maydelete() and the re-add dropped its last reference.

The .delayed leftover audit in local-crawl.sh now runs for every crawl
(--skip-delayed-audit is gone), and 39_local-delayed-cancel exercises
the cancel window: 4/10 runs failing on master, 10/10 clean with the
fix; 0 orphans in 35 direct-harness cancelled crawls against ~1 in 6
before.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

* review: never unlink a pre-existing target on a failed placeholder move

Windows rename() fails when the target exists; unlinking newname there
would destroy a previous run's completed file during an update crawl.
Drop only what the slot owns: the placeholder if the rename failed, the
moved partial if the reopen did.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

* review: comment conciseness pass

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

---------

Signed-off-by: Xavier Roche <roche@httrack.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

debian/changelog

@@ -10,6 +10,15 @@ httrack (3.49.11-1) unstable; urgency=medium
 
  -- Xavier Roche <xavier@debian.org>  Sun, 05 Jul 2026 00:03:18 +0200
 
+httrack (3.49.10-2) unstable; urgency=medium
+
+  * Fix FTBFS: tests/28_local-pause failed instead of skipping when python3 is
+    absent (the local-server tests need python3, which the buildds lack). Add
+    patches/skip-local-pause-test-without-python3.patch to guard the test on
+    python3 up front, like its siblings, so it skips cleanly.
+
+ -- Xavier Roche <xavier@debian.org>  Sun, 28 Jun 2026 20:18:46 +0200
+
 httrack (3.49.10-1) unstable; urgency=medium
 
   * New upstream release: new download-pacing and URL-handling options plus a

src/htsback.c

@@ -572,9 +572,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
         && back[p].r.size != back[p].r.totalsize && !opt->tolerant) {
       if (back[p].status == STATUS_READY) {
         hts_log_print(opt, LOG_WARNING,
-                      "file not stored in cache due to bogus state (broken size, expected "
-                      LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
-                      back[p].r.size, back[p].url_adr, back[p].url_fil);
+                      "incomplete transfer (expected " LLintP
+                      " bytes, got " LLintP
+                      "): file not cached, will be retried on the next update"
+                      " (use -%%B to cache anyway): %s%s",
+                      back[p].r.totalsize, back[p].r.size, back[p].url_adr,
+                      back[p].url_fil);
       } else {
         hts_log_print(opt, LOG_INFO,
                       "incomplete file not yet stored in cache (expected "
@@ -879,11 +882,12 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                          back[p].url_fil, NULL);
           } else {
             /* Partial file, but marked as "ok" ? */
-            hts_log_print(opt, LOG_WARNING,
-                          "file not stored in cache due to bogus state (incomplete type with %s (%d), size "
-                          LLintP "): %s%s", back[p].r.msg, back[p].r.statuscode,
-                          (LLint) back[p].r.size, back[p].url_adr,
-                          back[p].url_fil);
+            hts_log_print(
+                opt, LOG_WARNING,
+                "file with unresolved type not cached (%s (%d), size " LLintP
+                "): %s%s",
+                back[p].r.msg, back[p].r.statuscode, (LLint) back[p].r.size,
+                back[p].url_adr, back[p].url_fil);
           }
         }
 
@@ -1358,9 +1362,39 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
   return 0;
 }
 
+/* Move a still-writing .delayed placeholder to its final name (#483). */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname) {
+  hts_boolean renamed;
+
+  if (!back->r.is_write || back->tmpfile != NULL ||
+      !IS_DELAYED_EXT(back->url_sav) || strcmp(back->url_sav, newname) == 0)
+    return HTS_TRUE; /* nothing bound to the placeholder name */
+  if (back->r.out != NULL) {
+    fclose(back->r.out);
+    back->r.out = NULL;
+  }
+  renamed = RENAME(back->url_sav, newname) == 0 ? HTS_TRUE : HTS_FALSE;
+  if (renamed && (back->status == STATUS_READY ||
+                  (back->r.out = FOPEN(newname, "ab")) != NULL)) {
+    filenote(&opt->state.strc, newname, NULL);
+    hts_log_print(opt, LOG_DEBUG, "moved placeholder %s to %s", back->url_sav,
+                  newname);
+    return HTS_TRUE;
+  }
+  /* partial lost: drop only what we own (Windows rename won't overwrite) */
+  hts_log_print(opt, LOG_WARNING | LOG_ERRNO, "unable to move %s to %s",
+                back->url_sav, newname);
+  back->r.statuscode = STATUSCODE_INVALID;
+  strcpybuff(back->r.msg, "Write error on disk");
+  back->r.is_write = 0;
+  (void) UNLINK(renamed ? newname : back->url_sav);
+  return HTS_FALSE;
+}
+
 // effacer entrée
 /* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
-static void back_delayed_discard(httrackp *opt, lien_back *back) {
+void back_delayed_discard(httrackp *opt, lien_back *back) {
   if (back->r.out != NULL) {
     fclose(back->r.out);
     back->r.out = NULL;

src/htsback.h

@@ -113,6 +113,12 @@ void back_set_locked(struct_back * sback, const int p);
 void back_set_unlocked(struct_back * sback, const int p);
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p);
+/* Discard back's on-disk .delayed placeholder and its refname. */
+void back_delayed_discard(httrackp *opt, lien_back *back);
+/* Move back's .delayed placeholder (and open stream) to newname;
+   HTS_FALSE = file lost, slot flagged in error. */
+hts_boolean back_delayed_rename(httrackp *opt, lien_back *back,
+                                const char *newname);
 void back_index_unlock(struct_back * sback, const int p);
 int back_clear_entry(lien_back * back);
 int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,

src/htscache.c

@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
 #include "htscore.h"
 #include "htsbasenet.h"
 #include "htsmd5.h"
+#include <limits.h>
 #include <time.h>
 
 #include "htszlib.h"
@@ -220,23 +221,38 @@ struct cache_back_zip_entry {
 	} \
 } while(0)
 
-/* A cache (new.zip) write failed: storage is gone (disk full / dropped share),
-   so the mirror is doomed too. Abort it via exit_xh, don't crash as assertf
-   did. */
+/* Consecutive entry write failures before the cache stream is declared dead. */
+#define CACHE_MAX_WRITE_FAILURES 8
+
+/* Cache write failed: a fatal errno or a failure streak aborts the mirror
+   (exit_xh); an isolated failure only drops the current entry. */
 static void cache_zip_write_failed(httrackp *opt, cache_back *cache,
-                                   const char *what, int zErr) {
-  if (!cache->zipWriteFailed) {
-    cache->zipWriteFailed = HTS_TRUE;
-    if (check_fatal_io_errno()) {
-      hts_log_print(opt, LOG_ERROR,
-                    "Mirror aborted: disk full or filesystem problems");
-    } else {
-      hts_log_print(opt, LOG_ERROR,
-                    "Mirror aborted: cache write failed (%s): %s", what,
-                    hts_get_zerror(zErr));
+                                   const char *what, int zErr,
+                                   hts_boolean entry_open, const char *url_adr,
+                                   const char *url_fil) {
+  const int fatal_errno = zErr == ZIP_ERRNO && check_fatal_io_errno();
+
+  cache->zipWriteFailures++;
+  if (fatal_errno || cache->zipWriteFailures >= CACHE_MAX_WRITE_FAILURES) {
+    if (!cache->zipWriteFailed) {
+      cache->zipWriteFailed = HTS_TRUE;
+      if (fatal_errno) {
+        hts_log_print(opt, LOG_ERROR,
+                      "Mirror aborted: disk full or filesystem problems");
+      } else {
+        hts_log_print(opt, LOG_ERROR,
+                      "Mirror aborted: cache write failed (%s): %s", what,
+                      hts_get_zerror(zErr));
+      }
     }
+    opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
+  } else {
+    if (entry_open)
+      zipCloseFileInZip((zipFile) cache->zipOutput); /* abandon, best-effort */
+    hts_log_print(opt, LOG_WARNING,
+                  "cache write failed (%s: %s), entry not cached: %s%s", what,
+                  hts_get_zerror(zErr), url_adr, url_fil);
   }
-  opt->state.exit_xh = -1; /* fatal: stop the mirror, exit non-zero */
 }
 
 /* Ajout d'un fichier en cache */
@@ -286,10 +302,19 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
   if (r->size < 0)              // error
     return;
 
-  // data in cache
-  if (dataincache) {
-    assertf(((int) r->size) == r->size);
-    //entryBodySize = (int) r->size;
+  // data in cache: the body must fit the 32-bit zip write API
+  if (dataincache && (LLint) (int) r->size != r->size) {
+    if (r->is_write && url_save != NULL && strnotempty(url_save)) {
+      hts_log_print(opt, LOG_WARNING,
+                    "file too large for the cache, storing headers only: %s%s",
+                    url_adr, url_fil);
+      dataincache = 0;
+    } else {
+      hts_log_print(opt, LOG_WARNING,
+                    "entry too large for the cache, not cached: %s%s", url_adr,
+                    url_fil);
+      return;
+    }
   }
 
   /* Fields */
@@ -369,7 +394,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
                                    */
                                   headers, (uInt) strlen(headers), NULL, 0, NULL,       /* comment */
                                   Z_DEFLATED, Z_DEFAULT_COMPRESSION)) != Z_OK) {
-    cache_zip_write_failed(opt, cache, "opening a cache entry", zErr);
+    cache_zip_write_failed(opt, cache, "opening a cache entry", zErr, HTS_FALSE,
+                           url_adr, url_fil);
     return;
   }
 
@@ -380,7 +406,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
         if ((zErr =
              zipWriteInFileInZip((zipFile) cache->zipOutput, r->adr,
                                  (int) r->size)) != Z_OK) {
-          cache_zip_write_failed(opt, cache, "writing to the cache", zErr);
+          cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
+                                 HTS_TRUE, url_adr, url_fil);
           return;
         }
       }
@@ -402,8 +429,8 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
               if ((zErr =
                    zipWriteInFileInZip((zipFile) cache->zipOutput, buff,
                                        (int) nl)) != Z_OK) {
-                cache_zip_write_failed(opt, cache, "writing to the cache",
-                                       zErr);
+                cache_zip_write_failed(opt, cache, "writing to the cache", zErr,
+                                       HTS_TRUE, url_adr, url_fil);
                 fclose(fp);
                 return;
               }
@@ -419,15 +446,19 @@ void cache_add(httrackp * opt, cache_back * cache, const htsblk * r,
 
   /* Close */
   if ((zErr = zipCloseFileInZip((zipFile) cache->zipOutput)) != Z_OK) {
-    cache_zip_write_failed(opt, cache, "closing a cache entry", zErr);
+    cache_zip_write_failed(opt, cache, "closing a cache entry", zErr, HTS_FALSE,
+                           url_adr, url_fil);
     return;
   }
 
   /* Flush */
   if ((zErr = zipFlush((zipFile) cache->zipOutput)) != 0) {
-    cache_zip_write_failed(opt, cache, "flushing the cache", zErr);
+    cache_zip_write_failed(opt, cache, "flushing the cache", zErr, HTS_FALSE,
+                           url_adr, url_fil);
     return;
   }
+
+  cache->zipWriteFailures = 0; /* entry stored: reset the failure streak */
 }
 
 #else
@@ -768,6 +799,15 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
             strlcpybuff(return_save, previous_save, HTS_URLMAXSIZE * 2);
           }
 
+          /* A negative X-Size is corrupt; so is one >= INT_MAX when the data
+             is in the zip (the write path asserts int-sized). Headers-only
+             entries legitimately exceed INT_MAX (>2GB body on disk): keep
+             them, or every update would re-fetch the file. */
+          if (r.size < 0 || (dataincache && r.size >= INT_MAX)) {
+            r.statuscode = STATUSCODE_INVALID;
+            strcpybuff(r.msg, "Cache Read Error : Bad Size");
+          }
+
           /* Complete fields */
           r.totalsize = r.size;
           r.adr = NULL;
@@ -794,7 +834,8 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
             }                   // otherwise, the ZIP file is supposed to be consistent with data.
           }
           /* Read data ? */
-          else {                /* ne pas lire uniquement header */
+          else if (r.statuscode !=
+                   STATUSCODE_INVALID) { /* ne pas lire uniquement header */
             int ok = 0;
 
 #if HTS_DIRECTDISK
@@ -958,7 +999,10 @@ static htsblk cache_readex_new(httrackp * opt, cache_back * cache,
                     strcpybuff(r.msg,
                                "Previous cache file not found (empty filename)");
                   }
-                } else {        /* Read in memory from disk */
+                } else if (r.size >= INT_MAX) { /* too big to read in memory */
+                  r.statuscode = STATUSCODE_INVALID;
+                  strcpybuff(r.msg, "Cache Read Error : Bad Size");
+                } else { /* Read in memory from disk */
                   FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), previous_save), "rb");
 
                   if (fp != NULL) {
@@ -1420,6 +1464,86 @@ static int hts_rename(httrackp * opt, const char *a, const char *b) {
   return rename(a, b);
 }
 
+/* Pathname of a file inside the mirror dir (rotating concat buffer). */
+static char *reconcile_path(httrackp *opt, const char *name) {
+  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                 StringBuff(opt->path_log), name);
+}
+
+/* Interrupted-run heuristic: prefer the old generation when the new cache
+   stalled below NEW_TINY while the old one grew past OLD_SOLID (historical
+   arbitrary thresholds). */
+#define CACHE_RECONCILE_NEW_TINY 32768
+#define CACHE_RECONCILE_OLD_SOLID 65536
+
+/* Replace the new-generation file by the old one, when the old one exists. */
+static void reconcile_promote(httrackp *opt, const char *oldname,
+                              const char *newname) {
+  if (fexist(reconcile_path(opt, oldname))) {
+    remove(reconcile_path(opt, newname));
+    rename(reconcile_path(opt, oldname), reconcile_path(opt, newname));
+  }
+}
+
+void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode) {
+  switch (mode) {
+  case CACHE_RECONCILE_PROMOTE:
+    /* Previous run rotated new.* to old.* then died before writing: promote
+       the old generation back, whichever format it uses. */
+    if (!fexist(reconcile_path(opt, "hts-cache/new.zip")))
+      reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if ((!fexist(reconcile_path(opt, "hts-cache/new.dat")) ||
+         !fexist(reconcile_path(opt, "hts-cache/new.ndx"))) &&
+        fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    break;
+  case CACHE_RECONCILE_INTERRUPTED:
+    /* Aborted run: keep the larger generation when the new cache is
+       suspiciously small next to the old one. The new file must exist: fsize()
+       is -1 for a missing file, which would spuriously pass the "< TINY" test
+       and overwrite a solid old generation that PROMOTE/ROLLBACK should keep.
+     */
+    if (!opt->cache || !fexist(reconcile_path(opt, "hts-in_progress.lock")))
+      break;
+    if (fexist(reconcile_path(opt, "hts-cache/new.zip")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.zip")) &&
+        fsize(reconcile_path(opt, "hts-cache/new.zip")) <
+            CACHE_RECONCILE_NEW_TINY &&
+        fsize(reconcile_path(opt, "hts-cache/old.zip")) >
+            CACHE_RECONCILE_OLD_SOLID &&
+        fsize(reconcile_path(opt, "hts-cache/old.zip")) >
+            fsize(reconcile_path(opt, "hts-cache/new.zip")))
+      reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if (fexist(reconcile_path(opt, "hts-cache/new.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx")) &&
+        fsize(reconcile_path(opt, "hts-cache/new.dat")) <
+            CACHE_RECONCILE_NEW_TINY &&
+        fsize(reconcile_path(opt, "hts-cache/old.dat")) >
+            CACHE_RECONCILE_OLD_SOLID &&
+        fsize(reconcile_path(opt, "hts-cache/old.dat")) >
+            fsize(reconcile_path(opt, "hts-cache/new.dat"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    break;
+  case CACHE_RECONCILE_ROLLBACK:
+    /* Nothing transferred: restore the previous generation and sidecars. */
+    reconcile_promote(opt, "hts-cache/old.zip", "hts-cache/new.zip");
+    if (fexist(reconcile_path(opt, "hts-cache/old.dat")) &&
+        fexist(reconcile_path(opt, "hts-cache/old.ndx"))) {
+      reconcile_promote(opt, "hts-cache/old.dat", "hts-cache/new.dat");
+      reconcile_promote(opt, "hts-cache/old.ndx", "hts-cache/new.ndx");
+    }
+    reconcile_promote(opt, "hts-cache/old.lst", "hts-cache/new.lst");
+    reconcile_promote(opt, "hts-cache/old.txt", "hts-cache/new.txt");
+    break;
+  }
+}
+
 // renvoyer uniquement en tête, ou NULL si erreur
 // return NULL upon error, and set -1 to r.statuscode
 htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,

src/htscache.h

@@ -78,6 +78,17 @@ htsblk *cache_header(httrackp * opt, cache_back * cache, const char *adr,
                      const char *fil, htsblk * r);
 void cache_init(cache_back * cache, httrackp * opt);
 
+/* Which hts-cache/ generation (new.* vs old.*) is authoritative. */
+typedef enum {
+  CACHE_RECONCILE_PROMOTE,     /* no new cache: promote the old generation */
+  CACHE_RECONCILE_INTERRUPTED, /* aborted run: keep the larger generation */
+  CACHE_RECONCILE_ROLLBACK     /* nothing transferred: restore the old one */
+} hts_cache_reconcile_mode;
+
+/* Reconcile the on-disk cache generations according to mode; a no-op when
+   the involved files are absent. */
+void hts_cache_reconcile(httrackp *opt, hts_cache_reconcile_mode mode);
+
 int cache_writedata(FILE * cache_ndx, FILE * cache_dat, const char *str1,
                     const char *str2, char *outbuff, int len);
 int cache_readdata(cache_back * cache, const char *str1, const char *str2,

src/htscache_selftest.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 #include "htszlib.h"
 
 #include <errno.h>
+#include <limits.h>
 #include <stdio.h>
 #include <string.h>
 
@@ -321,6 +322,7 @@ typedef struct {
   size_t budget;  /**< bytes allowed through before writes start failing */
   int fail_errno; /**< errno set on the failing write (ENOSPC, EIO, ...) */
   int writes;     /**< zwrite call count, to detect re-entry into the stream */
+  int fail_once;  /**< recover (unlimited budget) after the first failure */
 } writefail_inject;
 
 /* zwrite that copies until the budget runs out, then fails with inj->fail_errno
@@ -335,6 +337,8 @@ static uLong selftest_failing_zwrite(voidpf opaque, voidpf stream,
     inj->budget -= (size_t) size;
     return (uLong) fwrite(buf, 1, (size_t) size, (FILE *) stream);
   }
+  if (inj->fail_once)
+    inj->budget = (size_t) -1; /* the backend recovers after this failure */
   errno = inj->fail_errno;
   return 0; /* short write -> the minizip op returns an error */
 }
@@ -373,9 +377,50 @@ static void writefail_store(httrackp *opt, cache_back *cache, const char *fil,
   freet(bodycopy);
 }
 
-/* #174/#219: a failing cache write used to crash via assertf(); it must instead
-   stop the mirror (exit_xh = -1) without crashing. Assert that, plus the cache
-   is flagged and a sibling write doesn't re-enter the broken stream. */
+/* Store an entry claiming a >2GB body; the degrade path never reads data. */
+static void writefail_store_oversized(httrackp *opt, cache_back *cache,
+                                      const char *fil, int is_write) {
+  htsblk r;
+  char locbuf[4];
+
+  hts_init_htsblk(&r);
+  r.statuscode = 200;
+  r.size = (LLint) INT_MAX + 1;
+  strcpybuff(r.msg, "OK");
+  strcpybuff(r.contenttype, "application/octet-stream");
+  locbuf[0] = '\0';
+  r.location = locbuf;
+  r.is_write = (short int) is_write;
+  cache_add(opt, cache, &r, "example.com", fil, "example.com/big.bin", 1, NULL);
+}
+
+/* Read back `entryname`: extra field (cached headers) and body. Returns the
+   body length, or -1 if the entry is absent or unreadable. */
+static int writefail_read_entry(const char *path, const char *entryname,
+                                char *extra, size_t extralen, char *body,
+                                size_t bodylen) {
+  unzFile z = unzOpen(path);
+  int n = -1;
+
+  if (z == NULL)
+    return -1;
+  if (unzLocateFile(z, entryname, 1) == UNZ_OK &&
+      unzOpenCurrentFile(z) == UNZ_OK) {
+    const int elen = unzGetLocalExtrafield(z, extra, (unsigned) (extralen - 1));
+
+    if (elen >= 0) {
+      extra[elen] = '\0';
+      n = unzReadCurrentFile(z, body, (unsigned) bodylen);
+    }
+    unzCloseCurrentFile(z);
+  }
+  unzClose(z);
+  return n;
+}
+
+/* Cache write-failure policy (#174/#219): fatal errno or a failure streak
+   stops the mirror (exit_xh=-1, no crash); isolated/oversized drops the entry.
+ */
 int cache_write_failure_selftest(httrackp *opt, const char *dir) {
   int fail = 0;
   char path[HTS_URLMAXSIZE];
@@ -388,9 +433,8 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
   gen_body(body, body_len, 1 /* incompressible */);
   fconcat(path, sizeof(path), dir, "/wfail.zip");
 
-  /* phase 0: fail on the body write, fatal errno (ENOSPC, the disk-full
-     branch). phase 1: fail on the open, non-fatal errno (EIO, dropped-share
-     branch). Both must abort the mirror. */
+  /* phase 0: fatal errno (ENOSPC) aborts at once; phase 1: persistent EIO
+     drops entries until the streak caps out, then aborts. */
   for (phase = 0; phase < 2; phase++) {
     cache_back cache;
     writefail_inject inj;
@@ -399,6 +443,7 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     inj.budget = (phase == 0) ? 4096 : 0;
     inj.fail_errno = (phase == 0) ? ENOSPC : EIO;
     inj.writes = 0;
+    inj.fail_once = 0;
     memset(&cache, 0, sizeof(cache));
     cache.type = 1;
     cache.log = stderr;
@@ -412,7 +457,25 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     }
 
     opt->state.exit_xh = 0; /* clear; the failing write must set it to -1 */
-    writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    if (phase == 0) {
+      writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    } else {
+      /* the abort must land exactly on the 8th consecutive failure */
+      int i;
+
+      for (i = 0; i < 7; i++) {
+        char fil[32];
+
+        snprintf(fil, sizeof(fil), "/b%d.bin", i);
+        writefail_store(opt, &cache, fil, body, 16);
+      }
+      if (cache.zipWriteFailed) {
+        fprintf(stderr, "cache-writefail: phase 1: aborted before the "
+                        "8th consecutive failure\n");
+        fail++;
+      }
+      writefail_store(opt, &cache, "/b7.bin", body, 16);
+    }
     if (!cache.zipWriteFailed) {
       fprintf(stderr, "cache-writefail: phase %d: write error not caught\n",
               phase);
@@ -443,6 +506,136 @@ int cache_write_failure_selftest(httrackp *opt, const char *dir) {
     }
   }
 
+  /* failures with successes in between reset the streak: never aborts */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    int i;
+
+    inj.budget = (size_t) -1;
+    inj.fail_errno = EIO;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    for (i = 0; i < 10; i++) {
+      char fil[32];
+
+      inj.budget = 0; /* this store fails */
+      snprintf(fil, sizeof(fil), "/s%d.bin", i);
+      writefail_store(opt, &cache, fil, body, 16);
+      inj.budget = (size_t) -1; /* this one succeeds and resets the streak */
+      snprintf(fil, sizeof(fil), "/ok%d.bin", i);
+      writefail_store(opt, &cache, fil, body, 16);
+    }
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: scattered: non-consecutive failures aborted "
+              "the mirror (flagged=%d, exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+  }
+
+  /* isolated failure: only that entry drops; a later sibling round-trips */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    char extra[8192];
+    char rbody[64];
+    int n;
+
+    inj.budget = 4096;
+    inj.fail_errno = EIO;
+    inj.writes = 0;
+    inj.fail_once = 1;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    writefail_store(opt, &cache, "/blob.bin", body, body_len);
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: skip: isolated failure aborted the mirror "
+              "(flagged=%d, exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    writefail_store(opt, &cache, "/blob2.bin", body, 16);
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+    n = writefail_read_entry(path, "http://example.com/blob2.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody));
+    if (n != 16 || memcmp(rbody, body, 16) != 0) {
+      fprintf(stderr,
+              "cache-writefail: skip: sibling entry lost after a skipped "
+              "entry (%d)\n",
+              n);
+      fail++;
+    }
+  }
+
+  /* >2GB bodies: in-memory drops the entry, on-disk degrades to headers-only */
+  {
+    cache_back cache;
+    writefail_inject inj;
+    char extra[8192];
+    char rbody[64];
+    int n;
+
+    inj.budget = (size_t) -1; /* no injected failure */
+    inj.fail_errno = 0;
+    inj.writes = 0;
+    inj.fail_once = 0;
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = stderr;
+    cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.zipOutput = selftest_open_failing_zip(path, &inj);
+    opt->state.exit_xh = 0;
+
+    writefail_store_oversized(opt, &cache, "/bigmem.bin", 0 /* in-memory */);
+    writefail_store_oversized(opt, &cache, "/bigdisk.bin", 1 /* on-disk */);
+    zipClose(cache.zipOutput, NULL);
+    cache.zipOutput = NULL;
+
+    if (cache.zipWriteFailed || opt->state.exit_xh != 0) {
+      fprintf(stderr,
+              "cache-writefail: oversize: mirror aborted (flagged=%d, "
+              "exit_xh=%d)\n",
+              (int) cache.zipWriteFailed, opt->state.exit_xh);
+      fail++;
+    }
+    if (writefail_read_entry(path, "http://example.com/bigmem.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody)) >= 0) {
+      fprintf(stderr,
+              "cache-writefail: oversize: in-memory entry was stored\n");
+      fail++;
+    }
+    n = writefail_read_entry(path, "http://example.com/bigdisk.bin", extra,
+                             sizeof(extra), rbody, sizeof(rbody));
+    if (n != 0 || strstr(extra, "X-In-Cache: 0") == NULL) {
+      fprintf(stderr,
+              "cache-writefail: oversize: on-disk entry not stored "
+              "headers-only (%d)\n",
+              n);
+      fail++;
+    }
+  }
+
   freet(body);
   return fail;
 }
@@ -716,3 +909,494 @@ int cache_golden_selftest(httrackp *opt, const char *dir, int regen) {
 
   return failures;
 }
+
+/* --- hts_cache_reconcile() policies -------------------------------------- */
+
+/* All reconcile inputs/outputs, wiped between cases. */
+static const char *const reconcile_files[] = {
+    "hts-cache/new.zip", "hts-cache/old.zip",   "hts-cache/new.dat",
+    "hts-cache/old.dat", "hts-cache/new.ndx",   "hts-cache/old.ndx",
+    "hts-cache/new.lst", "hts-cache/old.lst",   "hts-cache/new.txt",
+    "hts-cache/old.txt", "hts-in_progress.lock"};
+
+static char *reconcile_st_path(httrackp *opt, const char *name) {
+  return fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                 StringBuff(opt->path_log), name);
+}
+
+static void reconcile_wipe(httrackp *opt) {
+  size_t i;
+
+  for (i = 0; i < sizeof(reconcile_files) / sizeof(reconcile_files[0]); i++)
+    remove(reconcile_st_path(opt, reconcile_files[i]));
+}
+
+/* Create a filler file of exactly `size` bytes. */
+static void reconcile_put(httrackp *opt, const char *name, size_t size) {
+  FILE *const fp = fopen(reconcile_st_path(opt, name), "wb");
+  static const char filler[1024] = {'x'};
+
+  assertf(fp != NULL);
+  while (size > 0) {
+    const size_t n = size > sizeof(filler) ? sizeof(filler) : size;
+
+    assertf(fwrite(filler, 1, n, fp) == n);
+    size -= n;
+  }
+  fclose(fp);
+}
+
+/* Expect `name` to weigh `size` bytes, or be absent when size == -1. */
+static int reconcile_expect(httrackp *opt, const char *name, off_t size,
+                            const char *what) {
+  const off_t got = fsize(reconcile_st_path(opt, name));
+
+  if (got != size) {
+    fprintf(stderr, "cache-reconcile: %s: %s is %d bytes, expected %d\n", what,
+            name, (int) got, (int) size);
+    return 1;
+  }
+  return 0;
+}
+
+int cache_reconcile_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+
+  /* around the interrupted-run thresholds (new < 32768, old > 65536) */
+  static const off_t TINY = 1024, MID = 40000, SOLID = 131072;
+
+  golden_setup(opt, dir);
+#ifdef _WIN32
+  mkdir(reconcile_st_path(opt, "hts-cache"));
+#else
+  mkdir(reconcile_st_path(opt, "hts-cache"), HTS_PROTECT_FOLDER);
+#endif
+
+  /* PROMOTE: a zip old generation replaces a missing new one */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "promote-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "promote-zip");
+
+  /* PROMOTE: an existing new.zip is left alone */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "promote-zip-noop");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "promote-zip-noop");
+
+  /* PROMOTE: a pure-legacy old generation is promoted too (was dead when no
+     zip cache existed) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat");
+  failures += reconcile_expect(opt, "hts-cache/old.dat", -1, "promote-dat");
+
+  /* PROMOTE: a half-written legacy new pair is replaced by the old pair */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "promote-dat-partial");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", TINY, "promote-dat-partial");
+
+  /* INTERRUPTED: no lock file, no action */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-nolock");
+
+  /* INTERRUPTED: an absent new.zip must NOT promote old.zip (fsize(-1) would
+     spuriously pass "< TINY"); leave the solid old generation for ROLLBACK */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", -1, "interrupted-nonew");
+  failures +=
+      reconcile_expect(opt, "hts-cache/old.zip", SOLID, "interrupted-nonew");
+
+  /* INTERRUPTED: stalled tiny new.zip loses to a solid old.zip (was dead for
+     zip caches: the arm was gated on a legacy new.dat) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", SOLID, "interrupted-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "interrupted-zip");
+
+  /* INTERRUPTED: old below the confidence threshold, keep new */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", TINY, "interrupted-smallold");
+
+  /* INTERRUPTED: new big enough to trust, keep it */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.zip", MID);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.zip", MID, "interrupted-bignew");
+
+  /* INTERRUPTED: the legacy pair follows the same size rule (was dead code) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-in_progress.lock", 0);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.dat", SOLID, "interrupted-dat");
+  failures +=
+      reconcile_expect(opt, "hts-cache/new.ndx", MID, "interrupted-dat");
+
+  /* ROLLBACK: the old zip generation is restored (a zip cache used to lose
+     its only good generation here) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  reconcile_put(opt, "hts-cache/old.zip", SOLID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", SOLID, "rollback-zip");
+  failures += reconcile_expect(opt, "hts-cache/old.zip", -1, "rollback-zip");
+
+  /* ROLLBACK: sidecars are restored regardless of format */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.lst", TINY);
+  reconcile_put(opt, "hts-cache/old.lst", MID);
+  reconcile_put(opt, "hts-cache/old.txt", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-lst");
+  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-txt");
+
+  /* ROLLBACK: full legacy generation incl. sidecars (historical behavior) */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.dat", TINY);
+  reconcile_put(opt, "hts-cache/new.ndx", TINY);
+  reconcile_put(opt, "hts-cache/old.dat", SOLID);
+  reconcile_put(opt, "hts-cache/old.ndx", MID);
+  reconcile_put(opt, "hts-cache/old.lst", MID);
+  reconcile_put(opt, "hts-cache/old.txt", MID);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.dat", SOLID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.ndx", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.lst", MID, "rollback-dat");
+  failures += reconcile_expect(opt, "hts-cache/new.txt", MID, "rollback-dat");
+
+  /* ROLLBACK: nothing to restore, the new generation stays */
+  reconcile_wipe(opt);
+  reconcile_put(opt, "hts-cache/new.zip", TINY);
+  hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
+  failures += reconcile_expect(opt, "hts-cache/new.zip", TINY, "rollback-noop");
+
+  reconcile_wipe(opt);
+  return failures;
+}
+
+/* --- read-side corruption injection --------------------------------------- */
+
+/* canary read back intact after each corruption; victim gets the byte surgery
+ */
+#define CORRUPT_ADR "corrupt.example.com"
+static char corrupt_body_a[33 + 1];
+static char corrupt_body_b[44 + 1];
+
+/* Write a fresh two-entry cache: /canary.html then /victim.html. */
+static void corrupt_build(httrackp *opt) {
+  cache_back cache;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_b,
+              strlen(corrupt_body_b));
+  selftest_close(&cache);
+}
+
+/* Like corrupt_build, but the victim carries a 20-char Etag whose header line
+   is later overwritten with a forged oversized X-Size (same byte length). */
+static void corrupt_build_etag(httrackp *opt) {
+  cache_back cache;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  memset(corrupt_body_b, 'b', sizeof(corrupt_body_b) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  store_entry(opt, &cache, CORRUPT_ADR, "/victim.html", "victim.html", 200,
+              "OK", "text/html", "utf-8", "", "AAAAAAAAAAAAAAAAAAAA", "", "",
+              corrupt_body_b, strlen(corrupt_body_b));
+  selftest_close(&cache);
+}
+
+/* Like corrupt_build_etag, but the victim is headers-only (X-In-Cache: 0,
+   body on disk): the shape every non-html file is stored with. */
+static void corrupt_build_disk(httrackp *opt) {
+  cache_back cache;
+  htsblk w;
+  char locw[4];
+  char BIGSTK save[HTS_URLMAXSIZE * 2];
+  char BIGSTK catbuff[HTS_URLMAXSIZE * 2];
+  char *path;
+  FILE *fp;
+
+  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
+  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
+  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
+  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
+          CORRUPT_ADR "/victim.bin");
+  selftest_open_for_write(&cache, opt);
+  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
+              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
+              strlen(corrupt_body_a));
+  hts_init_htsblk(&w);
+  w.statuscode = 200;
+  w.size = (LLint) sizeof(corrupt_body_b) - 1;
+  strcpybuff(w.msg, "OK");
+  strcpybuff(w.contenttype, "application/octet-stream");
+  strcpybuff(w.etag, "AAAAAAAAAAAAAAAAAAAA");
+  locw[0] = '\0';
+  w.location = locw;
+  w.is_write = 0;
+  cache_add(opt, &cache, &w, CORRUPT_ADR, "/victim.bin", save,
+            0 /* all_in_cache */, NULL);
+  selftest_close(&cache);
+  /* the reader only checks this file exists; it never reads it here */
+  path = fconv(catbuff, sizeof(catbuff), save);
+  (void) structcheck(path);
+  fp = FOPEN(path, "wb");
+  assertf(fp != NULL);
+  fclose(fp);
+}
+
+/* Patch the nth of total occurrences of pat (same-length rep) in new.zip. */
+static void corrupt_patch(httrackp *opt, const char *pat, size_t patlen,
+                          const char *rep, size_t nth, size_t total) {
+  LLint fsz = 0;
+  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
+  const size_t n = (size_t) fsz;
+  size_t k, hits = 0, at = 0;
+  FILE *fp;
+
+  assertf(data != NULL);
+  for (k = 0; k + patlen <= n; k++) {
+    if (memcmp(data + k, pat, patlen) == 0) {
+      hits++;
+      if (hits == nth)
+        at = k;
+    }
+  }
+  assertf(hits == total);
+  memcpy(data + at, rep, patlen);
+  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, n, fp) == n);
+  fclose(fp);
+  freet(data);
+}
+
+/* Garbage the first bytes of the victim's deflated data (2nd local header). */
+static void corrupt_victim_body(httrackp *opt) {
+  LLint fsz = 0;
+  char *data = readfile2(reconcile_st_path(opt, "hts-cache/new.zip"), &fsz);
+  const size_t n = (size_t) fsz;
+  size_t k, hits = 0, off = 0;
+  FILE *fp;
+
+  assertf(data != NULL);
+  for (k = 0; k + 4 <= n; k++) {
+    if (memcmp(data + k, "PK\x03\x04", 4) == 0 && ++hits == 2) {
+      const size_t namelen =
+          (unsigned char) data[k + 26] | ((unsigned char) data[k + 27] << 8);
+      const size_t extralen =
+          (unsigned char) data[k + 28] | ((unsigned char) data[k + 29] << 8);
+
+      off = k + 30 + namelen + extralen;
+    }
+  }
+  assertf(hits == 2);
+  assertf(off != 0 && off + 4 <= n);
+  memset(data + off, 0xFF, 4);
+  fp = fopen(reconcile_st_path(opt, "hts-cache/new.zip"), "wb");
+  assertf(fp != NULL);
+  assertf(fwrite(data, 1, n, fp) == n);
+  fclose(fp);
+  freet(data);
+}
+
+/* Read the corrupt /victim.html and, in the SAME read session, the intact
+   /canary.html: the victim must be rejected (wantmsg pins which path) and the
+   canary must still decode byte-exact, proving one bad entry never taints a
+   sibling read. */
+static int corrupt_expect_victim_fil(httrackp *opt, const char *fil,
+                                     const char *wantmsg, const char *what) {
+  cache_back cache;
+  htsblk v, c;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  char BIGSTK lc[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = lc[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, fil, "", lv, NULL, 1);
+  if (v.statuscode != STATUSCODE_INVALID) {
+    fprintf(stderr, "%s: %s: victim: statuscode is %d, expected %d\n",
+            selftest_tag, what, v.statuscode, STATUSCODE_INVALID);
+    fail++;
+  }
+  if (wantmsg != NULL && strcmp(v.msg, wantmsg) != 0) {
+    fprintf(stderr, "%s: %s: victim: msg is '%s', expected '%s'\n",
+            selftest_tag, what, v.msg, wantmsg);
+    fail++;
+  }
+  c = cache_readex(opt, &cache, CORRUPT_ADR, "/canary.html", "", lc, NULL, 1);
+  if (c.statuscode != 200 || c.adr == NULL ||
+      c.size != (LLint) strlen(corrupt_body_a) ||
+      memcmp(c.adr, corrupt_body_a, strlen(corrupt_body_a)) != 0) {
+    fprintf(stderr, "%s: %s: canary tainted (status %d)\n", selftest_tag, what,
+            c.statuscode);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  if (c.adr != NULL)
+    freet(c.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
+static int corrupt_expect_victim(httrackp *opt, const char *wantmsg,
+                                 const char *what) {
+  return corrupt_expect_victim_fil(opt, "/victim.html", wantmsg, what);
+}
+
+/* Headers-only probe of the disk victim: must parse OK with the size kept. */
+static int corrupt_expect_disk_header(httrackp *opt, LLint wantsize,
+                                      const char *what) {
+  cache_back cache;
+  htsblk v;
+  char BIGSTK lv[HTS_URLMAXSIZE * 2];
+  int fail = 0;
+
+  selftest_open_for_read(&cache, opt);
+  lv[0] = '\0';
+  v = cache_readex(opt, &cache, CORRUPT_ADR, "/victim.bin", NULL, lv, NULL, 1);
+  if (v.statuscode != 200 || v.size != wantsize) {
+    fprintf(stderr,
+            "%s: %s: statuscode %d size " LLintP ", expected 200/" LLintP "\n",
+            selftest_tag, what, v.statuscode, (LLint) v.size, wantsize);
+    fail++;
+  }
+  if (v.adr != NULL)
+    freet(v.adr);
+  selftest_close(&cache);
+  return fail;
+}
+
+/* One zip corruption case: build, patch, then check victim+canary in-session.
+ */
+static int corrupt_case_zip(httrackp *opt, const char *pat, const char *rep,
+                            size_t nth, size_t total, const char *wantmsg,
+                            const char *what) {
+  corrupt_build(opt);
+  corrupt_patch(opt, pat, strlen(pat), rep, nth, total);
+  return corrupt_expect_victim(opt, wantmsg, what);
+}
+
+int cache_corruption_selftest(httrackp *opt, const char *dir) {
+  int failures = 0;
+
+  selftest_tag = "cache-corrupt";
+  golden_setup(opt, dir);
+
+  failures +=
+      corrupt_case_zip(opt, "X-Size: 44", "X-Size: 99", 1, 1,
+                       "Cache Read Error : Read Data", "oversized X-Size");
+  failures +=
+      corrupt_case_zip(opt, "X-Size: 44", "X-Size: -4", 1, 1,
+                       "Cache Read Error : Bad Size", "negative X-Size");
+  /* both entries carry the line; the victim's is the second */
+  failures += corrupt_case_zip(opt, "X-In-Cache: 1", "X-In-Cache: 0", 2, 2,
+                               "Previous cache file not found (empty filename)",
+                               "blanked X-In-Cache");
+  /* smashed local file header: the entry is dropped at index load */
+  failures +=
+      corrupt_case_zip(opt, "PK\x03\x04", "XK\x03\x04", 2, 2,
+                       "File Cache Entry Not Found", "smashed local header");
+
+  corrupt_build(opt);
+  corrupt_victim_body(opt);
+  failures += corrupt_expect_victim(opt, "Cache Read Error : Read Data",
+                                    "garbled deflate stream");
+
+  /* An X-Size above INT_MAX is positive as int64 (slips a bare sign check) but
+     truncates negative in the (int) cast the malloc uses: a wraparound alloc.
+     cache_add asserts size fits an int, so such a value only reaches the reader
+     from a corrupt/foreign cache; inject it by overwriting the victim's long
+     Etag line with a same-length forged X-Size line (the parser keeps the last
+     X-Size it sees), keeping the zip byte-length and offsets intact. */
+  corrupt_build_etag(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim(opt, "Cache Read Error : Bad Size",
+                                    "X-Size above INT_MAX");
+
+  /* A headers-only entry (X-In-Cache: 0) may carry an X-Size >= INT_MAX: that
+     is how every >2GB non-html file is stored. It must survive a header probe
+     (or every update re-fetches the file); an in-memory read still rejects. */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483648AAAAAAAA", 1, 1);
+  failures += corrupt_expect_disk_header(opt, (LLint) 2147483648LL,
+                                         "headers-only X-Size above INT_MAX");
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size above INT_MAX");
+
+  /* exactly INT_MAX pins the >= boundary: (int) r.size + 1 would overflow */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: 2147483647AAAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "in-memory X-Size at INT_MAX");
+
+  /* the negative check must stay global, headers-only included */
+  corrupt_build_disk(opt);
+  corrupt_patch(opt, "Etag: AAAAAAAAAAAAAAAAAAAA", 26,
+                "X-Size: -2147483648AAAAAAA", 1, 1);
+  failures += corrupt_expect_victim_fil(opt, "/victim.bin",
+                                        "Cache Read Error : Bad Size",
+                                        "headers-only negative X-Size");
+
+  return failures;
+}

src/htscache_selftest.h

@@ -52,10 +52,19 @@ int cache_selftests(httrackp *opt, const char *dir);
    committed file, never by the test). Returns the failed-check count. */
 int cache_golden_selftest(httrackp *opt, const char *dir, int regen);
 
-/* #174/#219: assert a failing cache write aborts the mirror cleanly instead of
-   crashing. Returns the failed-check count. */
+/* Cache write-failure policy (#174/#219): abort on fatal errno or a streak,
+   drop just the entry otherwise. Returns the failed-check count. */
 int cache_write_failure_selftest(httrackp *opt, const char *dir);
 
+/* Exercise the hts_cache_reconcile() generation policies on file fixtures
+   under <dir>. Returns the failed-check count. */
+int cache_reconcile_selftest(httrackp *opt, const char *dir);
+
+/* Inject read-side corruption (zip byte surgery: bad size, header, deflate)
+   under <dir> and assert every case degrades to STATUSCODE_INVALID without
+   tainting a sibling entry. */
+int cache_corruption_selftest(httrackp *opt, const char *dir);
+
 #endif
 
 #endif

src/htscore.c

@@ -2137,47 +2137,7 @@ int httpmirror(char *url1, httrackp * opt) {
     hts_log_print(opt, LOG_NOTICE,
                   "No data seems to have been transferred during this session! : restoring previous one!");
     XH_uninit;
-    if ((fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-cache/old.dat")))
-        &&
-        (fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-           "hts-cache/old.ndx")))) {
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.dat"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.ndx"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.lst"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.txt"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.dat"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.ndx"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.lst"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.lst"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.txt"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.txt"));
-    }
+    hts_cache_reconcile(opt, CACHE_RECONCILE_ROLLBACK);
     opt->state.exit_xh = 2;     /* interrupted (no connection detected) */
     return 1;
   }
@@ -2892,6 +2852,9 @@ int check_fatal_io_errno(void) {
 #endif
 #ifdef EROFS
   case EROFS:                  /* Read-only file system */
+#endif
+#ifdef EDQUOT
+  case EDQUOT: /* Disk quota exceeded */
 #endif
     return 1;
     break;

src/htscore.h

@@ -216,6 +216,7 @@ struct cache_back {
   int zipEntriesCapa;
   hts_boolean
       zipWriteFailed; /**< a cache write failed; stop touching the stream */
+  int zipWriteFailures; /**< consecutive entry write failures; reset on store */
 };
 
 #ifndef HTS_DEF_FWSTRUCT_hash_struct

src/htscoremain.c

@@ -544,69 +544,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
     }
   }
 
-  // Existence d'un cache - pas de new mais un old.. renommer
+  // No new cache but an old one? promote it
 #if DEBUG_STEPS
   printf("Checking cache\n");
 #endif
-  if (!fexist
-      (fconcat
-       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-       StringBuff(opt->path_log), "hts-cache/new.zip"))) {
-    if (fexist
-        (fconcat
-         (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-         StringBuff(opt->path_log), "hts-cache/old.zip"))) {
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.zip"));
-    }
-  } else
-    if ((!fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log), "hts-cache/new.dat")))
-        ||
-        (!fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log),
-           "hts-cache/new.ndx")))) {
-    if ((fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log), "hts-cache/old.dat")))
-        &&
-        (fexist
-         (fconcat
-          (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-          StringBuff(opt->path_log),
-           "hts-cache/old.ndx")))) {
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/new.dat"));
-      remove(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/new.ndx"));
-      //remove(fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-             StringBuff(opt->path_log),
-              "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.dat"));
-      rename(fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                            StringBuff(opt->path_log),
-                                            "hts-cache/new.ndx"));
-      //rename(fconcat(StringBuff(opt->path_log),"hts-cache/old.lst"),fconcat(StringBuff(opt->path_log),"hts-cache/new.lst"));
-    }
-  }
+  hts_cache_reconcile(opt, CACHE_RECONCILE_PROMOTE);
 
   /* Interrupted mirror detected */
   if (!opt->quiet) {
@@ -2554,109 +2496,8 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
   printf("Cache & log settings\n");
 #endif
 
-  // on utilise le cache..
-  // en cas de présence des deux versions, garder la version la plus avancée,
-  // cad la version contenant le plus de fichiers  
-  if (opt->cache) {
-    if (fexist(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-in_progress.lock"))) {        // problemes..
-      if (fexist
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-            "hts-cache/new.dat"))) {
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.zip"))) {
-          if (fsize
-              (fconcat
-               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                "hts-cache/new.zip")) < 32768) {
-            if (fsize
-                (fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.zip")) > 65536) {
-              if (fsize
-                  (fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/old.zip")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                          StringBuff(opt->
-                                                                     path_log),
-                                                          "hts-cache/new.zip")))
-              {
-                remove(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/new.zip"));
-                rename(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/old.zip"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                      StringBuff(opt->path_log),
-                                                      "hts-cache/new.zip"));
-              }
-            }
-          }
-        }
-      } else
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/new.dat"))
-            &&
-            fexist(fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/new.ndx"))) {
-        if (fexist
-            (fconcat
-             (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-              "hts-cache/old.dat"))
-            &&
-            fexist(fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/old.ndx"))) {
-          // switcher si new<32Ko et old>65Ko (tailles arbitraires) ?
-          // ce cas est peut être une erreur ou un crash d'un miroir ancien, prendre
-          // alors l'ancien cache
-          if (fsize
-              (fconcat
-               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                "hts-cache/new.dat")) < 32768) {
-            if (fsize
-                (fconcat
-                 (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                  "hts-cache/old.dat")) > 65536) {
-              if (fsize
-                  (fconcat
-                   (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                    "hts-cache/old.dat")) > fsize(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                          StringBuff(opt->
-                                                                     path_log),
-                                                          "hts-cache/new.dat")))
-              {
-                remove(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/new.dat"));
-                remove(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/new.ndx"));
-                rename(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/old.dat"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                      StringBuff(opt->path_log),
-                                                      "hts-cache/new.dat"));
-                rename(fconcat
-                       (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log),
-                        "hts-cache/old.ndx"), fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                                      StringBuff(opt->path_log),
-                                                      "hts-cache/new.ndx"));
-                //} else {  // ne rien faire
-                //  remove("hts-cache/old.dat");
-                //  remove("hts-cache/old.ndx");
-              }
-            }
-          }
-        }
-      }
-    }
-  }
+  // If both cache generations exist, keep the most complete one
+  hts_cache_reconcile(opt, CACHE_RECONCILE_INTERRUPTED);
   // Débuggage des en têtes
   if (_DEBUG_HEAD) {
     ioinfo =

src/htsparse.c

@@ -4566,6 +4566,12 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           back_maydelete(opt, cache, sback, b);    // cancel
           b = -1;
 
+          /* the cancel may leave the now-unreferenced placeholder on disk
+           * (#483) */
+          if (fexist_utf8(delayed_back.url_sav)) {
+            back_delayed_discard(opt, &delayed_back);
+          }
+
           /* Recompute filename with MIME type */
           afs->save[0] = '\0';
           url_savename(afs, former, heap(ptr)->adr,
@@ -4782,6 +4788,9 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
 
           /* Still have a back reference */
           if (b >= 0) {
+            /* move a still-writing placeholder before the url_sav patch
+               blinds every cleanup to it (#483) */
+            back_delayed_rename(opt, &back[b], afs->save);
             /* patch url_sav BEFORE finalize: it records/caches under this name
              */
             strcpybuff(back[b].url_sav, afs->save);

src/htsselftest.c

@@ -1347,6 +1347,30 @@ static int st_cache_writefail(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
+  int err;
+
+  if (argc < 1) {
+    fprintf(stderr, "cache-corrupt: needs a directory\n");
+    return 1;
+  }
+  err = cache_corruption_selftest(opt, argv[0]);
+  printf("cache-corrupt: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
+static int st_reconcile(httrackp *opt, int argc, char **argv) {
+  int err;
+
+  if (argc < 1) {
+    fprintf(stderr, "reconcile: needs a directory\n");
+    return 1;
+  }
+  err = cache_reconcile_selftest(opt, argv[0]);
+  printf("cache-reconcile: %s\n", err ? "FAIL" : "OK");
+  return err;
+}
+
 static int st_dns(httrackp *opt, int argc, char **argv) {
   const int err = dns_selftests(opt);
 
@@ -2119,6 +2143,10 @@ static const struct selftest_entry {
      st_cache_golden},
     {"cache-writefail", "<dir>", "cache write-failure handling self-test",
      st_cache_writefail},
+    {"reconcile", "<dir>", "cache generation reconcile policy self-test",
+     st_reconcile},
+    {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
+     st_cache_corrupt},
     {"dns", "", "DNS resolver/cache self-test", st_dns},
     {"cookies", "", "cookie request-header self-test", st_cookies},
     {"useragent", "", "default User-Agent self-test", st_useragent},

tests/01_engine-reconcile.test

@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Cache generation reconcile policies (httrack -#test=reconcile <dir>):
+# promote a stranded old generation, keep the larger one after an aborted
+# run, and restore the old one when an update transferred nothing.
+
+set -eu
+
+dir=$(mktemp -d)
+trap 'rm -rf "$dir"' EXIT
+
+out=$(httrack -#test=reconcile "$dir")
+
+test "$out" = "cache-reconcile: OK" || {
+    echo "expected 'cache-reconcile: OK', got: $out" >&2
+    exit 1
+}

tests/01_zlib-cache-corrupt.test

@@ -0,0 +1,19 @@
+#!/bin/bash
+#
+# Read-side cache corruption (httrack -#test=cache-corrupt <dir>): zip byte
+# surgery (bad/oversized X-Size, blanked X-In-Cache, smashed header, garbled
+# deflate) must each be rejected per-entry, never crash, never taint the sibling.
+
+set -eu
+
+dir=$(mktemp -d)
+trap 'rm -rf "$dir"' EXIT
+
+# the smashed-header case logs expected "Corrupted cache entry" warnings on
+# stdout; the verdict is the last line
+out=$(httrack -#test=cache-corrupt "$dir" 2>/dev/null | tail -n1)
+
+test "$out" = "cache-corrupt: OK" || {
+    echo "expected 'cache-corrupt: OK', got: $out" >&2
+    exit 1
+}

tests/01_zlib-cache-writefail.test

@@ -4,10 +4,9 @@
 # POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms and GNU-only
 # tool flags despite the #!/bin/bash above.
 
-# Cache write-failure handling (httrack -#test=cache-writefail <dir>). #174/#219.
-# A failing new.zip write (disk full) used to crash the process via assertf; it
-# must instead stop the mirror with a fatal error (exit_xh=-1), no crash. The
-# self-test asserts that; reverting the fix makes -#test=cache-writefail abort (SIGABRT) and fail.
+# Cache write-failure policy (-#test=cache-writefail <dir>). #174/#219: disk
+# full or a failure streak aborts cleanly; an isolated failure or an oversized
+# entry is only dropped.
 
 set -eu
 
@@ -22,3 +21,9 @@ printf '%s\n' "$out" | grep -qx "cache-writefail: OK" || {
     echo "expected 'cache-writefail: OK', got: $out" >&2
     exit 1
 }
+
+# A skipped entry must be warned about with its URL.
+printf '%s\n' "$out" | grep -q "entry not cached: example.com/" || {
+    echo "expected a URL-bearing skip warning" >&2
+    exit 1
+}

tests/22_local-broken-size.test

@@ -1,17 +1,19 @@
 #!/bin/bash
-# Issues #32/#41: a Content-Length that disagrees with the body warns "bogus
-# state (broken size)" and skips the cache; -%B (tolerant) accepts it.
+# Issues #32/#41: a Content-Length that disagrees with the body warns
+# "incomplete transfer" and skips the cache; -%B (tolerant) accepts it.
+
+set -euo pipefail
 
 : "${top_srcdir:=..}"
 
 # Default: warn, but the file is still written.
 bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
     --found 'size/oversize.bin' \
-    --log-found 'bogus state \(broken size' \
+    --log-found 'incomplete transfer \(expected' \
     httrack 'BASEURL/size/index.html'
 
 # -%B (tolerant): no warning, file written.
 bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
     --found 'size/oversize.bin' \
-    --log-not-found 'bogus state' \
+    --log-not-found 'incomplete transfer|not cached' \
     httrack 'BASEURL/size/index.html' '-%B'

tests/33_local-delayed.test

@@ -2,7 +2,7 @@
 #
 # Degenerate delayed-type paths (#5/#107 family): redirects that never resolve
 # a name must drop cleanly -- no .delayed leftovers (audited by local-crawl.sh),
-# no "bogus state" cache warnings, resolvable links still land correctly.
+# no "not cached" warnings, resolvable links still land correctly.
 
 set -euo pipefail
 
@@ -16,5 +16,5 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun --errors 0 \
     --not-found 'delayed/noloc.html' \
     --not-found 'delayed/selfloop.html' \
     --not-found 'delayed/chain9.pdf' \
-    --log-not-found 'bogus state' \
+    --log-not-found 'not cached' \
     httrack 'BASEURL/delayed/index.html'

tests/34_local-maxtime.test

@@ -7,10 +7,8 @@ set -euo pipefail
 
 : "${top_srcdir:=..}"
 
-# cancelled crawls can orphan .delayed placeholders (#483): skip that audit
 start=$(date +%s)
 bash "$top_srcdir/tests/local-crawl.sh" \
-    --skip-delayed-audit \
     --log-found 'More than 2 seconds passed' \
     httrack 'BASEURL/trickle/index.html' -E2 -c4
 wall=$(($(date +%s) - start))

tests/36_local-bigcrawl.test

@@ -44,7 +44,7 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --file-matches 'big/a/blk2.css' 'url\(blk2-bg\.png\)' \
     --file-matches 'big/p/5.html' "document\\.write\\('<a href=\"\\.\\./f5/dw\\.html\"" \
     --file-not-matches 'big/p/1.html' 'href="/big/' \
-    --log-not-found 'bogus state|[Pp]anic|assert' \
+    --log-not-found 'not cached|[Pp]anic|assert' \
     --log-found '\(404\) at link [^ ]*/big/e/404\.html' \
     --log-found '\(410\) at link [^ ]*/big/e/410\.html' \
     --log-found '\(500\) at link [^ ]*/big/e/500\.html' \

tests/37_local-cache-outage.test

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+# An update run against a dead server must not destroy the cache: the no-data
+# rollback restores the previous hts-cache generation (zip caches lost it).
+
+set -eu
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun-dead \
+    --found 'simple/basic.html' \
+    httrack 'BASEURL/simple/basic.html'

tests/38_local-update-304.test

@@ -0,0 +1,14 @@
+#!/bin/bash
+#
+# An all-304 update of a tiny site (headers under the 32K rollback threshold)
+# is a healthy run: it must not trip the no-data rollback as a fake outage.
+
+set -eu
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
+    --log-found 'no files updated' \
+    --log-not-found 'No data seems to have been transferred' \
+    --found 'mini304/index.html' --found 'mini304/page.html' \
+    httrack 'BASEURL/mini304/index.html'

tests/39_local-delayed-cancel.test

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# Cancelled delayed-type-checks must not orphan .delayed placeholders (#483).
+# Timing-dependent (hence two tries); -A keeps the window reachable.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+for _ in 1 2; do
+    bash "$top_srcdir/tests/local-crawl.sh" \
+        httrack 'BASEURL/dcancel/index.html' -E1 -c4 -A25000
+done

tests/Makefile.am

@@ -48,6 +48,7 @@ TESTS = \
 	01_engine-parse.test \
 	01_engine-pause.test \
 	01_engine-rcfile.test \
+	01_engine-reconcile.test \
 	01_engine-redirect.test \
 	01_engine-relative.test \
 	01_engine-robots.test \
@@ -63,6 +64,7 @@ TESTS = \
 	01_engine-useragent.test \
 	01_zlib-acceptencoding.test \
 	01_zlib-cache.test \
+	01_zlib-cache-corrupt.test \
 	01_zlib-cache-golden.test \
 	01_zlib-cache-writefail.test \
 	01_zlib-savename-cached.test \
@@ -99,6 +101,9 @@ TESTS = \
 	33_local-delayed.test \
 	34_local-maxtime.test \
 	35_local-maxsize.test \
-	36_local-bigcrawl.test
+	36_local-bigcrawl.test \
+	37_local-cache-outage.test \
+	38_local-update-304.test \
+	39_local-delayed-cancel.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -25,6 +25,8 @@
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
 # which the ephemeral port forces into the cookie domain) and passes it to
 # httrack via --cookies-file, to exercise preloaded cookies.
+# --rerun-dead re-runs with the server stopped: the no-data rollback must
+# restore the previous hts-cache generation byte-identical.
 
 set -u
 
@@ -37,6 +39,7 @@ key="${testdir}/server.key"
 tls=
 verbose=
 rerun=
+rerun_dead=
 tmpdir=
 serverpid=
 crawlpid=
@@ -94,7 +97,6 @@ tmpdir=$(mktemp -d "${tmptopdir}/httrack_local.XXXXXX") || die "could not create
 # --- parse leading control flags --------------------------------------------
 declare -a audit=()
 declare -a cookies=()
-skip_delayed_audit=""
 scheme=http
 pos=0
 args=("$@")
@@ -102,7 +104,8 @@ nargs=$#
 while test "$pos" -lt "$nargs"; do
     case "${args[$pos]}" in
     --debug) verbose=1 ;;
-    --rerun) rerun=1 ;; # run httrack a second time (update pass) before auditing
+    --rerun) rerun=1 ;;           # run httrack a second time (update pass) before auditing
+    --rerun-dead) rerun_dead=1 ;; # re-run with the server stopped (cache rollback)
     --no-purge)
         nopurge=1
         audit+=("--no-purge")
@@ -119,9 +122,6 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         cookies+=("${args[$pos]}")
         ;;
-    --skip-delayed-audit)
-        skip_delayed_audit=1
-        ;;
     --errors | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
@@ -241,6 +241,43 @@ if test -n "$rerun"; then
     fi
 fi
 
+# --- optional dead pass: server stopped, the cache must survive the rollback --
+if test -n "$rerun_dead"; then
+    zip="${out}/hts-cache/new.zip"
+    test -s "$zip" || die "no cache was written by the first pass"
+    cp "$zip" "${tmpdir}/cache-before.zip"
+    cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
+    kill "$serverpid" 2>/dev/null
+    wait "$serverpid" 2>/dev/null
+    serverpid=
+    info "re-running httrack against the stopped server"
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
+        "${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
+    crawlpid=$!
+    wait "$crawlpid" || true
+    crawlpid=
+    result "OK (dead pass ran)"
+    # The dead pass must have gone through the no-data rollback, not bailed out
+    # before the mirror loop (which would leave the cache trivially untouched).
+    info "checking the dead pass hit the rollback"
+    if grep -aq "No data seems to have been transferred" "${out}/hts-log.txt"; then
+        result "OK"
+    else
+        result "rollback notice not found in hts-log.txt"
+        exit 1
+    fi
+    info "checking the previous cache generation was restored"
+    if cmp -s "$zip" "${tmpdir}/cache-before.zip" &&
+        test ! -e "${out}/hts-cache/old.zip"; then
+        result "OK"
+    else
+        result "new.zip differs from the pre-outage cache (or old.zip left behind)"
+        exit 1
+    fi
+    # Audits below describe the healthy crawl, not the dead pass.
+    cp "${tmpdir}/log-before.txt" "${out}/hts-log.txt"
+fi
+
 # --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
 hostroot=
 for cand in "${out}/127.0.0.1_${port}" "${out}/127.0.0.1"; do
@@ -252,15 +289,12 @@ done
 test -n "$hostroot" || die "could not find host root under $out"
 debug "host root: $hostroot"
 
-# A completed crawl must leave no .delayed temporaries (issue #107).
-# --skip-delayed-audit: a cancelled crawl can orphan placeholders (issue #483)
-if test -z "$skip_delayed_audit"; then
-    info "checking for leftover .delayed files"
-    leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
-    if test -z "$leftovers"; then result "OK"; else
-        result "leftover: $leftovers"
-        exit 1
-    fi
+# No crawl, even a cancelled one, may leave .delayed temporaries (#107, #483).
+info "checking for leftover .delayed files"
+leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
+if test -z "$leftovers"; then result "OK"; else
+    result "leftover: $leftovers"
+    exit 1
 fi
 
 # --- audit -------------------------------------------------------------------

tests/local-server.py

@@ -781,7 +781,7 @@ class Handler(SimpleHTTPRequestHandler):
         self.send_raw(b"", "text/html")
 
     # broken Content-Length (#32/#41): declared size != bytes sent. httrack
-    # warns "bogus state (broken size)" and skips the cache unless -%B.
+    # warns "incomplete transfer" and skips the cache unless -%B.
     def route_size_index(self):
         self.send_html('\t<a href="oversize.bin">over</a>\n')
 
@@ -831,6 +831,16 @@ class Handler(SimpleHTTPRequestHandler):
     def route_redir_target(self):
         self.send_raw(b"<html><body>redirect target</body></html>\n", "text/html")
 
+    # --- /mini304/: tiny fully-cacheable site (an update gets only 304s) ---
+    def route_mini304_index(self):
+        self.big_send(
+            b'<html><body>\n\t<a href="page.html">page</a>\n</body></html>\n',
+            "text/html",
+        )
+
+    def route_mini304_page(self):
+        self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
+
     # --- delayed-type degenerate paths (issues #5/#107) --------------------
     def route_delayed_index(self):
         self.send_html(
@@ -903,6 +913,26 @@ class Handler(SimpleHTTPRequestHandler):
         except OSError:
             pass
 
+    # #483: trickled .bin pages so the -E stop lands in the type waiter's
+    # unlock-to-patch window with body bytes pending.
+    def route_dcancel_index(self):
+        self.send_bin_index()
+
+    def route_dcancel_page(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", "4096")
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            for _ in range(32):
+                self.wfile.write(b"z" * 128)
+                self.wfile.flush()
+                time.sleep(0.05)
+        except OSError:
+            pass
+
     # -M byte cap (#77): large fast files so a crawl overruns -M immediately.
     BIGFILE_BYTES = 640 * 1024
 
@@ -966,6 +996,15 @@ class Handler(SimpleHTTPRequestHandler):
         "/trickle/p5.bin": route_trickle_page,
         "/trickle/p6.bin": route_trickle_page,
         "/trickle/p7.bin": route_trickle_page,
+        "/dcancel/index.html": route_dcancel_index,
+        "/dcancel/p0.bin": route_dcancel_page,
+        "/dcancel/p1.bin": route_dcancel_page,
+        "/dcancel/p2.bin": route_dcancel_page,
+        "/dcancel/p3.bin": route_dcancel_page,
+        "/dcancel/p4.bin": route_dcancel_page,
+        "/dcancel/p5.bin": route_dcancel_page,
+        "/dcancel/p6.bin": route_dcancel_page,
+        "/dcancel/p7.bin": route_dcancel_page,
         "/bigfiles/index.html": route_bigfiles_index,
         "/bigfiles/p0.bin": route_bigfile,
         "/bigfiles/p1.bin": route_bigfile,
@@ -993,6 +1032,8 @@ class Handler(SimpleHTTPRequestHandler):
         "/redir/index.html": route_redir_index,
         "/redir/go.php": route_redir_go,
         "/redir/target.html": route_redir_target,
+        "/mini304/index.html": route_mini304_index,
+        "/mini304/page.html": route_mini304_page,
     }
 
     # --- /big/ seeded pseudo-site ------------------------------------------

tools/mkdeb.sh

@@ -211,7 +211,9 @@ main() {
     # lintian ourselves below as the real gate.
     local -a debuild_opts=(--no-lintian)
     local -a build_opts=()
-    [[ $source_only -eq 1 ]] && build_opts+=(-S)
+    # -d: a source build runs no debhelper, so don't require Build-Depends
+    # locally (the buildds and the --sbuild gate enforce them).
+    [[ $source_only -eq 1 ]] && build_opts+=(-S -d)
     if [[ $unsigned -eq 1 ]]; then
         build_opts+=(-us -uc)
     else
@@ -234,12 +236,15 @@ main() {
 
     # The real lintian gate (debuild only reports, it does not fail on tags).
     # --profile debian: CI runners are Ubuntu, whose vendor data would wrongly
-    # reject the Debian "unstable" distribution. newer-standards-version only
-    # means the local lintian is older than the buildds', not a package
-    # defect, so suppress it. set -e turns any error/warning tag into a failure.
+    # reject the Debian "unstable" distribution. Suppressed tags are stale-local-
+    # lintian skew, not package defects: newer-standards-version, and
+    # recommended-field (old lintian still wants the Priority field the sid
+    # lintian in CI accepts dropping). set -e turns any error/warning tag into
+    # a failure.
     info "running lintian gate (--fail-on=error,warning)"
     lintian --profile debian -I -i --fail-on=error,warning \
-        --suppress-tags newer-standards-version "${changes[@]}"
+        --suppress-tags newer-standards-version,recommended-field \
+        "${changes[@]}"
 
     dcmd cp -- "${changes[@]}" "$outdir/"