Add a -M byte-cap non-regression test

The -M limit had no test coverage. New /bigfiles/ fixture (8 fast 640KB files), a --max-mirror-bytes audit in local-crawl.sh, and a crawl that must log the "giving up" error and stay under the uncapped total. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com>
Enforce the -E time limit inside the transfer wait cycle (#482 )
2026-07-04 16:14:47 +03:00 · 2026-07-04 15:05:15 +02:00 · 2026-07-04 13:13:33 +02:00 · 2026-07-04 10:05:41 +02:00 · 2026-07-04 09:13:47 +02:00 · 2026-07-04 08:37:14 +02:00
26 changed files with 1540 additions and 203 deletions
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -62,7 +62,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htsname.c htsrobots.c htstools.c htswizard.c \
 	htsalias.c htsthread.c htsindex.c htsbauth.c \
 	htsmd5.c htszlib.c htswrap.c htsconcat.c \
-	htsmodules.c htscharset.c punycode.c htsencoding.c \
+	htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
 	md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
 	hts-indextmpl.h htsalias.h htsback.h htsbase.h htssafe.h \
@@ -70,7 +70,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htsconfig.h htscore.h htsparse.h htscoremain.h htsdefines.h  \
 	htsfilters.h htsftp.h htsglobal.h htshash.h coucal/coucal.h \
 	htshelp.h htsindex.h htslib.h htsmd5.h \
-	htsmodules.h htsname.h htsnet.h \
+	htsmodules.h htsname.h htsnet.h htssniff.h \
 	htsopt.h htsrobots.h htsthread.h  \
 	htstools.h htswizard.h htswrap.h htszlib.h  \
 	htsstrings.h htsarrays.h httrack-library.h \
--- a/src/htsback.c
+++ b/src/htsback.c
@@ -1359,6 +1359,18 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
 }

 // effacer entrée
+/* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
+static void back_delayed_discard(httrackp *opt, lien_back *back) {
+  if (back->r.out != NULL) {
+    fclose(back->r.out);
+    back->r.out = NULL;
+  }
+  back->r.is_write = 0;
+  if (opt != NULL)
+    url_savename_refname_remove(opt, back->url_adr, back->url_fil);
+  (void) UNLINK(back->url_sav);
+}
+
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                const int p) {
  lien_back *const back = sback->lnk;
@@ -1366,6 +1378,12 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,

  assertf(p >= 0 && p < back_max);
  if (p >= 0 && p < sback->count) {     // on sait jamais..
+    /* mid-write cancel: drop a .delayed placeholder; real-named partials
+       survive for resume (--continue) */
+    if (back[p].r.is_write && IS_DELAYED_EXT(back[p].url_sav) &&
+        (back[p].status != STATUS_READY || back[p].r.statuscode <= 0)) {
+      back_delayed_discard(opt, &back[p]);
+    }
    // Vérificateur d'intégrité
 #if DEBUG_CHECKINT
    _CHECKINT(&back[p], "Appel back_delete")
@@ -2237,12 +2255,13 @@ int host_wait(httrackp * opt, lien_back * back) {

 static int slot_can_be_cleaned(const lien_back * back) {
  return (back->status == STATUS_READY) // ready
-    /* Check autoclean */
-    && (!back->testmode)        // not test mode
-    && (strnotempty(back->url_sav))     // filename exists
-    && (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
-    && (back->r.size >= 0)      // size>=0
-    ;
+         /* Check autoclean */
+         && (!back->locked)   // not held by hts_wait_delayed (name pending)
+         && (!back->testmode) // not test mode
+         && (strnotempty(back->url_sav))     // filename exists
+         && (HTTP_IS_OK(back->r.statuscode)) // HTTP "OK"
+         && (back->r.size >= 0)              // size>=0
+      ;
 }

 static int slot_can_be_finalized(httrackp * opt, const lien_back * back) {
@@ -2418,6 +2437,34 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
  back_clean(opt, cache, sback);
 #endif

+  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
+     starves (#481). FTP slots stay, their thread owns the socket. */
+  if (!back_checkmirror(opt)) {
+    int aborted = 0;
+    unsigned int i;
+
+    for (i = 0; i < (unsigned int) back_max; i++) {
+      if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
+        if (back[i].r.soc != INVALID_SOCKET) {
+          deletehttp(&back[i].r);
+        }
+        back[i].r.soc = INVALID_SOCKET;
+        /* drop a .delayed placeholder; real partials survive for resume */
+        if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
+          back_delayed_discard(opt, &back[i]);
+        back[i].r.statuscode = STATUSCODE_TIMEOUT;
+        strcpybuff(back[i].r.msg, "Mirror Time Out");
+        back[i].status = STATUS_READY;
+        back_set_finished(sback, i);
+        aborted++;
+      }
+    }
+    if (aborted > 0)
+      hts_log_print(opt, LOG_WARNING,
+                    "time limit reached, %d transfer(s) aborted", aborted);
+    return;
+  }
+
  // recevoir tant qu'il y a des données (avec un maximum de max_loop boucles)
  do_wait = 0;
  gestion_timeout = 0;
@@ -2891,10 +2938,10 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
          // range size hack old location

 #if HTS_DIRECTDISK
-          // Court-circuit:
-          // Peut-on stocker le fichier directement sur disque?
-          // Ahh que ca serait vachement mieux et que ahh que la mémoire vous dit merci!
-          if (back[i].status) {
+          // Shortcut: store the file directly on disk when possible,
+          // sparing memory
+          if (back[i].status &&
+              !back[i].locked) { // name still pending when locked
            if (back[i].r.is_write == 0) {      // mode mémoire
              if (back[i].r.adr == NULL) {      // rien n'a été écrit
                if (!back[i].testmode) {        // pas mode test
@@ -3960,8 +4007,12 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                              && (back[i].r.adr = (char *) malloct(2))) {
                            back[i].r.adr[0] = 0;
                          }
-                          hts_log_print(opt, LOG_TRACE, "finalizing empty");
-                          back_finalize(opt, cache, sback, i);
+                          /* locked = name pending; the waiter finalizes after
+                             patching url_sav (else: cached as .delayed, #5) */
+                          if (!back[i].locked) {
+                            hts_log_print(opt, LOG_TRACE, "finalizing empty");
+                            back_finalize(opt, cache, sback, i);
+                          }
                        } else if (!back[i].r.is_chunk) {       // pas de chunk
                          //if (back[i].r.http11!=2) {    // pas de chunk
                          back[i].is_chunk = 0;
@@ -4159,6 +4210,11 @@ int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize)
  return 1;
 }

+/* Grace left to the smooth stop before in-flight transfers are aborted. */
+static int back_maxtime_grace(const int maxtime) {
+  return maximum(5, minimum(30, maxtime / 10));
+}
+
 int back_checkmirror(httrackp * opt) {
  // Check max size
  if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
@@ -4175,13 +4231,19 @@ int back_checkmirror(httrackp * opt) {
     */
  }
  // Check max time
-  if ((opt->maxtime > 0)
-      && ((time_local() - HTS_STAT.stat_timestart) >= opt->maxtime)) {
-    if (!opt->state.stop) {     /* not yet stopped */
-      hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-                    opt->maxtime);
-      /* cancel mirror smoothly */
-      hts_request_stop(opt, 0);
+  if (opt->maxtime > 0) {
+    const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
+
+    if (elapsed >= opt->maxtime) {
+      if (!opt->state.stop) { /* not yet stopped */
+        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                      opt->maxtime);
+        /* cancel mirror smoothly */
+        hts_request_stop(opt, 0);
+      }
+      /* smooth stop starved past the grace period: stop waiting (#481) */
+      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+        return 0;
    }
  }
  return 1;                     /* Ok, go on */
--- a/src/htsback.h
+++ b/src/htsback.h
@@ -136,6 +136,8 @@ void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
+/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
+   the -E deadline overran its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);

 #endif
--- a/src/htscache.c
+++ b/src/htscache.c
@@ -596,15 +596,18 @@ htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
  return cache_readex(opt, cache, adr, fil, save, location, NULL, 1);
 }

-htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
-                                   const char *adr, const char *fil) {
-  htsblk r = cache_read(opt, cache, adr, fil, NULL, NULL);
+htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
+                                   const char *adr, const char *fil,
+                                   char *return_save) {
+  htsblk r = cache_readex(opt, cache, adr, fil, NULL, NULL, return_save, 0);

  if (r.statuscode == -1) {
    lien_back *itemback = NULL;

    if (back_unserialize_ref(opt, adr, fil, &itemback) == 0) {
      r = itemback->r;
+      if (return_save != NULL)
+        strlcpybuff(return_save, itemback->url_sav, HTS_URLMAXSIZE * 2);
      /* cleanup */
      back_clear_entry(itemback);       /* delete entry content */
      freet(itemback);          /* delete item */
--- a/src/htscache.h
+++ b/src/htscache.h
@@ -66,8 +66,11 @@ htsblk cache_read(httrackp * opt, cache_back * cache, const char *adr,
                  const char *fil, const char *save, char *location);
 htsblk cache_read_ro(httrackp * opt, cache_back * cache, const char *adr,
                     const char *fil, const char *save, char *location);
-htsblk cache_read_including_broken(httrackp * opt, cache_back * cache,
-                                   const char *adr, const char *fil);
+/* Like cache_read, but also yields entries whose transfer broke; return_save
+   (optional, HTS_URLMAXSIZE*2) receives the entry's recorded save name. */
+htsblk cache_read_including_broken(httrackp *opt, cache_back *cache,
+                                   const char *adr, const char *fil,
+                                   char *return_save);
 htsblk cache_readex(httrackp * opt, cache_back * cache, const char *adr,
                    const char *fil, const char *save, char *location,
                    char *return_save, int readonly);
--- a/src/htsname.c
+++ b/src/htsname.c
@@ -41,6 +41,10 @@ Please visit our Website: http://www.httrack.com
 #include "htstools.h"
 #include "htscharset.h"
 #include "htsencoding.h"
+#include "htssniff.h"
+#if HTS_USEZLIB
+#include "htszlib.h"
+#endif
 #include <ctype.h>

 #define ADD_STANDARD_PATH \
@@ -70,30 +74,36 @@ static const char *hts_tbdev[] = {
  ""
 };

-#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET() do { \
-  int prev = opt->state._hts_in_html_parsing; \
-  while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
-    opt->state. _hts_in_html_parsing = 6; \
-    /* Wait .. */ \
-    back_wait(sback,opt,cache,0); \
-    /* Transfer rate */ \
-    engine_stats(); \
-    /* Refresh various stats */ \
-    HTS_STAT.stat_nsocket=back_nsoc(sback); \
-    HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
-    HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
-    HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
-    HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
-    HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
-    /* Check */ \
-    { \
-      if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,-1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
-        return -1; \
-      } \
-    } \
-  } \
-  opt->state._hts_in_html_parsing = prev; \
-} while(0)
+#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET()                                \
+  do {                                                                         \
+    int prev = opt->state._hts_in_html_parsing;                                \
+    while (back_pluggable_sockets_strict(sback, opt) <= 0) {                   \
+      opt->state._hts_in_html_parsing = 6;                                     \
+      /* Wait .. */                                                            \
+      back_wait(sback, opt, cache, 0);                                         \
+      /* time limit (-E) exceeded: stop waiting for a socket (#481) */         \
+      if (!back_checkmirror(opt))                                              \
+        break;                                                                 \
+      /* Transfer rate */                                                      \
+      engine_stats();                                                          \
+      /* Refresh various stats */                                              \
+      HTS_STAT.stat_nsocket = back_nsoc(sback);                                \
+      HTS_STAT.stat_errors = fspc(opt, NULL, "error");                         \
+      HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");                     \
+      HTS_STAT.stat_infos = fspc(opt, NULL, "info");                           \
+      HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);    \
+      HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);              \
+      /* Check */                                                              \
+      {                                                                        \
+        if (!RUN_CALLBACK7(                                                    \
+                opt, loop, sback->lnk, sback->count, -1, ptr, opt->lien_tot,   \
+                (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {  \
+          return -1;                                                           \
+        }                                                                      \
+      }                                                                        \
+    }                                                                          \
+    opt->state._hts_in_html_parsing = prev;                                    \
+  } while (0)

 /* Strip all // */
 static void cleanDoubleSlash(char *s) {
@@ -138,37 +148,191 @@ static void cleanEndingSpaceOrDot(char *s) {
  }
 }

-/* Should the wire Content-Type override the URL's own extension when naming the
-   saved file? True when the type is patchable (may_unknown2) and either the URL
-   extension implies no specific type or the server declared a disagreeing one.
-   A URL extension mapping to a specific non-HTML type is kept only when the
-   server declared NO type (the HTS_UNKNOWN_MIME sentinel; the #267 mangle
-   guard): a typeless .png stays .png, but a .pdf explicitly served as text/html
-   is named .html. The sentinel rides the cache, so updates stay consistent. */
-static int wire_patches_ext(httrackp *opt, const char *wiremime,
-                            const char *file) {
-  char urlmime[256];
+/* Wire Content-Type vs URL extension: a patchable wire type wins over an
+   unspecific ext, the HTS_UNKNOWN_MIME sentinel keeps a specific non-HTML ext
+   (#267 guard), a declared disagreement is CONTESTED (sniffed below). */
+typedef enum wire_verdict {
+  WIRE_KEEPS_EXT,
+  WIRE_WINS,
+  WIRE_CONTESTED
+} wire_verdict;

+static wire_verdict wire_ext_verdict(httrackp *opt, const char *wiremime,
+                                     const char *file, char *urlmime,
+                                     size_t urlmime_size) {
  if (may_unknown2(opt, wiremime, file))
-    return 0; /* type kept verbatim (keep-list / bogus-multiple) */
+    return WIRE_KEEPS_EXT; /* type kept verbatim (keep-list / bogus-multiple) */
  urlmime[0] = '\0';
  /* type implied by the URL extension, only when confidently known (flag 0) */
-  if (!get_httptype_sized(opt, urlmime, sizeof(urlmime), file, 0))
-    return 1; /* URL ext implies no known type: trust the wire type */
+  if (!get_httptype_sized(opt, urlmime, urlmime_size, file, 0))
+    return WIRE_WINS; /* URL ext implies no known type */
  if (strfield2(wiremime, urlmime))
-    return 0; /* wire agrees with the ext: keep it (no .htm->.html churn) */
-  /* wire disagrees with a specific non-HTML URL ext. Keep the ext only when
-     the server declared no type (the sentinel); an explicitly declared type,
-     even text/html, is trusted, so a binary-looking URL that really serves
-     HTML (login/error interstitial, soft-404) is named .html. */
+    return WIRE_KEEPS_EXT; /* agreement (no .htm->.html churn) */
  if (!is_hypertext_mime(opt, urlmime, file) &&
      strfield2(wiremime, HTS_UNKNOWN_MIME))
+    return WIRE_KEEPS_EXT; /* no declared type */
+  return WIRE_CONTESTED;
+}
+
+/* Optional evidence for a contested wire-vs-ext verdict. */
+typedef struct sniff_src {
+  struct_back *sback;       /* live backing (looked up by adr/fil) */
+  const lien_back *headers; /* snapshot: r.adr, else the url_sav file */
+  const char *adr, *fil;
+  const char *prev_save; /* previous run's save name (cache X-Save) */
+} sniff_src;
+
+#if HTS_USEZLIB
+/* Inflate the head of a gzip/zlib stream; 0 when undecodable. */
+static size_t sniff_inflate_head(const void *in, size_t in_len, void *out,
+                                 size_t out_len) {
+  z_stream zs;
+  size_t n = 0;
+  int err;
+
+  memset(&zs, 0, sizeof(zs));
+  if (inflateInit2(&zs, 47) != Z_OK) /* 47: gzip or zlib, autodetected */
    return 0;
+  zs.next_in = (const Bytef *) in;
+  zs.avail_in = (uInt) in_len;
+  zs.next_out = (Bytef *) out;
+  zs.avail_out = (uInt) out_len;
+  err = inflate(&zs, Z_SYNC_FLUSH);
+  if (err == Z_OK || err == Z_STREAM_END || err == Z_BUF_ERROR)
+    n = out_len - zs.avail_out;
+  inflateEnd(&zs);
+  return n;
+}
+#endif
+
+static size_t sniff_read_head(const char *path, void *buf, size_t len) {
+  char catbuff[CATBUFF_SIZE];
+  FILE *const fp = FOPEN(fconv(catbuff, sizeof(catbuff), path), "rb");
+  size_t n = 0;
+
+  if (fp != NULL) {
+    n = fread(buf, 1, len, fp);
+    fclose(fp);
+  }
+  return n;
+}
+
+/* Body head of one slot: memory, else its flushed on-disk file (url_sav, or
+   tmpfile for a compressed stream); inflated so the sniff sees the final body.
+ */
+static size_t sniff_slot_head(const lien_back *slot, void *buf, size_t len) {
+  const htsblk *const r = &slot->r;
+  size_t n = 0;
+
+  if (r->adr != NULL && r->size > 0) {
+    n = (size_t) r->size < len ? (size_t) r->size : len;
+    memcpy(buf, r->adr, n);
+  } else {
+    if (r->out != NULL)
+      fflush(r->out);
+    if (slot->url_sav[0] != '\0')
+      n = sniff_read_head(slot->url_sav, buf, len);
+    if (n == 0 && slot->tmpfile != NULL && slot->tmpfile[0] != '\0')
+      n = sniff_read_head(slot->tmpfile, buf, len);
+  }
+  if (n > 0 && r->compressed) {
+#if HTS_USEZLIB
+    unsigned char raw[HTS_SNIFF_LEN];
+
+    if (n > sizeof(raw))
+      n = sizeof(raw);
+    memcpy(raw, buf, n);
+    n = sniff_inflate_head(raw, n, buf, len);
+#else
+    n = 0;
+#endif
+  }
+  return n;
+}
+
+/* Up to len leading body bytes; 0 when unavailable, and always in
+   non-delayed mode (its HEAD-probe first run couldn't sniff either). */
+static size_t sniff_body_head(httrackp *opt, const sniff_src *src, void *buf,
+                              size_t len) {
+  size_t n = 0;
+
+  if (src == NULL || opt->savename_delayed == HTS_SAVENAME_DELAYED_NONE)
+    return 0;
+  /* live backing slot: a snapshot (back_copy_static) loses r.adr/r.out */
+  if (src->sback != NULL && src->adr != NULL && src->fil != NULL) {
+    const int b = back_index(opt, src->sback, src->adr, src->fil, NULL);
+
+    if (b >= 0)
+      n = sniff_slot_head(&src->sback->lnk[b], buf, len);
+  }
+  if (n == 0 && src->headers != NULL)
+    n = sniff_slot_head(src->headers, buf, len);
+  return n;
+}
+
+/* Contested verdicts: magic proving the URL ext keeps it, else wire wins. */
+static int wire_patches_ext(httrackp *opt, const sniff_src *src,
+                            const char *wiremime, const char *file) {
+  char urlmime[256];
+
+  switch (wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime))) {
+  case WIRE_KEEPS_EXT:
+    return 0;
+  case WIRE_WINS:
+    return 1;
+  case WIRE_CONTESTED:
+    break;
+  }
+  if (src != NULL) {
+    if (hts_sniff_mime_known(urlmime)) {
+      unsigned char head[HTS_SNIFF_LEN];
+      const size_t n = sniff_body_head(opt, src, head, sizeof(head));
+
+      if (n > 0)
+        return hts_sniff_mime_consistent(head, n, urlmime) ? 0 : 1;
+    }
+    /* no bytes: reproduce the previous run's verdict (cached X-Save name) */
+    if (src->prev_save != NULL && src->prev_save[0] != '\0') {
+      char prevmime[256];
+
+      prevmime[0] = '\0';
+      if (get_httptype_sized(opt, prevmime, sizeof(prevmime), src->prev_save,
+                             0) &&
+          strfield2(prevmime, urlmime))
+        return 0;
+    }
+  }
  return 1;
 }

-// forme le nom du fichier à sauver (save) à partir de fil et adr
-// système intelligent, qui renomme en cas de besoin (exemple: deux INDEX.HTML et index.html)
+int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime,
+                         const char *file) {
+  char urlmime[256];
+
+  return wiremime != NULL && strnotempty(wiremime) &&
+         wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime)) ==
+             WIRE_CONTESTED &&
+         hts_sniff_mime_known(urlmime);
+}
+
+/* Wire-metadata name change: a Content-Disposition filename wins (returns 2),
+   else the declared type's ext when wire_patches_ext() allows (returns 1),
+   else 0. ext receives the new extension or replacement filename. */
+static int resolve_extension(httrackp *opt, const sniff_src *src,
+                             const char *cdispo, const char *contenttype,
+                             const char *fil, char *ext, size_t ext_size) {
+  if (strnotempty(cdispo)) {
+    strlcpybuff(ext, cdispo, ext_size);
+    return 2;
+  }
+  if (wire_patches_ext(opt, src, contenttype, fil) &&
+      give_mimext(ext, ext_size, contenttype))
+    return 1;
+  return 0;
+}
+
+// Build the local save name (save) from adr/fil; renames on collision
+// (e.g. INDEX.HTML vs index.html).
 int url_savename(lien_adrfilsave *const afs,
                 lien_adrfil *const former,
                 const char *referer_adr, const char *referer_fil, 
@@ -405,45 +569,30 @@ int url_savename(lien_adrfilsave *const afs,

  // si option check_type activée
  if (is_html < 0 && opt->check_type && !ext_chg) {
-    int ishtest = 0;
-
    if (protocol != PROTOCOL_FILE
        && protocol != PROTOCOL_FTP
      ) {
      // tester type avec requète HEAD si on ne connait pas le type du fichier
      if (!((opt->check_type == 1) && (fil[strlen(fil) - 1] == '/')))   // slash doit être html?
        if (opt->savename_delayed == HTS_SAVENAME_DELAYED_HARD ||
-            (ishtest = ishtml(opt, fil)) <
-                0) { // unsure whether it's html or a file
+            ishtml(opt, fil) < 0) { // unsure whether it's html or a file
          // lire dans le cache
-          htsblk r = cache_read_including_broken(opt, cache, adr, fil); // test uniquement
+          char BIGSTK previous_save[HTS_URLMAXSIZE * 2];
+          htsblk r;

-          if (r.statuscode != -1) {     // pas d'erreur de lecture cache
-            char s[32];
+          previous_save[0] = '\0';
+          r = cache_read_including_broken(opt, cache, adr, fil,
+                                          previous_save); // test uniquement

-            s[0] = '\0';
+          if (r.statuscode != -1) { // cache entry read OK
            hts_log_print(opt, LOG_DEBUG, "Testing link type (from cache) %s%s",
                          adr_complete, fil_complete);
            if (!HTTP_IS_REDIRECT(r.statuscode)) {
-              if (strnotempty(r.cdispo)) {        /* filename given */
-                ext_chg = 2;      /* change filename */
-                strcpybuff(ext, r.cdispo);
-              } else if (wire_patches_ext(opt, r.contenttype, fil)) {
-                if (give_mimext(s, sizeof(s),
-                                r.contenttype)) { // recognized extension
-                  ext_chg = 1;
-                  strcpybuff(ext, s);
-                }
-              }
+              const sniff_src src = {sback, NULL, adr, fil, previous_save};
+
+              ext_chg = resolve_extension(opt, &src, r.cdispo, r.contenttype,
+                                          fil, ext, sizeof(ext));
            }
-#ifdef DEFAULT_BIN_EXT
-            // no extension and potentially bogus
-            else if (ishtest == -2) {
-              ext_chg = 1;
-              strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-            }
-#endif
-            //
          } else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_HARD &&
                     is_userknowntype(opt, fil)) { /* PATCH BY BRIAN SCHRÖDER.
                              Lookup mimetype not only by extension,
@@ -467,22 +616,13 @@ int url_savename(lien_adrfilsave *const afs,
          // fail later
          else if (opt->savename_delayed != HTS_SAVENAME_DELAYED_NONE &&
                   !opt->state.stop) {
-            // Check if the file is ready in backing. We basically take the same logic as later.
-            // FIXME: we should cleanup and factorize this unholy mess
+            // Check if the file is ready in backing.
            if (headers != NULL && headers->status >= 0 && !is_redirect) {
-              if (strnotempty(headers->r.cdispo)) {        /* filename given */
-                ext_chg = 2;      /* change filename */
-                strcpybuff(ext, headers->r.cdispo);
-              } else if (wire_patches_ext(opt, headers->r.contenttype,
-                                          headers->url_fil)) {
-                char s[16];
-                if (give_mimext(
-                        s, sizeof(s),
-                        headers->r.contenttype)) { // recognized extension
-                  ext_chg = 1;
-                  strcpybuff(ext, s);
-                }
-              }
+              const sniff_src src = {sback, headers, adr, fil, NULL};
+
+              ext_chg = resolve_extension(opt, &src, headers->r.cdispo,
+                                          headers->r.contenttype,
+                                          headers->url_fil, ext, sizeof(ext));
            }
            else if (mime_type != NULL) {
              ext[0] = '\0';
@@ -500,13 +640,6 @@ int url_savename(lien_adrfilsave *const afs,
                  if (!may_unknown2(opt, mime_type, fil)) {
                    ext_chg = 1;
                  }
-#ifdef DEFAULT_BIN_EXT
-                  // no extension and potentially bogus
-                  else if (ishtml(opt, fil) == -2) {
-                    ext_chg = 1;
-                    strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-                  }
-#endif
                } else {
                  ext_chg = 0;
                }
@@ -696,30 +829,10 @@ int url_savename(lien_adrfilsave *const afs,
                    // libérer emplacement backing
                  }

-                  {             // pas d'erreur, changer type?
-                    char s[16];
-
-                    s[0] = '\0';
-                    if (strnotempty(back[b].r.cdispo)) {        /* filename given */
-                      ext_chg = 2;      /* change filename */
-                      strcpybuff(ext, back[b].r.cdispo);
-                    } else if (wire_patches_ext(opt, back[b].r.contenttype,
-                                                back[b].url_fil)) {
-                      if (give_mimext(
-                              s, sizeof(s),
-                              back[b].r.contenttype)) { // recognized extension
-                        ext_chg = 1;
-                        strcpybuff(ext, s);
-                      }
-                    }
-#ifdef DEFAULT_BIN_EXT
-                    // no extension and potentially bogus
-                    else if (ishtest == -2) {
-                      ext_chg = 1;
-                      strcpybuff(ext, DEFAULT_BIN_EXT + 1);
-                    }
-#endif
-                  }
+                  // no error: change the type?
+                  ext_chg = resolve_extension(
+                      opt, NULL, back[b].r.cdispo, back[b].r.contenttype,
+                      back[b].url_fil, ext, sizeof(ext));
                }
                // FIN Si non déplacé, forcer type?

--- a/src/htsname.h
+++ b/src/htsname.h
@@ -100,6 +100,8 @@ void standard_name(char *b, size_t bsize, const char *dot_pos,
                   const char *nom_pos, const char *fil_complete,
                   int short_ver);
 void url_savename_addstr(char *d, const char *s);
+/* Contested wire-vs-ext verdict that a body sniff could settle (htssniff.h). */
+int hts_ext_sniff_wanted(httrackp *opt, const char *wiremime, const char *file);
 char *url_md5(char *digest_buffer, const char *fil_complete);
 void url_savename_refname(const char *adr, const char *fil, char *filename);
 char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
--- a/src/htsparse.c
+++ b/src/htsparse.c
@@ -49,6 +49,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsindex.h"
 #include "htscharset.h"
 #include "htsencoding.h"
+#include "htssniff.h"

 /* external modules */
 #include "htsmodules.h"
@@ -4076,6 +4077,9 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
    while(opt->state._hts_setpause || back_pluggable_sockets_strict(sback, opt) <= 0) { // on fait la pause..
      opt->state._hts_in_html_parsing = 6;
      back_wait(sback, opt, cache, HTS_STAT.stat_timestart);
+      /* time limit (-E) exceeded: stop waiting for a socket (#481) */
+      if (!back_checkmirror(opt))
+        break;

      // Transfer rate
      engine_stats();
@@ -4695,22 +4699,26 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
            if (!RUN_CALLBACK7
                (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
                 (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
+              back_set_unlocked(sback, b);
              return -1;
            } else if (opt->state._hts_cancel || !back_checkmirror(opt)) {      // cancel 2 ou 1 (cancel parsing)
              back_delete(opt, cache, sback, b);        // cancel test
              break;
            }
          }
-        } while(
-                 /* dns/connect/request */
-                 (back[b].status >= 99 && back[b].status <= 101)
-                 ||
-                 /* For redirects, wait for request to be terminated */
-                 (HTTP_IS_REDIRECT(back[b].r.statuscode) && back[b].status > 0)
-                 ||
-                 /* Same for errors */
-                 (HTTP_IS_ERROR(back[b].r.statuscode) && back[b].status > 0)
-          );
+        } while (
+            /* dns/connect/request */
+            (back[b].status >= 99 && back[b].status <= 101) ||
+            /* For redirects, wait for request to be terminated */
+            (HTTP_IS_REDIRECT(back[b].r.statuscode) && back[b].status > 0) ||
+            /* Same for errors */
+            (HTTP_IS_ERROR(back[b].r.statuscode) && back[b].status > 0) ||
+            /* Contested type: wait for a sniffable body head (or EOF) */
+            (back[b].r.statuscode == HTTP_OK && back[b].status > 0 &&
+             strnotempty(back[b].r.cdispo) == 0 &&
+             back[b].r.size < HTS_SNIFF_LEN &&
+             hts_ext_sniff_wanted(opt, back[b].r.contenttype,
+                                  back[b].url_fil)));
        if (b >= 0) {
          back_set_unlocked(sback, b);  // Unlocked entry
        }
@@ -4845,6 +4853,9 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,

          /* Still have a back reference */
          if (b >= 0) {
+            /* patch url_sav BEFORE finalize: it records/caches under this name
+             */
+            strcpybuff(back[b].url_sav, afs->save);
            /* Finalize now as we have the type */
            if (back[b].status == STATUS_READY) {
              if (!back[b].finalized) {
@@ -4852,8 +4863,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
                back_finalize(opt, cache, sback, b);
              }
            }
-            /* Patch destination filename for direct-to-disk mode */
-            strcpybuff(back[b].url_sav, afs->save);
          }

        }                       // b >= 0
--- a/src/htsparse.h
+++ b/src/htsparse.h
@@ -175,27 +175,33 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
  /* Apply changes */ \
  * str->ptr_ = ptr

-#define WAIT_FOR_AVAILABLE_SOCKET() do { \
-  int prev = opt->state._hts_in_html_parsing; \
-  while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
-    opt->state._hts_in_html_parsing = 6; \
-    /* Wait .. */ \
-    back_wait(sback,opt,cache,0); \
-    /* Transfer rate */ \
-    engine_stats(); \
-    /* Refresh various stats */ \
-    HTS_STAT.stat_nsocket=back_nsoc(sback); \
-    HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
-    HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
-    HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
-    HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
-    HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
-    /* Check */ \
-    if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count, -1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
-      return -1; \
-    } \
-  } \
-  opt->state._hts_in_html_parsing = prev; \
-} while(0)
+#define WAIT_FOR_AVAILABLE_SOCKET()                                            \
+  do {                                                                         \
+    int prev = opt->state._hts_in_html_parsing;                                \
+    while (back_pluggable_sockets_strict(sback, opt) <= 0) {                   \
+      opt->state._hts_in_html_parsing = 6;                                     \
+      /* Wait .. */                                                            \
+      back_wait(sback, opt, cache, 0);                                         \
+      /* time limit (-E) exceeded: stop waiting for a socket (#481) */         \
+      if (!back_checkmirror(opt))                                              \
+        break;                                                                 \
+      /* Transfer rate */                                                      \
+      engine_stats();                                                          \
+      /* Refresh various stats */                                              \
+      HTS_STAT.stat_nsocket = back_nsoc(sback);                                \
+      HTS_STAT.stat_errors = fspc(opt, NULL, "error");                         \
+      HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");                     \
+      HTS_STAT.stat_infos = fspc(opt, NULL, "info");                           \
+      HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);    \
+      HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);              \
+      /* Check */                                                              \
+      if (!RUN_CALLBACK7(                                                      \
+              opt, loop, sback->lnk, sback->count, -1, ptr, opt->lien_tot,     \
+              (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {    \
+        return -1;                                                             \
+      }                                                                        \
+    }                                                                          \
+    opt->state._hts_in_html_parsing = prev;                                    \
+  } while (0)

 #endif
--- a/src/htsselftest.c
+++ b/src/htsselftest.c
@@ -52,6 +52,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsencoding.h"
 #include "htsftp.h"
 #include "htsmd5.h"
+#include "htssniff.h"
 #if HTS_USEZLIB
 #include "htszlib.h"
 #endif
@@ -1093,35 +1094,218 @@ static int st_resolve(httrackp *opt, int argc, char **argv) {
  return 0;
 }

+/* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
+   no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
+   registering an already-crawled link (dedup/collision paths). */
+/* Parse raw response-header lines and print the naming-relevant fields. */
+static int st_header(httrackp *opt, int argc, char **argv) {
+  htsblk r;
+  int i;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "header: needs at least one raw header line\n");
+    return 1;
+  }
+  memset(&r, 0, sizeof(r));
+  for (i = 0; i < argc; i++) {
+    char BIGSTK line[HTS_URLMAXSIZE * 2];
+
+    strcpybuff(line, argv[i]);
+    treathead(NULL, "www.example.com", "/", &r, line);
+  }
+  printf("contenttype=%s cdispo=%s\n", r.contenttype, r.cdispo);
+  return 0;
+}
+
+/* Decode a body argument ("hex:FFD8.." or literal text) into buf. */
+static size_t st_decode_body(const char *arg, char *buf, size_t size) {
+  size_t n = 0;
+
+  if (strncmp(arg, "hex:", 4) == 0) {
+    const char *s = arg + 4;
+
+    for (; s[0] != '\0' && s[1] != '\0' && n + 1 < size; s += 2) {
+      unsigned int byte;
+
+      if (sscanf(s, "%2x", &byte) != 1)
+        break;
+      buf[n++] = (char) byte;
+    }
+  } else {
+    n = strlen(arg);
+    if (n >= size)
+      n = size - 1;
+    memcpy(buf, arg, n);
+  }
+  buf[n] = '\0';
+  return n;
+}
+
+static int st_sniff(httrackp *opt, int argc, char **argv) {
+  char BIGSTK body[1024];
+  size_t n;
+
+  (void) opt;
+  if (argc < 2) {
+    fprintf(stderr, "sniff: needs a content-type and a body\n");
+    return 1;
+  }
+  n = st_decode_body(argv[1], body, sizeof(body));
+  printf("sniff: known=%d consistent=%d\n",
+         hts_sniff_mime_known(argv[0]) == HTS_TRUE,
+         hts_sniff_mime_consistent(body, n, argv[0]) == HTS_TRUE);
+  return 0;
+}
+
 static int st_savename(httrackp *opt, int argc, char **argv) {
  lien_adrfilsave afs;
  cache_back cache;
  struct_back *sback;
  hash_struct hash;
  lien_back headers;
+  const char *adr = "www.example.com";
+  const char *cdispo = NULL;
+  const char *body = NULL;
+  const char *cached = NULL;
+  const char *bodyfile = "st-savename-body.tmp";
+  int statuscode = HTTP_OK, status = 0;
+  int i;

  if (argc < 2) {
    fprintf(stderr, "savename: needs a fil and a content-type\n");
    return 1;
  }
+  /* knobs first: hash_init and the prior links depend on them */
+  for (i = 2; i < argc; i++) {
+    const char *const a = argv[i];
+
+    if (strncmp(a, "adr=", 4) == 0)
+      adr = a + 4;
+    else if (strncmp(a, "cdispo=", 7) == 0)
+      cdispo = a + 7;
+    else if (strncmp(a, "statuscode=", 11) == 0)
+      statuscode = atoi(a + 11);
+    else if (strncmp(a, "status=", 7) == 0)
+      status = atoi(a + 7);
+    else if (strncmp(a, "strip=", 6) == 0)
+      StringCopy(opt->strip_query, a + 6);
+    else if (strncmp(a, "urlhack=", 8) == 0)
+      opt->urlhack = atoi(a + 8) ? HTS_TRUE : HTS_FALSE;
+    else if (strncmp(a, "no-www=", 7) == 0)
+      opt->no_www_dedup = atoi(a + 7) ? HTS_TRUE : HTS_FALSE;
+    else if (strncmp(a, "no-slash=", 9) == 0)
+      opt->no_slash_dedup = atoi(a + 9) ? HTS_TRUE : HTS_FALSE;
+    else if (strncmp(a, "no-query=", 9) == 0)
+      opt->no_query_dedup = atoi(a + 9) ? HTS_TRUE : HTS_FALSE;
+    else if (strncmp(a, "n83=", 4) == 0)
+      opt->savename_83 = atoi(a + 4);
+    else if (strncmp(a, "type=", 5) == 0)
+      opt->savename_type = atoi(a + 5);
+    else if (strncmp(a, "body=", 5) == 0)
+      body = a + 5;
+    else if (strncmp(a, "cached=", 7) == 0)
+      cached = a + 7;
+    else if (strncmp(a, "prior=", 6) != 0) {
+      fprintf(stderr, "savename: unknown arg '%s'\n", a);
+      return 1;
+    }
+  }
  memset(&afs, 0, sizeof(afs));
-  strcpybuff(afs.af.adr, "www.example.com");
+  strcpybuff(afs.af.adr, adr);
  strcpybuff(afs.af.fil, argv[0]);

  memset(&cache, 0, sizeof(cache));
-  cache.hashtable = (void *) coucal_new(0);
+  if (cached != NULL) { /* cached=<content-type>|<save name> */
+    char *dup = strdupt(cached);
+    char *const sep = strchr(dup, '|');
+    char locbuf[64] = "";
+    htsblk cr;
+
+    if (sep == NULL) {
+      fprintf(stderr, "savename: cached needs ctype|save\n");
+      return 1;
+    }
+    *sep = '\0';
+    /* one-entry cache in cwd, reopened read-only; body is PNG magic on
+       purpose: only the recorded name (X-Save) may drive the naming */
+    StringCopy(opt->path_log, "");
+    cache.type = 1;
+    cache.log = cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache_init(&cache, opt);
+    hts_init_htsblk(&cr);
+    cr.statuscode = HTTP_OK;
+    strcpybuff(cr.msg, "OK");
+    strcpybuff(cr.contenttype, dup);
+    cr.location = locbuf;
+    cr.adr = strdupt("\x89PNG\r\n\x1a\n");
+    cr.size = 8;
+    cache_add(opt, &cache, &cr, adr, argv[0], sep + 1, 1, NULL);
+    freet(cr.adr);
+    if (cache.zipOutput != NULL) {
+      zipClose(cache.zipOutput, NULL);
+      cache.zipOutput = NULL;
+    }
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.ro = 1;
+    cache_init(&cache, opt);
+    freet(dup);
+  } else {
+    cache.hashtable = (void *) coucal_new(0);
+  }

  sback = back_new(opt, opt->maxsoc * 32 + 1024);
+  /* same wiring as hts_mirror (htscore.c) */
  hash_init(opt, &hash, opt->urlhack);
+  hash.liens = (const lien_url *const *const *) &opt->liens;
+  opt->hash = &hash;
+  hts_record_init(opt);
+
+  for (i = 2; i < argc; i++) {
+    if (strncmp(argv[i], "prior=", 6) == 0) {
+      char *dup = strdupt(argv[i] + 6);
+      char *const p1 = strchr(dup, '|');
+      char *const p2 = p1 != NULL ? strchr(p1 + 1, '|') : NULL;
+
+      if (p2 == NULL) {
+        fprintf(stderr, "savename: prior needs adr|fil|sav\n");
+        return 1;
+      }
+      *p1 = *p2 = '\0';
+      if (!hts_record_link(opt, dup, p1 + 1, p2 + 1, "", "", NULL))
+        return 1;
+      freet(dup);
+    }
+  }

  memset(&headers, 0, sizeof(headers));
-  headers.status = 0;
-  headers.r.statuscode = HTTP_OK;
+  headers.status = status;
+  headers.r.statuscode = statuscode;
  strcpybuff(headers.r.contenttype, argv[1]);
+  if (cdispo != NULL)
+    strcpybuff(headers.r.cdispo, cdispo);
  strcpybuff(headers.url_fil, argv[0]);
+  if (body != NULL) { /* leading body bytes, read via url_sav */
+    char BIGSTK data[1024];
+    const size_t n = st_decode_body(body, data, sizeof(data));
+    FILE *const fp = fopen(bodyfile, "wb");
+
+    if (fp == NULL || fwrite(data, 1, n, fp) != n) {
+      fprintf(stderr, "savename: can not write %s\n", bodyfile);
+      return 1;
+    }
+    fclose(fp);
+    strcpybuff(headers.url_sav, bodyfile);
+  }

  url_savename(&afs, NULL, NULL, NULL, opt, sback, &cache, &hash, 0, 0,
               &headers);
+  if (body != NULL)
+    (void) UNLINK(bodyfile);
  printf("savename: %s\n", afs.save);
  return 0;
 }
@@ -1924,8 +2108,12 @@ static const struct selftest_entry {
     st_relative},
    {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
     st_resolve},
-    {"savename", "<fil> <content-type>", "local save-name for a URL",
-     st_savename},
+    {"header", "<raw-header-line> ...", "response header-line parsing",
+     st_header},
+    {"savename", "<fil> <content-type> [key=value ...]",
+     "local save-name for a URL", st_savename},
+    {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",
+     st_sniff},
    {"cache", "<dir>", "cache read/write round-trip self-test", st_cache},
    {"cache-golden", "<dir> [regen]", "frozen cache-format read self-test",
     st_cache_golden},
--- a/src/htssniff.c
+++ b/src/htssniff.c
@@ -0,0 +1,352 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998-2017 Xavier Roche and other contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Important notes:
+
+- We hereby ask people using this source NOT to use it in purpose of grabbing
+emails addresses, or collecting any other private information on persons.
+This would disgrace our work, and spoil the many hours we spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: MIME magic-byte consistency checks                     */
+/* Author: Xavier Roche                                         */
+/* ------------------------------------------------------------ */
+
+#include "htssniff.h"
+
+#include <string.h>
+
+#include "htslib.h"
+
+/* One magic rule: `len` bytes at `off` confirm `mime`. */
+typedef struct sniff_magic {
+  const char *mime;
+  unsigned short off;
+  unsigned char len;
+  const char *bytes;
+} sniff_magic;
+
+/* Direction is mime -> magic (verify a claim, never classify); types with
+   no reliable magic (plain text, css, js..) are deliberately absent. Patterns
+   follow the WHATWG MIME Sniffing Standard tables where it defines them
+   (https://mimesniff.spec.whatwg.org/); the rest covers httrack's wider MIME
+   set. Spec-only types absent from our MIME tables (EOT, font/collection)
+   are omitted as unreachable. */
+static const sniff_magic sniff_table[] = {
+    /* images */
+    {"image/jpeg", 0, 3, "\xff\xd8\xff"},
+    {"image/pipeg", 0, 3, "\xff\xd8\xff"},
+    {"image/pjpeg", 0, 3, "\xff\xd8\xff"},
+    {"image/png", 0, 8, "\x89PNG\r\n\x1a\n"},
+    {"image/gif", 0, 6, "GIF87a"},
+    {"image/gif", 0, 6, "GIF89a"},
+    {"image/bmp", 0, 2, "BM"},
+    {"image/tiff", 0, 4, "II*\0"},
+    {"image/tiff", 0, 4, "MM\0*"},
+    {"image/x-icon", 0, 4, "\0\0\1\0"},
+    {"image/x-icon", 0, 4, "\0\0\2\0"}, /* Windows cursor, per the spec */
+    {"image/x-portable-bitmap", 0, 2, "P1"},
+    {"image/x-portable-bitmap", 0, 2, "P4"},
+    {"image/x-portable-pixmap", 0, 2, "P3"},
+    {"image/x-portable-pixmap", 0, 2, "P6"},
+    {"image/x-xpixmap", 0, 9, "/* XPM */"},
+    {"image/x-xbitmap", 0, 7, "#define"},
+    {"image/x-rgb", 0, 2, "\x01\xda"},
+    {"image/x-cmu-raster", 0, 4, "\xf1\x00\x40\xbb"},
+    /* audio */
+    {"audio/mpeg", 0, 3, "ID3"},
+    {"audio/basic", 0, 4, ".snd"},
+    {"audio/mid", 0, 8, "MThd\0\0\0\6"},
+    {"audio/midi", 0, 8, "MThd\0\0\0\6"},
+    {"audio/x-pn-realaudio", 0, 4, ".ra\xfd"},
+    {"audio/x-pn-realaudio", 0, 4, ".RMF"},
+    {"audio/x-pn-realaudio-plugin", 0, 4, ".ra\xfd"},
+    {"audio/x-pn-realaudio-plugin", 0, 4, ".RMF"},
+    {"audio/flac", 0, 4, "fLaC"},
+    {"audio/aac", 0, 4, "ADIF"},
+    /* video */
+    {"video/mpeg", 0, 4, "\x00\x00\x01\xba"},
+    {"video/mpeg", 0, 4, "\x00\x00\x01\xb3"},
+    {"video/x-sgi-movie", 0, 4, "MOVI"},
+    /* archives / compression */
+    {"application/x-gzip", 0, 3, "\x1f\x8b\x08"},
+    {"multipart/x-gzip", 0, 3, "\x1f\x8b\x08"},
+    {"application/x-compressed", 0, 3, "\x1f\x8b\x08"},
+    {"application/x-compress", 0, 2, "\x1f\x9d"},
+    {"application/x-bzip2", 0, 3, "BZh"},
+    {"application/x-7z-compressed", 0, 6, "7z\xbc\xaf\x27\x1c"},
+    /* 6-byte prefix common to RAR4 (spec) and RAR5 */
+    {"application/x-rar-compressed", 0, 6, "Rar!\x1a\x07"},
+    {"application/zstd", 0, 4, "\x28\xb5\x2f\xfd"},
+    {"application/arj", 0, 2, "\x60\xea"},
+    {"application/x-cpio", 0, 6, "070701"},
+    {"application/x-cpio", 0, 6, "070707"},
+    {"application/x-cpio", 0, 2, "\xc7\x71"},
+    {"application/x-sv4cpio", 0, 6, "070701"},
+    {"application/x-sv4crc", 0, 6, "070702"},
+    {"application/x-stuffit", 0, 8, "StuffIt "},
+    {"application/x-stuffit", 0, 4, "SIT!"},
+    {"application/mac-binhex40", 0, 10, "(This file"},
+    /* documents */
+    {"application/pdf", 0, 5, "%PDF-"},
+    {"application/postscript", 0, 2, "%!"},
+    {"application/rtf", 0, 5, "{\\rtf"},
+    {"application/x-dvi", 0, 2, "\xf7\x02"},
+    {"application/x-hdf", 0, 4, "\x0e\x03\x13\x01"},
+    {"application/x-hdf", 0, 8, "\x89HDF\r\n\x1a\n"},
+    {"application/x-netcdf", 0, 4, "CDF\x01"},
+    {"application/x-netcdf", 0, 4, "CDF\x02"},
+    {"application/x-msaccess", 0, 19, "\0\1\0\0Standard Jet DB"},
+    /* fonts */
+    {"font/woff", 0, 4, "wOFF"},
+    {"font/woff2", 0, 4, "wOF2"},
+    {"font/ttf", 0, 4, "\0\1\0\0"},
+    {"font/ttf", 0, 4, "true"},
+    {"font/otf", 0, 4, "OTTO"},
+    /* misc */
+    {"application/x-shockwave-flash", 0, 3, "FWS"},
+    {"application/x-shockwave-flash", 0, 3, "CWS"},
+    {"application/x-shockwave-flash", 0, 3, "ZWS"},
+    {"application/futuresplash", 0, 3, "FWS"},
+    {"application/x-director", 0, 4, "RIFX"},
+    {"application/x-director", 0, 4, "XFIR"},
+    {"application/x-java-vm", 0, 4, "\xca\xfe\xba\xbe"},
+    {"application/wasm", 0, 4, "\0asm"},
+    {"application/x-msmetafile", 0, 4, "\xd7\xcd\xc6\x9a"},
+    {"application/x-msmetafile", 0, 4, "\x01\x00\x09\x00"},
+    {"application/x-x509-ca-cert", 0, 2, "\x30\x82"},
+    {"application/x-pkcs12", 0, 2, "\x30\x82"},
+    {"application/x-pkcs7-mime", 0, 2, "\x30\x82"},
+    {"application/x-pkcs7-signature", 0, 2, "\x30\x82"},
+    {"application/x-pkcs7-certificates", 0, 2, "\x30\x82"},
+    {"x-world/x-vrml", 0, 5, "#VRML"},
+    {"application/x-bittorrent", 0, 11, "d8:announce"},
+    {"drawing/x-dwf", 0, 4, "(DWF"},
+    {"application/acad", 0, 4, "AC10"},
+    {NULL, 0, 0, NULL}};
+
+/* MIME families sharing a container magic */
+static const char *const zip_mimes[] = {
+    "application/zip", "application/x-zip-compressed", "multipart/x-zip", NULL};
+static const char *const zip_mime_prefixes[] = {
+    "application/vnd.openxmlformats-officedocument.",
+    "application/vnd.oasis.opendocument.", NULL};
+static const char *const ole_mimes[] = {"application/msword",
+                                        "application/excel",
+                                        "application/vnd.ms-excel",
+                                        "application/powerpoint",
+                                        "application/vnd.ms-powerpoint",
+                                        "application/vnd.ms-project",
+                                        "application/vnd.ms-works",
+                                        "application/x-msmoney",
+                                        "application/x-mspublisher",
+                                        NULL};
+static const char *const tar_mimes[] = {
+    "application/x-tar", "application/x-ustar", "application/x-gtar", NULL};
+static const char *const ogg_mimes[] = {"application/ogg", "audio/ogg",
+                                        "video/ogg", "audio/opus", NULL};
+static const char *const ebml_mimes[] = {"video/webm", "audio/webm", NULL};
+/* ISO-BMFF, any 'ftyp' brand: containers overlap too much to split */
+static const char *const bmff_mimes[] = {"video/mp4", "audio/mp4",
+                                         "video/quicktime", NULL};
+static const char *const avif_mimes[] = {"image/avif", NULL};
+static const char *const heic_mimes[] = {"image/heic", NULL};
+static const char *const asf_mimes[] = {"video/x-ms-asf", "video/x-ms-wmv",
+                                        "video/x-la-asf", NULL};
+static const char *const xml_mimes[] = {"application/xml", "text/xml",
+                                        "image/svg+xml", "image/svg-xml", NULL};
+static const char *const svg_mimes[] = {"image/svg+xml", "image/svg-xml", NULL};
+static const char *const html_mimes[] = {"text/html", NULL};
+static const char *const pem_mimes[] = {
+    "application/x-x509-ca-cert", "application/x-pkcs7-certificates",
+    "application/x-pkcs7-mime", "application/x-pkcs7-signature", NULL};
+
+static hts_boolean mime_in(const char *const *list, const char *mime) {
+  size_t i;
+
+  for (i = 0; list[i] != NULL; i++)
+    if (strfield2(list[i], mime))
+      return HTS_TRUE;
+  return HTS_FALSE;
+}
+
+static hts_boolean mime_in_prefix(const char *const *list, const char *mime) {
+  size_t i;
+
+  for (i = 0; list[i] != NULL; i++)
+    if (strfield(mime, list[i]))
+      return HTS_TRUE;
+  return HTS_FALSE;
+}
+
+static hts_boolean has_bytes(const unsigned char *d, size_t n, size_t off,
+                             const char *bytes, size_t len) {
+  /* overflow-safe: untrusted n alone on one side */
+  return n >= off && len <= n - off && memcmp(d + off, bytes, len) == 0
+             ? HTS_TRUE
+             : HTS_FALSE;
+}
+
+static unsigned char ascii_lower(unsigned char c) {
+  return c >= 'A' && c <= 'Z' ? (unsigned char) (c + 32) : c;
+}
+
+/* Case-insensitive text prefix after an optional UTF-8 BOM and whitespace. */
+static hts_boolean has_text_prefix(const unsigned char *d, size_t n,
+                                   const char *prefix) {
+  const size_t len = strlen(prefix);
+  size_t i, k;
+
+  i = n >= 3 && memcmp(d, "\xef\xbb\xbf", 3) == 0 ? 3 : 0;
+  while (i < n && (d[i] == ' ' || d[i] == '\t' || d[i] == '\r' || d[i] == '\n'))
+    i++;
+  if (len > n - i) /* i <= n from the loop above */
+    return HTS_FALSE;
+  for (k = 0; k < len; k++)
+    if (ascii_lower(d[i + k]) != ascii_lower((unsigned char) prefix[k]))
+      return HTS_FALSE;
+  return HTS_TRUE;
+}
+
+typedef enum sniff_op {
+  SNIFF_QUERY_KNOWN, /* is any rule defined for this MIME? */
+  SNIFF_QUERY_MATCH  /* do the bytes confirm this MIME? */
+} sniff_op;
+
+/* Single walk for both queries so the rule set can't drift apart. */
+static hts_boolean sniff_eval(sniff_op op, const unsigned char *d, size_t n,
+                              const char *mime) {
+  size_t i;
+
+/* KNOWN short-circuits; MATCH tests the magic */
+#define SNIFF_RULE(cond)                                                       \
+  do {                                                                         \
+    if (op == SNIFF_QUERY_KNOWN)                                               \
+      return HTS_TRUE;                                                         \
+    if (cond)                                                                  \
+      return HTS_TRUE;                                                         \
+  } while (0)
+
+  for (i = 0; sniff_table[i].mime != NULL; i++) {
+    if (strfield2(sniff_table[i].mime, mime)) {
+      SNIFF_RULE(has_bytes(d, n, sniff_table[i].off, sniff_table[i].bytes,
+                           sniff_table[i].len));
+    }
+  }
+  if (mime_in(zip_mimes, mime) || mime_in_prefix(zip_mime_prefixes, mime)) {
+    SNIFF_RULE(has_bytes(d, n, 0, "PK\3\4", 4) ||
+               has_bytes(d, n, 0, "PK\5\6", 4));
+  }
+  if (mime_in(ole_mimes, mime)) {
+    SNIFF_RULE(has_bytes(d, n, 0, "\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1", 8));
+  }
+  if (mime_in(tar_mimes, mime)) {
+    SNIFF_RULE(has_bytes(d, n, 257, "ustar", 5));
+  }
+  if (mime_in(ogg_mimes, mime)) {
+    SNIFF_RULE(has_bytes(d, n, 0, "OggS\0", 5));
+  }
+  if (mime_in(ebml_mimes, mime)) {
+    SNIFF_RULE(has_bytes(d, n, 0, "\x1a\x45\xdf\xa3", 4));
+  }
+  if (mime_in(bmff_mimes, mime)) {
+    SNIFF_RULE(has_bytes(d, n, 4, "ftyp", 4));
+  }
+  if (mime_in(avif_mimes, mime)) {
+    SNIFF_RULE(has_bytes(d, n, 4, "ftypavif", 8) ||
+               has_bytes(d, n, 4, "ftypavis", 8));
+  }
+  if (mime_in(heic_mimes, mime)) {
+    SNIFF_RULE(
+        has_bytes(d, n, 4, "ftyphei", 7) || has_bytes(d, n, 4, "ftyphev", 7) ||
+        has_bytes(d, n, 4, "ftypmif1", 8) || has_bytes(d, n, 4, "ftypmsf1", 8));
+  }
+  if (mime_in(asf_mimes, mime)) {
+    SNIFF_RULE(has_bytes(d, n, 0, "\x30\x26\xb2\x75\x8e\x66\xcf\x11", 8));
+  }
+  if (strfield2("audio/x-wav", mime)) {
+    SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) && has_bytes(d, n, 8, "WAVE", 4));
+  }
+  if (strfield2("video/x-msvideo", mime)) {
+    SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) && has_bytes(d, n, 8, "AVI ", 4));
+  }
+  if (strfield2("image/webp", mime)) {
+    SNIFF_RULE(has_bytes(d, n, 0, "RIFF", 4) &&
+               has_bytes(d, n, 8, "WEBPVP", 6));
+  }
+  if (strfield2("image/x-portable-anymap", mime)) {
+    SNIFF_RULE(n >= 2 && d[0] == 'P' && d[1] >= '1' && d[1] <= '6');
+  }
+  if (strfield2("audio/x-aiff", mime)) {
+    SNIFF_RULE(
+        has_bytes(d, n, 0, "FORM", 4) &&
+        (has_bytes(d, n, 8, "AIFF", 4) || has_bytes(d, n, 8, "AIFC", 4)));
+  }
+  if (strfield2("audio/mpeg", mime)) {
+    /* MPEG audio frame sync (11 bits), valid layer and bitrate fields */
+    SNIFF_RULE(n >= 2 && d[0] == 0xff && (d[1] & 0xe0) == 0xe0 &&
+               (d[1] & 0x06) != 0);
+  }
+  if (strfield2("audio/aac", mime)) {
+    /* ADTS sync */
+    SNIFF_RULE(n >= 2 && d[0] == 0xff && (d[1] & 0xf6) == 0xf0);
+  }
+  if (strfield2("video/mp2t", mime)) {
+    SNIFF_RULE(n >= 1 && d[0] == 0x47 && (n <= 188 || d[188] == 0x47));
+  }
+  if (mime_in(xml_mimes, mime)) {
+    SNIFF_RULE(has_text_prefix(d, n, "<?xml"));
+  }
+  if (mime_in(svg_mimes, mime)) {
+    SNIFF_RULE(has_text_prefix(d, n, "<svg") ||
+               has_text_prefix(d, n, "<!DOCTYPE svg"));
+  }
+  if (mime_in(html_mimes, mime)) {
+    SNIFF_RULE(has_text_prefix(d, n, "<!DOCTYPE") ||
+               has_text_prefix(d, n, "<html") ||
+               has_text_prefix(d, n, "<head"));
+  }
+  if (mime_in(pem_mimes, mime)) {
+    SNIFF_RULE(has_text_prefix(d, n, "-----BEGIN"));
+  }
+  if (strfield2("audio/x-mpegurl", mime)) {
+    SNIFF_RULE(has_text_prefix(d, n, "#EXTM3U"));
+  }
+  if (strfield2("text/x-vcard", mime)) {
+    SNIFF_RULE(has_text_prefix(d, n, "BEGIN:VCARD"));
+  }
+#undef SNIFF_RULE
+  return HTS_FALSE;
+}
+
+hts_boolean hts_sniff_mime_known(const char *mime) {
+  if (mime == NULL || *mime == '\0')
+    return HTS_FALSE;
+  return sniff_eval(SNIFF_QUERY_KNOWN, NULL, 0, mime);
+}
+
+hts_boolean hts_sniff_mime_consistent(const void *data, size_t size,
+                                      const char *mime) {
+  if (data == NULL || size == 0 || mime == NULL || *mime == '\0')
+    return HTS_FALSE;
+  return sniff_eval(SNIFF_QUERY_MATCH, (const unsigned char *) data, size,
+                    mime);
+}
--- a/src/htssniff.h
+++ b/src/htssniff.h
@@ -0,0 +1,50 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 1998-2017 Xavier Roche and other contributors
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Important notes:
+
+- We hereby ask people using this source NOT to use it in purpose of grabbing
+emails addresses, or collecting any other private information on persons.
+This would disgrace our work, and spoil the many hours we spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: MIME magic-byte consistency checks                     */
+/* Author: Xavier Roche                                         */
+/* ------------------------------------------------------------ */
+
+#ifndef HTSSNIFF_DEFH
+#define HTSSNIFF_DEFH
+
+#include <stddef.h>
+#include "htsglobal.h"
+
+/* Leading-body window read to arbitrate a wire/extension MIME conflict. */
+#define HTS_SNIFF_LEN 512
+
+/* Can a magic rule ever confirm this MIME? (whether sniffing is worth it) */
+hts_boolean hts_sniff_mime_known(const char *mime);
+
+/* TRUE when the leading body bytes are consistent with the claimed MIME;
+   FALSE on unknown MIME, unknown magic, or too-short data (fail-safe). */
+hts_boolean hts_sniff_mime_consistent(const void *data, size_t size,
+                                      const char *mime);
+
+#endif
--- a/tests/01_engine-header.test
+++ b/tests/01_engine-header.test
@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Response header-line parsing (treathead via -#test=header <raw-line> ...).
+# Isolates the wire layer from url_savename, which strips traversal on its own.
+
+hdr() {
+    local want="$1"
+    shift
+    out="$(httrack -O /dev/null -#test=header "$@" | grep '^contenttype=')"
+    test "$out" == "$want" || {
+        echo "FAIL: $* -> '$out' (want '$want')"
+        exit 1
+    }
+}
+
+hdr 'contenttype=application/pdf cdispo=' 'Content-Type: application/pdf'
+
+# filename= is honored quoted or bare.
+hdr 'contenttype= cdispo=report.pdf' \
+    'Content-Disposition: attachment; filename="report.pdf"'
+hdr 'contenttype= cdispo=report.pdf' \
+    'Content-Disposition: attachment; filename=report.pdf'
+
+# Path components in the filename are dropped on the wire (RFC 2616).
+hdr 'contenttype= cdispo=evil.pdf' \
+    'Content-Disposition: attachment; filename="../../evil.pdf"'
--- a/tests/01_engine-savename.test
+++ b/tests/01_engine-savename.test
@@ -3,13 +3,38 @@

 set -euo pipefail

-# Local save-name extension resolution (url_savename via -#test=savename <fil> <content-type>).
-# Asserts on the basename of "savename: <path>".
+# Local save-name resolution (url_savename via -#test=savename <fil> <content-type> [key=value ...]).
+# name() asserts on the basename, full() on the whole path; prior= registers an
+# already-crawled link whose sav is rooted under the -O path (/dev/null here).
+
+# resolve httrack before cd: make check puts a RELATIVE ../src on PATH
+httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
+
+# scratch dir: body= and cached= write temp files (st-savename-body.tmp, hts-cache/)
+scratch=$(mktemp -d)
+trap 'rm -rf "$scratch"' EXIT
+cd "$scratch"
+
+run() {
+    "$httrack_bin" -O /dev/null -#test=savename "$@" | sed -n 's/^savename: //p'
+}

 name() {
-    out="$(httrack -O /dev/null -#test=savename "$1" "$2" | sed -n 's/^savename: //p')"
-    test "${out##*/}" == "$3" || {
-        echo "FAIL: '$1' '$2' -> '$out' (want '$3')"
+    local fil="$1" ctype="$2" want="$3"
+    shift 3
+    out="$(run "$fil" "$ctype" "$@")"
+    test "${out##*/}" == "$want" || {
+        echo "FAIL: '$fil' '$ctype' $* -> '$out' (want '$want')"
+        exit 1
+    }
+}
+
+full() {
+    local fil="$1" ctype="$2" want="$3"
+    shift 3
+    out="$(run "$fil" "$ctype" "$@")"
+    test "$out" == "$want" || {
+        echo "FAIL: '$fil' '$ctype' $* -> '$out' (want '$want')"
        exit 1
    }
 }
@@ -39,3 +64,95 @@ name '/types/data.json' 'application/json' 'data.json'

 # Agreeing type must not rewrite the extension's casing (no strip-and-reappend).
 name '/x.JPG' 'image/jpeg' 'x.JPG'
+
+# A Content-Disposition filename replaces the URL name outright.
+name '/x.php' 'application/pdf' 'report.pdf' cdispo=report.pdf
+name '/download' 'text/html' 'setup.exe' cdispo=setup.exe
+
+# Reserved characters in a hostile Content-Disposition name are sanitized.
+name '/x.php' 'application/pdf' 'set_up.exe' 'cdispo=set:up.exe'
+
+# The md5-of-query suffix lands inside a Content-Disposition name too.
+name '/x.php?id=1' 'application/pdf' 'report681a.pdf' cdispo=report.pdf
+
+# Still-downloading path (status=-1): mime drives the ext, cdispo is ignored
+# there (the deliberately unfolded 4th resolve_extension variant).
+name '/x.pdf' 'text/html' 'x.html' status=-1
+name '/x.html' 'text/html' 'x.html' status=-1
+name '/x.php' 'application/pdf' 'x.pdf' status=-1 cdispo=report.pdf
+
+# Contested type (wire disagrees with a specific ext): magic bytes proving the
+# extension right keep it, anything else trusts the wire as before.
+name '/photo.jpg' 'image/png' 'photo.jpg' body=hex:FFD8FFE000104A46
+name '/photo.jpg' 'image/png' 'photo.png' body=hex:89504E470D0A1A0A
+name '/photo.jpg' 'image/png' 'photo.png'
+name '/doc.pdf' 'text/html' 'doc.pdf' body=hex:255044462D312E34
+name '/doc.pdf' 'text/html' 'doc.html' 'body=<html><body>soft 404</body></html>'
+name '/style.css' 'image/png' 'style.png' 'body=body { }' # no rule for css: wire wins
+
+# A redirect answer resolves nothing: delayed placeholder name.
+name '/x.php' 'text/html' 'x.0.delayed' statuscode=301
+
+# Root and query-only URLs get index + the md5-of-query suffix.
+name '/' 'text/html' 'index.html'
+name '/?a=1' 'text/html' 'index3872.html'
+
+# Same URL crawled before: reuse its sav verbatim (case preserved).
+full '/X.PHP' 'text/html' 'www.example.com/CASE.HTML' \
+    'prior=www.example.com|/X.PHP|www.example.com/CASE.HTML'
+
+# Another URL owns the name: collision suffix -2, then -3, case-insensitively.
+name '/x.php' 'text/html' 'x-2.html' \
+    'prior=www.example.com|/other.html|/dev/null/www.example.com/x.html'
+name '/x.php' 'text/html' 'x-3.html' \
+    'prior=www.example.com|/o1.html|/dev/null/www.example.com/x.html' \
+    'prior=www.example.com|/o2.html|/dev/null/www.example.com/x-2.html'
+name '/INDEX.HTML' 'text/html' 'INDEX-2.HTML' \
+    'prior=www.example.com|/index.html|/dev/null/www.example.com/index.html'
+
+# Same basename in another directory is NOT a collision.
+name '/x.php' 'text/html' 'x.html' \
+    'prior=www.example.com|/sub/x.html|/dev/null/www.example.com/sub/x.html'
+
+# 8-3 modes: DOS truncates every component to 8+3, ISO9660 level 2 to 31.
+full '/directory-long/verylongfilename.html' 'text/html' \
+    '/dev/null/EXAMPLE/DIRECTOR/VERYLONG.HTM' n83=1
+full '/directory-long/verylongfilename.html' 'text/html' \
+    '/dev/null/EXAMPLE_C/DIRECTORY_LONG/VERYLONGFILENAME.HTM' n83=2
+name '/verylongfilename.php' 'text/html' 'VERYLO-2.HTM' n83=1 \
+    'prior=www.example.com|/other.html|/dev/null/EXAMPLE/VERYLONG.HTM'
+
+# urlhack dedup (#271): // collapse and www-strip map to the prior link's sav;
+# the per-feature negatives opt out and take a fresh name.
+full '/a//b.php' 'text/html' '/dev/null/www.example.com/a/PRIOR.html' \
+    'prior=www.example.com|/a/b.php|/dev/null/www.example.com/a/PRIOR.html'
+full '/a//b.php' 'text/html' '/dev/null/www.example.com/a/b.html' no-slash=1 \
+    'prior=www.example.com|/a/b.php|/dev/null/www.example.com/a/PRIOR.html'
+full '/w.php' 'text/html' '/dev/null/www.example.com/W-PRIOR.html' adr=example.com \
+    'prior=www.example.com|/w.php|/dev/null/www.example.com/W-PRIOR.html'
+full '/w.php' 'text/html' '/dev/null/example.com/w.html' adr=example.com no-www=1 \
+    'prior=www.example.com|/w.php|/dev/null/www.example.com/W-PRIOR.html'
+
+# Distinct URLs must stay distinct under urlhack (no over-normalization).
+full '/a//b.php' 'text/html' '/dev/null/www.example.com/a/b.html' \
+    'prior=www.example.com|/a/c.php|/dev/null/www.example.com/a/C-PRIOR.html'
+
+# --strip-query (#112): stripped key dedups onto the prior sav; without the
+# option the same URLs stay distinct.
+full '/page.php?id=3&sid=42' 'text/html' '/dev/null/www.example.com/PAGE-PRIOR.html' \
+    strip=sid 'prior=www.example.com|/page.php?id=3|/dev/null/www.example.com/PAGE-PRIOR.html'
+full '/page.php?id=3&sid=42' 'text/html' '/dev/null/www.example.com/page475b.html' \
+    'prior=www.example.com|/page.php?id=3|/dev/null/www.example.com/PAGE-PRIOR.html'
+
+# A kept key that differs must still block the dedup (no over-stripping).
+full '/page.php?id=3&sid=42' 'text/html' '/dev/null/www.example.com/page475b.html' \
+    strip=sid 'prior=www.example.com|/page.php?id=4|/dev/null/www.example.com/PAGE-PRIOR.html'
+
+# Hostile fils stay rooted under the mirror: ../ (raw or %2e-encoded) drops out,
+# control characters become spaces, oversized names cap at 210 chars (the cap
+# can chop the extension off entirely).
+full '/../../etc/passwd' 'text/html' '/dev/null/www.example.com///etc/passwd.html'
+full '/%2e%2e/%2e%2e/etc/passwd' 'text/html' '/dev/null/www.example.com///etc/passwd.html'
+full '/x.php' 'application/pdf' '/dev/null/www.example.com///evil.exe' 'cdispo=../../evil.exe'
+name $'/evil\rname\t.php' 'text/html' 'evil name .html'
+name "/$(printf 'a%.0s' {1..300}).php" 'text/html' "$(printf 'a%.0s' {1..210})"
--- a/tests/01_engine-sniff.test
+++ b/tests/01_engine-sniff.test
@@ -0,0 +1,87 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# MIME magic consistency (-#test=sniff <content-type> <hex:..|text>), the
+# tie-break behind htsname's wire-vs-extension naming.
+
+chk() {
+    local mime="$1" body="$2" want="$3"
+    out="$(httrack -#test=sniff "$mime" "$body" | sed -n 's/^sniff: //p')"
+    test "$out" == "$want" || {
+        echo "FAIL: '$mime' '$body' -> '$out' (want '$want')"
+        exit 1
+    }
+}
+
+yes='known=1 consistent=1'
+no='known=1 consistent=0'
+unk='known=0 consistent=0'
+
+# images
+chk image/jpeg hex:FFD8FFE000104A46 "$yes"
+chk image/png hex:89504E470D0A1A0A "$yes"
+chk image/png hex:FFD8FFE000104A46 "$no" # jpeg bytes are not a png
+chk image/gif 'GIF89a' "$yes"
+chk image/bmp 'BMxxxx' "$yes"
+chk image/tiff hex:49492A00 "$yes"
+chk image/tiff hex:4D4D002A "$yes" # both endians
+chk image/x-icon hex:00000100 "$yes"
+chk image/x-icon hex:00000200 "$yes" # Windows cursor, spec maps to x-icon
+chk image/webp 'RIFFxxxxWEBPVP' "$yes"
+chk image/webp 'RIFFxxxxWAVE' "$no" # riff subtype discriminates
+chk image/avif hex:0000001C6674797061766966 "$yes"
+chk image/avif hex:0000001C6674797068656963 "$no" # heic brand is not avif
+chk image/heic hex:0000001C6674797068656963 "$yes"
+chk image/svg+xml '<svg xmlns="x">' "$yes"
+chk image/svg+xml $'\xef\xbb\xbf  <?xml version="1.0"?>' "$yes" # BOM+ws skip
+
+# audio / video
+chk audio/mpeg 'ID3xxx' "$yes"
+chk audio/mpeg hex:FFFB9000 "$yes" # bare frame sync
+chk audio/aac hex:FFF15080 "$yes"
+chk audio/flac 'fLaC' "$yes"
+chk audio/ogg hex:4F67675300 "$yes"
+chk audio/x-wav 'RIFFxxxxWAVE' "$yes"
+chk video/x-msvideo 'RIFFxxxxAVI ' "$yes"
+chk video/x-msvideo 'RIFFxxxxWAVE' "$no"
+chk video/mp4 hex:000000186674797069736F6D "$yes"
+chk video/webm hex:1A45DFA3 "$yes"
+chk video/mpeg hex:000001BA "$yes"
+chk video/x-ms-wmv hex:3026B2758E66CF11 "$yes"
+
+# archives; zip magic covers the office-container families
+chk application/zip hex:504B0304 "$yes"
+chk application/vnd.openxmlformats-officedocument.wordprocessingml.document hex:504B0304 "$yes"
+chk application/vnd.oasis.opendocument.text hex:504B0304 "$yes"
+chk application/msword hex:D0CF11E0A1B11AE1 "$yes"
+chk application/msword hex:504B0304 "$no" # legacy .doc is OLE, not zip
+chk application/x-gzip hex:1F8B08 "$yes"
+chk application/x-bzip2 'BZh9' "$yes"
+chk application/x-7z-compressed hex:377ABCAF271C "$yes"
+chk application/x-rar-compressed hex:526172211A07 "$yes"
+chk application/zstd hex:28B52FFD "$yes"
+chk application/x-tar "hex:$(printf '00%.0s' {1..257})7573746172" "$yes" # ustar at 257
+chk application/x-tar hex:7573746172 "$no"
+
+# documents, fonts, misc
+chk application/pdf '%PDF-1.7' "$yes"
+chk application/pdf '<html><body>soft 404</body></html>' "$no"
+chk application/postscript '%!PS-Adobe' "$yes"
+chk application/rtf '{\rtf1' "$yes"
+chk font/woff2 'wOF2' "$yes"
+chk font/otf 'OTTO' "$yes"
+chk font/ttf hex:0001000000 "$yes"
+chk application/x-shockwave-flash 'CWSx' "$yes"
+chk application/x-java-vm hex:CAFEBABE "$yes"
+chk application/wasm hex:0061736D "$yes"
+chk text/html $'  \r\n<!DOCTYPE html><html>' "$yes"
+chk text/html '<html lang="en">' "$yes"
+chk text/html 'plain text, no markup' "$no"
+chk text/xml '<?xml version="1.0"?>' "$yes"
+
+# no magic rule at all: never confirmed, never blocks the wire type
+chk text/css 'body { }' "$unk"
+chk text/plain 'hello' "$unk"
+chk application/x-javascript 'var x;' "$unk"
--- a/tests/01_zlib-savename-cached.test
+++ b/tests/01_zlib-savename-cached.test
@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Update-run naming from a real cache entry (-#test=savename cached=<ctype>|<save>).
+# Named 01_zlib-*: the cache writer needs zlib, which the MSan job can't run.
+
+# resolve httrack before cd: make check puts a RELATIVE ../src on PATH
+httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
+
+scratch=$(mktemp -d)
+trap 'rm -rf "$scratch"' EXIT
+cd "$scratch"
+
+name() {
+    local fil="$1" ctype="$2" want="$3"
+    shift 3
+    out="$("$httrack_bin" -O /dev/null -#test=savename "$fil" "$ctype" "$@" | sed -n 's/^savename: //p')"
+    test "${out##*/}" == "$want" || {
+        echo "FAIL: '$fil' '$ctype' $* -> '$out' (want '$want')"
+        exit 1
+    }
+}
+
+# No live bytes: the recorded save name (X-Save) reproduces the previous
+# verdict; cached body bytes (PNG magic) are ignored; css has no magic rule.
+name '/photo.jpg' 'image/png' 'photo.jpg' 'cached=image/png|www.example.com/photo.jpg'
+name '/photo.jpg' 'image/png' 'photo.png' 'cached=image/png|www.example.com/photo.png'
+name '/photo.jpg' 'image/jpeg' 'photo.jpg' 'cached=image/jpeg|www.example.com/photo.png'
+name '/style.css' 'image/png' 'style.css' 'cached=image/png|www.example.com/style.css'
+# agreement keeps the URL ext verbatim (.jpeg), never canonicalized to .jpg
+name '/photo.jpeg' 'image/jpeg' 'photo.jpeg' 'cached=image/jpeg|www.example.com/photo.jpeg'
--- a/tests/15_local-types.test
+++ b/tests/15_local-types.test
@@ -1,11 +1,10 @@
 #!/bin/bash
 #
-# Content-Type vs URL-extension naming (issue #267 family) under the default
-# delayed type check (-%N2). Policy: a MISSING Content-Type must not clobber a
-# URL extension that maps to a specific non-HTML type (.png/.pdf stay as-is);
-# an explicitly DECLARED type is trusted, so a binary-looking URL that really
-# serves HTML (text/html on .pdf/.jpg) is named .html. The "wrong" names are
-# asserted absent so a regression in either direction fails here.
+# Content-Type vs URL-extension naming (#267 family, default -%N2). A MISSING
+# type keeps a specific non-HTML ext; a DECLARED disagreeing type is trusted
+# unless magic bytes prove the ext right (lie/wrongtype/packed keep theirs),
+# so a real HTML body (report.pdf) still becomes .html. Wrong names are
+# asserted absent so a regression in either direction fails.

 : "${top_srcdir:=..}"

@@ -14,7 +13,11 @@ bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
    --found 'types/notype.pdf' --not-found 'types/notype.html' \
    --found 'types/photo.png' \
    --found 'types/doc.pdf' \
-    --found 'types/lie.html' --not-found 'types/lie.png' \
+    --found 'types/lie.png' --not-found 'types/lie.html' \
+    --found 'types/wrongtype.jpg' --not-found 'types/wrongtype.png' \
+    --found 'types/bigtype.jpg' --not-found 'types/bigtype.png' \
+    --found 'types/mutant.jpg' --not-found 'types/mutant.png' \
+    --found 'types/packed.jpg' --not-found 'types/packed.png' \
    --found 'types/report.html' --not-found 'types/report.pdf' \
    --found 'types/page.htm' --not-found 'types/page.html' \
    --found 'types/script.js' \
--- a/tests/18_local-update.test
+++ b/tests/18_local-update.test
@@ -1,15 +1,18 @@
 #!/bin/bash
 #
-# A second (update) pass must keep the names the first crawl chose. The stored
-# Content-Type rides the cache, so the update reads back the same value -- the
-# unknown/unknown sentinel for a typeless response, the declared type otherwise
-# -- and names consistently: a declared-text/html .pdf stays .html and a
-# typeless .png stays .png across the update rather than reverting.
+# An update pass keeps the names the first crawl chose: type and save name
+# ride the cache, so a declared-text/html .pdf stays .html, a typeless .png
+# stays .png, and a sniff-kept ext is reproduced from X-Save even when the
+# refetched content changed (mutant.jpg serves PNG bytes on the rerun).

 : "${top_srcdir:=..}"

 bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
    --found 'types/report.html' --not-found 'types/report.pdf' \
    --found 'types/notype.png' --not-found 'types/notype.html' \
-    --found 'types/lie.html' \
+    --found 'types/lie.png' --not-found 'types/lie.html' \
+    --found 'types/wrongtype.jpg' --not-found 'types/wrongtype.png' \
+    --found 'types/bigtype.jpg' --not-found 'types/bigtype.png' \
+    --found 'types/packed.jpg' --not-found 'types/packed.png' \
+    --found 'types/mutant.jpg' --not-found 'types/mutant.png' \
    httrack 'BASEURL/types/index.html'
--- a/tests/32_local-cdispo.test
+++ b/tests/32_local-cdispo.test
@@ -0,0 +1,17 @@
+#!/bin/bash
+#
+# Content-Disposition names the saved file: the attachment filename replaces
+# the URL-derived name, and a traversal filename is reduced to its last
+# component, inside the mirror.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
+    --found 'cdispo/report.pdf' \
+    --file-matches 'cdispo/report.pdf' '%PDF' \
+    --not-found 'cdispo/fetch.pdf' \
+    --found 'cdispo/evil.pdf' \
+    --not-found 'evil.pdf' \
+    httrack 'BASEURL/cdispo/index.html'
--- a/tests/33_local-delayed.test
+++ b/tests/33_local-delayed.test
@@ -0,0 +1,20 @@
+#!/bin/bash
+#
+# Degenerate delayed-type paths (#5/#107 family): redirects that never resolve
+# a name must drop cleanly -- no .delayed leftovers (audited by local-crawl.sh),
+# no "bogus state" cache warnings, resolvable links still land correctly.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --rerun --errors 0 \
+    --found 'delayed/real.pdf' \
+    --file-matches 'delayed/real.pdf' '%PDF' \
+    --found 'delayed/notype.bin.html' \
+    --found 'delayed/empty.html' \
+    --not-found 'delayed/noloc.html' \
+    --not-found 'delayed/selfloop.html' \
+    --not-found 'delayed/chain9.pdf' \
+    --log-not-found 'bogus state' \
+    httrack 'BASEURL/delayed/index.html'
--- a/tests/34_local-maxtime.test
+++ b/tests/34_local-maxtime.test
@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# -E time limit (#481): server pages trickle for minutes; the engine must stop
+# on its own at -E plus grace, aborting the in-flight transfers.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# cancelled crawls can orphan .delayed placeholders (#483): skip that audit
+start=$(date +%s)
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --skip-delayed-audit \
+    --log-found 'More than 2 seconds passed' \
+    httrack 'BASEURL/trickle/index.html' -E2 -c4
+wall=$(($(date +%s) - start))
+# hard stop is due at -E2 + 5s grace; near TRICKLE_SECONDS means it never fired
+if [ "$wall" -ge 30 ]; then
+    echo "crawl took ${wall}s, -E hard stop did not engage" >&2
+    exit 1
+fi
--- a/tests/35_local-maxsize.test
+++ b/tests/35_local-maxsize.test
@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# -M byte cap (#77): the crawl must stop with the "giving up" error and keep
+# the mirror well under the 8 x 640KB the fixture totals uncapped.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'More than 400000 bytes have been transferred.. giving up' \
+    --max-mirror-bytes 4000000 \
+    httrack 'BASEURL/bigfiles/index.html' -M400000 -c4
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -38,6 +38,7 @@ TESTS = \
 	01_engine-ftp-line.test \
 	01_engine-ftp-userpass.test \
 	01_engine-hashtable.test \
+	01_engine-header.test \
 	01_engine-idna.test \
 	01_engine-escape-room.test \
 	01_engine-inplace-escape.test \
@@ -53,6 +54,7 @@ TESTS = \
 	01_engine-savename.test \
 	01_engine-selftest-dispatch.test \
 	01_engine-simplify.test \
+	01_engine-sniff.test \
 	01_engine-status.test \
 	01_engine-stripquery.test \
 	01_engine-strsafe.test \
@@ -63,6 +65,7 @@ TESTS = \
 	01_zlib-cache.test \
 	01_zlib-cache-golden.test \
 	01_zlib-cache-writefail.test \
+	01_zlib-savename-cached.test \
 	02_manpage-regen.test \
 	02_update-cache.test \
 	10_crawl-simple.test \
@@ -91,6 +94,10 @@ TESTS = \
 	28_local-pause.test \
 	29_local-redirect-fragment.test \
 	30_local-fragment-link.test \
-	31_local-javaclass.test
+	31_local-javaclass.test \
+	32_local-cdispo.test \
+	33_local-delayed.test \
+	34_local-maxtime.test \
+	35_local-maxsize.test

 CLEANFILES = check-network_sh.cache
--- a/tests/pycache/local-server.cpython-313.pyc
+++ b/tests/pycache/local-server.cpython-313.pyc
--- a/tests/local-crawl.sh
+++ b/tests/local-crawl.sh
@@ -16,8 +16,10 @@
 #       --errors N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
+#       --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
+# --max-mirror-bytes asserts the mirrored content (host root) stays under N.
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
@@ -92,6 +94,7 @@ tmpdir=$(mktemp -d "${tmptopdir}/httrack_local.XXXXXX") || die "could not create
 # --- parse leading control flags --------------------------------------------
 declare -a audit=()
 declare -a cookies=()
+skip_delayed_audit=""
 scheme=http
 pos=0
 args=("$@")
@@ -116,11 +119,14 @@ while test "$pos" -lt "$nargs"; do
        pos=$((pos + 1))
        cookies+=("${args[$pos]}")
        ;;
+    --skip-delayed-audit)
+        skip_delayed_audit=1
+        ;;
    --errors | --files)
        audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
        pos=$((pos + 1))
        ;;
-    --found | --not-found | --directory | --log-found | --log-not-found)
+    --found | --not-found | --directory | --log-found | --log-not-found | --max-mirror-bytes)
        audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
        pos=$((pos + 1))
        ;;
@@ -246,6 +252,17 @@ done
 test -n "$hostroot" || die "could not find host root under $out"
 debug "host root: $hostroot"

+# A completed crawl must leave no .delayed temporaries (issue #107).
+# --skip-delayed-audit: a cancelled crawl can orphan placeholders (issue #483)
+if test -z "$skip_delayed_audit"; then
+    info "checking for leftover .delayed files"
+    leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
+    if test -z "$leftovers"; then result "OK"; else
+        result "leftover: $leftovers"
+        exit 1
+    fi
+fi
+
 # --- audit -------------------------------------------------------------------
 i=0
 while test "$i" -lt "${#audit[@]}"; do
@@ -301,6 +318,15 @@ while test "$i" -lt "${#audit[@]}"; do
            exit 1
        else result "OK"; fi
        ;;
+    --max-mirror-bytes)
+        i=$((i + 1))
+        sz=$(find "$hostroot" -type f -exec cat {} + | wc -c | tr -d '[:space:]')
+        info "checking mirror size ${sz} <= ${audit[$i]} bytes"
+        if test "$sz" -le "${audit[$i]}"; then result "OK"; else
+            result "mirror too big"
+            exit 1
+        fi
+        ;;
    --file-matches)
        path="${audit[$((i + 1))]}"
        i=$((i + 2))
--- a/tests/local-server.py
+++ b/tests/local-server.py
@@ -14,6 +14,7 @@ stdlib only (http.server + ssl) -- no new build or runtime dependency.
 """

 import argparse
+import gzip
 import os
 import time
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
@@ -134,12 +135,14 @@ class Handler(SimpleHTTPRequestHandler):

    # --- type/extension matrix (issue #267 family) -------------------------

-    def send_raw(self, body, content_type):
+    def send_raw(self, body, content_type, extra_headers=()):
        """Send a raw body with an explicit Content-Type, or none at all when
        content_type is None (to observe httrack's typeless-file naming)."""
        self.send_response(200)
        if content_type is not None:
            self.send_header("Content-Type", content_type)
+        for name, value in extra_headers:
+            self.send_header(name, value)
        self.send_header("Content-Length", str(len(body)))
        self.end_headers()
        if self.command != "HEAD":
@@ -148,6 +151,8 @@ class Handler(SimpleHTTPRequestHandler):
    # Fake-binary blobs for the image/pdf/typeless cases.
    FAKE_PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 64
    FAKE_PDF = b"%PDF-1.4\n" + b"\x00" * 64
+    FAKE_JPEG = b"\xff\xd8\xff\xe0" + b"\x00" * 64
+    BIG_JPEG = b"\xff\xd8\xff\xe0" + bytes(range(256)) * 64  # > sniff window

    # path -> (body, content_type); None sends no header, "" sends an empty
    # Content-Type value (no usable type, must be treated like None).
@@ -159,6 +164,8 @@ class Handler(SimpleHTTPRequestHandler):
        "/types/notype.pdf": (FAKE_PDF, None),
        "/types/emptyct.png": (FAKE_PNG, ""),
        "/types/lie.png": (FAKE_PNG, "text/html"),
+        "/types/wrongtype.jpg": (FAKE_JPEG, "image/png"),
+        "/types/bigtype.jpg": (BIG_JPEG, "image/png"),
        "/types/report.pdf": (b"<html><body>real page</body></html>", "text/html"),
        "/types/page.htm": (b"<html><body>htm page</body></html>", "text/html"),
        "/types/script.js": (b"var x = 1;\n", "application/javascript"),
@@ -176,6 +183,10 @@ class Handler(SimpleHTTPRequestHandler):
            '\t<a href="notype.pdf">notypepdf</a>\n'
            '\t<img src="emptyct.png" />\n'
            '\t<img src="lie.png" />\n'
+            '\t<img src="wrongtype.jpg" />\n'
+            '\t<img src="bigtype.jpg" />\n'
+            '\t<img src="mutant.jpg" />\n'
+            '\t<img src="packed.jpg" />\n'
            '\t<a href="report.pdf">report</a>\n'
            '\t<a href="page.htm">htm</a>\n'
            '\t<script src="script.js"></script>\n'
@@ -190,6 +201,25 @@ class Handler(SimpleHTTPRequestHandler):
        body, ctype = self.TYPE_MATRIX[path]
        self.send_raw(body, ctype)

+    # content changes between crawls: run 1 sniffs JPEG, the update pass must
+    # keep the run-1 name (recorded verdict) even though the body is now PNG
+    MUTANT_SEEN = set()
+
+    def route_types_mutant(self):
+        path = urlsplit(self.path).path
+        body = self.FAKE_PNG if path in self.MUTANT_SEEN else self.FAKE_JPEG
+        if self.command != "HEAD":
+            self.MUTANT_SEEN.add(path)
+        self.send_raw(body, "image/png")
+
+    # gzip on the wire: the sniff must see the decoded body, not the stream
+    def route_types_packed(self):
+        self.send_raw(
+            gzip.compress(self.FAKE_JPEG),
+            "image/png",
+            extra_headers=[("Content-Encoding", "gzip")],
+        )
+
    # --- MIME-type exclusion abort (issue #58) -----------------------------
    # A -mime:application/pdf filter must abort the transfer once the header
    # arrives, not download the whole body and discard it.
@@ -354,6 +384,27 @@ class Handler(SimpleHTTPRequestHandler):
        if self.command != "HEAD":
            self.wfile.write(body)

+    # Content-Disposition naming: the attachment filename replaces the
+    # URL-derived name; path components in it are stripped (RFC 2616).
+    CDISPO_NAMES = {
+        "/cdispo/fetch.php": "report.pdf",
+        "/cdispo/evil.php": "../../evil.pdf",
+    }
+
+    def route_cdispo_index(self):
+        self.send_html(
+            '\t<a href="fetch.php">report</a>\n' '\t<a href="evil.php">evil</a>\n'
+        )
+
+    def route_cdispo(self):
+        filename = self.CDISPO_NAMES[urlsplit(self.path).path]
+        cdispo = 'attachment; filename="%s"' % filename
+        self.send_raw(
+            self.FAKE_PDF,
+            "application/pdf",
+            extra_headers=[("Content-Disposition", cdispo)],
+        )
+
    # 302 whose Location carries a #fragment (#204): the fragment is a UA anchor
    # that must be dropped before the target is fetched. A leaked '#' reaches the
    # strict-server guard below and 400s.
@@ -369,6 +420,85 @@ class Handler(SimpleHTTPRequestHandler):
    def route_redir_target(self):
        self.send_raw(b"<html><body>redirect target</body></html>\n", "text/html")

+    # --- delayed-type degenerate paths (issues #5/#107) --------------------
+    def route_delayed_index(self):
+        self.send_html(
+            '\t<a href="noloc.php">noloc</a>\n'
+            '\t<a href="selfloop.php">selfloop</a>\n'
+            '\t<a href="chain1.php">chain</a>\n'
+            '\t<a href="redir.php">redir</a>\n'
+            '\t<a href="notype.bin">notype</a>\n'
+            '\t<a href="empty.php">empty</a>\n'
+        )
+
+    def send_redirect(self, location):
+        self.send_response(302, "Found")
+        if location is not None:
+            self.send_header("Location", location)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
+    def route_delayed_noloc(self):
+        self.send_redirect(None)  # 302 without Location: name never resolves
+
+    def route_delayed_selfloop(self):
+        self.send_redirect("selfloop.php")
+
+    def route_delayed_chain(self):
+        # chain1..chain9: one more hop than the type-check redirect budget
+        n = int(urlsplit(self.path).path.rsplit("chain", 1)[1].split(".")[0])
+        if n < 9:
+            self.send_redirect("chain%d.php" % (n + 1))
+        else:
+            self.send_raw(self.FAKE_PDF, "application/pdf")
+
+    def route_delayed_redir(self):
+        self.send_redirect("real.pdf")
+
+    def route_delayed_realpdf(self):
+        self.send_raw(self.FAKE_PDF, "application/pdf")
+
+    def route_delayed_notype(self):
+        self.send_raw(self.FAKE_PDF, None)
+
+    def route_delayed_empty(self):
+        self.send_raw(b"", "text/html")  # 200 + Content-Length: 0
+
+    # -E time-limit (#481): pages that trickle far longer than any -E budget,
+    # so only an engine-side abort can end the crawl.
+    TRICKLE_SECONDS = 60
+
+    def route_trickle_index(self):
+        self.send_html(
+            "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(8))
+        )
+
+    def route_trickle_page(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(2 * self.TRICKLE_SECONDS))
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            for _ in range(self.TRICKLE_SECONDS):
+                self.wfile.write(b"xy")
+                self.wfile.flush()
+                time.sleep(1.0)
+        except OSError:
+            pass
+
+    # -M byte cap (#77): large fast files so a crawl overruns -M immediately.
+    BIGFILE_BYTES = 640 * 1024
+
+    def route_bigfiles_index(self):
+        self.send_html(
+            "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(8))
+        )
+
+    def route_bigfile(self):
+        self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
+
    ROUTES = {
        "/cookies/entrance.php": route_entrance,
        "/cookies/second.php": route_second,
@@ -384,6 +514,10 @@ class Handler(SimpleHTTPRequestHandler):
        "/types/notype.pdf": route_types,
        "/types/emptyct.png": route_types,
        "/types/lie.png": route_types,
+        "/types/wrongtype.jpg": route_types,
+        "/types/bigtype.jpg": route_types,
+        "/types/mutant.jpg": route_types_mutant,
+        "/types/packed.jpg": route_types_packed,
        "/types/report.pdf": route_types,
        "/types/page.htm": route_types,
        "/types/script.js": route_types,
@@ -406,6 +540,43 @@ class Handler(SimpleHTTPRequestHandler):
        "/mimex/index.html": route_mimex_index,
        "/mimex/blob.pdf": route_mimex_blob,
        "/mimex/real.html": route_mimex_real,
+        "/cdispo/index.html": route_cdispo_index,
+        "/cdispo/fetch.php": route_cdispo,
+        "/cdispo/evil.php": route_cdispo,
+        "/delayed/index.html": route_delayed_index,
+        "/trickle/index.html": route_trickle_index,
+        "/trickle/p0.bin": route_trickle_page,
+        "/trickle/p1.bin": route_trickle_page,
+        "/trickle/p2.bin": route_trickle_page,
+        "/trickle/p3.bin": route_trickle_page,
+        "/trickle/p4.bin": route_trickle_page,
+        "/trickle/p5.bin": route_trickle_page,
+        "/trickle/p6.bin": route_trickle_page,
+        "/trickle/p7.bin": route_trickle_page,
+        "/bigfiles/index.html": route_bigfiles_index,
+        "/bigfiles/p0.bin": route_bigfile,
+        "/bigfiles/p1.bin": route_bigfile,
+        "/bigfiles/p2.bin": route_bigfile,
+        "/bigfiles/p3.bin": route_bigfile,
+        "/bigfiles/p4.bin": route_bigfile,
+        "/bigfiles/p5.bin": route_bigfile,
+        "/bigfiles/p6.bin": route_bigfile,
+        "/bigfiles/p7.bin": route_bigfile,
+        "/delayed/noloc.php": route_delayed_noloc,
+        "/delayed/selfloop.php": route_delayed_selfloop,
+        "/delayed/redir.php": route_delayed_redir,
+        "/delayed/real.pdf": route_delayed_realpdf,
+        "/delayed/notype.bin": route_delayed_notype,
+        "/delayed/empty.php": route_delayed_empty,
+        "/delayed/chain1.php": route_delayed_chain,
+        "/delayed/chain2.php": route_delayed_chain,
+        "/delayed/chain3.php": route_delayed_chain,
+        "/delayed/chain4.php": route_delayed_chain,
+        "/delayed/chain5.php": route_delayed_chain,
+        "/delayed/chain6.php": route_delayed_chain,
+        "/delayed/chain7.php": route_delayed_chain,
+        "/delayed/chain8.php": route_delayed_chain,
+        "/delayed/chain9.php": route_delayed_chain,
        "/redir/index.html": route_redir_index,
        "/redir/go.php": route_redir_go,
        "/redir/target.html": route_redir_target,
Author	SHA1	Message	Date
Xavier Roche	6da801e3b9	Add a -M byte-cap non-regression test The -M limit had no test coverage. New /bigfiles/ fixture (8 fast 640KB files), a --max-mirror-bytes audit in local-crawl.sh, and a crawl that must log the "giving up" error and stay under the uncapped total. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com>	2026-07-04 15:05:15 +02:00
Xavier Roche	dfafe28002	Enforce the -E time limit inside the transfer wait cycle (#482 ) -E was only evaluated at per-link boundaries, so a slow or throttling server starved the check for minutes, and the smooth stop it finally requested drained the remaining transfers at server pace with no bound. back_wait now checks the deadline every cycle and, once a short grace period expires, aborts the in-flight HTTP transfers like the -T timeout path does (FTP slots stay with their owning thread). back_checkmirror's 0 return, previously dead, now carries the hard stop. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-04 13:13:33 +02:00
Xavier Roche	a3f04bde72	Sniff magic bytes before the wire type renames a URL extension (#478 ) A URL whose extension maps to a specific type but is served with a disagreeing specific Content-Type was always renamed after the wire (photo.jpg served as image/png became photo.png). The contested verdict (#480) is now settled by the leading body bytes: magic proving the extension's own type keeps it, anything inconclusive trusts the wire as before, and the #267 soft-404 guard is unchanged. New htssniff.c covers the magic-sniffable part of the supported MIME set (images, A/V containers by RIFF subtype and ftyp brand, zip/OLE document containers, archives, fonts, conservative text prefixes). hts_wait_delayed waits for a sniffable head (or EOF) only on contested verdicts; the head is read from the live backing slot (memory, url_sav, or the compressed-stream tmpfile, inflated in memory). Update runs never re-read bytes: they reproduce the previous run's verdict from the recorded X-Save name (cache_read_including_broken grows a return_save), so names never churn across updates or upgrades. Non-delayed mode never sniffs; its HEAD probe has no body on the first run. Also unlock the waiter's slot on the user-cancel abort. Tests flip the #480 contract pins to the sniffed outcomes (wrongtype/ bigtype/packed/mutant keep their extension, lie.png stays png), add -#test=sniff table rows, and pin the recorded-verdict proxy in 01_zlib-savename-cached (kept out of the MSan job: uninstrumented zlib). All discriminate against the pre-sniff binary. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-04 10:05:41 +02:00
Xavier Roche	11beef52e1	Name the contested case in extension naming and pin the current contract (#480 ) * Make the wire-type-vs-extension naming decision an explicit verdict Behavior-preserving refactor of wire_patches_ext: the decision becomes a three-way wire_ext_verdict (ext kept / wire wins / contested), with the contested case, a specific declared type disagreeing with a specific URL extension, named explicitly instead of falling through. Today a contested verdict trusts the wire, unchanged. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * Pin the naming contract: knobs and fixtures for content-independent naming -#test=savename gains body= (leading body bytes via a temp url_sav file) and cached= (a real one-entry cache, reopened read-only, whose stored body is PNG magic); new rows and 01_zlib-savename-cached.test pin that naming never depends on content or on the previously recorded save name, only on headers. e2e fixtures (wrongtype.jpg served as image/png, a gzip variant, a 16 KiB body, content that changes between crawls) pin the wire-wins outcome across fresh and update passes. Any future content-based tie-break must flip these rows explicitly. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> --------- Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-04 09:13:47 +02:00
Xavier Roche	d7c4eab1f5	Delayed type check finalizes fast transfers under their .delayed placeholder name (#479 ) * Fix delayed-slot races: lock guards and finalize order A slot whose savename is still the .delayed placeholder could be finalized and recorded under that transient name when the transfer completed before hts_wait_delayed resolved the type (fast servers): back_clean ignored the lock hts_wait_delayed holds, the direct-to-disk shortcut bound the output file to the placeholder name, and the type-known finalize ran before url_sav was patched, caching the entry under the wrong name (the 'bogus state (incomplete type)' warnings). Skip locked slots in back_clean and in the disk shortcut, and patch url_sav before finalizing. The delayed-naming body sniff (next commit) waits for body bytes and would hit these races deterministically; 15_local-types and 18_local-update cover the flow end to end. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * Exercise the degenerate delayed-type paths; audit .delayed leftovers everywhere New delayed/ fixtures (302 without Location, self-loop, a chain one hop past the redirect budget, a proper redirect, a typeless unknown-ext file) with 33_local-delayed.test, run twice via --rerun. local-crawl.sh now fails ANY local test whose finished mirror contains a .delayed temporary, guarding the whole suite against the #107 class. Probing the degenerate paths settled a review question: afs->save can still be a fresh placeholder when url_sav is patched (redirects that never resolve), but nothing name-keyed is stored there -- non-OK entries are cached header-only with an empty X-Save, the link is dropped, no file is left. The test pins exactly that. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> Empty responses finalized delayed slots under the .delayed name Found while probing for a harness trigger of the finalize race: a delayed-type URL answered 200 with Content-Length: 0 takes the empty-response branch, which reaches READY inside the header round and calls back_finalize directly, while url_sav still holds the .delayed placeholder. Deterministic on any such URL: the 'bogus state (incomplete type)' warning and an entry recorded under the placeholder name. Empty responses are common in the wild (beacons, placeholder files), so this is the reproducible face of #5. Defer the finalize when the slot is locked, matching the other guards: the waiter finalizes right after patching url_sav. The new empty.php fixture makes 33_local-delayed fail on master and pass here; the timing race itself remains untriggerable from the harness (headers and body advance in separate back_wait rounds; probe notes in the PR). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> --------- Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-04 08:37:14 +02:00
Xavier Roche	2eac19655b	Widen the savename self-test and cover Content-Disposition end to end (#477 ) * tests: widen the savename self-test and cover Content-Disposition end to end -#test=savename grows key=value knobs (cdispo=, statuscode=, status=, adr=, strip=, urlhack= and its negatives, n83=, type=) plus repeatable prior=adr\|fil\|sav rows that register an already-crawled link, so the .test can pin the still-downloading mime path, redirect delayed naming, dedup and collision suffixing, 8-3 modes, --strip-query dedup and hostile fils (traversal, control chars, oversized names) - the regression net for the upcoming resolve_extension work. A new cdispo/ endpoint in local-server.py and 32_local-cdispo.test give the Content-Disposition branches their first end-to-end coverage, including a traversal filename reduced to its last component. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * review follow-ups: isolate the wire cdispo strip, add over-match negatives The review found the e2e traversal row masked by two independent layers (the wire parser and url_savename both strip path components), so a new -#test=header self-test pins treathead's Content-Disposition parse alone. Three negative rows keep dedup honest: a kept query key that differs, a distinct URL under urlhack, and a same-basename-different-directory prior must all produce a fresh name, not a false match. route_cdispo now reuses send_raw via an extra_headers argument. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> --------- Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-03 19:42:32 +02:00
Xavier Roche	83c231d50e	Fold the type-probe's triplicated extension decision into one function (#476 ) * htsname: extract resolve_extension() from the type-probe block The cache, backing-headers and live-probe paths each carried the same cdispo / wire_patches_ext / give_mimext decision, per the block's own FIXME ("factorize this unholy mess"); fold the three copies into one static resolve_extension(). Drop the dead DEFAULT_BIN_EXT arms (the define has been commented out in htsconfig.h for years) and the ishtest variable only they read. -#test=savename gains an optional Content-Disposition argument, and the engine test now pins the filename-replacement path. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * tests: pin cdispo reserved-char sanitization in the savename table Review follow-up: adds a row where the expected name differs from the Content-Disposition input, exercising the hostile-cdispo cleaning path. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> --------- Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Fable 5 <noreply@anthropic.com>	2026-07-03 08:38:30 +02:00