Pin the naming contract: knobs and fixtures for content-independent naming

-#test=savename gains body= (leading body bytes via a temp url_sav
file) and cached= (a real one-entry cache, reopened read-only, whose
stored body is PNG magic); new rows and 01_zlib-savename-cached.test
pin that naming never depends on content or on the previously recorded
save name, only on headers. e2e fixtures (wrongtype.jpg served as
image/png, a gzip variant, a 16 KiB body, content that changes between
crawls) pin the wire-wins outcome across fresh and update passes. Any
future content-based tie-break must flip these rows explicitly.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsname.c

@@ -138,32 +138,45 @@ static void cleanEndingSpaceOrDot(char *s) {
   }
 }
 
-/* Should the wire Content-Type override the URL's own extension when naming the
-   saved file? True when the type is patchable (may_unknown2) and either the URL
-   extension implies no specific type or the server declared a disagreeing one.
-   A URL extension mapping to a specific non-HTML type is kept only when the
-   server declared NO type (the HTS_UNKNOWN_MIME sentinel; the #267 mangle
-   guard): a typeless .png stays .png, but a .pdf explicitly served as text/html
-   is named .html. The sentinel rides the cache, so updates stay consistent. */
+/* Wire Content-Type vs URL extension: a patchable wire type wins over an
+   unspecific ext, the HTS_UNKNOWN_MIME sentinel keeps a specific non-HTML ext
+   (#267 guard), a declared disagreement is CONTESTED. Sentinel and verdict
+   ride the cache, so updates stay consistent. */
+typedef enum wire_verdict {
+  WIRE_KEEPS_EXT,
+  WIRE_WINS,
+  WIRE_CONTESTED
+} wire_verdict;
+
+static wire_verdict wire_ext_verdict(httrackp *opt, const char *wiremime,
+                                     const char *file, char *urlmime,
+                                     size_t urlmime_size) {
+  if (may_unknown2(opt, wiremime, file))
+    return WIRE_KEEPS_EXT; /* type kept verbatim (keep-list / bogus-multiple) */
+  urlmime[0] = '\0';
+  /* type implied by the URL extension, only when confidently known (flag 0) */
+  if (!get_httptype_sized(opt, urlmime, urlmime_size, file, 0))
+    return WIRE_WINS; /* URL ext implies no known type */
+  if (strfield2(wiremime, urlmime))
+    return WIRE_KEEPS_EXT; /* agreement (no .htm->.html churn) */
+  if (!is_hypertext_mime(opt, urlmime, file) &&
+      strfield2(wiremime, HTS_UNKNOWN_MIME))
+    return WIRE_KEEPS_EXT; /* no declared type */
+  return WIRE_CONTESTED;
+}
+
 static int wire_patches_ext(httrackp *opt, const char *wiremime,
                             const char *file) {
   char urlmime[256];
 
-  if (may_unknown2(opt, wiremime, file))
-    return 0; /* type kept verbatim (keep-list / bogus-multiple) */
-  urlmime[0] = '\0';
-  /* type implied by the URL extension, only when confidently known (flag 0) */
-  if (!get_httptype_sized(opt, urlmime, sizeof(urlmime), file, 0))
-    return 1; /* URL ext implies no known type: trust the wire type */
-  if (strfield2(wiremime, urlmime))
-    return 0; /* wire agrees with the ext: keep it (no .htm->.html churn) */
-  /* wire disagrees with a specific non-HTML URL ext. Keep the ext only when
-     the server declared no type (the sentinel); an explicitly declared type,
-     even text/html, is trusted, so a binary-looking URL that really serves
-     HTML (login/error interstitial, soft-404) is named .html. */
-  if (!is_hypertext_mime(opt, urlmime, file) &&
-      strfield2(wiremime, HTS_UNKNOWN_MIME))
+  switch (wire_ext_verdict(opt, wiremime, file, urlmime, sizeof(urlmime))) {
+  case WIRE_KEEPS_EXT:
     return 0;
+  case WIRE_WINS:
+    return 1;
+  case WIRE_CONTESTED:
+    break; /* no content evidence is consulted today: trust the wire */
+  }
   return 1;
 }
 

src/htsselftest.c

@@ -1117,6 +1117,30 @@ static int st_header(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Decode a body argument ("hex:FFD8.." or literal text) into buf. */
+static size_t st_decode_body(const char *arg, char *buf, size_t size) {
+  size_t n = 0;
+
+  if (strncmp(arg, "hex:", 4) == 0) {
+    const char *s = arg + 4;
+
+    for (; s[0] != '\0' && s[1] != '\0' && n + 1 < size; s += 2) {
+      unsigned int byte;
+
+      if (sscanf(s, "%2x", &byte) != 1)
+        break;
+      buf[n++] = (char) byte;
+    }
+  } else {
+    n = strlen(arg);
+    if (n >= size)
+      n = size - 1;
+    memcpy(buf, arg, n);
+  }
+  buf[n] = '\0';
+  return n;
+}
+
 static int st_savename(httrackp *opt, int argc, char **argv) {
   lien_adrfilsave afs;
   cache_back cache;
@@ -1125,6 +1149,9 @@ static int st_savename(httrackp *opt, int argc, char **argv) {
   lien_back headers;
   const char *adr = "www.example.com";
   const char *cdispo = NULL;
+  const char *body = NULL;
+  const char *cached = NULL;
+  const char *bodyfile = "st-savename-body.tmp";
   int statuscode = HTTP_OK, status = 0;
   int i;
 
@@ -1158,6 +1185,10 @@ static int st_savename(httrackp *opt, int argc, char **argv) {
       opt->savename_83 = atoi(a + 4);
     else if (strncmp(a, "type=", 5) == 0)
       opt->savename_type = atoi(a + 5);
+    else if (strncmp(a, "body=", 5) == 0)
+      body = a + 5;
+    else if (strncmp(a, "cached=", 7) == 0)
+      cached = a + 7;
     else if (strncmp(a, "prior=", 6) != 0) {
       fprintf(stderr, "savename: unknown arg '%s'\n", a);
       return 1;
@@ -1168,7 +1199,47 @@ static int st_savename(httrackp *opt, int argc, char **argv) {
   strcpybuff(afs.af.fil, argv[0]);
 
   memset(&cache, 0, sizeof(cache));
-  cache.hashtable = (void *) coucal_new(0);
+  if (cached != NULL) { /* cached=<content-type>|<save name> */
+    char *dup = strdupt(cached);
+    char *const sep = strchr(dup, '|');
+    char locbuf[64] = "";
+    htsblk cr;
+
+    if (sep == NULL) {
+      fprintf(stderr, "savename: cached needs ctype|save\n");
+      return 1;
+    }
+    *sep = '\0';
+    /* one-entry cache in cwd, reopened read-only; body is PNG magic on
+       purpose: naming must not depend on stored content */
+    StringCopy(opt->path_log, "");
+    cache.type = 1;
+    cache.log = cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache_init(&cache, opt);
+    hts_init_htsblk(&cr);
+    cr.statuscode = HTTP_OK;
+    strcpybuff(cr.msg, "OK");
+    strcpybuff(cr.contenttype, dup);
+    cr.location = locbuf;
+    cr.adr = strdupt("\x89PNG\r\n\x1a\n");
+    cr.size = 8;
+    cache_add(opt, &cache, &cr, adr, argv[0], sep + 1, 1, NULL);
+    freet(cr.adr);
+    if (cache.zipOutput != NULL) {
+      zipClose(cache.zipOutput, NULL);
+      cache.zipOutput = NULL;
+    }
+    memset(&cache, 0, sizeof(cache));
+    cache.type = 1;
+    cache.log = cache.errlog = stderr;
+    cache.hashtable = coucal_new(0);
+    cache.ro = 1;
+    cache_init(&cache, opt);
+    freet(dup);
+  } else {
+    cache.hashtable = (void *) coucal_new(0);
+  }
 
   sback = back_new(opt, opt->maxsoc * 32 + 1024);
   /* same wiring as hts_mirror (htscore.c) */
@@ -1201,9 +1272,23 @@ static int st_savename(httrackp *opt, int argc, char **argv) {
   if (cdispo != NULL)
     strcpybuff(headers.r.cdispo, cdispo);
   strcpybuff(headers.url_fil, argv[0]);
+  if (body != NULL) { /* leading body bytes, exposed via url_sav */
+    char BIGSTK data[1024];
+    const size_t n = st_decode_body(body, data, sizeof(data));
+    FILE *const fp = fopen(bodyfile, "wb");
+
+    if (fp == NULL || fwrite(data, 1, n, fp) != n) {
+      fprintf(stderr, "savename: can not write %s\n", bodyfile);
+      return 1;
+    }
+    fclose(fp);
+    strcpybuff(headers.url_sav, bodyfile);
+  }
 
   url_savename(&afs, NULL, NULL, NULL, opt, sback, &cache, &hash, 0, 0,
                &headers);
+  if (body != NULL)
+    (void) UNLINK(bodyfile);
   printf("savename: %s\n", afs.save);
   return 0;
 }

tests/01_engine-savename.test

@@ -7,8 +7,16 @@ set -euo pipefail
 # name() asserts on the basename, full() on the whole path; prior= registers an
 # already-crawled link whose sav is rooted under the -O path (/dev/null here).
 
+# resolve httrack before cd: make check puts a RELATIVE ../src on PATH
+httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
+
+# scratch dir: body= and cached= write temp files (st-savename-body.tmp, hts-cache/)
+scratch=$(mktemp -d)
+trap 'rm -rf "$scratch"' EXIT
+cd "$scratch"
+
 run() {
-    httrack -O /dev/null -#test=savename "$@" | sed -n 's/^savename: //p'
+    "$httrack_bin" -O /dev/null -#test=savename "$@" | sed -n 's/^savename: //p'
 }
 
 name() {
@@ -73,6 +81,16 @@ name '/x.pdf' 'text/html' 'x.html' status=-1
 name '/x.html' 'text/html' 'x.html' status=-1
 name '/x.php' 'application/pdf' 'x.pdf' status=-1 cdispo=report.pdf
 
+# Contested type (wire disagrees with a specific ext): the wire is trusted and
+# body bytes are not consulted; pinned so a content-based tie-break shows up
+# as an explicit flip of these rows.
+name '/photo.jpg' 'image/png' 'photo.png' body=hex:FFD8FFE000104A46
+name '/photo.jpg' 'image/png' 'photo.png' body=hex:89504E470D0A1A0A
+name '/photo.jpg' 'image/png' 'photo.png'
+name '/doc.pdf' 'text/html' 'doc.html' body=hex:255044462D312E34
+name '/doc.pdf' 'text/html' 'doc.html' 'body=<html><body>soft 404</body></html>'
+name '/style.css' 'image/png' 'style.png' 'body=body { }'
+
 # A redirect answer resolves nothing: delayed placeholder name.
 name '/x.php' 'text/html' 'x.0.delayed' statuscode=301
 

tests/01_zlib-savename-cached.test

@@ -0,0 +1,33 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Update-run naming from a real cache entry (-#test=savename cached=<ctype>|<save>).
+# Named 01_zlib-*: the cache writer needs zlib, which the MSan job can't run.
+
+# resolve httrack before cd: make check puts a RELATIVE ../src on PATH
+httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
+
+scratch=$(mktemp -d)
+trap 'rm -rf "$scratch"' EXIT
+cd "$scratch"
+
+name() {
+    local fil="$1" ctype="$2" want="$3"
+    shift 3
+    out="$("$httrack_bin" -O /dev/null -#test=savename "$fil" "$ctype" "$@" | sed -n 's/^savename: //p')"
+    test "${out##*/}" == "$want" || {
+        echo "FAIL: '$fil' '$ctype' $* -> '$out' (want '$want')"
+        exit 1
+    }
+}
+
+# Names are re-derived from the stored headers on every run: neither the
+# recorded save name nor the cached body bytes change the verdict (pinned).
+name '/photo.jpg' 'image/png' 'photo.png' 'cached=image/png|www.example.com/photo.jpg'
+name '/photo.jpg' 'image/png' 'photo.png' 'cached=image/png|www.example.com/photo.png'
+name '/photo.jpg' 'image/jpeg' 'photo.jpg' 'cached=image/jpeg|www.example.com/photo.png'
+name '/style.css' 'image/png' 'style.png' 'cached=image/png|www.example.com/style.css'
+# agreement keeps the URL ext verbatim (.jpeg), never canonicalized to .jpg
+name '/photo.jpeg' 'image/jpeg' 'photo.jpeg' 'cached=image/jpeg|www.example.com/photo.jpeg'

tests/15_local-types.test

@@ -15,6 +15,10 @@ bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
     --found 'types/photo.png' \
     --found 'types/doc.pdf' \
     --found 'types/lie.html' --not-found 'types/lie.png' \
+    --found 'types/wrongtype.png' --not-found 'types/wrongtype.jpg' \
+    --found 'types/bigtype.png' --not-found 'types/bigtype.jpg' \
+    --found 'types/packed.png' --not-found 'types/packed.jpg' \
+    --found 'types/mutant.png' --not-found 'types/mutant.jpg' \
     --found 'types/report.html' --not-found 'types/report.pdf' \
     --found 'types/page.htm' --not-found 'types/page.html' \
     --found 'types/script.js' \

tests/18_local-update.test

@@ -12,4 +12,7 @@ bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
     --found 'types/report.html' --not-found 'types/report.pdf' \
     --found 'types/notype.png' --not-found 'types/notype.html' \
     --found 'types/lie.html' \
+    --found 'types/wrongtype.png' --not-found 'types/wrongtype.jpg' \
+    --found 'types/packed.png' --not-found 'types/packed.jpg' \
+    --found 'types/mutant.png' --not-found 'types/mutant.jpg' \
     httrack 'BASEURL/types/index.html'

tests/Makefile.am

@@ -64,6 +64,7 @@ TESTS = \
 	01_zlib-cache.test \
 	01_zlib-cache-golden.test \
 	01_zlib-cache-writefail.test \
+	01_zlib-savename-cached.test \
 	02_manpage-regen.test \
 	02_update-cache.test \
 	10_crawl-simple.test \

tests/local-server.py

@@ -14,6 +14,7 @@ stdlib only (http.server + ssl) -- no new build or runtime dependency.
 """
 
 import argparse
+import gzip
 import os
 import time
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
@@ -150,6 +151,8 @@ class Handler(SimpleHTTPRequestHandler):
     # Fake-binary blobs for the image/pdf/typeless cases.
     FAKE_PNG = b"\x89PNG\r\n\x1a\n" + b"\x00" * 64
     FAKE_PDF = b"%PDF-1.4\n" + b"\x00" * 64
+    FAKE_JPEG = b"\xff\xd8\xff\xe0" + b"\x00" * 64
+    BIG_JPEG = b"\xff\xd8\xff\xe0" + bytes(range(256)) * 64  # > sniff window
 
     # path -> (body, content_type); None sends no header, "" sends an empty
     # Content-Type value (no usable type, must be treated like None).
@@ -161,6 +164,8 @@ class Handler(SimpleHTTPRequestHandler):
         "/types/notype.pdf": (FAKE_PDF, None),
         "/types/emptyct.png": (FAKE_PNG, ""),
         "/types/lie.png": (FAKE_PNG, "text/html"),
+        "/types/wrongtype.jpg": (FAKE_JPEG, "image/png"),
+        "/types/bigtype.jpg": (BIG_JPEG, "image/png"),
         "/types/report.pdf": (b"<html><body>real page</body></html>", "text/html"),
         "/types/page.htm": (b"<html><body>htm page</body></html>", "text/html"),
         "/types/script.js": (b"var x = 1;\n", "application/javascript"),
@@ -178,6 +183,10 @@ class Handler(SimpleHTTPRequestHandler):
             '\t<a href="notype.pdf">notypepdf</a>\n'
             '\t<img src="emptyct.png" />\n'
             '\t<img src="lie.png" />\n'
+            '\t<img src="wrongtype.jpg" />\n'
+            '\t<img src="bigtype.jpg" />\n'
+            '\t<img src="mutant.jpg" />\n'
+            '\t<img src="packed.jpg" />\n'
             '\t<a href="report.pdf">report</a>\n'
             '\t<a href="page.htm">htm</a>\n'
             '\t<script src="script.js"></script>\n'
@@ -192,6 +201,25 @@ class Handler(SimpleHTTPRequestHandler):
         body, ctype = self.TYPE_MATRIX[path]
         self.send_raw(body, ctype)
 
+    # content changes between crawls: run 1 sniffs JPEG, the update pass must
+    # keep the run-1 name (recorded verdict) even though the body is now PNG
+    MUTANT_SEEN = set()
+
+    def route_types_mutant(self):
+        path = urlsplit(self.path).path
+        body = self.FAKE_PNG if path in self.MUTANT_SEEN else self.FAKE_JPEG
+        if self.command != "HEAD":
+            self.MUTANT_SEEN.add(path)
+        self.send_raw(body, "image/png")
+
+    # gzip on the wire: the sniff must see the decoded body, not the stream
+    def route_types_packed(self):
+        self.send_raw(
+            gzip.compress(self.FAKE_JPEG),
+            "image/png",
+            extra_headers=[("Content-Encoding", "gzip")],
+        )
+
     # --- MIME-type exclusion abort (issue #58) -----------------------------
     # A -mime:application/pdf filter must abort the transfer once the header
     # arrives, not download the whole body and discard it.
@@ -451,6 +479,10 @@ class Handler(SimpleHTTPRequestHandler):
         "/types/notype.pdf": route_types,
         "/types/emptyct.png": route_types,
         "/types/lie.png": route_types,
+        "/types/wrongtype.jpg": route_types,
+        "/types/bigtype.jpg": route_types,
+        "/types/mutant.jpg": route_types_mutant,
+        "/types/packed.jpg": route_types_packed,
         "/types/report.pdf": route_types,
         "/types/page.htm": route_types,
         "/types/script.js": route_types,