Use an unknown/unknown sentinel for an absent Content-Type (#412)

#409 distinguished "the server declared text/html" from "no Content-Type,
defaulted to text/html" with a new htsblk.contenttype_given flag, so a
binary-looking URL that really serves HTML is saved .html while a typeless
response keeps its URL extension. That worked on a fresh crawl but had two
costs: the flag was never persisted, so on --update the cache read it as
unset and the names reverted (report.html became report.pdf again, and the
two passes disagreed); and it was an installed-struct ABI break (soname 4,
libhttrack4).

Replace the flag with a sentinel: when no Content-Type is received, store
"unknown/unknown" as the type instead of text/html. The sentinel is treated
as html for every type test (added to is_html_mime_type), so parsing,
storage and filtering of a typeless response are unchanged; only the naming
code (wire_patches_ext) reads it as "no declared type" and keeps the URL
extension. Because the type string rides the cache, an update reads the same
sentinel and names consistently -- the revert is fixed at the source.

The sentinel never reaches a consumer as a real type: a single helper,
hts_effective_mime(), maps it back to text/html wherever a stored type is
derived (give_mimext) or emitted/persisted -- the httrack stdout serve, the
ProxyTrack live serve, and the ProxyTrack .arc export (both the replayed
response header and the index record). The .arc export was caught by an
adversarial spill audit; without the map a typeless page archived via
proxytrack would carry Content-Type: unknown/unknown.

Since the sentinel makes contenttype_given unnecessary, #409's ABI break is
undone: the field is removed, soname returns to 3, and the Debian package
reverts libhttrack4 -> libhttrack3. soname 4 was never released (Debian NEW
carries libhttrack3), so this re-aligns master with the archive rather than
flip-flopping anything downstream.

Tests: 18_local-update re-mirrors and asserts the names survive the update
pass; 15_local-types gains a notype.html negative control; 17_local-empty-ct
stays green. Full make check: 27 pass, 0 fail.

One accepted behavior change: a mime filter matching exactly text/html no
longer matches a typeless response (its type is the sentinel, html-ish but
not literally text/html); the response is still parsed and crawled as html.

Signed-off-by: Xavier Roche <roche@httrack.com>
Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>

configure.ac

@@ -29,9 +29,9 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 4:0:0: htsblk gained the contenttype_given field, an incompatible ABI break,
-# so bump current and reset revision/age.
-VERSION_INFO="4:0:0"
+# 3:0:0: htsblk layout changed (contenttype/charset/contentencoding widened to
+# 128), an incompatible ABI break, so bump current and reset revision/age.
+VERSION_INFO="3:0:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 

debian/changelog

@@ -1,13 +1,3 @@
-httrack (3.49.8-3) unstable; urgency=medium
-
-  * Rename libhttrack3 to libhttrack4 to follow the SONAME bump to
-    libhttrack.so.4: htsblk gained a contenttype_given field, an
-    incompatible ABI change (VERSION_INFO 3 -> 4). The .files wildcard
-    now tracks .so.4* so the runtime libraries land in the right
-    package. New binary package, via NEW.
-
- -- Xavier Roche <xavier@debian.org>  Sat, 20 Jun 2026 19:46:16 +0200
-
 httrack (3.49.8-2) unstable; urgency=medium
 
   * Rename libhttrack2 to libhttrack3 to follow the SONAME, which the 3.49.8

debian/control

@@ -58,13 +58,13 @@ Description: webhttrack common files
  This package is the common files of webhttrack, website copier and
  mirroring utility
 
-Package: libhttrack4
+Package: libhttrack3
 Architecture: any
 Multi-Arch: same
 Section: libs
 Depends: ${misc:Depends}, ${shlibs:Depends}
-Replaces: libhttrack3, httrack (<< 3.49.8-3~)
-Breaks: libhttrack3, httrack (<< 3.49.8-3~)
+Replaces: libhttrack2, httrack (<< 3.49.8-2~)
+Breaks: libhttrack2, httrack (<< 3.49.8-2~)
 Description: Httrack website copier library
  This package is the library part of httrack, website copier and mirroring
  utility

debian/libhttrack3.files

@@ -0,0 +1,3 @@
+usr/lib/*/libhttrack.so.3*
+usr/lib/*/libhtsjava.so.3*
+usr/share/httrack/templates

debian/libhttrack3.lintian-overrides

@@ -1,3 +1,3 @@
 # The shared libraries ship without a versioned symbols control file (ABI is
 # tracked via the SONAME plus a >= upstream-version dependency, see debian/rules).
-libhttrack4: no-symbols-control-file usr/lib/*
+libhttrack3: no-symbols-control-file usr/lib/*

debian/libhttrack4.files

@@ -1,3 +0,0 @@
-usr/lib/*/libhttrack.so.4*
-usr/lib/*/libhtsjava.so.4*
-usr/share/httrack/templates

debian/rules

@@ -135,7 +135,7 @@ binary-arch: build install
 	dh_makeshlibs -a -X/usr/lib/$(DEB_HOST_MULTIARCH)/httrack/libtest --version-info
 	dh_installdeb -a
 	# we depend on the current version (ABI may change)
-	dh_shlibdeps -a -ldebian/libhttrack4/usr/lib/$(DEB_HOST_MULTIARCH)
+	dh_shlibdeps -a -ldebian/libhttrack3/usr/lib/$(DEB_HOST_MULTIARCH)
 	dh_gencontrol -a
 	dh_md5sums -a
 	dh_builddeb -a

src/htscoremain.c

@@ -2579,7 +2579,7 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                                         (r.size >= 0) ? r.size : (-r.size));
                                 if (r.contenttype >= 0) {
                                   fprintf(stdout, "Content-Type: %s\r\n",
-                                          r.contenttype);
+                                          hts_effective_mime(r.contenttype));
                                 }
                                 if (r.cdispo[0]) {
                                   fprintf(stdout, "Content-Disposition: %s\r\n",

src/htslib.c

@@ -1396,8 +1396,6 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
 void treatfirstline(htsblk * retour, const char *rcvd) {
   const char *a = rcvd;
 
-  retour->contenttype_given = HTS_FALSE; /* set when a Content-Type is seen */
-
   // exemple:
   // HTTP/1.0 200 OK
   if (*a) {
@@ -1425,7 +1423,7 @@ void treatfirstline(htsblk * retour, const char *rcvd) {
           else
             infostatuscode(retour->msg, retour->statuscode);
           // type MIME par défaut2
-          strcpybuff(retour->contenttype, HTS_HYPERTEXT_DEFAULT_MIME);
+          strcpybuff(retour->contenttype, HTS_UNKNOWN_MIME);
         } else {                // pas de code!
           retour->statuscode = STATUSCODE_INVALID;
           strcpybuff(retour->msg, "Unknown response structure");
@@ -1440,7 +1438,7 @@ void treatfirstline(htsblk * retour, const char *rcvd) {
         retour->statuscode = HTTP_OK;
         retour->keep_alive = 0;
         strcpybuff(retour->msg, "Unknown, assuming junky server");
-        strcpybuff(retour->contenttype, HTS_HYPERTEXT_DEFAULT_MIME);
+        strcpybuff(retour->contenttype, HTS_UNKNOWN_MIME);
       } else if (strnotempty(a)) {
         retour->statuscode = STATUSCODE_INVALID;
         strcpybuff(retour->msg, "Unknown (not HTTP/xx) response structure");
@@ -1449,7 +1447,7 @@ void treatfirstline(htsblk * retour, const char *rcvd) {
         retour->statuscode = HTTP_OK;
         retour->keep_alive = 0;
         strcpybuff(retour->msg, "Unknown, assuming junky server");
-        strcpybuff(retour->contenttype, HTS_HYPERTEXT_DEFAULT_MIME);
+        strcpybuff(retour->contenttype, HTS_UNKNOWN_MIME);
       }
     }
   } else {                      // vide!
@@ -1460,7 +1458,7 @@ void treatfirstline(htsblk * retour, const char *rcvd) {
     /* This is dirty .. */
     retour->statuscode = HTTP_OK;
     strcpybuff(retour->msg, "Unknown, assuming junky server");
-    strcpybuff(retour->contenttype, HTS_HYPERTEXT_DEFAULT_MIME);
+    strcpybuff(retour->contenttype, HTS_UNKNOWN_MIME);
   }
 }
 
@@ -1591,16 +1589,14 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           }
         }
       }
-      // An empty/whitespace Content-Type value yields no token; keep the
-      // default type and the "not given" flag instead of reading uninit tempo.
+      // An empty/whitespace Content-Type value yields no token: keep the
+      // sentinel default rather than reading an uninitialized tempo.
       if (sscanf(rcvd + p, "%s", tempo) == 1) {
         if (strlen(tempo) < sizeof(retour->contenttype) - 2) // pas trop long!!
           strcpybuff(retour->contenttype, tempo);
         else
           strcpybuff(retour->contenttype,
                      "application/octet-stream-unknown"); // erreur
-        retour->contenttype_given =
-            HTS_TRUE; /* server declared a usable type */
       }
     }
   } else if ((p = strfield(rcvd, "Content-Range:")) != 0) {
@@ -4318,6 +4314,7 @@ int give_mimext(char *s, size_t ssize, const char *st) {
   int ok = 0;
   int j = 0;
 
+  st = hts_effective_mime(st); /* no declared type: derive an html ext */
   s[0] = '\0';
   while((!ok) && (strnotempty(hts_mime[j][1]))) {
     if (strfield2(hts_mime[j][0], st)) {

src/htslib.h

@@ -481,10 +481,22 @@ HTS_STATIC int strcmpnocase(const char *a, const char *b) {
 
 // is this MIME an hypertext MIME (text/html), html/js-style or other script/text type?
 #define HTS_HYPERTEXT_DEFAULT_MIME "text/html"
+/* Sentinel stored when the server declared no Content-Type. It is html-ish
+   for every type test (so a typeless response still parses/stores as today),
+   but the naming code (wire_patches_ext) treats it as "no declared type" and
+   keeps the URL extension. It rides the cache, so updates name consistently. */
+#define HTS_UNKNOWN_MIME "unknown/unknown"
+/* Map the no-declared-type sentinel back to a real type for any header or
+   record we EMIT or PERSIST, so "unknown/unknown" never reaches a consumer
+   (a served Content-Type, a ProxyTrack .arc record, ...). */
+#define hts_effective_mime(m)                                                  \
+  (strfield2((m), HTS_UNKNOWN_MIME) ? HTS_HYPERTEXT_DEFAULT_MIME : (m))
 
-#define is_html_mime_type(a) \
-  ( (strfield2((a),"text/html")!=0)\
-  || (strfield2((a),"application/xhtml+xml")!=0) \
+#define is_html_mime_type(a)                                                   \
+  ((strfield2((a), "text/html") != 0) ||                                       \
+   (strfield2((a), "application/xhtml+xml") != 0) ||                           \
+   (strfield2((a), HTS_UNKNOWN_MIME) !=                                        \
+    0) /* no declared type: treat as html */                                   \
   )
 #define is_hypertext_mime__(a) \
   ( \

src/htsname.c

@@ -142,10 +142,11 @@ static void cleanEndingSpaceOrDot(char *s) {
    saved file? True when the type is patchable (may_unknown2) and either the URL
    extension implies no specific type or the server declared a disagreeing one.
    A URL extension mapping to a specific non-HTML type is kept only when the
-   server sent NO Content-Type (the #267 mangle guard): a typeless .png stays
-   .png, but a .pdf explicitly served as text/html is named .html. */
+   server declared NO type (the HTS_UNKNOWN_MIME sentinel; the #267 mangle
+   guard): a typeless .png stays .png, but a .pdf explicitly served as text/html
+   is named .html. The sentinel rides the cache, so updates stay consistent. */
 static int wire_patches_ext(httrackp *opt, const char *wiremime,
-                            const char *file, int contenttype_given) {
+                            const char *file) {
   char urlmime[256];
 
   if (may_unknown2(opt, wiremime, file))
@@ -157,11 +158,11 @@ static int wire_patches_ext(httrackp *opt, const char *wiremime,
   if (strfield2(wiremime, urlmime))
     return 0; /* wire agrees with the ext: keep it (no .htm->.html churn) */
   /* wire disagrees with a specific non-HTML URL ext. Keep the ext only when
-     the server sent NO Content-Type: a missing type is defaulted to text/html
-     upstream and must not clobber e.g. a .png. An explicitly declared type is
-     trusted, so a binary-looking URL that really serves HTML (login/error
-     interstitial, soft-404) is named .html instead of kept as .pdf/.jpg. */
-  if (!is_hypertext_mime(opt, urlmime, file) && !contenttype_given)
+     the server declared no type (the sentinel); an explicitly declared type,
+     even text/html, is trusted, so a binary-looking URL that really serves
+     HTML (login/error interstitial, soft-404) is named .html. */
+  if (!is_hypertext_mime(opt, urlmime, file) &&
+      strfield2(wiremime, HTS_UNKNOWN_MIME))
     return 0;
   return 1;
 }
@@ -411,8 +412,7 @@ int url_savename(lien_adrfilsave *const afs,
               if (strnotempty(r.cdispo)) {        /* filename given */
                 ext_chg = 2;      /* change filename */
                 strcpybuff(ext, r.cdispo);
-              } else if (wire_patches_ext(opt, r.contenttype, fil,
-                                          r.contenttype_given)) {
+              } else if (wire_patches_ext(opt, r.contenttype, fil)) {
                 if (give_mimext(s, sizeof(s),
                                 r.contenttype)) { // recognized extension
                   ext_chg = 1;
@@ -458,8 +458,7 @@ int url_savename(lien_adrfilsave *const afs,
                 ext_chg = 2;      /* change filename */
                 strcpybuff(ext, headers->r.cdispo);
               } else if (wire_patches_ext(opt, headers->r.contenttype,
-                                          headers->url_fil,
-                                          headers->r.contenttype_given)) {
+                                          headers->url_fil)) {
                 char s[16];
                 if (give_mimext(
                         s, sizeof(s),
@@ -675,7 +674,8 @@ int url_savename(lien_adrfilsave *const afs,
                 if (!has_been_moved) {
                   if (back[b].r.statuscode != -10) {    // erreur
                     if (strnotempty(back[b].r.contenttype) == 0)
-                      strcpybuff(back[b].r.contenttype, "text/html");   // message d'erreur en html
+                      strcpybuff(back[b].r.contenttype,
+                                 HTS_UNKNOWN_MIME); // no declared type
                     // Finalement on, renvoie un erreur, pour ne toucher à rien dans le code
                     // libérer emplacement backing
                   }
@@ -688,8 +688,7 @@ int url_savename(lien_adrfilsave *const afs,
                       ext_chg = 2;      /* change filename */
                       strcpybuff(ext, back[b].r.cdispo);
                     } else if (wire_patches_ext(opt, back[b].r.contenttype,
-                                                back[b].url_fil,
-                                                back[b].r.contenttype_given)) {
+                                                back[b].url_fil)) {
                       if (give_mimext(
                               s, sizeof(s),
                               back[b].r.contenttype)) { // recognized extension

src/htsopt.h

@@ -651,8 +651,6 @@ struct htsblk {
   int debugid;           /**< connection debug id */
   /* */
   htsrequest req; /**< parameters used for the request */
-  /* a Content-Type header was received (else contenttype holds a default) */
-  hts_boolean contenttype_given;
   /*char digest[32+2];   // md5 digest generated by the engine ("" if none) */
 };
 

src/proxy/proxytrack.c

@@ -1176,11 +1176,15 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
         if (element != NULL) {
           msgCode = element->statuscode;
           StringRoom(headers, 8192);
-          sprintf(StringBuffRW(headers), "HTTP/1.1 %d %s\r\n"
+          sprintf(StringBuffRW(headers),
+                  "HTTP/1.1 %d %s\r\n"
 #ifndef NO_WEBDAV
                   "%s"
 #endif
-                  "Content-Type: %s%s%s%s\r\n" "%s%s%s" "%s%s%s" "%s%s%s",
+                  "Content-Type: %s%s%s%s\r\n"
+                  "%s%s%s"
+                  "%s%s%s"
+                  "%s%s%s",
                   /* */
                   msgCode, element->msg,
 #ifndef NO_WEBDAV
@@ -1188,16 +1192,18 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
                   StringBuff(davHeaders),
 #endif
                   /* Content-type: foo; [ charset=bar ] */
-                  element->contenttype,
+                  hts_effective_mime(element->contenttype),
                   ((element->charset[0]) ? "; charset=\"" : ""),
                   element->charset, ((element->charset[0]) ? "\"" : ""),
                   /* location */
-                  ((element->location != NULL
-                    && element->location[0]) ? "Location: " : ""),
-                  ((element->location != NULL
-                    && element->location[0]) ? element->location : ""),
-                  ((element->location != NULL
-                    && element->location[0]) ? "\r\n" : ""),
+                  ((element->location != NULL && element->location[0])
+                       ? "Location: "
+                       : ""),
+                  ((element->location != NULL && element->location[0])
+                       ? element->location
+                       : ""),
+                  ((element->location != NULL && element->location[0]) ? "\r\n"
+                                                                       : ""),
                   /* last-modified */
                   ((element->lastmodified[0]) ? "Last-Modified: " : ""),
                   ((element->lastmodified[0]) ? element->lastmodified : ""),
@@ -1205,8 +1211,7 @@ static void proxytrack_process_HTTP(PT_Indexes indexes, T_SOC soc_c) {
                   /* etag */
                   ((element->etag[0]) ? "ETag: " : ""),
                   ((element->etag[0]) ? element->etag : ""),
-                  ((element->etag[0]) ? "\r\n" : "")
-            );
+                  ((element->etag[0]) ? "\r\n" : ""));
           StringLength(headers) = (int) strlen(StringBuff(headers));
         } else {
           /* No query string, no ending / : check the the <url>/ page */

src/proxy/store.c

@@ -52,6 +52,7 @@ Please visit our Website: http://www.httrack.com
 
 #include "htscore.h"
 #include "htsback.h"
+#include "htslib.h" /* hts_effective_mime */
 
 #include "store.h"
 #include "proxystrings.h"
@@ -2289,10 +2290,17 @@ static int PT_SaveCache__Arc_Fun(void *arg, const char *url, PT_Element element)
   int size_headers;
 
   sprintf(st->headers,
-          "HTTP/1.0 %d %s" "\r\n" "X-Server: ProxyTrack " PROXYTRACK_VERSION
-          "\r\n" "Content-type: %s%s%s%s" "\r\n" "Last-modified: %s" "\r\n"
-          "Content-length: %d" "\r\n", element->statuscode, element->msg,
-          /**/ element->contenttype,
+          "HTTP/1.0 %d %s"
+          "\r\n"
+          "X-Server: ProxyTrack " PROXYTRACK_VERSION "\r\n"
+          "Content-type: %s%s%s%s"
+          "\r\n"
+          "Last-modified: %s"
+          "\r\n"
+          "Content-length: %d"
+          "\r\n",
+          element->statuscode, element->msg,
+          /**/ hts_effective_mime(element->contenttype),
           (element->charset[0] ? "; charset=\"" : ""),
           (element->charset[0] ? element->charset : ""),
           (element->charset[0] ? "\"" : ""), /**/ element->lastmodified,
@@ -2328,10 +2336,10 @@ static int PT_SaveCache__Arc_Fun(void *arg, const char *url, PT_Element element)
           /* args */
           (link_has_authority(url) ? "" : "http://"), url, "0.0.0.0",
           tm->tm_year + 1900, tm->tm_mon + 1, tm->tm_mday, tm->tm_hour,
-          tm->tm_min, tm->tm_sec, element->contenttype, element->statuscode,
-          st->md5, (element->location ? element->location : "-"),
-          (long int) ftell(fp), st->filename,
-          (long int) (size_headers + element->size));
+          tm->tm_min, tm->tm_sec, hts_effective_mime(element->contenttype),
+          element->statuscode, st->md5,
+          (element->location ? element->location : "-"), (long int) ftell(fp),
+          st->filename, (long int) (size_headers + element->size));
   /* network_doc */
   if (fwrite(st->headers, 1, size_headers, fp) != size_headers
       || (element->size > 0

tests/15_local-types.test

@@ -10,7 +10,7 @@
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
-    --found 'types/notype.png' \
+    --found 'types/notype.png' --not-found 'types/notype.html' \
     --found 'types/notype.pdf' --not-found 'types/notype.html' \
     --found 'types/photo.png' \
     --found 'types/doc.pdf' \

tests/18_local-update.test

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# A second (update) pass must keep the names the first crawl chose. The stored
+# Content-Type rides the cache, so the update reads back the same value -- the
+# unknown/unknown sentinel for a typeless response, the declared type otherwise
+# -- and names consistently: a declared-text/html .pdf stays .html and a
+# typeless .png stays .png across the update rather than reverting.
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 --rerun \
+    --found 'types/report.html' --not-found 'types/report.pdf' \
+    --found 'types/notype.png' --not-found 'types/notype.html' \
+    --found 'types/lie.html' \
+    httrack 'BASEURL/types/index.html'

tests/Makefile.am

@@ -54,6 +54,7 @@ TESTS = \
 	14_local-https.test \
 	15_local-types.test \
 	16_local-assume.test \
-	17_local-empty-ct.test
+	17_local-empty-ct.test \
+	18_local-update.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -26,6 +26,7 @@ key="${testdir}/server.key"
 
 tls=
 verbose=
+rerun=
 tmpdir=
 serverpid=
 crawlpid=
@@ -89,6 +90,7 @@ nargs=$#
 while test "$pos" -lt "$nargs"; do
     case "${args[$pos]}" in
     --debug) verbose=1 ;;
+    --rerun) rerun=1 ;; # run httrack a second time (update pass) before auditing
     --no-purge)
         nopurge=1
         audit+=("--no-purge")
@@ -180,6 +182,22 @@ test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || {
 result "OK"
 grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
 
+# --- optional second pass: re-mirror into the same dir (cache/update path) ----
+if test -n "$rerun"; then
+    info "re-running httrack (update pass)"
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
+        "${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
+    crawlpid=$!
+    wait "$crawlpid"
+    crawlres=$?
+    crawlpid=
+    test "$crawlres" -eq 0 || ! result "update pass exited $crawlres" || {
+        cat "${log}.2" >&2
+        exit 1
+    }
+    result "OK (update)"
+fi
+
 # --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
 hostroot=
 for cand in "${out}/127.0.0.1_${port}" "${out}/127.0.0.1"; do