tests: international crawl tracked the double-encoded behavior

The default.html sub-crawl asserted the pre-fix output: two mojibake
café*.html filenames from double-encoded fetches and 4 errors where
the fix leaves 2 (the caf%e9.html Latin-1 escapes the server really
lacks). Update to the corrected names and counts; the other three
sub-crawls are unchanged. Only runs where ut.httrack.com is reachable
(in-tree builds), which is why just the deb CI job caught it.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

configure.ac

@@ -1,6 +1,6 @@
 AC_PREREQ([2.71])
 
-AC_INIT([httrack], [3.49.12], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
+AC_INIT([httrack], [3.49.11], [roche+packaging@httrack.com], [httrack], [http://www.httrack.com/])
 AC_COPYRIGHT([
 HTTrack Website Copier, Offline Browser for Windows and Unix
 Copyright (C) 1998-2015 Xavier Roche and other contributors
@@ -29,11 +29,11 @@ AC_CONFIG_SRCDIR(src/httrack.c)
 AC_CONFIG_MACRO_DIR([m4])
 AC_CONFIG_HEADERS(config.h)
 AM_INIT_AUTOMAKE([subdir-objects])
-# 3:4:0: 3.49.12 tail-appends one field (why_url) to httrackp and otherwise only
-# deletes dead comments; no struct layout or exported signature broke vs 3.49.11,
-# so it stays soname .so.3; bump revision.
+# 3:3:0: 3.49.11 only adds enum values, macros and inline helpers to the
+# installed headers (no struct layout or exported signature changed vs
+# 3.49.10), so it stays soname .so.3; bump revision.
 # (3:0:0 was the htsblk mime-buffer widening, the ABI break that moved .so.2 -> .so.3.)
-VERSION_INFO="3:4:0"
+VERSION_INFO="3:3:0"
 AM_MAINTAINER_MODE
 AC_USE_SYSTEM_EXTENSIONS
 

debian/changelog

@@ -1,13 +1,3 @@
-httrack (3.49.12-1) unstable; urgency=medium
-
-  * New upstream release: security and crawl-correctness fixes (remote stack
-    overflow in Content-Type/-Encoding parsing, fuzzer-found parser over-reads,
-    world-readable cookies.txt, filter-pattern denial of service) plus a new
-    --why filter diagnostic; full list in history.txt.
-  * Build with _FORTIFY_SOURCE and -fstack-protector-strong.
-
- -- Xavier Roche <xavier@debian.org>  Fri, 10 Jul 2026 21:11:09 +0200
-
 httrack (3.49.11-1) unstable; urgency=medium
 
   * New upstream release: crawl correctness and security fixes (network-facing

history.txt

@@ -4,23 +4,6 @@ HTTrack Website Copier release history:
 
 This file lists all changes and fixes that have been made for HTTrack
 
-3.49-12
-+ New: --why explains which filter rule accepts or rejects a given URL, then exits (#505)
-+ Fixed: links carrying raw UTF-8 bytes were fetched double-encoded and 404'd (#516)
-+ Fixed: an uncompressed body mislabeled as gzip no longer loses the page (#515)
-+ Fixed: remote stack overflow and uninitialized read in Content-Type/-Encoding parsing (#506)
-+ Fixed: several over-reads and a leak in the filter, URL and IDNA parsers (#499)
-+ Fixed: bound the cache-index (.ndx) parser to its buffer (#507)
-+ Fixed: catastrophic backtracking on '*'-heavy filter patterns (#513)
-+ Fixed: cookies.txt was created world-readable (#511)
-+ Fixed: a single corrupt cache entry no longer aborts the whole mirror (#494)
-+ Fixed: cache-reconcile policy was broken for zip caches (#491, #493, #495)
-+ Fixed: cancelling a crawl mid type-check no longer orphans .delayed placeholders (#496)
-+ Fixed: detect URLs after the first inline script and in mid-tag attributes (#497)
-+ Changed: build with _FORTIFY_SOURCE and -fstack-protector-strong (#504)
-+ Changed: removed the pre-3.31 (.dat/.ndx) cache import (#512)
-+ Changed: multiple internal hardening and build improvements (libFuzzer harnesses, CodeQL, dead-code removal)
-
 3.49-11
 + New: parse robots.txt Allow rules and path wildcards per RFC 9309 (#452)
 + New: advertise deflate in Accept-Encoding and decode deflate responses (#450)

src/htsback.c

@@ -68,15 +68,6 @@ static int slot_can_be_cached_on_disk(const lien_back * back);
 static int slot_can_be_cleaned(const lien_back * back);
 static int slot_can_be_finalized(httrackp * opt, const lien_back * back);
 
-/* Which hard quota, if any, is currently aborting the mirror. */
-typedef enum {
-  HTS_MIRROR_LIMIT_NONE = 0,
-  HTS_MIRROR_LIMIT_SIZE,
-  HTS_MIRROR_LIMIT_TIME,
-} hts_mirror_limit;
-
-static hts_mirror_limit back_mirror_limit(httrackp *opt);
-
 struct_back *back_new(httrackp *opt, int back_max) {
   int i;
   struct_back *sback = calloct(1, sizeof(struct_back));
@@ -533,22 +524,22 @@ int back_nsoc_overall(const struct_back * sback) {
 
 /* generate temporary file on lien_back */
 /* Note: utf-8 */
-static int create_back_tmpfile(httrackp *opt, lien_back *const back,
-                               const char *ext) {
+static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
   // do not use tempnam() but a regular filename
   back->tmpfile_buffer[0] = '\0';
   if (back->url_sav != NULL && back->url_sav[0] != '\0') {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
-             back->url_sav, ext);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.z", 
+             back->url_sav);
     back->tmpfile = back->tmpfile_buffer;
     if (structcheck(back->tmpfile) != 0) {
-      hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
+      hts_log_print(opt, LOG_WARNING, "can not create directory to %s", 
                     back->tmpfile);
       return -1;
     }
   } else {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
-             StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer),
+             "%s/tmp%d.z", StringBuff(opt->path_html_utf8),
+             opt->state.tmpnameid++);
     back->tmpfile = back->tmpfile_buffer;
   }
   /* OK */
@@ -556,32 +547,6 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
   return 0;
 }
 
-/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
-   existing file moved the good copy to back->tmpfile before truncating url_sav.
-   commit keeps the new file and drops the backup; else restore it so an aborted
-   transfer leaves the previous copy intact. Skips the zlib .z temp. */
-static void back_finalize_backup(httrackp *opt, lien_back *const back,
-                                 const hts_boolean commit) {
-  if (back->tmpfile == NULL || back->r.compressed)
-    return;
-  if (commit) {
-    (void) UNLINK(back->tmpfile); /* new copy is good; drop the backup */
-  } else {
-    if (back->r.out != NULL) {
-      fclose(back->r.out);
-      back->r.out = NULL;
-    }
-    (void) UNLINK(back->url_sav); /* drop the failed partial */
-    /* On rename failure keep the backup: it still holds the good copy, so
-       losing it would be worse than an orphaned temp. */
-    if (RENAME(back->tmpfile, back->url_sav) != 0)
-      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
-                    "could not restore %s; previous copy kept as %s",
-                    back->url_sav, back->tmpfile);
-  }
-  back->tmpfile = NULL;
-}
-
 // objet (lien) téléchargé ou transféré depuis le cache
 //
 // fermer les paramètres de transfert,
@@ -617,7 +582,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                       LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
                       back[p].r.size, back[p].url_adr, back[p].url_fil);
       }
-      back_finalize_backup(opt, &back[p], HTS_FALSE);
       return -1;
     }
 
@@ -636,7 +600,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
             // en mémoire -> passage sur disque
             if (!back[p].r.is_write) {
               // do not use tempnam() but a regular filename
-              if (create_back_tmpfile(opt, &back[p], "z") == 0) {
+              if (create_back_tmpfile(opt, &back[p]) == 0) {
                 assertf(back[p].tmpfile != NULL);
                 /* note: tmpfile is utf-8 */
                 back[p].r.out = FOPEN(back[p].tmpfile, "wb");
@@ -709,9 +673,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
           }
         }
 #endif
-        /* Body fully received: keep the freshly written url_sav, drop the
-           backup of the previous copy. */
-        back_finalize_backup(opt, &back[p], HTS_TRUE);
         /* Write mode to disk */
         if (back[p].r.is_write && back[p].r.adr != NULL) {
           freet(back[p].r.adr);
@@ -940,9 +901,6 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
       }
     }
   }
-  /* Aborted, error, or not ready: url_sav (if written) is broken; restore the
-     previous copy from the backup. */
-  back_finalize_backup(opt, &back[p], HTS_FALSE);
   return -1;
 }
 
@@ -2092,9 +2050,13 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 
       if (!back_trylive(opt, cache, sback, p)) {
 #if HTS_XGETHOST
-        back[p].status = STATUS_WAIT_DNS; // host name resolution attempt
-        soc = INVALID_SOCKET;             // not opened yet
-        if (host_wait(opt, &back[p])) {   // ready (file, or cached dns)
+#if HDEBUG
+        printf("back_solve..\n");
+#endif
+        back[p].status = STATUS_WAIT_DNS;       // tentative de résolution du nom de host
+        soc = INVALID_SOCKET;   // pas encore ouverte
+        back_solve(opt, &back[p]);      // préparer
+        if (host_wait(opt, &back[p])) { // prêt, par ex fichier ou dispo dans dns
 #if HDEBUG
           printf("ok, dns cache ready..\n");
 #endif
@@ -2237,9 +2199,37 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
 }
 
 #if HTS_XGETHOST
-// Resolution is synchronous inside the connect path; no pre-resolve step, so
-// the host is always immediately ready.
-int host_wait(httrackp *opt, lien_back *back) { return 1; }
+// attendre que le host (ou celui du proxy) ait été résolu
+// si c'est un fichier, la résolution est immédiate
+// idem pour ftp://
+void back_solve(httrackp * opt, lien_back * back) {
+  assertf(opt != NULL);
+  assertf(back != NULL);
+  if ((!strfield(back->url_adr, "file://"))
+      && !strfield(back->url_adr, "ftp://")
+    ) {
+    const char *a;
+
+    if (!(back->r.req.proxy.active))
+      a = back->url_adr;
+    else
+      a = back->r.req.proxy.name;
+    assertf(a != NULL);
+    a = jump_protocol_const(a);
+    if (check_hostname_dns(a)) {
+      hts_log_print(opt, LOG_DEBUG, "resolved: %s", a);
+    } else {
+      hts_log_print(opt, LOG_DEBUG, "failed to resolve: %s", a);
+    }
+  }
+}
+
+// détermine si le host a pu être résolu
+int host_wait(httrackp * opt, lien_back * back) {
+  // Always synchronous. No more background DNS resolution
+  // (does not really improve performances)
+  return 1;
+}
 #endif
 
 // élimine les fichiers non html en backing (anticipation)
@@ -2426,15 +2416,9 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
-  /* Time/size limit exceeded past grace: abort in-flight transfers so no wait
-     loop starves (#481, #77). FTP slots stay, their thread owns the socket. */
+  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
+     starves (#481). FTP slots stay, their thread owns the socket. */
   if (!back_checkmirror(opt)) {
-    const hts_mirror_limit limit = back_mirror_limit(opt);
-    const char *const reason =
-        (limit == HTS_MIRROR_LIMIT_SIZE) ? "size limit" : "time limit";
-    const char *const slotmsg = (limit == HTS_MIRROR_LIMIT_SIZE)
-                                    ? "Mirror Size Limit"
-                                    : "Mirror Time Out";
     int aborted = 0;
     unsigned int i;
 
@@ -2448,15 +2432,15 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
         if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
           back_delayed_discard(opt, &back[i]);
         back[i].r.statuscode = STATUSCODE_TIMEOUT;
-        strcpybuff(back[i].r.msg, slotmsg);
+        strcpybuff(back[i].r.msg, "Mirror Time Out");
         back[i].status = STATUS_READY;
         back_set_finished(sback, i);
         aborted++;
       }
     }
     if (aborted > 0)
-      hts_log_print(opt, LOG_WARNING, "%s reached, %d transfer(s) aborted",
-                    reason, aborted);
+      hts_log_print(opt, LOG_WARNING,
+                    "time limit reached, %d transfer(s) aborted", aborted);
     return;
   }
 
@@ -2946,8 +2930,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                 strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "gz") == 0
                                 && strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "tgz") == 0
                               ) {
-                              if (create_back_tmpfile(opt, &back[i], "z") ==
-                                  0) {
+                              if (create_back_tmpfile(opt, &back[i]) == 0) {
                                 assertf(back[i].tmpfile != NULL);
                                 /* note: tmpfile is utf-8 */
                                 if ((back[i].r.out =
@@ -2960,20 +2943,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
-                              /* Re-fetch over an existing file (#77 follow-up):
-                                 move the good copy aside before truncating it
-                                 so an aborted transfer can restore it. url_sav
-                                 is still written normally (file list intact).
-                               */
-                              back[i].tmpfile = NULL;
-                              if (fexist_utf8(back[i].url_sav)) {
-                                if (create_back_tmpfile(opt, &back[i], "bak") !=
-                                        0 ||
-                                    RENAME(back[i].url_sav, back[i].tmpfile) !=
-                                        0) {
-                                  back[i].tmpfile = NULL;
-                                }
-                              }
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {
@@ -3540,20 +3509,22 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                   if (back[i].r.statuscode == 406) {    // 'Not Acceptable'
                     back[i].r.statuscode = HTTP_OK;
                   }
-                  // On update, keep the good copy on error (mask as 304); skip
-                  // resume paths so a stale-partial 416 still re-fetches.
-                  if (HTTP_IS_ERROR(back[i].r.statuscode) &&
-                      back[i].is_update && !back[i].testmode &&
-                      back[i].range_req_size == 0 && back[i].url_sav[0] &&
-                      fexist_utf8(back[i].url_sav)) {
-                    hts_log_print(opt, LOG_DEBUG,
-                                  "Error %d (%s) ignored on update, keeping "
-                                  "existing copy: %s%s",
-                                  back[i].r.statuscode, back[i].r.msg,
-                                  back[i].url_adr, back[i].url_fil);
-                    back[i].r.statuscode = HTTP_NOT_MODIFIED;
-                    deletehttp(&back[i].r);
-                    back[i].r.soc = INVALID_SOCKET;
+                  // 'do not erase already downloaded file'
+                  // on an updated file
+                  // with an error : consider a 304 error
+                  if (!opt->delete_old) {
+                    if (HTTP_IS_ERROR(back[i].r.statuscode) && back[i].is_update
+                        && !back[i].testmode) {
+                      if (back[i].url_sav[0] && fexist_utf8(back[i].url_sav)) {
+                        hts_log_print(opt, LOG_DEBUG,
+                                      "Error ignored %d (%s) because of 'no purge' option for %s%s",
+                                      back[i].r.statuscode, back[i].r.msg,
+                                      back[i].url_adr, back[i].url_fil);
+                        back[i].r.statuscode = HTTP_NOT_MODIFIED;
+                        deletehttp(&back[i].r);
+                        back[i].r.soc = INVALID_SOCKET;
+                      }
+                    }
                   }
                   // Various hacks to limit re-transfers when updating a mirror
                   // Force update if same size detected
@@ -4174,43 +4145,38 @@ static int back_maxtime_grace(const int maxtime) {
   return maximum(5, minimum(30, maxtime / 10));
 }
 
-/* Bytes the smooth stop may overrun before in-flight transfers are aborted.
-   No floor (unlike the time grace): a size overrun should abort promptly. */
-static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
-
-/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481). */
-static hts_mirror_limit back_mirror_limit(httrackp *opt) {
-  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
-      HTS_STAT.stat_bytes - opt->maxsite >= back_maxsize_grace(opt->maxsite))
-    return HTS_MIRROR_LIMIT_SIZE;
+int back_checkmirror(httrackp * opt) {
+  // Check max size
+  if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
+    if (!opt->state.stop) {     /* not yet stopped */
+      hts_log_print(opt, LOG_ERROR,
+                    "More than " LLintP
+                    " bytes have been transferred.. giving up",
+                    (LLint) opt->maxsite);
+      /* cancel mirror smoothly */
+      hts_request_stop(opt, 0);
+    }
+    return 1;                   /* don'k break mirror too sharply for size limits, but stop requested */
+    /*return 0;
+     */
+  }
+  // Check max time
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
 
-    if (elapsed >= opt->maxtime &&
-        elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
-      return HTS_MIRROR_LIMIT_TIME;
+    if (elapsed >= opt->maxtime) {
+      if (!opt->state.stop) { /* not yet stopped */
+        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                      opt->maxtime);
+        /* cancel mirror smoothly */
+        hts_request_stop(opt, 0);
+      }
+      /* smooth stop starved past the grace period: stop waiting (#481) */
+      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+        return 0;
+    }
   }
-  return HTS_MIRROR_LIMIT_NONE;
-}
-
-int back_checkmirror(httrackp *opt) {
-  /* request a smooth stop the first time each cap is reached */
-  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
-      !opt->state.stop) {
-    hts_log_print(opt, LOG_ERROR,
-                  "More than " LLintP
-                  " bytes have been transferred.. giving up",
-                  (LLint) opt->maxsite);
-    hts_request_stop(opt, 0);
-  }
-  if (opt->maxtime > 0 && !opt->state.stop &&
-      (time_local() - HTS_STAT.stat_timestart) >= opt->maxtime) {
-    hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-                  opt->maxtime);
-    hts_request_stop(opt, 0);
-  }
-  /* hard stop once a cap overruns its grace (callers must stop waiting) */
-  return back_mirror_limit(opt) == HTS_MIRROR_LIMIT_NONE;
+  return 1;                     /* Ok, go on */
 }
 
 // octets transférés + add

src/htsback.h

@@ -138,11 +138,12 @@ LLint back_transferred(LLint add, struct_back * sback);
 
 // hostback
 #if HTS_XGETHOST
+void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
-/* Enforce -M/-E quotas: smooth-stops when reached; returns 0 once the -M cap
-   or -E deadline overruns its grace period (callers must stop waiting). */
+/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
+   the -E deadline overran its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htscore.c

@@ -1904,11 +1904,8 @@ int httpmirror(char *url1, httrackp * opt) {
             }
           }
 
-          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
-          // An empty body must not overwrite the file when the transfer failed
-          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
-          // copy to 0 (#77 follow-up).
-          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
+          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!          
+          if (r.adr != NULL || r.size == 0) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {
@@ -1929,6 +1926,7 @@ int httpmirror(char *url1, httrackp * opt) {
                */
             }
           }
+
         }
 
         /* Parsing of other media types (java, ram..) */

src/htsdns_selftest.c

@@ -297,26 +297,6 @@ int dns_selftests(httrackp *opt) {
     IPV6_resolver = 0;
   }
 
-  /* "host:port" resolves as the bare host: the cache/connect path strips
-     ":port" before both the backend and the cache key (#181). */
-  mock_reset_calls();
-  {
-    SOCaddr a;
-    char ip[64];
-    const char *err = NULL;
-
-    /* cache miss: the backend is hit with the stripped host, not "host:port" */
-    CHECK(hts_dns_resolve2(opt, "v6only.test:8080", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* the bare host shares that one cache entry (stripped key) */
-    CHECK(hts_dns_resolve2(opt, "v6only.test", &a, &err) != NULL);
-    CHECK(mock_find("v6only.test")->calls == 1);
-    /* a port variant of an already-cached host also hits, right address */
-    CHECK(hts_dns_resolve2(opt, "v4only.test:1234", &a, &err) != NULL);
-    SOCaddr_inetntoa(ip, sizeof(ip), a);
-    CHECK(strcmp(ip, "1.2.3.4") == 0);
-  }
-
   /* newhttp_addr() must connect to the addr_index-th address, not always the
      first: this is what back_connect_next relies on to reach the fallback. */
   {

src/htsglobal.h

@@ -43,8 +43,8 @@ Please visit our Website: http://www.httrack.com
    configure.ac, decoupled from these). VERSION is the display form, VERSIONID
    the dotted numeric form, AFF_VERSION the short form shown in footers,
    LIB_VERSION the data/cache format generation. */
-#define HTTRACK_VERSION "3.49-12"
-#define HTTRACK_VERSIONID "3.49.12"
+#define HTTRACK_VERSION "3.49-11"
+#define HTTRACK_VERSIONID "3.49.11"
 #define HTTRACK_AFF_VERSION "3.x"
 #define HTTRACK_LIB_VERSION "2.0"
 
@@ -56,10 +56,6 @@ Please visit our Website: http://www.httrack.com
 // Platform detection (sizes, feature macros)
 #include "htsconfig.h"
 
-// Fixed-width integer types + PRI* format macros for the LLint/TStamp typedefs
-#include <stdint.h>
-#include <inttypes.h>
-
 // WIN32 types
 #ifdef _WIN32
 #ifndef SIZEOF_LONG
@@ -115,6 +111,29 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DO_NOT_USE_UID
 #endif
 
+#ifndef HTS_LONGLONG
+#ifdef SIZEOF_LONG_LONG
+#if SIZEOF_LONG_LONG == 8
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+#ifndef HTS_LONGLONG
+#ifdef __sun
+#define HTS_LONGLONG 0
+#endif
+#ifdef __osf__
+#define HTS_LONGLONG 0
+#endif
+#ifdef __linux
+#define HTS_LONGLONG 1
+#endif
+#ifdef _WIN32
+#define HTS_LONGLONG 1
+#endif
+#endif
+#endif
+
 #ifdef DLLIB
 #define HTS_DLOPEN 1
 #else
@@ -307,11 +326,59 @@ typedef int hts_tristate;
 #define HTS_DEPRECATED(msg)
 #endif
 
-/* LLint/TStamp: signed exact-width 64-bit; -1 is a sentinel engine-wide. */
-typedef int64_t LLint;
-typedef int64_t TStamp;
-/* Full printf conversion, '%' included (PRId64 has none): "X: " LLintP. */
-#define LLintP "%" PRId64
+#ifndef HTS_LONGLONG
+#ifdef HTS_NO_64_BIT
+#define HTS_LONGLONG 0
+#else
+#define HTS_LONGLONG 1
+#endif
+#endif
+
+/* Wide integer types, chosen per platform.
+   LLint:  signed 64-bit counter for byte counts and large sizes (falls back to
+           plain int where 64-bit is unavailable).
+   TStamp: timestamp/duration in the same width (a double in the no-64-bit
+           fallback).
+   LLintP: the printf conversion for an LLint. */
+#if HTS_LONGLONG
+#ifdef LLINT_FORMAT
+typedef LLINT_TYPE LLint;
+
+typedef LLINT_TYPE TStamp;
+
+#define LLintP LLINT_FORMAT
+#else
+
+#ifdef _WIN32
+typedef __int64 LLint;
+
+typedef __int64 TStamp;
+
+#define LLintP "%I64d"
+#elif (defined(_LP64) || defined(__x86_64__) || defined(__powerpc64__) ||      \
+       defined(__64BIT__))
+
+typedef long int LLint;
+
+typedef long int TStamp;
+
+#define LLintP "%ld"
+#else
+typedef long long int LLint;
+
+typedef long long int TStamp;
+
+#define LLintP "%lld"
+#endif
+
+#endif /* HTS_LONGLONG */
+
+#else
+typedef int LLint;
+
+#define LLintP "%d"
+typedef double TStamp;
+#endif
 
 /* Integer type for file offsets/sizes passed to the C library. Widens to
    LLint (with HTS_FSEEKO for fseeko/ftello) under large-file support, plain

src/htslib.c

@@ -3038,11 +3038,12 @@ HTSEXT_API char *int2char(strc_int2bytes2 * strc, int n) {
 /* See http://physics.nist.gov/cuu/Units/binary.html */
 #define ToLLint(a) ((LLint)(a))
 #define ToLLintKiB (ToLLint(1024))
-#define ToLLintMiB (ToLLintKiB * ToLLintKiB)
-#define ToLLintGiB (ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#define ToLLintTiB (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
-#define ToLLintPiB                                                             \
-  (ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB * ToLLintKiB)
+#define ToLLintMiB (ToLLintKiB*ToLLintKiB)
+#ifdef HTS_LONGLONG
+#define ToLLintGiB (ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#define ToLLintTiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#define ToLLintPiB (ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB*ToLLintKiB)
+#endif
 HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
   if (n < ToLLintKiB) {
     sprintf(strc->buff1, "%d", (int) (LLint) n);
@@ -3051,7 +3052,9 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / ToLLintKiB)),
             (int) ((LLint) ((n % ToLLintKiB) * 100) / ToLLintKiB));
     strcpybuff(strc->buff2, "KiB");
-  } else if (n < ToLLintGiB) {
+  }
+#ifdef HTS_LONGLONG
+  else if (n < ToLLintGiB) {
     sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
             (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
     strcpybuff(strc->buff2, "MiB");
@@ -3068,6 +3071,13 @@ HTSEXT_API char **int2bytes2(strc_int2bytes2 * strc, LLint n) {
             (int) ((LLint) (((n % (ToLLintPiB)) * 100) / (ToLLintPiB))));
     strcpybuff(strc->buff2, "PiB");
   }
+#else
+  else {
+    sprintf(strc->buff1, "%d,%02d", (int) ((LLint) (n / (ToLLintMiB))),
+            (int) ((LLint) (((n % (ToLLintMiB)) * 100) / (ToLLintMiB))));
+    strcpybuff(strc->buff2, "MiB");
+  }
+#endif
   strc->buffadr[0] = strc->buff1;
   strc->buffadr[1] = strc->buff2;
   return strc->buffadr;

src/htsselftest.c

@@ -2147,19 +2147,6 @@ static int ae_write_collision(const char *path, const unsigned char *src,
   return ok ? 0 : 1;
 }
 
-/* Write src[0..len) to path as-is; 0 on success. */
-static int ae_write_raw(const char *path, const unsigned char *src,
-                        size_t len) {
-  FILE *const f = FOPEN(path, "wb");
-  int ok;
-
-  if (f == NULL)
-    return 1;
-  ok = fwrite(src, 1, len, f) == len;
-  fclose(f);
-  return ok ? 0 : 1;
-}
-
 /* Compare path's bytes to expect[0..len); 0 if equal. Streams (large files). */
 static int ae_check_decoded(const char *path, const unsigned char *expect,
                             size_t len) {
@@ -2227,46 +2214,6 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
     assertf(ae_write_collision(inpath, body, 64) == 0);
     assertf(hts_zunpack(inpath, outpath) == 64);
     assertf(ae_check_decoded(outpath, body, 64) == 0);
-    /* Identity fallback (#47): a plain body mislabeled as compressed is kept
-       verbatim, small and multi-chunk (> one 8 KiB fread). */
-    assertf(ae_write_raw(inpath, small, slen) == 0);
-    assertf(hts_zunpack(inpath, outpath) == (int) slen);
-    assertf(ae_check_decoded(outpath, small, slen) == 0);
-    {
-      const size_t ilen = 16 * 1024;
-      unsigned char *idbody = malloct(ilen);
-
-      assertf(idbody != NULL);
-      for (i = 0; i < ilen; i++)
-        idbody[i] = small[i % slen];
-      assertf(ae_write_raw(inpath, idbody, ilen) == 0);
-      assertf(hts_zunpack(inpath, outpath) == (int) ilen);
-      assertf(ae_check_decoded(outpath, idbody, ilen) == 0);
-      freet(idbody);
-    }
-    /* Truncated gzip (CRC+ISIZE cut), zlib (ADLER32 cut) and raw deflate
-       must all still fail, not fall back to a verbatim copy. */
-    {
-      static const struct {
-        int wb;
-        size_t cut;
-      } tr[] = {{16 + MAX_WBITS, 8}, {MAX_WBITS, 4}, {-MAX_WBITS, 5}};
-
-      for (i = 0; i < sizeof(tr) / sizeof(tr[0]); i++) {
-        unsigned char z[512];
-        size_t zlen;
-        FILE *f;
-
-        assertf(ae_write_packed(inpath, tr[i].wb, small, slen) == 0);
-        f = FOPEN(inpath, "rb");
-        assertf(f != NULL);
-        zlen = fread(z, 1, sizeof(z), f);
-        fclose(f);
-        assertf(zlen > tr[i].cut && zlen < sizeof(z));
-        assertf(ae_write_raw(inpath, z, zlen - tr[i].cut) == 0);
-        assertf(hts_zunpack(inpath, outpath) < 0);
-      }
-    }
     freet(body);
   }
 #else

src/htszlib.c

@@ -48,7 +48,6 @@ Please visit our Website: http://www.httrack.com
 
 /*
   Unpack file into a new file (gzip, zlib RFC1950 or raw deflate RFC1951).
-  A body provably in no deflate framing is copied verbatim (identity).
   Return value: size of the new file, or -1 if an error occurred
 */
 /* Note: utf-8 */
@@ -69,10 +68,6 @@ int hts_zunpack(char *filename, char *newfile) {
             ((inbuf[0] & 0x0f) == Z_DEFLATED &&
              (((unsigned) inbuf[0] << 8 | inbuf[1]) % 31) == 0)));
       int attempt;
-      /* not_deflate: the raw attempt hit a data error, so the body is in no
-         deflate framing at all. env_error: local I/O or memory failure. */
-      hts_boolean not_deflate = HTS_FALSE;
-      hts_boolean env_error = HTS_FALSE;
 
       /* deflate is ambiguous; on failure retry with the other windowBits */
       for (attempt = 0; attempt < 2 && ret < 0; attempt++) {
@@ -83,20 +78,15 @@ int hts_zunpack(char *filename, char *newfile) {
 
         if (attempt > 0) {
           /* rewind input; reopening fpout "wb" discards the partial output */
-          if (fseek(in, 0, SEEK_SET) != 0) {
-            env_error = HTS_TRUE;
+          if (fseek(in, 0, SEEK_SET) != 0)
             break;
-          }
           navail = fread(inbuf, 1, sizeof(inbuf), in);
         }
         fpout = FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
-        if (fpout == NULL) {
-          env_error = HTS_TRUE;
+        if (fpout == NULL)
           break;
-        }
         memset(&strm, 0, sizeof(strm));
         if (inflateInit2(&strm, windowBits) != Z_OK) {
-          env_error = HTS_TRUE;
           fclose(fpout);
           break;
         }
@@ -118,17 +108,12 @@ int hts_zunpack(char *filename, char *newfile) {
               zerr = inflate(&strm, Z_NO_FLUSH);
               if (zerr == Z_NEED_DICT || zerr == Z_DATA_ERROR ||
                   zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR) {
-                if (zerr == Z_MEM_ERROR || zerr == Z_STREAM_ERROR)
-                  env_error = HTS_TRUE;
-                else if (windowBits < 0)
-                  not_deflate = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
               produced = sizeof(outbuf) - strm.avail_out;
               if (produced > 0 &&
                   fwrite(outbuf, 1, produced, fpout) != produced) {
-                env_error = HTS_TRUE;
                 ok = HTS_FALSE;
                 break;
               }
@@ -144,28 +129,6 @@ int hts_zunpack(char *filename, char *newfile) {
         inflateEnd(&strm);
         fclose(fpout);
       }
-      /* keep a mislabeled identity body verbatim only when provably not
-         deflate; truncation or a local failure must keep failing (#47) */
-      if (ret < 0 && !wrapped && not_deflate && !env_error && !ferror(in)) {
-        FILE *const fpout =
-            FOPEN(fconv(catbuff, sizeof(catbuff), newfile), "wb");
-
-        if (fpout != NULL && fseek(in, 0, SEEK_SET) == 0) {
-          int size = 0;
-
-          while ((navail = fread(inbuf, 1, sizeof(inbuf), in)) > 0) {
-            if (fwrite(inbuf, 1, navail, fpout) != navail) {
-              size = -1;
-              break;
-            }
-            size += (int) navail;
-          }
-          if (size >= 0 && !ferror(in))
-            ret = size;
-        }
-        if (fpout != NULL)
-          fclose(fpout);
-      }
       fclose(in);
     }
   }

tests/19_local-connect-fallback.test

@@ -66,12 +66,9 @@ test -n "$port" || {
 }
 
 out="$tmpdir/crawl"
-# macOS has only 127.0.0.1, so the dead 127.0.0.x connects block to the timeout
-# instead of refusing instantly (as on Linux); a small --timeout + --retries=0
-# bound the stall without changing what the fallback exercises.
 HTTRACK_DEBUG_RESOLVE="deadhost:127.0.0.2,127.0.0.1" \
     httrack "http://deadhost:$port/simple/basic.html" -O "$out" \
-    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log" 2>&1
+    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log" 2>&1
 
 log="$out/hts-log.txt"
 
@@ -95,12 +92,12 @@ test -f "$out/deadhost_$port/simple/basic.html" || {
     exit 1
 }
 
-# every address dead: the slot exhausts the list, then errors out (the harness
-# timeout would catch a hang/loop)
+# every address refused: the slot exhausts the list, then errors out (the
+# harness timeout would catch a hang/loop; refused connects are instant)
 out2="$tmpdir/crawl2"
 HTTRACK_DEBUG_RESOLVE="alldead:127.0.0.2,127.0.0.3" \
     httrack "http://alldead:$port/simple/basic.html" -O "$out2" \
-    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log2" 2>&1
+    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log2" 2>&1
 log2="$out2/hts-log.txt"
 
 grep -q "trying next address" "$log2" || {

tests/31_local-javaclass.test

@@ -17,10 +17,7 @@ printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
 printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
     >"$tmproot/javaclass/Foo.class"
 
-# --log-found first: a plugin-load failure (wrong soname, shadowing system
-# libhtsjava) trips here on the banner suffix, not as an opaque "not found".
 bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
-    --log-found '\+libhtsjava' \
     --found 'javaclass/Foo.class' \
     --found 'javaclass/hello.gif' \
     httrack 'BASEURL/javaclass/index.html'

tests/36_local-bigcrawl.test

@@ -1,7 +1,7 @@
 #!/bin/bash
 #
 # Diverse seeded /big/ crawl: 12 pattern families, decoy absence, update pass
-# must 304-revalidate. 361 = 1 index + 96 pages + 192 imgs + 5 shared + 61
+# must 304-revalidate. 360 = 1 index + 96 pages + 192 imgs + 5 shared + 60
 # family + 6 singles; the 4 planted errors write -o1 pages, not counted.
 
 set -euo pipefail
@@ -9,7 +9,7 @@ set -euo pipefail
 : "${top_srcdir:=..}"
 
 bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --errors-content 4 --files 361 \
+    --errors 4 --files 360 \
     --found 'big/p/95.html' \
     --found 'big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png' \
     --found 'big/a/f2-2x.png' \
@@ -40,7 +40,6 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --not-found 'big/x/bj.jar' \
     --not-found 'big/x/is1.png' \
     --not-found 'big/x/concat.html' \
-    --file-matches 'big/f1/gzid.html' '<p>gzid</p>' \
     --file-matches 'big/p/2.html' 'srcset="\.\./a/f2-1x\.png 1x, \.\./a/f2-2x\.png 2x"' \
     --file-matches 'big/a/blk2.css' 'url\(blk2-bg\.png\)' \
     --file-matches 'big/p/5.html' "document\\.write\\('<a href=\"\\.\\./f5/dw\\.html\"" \

tests/42_local-maxsize-slow.test

@@ -1,21 +0,0 @@
-#!/bin/bash
-#
-# -M byte cap under a slow server (#77): p0 overruns -M at once while p1..p3
-# trickle for a minute; the cap must abort the in-flight transfers, not wait
-# them out. Unfixed, the smooth stop drains the trickle at server pace.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-start=$(date +%s)
-bash "$top_srcdir/tests/local-crawl.sh" \
-    --log-found 'More than 400000 bytes have been transferred.. giving up' \
-    httrack 'BASEURL/bigtrickle/index.html' -M400000 -c4
-wall=$(($(date +%s) - start))
-# trickle runs 60s; a smooth-only stop waits it out, the hard stop lands near 8s.
-# Wall clock is the discriminating check: the log line above fires either way.
-if [ "$wall" -ge 30 ]; then
-    echo "crawl took ${wall}s, -M hard stop did not engage" >&2
-    exit 1
-fi

tests/43_local-update-truncate.test

@@ -1,24 +0,0 @@
-#!/bin/bash
-#
-# -M hard-abort must not destroy already-complete files (#77 follow-up).
-# Pass 1 mirrors bigtrunc fully. Pass 2 re-fetches under --update -M: fast.bin
-# overruns the cap, its abort tears down slow.bin's stalled re-fetch. Both were
-# complete on disk from pass 1, and both must survive the aborted update:
-#   - slow.bin's re-fetch is incomplete; the direct-to-disk write must not be
-#     left truncated (the previous copy is restored).
-#   - fast.bin fully arrives but is torn down; the aborted slot must not overwrite
-#     it with an empty body.
-# Unfixed, slow.bin is truncated and fast.bin is zeroed while the cache still
-# records them complete.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-bash "$top_srcdir/tests/local-crawl.sh" \
-    --rerun-args '--update -M400000' \
-    --log-found 'More than 400000 bytes have been transferred.. giving up' \
-    --found bigtrunc/slow.bin \
-    --file-min-bytes bigtrunc/slow.bin 655360 \
-    --file-min-bytes bigtrunc/fast.bin 655360 \
-    httrack 'BASEURL/bigtrunc/index.html' -c2

tests/44_local-update-errormask.test

@@ -1,13 +0,0 @@
-#!/bin/bash
-# Issue #176: on --update a page that 200'd on the first crawl but now 403s must
-# keep its good local copy, not be overwritten by the error body nor purged.
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-bash "$top_srcdir/tests/local-crawl.sh" --rerun \
-    --found 'errmask/keep.dat' \
-    --file-min-bytes 'errmask/keep.dat' 1024 \
-    --file-matches 'errmask/keep.dat' '^KEEP' \
-    --file-not-matches 'errmask/keep.dat' 'error 403' \
-    httrack 'BASEURL/errmask/index.html'

tests/Makefile.am

@@ -110,9 +110,6 @@ TESTS = \
 	38_local-update-304.test \
 	39_local-delayed-cancel.test \
 	40_local-why.test \
-	41_local-utf8-link.test \
-	42_local-maxsize-slow.test \
-	43_local-update-truncate.test \
-	44_local-update-errormask.test
+	41_local-utf8-link.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -13,21 +13,15 @@
 #
 # Usage:
 #   bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
-#       [--rerun-args 'ARGS'] \
-#       --errors N --errors-content N --files N --found PATH ... --directory PATH ... \
+#       --errors N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
-#       --file-min-bytes PATH N --max-mirror-bytes N \
+#       --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
-# --errors counts every "Error:" log line; --errors-content drops transient
-# network failures (codes -2..-6) that flake on busy loopback under -c8.
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
 # --max/--min-mirror-bytes bound the mirrored content bytes (host root).
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
-# --file-min-bytes asserts a mirrored file (PATH) is at least N bytes.
-# --rerun-args runs a second pass (same server and mirror dir) with the given
-# extra httrack args appended, e.g. an --update run under a cap.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
 # which the ephemeral port forces into the cookie domain) and passes it to
 # httrack via --cookies-file, to exercise preloaded cookies.
@@ -45,7 +39,6 @@ key="${testdir}/server.key"
 tls=
 verbose=
 rerun=
-rerun_args=
 rerun_dead=
 tmpdir=
 serverpid=
@@ -129,11 +122,7 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         cookies+=("${args[$pos]}")
         ;;
-    --rerun-args)
-        pos=$((pos + 1))
-        rerun_args="${args[$pos]}"
-        ;;
-    --errors | --errors-content | --files)
+    --errors | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -141,7 +130,7 @@ while test "$pos" -lt "$nargs"; do
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --file-matches | --file-not-matches | --file-min-bytes)
+    --file-matches | --file-not-matches)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}" "${args[$((pos + 2))]}")
         pos=$((pos + 2))
         ;;
@@ -252,25 +241,6 @@ if test -n "$rerun"; then
     fi
 fi
 
-# --- optional second pass with extra args (e.g. an --update run under a cap) ---
-# Same server and mirror dir as the first pass, so the second pass sees the
-# cache the first pass wrote. Used to exercise re-fetch/update behaviour.
-if test -n "$rerun_args"; then
-    read -ra extra <<<"$rerun_args"
-    info "re-running httrack with ${rerun_args}"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
-        "${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
-    crawlpid=$!
-    wait "$crawlpid"
-    crawlres=$?
-    crawlpid=
-    test "$crawlres" -eq 0 || ! result "second pass exited $crawlres" || {
-        cat "${log}.2" >&2
-        exit 1
-    }
-    result "OK (second pass)"
-fi
-
 # --- optional dead pass: server stopped, the cache must survive the rollback --
 if test -n "$rerun_dead"; then
     zip="${out}/hts-cache/new.zip"
@@ -336,14 +306,6 @@ while test "$i" -lt "${#audit[@]}"; do
         assert_equals "checking errors" "${audit[$i]}" \
             "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
         ;;
-    --errors-content)
-        i=$((i + 1))
-        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
-        # transient network failures (statuscode -2..-6) flake on busy loopback;
-        # the code parens are followed by " at link" or " after N retries at link"
-        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
-        assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
-        ;;
     --files)
         i=$((i + 1))
         nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
@@ -426,16 +388,6 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         else result "OK"; fi
         ;;
-    --file-min-bytes)
-        path="${audit[$((i + 1))]}"
-        i=$((i + 2))
-        sz=$(wc -c <"${hostroot}/${path}" 2>/dev/null | tr -d '[:space:]')
-        info "checking ${path} size ${sz:-0} >= ${audit[$i]} bytes"
-        if test -n "$sz" && test "$sz" -ge "${audit[$i]}"; then result "OK"; else
-            result "file too small (or missing)"
-            exit 1
-        fi
-        ;;
     esac
     i=$((i + 1))
 done

tests/local-server.py

@@ -374,7 +374,6 @@ def big_index(port):
         '<img src="/big/a/d1/d2/d3/d4/d5/d6/d7/d8/deep.png">'
         '<a href="/big/f1/long.html?x=%s">long</a>'
         '<a href="/big/f1/gzok.html">gzok</a>'
-        '<a href="/big/f1/gzid.html">gzid</a>'
         '<a href="//127.0.0.1:%d/big/f1/protorel.html">protorel</a>'
         '<a href="http://127.0.0.1:%d/big/f1/abshost.html">abshost</a>'
         '<a href="/big/e/404.html">e404</a>'
@@ -400,7 +399,6 @@ BIG_SIMPLE_PAGES = {
     "/big/f1/dir/": "dir index",
     "/big/f1/long.html": "long",
     "/big/f1/gzok.html": "gzok",
-    "/big/f1/gzid.html": "gzid",
     "/big/f1/protorel.html": "protorel",
     "/big/f1/abshost.html": "abshost",
     "/big/f5/dw.html": "dw target",
@@ -934,33 +932,6 @@ class Handler(SimpleHTTPRequestHandler):
     def route_mini304_page(self):
         self.big_send(b"<html><body>tiny cacheable page</body></html>\n", "text/html")
 
-    # --- /errmask/: issue #176 — a page that 200'd on the first crawl but 403s
-    # on the update fetch must keep its good copy, not be overwritten nor purged.
-    ERRMASK_GOOD = b"KEEP" + b"." * 1020  # 1024 B distinctive non-HTML body
-    ERRMASK_ERR = b"<html><body>error 403</body></html>\n"
-
-    def route_errmask_index(self):
-        self.send_html('\t<a href="keep.dat">keep</a>\n')
-
-    def route_errmask_keep(self):
-        # First crawl (no validator) gets the 1024 B body + Last-Modified; the
-        # update sends a conditional and gets a 403 error page.
-        if self.headers.get("If-Modified-Since") or self.headers.get("If-None-Match"):
-            self.send_response(403, "Forbidden")
-            self.send_header("Content-Type", "text/html")
-            self.send_header("Content-Length", str(len(self.ERRMASK_ERR)))
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.ERRMASK_ERR)
-            return
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Last-Modified", BIG_LASTMOD)
-        self.send_header("Content-Length", str(len(self.ERRMASK_GOOD)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(self.ERRMASK_GOOD)
-
     # --- delayed-type degenerate paths (issues #5/#107) --------------------
     def route_delayed_index(self):
         self.send_html(
@@ -1062,48 +1033,6 @@ class Handler(SimpleHTTPRequestHandler):
     def route_bigfile(self):
         self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
 
-    # -M under a slow server (#77): p0 is a fast 640KB file that alone overruns
-    # -M; p1..p3 trickle for a minute. The cap must abort those in-flight
-    # transfers, not wait them out.
-    def route_bigtrickle_index(self):
-        self.send_html(
-            "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(4))
-        )
-
-    # -M hard-abort must not destroy an already-complete file (#77 follow-up).
-    # "fast.bin" alone overruns -M and completes; "slow.bin" transfers fully on
-    # its first fetch (initial mirror) but stalls on every later fetch (the
-    # --update re-fetch), so the -M abort tears it down mid-body. An engine that
-    # truncates the good local copy on the aborted re-fetch loses data.
-    slow_seen = 0
-
-    def route_bigtrunc_index(self):
-        self.send_html('\t<a href="fast.bin">fast</a>\n\t<a href="slow.bin">slow</a>\n')
-
-    def route_bigtrunc_slow(self):
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(self.BIGFILE_BYTES))
-        self.end_headers()
-        if self.command == "HEAD":
-            return
-        # Count body fetches only, so a stray HEAD can't shift which pass stalls.
-        Handler.slow_seen += 1
-        first = Handler.slow_seen == 1
-        try:
-            if first:
-                self.wfile.write(b"x" * self.BIGFILE_BYTES)
-                self.wfile.flush()
-            else:
-                self.wfile.write(b"x" * 4096)
-                self.wfile.flush()
-                for _ in range(120):
-                    self.wfile.write(b"x")
-                    self.wfile.flush()
-                    time.sleep(1.0)
-        except OSError:
-            pass
-
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -1176,14 +1105,6 @@ class Handler(SimpleHTTPRequestHandler):
         "/bigfiles/p5.bin": route_bigfile,
         "/bigfiles/p6.bin": route_bigfile,
         "/bigfiles/p7.bin": route_bigfile,
-        "/bigtrickle/index.html": route_bigtrickle_index,
-        "/bigtrickle/p0.bin": route_bigfile,
-        "/bigtrickle/p1.bin": route_trickle_page,
-        "/bigtrickle/p2.bin": route_trickle_page,
-        "/bigtrickle/p3.bin": route_trickle_page,
-        "/bigtrunc/index.html": route_bigtrunc_index,
-        "/bigtrunc/fast.bin": route_bigfile,
-        "/bigtrunc/slow.bin": route_bigtrunc_slow,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,
@@ -1204,8 +1125,6 @@ class Handler(SimpleHTTPRequestHandler):
         "/redir/target.html": route_redir_target,
         "/mini304/index.html": route_mini304_index,
         "/mini304/page.html": route_mini304_page,
-        "/errmask/index.html": route_errmask_index,
-        "/errmask/keep.dat": route_errmask_keep,
     }
 
     # --- /big/ seeded pseudo-site ------------------------------------------
@@ -1251,13 +1170,6 @@ class Handler(SimpleHTTPRequestHandler):
                     "text/html",
                     extra=[("Content-Encoding", "gzip")],
                 )
-            elif path == "/big/f1/gzid.html":
-                # Plain body mislabeled as gzip: identity fallback keeps it (#47)
-                self.big_send(
-                    body,
-                    "text/html",
-                    extra=[("Content-Encoding", "gzip")],
-                )
             else:
                 self.big_send(body, "text/html")
         elif path == "/big/f1/list.html":