Interrupted mirror loses a file when a repaired cache meets an unusable 206 resume (#581)

#581 reports a Windows-only data loss: an interrupted mirror that resumes into an unusable Content-Range 206 drops its partial and never refetches, where the same sequence recovers the file whole on Linux. The trigger is the damaged cache a hard TerminateProcess leaves behind (MSYS can't signal a native exe), which pass 2 has to repair before resuming.

This does not fix the engine. I couldn't reproduce the failure on Linux, so an engine change would be guesswork. What it adds is a deterministic test that drives the damaged-cache regime, plus an analysis of where the two platforms part.

Test 71 leaves a partial and a temp-ref in pass 1, truncates `new.zip` past its last local entry so the central directory a hard kill never wrote is gone and the repair path runs, then resumes into the hostile 206. On Linux that fires the repair, takes the "unusable range -> restart whole" branch, and recovers the file whole every time, across every damage severity I tried, including a repair that recovers zero entries or fails outright. So the cache repair and the restart-whole logic are not themselves where Linux and Windows differ.

Root cause, as far as I can pin it from Linux: restart-whole doesn't refetch. It removes the partial and the temp-ref, flags `STATUSCODE_NON_FATAL`, and leans on the ordinary retry to requeue the URL. The requeued attempt rebuilds the request from the cache, then the temp-ref, then the on-disk partial. On Linux the removals leave none of those, so the retry is a clean whole-file GET and it succeeds. For the file to be lost, the retried attempt has to send a Range again: a second unusable 206, a second -5, and once the retry budget is spent the partial is already gone. That second Range can only come from a temp-ref or partial that outlived the restart-whole removal, which points at a Windows-specific removal or path effect I can't confirm from here.

For the maintainer: the fragile hinge is that restart-whole depends on a budget-consuming retry that re-derives its Range state from disk. A sturdier fix would make the retried attempt refuse to resume, via a per-link "refetch whole, no Range" flag the request builder honors, so a leftover temp-ref or partial can't re-enter the 206 loop whatever the removal quirk turns out to be. Checking that `UNLINK` and `url_savename_refname_remove` actually succeed on Windows would confirm the mechanism first.

Test 71 carries the same Windows skip as test 48. Lifting that skip should reproduce the failure, and turn the test into the fix's verification.

Refs #581.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

.github/workflows/windows-build.yml

@@ -188,7 +188,7 @@ jobs:
 
           # Every gate here exits 77, so an all-skipped suite would report green having
           # tested nothing: pin the skips, and floor the passes in case the glob empties.
-          expected_skips=" 48_local-crange-memresume.test"  # pending #581
+          expected_skips=" 48_local-crange-memresume.test 71_local-crange-repaircache.test"  # pending #581
           [ "$pass" -ge 90 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
           [ "$skipped" = "$expected_skips" ] || { echo "::error::unexpected skips:$skipped"; exit 1; }
           [ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }

src/htscharset.c

@@ -373,7 +373,8 @@ char *hts_convertStringCPFromUTF8(const char *s, size_t size, UINT cp) {
   return NULL;
 }
 
-char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
+HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
+                                         const char *charset) {
   const UINT cp = hts_getCodepage(charset);
 
   return hts_convertStringCPToUTF8(s, size, cp);
@@ -554,7 +555,8 @@ static char *hts_convertStringCharset(const char *s, size_t size,
   return NULL;
 }
 
-char *hts_convertStringToUTF8(const char *s, size_t size, const char *charset) {
+HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
+                                         const char *charset) {
   /* Empty string ? */
   if (size == 0) {
     return strdup("");

src/htscharset.h

@@ -51,8 +51,8 @@ typedef unsigned int hts_UCS4;
  * Convert the string "s" from charset "charset" to UTF-8.
  * Return NULL upon error.
  **/
-extern char *hts_convertStringToUTF8(const char *s, size_t size,
-                                     const char *charset);
+HTSEXT_API char *hts_convertStringToUTF8(const char *s, size_t size,
+                                         const char *charset);
 
 /**
  * Convert the string "s" from UTF-8 to charset "charset".

src/htscoremain.c

@@ -2368,30 +2368,28 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       opt->errlog = stderr;
     } else if (httrack_logmode >= 2) {
       // deux fichiers log
-      structcheck(StringBuff(opt->path_log));
-      if (fexist
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt")))
-        remove(fconcat
-               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"));
-      if (fexist
-          (fconcat
-           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt")))
-        remove(fconcat
-               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"));
+      // path_log holds UTF-8 bytes (argv is UTF-8): the ANSI file calls would
+      // read them as the codepage and drop the logs into a mangled twin (#630).
+      structcheck_utf8(StringBuff(opt->path_log));
+      if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                              StringBuff(opt->path_log), "hts-log.txt")))
+        UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                       StringBuff(opt->path_log), "hts-log.txt"));
+      if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                              StringBuff(opt->path_log), "hts-err.txt")))
+        UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                       StringBuff(opt->path_log), "hts-err.txt"));
 
       /* Check FS directory structure created */
-      structcheck(StringBuff(opt->path_log));
+      structcheck_utf8(StringBuff(opt->path_log));
 
-      opt->log =
-        fopen(fconcat
-              (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"),
-              "w");
+      opt->log = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                               StringBuff(opt->path_log), "hts-log.txt"),
+                       "w");
       if (httrack_logmode == 2)
-        opt->errlog =
-          fopen(fconcat
-                (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"),
-                "w");
+        opt->errlog = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                                    StringBuff(opt->path_log), "hts-err.txt"),
+                            "w");
       else
         opt->errlog = opt->log;
       if (opt->log == NULL) {

src/htsname.c

@@ -1532,8 +1532,30 @@ int url_savename(lien_adrfilsave *const afs,
       // last segment
       wsave[j++] = '/';
 #define MAX_UTF8_SEQ_CHARS 4
-      for(i = lastSeg; wsave[i] != '\0' && j < maxLen; i++) {
-        wsave[j++] = wsave[i];
+      {
+        // #623: the ".delayed" placeholder marker sits at the tail; cutting
+        // through it drops IS_DELAYED_EXT, so the file is never renamed to its
+        // final name. Reserve the trailing ".<id>.delayed" across the cut.
+        size_t markStart = wsaveLen;
+        if (IS_DELAYED_EXT(afs->save)) {
+          const size_t extDot = wsaveLen - strlen("." DELAYED_EXT);
+          size_t p = extDot; /* walk back over a dot-separated ".<hexid>" tag */
+
+          while (p > lastSeg && ((wsave[p - 1] >= '0' && wsave[p - 1] <= '9') ||
+                                 (wsave[p - 1] >= 'a' && wsave[p - 1] <= 'f')))
+            p--;
+          // keep the tag only if truly ".<hexid>.delayed"; else the bare marker
+          // (a wholly-hex base, e.g. a hashed #133 name, must not be absorbed)
+          markStart = (p > lastSeg && p < extDot && wsave[p - 1] == '.')
+                          ? p - 1
+                          : extDot;
+        }
+        // head, bounded so the marker still fits, then the marker itself
+        for (i = lastSeg; i < markStart && j + (wsaveLen - markStart) < maxLen;
+             i++)
+          wsave[j++] = wsave[i];
+        for (i = markStart; i < wsaveLen && j < maxLen; i++)
+          wsave[j++] = wsave[i];
       }
       // terminating \0
       wsave[j++] = '\0';

src/htsparse.c

@@ -315,6 +315,43 @@ static void escape_url_parens(char *const s, const size_t size) {
   strlcpybuff(s, buff, size);
 }
 
+/* Strip a default ":80" from lien's authority in place. Any spelling that
+   range-parses to 80 (":80", ":080") is dropped by its matched length, not a
+   hardcoded 3 chars; a value that merely wraps to 80 as an int (#614) stays. */
+void hts_strip_default_port(char *lien, size_t size) {
+  char *a;
+
+  if (!link_has_authority(lien))
+    return;
+  a = strstr(lien, "//"); // "//" authority
+  if (a)
+    a += 2;
+  else
+    a = lien;
+  a = jump_toport(a);
+  if (a) { // port present
+    char *b = a + 1;
+    char saved;
+    int port;
+    hts_boolean is_default;
+
+    while (isdigit((unsigned char) *b))
+      b++;
+    saved = *b;
+    *b = '\0';
+    is_default = hts_parse_url_port(a + 1, &port) && port == 80;
+    *b = saved;
+    if (is_default) { // default port, strip it
+      char BIGSTK tempo[HTS_URLMAXSIZE * 2];
+
+      tempo[0] = '\0';
+      strncatbuff(tempo, lien, a - lien);
+      strcatbuff(tempo, b); // skip the whole matched :port
+      strlcpybuff(lien, tempo, size);
+    }
+  }
+}
+
 /* Main parser */
 int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
   char catbuff[CATBUFF_SIZE];
@@ -2138,38 +2175,8 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                     } while((b != a) && (b));
                   }
                 }
-                // éliminer les éventuels :80 (port par défaut!)
-                if (link_has_authority(lien)) {
-                  char *a;
-
-                  a = strstr(lien, "//");       // "//" authority
-                  if (a)
-                    a += 2;
-                  else
-                    a = lien;
-                  a = jump_toport(a);
-                  if (a) {      // port
-                    int port = 0;
-                    int defport = 80;
-                    char *b = a + 1;
-
-#if HTS_USEOPENSSL
-#endif
-                    while(isdigit((unsigned char) *b)) {
-                      port *= 10;
-                      port += (int) (*b - '0');
-                      b++;
-                    }
-                    if (port == defport) {      // port 80, default - c'est débile
-                      char BIGSTK tempo[HTS_URLMAXSIZE * 2];
-
-                      tempo[0] = '\0';
-                      strncatbuff(tempo, lien, a - lien);
-                      strcatbuff(tempo, a + 3); // sauter :80
-                      strcpybuff(lien, tempo);
-                    }
-                  }
-                }
+                // drop a default :80 port from the authority
+                hts_strip_default_port(lien, sizeof(lien));
                 // filtrer les parazites (mailto & cie)
                 /*
                    if (strfield(lien,"mailto:")) {  // ne pas traiter

src/htsparse.h

@@ -106,6 +106,10 @@ struct htsmoduleStructExtended {
 */
 int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre);
 
+/* Strip a default ":80" (any spelling) from an absolute link's authority, in
+   place into a buffer of the given size. */
+void hts_strip_default_port(char *lien, size_t size);
+
 /*
   Check for 301,302.. errors ("moved") and handle them; re-isuue requests, make
   rediretc file, handle filters considerations..

src/htsselftest.c

@@ -1589,6 +1589,41 @@ static int st_identabs(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Default-port strip (#627): a genuine 80 (any spelling) is removed by its
+   matched length, host preserved; a non-80 port or one that only wraps to 80 as
+   a 32-bit int (#614) is left intact. Guards the old bug where ":080"/":0080"
+   dropped a hardcoded 3 chars and glued the leftover digits onto the host. */
+static int st_stripport(httrackp *opt, int argc, char **argv) {
+  static const struct {
+    const char *in, *out;
+  } cases[] = {
+      {"http://127.0.0.1:80/x", "http://127.0.0.1/x"},
+      {"http://127.0.0.1:080/x", "http://127.0.0.1/x"},
+      {"http://127.0.0.1:0080/x", "http://127.0.0.1/x"},
+      {"http://127.0.0.1:80", "http://127.0.0.1"},
+      {"http://127.0.0.1:0081/x", "http://127.0.0.1:0081/x"},
+      {"http://127.0.0.1:81/x", "http://127.0.0.1:81/x"},
+      {"http://127.0.0.1:8080/x", "http://127.0.0.1:8080/x"},
+      {"http://127.0.0.1:4294967376/x", "http://127.0.0.1:4294967376/x"},
+      {"http://127.0.0.1/x", "http://127.0.0.1/x"},
+  };
+
+  size_t k;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  for (k = 0; k < sizeof(cases) / sizeof(cases[0]); k++) {
+    char BIGSTK buff[HTS_URLMAXSIZE * 2];
+
+    strcpybuff(buff, cases[k].in);
+    hts_strip_default_port(buff, sizeof(buff));
+    assertf(strcmp(buff, cases[k].out) == 0);
+  }
+  printf("stripport self-test OK\n");
+  return 0;
+}
+
 /* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
    no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
    registering an already-crawled link (dedup/collision paths). */
@@ -2111,6 +2146,52 @@ static int st_cache_corrupt(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
+/* Drives unzRepair over a damaged local file header whose CRC field's high
+   16-bit word has bit 15 set. Before the READ_32 fix that shifted an int and
+   overflowed, so UBSan aborts here; after it, repair recovers the one entry. */
+static int st_zip_repair_shift(httrackp *opt, int argc, char **argv) {
+  static const unsigned char zip[] = {
+      0x50, 0x4b, 0x03, 0x04, /* local file header signature */
+      0x14, 0x00,             /* version needed */
+      0x00, 0x00,             /* general purpose flag */
+      0x00, 0x00,             /* method */
+      0x00, 0x00,             /* time */
+      0x00, 0x00,             /* date */
+      0x00, 0x00, 0xe8, 0x8a, /* crc: high word 0x8ae8, bit 15 set */
+      0x00, 0x00, 0x00, 0x00, /* compressed size */
+      0x00, 0x00, 0x00, 0x00, /* uncompressed size */
+      0x01, 0x00,             /* filename length */
+      0x00, 0x00,             /* extra field length */
+      0x61                    /* filename "a" */
+  };
+  char in[HTS_URLMAXSIZE], out[HTS_URLMAXSIZE], tmp[HTS_URLMAXSIZE];
+  uLong nrec = 0, bytes = 0;
+  FILE *fp;
+  int err;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "zip-repair-shift: needs a directory\n");
+    return 1;
+  }
+  snprintf(in, sizeof(in), "%s/damaged.zip", argv[0]);
+  snprintf(out, sizeof(out), "%s/repair.zip", argv[0]);
+  snprintf(tmp, sizeof(tmp), "%s/repair.tmp", argv[0]);
+  fp = fopen(in, "wb");
+  if (fp == NULL || fwrite(zip, 1, sizeof(zip), fp) != sizeof(zip)) {
+    if (fp != NULL)
+      fclose(fp);
+    fprintf(stderr, "zip-repair-shift: cannot write %s\n", in);
+    return 1;
+  }
+  fclose(fp);
+  err = unzRepair(in, out, tmp, &nrec, &bytes);
+  printf("zip-repair-shift: %s (recovered %lu entr%s)\n",
+         (err == Z_OK && nrec == 1) ? "OK" : "FAIL", (unsigned long) nrec,
+         nrec == 1 ? "y" : "ies");
+  return (err == Z_OK && nrec == 1) ? 0 : 1;
+}
+
 static int st_cache_legacy(httrackp *opt, int argc, char **argv) {
   int err;
 
@@ -3189,6 +3270,8 @@ static const struct selftest_entry {
      st_socks5},
     {"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
      st_identabs},
+    {"stripport", "", "default :80 port strip preserves host (#627)",
+     st_stripport},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
     {"headerlong", "[header-name:]",
@@ -3216,6 +3299,9 @@ static const struct selftest_entry {
      st_cache_legacy},
     {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
      st_cache_corrupt},
+    {"zip-repair-shift", "<dir>",
+     "cache zip-repair header read must not overflow a signed shift",
+     st_zip_repair_shift},
     {"dns", "", "DNS resolver/cache self-test", st_dns},
     {"dnstimeout", "", "a slow DNS resolve is bounded and holds no lock",
      st_dnstimeout},

src/htsserver.c

@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
 
 #include "htsnet.h"
 #include "htslib.h"
+#include "htscharset.h"
 #include <stdio.h>
 #include <stdlib.h>
 #include <string.h>
@@ -749,7 +750,14 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
                            RUN THE SERVER
                          */
                         if (strcmp((char *) adrcd, "start") == 0) {
-                          webhttrack_main((char *) adr + p);
+                          /* POST body is in the form's charset, not the
+                             UTF-8 argv the engine now assumes (#629). */
+                          char *const cmdl = (char *) adr + p;
+                          char *cmdlUtf8 = hts_convertStringToUTF8(
+                              cmdl, strlen(cmdl), LANGSEL("LANGUAGE_CHARSET"));
+
+                          webhttrack_main(cmdlUtf8 != NULL ? cmdlUtf8 : cmdl);
+                          freet(cmdlUtf8);
                         } else {
                           commandRunning = 0;
                           commandEnd = 1;

src/htsweb.c

@@ -65,6 +65,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsserver.h"
 #include "htsurlport.h"
 #include "htsweb.h"
+#include "htscharset.h"
 
 #if USE_BEGINTHREAD==0
 #error fatal: no threads support
@@ -156,6 +157,7 @@ int main(int argc, char *argv[]) {
   printf("Initializing the server..\n");
 
 #ifdef _WIN32
+  hts_argv_utf8(&argc, &argv);
   {
     WORD wVersionRequested;     // requested version WinSock API
     WSADATA wsadata;            // Windows Sockets API data

src/minizip/mztools.c

@@ -14,7 +14,7 @@
 
 #define READ_8(adr)  ((unsigned char)*(adr))
 #define READ_16(adr) ( READ_8(adr) | (READ_8(adr+1) << 8) )
-#define READ_32(adr) ( READ_16(adr) | (READ_16((adr)+2) << 16) )
+#define READ_32(adr) ((uLong) READ_16(adr) | ((uLong) READ_16((adr) + 2) << 16))
 
 #define WRITE_8(buff, n) do { \
   *((unsigned char*)(buff)) = (unsigned char) ((n) & 0xff); \

tests/01_zlib-repair-shift.test

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# Keep this POSIX-portable: the harness runs it via $(BASH), which is a plain
+# POSIX /bin/sh on some platforms (e.g. macOS), so avoid bashisms and GNU-only
+# tool flags despite the #!/bin/bash above.
+
+# unzRepair header read must not overflow a signed shift (-#test=zip-repair-shift
+# <dir>). A damaged local file header whose CRC high word has bit 15 set made
+# READ_32 shift an int past INT_MAX; UBSan aborts before the fix casts to uLong.
+
+set -eu
+
+dir=$(mktemp -d)
+trap 'rm -rf "$dir"' EXIT
+
+out=$(httrack -#test=zip-repair-shift "$dir")
+
+printf '%s\n' "$out" | grep -qx "zip-repair-shift: OK (recovered 1 entry)" || {
+    echo "expected 'zip-repair-shift: OK (recovered 1 entry)', got: $out" >&2
+    exit 1
+}

tests/66_engine-port80-strip.test

@@ -0,0 +1,11 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Stripping a default :80 from a crawled link used to skip a hardcoded 3 chars
+# (#627): ":080"/":0080" lost only ":80" and glued the rest onto the host
+# (127.0.0.1:080/x -> 127.0.0.10/x), and a value wrapping to 80 as a 32-bit int
+# (#614) was stripped as if it were the default. All assertions live in the
+# engine self-test.
+httrack -O /dev/null -#test=stripport | grep -q "stripport self-test OK"

tests/67_engine-delayed-truncate.test

@@ -0,0 +1,72 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# #623: url_savename enforces the 236-char path ceiling by cutting the tail of
+# the last segment, where the mandatory ".delayed" placeholder marker lives. A
+# cut marker fails IS_DELAYED_EXT, so back_delayed_rename never renames the file
+# to its final name and the download is lost. The marker must survive the cut.
+
+# statuscode=302 status=-1 = a redirect answer still downloading: no type is
+# resolved, so the name gets a ".<id>.delayed" placeholder (see 01_engine-savename).
+CEIL=236
+
+httrack_bin=$(cd "$(dirname "$(command -v httrack)")" && pwd)/httrack
+scratch=$(mktemp -d)
+trap 'rm -rf "$scratch"' EXIT
+cd "$scratch"
+
+run() {
+    "$httrack_bin" -O /dev/null -#test=savename "$@" | sed -n 's/^savename: //p'
+}
+
+# A deep path ending in a long segment that overruns the ceiling.
+deep="/d1$(printf 'a%.0s' {1..40})/d2$(printf 'b%.0s' {1..40})"
+deep="$deep/d3$(printf 'c%.0s' {1..40})/d4$(printf 'd%.0s' {1..40})"
+long="$deep/$(printf 'z%.0s' {1..90})"
+
+out="$(run "$long" text/html statuscode=302 status=-1)"
+case "$out" in
+*.delayed) ;;
+*)
+    echo "FAIL: delayed marker cut by truncation: '$out'"
+    exit 1
+    ;;
+esac
+test "${#out}" -le "$CEIL" || {
+    echo "FAIL: truncated name ${#out} > $CEIL ceiling: '$out'"
+    exit 1
+}
+
+# #133-style hashed name: an all-hex last segment must keep the marker too (the
+# ".<id>." tag is not mistaken for part of the hash).
+hexseg=$(printf 'a1b2c3d4e5f60718%.0s' {1..8})
+hexpath="/d1$(printf 'x%.0s' {1..30})/d2$(printf 'y%.0s' {1..30})/$hexseg"
+out="$(run "$hexpath" text/html statuscode=302 status=-1)"
+case "$out" in
+*.delayed) ;;
+*)
+    echo "FAIL: hashed-name marker cut by truncation: '$out'"
+    exit 1
+    ;;
+esac
+test "${#out}" -le "$CEIL" || {
+    echo "FAIL: hashed name ${#out} > $CEIL ceiling: '$out'"
+    exit 1
+}
+
+# A non-delayed name of the same shape still truncates, with no marker to keep.
+out="$(run "$long.html" text/html)"
+test "${#out}" -le "$CEIL" || {
+    echo "FAIL: non-delayed name ${#out} > $CEIL ceiling: '$out'"
+    exit 1
+}
+case "$out" in
+*.delayed)
+    echo "FAIL: non-delayed name grew a .delayed marker: '$out'"
+    exit 1
+    ;;
+esac
+
+echo "delayed-truncate OK"

tests/68_webhttrack-outdir-charset.test

@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# webhttrack's POST body arrives in the form's declared charset, not UTF-8, yet
+# the engine now assumes UTF-8 argv. A non-ASCII -O output dir submitted through
+# the web UI must land under the UTF-8 directory, not an ISO-8859-1 twin (#629).
+# Drives the real htsserver over HTTP; the default (English) form is served
+# ISO-8859-1, so 'café' travels as the single byte 0xE9. The crawl target is a
+# dead port: only the -O directory name is under test, and htsserver creates it
+# from the decoded path before any fetch.
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
+distdir=$(cd "${distdir}" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+command -v htsserver >/dev/null || fail "no htsserver in PATH"
+python=$(find_python) || {
+    echo "python3 not found; skipping" >&2
+    exit 77
+}
+
+work=$(mktemp -d "${TMPDIR:-/tmp}/webhttrack_charset.XXXXXX") || fail "no tmpdir"
+srvlog=$(mktemp)
+srv=
+cleanup() {
+    # htsserver keeps SIGTERM ignored across its exec, so only -9 reaps it.
+    test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
+    wait "${srv}" 2>/dev/null || true # absorb bash's async "Killed" notice
+    rm -rf "${work}" "${srvlog}"
+}
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+# webhttrack server on a pre-picked port; an isolated HOME keeps a stray
+# ~/.httrack.ini out of it.
+sport=$("${python}" -c 'import socket
+s = socket.socket()
+s.bind(("127.0.0.1", 0))
+print(s.getsockname()[1])
+s.close()')
+(
+    trap '' TERM TTOU
+    export HOME="${work}"
+    exec htsserver "${distdir}/" --port "${sport}" >"${srvlog}" 2>&1
+) &
+srv=$!
+for _ in $(seq 1 40); do
+    url=$(sed -n 's/^URL=//p' "${srvlog}") && test -n "${url}" && break
+    kill -0 "${srv}" 2>/dev/null || break
+    sleep 0.25
+done
+test -n "${url:-}" || fail "htsserver did not start: $(cat "${srvlog}")"
+
+# Post a "start" whose -O dir is 'café' in the form's ISO-8859-1 charset.
+"${python}" - "${url}" "${work}" <<'PY'
+import sys, urllib.parse, urllib.request
+
+url, work = sys.argv[1], sys.argv[2]
+outdir = work + "/caf\xe9"  # 'café' as the single byte the browser would send
+# Port 1 refuses at once: only the decoded -O dir matters, not the fetch.
+cmd = "httrack --quiet --robots=0 http://127.0.0.1:1/x.html -O " + outdir
+fields = [("path", work), ("projname", "proj"), ("command_do", "start"),
+          ("winprofile", "x"), ("command", cmd)]
+body = "&".join("%s=%s" % (k, urllib.parse.quote(v, safe="", encoding="latin-1"))
+                for k, v in fields)
+req = urllib.request.Request(url + "step4.html", data=body.encode("latin-1"),
+                             method="POST")
+urllib.request.urlopen(req, timeout=20).read()
+PY
+
+# The crawl runs inside htsserver; wait for the -O dir, then check it chose the
+# UTF-8 name and not the ISO-8859-1 twin.
+utf8dir="${work}/café"
+latin1dir="${work}/caf"$'\xe9'
+for _ in $(seq 1 80); do
+    test -e "${utf8dir}" && break
+    kill -0 "${srv}" 2>/dev/null || break
+    sleep 0.25
+done
+test -e "${utf8dir}" || fail "-O dir never landed as UTF-8 café/: $(find "${work}" -maxdepth 2 2>/dev/null)"
+test ! -e "${latin1dir}" || fail "-O dir created as the ISO-8859-1 twin instead"
+
+echo "PASS"

tests/69_local-intl-logdir.test

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# A single non-ASCII -O sets both path_html and path_log to "café" (#630). The
+# logs (hts-log.txt/hts-err.txt) must land there, not in an ANSI-mangled twin:
+# on Windows path_log holds UTF-8 bytes the raw file calls read as the codepage.
+# The --errors/--files/--log-found audits all grep logroot=café/hts-log.txt, so
+# a log written to the twin fails them. POSIX has no twin, so this bites on the
+# Windows CI leg (like test 64). The cache twin is a separate, larger fix.
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --outdir-intl 'café' --errors 0 --files 5 \
+    --found 'simple/basic.html' \
+    --log-found 'mirror complete in' \
+    httrack 'BASEURL/simple/basic.html'

tests/71_local-crange-repaircache.test

@@ -0,0 +1,134 @@
+#!/bin/bash
+# #581: an interrupted mirror can leave a truncated new.zip that pass 2 must
+# repair before resuming. The repaired-cache resume then hits the same hostile
+# 206 as test 48, so restart-whole must still drop the partial and refetch the
+# whole file. Pass 1 leaves a partial + temp-ref; we truncate new.zip past its
+# last local entry (dropping the central directory) so unzOpen fails and the
+# repair path runs; pass 2 resumes, rejects the range, and refetches whole.
+set -u
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+# shellcheck source=tests/testlib.sh
+. "${testdir}/testlib.sh"
+server=$(nativepath "${testdir}/local-server.py")
+root=$(nativepath "${testdir}/server-root")
+
+python=$(find_python) || ! echo "python3 not found; skipping" >&2 || exit 77
+
+# On Windows the pass-1 interrupt is a hard kill (MSYS can't signal a native
+# exe) and the restart-whole path fails on the repaired cache (#581) -- the very
+# bug this exercises; skip until the engine fix lands.
+if is_windows; then
+    echo "Windows: restart-whole fails on a repaired cache (#581), skipping"
+    exit 77
+fi
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangerep.XXXXXX") || exit 1
+serverpid=
+crawlpid=
+cleanup() {
+    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
+    stop_server "$serverpid"
+    rm -rf "$tmpdir"
+}
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+serverlog="${tmpdir}/server.log"
+: >"$serverlog"
+"$python" "$server" --root "$root" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$serverlog" 2>/dev/null)
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$serverlog")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+base="http://127.0.0.1:${port}"
+
+which httrack >/dev/null || {
+    echo "could not find httrack"
+    exit 1
+}
+out="${tmpdir}/crawl"
+mkdir "$out"
+common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
+refdir="${out}/hts-cache/ref"
+newzip="${out}/hts-cache/new.zip"
+
+# --- pass 1: crawl, interrupt once the blob download is underway -------------
+printf '[pass 1: interrupt mid-download] ..\t'
+httrack "${common[@]}" "${base}/crange206mem/index.html" >"${tmpdir}/log1" 2>&1 &
+crawlpid=$!
+for _ in $(seq 1 300); do
+    test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
+    kill -0 "$crawlpid" 2>/dev/null || break
+    sleep 0.1
+done
+sleep 0.3
+kill -TERM "$crawlpid" 2>/dev/null
+wait "$crawlpid" 2>/dev/null
+crawlpid=
+test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
+    echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
+    exit 1
+}
+echo "OK (temp-ref present)"
+
+# --- damage the cache: drop the central directory a hard kill never wrote ----
+printf '[damage new.zip -> forces repair] ..\t'
+"$python" - "$newzip" <<'PY' || exit 1
+import os, sys
+path = sys.argv[1]
+data = open(path, "rb").read()
+cut = data.find(b"PK\x01\x02")  # first central-directory header
+if cut <= 0:
+    sys.exit("no central directory to drop in %s" % path)
+os.truncate(path, cut)
+PY
+echo "OK"
+
+# --- pass 2: --continue -> repair -> resume -> hostile 206 -> refetch whole ---
+printf '[pass 2: repair, reject range, refetch] ..\t'
+rc=0
+httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1 || rc=$?
+test "$rc" -eq 0 || {
+    echo "FAIL: httrack exited $rc"
+    cat "${tmpdir}/log2" >&2
+    exit 1
+}
+echo "OK (terminated)"
+
+# The repair path must actually have run, else this is just a copy of test 48.
+printf '[cache repair fired] ..\t'
+grep -q 'damaged cache' "${out}/hts-log.txt" 2>/dev/null || {
+    echo "FAIL: repair path did not run; damage was ineffective"
+    exit 1
+}
+echo "OK"
+
+blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
+full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
+
+printf '[file recovered whole] ..\t'
+test -s "$blob" || {
+    echo "FAIL: blob.bin missing after the repaired-cache resume"
+    exit 1
+}
+got=$(wc -c <"$blob")
+test "$got" -eq "$full" || {
+    echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
+    exit 1
+}
+echo "OK (${got} bytes)"

tests/Makefile.am

@@ -84,6 +84,7 @@ TESTS = \
 	01_zlib-cache-legacy.test \
 	01_zlib-cache-golden.test \
 	01_zlib-cache-writefail.test \
+	01_zlib-repair-shift.test \
 	01_zlib-savename-cached.test \
 	02_manpage-regen.test \
 	02_update-cache.test \
@@ -146,6 +147,11 @@ TESTS = \
 	62_lang-integrity.test \
 	63_webhttrack-home.test \
 	64_local-intl-outdir.test \
-	65_port-siblings.test
+	65_port-siblings.test \
+	66_engine-port80-strip.test \
+	67_engine-delayed-truncate.test \
+	68_webhttrack-outdir-charset.test \
+	69_local-intl-logdir.test \
+	71_local-crange-repaircache.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -48,6 +48,7 @@ key="${testdir}/server.key"
 tls=
 verbose=
 html_subdir=
+outdir_intl=
 rerun=
 rerun_args=
 rerun_dead=
@@ -140,6 +141,13 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         html_subdir="${args[$pos]}"
         ;;
+    --outdir-intl)
+        # Single non-ASCII -O "$out/NAME": path_html AND path_log are NAME, so
+        # the logs (and the harness reads of them) go through the non-ASCII path
+        # (#630). Distinct from --html-subdir, which keeps path_log ASCII.
+        pos=$((pos + 1))
+        outdir_intl="${args[$pos]}"
+        ;;
     --errors | --errors-content | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
@@ -216,13 +224,18 @@ test -n "$ver" || die "could not run httrack"
 out="${tmpdir}/crawl"
 mkdir "$out" || die "could not create $out"
 # path_html holds the mirror + index; path_log holds hts-cache/hts-log.txt.
-# Default: both are "$out". With --html-subdir, path_html becomes "$out/NAME"
-# (the mirror root the audits inspect) while path_log stays "$out".
+# Default: both are "$out". --html-subdir moves path_html to "$out/NAME" while
+# path_log (logroot) stays "$out"; --outdir-intl moves both to "$out/NAME".
 mirrorroot="$out"
+logroot="$out"
 odir="$out"
 if test -n "$html_subdir"; then
     mirrorroot="${out}/${html_subdir}"
     odir="${mirrorroot},${out}"
+elif test -n "$outdir_intl"; then
+    mirrorroot="${out}/${outdir_intl}"
+    logroot="$mirrorroot"
+    odir="$mirrorroot"
 fi
 # Localhost is fast; disable the rate/bandwidth safety limits but keep a
 # max-time backstop so a hang cannot wedge the suite.
@@ -241,7 +254,7 @@ test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || {
     exit 1
 }
 result "OK"
-grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
+grep -iE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt" >&2
 
 # --- optional second pass: re-mirror into the same dir (cache/update path) ----
 if test -n "$rerun"; then
@@ -260,7 +273,7 @@ if test -n "$rerun"; then
     # The update summary reports "files updated"; a fresh crawl never does. Assert
     # it so a regression that bypasses the cache (re-crawls fresh) can't pass.
     info "checking update used the cache"
-    if grep -aqE "mirror complete in .*files updated" "${out}/hts-log.txt"; then
+    if grep -aqE "mirror complete in .*files updated" "${logroot}/hts-log.txt"; then
         result "OK"
     else
         result "update pass did not report cache activity"
@@ -292,7 +305,7 @@ if test -n "$rerun_dead"; then
     zip="${out}/hts-cache/new.zip"
     test -s "$zip" || die "no cache was written by the first pass"
     cp "$zip" "${tmpdir}/cache-before.zip"
-    cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
+    cp "${logroot}/hts-log.txt" "${tmpdir}/log-before.txt"
     stop_server "$serverpid"
     serverpid=
     info "re-running httrack against the stopped server"
@@ -305,7 +318,7 @@ if test -n "$rerun_dead"; then
     # The dead pass must have gone through the no-data rollback, not bailed out
     # before the mirror loop (which would leave the cache trivially untouched).
     info "checking the dead pass hit the rollback"
-    if grep -aq "No data seems to have been transferred" "${out}/hts-log.txt"; then
+    if grep -aq "No data seems to have been transferred" "${logroot}/hts-log.txt"; then
         result "OK"
     else
         result "rollback notice not found in hts-log.txt"
@@ -320,7 +333,7 @@ if test -n "$rerun_dead"; then
         exit 1
     fi
     # Audits below describe the healthy crawl, not the dead pass.
-    cp "${tmpdir}/log-before.txt" "${out}/hts-log.txt"
+    cp "${tmpdir}/log-before.txt" "${logroot}/hts-log.txt"
 fi
 
 # --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
@@ -350,19 +363,19 @@ while test "$i" -lt "${#audit[@]}"; do
     --errors)
         i=$((i + 1))
         assert_equals "checking errors" "${audit[$i]}" \
-            "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
+            "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")"
         ;;
     --errors-content)
         i=$((i + 1))
-        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
+        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")
         # transient network failures (statuscode -2..-6) flake on busy loopback;
         # the code parens are followed by " at link" or " after N retries at link"
-        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
+        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${logroot}/hts-log.txt" || true)
         assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
         ;;
     --files)
         i=$((i + 1))
-        nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
+        nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${logroot}/hts-log.txt" |
             sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
         assert_equals "checking files" "${audit[$i]}" "$nFiles"
         ;;
@@ -393,7 +406,7 @@ while test "$i" -lt "${#audit[@]}"; do
     --log-found)
         i=$((i + 1))
         info "checking log matches ${audit[$i]}"
-        if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then result "OK"; else
+        if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then result "OK"; else
             result "not in log"
             exit 1
         fi
@@ -401,7 +414,7 @@ while test "$i" -lt "${#audit[@]}"; do
     --log-not-found)
         i=$((i + 1))
         info "checking log lacks ${audit[$i]}"
-        if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then
+        if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then
             result "present in log"
             exit 1
         else result "OK"; fi