tests: assert the retry exhaustion, not the per-platform failure message

The message a cut connection produces depends on whether any bytes arrived, so
matching it would fail on a runner that sees none.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htscodec.c

@@ -72,7 +72,7 @@ hts_codec hts_codec_parse(const char *encoding) {
     return HTS_CODEC_IDENTITY;
   if (strfield2(encoding, "gzip") || strfield2(encoding, "x-gzip") ||
       strfield2(encoding, "deflate") || strfield2(encoding, "x-deflate"))
-    return HTS_CODEC_DEFLATE;
+    return HTS_USEZLIB ? HTS_CODEC_DEFLATE : HTS_CODEC_UNSUPPORTED;
   if (strfield2(encoding, "br"))
     return HTS_USEBROTLI ? HTS_CODEC_BROTLI : HTS_CODEC_UNSUPPORTED;
   if (strfield2(encoding, "zstd"))
@@ -98,11 +98,16 @@ hts_codec hts_codec_parse(const char *encoding) {
 const char *hts_acceptencoding(hts_boolean compressible, hts_boolean secure) {
   if (!compressible)
     return "identity";
+#if HTS_USEZLIB
   /* br and zstd over TLS only, as browsers do: a cleartext intermediary that
      rewrites a coding it can not read would corrupt the mirror. */
   if (secure)
     return "gzip, deflate" HTS_AE_BROTLI HTS_AE_ZSTD ", identity;q=0.9";
   return "gzip, deflate, identity;q=0.9";
+#else
+  (void) secure;
+  return "identity";
+#endif
 }
 
 hts_boolean hts_codec_is_archive_ext(hts_codec codec, const char *ext) {
@@ -295,7 +300,11 @@ int hts_codec_unpack(hts_codec codec, const char *filename,
     return -1;
   switch (codec) {
   case HTS_CODEC_DEFLATE:
+#if HTS_USEZLIB
     return hts_zunpack(filename, newfile);
+#else
+    return -1;
+#endif
   case HTS_CODEC_BROTLI:
   case HTS_CODEC_ZSTD:
     break;
@@ -339,8 +348,10 @@ size_t hts_codec_head(hts_codec codec, const void *in, size_t in_len, void *out,
   if (in == NULL || in_len == 0 || out == NULL || out_len == 0)
     return 0;
   switch (codec) {
+#if HTS_USEZLIB
   case HTS_CODEC_DEFLATE:
     return hts_zhead(in, in_len, out, out_len);
+#endif
 #if HTS_USEBROTLI
   case HTS_CODEC_BROTLI:
     return codec_head_brotli(in, in_len, out, out_len);

src/htscore.c

@@ -1980,10 +1980,9 @@ int httpmirror(char *url1, httrackp * opt) {
           }
 
           // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
-          // An empty body must not overwrite the file when the transfer failed
-          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
-          // copy to 0 (#77 follow-up).
-          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
+          // A failed transfer has no body: r.adr holds debris from the aborted
+          // read, which would destroy the copy being re-fetched (#748).
+          if (r.statuscode > 0 && (r.adr != NULL || r.size == 0)) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {

src/htslib.c

@@ -1131,10 +1131,12 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
 
       // Compression accepted ?
       if (retour->req.http11) {
-        hts_boolean compressible =
-            (!retour->req.range_used && !retour->req.nocompression);
+        hts_boolean compressible = HTS_FALSE;
         hts_boolean secure = HTS_FALSE;
 
+#if HTS_USEZLIB
+        compressible = (!retour->req.range_used && !retour->req.nocompression);
+#endif
 #if HTS_USEOPENSSL
         secure = retour->ssl ? HTS_TRUE : HTS_FALSE;
 #endif

src/htsname.c

@@ -43,7 +43,9 @@ Please visit our Website: http://www.httrack.com
 #include "htsencoding.h"
 #include "htssniff.h"
 #include "htscodec.h"
+#if HTS_USEZLIB
 #include "htszlib.h"
+#endif
 #include <ctype.h>
 #include <limits.h>
 

src/htsselftest.c

@@ -63,7 +63,9 @@ Please visit our Website: http://www.httrack.com
 #include "htswarc.h"
 #include "htschanges.h"
 #include "htssinglefile.h"
+#if HTS_USEZLIB
 #include "htszlib.h"
+#endif
 #if HTS_USEZSTD
 #include <zstd.h>
 #endif
@@ -3375,6 +3377,7 @@ static int st_status(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+#if HTS_USEZLIB
 /* Deflate src->path at windowBits (16+ gzip, + zlib, - raw); 0 on success. */
 static int ae_write_packed(const char *path, int windowBits,
                            const unsigned char *src, size_t len) {
@@ -3448,6 +3451,7 @@ static int ae_write_collision(const char *path, const unsigned char *src,
   freet(buf);
   return ok ? 0 : 1;
 }
+#endif
 
 /* Write src[0..len) to path as-is; 0 on success. */
 static int ae_write_raw(const char *path, const unsigned char *src,
@@ -3497,6 +3501,7 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
   assertf(strstr(on, "br") == NULL && strstr(on, "zstd") == NULL);
   assertf((strstr(tls, ", br") != NULL) == (HTS_USEBROTLI != 0));
   assertf((strstr(tls, "zstd") != NULL) == (HTS_USEZSTD != 0));
+#if HTS_USEZLIB
   if (argc >= 1) {
     static const int windowBits[] = {16 + MAX_WBITS, MAX_WBITS, -MAX_WBITS};
     const unsigned char small[] =
@@ -3575,6 +3580,10 @@ static int st_acceptencoding(httrackp *opt, int argc, char **argv) {
     }
     freet(body);
   }
+#else
+  (void) argc;
+  (void) argv;
+#endif
   printf("acceptencoding self-test OK: %s\n", on);
   return 0;
 }
@@ -3919,6 +3928,7 @@ static int st_sitemap(httrackp *opt, int argc, char **argv) {
     freet(big);
   }
 
+#if HTS_USEZLIB
   /* A highly compressible document decodes without running away: the ratio
      budget cannot bind (deflate tops out near 1032:1), so this pins the
      decompression path itself rather than the 64 MiB ceiling. */
@@ -3969,11 +3979,13 @@ static int st_sitemap(httrackp *opt, int argc, char **argv) {
     freet(z);
     freet(x);
   }
+#endif
 
   /* An unterminated <loc> at end of buffer must not read past it. */
   assertf(sm_scan("<urlset><loc>http://h.test/a", 100, &idx, &c) == 0);
   assertf(sm_scan("<urlset><lo", 100, &idx, &c) == 0);
 
+#if HTS_USEZLIB
   /* A gzip-framed document is decompressed before scanning. */
   {
     const char *const xml =
@@ -4003,6 +4015,7 @@ static int st_sitemap(httrackp *opt, int argc, char **argv) {
     assertf(hts_sitemap_scan(z, 4, 100, &idx, sm_take, &c) == -1);
     freet(z);
   }
+#endif
 
   /* robots.txt: only Sitemap: records, comments stripped, case-insensitive,
      and group-independent (no User-agent line needed). */

src/htsweb.c

@@ -224,7 +224,9 @@ int main(int argc, char *argv[]) {
 #ifdef HTS_USESWF
   smallserver_setkey("USESWF", "1");
 #endif
+#ifdef HTS_USEZLIB
   smallserver_setkey("USEZLIB", "1");
+#endif
 #ifdef _WIN32
   smallserver_setkey("WIN32", "1");
 #endif

src/htszlib.c

@@ -40,6 +40,7 @@ Please visit our Website: http://www.httrack.com
 #include "htscodec.h"
 #include "htszlib.h"
 
+#if HTS_USEZLIB
 /* zlib */
 /*
 #include <zlib.h>
@@ -273,3 +274,4 @@ const char *hts_get_zerror(int err) {
     break;
   }
 }
+#endif

tests/93_local-changes.test

@@ -94,10 +94,7 @@ expect_counts_match() {
     echo "OK"
 }
 lines() { printf '%s\n' "$@" | sort; }
-# A connection killed before the status line surfaces differently per platform
-# (macOS truncates the file, #748; Linux leaves it), so reset.bin is asserted on
-# its own and kept out of the exact lists.
-listed_but_reset() { listed "$1" | grep -v '/reset\.bin$' | sort; }
+digest() { cksum <"$1" | tr -d '[:space:]'; }
 
 # --- pass 1: nothing to compare against --------------------------------------
 httrack "${common[@]}" -O "$out" --purge-old=0 "${base}/changes/index.html" \
@@ -120,6 +117,7 @@ grep -aq "first crawl" "${out}/hts-log.txt" || {
 }
 
 host="127.0.0.1_${port}"
+resetdigest=$(digest "${out}/${host}/changes/reset.bin")
 # A leftover from some earlier failed attempt, at a name pass 2 fetches for the
 # first time: on disk, but never part of the previous mirror, so it is new.
 printf 'leftover junk' >"${out}/${host}/changes/fresh.html"
@@ -144,21 +142,19 @@ expect "gone is doomed.html and the old redirect target" \
 expect "changed is exactly the four moved resources" \
     "$(lines "${host}/changes/coded.bin" "${host}/changes/index.html" \
         "${host}/changes/moved.bin" "${host}/changes/moved.html")" \
-    "$(listed_but_reset changed)"
+    "$(listed changed | sort)"
 # Re-served byte for byte, 200, no Last-Modified, no ETag. flaky.bin gets there
 # through a failed transfer and a retry, so its file is notified twice;
 # redirtarget.html arrives behind a 302. sized.html has a byte-identical
 # payload, but the rewritten link to the renamed redirect target makes the file
-# on disk change length.
-expect "unchanged is exactly the six stable resources" \
+# on disk change length. reset.bin never completes a transfer (#746), so its
+# previous copy stands.
+expect "unchanged is exactly the seven stable resources" \
     "$(lines "${host}/changes/codedstable.bin" "${host}/changes/flaky.bin" \
-        "${host}/changes/redirtarget.html" "${host}/changes/sized.html" \
-        "${host}/changes/stable.bin" "${host}/changes/stable.html")" \
-    "$(listed_but_reset unchanged)"
-# reset.bin never completes a transfer, so it drops out of new.lst with its
-# mirrored copy still there. Whatever else it is, it is not a deletion.
-expect "a failed re-fetch is never reported gone" "" \
-    "$(listed gone | grep '/reset\.bin$' || true)"
+        "${host}/changes/redirtarget.html" "${host}/changes/reset.bin" \
+        "${host}/changes/sized.html" "${host}/changes/stable.bin" \
+        "${host}/changes/stable.html")" \
+    "$(listed unchanged | sort)"
 expect_counts_match "pass 2 counts match its lists"
 
 # Every mirrored file appears once and only once, across all four buckets.
@@ -192,19 +188,15 @@ test -f "${out}/${host}/changes/doomed.html" || {
     exit 1
 }
 echo "OK"
-printf '[a failed transfer keeps its file] ..\t'
-test -f "${out}/${host}/changes/reset.bin" || {
-    echo "FAIL: reset.bin lost its previous copy"
-    exit 1
-}
-echo "OK"
+expect "a failed transfer keeps its bytes" "$resetdigest" \
+    "$(digest "${out}/${host}/changes/reset.bin")"
 
 # --- pass 3: the same run with purging on ------------------------------------
 httrack "${common[@]}" -O "$out" --update "${base}/changes/index.html" \
     >"${tmpdir}/log3" 2>&1
 expect "the report says files were purged" "true" "$(field purged)"
 expect "gone is the page that only pass 2 linked" \
-    "${host}/changes/transient.html" "$(listed_but_reset gone)"
+    "${host}/changes/transient.html" "$(listed gone | sort)"
 printf '[a purged file is off disk] ..\t'
 test ! -f "${out}/${host}/changes/transient.html" || {
     echo "FAIL: transient.html survived --purge-old"

tests/96_local-refetch-keep.test

@@ -0,0 +1,28 @@
+#!/bin/bash
+#
+# A re-fetch cut mid-header must leave the mirrored file alone (#748): unfixed,
+# the aborted read's leftover buffer lands on it. err.bin is the control, an
+# HTTP 500 on the same resource already keeping its copy; stay.bin answers
+# normally with a new body, so a fix that stopped overwriting anything fails.
+#
+# Purging is off: an update purge deletes both files for an unrelated reason
+# (#746), which would hide what this asserts.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update --purge-old=0' \
+    --log-found 'after 2 retries at link .*keep/page\.html' \
+    --log-found 'after 2 retries at link .*keep/data\.bin' \
+    --file-matches 'keep/page.html' 'KEEP-PAGE-V1' \
+    --file-not-matches 'keep/page.html' 'HTTP/1\.0 200' \
+    --file-matches 'keep/data.bin' 'KEEP-BIN-V1' \
+    --file-not-matches 'keep/data.bin' 'HTTP/1\.0 200' \
+    --file-min-bytes 'keep/data.bin' 2048 \
+    --file-matches 'keep/err.bin' 'KEEP-ERR-V1' \
+    --file-min-bytes 'keep/err.bin' 2048 \
+    --file-matches 'keep/stay.bin' 'KEEP-STAY-V2' \
+    --file-min-bytes 'keep/stay.bin' 2048 \
+    httrack 'BASEURL/keep/index.html' --retries=2

tests/Makefile.am

@@ -194,6 +194,7 @@ TESTS = \
 	92_local-proxytrack-ndx-fields.test \
 	93_local-changes.test \
 	94_local-single-file.test \
-	95_local-sitemap.test
+	95_local-sitemap.test \
+	96_local-refetch-keep.test
 
 CLEANFILES = check-network_sh.cache

tests/local-server.py

@@ -1000,6 +1000,62 @@ class Handler(SimpleHTTPRequestHandler):
         v = 1 if self.refetch_pass() == 1 else 2
         self.send_raw(b"<html><body><p>STAY-V%d</p></body></html>" % v, "text/html")
 
+    # --- re-fetch cut mid-header, so nothing is stored (#746, #748) ---------
+    KEEP_PAGE = b"<html><body><p>KEEP-PAGE-V1</p></body></html>"
+    KEEP_BIN = b"KEEP-BIN-V1\n" + b"\x51\x52\x53\x54" * 512
+    KEEP_ERR = b"KEEP-ERR-V1\n" + b"\x61\x62\x63\x64" * 512
+
+    def send_cut_headers(self):
+        """Hang up mid-header: the response never becomes parseable."""
+        self.close_connection = True
+        try:
+            self.wfile.write(b"HTTP/1.0 200 OK\r\nContent-Ty")
+            self.wfile.flush()
+        except OSError:
+            pass
+        self.connection.close()
+
+    def route_keep_index(self):
+        self.refetch_pass()
+        self.send_html(
+            '\t<a href="page.html">page</a>\n'
+            '\t<a href="data.bin">data</a>\n'
+            '\t<a href="err.bin">err</a>\n'
+            '\t<a href="stay.bin">stay</a>\n'
+        )
+
+    def route_keep_page(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.KEEP_PAGE, "text/html")
+        else:
+            self.send_cut_headers()
+
+    def route_keep_data(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.KEEP_BIN, "application/octet-stream")
+        else:
+            self.send_cut_headers()
+
+    # Control: an HTTP error on the same resource already keeps the copy, so
+    # the cut-header routes above must end up indistinguishable from it.
+    def route_keep_err(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.KEEP_ERR, "application/octet-stream")
+        else:
+            self.send_response(500)
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", "0")
+            self.end_headers()
+
+    # Control: answers normally, with a new body on pass 2, so a fix that
+    # stopped overwriting mirrored files altogether would be caught.
+    def route_keep_stay(self):
+        v = 1 if self.refetch_pass() == 1 else 2
+        self.send_raw(
+            b"KEEP-STAY-V%d\n" % v + b"\x71\x72\x73\x74" * 512,
+            "application/octet-stream",
+        )
+
     # Echo what httrack advertised, so a crawl can assert the header.
     def route_codec_ae(self):
         self.send_raw(
@@ -1950,6 +2006,11 @@ class Handler(SimpleHTTPRequestHandler):
         "/uptrunc/page.html": route_uptrunc_page,
         "/uptrunc/file.bin": route_uptrunc_file,
         "/uptrunc/stay.html": route_uptrunc_stay,
+        "/keep/index.html": route_keep_index,
+        "/keep/page.html": route_keep_page,
+        "/keep/data.bin": route_keep_data,
+        "/keep/err.bin": route_keep_err,
+        "/keep/stay.bin": route_keep_stay,
         "/types/index.html": route_types_index,
         "/types/control.php": route_types,
         "/types/photo.png": route_types,