tests: share one fetch-pass counter across the update-refetch routes

The #562 /uptrunc/ routes copied upcodec's per-path body-fetch counter verbatim. Fold both onto one refetch_pass() helper and one dict. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com>
Reflow the #562 comment to satisfy clang-format
2026-07-16 13:51:46 +03:00 · 2026-07-15 09:39:51 +02:00 · 2026-07-15 08:59:29 +02:00 · 2026-07-15 08:54:16 +02:00
4 changed files with 88 additions and 12 deletions
--- a/src/htsback.c
+++ b/src/htsback.c
@@ -628,6 +628,10 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                      back[p].r.size, back[p].url_adr, back[p].url_fil);
      }
      back_finalize_backup(opt, &back[p], HTS_FALSE);
+      /* Keep the surviving copy in new.lst, else the update purge drops the
+         file we refused to overwrite with the partial body (#562). */
+      if (fexist_utf8(back[p].url_sav))
+        filenote(&opt->state.strc, back[p].url_sav, NULL);
      return -1;
    }

--- a/tests/54_local-update-truncate-purge.test
+++ b/tests/54_local-update-truncate-purge.test
@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# An --update re-fetch that comes in short must not delete the mirrored file (#562).
+# Pass 1 mirrors page.html (in-memory) and file.bin (direct-to-disk); pass 2 serves
+# both with the full declared Content-Length but half the body, then closes. httrack
+# refuses the partial ("will be retried") — and unfixed, the end-of-update purge then
+# deletes the good pass-1 copy, which was never overwritten. stay.html is the control:
+# fully served both passes, it must update to V2.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update' \
+    --log-found 'incomplete transfer.*uptrunc/page\.html' \
+    --log-found 'incomplete transfer.*uptrunc/file\.bin' \
+    --file-matches 'uptrunc/page.html' 'MIRRORED-PAGE-V1' \
+    --file-matches 'uptrunc/file.bin' 'MIRRORED-BIN-V1' \
+    --file-min-bytes 'uptrunc/file.bin' 32768 \
+    --file-matches 'uptrunc/stay.html' 'STAY-V2' \
+    --file-not-matches 'uptrunc/stay.html' 'STAY-V1' \
+    httrack 'BASEURL/uptrunc/index.html'
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -130,6 +130,7 @@ TESTS = \
 	50_local-contentcodings.test \
 	51_local-update-codec.test \
 	52_local-socks5.test \
-	53_local-proxytrack-cache-corrupt.test
+	53_local-proxytrack-cache-corrupt.test \
+	54_local-update-truncate-purge.test

 CLEANFILES = check-network_sh.cache
--- a/tests/local-server.py
+++ b/tests/local-server.py
@@ -700,15 +700,17 @@ class Handler(SimpleHTTPRequestHandler):
    # Pass 1 mirrors each file from a valid gzip body; pass 2 (--update) serves
    # a body that cannot be decoded. The previously-mirrored copy must survive.
    # fresh.html is the control: its pass-2 body decodes, so it must be updated.
-    UPCODEC_SEEN = {}
+    # Per-path body-fetch counter shared by the update-refetch routes; paths are
+    # distinct so one dict serves all of them.
+    REFETCH_SEEN = {}

-    def upcodec_pass(self):
-        """1 on the first body fetch of this path, 2 on the next ones. HEADs
-        don't count, so a stray one can't shift which pass gets the bad body."""
+    def refetch_pass(self):
+        """1 on the first body fetch of this path, N on the Nth. HEADs don't
+        count, so a stray one can't shift which pass gets the special body."""
        if self.command == "HEAD":
            return 1
-        seen = Handler.UPCODEC_SEEN.get(self.path, 0) + 1
-        Handler.UPCODEC_SEEN[self.path] = seen
+        seen = Handler.REFETCH_SEEN.get(self.path, 0) + 1
+        Handler.REFETCH_SEEN[self.path] = seen
        return seen

    @staticmethod
@@ -740,36 +742,78 @@ class Handler(SimpleHTTPRequestHandler):
    UNSUP_V1 = b"<html><body><p>MIRRORED-UNSUP-V1</p></body></html>"

    def route_upcodec_mem(self):
-        if self.upcodec_pass() == 1:
+        if self.refetch_pass() == 1:
            self.send_coded(self.gzipped(self.MEM_V1), "text/html")
        else:
            self.send_coded(self.bad_gzip(self.MEM_V1), "text/html")

    def route_upcodec_disk(self):
-        if self.upcodec_pass() == 1:
+        if self.refetch_pass() == 1:
            self.send_coded(self.gzipped(self.DISK_V1), "application/octet-stream")
        else:
            self.send_coded(self.bad_gzip(self.DISK_V1), "application/octet-stream")

    # Pass 2 switches to a coding we have no decoder for.
    def route_upcodec_unsup(self):
-        if self.upcodec_pass() == 1:
+        if self.refetch_pass() == 1:
            self.send_coded(self.gzipped(self.UNSUP_V1), "text/html")
        else:
            self.send_coded(self.UNSUP_V1, "text/html", coding="compress")

    def route_upcodec_fresh(self):
-        pass1 = self.upcodec_pass() == 1
+        pass1 = self.refetch_pass() == 1
        body = b"<html><body><p>FRESH-V%d</p></body></html>" % (1 if pass1 else 2)
        self.send_coded(self.gzipped(body), "text/html")

    # Same, direct-to-disk: the update pass decodes, so the temp is renamed over
    # an existing mirror file.
    def route_upcodec_freshdisk(self):
-        pass1 = self.upcodec_pass() == 1
+        pass1 = self.refetch_pass() == 1
        body = b"FRESHDISK-V%d\n" % (1 if pass1 else 2) + b"\x03\x02\x01\xfe" * 8192
        self.send_coded(self.gzipped(body), "application/octet-stream")

+    # #562: pass 1 mirrors fully; pass 2 (--update) declares the full
+    # Content-Length but delivers half then closes, so httrack refuses the partial.
+    PAGE_V1 = b"<html><body><p>MIRRORED-PAGE-V1</p></body></html>"
+    BIN_V1 = b"MIRRORED-BIN-V1\n" + b"\x00\x01\x02\xff" * 8192
+
+    def route_uptrunc_index(self):
+        self.send_html(
+            '\t<a href="page.html">page</a>\n'
+            '\t<a href="file.bin">file</a>\n'
+            '\t<a href="stay.html">stay</a>\n'
+        )
+
+    def send_truncated(self, body, content_type):
+        self.send_response(200)
+        self.send_header("Content-Type", content_type)
+        self.send_header("Content-Length", str(len(body)))
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            self.wfile.write(body[: len(body) // 2])  # short, then close
+            self.wfile.flush()
+        except OSError:
+            pass
+
+    def route_uptrunc_page(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.PAGE_V1, "text/html")
+        else:
+            self.send_truncated(self.PAGE_V1, "text/html")
+
+    def route_uptrunc_file(self):
+        if self.refetch_pass() == 1:
+            self.send_raw(self.BIN_V1, "application/octet-stream")
+        else:
+            self.send_truncated(self.BIN_V1, "application/octet-stream")
+
+    # Control: fully served both passes, so a normal --update still lands.
+    def route_uptrunc_stay(self):
+        v = 1 if self.refetch_pass() == 1 else 2
+        self.send_raw(b"<html><body><p>STAY-V%d</p></body></html>" % v, "text/html")
+
    # Echo what httrack advertised, so a crawl can assert the header.
    def route_codec_ae(self):
        self.send_raw(
@@ -1473,6 +1517,10 @@ class Handler(SimpleHTTPRequestHandler):
        "/upcodec/unsup.html": route_upcodec_unsup,
        "/upcodec/fresh.html": route_upcodec_fresh,
        "/upcodec/freshdisk.bin": route_upcodec_freshdisk,
+        "/uptrunc/index.html": route_uptrunc_index,
+        "/uptrunc/page.html": route_uptrunc_page,
+        "/uptrunc/file.bin": route_uptrunc_file,
+        "/uptrunc/stay.html": route_uptrunc_stay,
        "/types/index.html": route_types_index,
        "/types/control.php": route_types,
        "/types/photo.png": route_types,