Don't destroy already-complete files when -M/-E aborts an --update

An --update run stopped by the -M size cap (or -E time cap) could wipe out
files that were already fully mirrored, via two paths:

  - An incomplete re-fetch opens the real file "wb" (truncating the good copy)
    before any body byte arrives; the hard-abort then leaves it truncated.
  - A file that fully arrives is committed, but the aborted slot then reaches
    the main-loop save with statuscode <= 0 and size 0, and filesave() writes a
    zero-byte file over it -- leaving the file empty on disk while the cache
    still records it 200 OK, so a later --update never re-fetches it (permanent
    loss).

For the first path, a re-fetch over an existing file now moves the good copy
aside before truncating; back_finalize commits the fresh file on a full body and
restores the previous copy on an aborted/incomplete transfer. url_sav is still
written the normal way, so the file list, update purge and cache bookkeeping are
unchanged. For the second, an empty body only overwrites the file on a real
response (statuscode > 0), never on a failed transfer.

Adds 43_local-update-truncate.test with a stateful bigtrunc fixture that
mirrors, then re-fetches under --update -M and asserts both the completed and
the incomplete-refetch file keep their full size across the abort. The harness
gains --rerun-args (a second pass with extra args) and --file-min-bytes.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsback.c

@@ -533,22 +533,22 @@ int back_nsoc_overall(const struct_back * sback) {
 
 /* generate temporary file on lien_back */
 /* Note: utf-8 */
-static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
+static int create_back_tmpfile(httrackp *opt, lien_back *const back,
+                               const char *ext) {
   // do not use tempnam() but a regular filename
   back->tmpfile_buffer[0] = '\0';
   if (back->url_sav != NULL && back->url_sav[0] != '\0') {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.z", 
-             back->url_sav);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s.%s",
+             back->url_sav, ext);
     back->tmpfile = back->tmpfile_buffer;
     if (structcheck(back->tmpfile) != 0) {
-      hts_log_print(opt, LOG_WARNING, "can not create directory to %s", 
+      hts_log_print(opt, LOG_WARNING, "can not create directory to %s",
                     back->tmpfile);
       return -1;
     }
   } else {
-    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer),
-             "%s/tmp%d.z", StringBuff(opt->path_html_utf8),
-             opt->state.tmpnameid++);
+    snprintf(back->tmpfile_buffer, sizeof(back->tmpfile_buffer), "%s/tmp%d.%s",
+             StringBuff(opt->path_html_utf8), opt->state.tmpnameid++, ext);
     back->tmpfile = back->tmpfile_buffer;
   }
   /* OK */
@@ -556,6 +556,32 @@ static int create_back_tmpfile(httrackp * opt, lien_back *const back) {
   return 0;
 }
 
+/* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
+   existing file moved the good copy to back->tmpfile before truncating url_sav.
+   commit keeps the new file and drops the backup; else restore it so an aborted
+   transfer leaves the previous copy intact. Skips the zlib .z temp. */
+static void back_finalize_backup(httrackp *opt, lien_back *const back,
+                                 const hts_boolean commit) {
+  if (back->tmpfile == NULL || back->r.compressed)
+    return;
+  if (commit) {
+    (void) UNLINK(back->tmpfile); /* new copy is good; drop the backup */
+  } else {
+    if (back->r.out != NULL) {
+      fclose(back->r.out);
+      back->r.out = NULL;
+    }
+    (void) UNLINK(back->url_sav); /* drop the failed partial */
+    /* On rename failure keep the backup: it still holds the good copy, so
+       losing it would be worse than an orphaned temp. */
+    if (RENAME(back->tmpfile, back->url_sav) != 0)
+      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
+                    "could not restore %s; previous copy kept as %s",
+                    back->url_sav, back->tmpfile);
+  }
+  back->tmpfile = NULL;
+}
+
 // objet (lien) téléchargé ou transféré depuis le cache
 //
 // fermer les paramètres de transfert,
@@ -591,6 +617,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                       LLintP " got " LLintP "): %s%s", back[p].r.totalsize,
                       back[p].r.size, back[p].url_adr, back[p].url_fil);
       }
+      back_finalize_backup(opt, &back[p], HTS_FALSE);
       return -1;
     }
 
@@ -609,7 +636,7 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
             // en mémoire -> passage sur disque
             if (!back[p].r.is_write) {
               // do not use tempnam() but a regular filename
-              if (create_back_tmpfile(opt, &back[p]) == 0) {
+              if (create_back_tmpfile(opt, &back[p], "z") == 0) {
                 assertf(back[p].tmpfile != NULL);
                 /* note: tmpfile is utf-8 */
                 back[p].r.out = FOPEN(back[p].tmpfile, "wb");
@@ -682,6 +709,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
           }
         }
 #endif
+        /* Body fully received: keep the freshly written url_sav, drop the
+           backup of the previous copy. */
+        back_finalize_backup(opt, &back[p], HTS_TRUE);
         /* Write mode to disk */
         if (back[p].r.is_write && back[p].r.adr != NULL) {
           freet(back[p].r.adr);
@@ -910,6 +940,9 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
       }
     }
   }
+  /* Aborted, error, or not ready: url_sav (if written) is broken; restore the
+     previous copy from the backup. */
+  back_finalize_backup(opt, &back[p], HTS_FALSE);
   return -1;
 }
 
@@ -2913,7 +2946,8 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                 strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "gz") == 0
                                 && strfield(get_ext(catbuff, sizeof(catbuff), back[i].url_sav), "tgz") == 0
                               ) {
-                              if (create_back_tmpfile(opt, &back[i]) == 0) {
+                              if (create_back_tmpfile(opt, &back[i], "z") ==
+                                  0) {
                                 assertf(back[i].tmpfile != NULL);
                                 /* note: tmpfile is utf-8 */
                                 if ((back[i].r.out =
@@ -2926,6 +2960,20 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                                           back[i].url_sav, 1, 1,
                                           back[i].r.notmodified);
                               back[i].r.compressed = 0;
+                              /* Re-fetch over an existing file (#77 follow-up):
+                                 move the good copy aside before truncating it
+                                 so an aborted transfer can restore it. url_sav
+                                 is still written normally (file list intact).
+                               */
+                              back[i].tmpfile = NULL;
+                              if (fexist_utf8(back[i].url_sav)) {
+                                if (create_back_tmpfile(opt, &back[i], "bak") !=
+                                        0 ||
+                                    RENAME(back[i].url_sav, back[i].tmpfile) !=
+                                        0) {
+                                  back[i].tmpfile = NULL;
+                                }
+                              }
                               if ((back[i].r.out =
                                    filecreate(&opt->state.strc,
                                               back[i].url_sav)) == NULL) {

src/htscore.c

@@ -1904,8 +1904,11 @@ int httpmirror(char *url1, httrackp * opt) {
             }
           }
 
-          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!          
-          if (r.adr != NULL || r.size == 0) {
+          // ATTENTION C'EST ICI QU'ON SAUVE LE FICHIER!!
+          // An empty body must not overwrite the file when the transfer failed
+          // (statuscode <= 0, e.g. an -M hard-stop): it would truncate a good
+          // copy to 0 (#77 follow-up).
+          if (r.adr != NULL || (r.size == 0 && r.statuscode > 0)) {
             file_notify(opt, urladr(), urlfil(), savename(), 1, 1, r.notmodified);
             if (filesave(opt, r.adr, (int) r.size, savename(), urladr(), urlfil()) !=
                 0) {
@@ -1926,7 +1929,6 @@ int httpmirror(char *url1, httrackp * opt) {
                */
             }
           }
-
         }
 
         /* Parsing of other media types (java, ram..) */

tests/43_local-update-truncate.test

@@ -0,0 +1,24 @@
+#!/bin/bash
+#
+# -M hard-abort must not destroy already-complete files (#77 follow-up).
+# Pass 1 mirrors bigtrunc fully. Pass 2 re-fetches under --update -M: fast.bin
+# overruns the cap, its abort tears down slow.bin's stalled re-fetch. Both were
+# complete on disk from pass 1, and both must survive the aborted update:
+#   - slow.bin's re-fetch is incomplete; the direct-to-disk write must not be
+#     left truncated (the previous copy is restored).
+#   - fast.bin fully arrives but is torn down; the aborted slot must not overwrite
+#     it with an empty body.
+# Unfixed, slow.bin is truncated and fast.bin is zeroed while the cache still
+# records them complete.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update -M400000' \
+    --log-found 'More than 400000 bytes have been transferred.. giving up' \
+    --found bigtrunc/slow.bin \
+    --file-min-bytes bigtrunc/slow.bin 655360 \
+    --file-min-bytes bigtrunc/fast.bin 655360 \
+    httrack 'BASEURL/bigtrunc/index.html' -c2

tests/Makefile.am

@@ -111,6 +111,7 @@ TESTS = \
 	39_local-delayed-cancel.test \
 	40_local-why.test \
 	41_local-utf8-link.test \
-	42_local-maxsize-slow.test
+	42_local-maxsize-slow.test \
+	43_local-update-truncate.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -13,15 +13,19 @@
 #
 # Usage:
 #   bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
+#       [--rerun-args 'ARGS'] \
 #       --errors N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
-#       --max-mirror-bytes N \
+#       --file-min-bytes PATH N --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
 # --max/--min-mirror-bytes bound the mirrored content bytes (host root).
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
+# --file-min-bytes asserts a mirrored file (PATH) is at least N bytes.
+# --rerun-args runs a second pass (same server and mirror dir) with the given
+# extra httrack args appended, e.g. an --update run under a cap.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
 # which the ephemeral port forces into the cookie domain) and passes it to
 # httrack via --cookies-file, to exercise preloaded cookies.
@@ -39,6 +43,7 @@ key="${testdir}/server.key"
 tls=
 verbose=
 rerun=
+rerun_args=
 rerun_dead=
 tmpdir=
 serverpid=
@@ -122,6 +127,10 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         cookies+=("${args[$pos]}")
         ;;
+    --rerun-args)
+        pos=$((pos + 1))
+        rerun_args="${args[$pos]}"
+        ;;
     --errors | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
@@ -130,7 +139,7 @@ while test "$pos" -lt "$nargs"; do
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --file-matches | --file-not-matches)
+    --file-matches | --file-not-matches | --file-min-bytes)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}" "${args[$((pos + 2))]}")
         pos=$((pos + 2))
         ;;
@@ -241,6 +250,25 @@ if test -n "$rerun"; then
     fi
 fi
 
+# --- optional second pass with extra args (e.g. an --update run under a cap) ---
+# Same server and mirror dir as the first pass, so the second pass sees the
+# cache the first pass wrote. Used to exercise re-fetch/update behaviour.
+if test -n "$rerun_args"; then
+    read -ra extra <<<"$rerun_args"
+    info "re-running httrack with ${rerun_args}"
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
+        "${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
+    crawlpid=$!
+    wait "$crawlpid"
+    crawlres=$?
+    crawlpid=
+    test "$crawlres" -eq 0 || ! result "second pass exited $crawlres" || {
+        cat "${log}.2" >&2
+        exit 1
+    }
+    result "OK (second pass)"
+fi
+
 # --- optional dead pass: server stopped, the cache must survive the rollback --
 if test -n "$rerun_dead"; then
     zip="${out}/hts-cache/new.zip"
@@ -388,6 +416,16 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         else result "OK"; fi
         ;;
+    --file-min-bytes)
+        path="${audit[$((i + 1))]}"
+        i=$((i + 2))
+        sz=$(wc -c <"${hostroot}/${path}" 2>/dev/null | tr -d '[:space:]')
+        info "checking ${path} size ${sz:-0} >= ${audit[$i]} bytes"
+        if test -n "$sz" && test "$sz" -ge "${audit[$i]}"; then result "OK"; else
+            result "file too small (or missing)"
+            exit 1
+        fi
+        ;;
     esac
     i=$((i + 1))
 done

tests/local-server.py

@@ -1043,6 +1043,40 @@ class Handler(SimpleHTTPRequestHandler):
             "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(4))
         )
 
+    # -M hard-abort must not destroy an already-complete file (#77 follow-up).
+    # "fast.bin" alone overruns -M and completes; "slow.bin" transfers fully on
+    # its first fetch (initial mirror) but stalls on every later fetch (the
+    # --update re-fetch), so the -M abort tears it down mid-body. An engine that
+    # truncates the good local copy on the aborted re-fetch loses data.
+    slow_seen = 0
+
+    def route_bigtrunc_index(self):
+        self.send_html('\t<a href="fast.bin">fast</a>\n\t<a href="slow.bin">slow</a>\n')
+
+    def route_bigtrunc_slow(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(self.BIGFILE_BYTES))
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        # Count body fetches only, so a stray HEAD can't shift which pass stalls.
+        Handler.slow_seen += 1
+        first = Handler.slow_seen == 1
+        try:
+            if first:
+                self.wfile.write(b"x" * self.BIGFILE_BYTES)
+                self.wfile.flush()
+            else:
+                self.wfile.write(b"x" * 4096)
+                self.wfile.flush()
+                for _ in range(120):
+                    self.wfile.write(b"x")
+                    self.wfile.flush()
+                    time.sleep(1.0)
+        except OSError:
+            pass
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -1120,6 +1154,9 @@ class Handler(SimpleHTTPRequestHandler):
         "/bigtrickle/p1.bin": route_trickle_page,
         "/bigtrickle/p2.bin": route_trickle_page,
         "/bigtrickle/p3.bin": route_trickle_page,
+        "/bigtrunc/index.html": route_bigtrunc_index,
+        "/bigtrunc/fast.bin": route_bigfile,
+        "/bigtrunc/slow.bin": route_bigtrunc_slow,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,