Stop the 412/416 partial-reget loop on continue/update (#206)

On resume, the Range request is rebuilt by back_add from a temp-ref keyed on
(adr,fil) that records the partial download's real save name. A 412/416
("Range Not Satisfiable") means that partial is stale and the whole file
must be re-fetched. The handler only removed heap->sav, so when the resume
pass recomputed a save name different from the temp-ref's (the default
delayed-type machinery renames freely), the partial was never cleared:
back_add re-sent the same Range, earned the same 416, and the link was
re-recorded forever, growing the scan counter without bound.

Clear the whole partial wherever it lives -- the temp-ref and the file it
points at, plus heap->sav -- so the re-record falls through to a plain full
GET. Re-get only when there was a partial to discard and both Range triggers
(the ref and the on-disk file) are actually gone; once they are, a fresh 416
with nothing left to drop means the whole-file GET itself failed, so the link
gives up cleanly instead of re-queueing. A failed removal (read-only or full
cache) also gives up rather than looping, since back_add would otherwise
re-Range the surviving ref; url_savename_refname_remove now reports the
removal result so the handler can tell. (The request's range_used flag would
be the natural one-shot signal, but it does not survive the delayed-type
two-pass, so we key off the partial instead.)

tests/20_local-resume-loop.test drives it offline: pass 1 is interrupted
(SIGTERM, so the exit handler finalizes the cache and the temp-ref) to leave
a partial, then pass 2 --continue gets 416 on every resume request. A
portable watchdog kills pass 2 if it loops; the test asserts it terminates
and attempts exactly one whole-file re-get (2 <= requests <= 8). It fails on
the pre-fix handler (loops) and on a re-get that silently drops the link.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htscoremain.c

@@ -2468,6 +2468,44 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 htsmain_free();
                 return err;
               } break;
+              case 'N': { // url_savename name resolution: httrack -#N <fil>
+                          // <content-type>
+                if (na + 2 < argc) {
+                  lien_adrfilsave afs;
+                  cache_back cache;
+                  struct_back *sback;
+                  hash_struct hash;
+                  lien_back headers;
+
+                  memset(&afs, 0, sizeof(afs));
+                  strcpybuff(afs.af.adr, "www.example.com");
+                  strcpybuff(afs.af.fil, argv[na + 1]);
+
+                  memset(&cache, 0, sizeof(cache));
+                  cache.hashtable = (void *) coucal_new(0);
+
+                  sback = back_new(opt, opt->maxsoc * 32 + 1024);
+                  hash_init(opt, &hash, opt->urlhack);
+
+                  memset(&headers, 0, sizeof(headers));
+                  headers.status = 0;
+                  headers.r.statuscode = HTTP_OK;
+                  strcpybuff(headers.r.contenttype, argv[na + 2]);
+                  strcpybuff(headers.url_fil, argv[na + 1]);
+
+                  url_savename(&afs, NULL, NULL, NULL, opt, sback, &cache,
+                               &hash, 0, 0, &headers);
+                  printf("savename: %s\n", afs.save);
+                  htsmain_free();
+                  return 0;
+                } else {
+                  fprintf(
+                      stderr,
+                      "Option #N requires <fil> <content-type> arguments\n");
+                  htsmain_free();
+                  return 1;
+                }
+              } break;
               case 'C':        // list cache files : httrack -#C '*spid*.gif' will attempt to find the matching file
                 {
                   int hasFilter = 0;

src/htsname.c

@@ -760,9 +760,9 @@ int url_savename(lien_adrfilsave *const afs,
         strcatbuff(fil, DEFAULT_HTML);  // nommer page par défaut (à priori ici html depuis un proxy http)
     }
   }
-  // Changer extension?
-  // par exemple, php3 sera sauvé en html, cgi en html ou gif, xbm etc.. selon les cas
-  if (ext_chg && !opt->no_type_change) {                // changer ext
+  // Change the extension? e.g. php3 saved as html, cgi as html or gif/xbm
+  // depending on the resolved type.
+  if (ext_chg && !opt->no_type_change) {
     char *a = fil + strlen(fil) - 1;
 
     if ((opt->debug > 1) && (opt->log != NULL)) {
@@ -774,11 +774,19 @@ int url_savename(lien_adrfilsave *const afs,
                       adr_complete, fil_complete, ext);
     }
     if (ext_chg == 1) {
+      // Cut the old extension only when it is empty (a bare trailing dot), the
+      // new one, or a recognized one; an unknown trailing ".token" (e.g.
+      // /article-1.884291, #115) is part of the name, not an extension.
+      const char *const old_ext = get_ext(catbuff, sizeof(catbuff), fil);
+      const int known_ext = !*old_ext || strfield2(old_ext, ext) ||
+                            is_knowntype(opt, fil) || is_dyntype(old_ext) ||
+                            ishtml_ext(old_ext) != -1;
+
       while((a > fil) && (*a != '.') && (*a != '/'))
         a--;
-      if (*a == '.')
-        *a = '\0';              // couper
-      strcatbuff(fil, ".");     // recopier point
+      if (*a == '.' && known_ext)
+        *a = '\0';          // cut
+      strcatbuff(fil, "."); // re-add the dot
     } else {
       while((a > fil) && (*a != '/'))
         a--;
@@ -786,7 +794,7 @@ int url_savename(lien_adrfilsave *const afs,
         a++;
       *a = '\0';
     }
-    strcatbuff(fil, ext);       // copier ext/nom
+    strcatbuff(fil, ext); // append ext/name
   }
   // Rechercher premier / et dernier .
   {
@@ -1721,10 +1729,10 @@ char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
     StringBuff(opt->path_log), digest_filename);
 }
 
-/* remove refname if any */
-void url_savename_refname_remove(httrackp * opt, const char *adr,
-                                 const char *fil) {
+/* remove refname if any; HTS_TRUE if it was removed */
+hts_boolean url_savename_refname_remove(httrackp *opt, const char *adr,
+                                        const char *fil) {
   char *filename = url_savename_refname_fullpath(opt, adr, fil);
 
-  (void) UNLINK(filename);
+  return UNLINK(filename) == 0 ? HTS_TRUE : HTS_FALSE;
 }

src/htsname.h

@@ -104,8 +104,9 @@ char *url_md5(char *digest_buffer, const char *fil_complete);
 void url_savename_refname(const char *adr, const char *fil, char *filename);
 char *url_savename_refname_fullpath(httrackp * opt, const char *adr,
                                     const char *fil);
-void url_savename_refname_remove(httrackp * opt, const char *adr,
-                                 const char *fil);
+/* Remove the temp-ref for (adr,fil); HTS_TRUE if it was removed. */
+hts_boolean url_savename_refname_remove(httrackp *opt, const char *adr,
+                                        const char *fil);
 #endif
 
 #endif

src/htsparse.c

@@ -3749,44 +3749,60 @@ int hts_mirror_check_moved(htsmoduleStruct * str,
 
       }                         // bloc
       // erreur HTTP (ex: 404, not found)
-    } else if ((r->statuscode == HTTP_PRECONDITION_FAILED)
-               || (r->statuscode == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE)
-      ) {                       // Precondition Failed, c'est à dire pour nous redemander TOUT le fichier
-      if (fexist_utf8(heap(ptr)->sav)) {
-        remove(heap(ptr)->sav);        // Eliminer
-      } else {
-        hts_log_print(opt, LOG_WARNING,
-                      "Unexpected 412/416 error (%s) for %s%s, '%s' could not be found on disk",
-                      r->msg, urladr(), urlfil(),
-                      heap(ptr)->sav != NULL ? heap(ptr)->sav : "");
+    } else if ((r->statuscode == HTTP_PRECONDITION_FAILED) ||
+               (r->statuscode == HTTP_REQUESTED_RANGE_NOT_SATISFIABLE)) {
+      // 412/416: the resume partial is stale; re-get the whole file (#206)
+      lien_back *itemback = NULL;
+      int had_partial = 0;
+      int ref_existed = 0;
+      int ref_gone;
+
+      // Drop the temp-ref, its partial, and heap->sav so the re-get carries no
+      // Range; else back_add rebuilds the same Range and loops.
+      if (back_unserialize_ref(opt, heap(ptr)->adr, heap(ptr)->fil,
+                               &itemback) == 0) {
+        had_partial = 1;
+        ref_existed = 1;
+        // best-effort: an orphaned partial cannot re-Range once the ref is gone
+        if (fexist_utf8(itemback->url_sav))
+          (void) UNLINK(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                              itemback->url_sav));
+        back_clear_entry(itemback);
+        freet(itemback);
       }
-      if (!fexist_utf8(heap(ptr)->sav)) {    // Bien éliminé? (sinon on boucle..)
-#if HDEBUG
-        printf("Partial content NOT up-to-date, reget all file for %s\n",
-               heap(ptr)->sav);
-#endif
+      // don't re-record if the ref survived (it would re-Range and loop)
+      ref_gone =
+          url_savename_refname_remove(opt, heap(ptr)->adr, heap(ptr)->fil) ||
+          !ref_existed;
+      if (fexist_utf8(heap(ptr)->sav)) {
+        had_partial = 1;
+        remove(heap(ptr)->sav);
+      }
+
+      // Re-get once, only if a partial existed and both Range triggers are
+      // gone; a failed removal gives up rather than looping. range_used is
+      // unreliable (it does not survive the delayed-type two-pass).
+      if (had_partial && ref_gone && !fexist_utf8(heap(ptr)->sav)) {
         hts_log_print(opt, LOG_DEBUG, "Partial file reget (%s) for %s%s",
                       r->msg, urladr(), urlfil());
-        // enregistrer le MEME lien
         if (hts_record_link(opt, heap(ptr)->adr, heap(ptr)->fil, heap(ptr)->sav, "", "", NULL)) {
-          heap_top()->testmode = heap(ptr)->testmode;   // mode test?
-          heap_top()->link_import = 0;   // pas mode import
+          heap_top()->testmode = heap(ptr)->testmode;
+          heap_top()->link_import = 0;
           heap_top()->depth = heap(ptr)->depth;
           heap_top()->pass2 = max(heap(ptr)->pass2, numero_passe);
           heap_top()->retry = heap(ptr)->retry;
           heap_top()->premier = heap(ptr)->premier;
           heap_top()->precedent = ptr;
-          //
-          // canceller lien actuel
           error = 1;
-          hts_invalidate_link(opt, ptr);  // invalidate hashtable entry
-          //
-        } else {              // oups erreur, plus de mémoire!!
-          XH_uninit;          // désallocation mémoire & buffers
+          hts_invalidate_link(opt, ptr); // invalidate hashtable entry
+        } else {                         // out of memory
+          XH_uninit;
           return 0;
         }
       } else {
-        hts_log_print(opt, LOG_ERROR, "Can not remove old file %s", urlfil());
+        hts_log_print(opt, LOG_WARNING,
+                      "Giving up on partial reget (%s) for %s%s", r->msg,
+                      urladr(), urlfil());
         error = 1;
       }
 

tests/01_engine-savename.test

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Local save-name extension resolution (url_savename via -#N <fil> <content-type>).
+# Asserts on the basename of "savename: <path>".
+
+name() {
+    out="$(httrack -O /dev/null -#N "$1" "$2" | sed -n 's/^savename: //p')"
+    test "${out##*/}" == "$3" || {
+        echo "FAIL: '$1' '$2' -> '$out' (want '$3')"
+        exit 1
+    }
+}
+
+# #115: an unknown trailing ".token" is part of the name, keep it and append the type.
+name '/article-1.884291' 'text/html' 'article-1.884291.html'
+name '/news/story-12345.987654' 'text/html' 'story-12345.987654.html'
+
+# Recognized extensions still collapse to the resolved type.
+name '/page.php' 'text/html' 'page.html'
+name '/page.asp' 'text/html' 'page.html'
+name '/foo' 'text/html' 'foo.html'
+
+# A bare trailing dot is not a tail to keep.
+name '/page.' 'text/html' 'page.html'
+
+# Soft-404 (#267/#408): a binary URL served as HTML is named .html.
+name '/x.pdf' 'text/html' 'x.html'
+name '/x.gif' 'text/html' 'x.html'
+
+# Type agrees with the extension: keep it, no churn, no double extension.
+name '/x.pdf' 'application/pdf' 'x.pdf'
+name '/x.jpg' 'image/jpeg' 'x.jpg'
+name '/x.html' 'text/html' 'x.html'
+name '/x.js' 'application/x-javascript' 'x.js'
+name '/types/data.json' 'application/json' 'data.json'
+
+# Agreeing type must not rewrite the extension's casing (no strip-and-reappend).
+name '/x.JPG' 'image/jpeg' 'x.JPG'

tests/20_local-resume-loop.test

@@ -0,0 +1,113 @@
+#!/bin/bash
+# Issue #206: a continue/update crawl looped forever when the resume Range got a
+# 416. Pass 1 leaves a partial + temp-ref; pass 2 must terminate and not loop.
+set -u
+
+: "${top_srcdir:=..}"
+testdir=$(cd "$(dirname "$0")" && pwd)
+server="${testdir}/local-server.py"
+
+command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
+
+tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_206.XXXXXX") || exit 1
+serverpid=
+crawlpid=
+cleanup() {
+    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
+    if test -n "$serverpid"; then
+        kill "$serverpid" 2>/dev/null
+        wait "$serverpid" 2>/dev/null
+    fi
+    rm -rf "$tmpdir"
+}
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+# --- start the server, discover its ephemeral port --------------------------
+# RESUME_COUNTER gets a byte per /resume/blob.txt request (pass-2 delta bounds re-gets).
+serverlog="${tmpdir}/server.log"
+counter="${tmpdir}/blobcount"
+RESUME_COUNTER="$counter" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
+serverpid=$!
+port=
+for _ in $(seq 1 50); do
+    line=$(head -n1 "$serverlog" 2>/dev/null)
+    if test "${line%% *}" == "PORT"; then
+        port="${line#PORT }"
+        break
+    fi
+    kill -0 "$serverpid" 2>/dev/null || {
+        echo "server exited early: $(cat "$serverlog")"
+        exit 1
+    }
+    sleep 0.1
+done
+test -n "$port" || {
+    echo "could not discover server port"
+    exit 1
+}
+base="http://127.0.0.1:${port}"
+
+which httrack >/dev/null || {
+    echo "could not find httrack"
+    exit 1
+}
+out="${tmpdir}/crawl"
+mkdir "$out"
+common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=0)
+refdir="${out}/hts-cache/ref"
+
+# --- pass 1: crawl, interrupt once the blob download is underway -------------
+printf '[pass 1: interrupt mid-download] ..\t'
+httrack "${common[@]}" "${base}/resume/index.html" >"${tmpdir}/log1" 2>&1 &
+crawlpid=$!
+# Wait until blob.txt is requested, then SIGTERM so httrack's exit handler
+# finalizes the cache and serializes the temp-ref.
+for _ in $(seq 1 300); do
+    test -s "$counter" && break
+    kill -0 "$crawlpid" 2>/dev/null || break
+    sleep 0.1
+done
+sleep 0.5
+kill -TERM "$crawlpid" 2>/dev/null
+wait "$crawlpid" 2>/dev/null
+crawlpid=
+test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
+    echo "FAIL: no temp-ref survived pass 1; cannot drive #206"
+    exit 1
+}
+echo "OK (temp-ref present)"
+before=$(wc -c <"$counter" 2>/dev/null || echo 0)
+
+# --- pass 2: --continue -> resume Range -> 416, bounded against the #206 loop -
+# Kill pass 2 after a deadline (portable stand-in for `timeout`, absent on macOS).
+printf '[pass 2: resume must terminate] ..\t'
+HANG_RC=137 # 128 + SIGKILL
+httrack "${common[@]}" --continue "${base}/resume/index.html" >"${tmpdir}/log2" 2>&1 &
+crawlpid=$!
+(sleep 30 && kill -9 "$crawlpid" 2>/dev/null) &
+guard=$!
+rc=0
+wait "$crawlpid" 2>/dev/null || rc=$?
+crawlpid=
+kill "$guard" 2>/dev/null || true
+wait "$guard" 2>/dev/null || true
+if test "$rc" -eq "$HANG_RC"; then
+    echo "FAIL: pass 2 did not terminate (#206 resume->416 loop)"
+    exit 1
+fi
+echo "OK (terminated, rc=$rc)"
+
+# The fix re-gets once (resume Range + range-less re-get = 2): the lower bound
+# rejects a drop-the-link non-fix (1), the upper bound rejects the loop (many).
+after=$(wc -c <"$counter" 2>/dev/null || echo 0)
+hits=$((after - before))
+printf '[bounded re-get count] ..\t'
+if test "$hits" -lt 2; then
+    echo "FAIL: only ${hits} pass-2 request(s); the stale partial was not re-got"
+    exit 1
+fi
+if test "$hits" -gt 8; then
+    echo "FAIL: ${hits} pass-2 requests for blob.txt (resume is looping)"
+    exit 1
+fi
+echo "OK (${hits} requests)"

tests/Makefile.am

@@ -40,6 +40,7 @@ TESTS = \
 	01_engine-parse.test \
 	01_engine-rcfile.test \
 	01_engine-relative.test \
+	01_engine-savename.test \
 	01_engine-simplify.test \
 	01_engine-strsafe.test \
 	02_manpage-regen.test \
@@ -58,6 +59,7 @@ TESTS = \
 	16_local-assume.test \
 	17_local-empty-ct.test \
 	18_local-update.test \
-	19_local-connect-fallback.test
+	19_local-connect-fallback.test \
+	20_local-resume-loop.test
 
 CLEANFILES = check-network_sh.cache

tests/local-server.py

@@ -15,6 +15,7 @@ stdlib only (http.server + ssl) -- no new build or runtime dependency.
 
 import argparse
 import os
+import time
 from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer
 from urllib.parse import quote, unquote, urlsplit
 
@@ -176,6 +177,43 @@ class Handler(SimpleHTTPRequestHandler):
         body, ctype = self.TYPE_MATRIX[path]
         self.send_raw(body, ctype)
 
+    # resume / 416 loop (#206): the first GET stalls after a prefix so the crawl
+    # can be interrupted (partial + temp-ref); every later request is 416.
+    RESUME_PREFIX = b"PARTIAL-" + b"x" * 4096  # flushed before the stall
+    RESUME_LEN = len(RESUME_PREFIX) + 4096  # declared length never delivered
+    _resume_started = False
+
+    def route_resume_index(self):
+        self.send_html('\t<a href="blob.txt">blob</a>')
+
+    def route_resume(self):
+        counter = os.environ.get("RESUME_COUNTER")
+        if counter:
+            with open(counter, "a") as fp:
+                fp.write("x")
+        # First GET: stall mid-body so the crawl can be interrupted with a partial.
+        if not Handler._resume_started:
+            Handler._resume_started = True
+            self.send_response(200)
+            self.send_header("Content-Type", "image/png")
+            self.send_header("Content-Length", str(self.RESUME_LEN))
+            self.send_header("Accept-Ranges", "bytes")
+            self.end_headers()
+            if self.command != "HEAD":
+                self.wfile.write(self.RESUME_PREFIX)
+                self.wfile.flush()
+                try:
+                    while True:
+                        time.sleep(3600)
+                except OSError:
+                    pass
+            return
+        self.send_response(416, "Requested Range Not Satisfiable")
+        self.send_header("Content-Type", "image/png")
+        self.send_header("Content-Range", "bytes */%d" % self.RESUME_LEN)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -195,6 +233,8 @@ class Handler(SimpleHTTPRequestHandler):
         "/types/style.css": route_types,
         "/types/data.json": route_types,
         "/types/gen.php": route_types,
+        "/resume/index.html": route_resume_index,
+        "/resume/blob.txt": route_resume,
     }
 
     # --- dispatch ----------------------------------------------------------