Add a -M byte-cap non-regression test

The -M limit had no test coverage. New /bigfiles/ fixture (8 fast 640KB
files), a --max-mirror-bytes audit in local-crawl.sh, and a crawl that
must log the "giving up" error and stay under the uncapped total.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.gitignore

@@ -39,3 +39,6 @@ Makefile
 
 # Editor / autotools backup files.
 *~
+
+# Python bytecode (tests/local-server.py).
+__pycache__/

src/htsback.c

@@ -1359,6 +1359,18 @@ int back_flush_output(httrackp * opt, cache_back * cache, struct_back * sback,
 }
 
 // effacer entrée
+/* Discard a cancelled mid-write .delayed placeholder (unusable across runs). */
+static void back_delayed_discard(httrackp *opt, lien_back *back) {
+  if (back->r.out != NULL) {
+    fclose(back->r.out);
+    back->r.out = NULL;
+  }
+  back->r.is_write = 0;
+  if (opt != NULL)
+    url_savename_refname_remove(opt, back->url_adr, back->url_fil);
+  (void) UNLINK(back->url_sav);
+}
+
 int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
                 const int p) {
   lien_back *const back = sback->lnk;
@@ -1366,6 +1378,12 @@ int back_delete(httrackp * opt, cache_back * cache, struct_back * sback,
 
   assertf(p >= 0 && p < back_max);
   if (p >= 0 && p < sback->count) {     // on sait jamais..
+    /* mid-write cancel: drop a .delayed placeholder; real-named partials
+       survive for resume (--continue) */
+    if (back[p].r.is_write && IS_DELAYED_EXT(back[p].url_sav) &&
+        (back[p].status != STATUS_READY || back[p].r.statuscode <= 0)) {
+      back_delayed_discard(opt, &back[p]);
+    }
     // Vérificateur d'intégrité
 #if DEBUG_CHECKINT
     _CHECKINT(&back[p], "Appel back_delete")
@@ -2419,6 +2437,34 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
   back_clean(opt, cache, sback);
 #endif
 
+  /* Time limit exceeded past grace: abort in-flight transfers so no wait loop
+     starves (#481). FTP slots stay, their thread owns the socket. */
+  if (!back_checkmirror(opt)) {
+    int aborted = 0;
+    unsigned int i;
+
+    for (i = 0; i < (unsigned int) back_max; i++) {
+      if (back[i].status > 0 && back[i].status < STATUS_FTP_TRANSFER) {
+        if (back[i].r.soc != INVALID_SOCKET) {
+          deletehttp(&back[i].r);
+        }
+        back[i].r.soc = INVALID_SOCKET;
+        /* drop a .delayed placeholder; real partials survive for resume */
+        if (back[i].r.is_write && IS_DELAYED_EXT(back[i].url_sav))
+          back_delayed_discard(opt, &back[i]);
+        back[i].r.statuscode = STATUSCODE_TIMEOUT;
+        strcpybuff(back[i].r.msg, "Mirror Time Out");
+        back[i].status = STATUS_READY;
+        back_set_finished(sback, i);
+        aborted++;
+      }
+    }
+    if (aborted > 0)
+      hts_log_print(opt, LOG_WARNING,
+                    "time limit reached, %d transfer(s) aborted", aborted);
+    return;
+  }
+
   // recevoir tant qu'il y a des données (avec un maximum de max_loop boucles)
   do_wait = 0;
   gestion_timeout = 0;
@@ -4164,6 +4210,11 @@ int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize)
   return 1;
 }
 
+/* Grace left to the smooth stop before in-flight transfers are aborted. */
+static int back_maxtime_grace(const int maxtime) {
+  return maximum(5, minimum(30, maxtime / 10));
+}
+
 int back_checkmirror(httrackp * opt) {
   // Check max size
   if ((opt->maxsite > 0) && (HTS_STAT.stat_bytes >= opt->maxsite)) {
@@ -4180,13 +4231,19 @@ int back_checkmirror(httrackp * opt) {
      */
   }
   // Check max time
-  if ((opt->maxtime > 0)
-      && ((time_local() - HTS_STAT.stat_timestart) >= opt->maxtime)) {
-    if (!opt->state.stop) {     /* not yet stopped */
-      hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
-                    opt->maxtime);
-      /* cancel mirror smoothly */
-      hts_request_stop(opt, 0);
+  if (opt->maxtime > 0) {
+    const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
+
+    if (elapsed >= opt->maxtime) {
+      if (!opt->state.stop) { /* not yet stopped */
+        hts_log_print(opt, LOG_ERROR, "More than %d seconds passed.. giving up",
+                      opt->maxtime);
+        /* cancel mirror smoothly */
+        hts_request_stop(opt, 0);
+      }
+      /* smooth stop starved past the grace period: stop waiting (#481) */
+      if (elapsed - opt->maxtime >= back_maxtime_grace(opt->maxtime))
+        return 0;
     }
   }
   return 1;                     /* Ok, go on */

src/htsback.h

@@ -136,6 +136,8 @@ void back_solve(httrackp * opt, lien_back * sback);
 int host_wait(httrackp * opt, lien_back * sback);
 #endif
 int back_checksize(httrackp * opt, lien_back * eback, int check_only_totalsize);
+/* Enforce -M/-E quotas: requests a smooth stop when reached; returns 0 once
+   the -E deadline overran its grace period (callers must stop waiting). */
 int back_checkmirror(httrackp * opt);
 
 #endif

src/htsname.c

@@ -74,30 +74,36 @@ static const char *hts_tbdev[] = {
   ""
 };
 
-#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET() do { \
-  int prev = opt->state._hts_in_html_parsing; \
-  while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
-    opt->state. _hts_in_html_parsing = 6; \
-    /* Wait .. */ \
-    back_wait(sback,opt,cache,0); \
-    /* Transfer rate */ \
-    engine_stats(); \
-    /* Refresh various stats */ \
-    HTS_STAT.stat_nsocket=back_nsoc(sback); \
-    HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
-    HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
-    HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
-    HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
-    HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
-    /* Check */ \
-    { \
-      if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count,-1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
-        return -1; \
-      } \
-    } \
-  } \
-  opt->state._hts_in_html_parsing = prev; \
-} while(0)
+#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET()                                \
+  do {                                                                         \
+    int prev = opt->state._hts_in_html_parsing;                                \
+    while (back_pluggable_sockets_strict(sback, opt) <= 0) {                   \
+      opt->state._hts_in_html_parsing = 6;                                     \
+      /* Wait .. */                                                            \
+      back_wait(sback, opt, cache, 0);                                         \
+      /* time limit (-E) exceeded: stop waiting for a socket (#481) */         \
+      if (!back_checkmirror(opt))                                              \
+        break;                                                                 \
+      /* Transfer rate */                                                      \
+      engine_stats();                                                          \
+      /* Refresh various stats */                                              \
+      HTS_STAT.stat_nsocket = back_nsoc(sback);                                \
+      HTS_STAT.stat_errors = fspc(opt, NULL, "error");                         \
+      HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");                     \
+      HTS_STAT.stat_infos = fspc(opt, NULL, "info");                           \
+      HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);    \
+      HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);              \
+      /* Check */                                                              \
+      {                                                                        \
+        if (!RUN_CALLBACK7(                                                    \
+                opt, loop, sback->lnk, sback->count, -1, ptr, opt->lien_tot,   \
+                (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {  \
+          return -1;                                                           \
+        }                                                                      \
+      }                                                                        \
+    }                                                                          \
+    opt->state._hts_in_html_parsing = prev;                                    \
+  } while (0)
 
 /* Strip all // */
 static void cleanDoubleSlash(char *s) {

src/htsparse.c

@@ -4077,6 +4077,9 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
     while(opt->state._hts_setpause || back_pluggable_sockets_strict(sback, opt) <= 0) { // on fait la pause..
       opt->state._hts_in_html_parsing = 6;
       back_wait(sback, opt, cache, HTS_STAT.stat_timestart);
+      /* time limit (-E) exceeded: stop waiting for a socket (#481) */
+      if (!back_checkmirror(opt))
+        break;
 
       // Transfer rate
       engine_stats();

src/htsparse.h

@@ -175,27 +175,33 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
   /* Apply changes */ \
   * str->ptr_ = ptr
 
-#define WAIT_FOR_AVAILABLE_SOCKET() do { \
-  int prev = opt->state._hts_in_html_parsing; \
-  while(back_pluggable_sockets_strict(sback, opt) <= 0) { \
-    opt->state._hts_in_html_parsing = 6; \
-    /* Wait .. */ \
-    back_wait(sback,opt,cache,0); \
-    /* Transfer rate */ \
-    engine_stats(); \
-    /* Refresh various stats */ \
-    HTS_STAT.stat_nsocket=back_nsoc(sback); \
-    HTS_STAT.stat_errors=fspc(opt,NULL,"error"); \
-    HTS_STAT.stat_warnings=fspc(opt,NULL,"warning"); \
-    HTS_STAT.stat_infos=fspc(opt,NULL,"info"); \
-    HTS_STAT.nbk=backlinks_done(sback,opt->liens,opt->lien_tot,ptr); \
-    HTS_STAT.nb=back_transferred(HTS_STAT.stat_bytes,sback); \
-    /* Check */ \
-    if (!RUN_CALLBACK7(opt, loop, sback->lnk, sback->count, -1,ptr,opt->lien_tot,(int) (time_local()-HTS_STAT.stat_timestart),&HTS_STAT)) { \
-      return -1; \
-    } \
-  } \
-  opt->state._hts_in_html_parsing = prev; \
-} while(0)
+#define WAIT_FOR_AVAILABLE_SOCKET()                                            \
+  do {                                                                         \
+    int prev = opt->state._hts_in_html_parsing;                                \
+    while (back_pluggable_sockets_strict(sback, opt) <= 0) {                   \
+      opt->state._hts_in_html_parsing = 6;                                     \
+      /* Wait .. */                                                            \
+      back_wait(sback, opt, cache, 0);                                         \
+      /* time limit (-E) exceeded: stop waiting for a socket (#481) */         \
+      if (!back_checkmirror(opt))                                              \
+        break;                                                                 \
+      /* Transfer rate */                                                      \
+      engine_stats();                                                          \
+      /* Refresh various stats */                                              \
+      HTS_STAT.stat_nsocket = back_nsoc(sback);                                \
+      HTS_STAT.stat_errors = fspc(opt, NULL, "error");                         \
+      HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");                     \
+      HTS_STAT.stat_infos = fspc(opt, NULL, "info");                           \
+      HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);    \
+      HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);              \
+      /* Check */                                                              \
+      if (!RUN_CALLBACK7(                                                      \
+              opt, loop, sback->lnk, sback->count, -1, ptr, opt->lien_tot,     \
+              (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {    \
+        return -1;                                                             \
+      }                                                                        \
+    }                                                                          \
+    opt->state._hts_in_html_parsing = prev;                                    \
+  } while (0)
 
 #endif

tests/34_local-maxtime.test

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# -E time limit (#481): server pages trickle for minutes; the engine must stop
+# on its own at -E plus grace, aborting the in-flight transfers.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# cancelled crawls can orphan .delayed placeholders (#483): skip that audit
+start=$(date +%s)
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --skip-delayed-audit \
+    --log-found 'More than 2 seconds passed' \
+    httrack 'BASEURL/trickle/index.html' -E2 -c4
+wall=$(($(date +%s) - start))
+# hard stop is due at -E2 + 5s grace; near TRICKLE_SECONDS means it never fired
+if [ "$wall" -ge 30 ]; then
+    echo "crawl took ${wall}s, -E hard stop did not engage" >&2
+    exit 1
+fi

tests/35_local-maxsize.test

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# -M byte cap (#77): the crawl must stop with the "giving up" error and keep
+# the mirror well under the 8 x 640KB the fixture totals uncapped.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# cap = -M + the 4 in-flight files the smooth stop lets finish + one of margin
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'More than 400000 bytes have been transferred.. giving up' \
+    --found bigfiles/p0.bin \
+    --max-mirror-bytes 3700000 \
+    httrack 'BASEURL/bigfiles/index.html' -M400000 -c4

tests/Makefile.am

@@ -96,6 +96,8 @@ TESTS = \
 	30_local-fragment-link.test \
 	31_local-javaclass.test \
 	32_local-cdispo.test \
-	33_local-delayed.test
+	33_local-delayed.test \
+	34_local-maxtime.test \
+	35_local-maxsize.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -16,8 +16,10 @@
 #       --errors N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
+#       --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
+# --max-mirror-bytes asserts the mirrored content (host root) stays under N.
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
@@ -92,6 +94,7 @@ tmpdir=$(mktemp -d "${tmptopdir}/httrack_local.XXXXXX") || die "could not create
 # --- parse leading control flags --------------------------------------------
 declare -a audit=()
 declare -a cookies=()
+skip_delayed_audit=""
 scheme=http
 pos=0
 args=("$@")
@@ -116,11 +119,14 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         cookies+=("${args[$pos]}")
         ;;
+    --skip-delayed-audit)
+        skip_delayed_audit=1
+        ;;
     --errors | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --found | --not-found | --directory | --log-found | --log-not-found)
+    --found | --not-found | --directory | --log-found | --log-not-found | --max-mirror-bytes)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -246,12 +252,15 @@ done
 test -n "$hostroot" || die "could not find host root under $out"
 debug "host root: $hostroot"
 
-# A completed crawl must leave no .delayed temporaries (issue #107)
-info "checking for leftover .delayed files"
-leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
-if test -z "$leftovers"; then result "OK"; else
-    result "leftover: $leftovers"
-    exit 1
+# A completed crawl must leave no .delayed temporaries (issue #107).
+# --skip-delayed-audit: a cancelled crawl can orphan placeholders (issue #483)
+if test -z "$skip_delayed_audit"; then
+    info "checking for leftover .delayed files"
+    leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
+    if test -z "$leftovers"; then result "OK"; else
+        result "leftover: $leftovers"
+        exit 1
+    fi
 fi
 
 # --- audit -------------------------------------------------------------------
@@ -309,6 +318,15 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         else result "OK"; fi
         ;;
+    --max-mirror-bytes)
+        i=$((i + 1))
+        sz=$(find "$hostroot" -type f -exec cat {} + | wc -c | tr -d '[:space:]')
+        info "checking mirror size ${sz} <= ${audit[$i]} bytes"
+        if test "$sz" -le "${audit[$i]}"; then result "OK"; else
+            result "mirror too big"
+            exit 1
+        fi
+        ;;
     --file-matches)
         path="${audit[$((i + 1))]}"
         i=$((i + 2))

tests/local-server.py

@@ -464,6 +464,43 @@ class Handler(SimpleHTTPRequestHandler):
     def route_delayed_empty(self):
         self.send_raw(b"", "text/html")  # 200 + Content-Length: 0
 
+    # -E time-limit (#481): pages that trickle far longer than any -E budget,
+    # so only an engine-side abort can end the crawl.
+    TRICKLE_SECONDS = 60
+
+    def send_bin_index(self):
+        """Index page linking p0.bin..p7.bin (shared by trickle and bigfiles)."""
+        self.send_html(
+            "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(8))
+        )
+
+    def route_trickle_index(self):
+        self.send_bin_index()
+
+    def route_trickle_page(self):
+        self.send_response(200)
+        self.send_header("Content-Type", "application/octet-stream")
+        self.send_header("Content-Length", str(2 * self.TRICKLE_SECONDS))
+        self.end_headers()
+        if self.command == "HEAD":
+            return
+        try:
+            for _ in range(self.TRICKLE_SECONDS):
+                self.wfile.write(b"xy")
+                self.wfile.flush()
+                time.sleep(1.0)
+        except OSError:
+            pass
+
+    # -M byte cap (#77): large fast files so a crawl overruns -M immediately.
+    BIGFILE_BYTES = 640 * 1024
+
+    def route_bigfiles_index(self):
+        self.send_bin_index()
+
+    def route_bigfile(self):
+        self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -509,6 +546,24 @@ class Handler(SimpleHTTPRequestHandler):
         "/cdispo/fetch.php": route_cdispo,
         "/cdispo/evil.php": route_cdispo,
         "/delayed/index.html": route_delayed_index,
+        "/trickle/index.html": route_trickle_index,
+        "/trickle/p0.bin": route_trickle_page,
+        "/trickle/p1.bin": route_trickle_page,
+        "/trickle/p2.bin": route_trickle_page,
+        "/trickle/p3.bin": route_trickle_page,
+        "/trickle/p4.bin": route_trickle_page,
+        "/trickle/p5.bin": route_trickle_page,
+        "/trickle/p6.bin": route_trickle_page,
+        "/trickle/p7.bin": route_trickle_page,
+        "/bigfiles/index.html": route_bigfiles_index,
+        "/bigfiles/p0.bin": route_bigfile,
+        "/bigfiles/p1.bin": route_bigfile,
+        "/bigfiles/p2.bin": route_bigfile,
+        "/bigfiles/p3.bin": route_bigfile,
+        "/bigfiles/p4.bin": route_bigfile,
+        "/bigfiles/p5.bin": route_bigfile,
+        "/bigfiles/p6.bin": route_bigfile,
+        "/bigfiles/p7.bin": route_bigfile,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,