Add a -M byte-cap non-regression test

The -M limit had no test coverage. New /bigfiles/ fixture (8 fast 640KB
files), a --max-mirror-bytes audit in local-crawl.sh, and a crawl that
must log the "giving up" error and stay under the uncapped total.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.gitignore

@@ -39,3 +39,6 @@ Makefile
 
 # Editor / autotools backup files.
 *~
+
+# Python bytecode (tests/local-server.py).
+__pycache__/

src/htscore.c

@@ -3371,41 +3371,6 @@ int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt) {
   return n;
 }
 
-/* One engine-loop tick: refresh the transfer stats and run the loop callback
-   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
-hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr) {
-  engine_stats();
-  HTS_STAT.stat_nsocket = back_nsoc(sback);
-  HTS_STAT.stat_errors = fspc(opt, NULL, "error");
-  HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
-  HTS_STAT.stat_infos = fspc(opt, NULL, "info");
-  HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
-  HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
-  return RUN_CALLBACK7(
-             opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
-             (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)
-             ? HTS_TRUE
-             : HTS_FALSE;
-}
-
-/* Single implementation of the historical WAIT_FOR_AVAILABLE_SOCKET macros. */
-hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
-                                      cache_back *cache, int ptr) {
-  const int prev = opt->state._hts_in_html_parsing;
-
-  while (back_pluggable_sockets_strict(sback, opt) <= 0) {
-    opt->state._hts_in_html_parsing = 6;
-    back_wait(sback, opt, cache, 0);
-    /* time limit (-E) exceeded: stop waiting for a socket (#481) */
-    if (!back_checkmirror(opt))
-      break;
-    if (!hts_loop_tick(sback, opt, -1, ptr))
-      return HTS_FALSE;
-  }
-  opt->state._hts_in_html_parsing = prev;
-  return HTS_TRUE;
-}
-
 int back_pluggable_sockets(struct_back * sback, httrackp * opt) {
   int n;
 

src/htscore.h

@@ -432,15 +432,6 @@ int back_pluggable_sockets(struct_back * sback, httrackp * opt);
 
 int back_pluggable_sockets_strict(struct_back * sback, httrackp * opt);
 
-/* One engine-loop tick: refresh the transfer stats and run the loop callback
-   for slot b (-1 = none). HTS_FALSE = the callback requested an abort. */
-hts_boolean hts_loop_tick(struct_back *sback, httrackp *opt, int b, int ptr);
-
-/* Wait until a test socket can be plugged, pumping transfers, stats and the
-   loop callback; gives up past the -E deadline. HTS_FALSE = callback abort. */
-hts_boolean hts_wait_available_socket(struct_back *sback, httrackp *opt,
-                                      cache_back *cache, int ptr);
-
 /* Randomized inter-file pause target in [min_ms,max_ms] (#185), derived from a
    timestamp seed so it is stable within one gap and rerolls per launch. */
 int hts_pause_target_ms(TStamp seed, int min_ms, int max_ms);

src/htsname.c

@@ -74,6 +74,37 @@ static const char *hts_tbdev[] = {
   ""
 };
 
+#define URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET()                                \
+  do {                                                                         \
+    int prev = opt->state._hts_in_html_parsing;                                \
+    while (back_pluggable_sockets_strict(sback, opt) <= 0) {                   \
+      opt->state._hts_in_html_parsing = 6;                                     \
+      /* Wait .. */                                                            \
+      back_wait(sback, opt, cache, 0);                                         \
+      /* time limit (-E) exceeded: stop waiting for a socket (#481) */         \
+      if (!back_checkmirror(opt))                                              \
+        break;                                                                 \
+      /* Transfer rate */                                                      \
+      engine_stats();                                                          \
+      /* Refresh various stats */                                              \
+      HTS_STAT.stat_nsocket = back_nsoc(sback);                                \
+      HTS_STAT.stat_errors = fspc(opt, NULL, "error");                         \
+      HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");                     \
+      HTS_STAT.stat_infos = fspc(opt, NULL, "info");                           \
+      HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);    \
+      HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);              \
+      /* Check */                                                              \
+      {                                                                        \
+        if (!RUN_CALLBACK7(                                                    \
+                opt, loop, sback->lnk, sback->count, -1, ptr, opt->lien_tot,   \
+                (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {  \
+          return -1;                                                           \
+        }                                                                      \
+      }                                                                        \
+    }                                                                          \
+    opt->state._hts_in_html_parsing = prev;                                    \
+  } while (0)
+
 /* Strip all // */
 static void cleanDoubleSlash(char *s) {
   int i, j;
@@ -627,10 +658,11 @@ int url_savename(lien_adrfilsave *const afs,
             int has_been_moved = 0;
             lien_adrfil current;
 
-            /* Wait for an available test slot, honoring the connection limits
+            /* Ensure we don't use too many sockets by using a "testing" one
+               If we have only 1 simultaneous connection authorized, wait for pending download
+               Wait for an available slot 
              */
-            if (!hts_wait_available_socket(sback, opt, cache, ptr))
-              return -1;
+            URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
 
             /* Rock'in */
             current.adr[0] = current.fil[0] = '\0';
@@ -660,11 +692,24 @@ int url_savename(lien_adrfilsave *const afs,
                   if (ptr >= 0) {
                     back_fillmax(sback, opt, cache, ptr, numero_passe);
                   }
-                  if (!hts_loop_tick(sback, opt, b, ptr)) {
+                  // on est obligé d'appeler le shell pour le refresh..
+                  // Transfer rate
+                  engine_stats();
+
+                  // Refresh various stats
+                  HTS_STAT.stat_nsocket = back_nsoc(sback);
+                  HTS_STAT.stat_errors = fspc(opt, NULL, "error");
+                  HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
+                  HTS_STAT.stat_infos = fspc(opt, NULL, "info");
+                  HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
+                  HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
+
+                  if (!RUN_CALLBACK7
+                      (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
+                       (int) (time_local() - HTS_STAT.stat_timestart),
+                       &HTS_STAT)) {
                     return -1;
-                  } else if (opt->state._hts_cancel ||
-                             !back_checkmirror(
-                                 opt)) { // cancel level 2 or 1 (cancel parsing)
+                  } else if (opt->state._hts_cancel || !back_checkmirror(opt)) {        // cancel 2 ou 1 (cancel parsing)
                     back_delete(opt, cache, sback, b);  // cancel test
                     stop_looping = 1;
                   }
@@ -729,9 +774,8 @@ int url_savename(lien_adrfilsave *const afs,
                                                 "Loop with HEAD request (during prefetch) at %s%s",
                                                 current.adr, current.fil);
                                 }
-                                if (!hts_wait_available_socket(sback, opt,
-                                                               cache, ptr))
-                                  return -1;
+                                // Ajouter
+                                URLSAVENAME_WAIT_FOR_AVAILABLE_SOCKET();
                                 if (back_add(sback, opt, cache, moved.adr, moved.fil, methode, referer_adr, referer_fil, 1) != -1) {        // OK
                                   hts_log_print(opt, LOG_DEBUG,
                                                 "(during prefetch) %s (%d) to link %s at %s%s",

src/htsparse.c

@@ -3399,7 +3399,20 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
           back_wait(sback, opt, cache, HTS_STAT.stat_timestart);
           back_fillmax(sback, opt, cache, ptr, numero_passe);
 
-          if (!hts_loop_tick(sback, opt, 0, ptr)) {
+          // Transfer rate
+          engine_stats();
+
+          // Refresh various stats
+          HTS_STAT.stat_nsocket = back_nsoc(sback);
+          HTS_STAT.stat_errors = fspc(opt, NULL, "error");
+          HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
+          HTS_STAT.stat_infos = fspc(opt, NULL, "info");
+          HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
+          HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
+
+          if (!RUN_CALLBACK7
+              (opt, loop, sback->lnk, sback->count, 0, ptr, opt->lien_tot,
+               (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
             hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
             *stre->exit_xh_ = 1;        // exit requested
             XH_uninit;
@@ -3410,6 +3423,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             nofollow = 1;       // moins violent
             opt->state._hts_cancel = 0;
           }
+
         }
         // refresh the backing system each 2 seconds
         if (engine_stats()) {
@@ -3946,8 +3960,22 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
         {
           back_wait(sback, opt, cache, HTS_STAT.stat_timestart);
 
+          // Transfer rate
+          engine_stats();
+
+          // Refresh various stats
+          HTS_STAT.stat_nsocket = back_nsoc(sback);
+          HTS_STAT.stat_errors = fspc(opt, NULL, "error");
+          HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
+          HTS_STAT.stat_infos = fspc(opt, NULL, "info");
+          HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
+          HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
+
           b = 0;
-          if (!hts_loop_tick(sback, opt, b, ptr) || !back_checkmirror(opt)) {
+          if (!RUN_CALLBACK7
+              (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
+               (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)
+              || !back_checkmirror(opt)) {
             hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
             *stre->exit_xh_ = 1;        // exit requested
             XH_uninit;
@@ -4053,7 +4081,20 @@ void hts_mirror_process_user_interaction(htsmoduleStruct * str,
       if (!back_checkmirror(opt))
         break;
 
-      if (!hts_loop_tick(sback, opt, b, ptr)) {
+      // Transfer rate
+      engine_stats();
+
+      // Refresh various stats
+      HTS_STAT.stat_nsocket = back_nsoc(sback);
+      HTS_STAT.stat_errors = fspc(opt, NULL, "error");
+      HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
+      HTS_STAT.stat_infos = fspc(opt, NULL, "info");
+      HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
+      HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
+
+      if (!RUN_CALLBACK7
+          (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
+           (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
         hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
         *stre->exit_xh_ = 1;    // exit requested
         XH_uninit;
@@ -4240,12 +4281,26 @@ int hts_mirror_wait_for_next_file(htsmoduleStruct * str,
           freet(s);
         }
 
-        if (!hts_loop_tick(sback, opt, b, ptr)) {
+        // Transfer rate
+        engine_stats();
+
+        // Refresh various stats
+        HTS_STAT.stat_nsocket = back_nsoc(sback);
+        HTS_STAT.stat_errors = fspc(opt, NULL, "error");
+        HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
+        HTS_STAT.stat_infos = fspc(opt, NULL, "info");
+        HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
+        HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
+
+        if (!RUN_CALLBACK7
+            (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
+             (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
           hts_log_print(opt, LOG_ERROR, "Exit requested by shell or user");
           *stre->exit_xh_ = 1;  // exit requested
           XH_uninit;
           return 0;
         }
+
       }
 
 #if HTS_POLL
@@ -4478,9 +4533,10 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
         IS_DELAYED_EXT(afs->save) && continue_loop && loops < 7; loops++) {
       continue_loop = 0;
 
-      /* Wait for an available slot */
-      if (!hts_wait_available_socket(sback, opt, cache, ptr))
-        return -1;
+      /*
+         Wait for an available slot 
+       */
+      WAIT_FOR_AVAILABLE_SOCKET();
 
       /* We can lookup directly in the cache to speedup this mess */
       if (opt->delayed_cached) {
@@ -4626,14 +4682,29 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           if (ptr >= 0) {
             back_fillmax(sback, opt, cache, ptr, numero_passe);
           }
-          if (!hts_loop_tick(sback, opt, b, ptr)) {
-            back_set_unlocked(sback, b);
-            return -1;
-          } else if (opt->state._hts_cancel ||
-                     !back_checkmirror(
-                         opt)) { // cancel level 2 or 1 (cancel parsing)
-            back_delete(opt, cache, sback, b); // cancel test
-            break;
+          // on est obligé d'appeler le shell pour le refresh..
+          {
+
+            // Transfer rate
+            engine_stats();
+
+            // Refresh various stats
+            HTS_STAT.stat_nsocket = back_nsoc(sback);
+            HTS_STAT.stat_errors = fspc(opt, NULL, "error");
+            HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");
+            HTS_STAT.stat_infos = fspc(opt, NULL, "info");
+            HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);
+            HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);
+
+            if (!RUN_CALLBACK7
+                (opt, loop, sback->lnk, sback->count, b, ptr, opt->lien_tot,
+                 (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {
+              back_set_unlocked(sback, b);
+              return -1;
+            } else if (opt->state._hts_cancel || !back_checkmirror(opt)) {      // cancel 2 ou 1 (cancel parsing)
+              back_delete(opt, cache, sback, b);        // cancel test
+              break;
+            }
           }
         } while (
             /* dns/connect/request */

src/htsparse.h

@@ -175,4 +175,33 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
   /* Apply changes */ \
   * str->ptr_ = ptr
 
+#define WAIT_FOR_AVAILABLE_SOCKET()                                            \
+  do {                                                                         \
+    int prev = opt->state._hts_in_html_parsing;                                \
+    while (back_pluggable_sockets_strict(sback, opt) <= 0) {                   \
+      opt->state._hts_in_html_parsing = 6;                                     \
+      /* Wait .. */                                                            \
+      back_wait(sback, opt, cache, 0);                                         \
+      /* time limit (-E) exceeded: stop waiting for a socket (#481) */         \
+      if (!back_checkmirror(opt))                                              \
+        break;                                                                 \
+      /* Transfer rate */                                                      \
+      engine_stats();                                                          \
+      /* Refresh various stats */                                              \
+      HTS_STAT.stat_nsocket = back_nsoc(sback);                                \
+      HTS_STAT.stat_errors = fspc(opt, NULL, "error");                         \
+      HTS_STAT.stat_warnings = fspc(opt, NULL, "warning");                     \
+      HTS_STAT.stat_infos = fspc(opt, NULL, "info");                           \
+      HTS_STAT.nbk = backlinks_done(sback, opt->liens, opt->lien_tot, ptr);    \
+      HTS_STAT.nb = back_transferred(HTS_STAT.stat_bytes, sback);              \
+      /* Check */                                                              \
+      if (!RUN_CALLBACK7(                                                      \
+              opt, loop, sback->lnk, sback->count, -1, ptr, opt->lien_tot,     \
+              (int) (time_local() - HTS_STAT.stat_timestart), &HTS_STAT)) {    \
+        return -1;                                                             \
+      }                                                                        \
+    }                                                                          \
+    opt->state._hts_in_html_parsing = prev;                                    \
+  } while (0)
+
 #endif

tests/35_local-maxsize.test

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# -M byte cap (#77): the crawl must stop with the "giving up" error and keep
+# the mirror well under the 8 x 640KB the fixture totals uncapped.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# cap = -M + the 4 in-flight files the smooth stop lets finish + one of margin
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'More than 400000 bytes have been transferred.. giving up' \
+    --found bigfiles/p0.bin \
+    --max-mirror-bytes 3700000 \
+    httrack 'BASEURL/bigfiles/index.html' -M400000 -c4

tests/Makefile.am

@@ -97,6 +97,7 @@ TESTS = \
 	31_local-javaclass.test \
 	32_local-cdispo.test \
 	33_local-delayed.test \
-	34_local-maxtime.test
+	34_local-maxtime.test \
+	35_local-maxsize.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -16,8 +16,10 @@
 #       --errors N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
+#       --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
+# --max-mirror-bytes asserts the mirrored content (host root) stays under N.
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
@@ -124,7 +126,7 @@ while test "$pos" -lt "$nargs"; do
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --found | --not-found | --directory | --log-found | --log-not-found)
+    --found | --not-found | --directory | --log-found | --log-not-found | --max-mirror-bytes)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -316,6 +318,15 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         else result "OK"; fi
         ;;
+    --max-mirror-bytes)
+        i=$((i + 1))
+        sz=$(find "$hostroot" -type f -exec cat {} + | wc -c | tr -d '[:space:]')
+        info "checking mirror size ${sz} <= ${audit[$i]} bytes"
+        if test "$sz" -le "${audit[$i]}"; then result "OK"; else
+            result "mirror too big"
+            exit 1
+        fi
+        ;;
     --file-matches)
         path="${audit[$((i + 1))]}"
         i=$((i + 2))

tests/local-server.py

@@ -468,11 +468,15 @@ class Handler(SimpleHTTPRequestHandler):
     # so only an engine-side abort can end the crawl.
     TRICKLE_SECONDS = 60
 
-    def route_trickle_index(self):
+    def send_bin_index(self):
+        """Index page linking p0.bin..p7.bin (shared by trickle and bigfiles)."""
         self.send_html(
             "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(8))
         )
 
+    def route_trickle_index(self):
+        self.send_bin_index()
+
     def route_trickle_page(self):
         self.send_response(200)
         self.send_header("Content-Type", "application/octet-stream")
@@ -488,6 +492,15 @@ class Handler(SimpleHTTPRequestHandler):
         except OSError:
             pass
 
+    # -M byte cap (#77): large fast files so a crawl overruns -M immediately.
+    BIGFILE_BYTES = 640 * 1024
+
+    def route_bigfiles_index(self):
+        self.send_bin_index()
+
+    def route_bigfile(self):
+        self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -542,6 +555,15 @@ class Handler(SimpleHTTPRequestHandler):
         "/trickle/p5.bin": route_trickle_page,
         "/trickle/p6.bin": route_trickle_page,
         "/trickle/p7.bin": route_trickle_page,
+        "/bigfiles/index.html": route_bigfiles_index,
+        "/bigfiles/p0.bin": route_bigfile,
+        "/bigfiles/p1.bin": route_bigfile,
+        "/bigfiles/p2.bin": route_bigfile,
+        "/bigfiles/p3.bin": route_bigfile,
+        "/bigfiles/p4.bin": route_bigfile,
+        "/bigfiles/p5.bin": route_bigfile,
+        "/bigfiles/p6.bin": route_bigfile,
+        "/bigfiles/p7.bin": route_bigfile,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,