Add a -M byte-cap non-regression test

The -M limit had no test coverage. New /bigfiles/ fixture (8 fast 640KB
files), a --max-mirror-bytes audit in local-crawl.sh, and a crawl that
must log the "giving up" error and stay under the uncapped total.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.gitignore

@@ -39,3 +39,6 @@ Makefile
 
 # Editor / autotools backup files.
 *~
+
+# Python bytecode (tests/local-server.py).
+__pycache__/

tests/35_local-maxsize.test

@@ -0,0 +1,15 @@
+#!/bin/bash
+#
+# -M byte cap (#77): the crawl must stop with the "giving up" error and keep
+# the mirror well under the 8 x 640KB the fixture totals uncapped.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# cap = -M + the 4 in-flight files the smooth stop lets finish + one of margin
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --log-found 'More than 400000 bytes have been transferred.. giving up' \
+    --found bigfiles/p0.bin \
+    --max-mirror-bytes 3700000 \
+    httrack 'BASEURL/bigfiles/index.html' -M400000 -c4

tests/Makefile.am

@@ -97,6 +97,7 @@ TESTS = \
 	31_local-javaclass.test \
 	32_local-cdispo.test \
 	33_local-delayed.test \
-	34_local-maxtime.test
+	34_local-maxtime.test \
+	35_local-maxsize.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -16,8 +16,10 @@
 #       --errors N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
+#       --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
+# --max-mirror-bytes asserts the mirrored content (host root) stays under N.
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
@@ -124,7 +126,7 @@ while test "$pos" -lt "$nargs"; do
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --found | --not-found | --directory | --log-found | --log-not-found)
+    --found | --not-found | --directory | --log-found | --log-not-found | --max-mirror-bytes)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
@@ -316,6 +318,15 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         else result "OK"; fi
         ;;
+    --max-mirror-bytes)
+        i=$((i + 1))
+        sz=$(find "$hostroot" -type f -exec cat {} + | wc -c | tr -d '[:space:]')
+        info "checking mirror size ${sz} <= ${audit[$i]} bytes"
+        if test "$sz" -le "${audit[$i]}"; then result "OK"; else
+            result "mirror too big"
+            exit 1
+        fi
+        ;;
     --file-matches)
         path="${audit[$((i + 1))]}"
         i=$((i + 2))

tests/local-server.py

@@ -468,11 +468,15 @@ class Handler(SimpleHTTPRequestHandler):
     # so only an engine-side abort can end the crawl.
     TRICKLE_SECONDS = 60
 
-    def route_trickle_index(self):
+    def send_bin_index(self):
+        """Index page linking p0.bin..p7.bin (shared by trickle and bigfiles)."""
         self.send_html(
             "".join('\t<a href="p%d.bin">p%d</a>\n' % (i, i) for i in range(8))
         )
 
+    def route_trickle_index(self):
+        self.send_bin_index()
+
     def route_trickle_page(self):
         self.send_response(200)
         self.send_header("Content-Type", "application/octet-stream")
@@ -488,6 +492,15 @@ class Handler(SimpleHTTPRequestHandler):
         except OSError:
             pass
 
+    # -M byte cap (#77): large fast files so a crawl overruns -M immediately.
+    BIGFILE_BYTES = 640 * 1024
+
+    def route_bigfiles_index(self):
+        self.send_bin_index()
+
+    def route_bigfile(self):
+        self.send_raw(b"x" * self.BIGFILE_BYTES, "application/octet-stream")
+
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -542,6 +555,15 @@ class Handler(SimpleHTTPRequestHandler):
         "/trickle/p5.bin": route_trickle_page,
         "/trickle/p6.bin": route_trickle_page,
         "/trickle/p7.bin": route_trickle_page,
+        "/bigfiles/index.html": route_bigfiles_index,
+        "/bigfiles/p0.bin": route_bigfile,
+        "/bigfiles/p1.bin": route_bigfile,
+        "/bigfiles/p2.bin": route_bigfile,
+        "/bigfiles/p3.bin": route_bigfile,
+        "/bigfiles/p4.bin": route_bigfile,
+        "/bigfiles/p5.bin": route_bigfile,
+        "/bigfiles/p6.bin": route_bigfile,
+        "/bigfiles/p7.bin": route_bigfile,
         "/delayed/noloc.php": route_delayed_noloc,
         "/delayed/selfloop.php": route_delayed_selfloop,
         "/delayed/redir.php": route_delayed_redir,