ci: harden the webhttrack smoke test and drop its leaky watchdog

Follow-up to #540. A narrow-charter review found two issues in the smoke test
as merged:

- The content check matched only the HTTrack brand string, which is a literal
  in every page header, so a truncated or degraded template served 200 would
  still pass. Also require the step-2 form action, which such a page lacks.
- The hard watchdog subshell was un-redirected, so its orphaned sleep kept the
  CI step stdout open ~35s after every run. The poll loop and teardown already
  bound the run, so drop the watchdog and SIGKILL webhttrack if it ignores TERM.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -135,21 +135,52 @@ jobs:
       - name: Build
         run: make -j"$(sysctl -n hw.ncpu)"
 
+      - name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
+        # 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
+        # instantly like Linux; alias them onto lo0 so they don't stall to timeout.
+        run: |
+          set -euo pipefail
+          sudo ifconfig lo0 alias 127.0.0.2 up
+          sudo ifconfig lo0 alias 127.0.0.3 up
+
       - name: Test
-        # bigcrawl's sustained -c8 crawl drops fetches on macOS's loopback when
-        # it competes with other crawls, flaking its exact file count (the #527
-        # macOS drop). Run everything else in parallel, then bigcrawl alone (its
-        # serial-safe condition). Linux tolerates the full parallel run.
         run: |
           jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          rest=$(cd tests && ls *.test | grep -v '^36_local-bigcrawl\.test$' | tr '\n' ' ')
-          make check -j"$jobs" TESTS="$rest"
-          make check TESTS=36_local-bigcrawl.test
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # Runtime smoke of the WebHTTrack launcher on macOS: it carries a Darwin-only
+  # browser path (open -W) and nothing else exercises htsserver. Install into a
+  # temp prefix, then check webhttrack brings up htsserver and serves the UI.
+  webhttrack-macos:
+    name: webhttrack smoke (macOS arm64)
+    runs-on: macos-14
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          brew install autoconf automake libtool autoconf-archive
+
+      - name: Build and install into a temp prefix
+        run: |
+          set -euo pipefail
+          ssl="$(brew --prefix openssl@3)"
+          autoreconf -fi
+          ./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib" \
+            --prefix="$RUNNER_TEMP/inst"
+          make -j"$(sysctl -n hw.ncpu)"
+          make install
+
+      - name: Smoke-test webhttrack
+        run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
+
   # Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
   # -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
   # and bounds math can truncate or wrap in ways 64-bit never reveals (the axis

AGENTS.md

@@ -14,10 +14,10 @@ the operational checklist: toolchain, invariants, and how to ship a change.
   automatically; only a test slower than the current longest raises the floor.
   On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
   spend much of their wall time asleep (server trickles, httrack self-pacing),
-  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)`. macOS runs
-  36_local-bigcrawl alone in a second pass: its sustained `-c8` crawl overloads
-  the macOS loopback when it competes with other crawls and flakes its exact
-  file count (Linux tolerates the full parallel run).
+  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
+  platform, macOS included: the test server raises its listen backlog
+  (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
+  `-c16` bigcrawl the way Python's default backlog of 5 did.
   Or run `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants

src/htsfilters.c

@@ -121,12 +121,18 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
   return use1 ? jok1 : jok2;
 }
 
+/* Real filters/URLs fit HTS_URLMAXSIZE*2; past it a hostile pattern recurses
+   O(len) deep or runs O(n^2*stars). Cap length (depth) and steps (work). */
+#define STRJOKER_MAXLEN (HTS_URLMAXSIZE * 2)
+#define STRJOKER_MAXSTEPS 2000000u
+
 /* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
    pair keeps star-heavy patterns polynomial instead of exponential (#501). */
 typedef struct strjoker_memo {
   const char *chaine0, *joker0; /* offsets are relative to these bases */
   size_t stride;                /* strlen(joker0) + 1 */
   unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
+  size_t *nsteps; /* shared work counter; NULL: unbounded (oracle) */
 } strjoker_memo;
 
 static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
@@ -141,6 +147,8 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
   size_t bit = 0;
   const char *adr;
 
+  if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
+    return NULL; /* work budget spent: fail the match safely */
   if (memo->failed) {
     bit = (size_t) (chaine - memo->chaine0) * memo->stride +
           (size_t) (joker - memo->joker0);
@@ -153,22 +161,24 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
   return adr;
 }
 
-// wildcard comparator: match chaine against joker (pattern), case-insensitive
-// returns the address of the first matched letter past any leading joker
-// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
-                                LLint *size, int *size_flag) {
-  strjoker_memo memo = {chaine, joker, 0, NULL};
+/* Match chaine against joker with a shared work budget *nsteps (a strjokerfind
+   sweep passes one counter, bounding the whole scan). */
+static const char *strjoker_bounded(const char *chaine, const char *joker,
+                                    LLint *size, int *size_flag,
+                                    size_t *nsteps) {
+  strjoker_memo memo = {chaine, joker, 0, NULL, nsteps};
   unsigned char stackbits[2048];
   hts_boolean onheap = HTS_FALSE;
   const char *adr;
 
   if (chaine != NULL && joker != NULL) {
-    const size_t n1 = strlen(chaine) + 1;
+    const size_t l1 = strlen(chaine), l2 = strlen(joker);
 
-    memo.stride = strlen(joker) + 1;
-    if (n1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
-      const size_t bytes = (n1 * memo.stride + 7) / 8;
+    if (l1 > STRJOKER_MAXLEN || l2 > STRJOKER_MAXLEN)
+      return NULL; /* beyond any real filter/URL: bound depth+work */
+    memo.stride = l2 + 1;
+    if (l1 + 1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
+      const size_t bytes = ((l1 + 1) * memo.stride + 7) / 8;
 
       if (bytes <= sizeof(stackbits)) {
         memset(stackbits, 0, bytes);
@@ -185,6 +195,16 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
   return adr;
 }
 
+// wildcard comparator: match chaine against joker (pattern), case-insensitive
+// returns the address of the first matched letter past any leading joker
+// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
+                                LLint *size, int *size_flag) {
+  size_t nsteps = 0;
+
+  return strjoker_bounded(chaine, joker, size, size_flag, &nsteps);
+}
+
 /* Test-only oracle: the same matcher without the failure memo. */
 const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
                             int *size_flag) {
@@ -431,12 +451,18 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 // d'un strcpy sur une variable ayant un nom en lettres et copiant une chaine de chiffres
 // ATTENTION!! Eviter les jokers en début, où gare au temps machine!
 const char *strjokerfind(const char *chaine, const char *joker) {
+  size_t nsteps = 0; /* one budget for the whole scan */
   const char *adr;
 
+  if (chaine != NULL && strlen(chaine) > STRJOKER_MAXLEN)
+    return NULL;
   while(*chaine) {
-    if ((adr = strjoker(chaine, joker, NULL, NULL))) {  // ok trouvé
+    if ((adr = strjoker_bounded(chaine, joker, NULL, NULL,
+                                &nsteps))) { // ok trouvé
       return adr;
     }
+    if (nsteps > STRJOKER_MAXSTEPS) // scan budget spent: no match
+      return NULL;
     chaine++;
   }
   return NULL;

src/htslib.c

@@ -2575,6 +2575,11 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
     // recopier adrfil->adresse www..
     strncatbuff(adrfil->adr, p, ((int) (q - p)));
     // *( adrfil->adr+( ((int) q) - ((int) p) ) )=0;  // faut arrêter la fumette!
+
+    // fil[] holds the path plus a possible leading '/' the top strlen() misses.
+    if (strlen(q) >= sizeof(adrfil->fil) - (q[0] != '/' ? 1 : 0))
+      return -1;
+
     // recopier chemin /pub/..
     if (q[0] != '/')            // page par défaut (/)
       strcatbuff(adrfil->fil, "/");

src/htsselftest.c

@@ -740,6 +740,40 @@ static int st_filterdual(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Length/work caps stop a hostile pattern stack-overflowing or hanging the
+   process (OSS-Fuzz 5060751291908096 / 5745936014573568). */
+static int st_filterbounds(httrackp *opt, int argc, char **argv) {
+  const size_t big = 100000; /* well past the length cap */
+  const size_t stars = 900;  /* star-heavy pattern, under the cap */
+  char *subj = malloct(big + 1);
+  char *pat = malloct(2 * stars + 2);
+  size_t i;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  memset(subj, 'a', big);
+  subj[big] = '\0';
+  /* '*' matches anything, but an over-length subject is refused by the cap */
+  assertf(strjoker(subj, "*", NULL, NULL) == NULL);
+  assertf(strjokerfind(subj, "*") == NULL);
+  /* within-cap star-heavy dead-end: the step budget keeps it bounded (a hang
+     would time the test out) */
+  for (i = 0; i < stars; i++) {
+    pat[2 * i] = '*';
+    pat[2 * i + 1] = 'a';
+  }
+  pat[2 * stars] = 'b'; /* never matches an all-'a' subject */
+  pat[2 * stars + 1] = '\0';
+  subj[stars] = '\0';
+  assertf(strjoker(subj, pat, NULL, NULL) == NULL);
+  assertf(strjokerfind(subj, pat) == NULL);
+  freet(pat);
+  freet(subj);
+  printf("filterbounds: OK\n");
+  return 0;
+}
+
 static int st_simplify(httrackp *opt, int argc, char **argv) {
   (void) opt;
   if (argc < 1) {
@@ -1266,6 +1300,30 @@ static int st_identurl(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Regression for the one-byte fil[] overflow: a 2047-byte hostless "?"-URL used
+   to abort in strncat_safe_ when the missing leading '/' pushed fil to 2048. */
+static int st_identabs(httrackp *opt, int argc, char **argv) {
+  lien_adrfil af;
+  const size_t len =
+      sizeof(af.fil) - 1; /* 2047: max URL the top guard admits */
+  char *url = malloct(len + 1);
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  url[0] = '?';
+  memset(url + 1, 'a', len - 1);
+  url[len] = '\0';
+  assertf(ident_url_absolute(url, &af) == -1);
+  freet(url);
+  /* valid URLs still parse, so the guard is not over-rejecting */
+  assertf(ident_url_absolute("http://www.example.com/a/b/c.html?x=1", &af) ==
+          0);
+  assertf(ident_url_absolute("www.foo.com?bar=1", &af) == 0);
+  printf("identabs self-test OK\n");
+  return 0;
+}
+
 /* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
    no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
    registering an already-crawled link (dedup/collision paths). */
@@ -2537,6 +2595,8 @@ static const struct selftest_entry {
      "memoized vs unmemoized matcher differential", st_filtermemo},
     {"filterdual", "<string1> <string2> <filter>...",
      "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
+    {"filterbounds", "", "matcher length/work caps reject hostile patterns",
+     st_filterbounds},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
@@ -2567,6 +2627,8 @@ static const struct selftest_entry {
     {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
      st_resolve},
     {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
+    {"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
+     st_identabs},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
     {"headerlong", "[header-name:]",

tests/01_engine-filter.test

@@ -192,3 +192,6 @@ test "$(with_timeout httrack -O /dev/null -#test=filter "$pat" "$subj")" == "$su
 # (guards against a single-backtrack-point matcher rewrite)
 match '*[aX]X*[a]Y' 'aXaXaY'
 nomatch '*[aX]X*[a]Y' 'aXbXaY'
+
+# hostile patterns must not stack-overflow or hang: length + work caps
+test "$(with_timeout httrack -O /dev/null -#test=filterbounds)" == "filterbounds: OK" || exit 1

tests/01_engine-identabs.test

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# A 2047-byte hostless "?"-URL used to overflow fil[] by one byte and abort in
+# strncat_safe_; the guard now returns -1 while valid URLs still parse.
+test "$(httrack -O /dev/null -#test=identabs)" == "identabs self-test OK" || exit 1

tests/36_local-bigcrawl.test

@@ -53,4 +53,4 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --log-found ', no files updated' \
     --max-mirror-bytes 700000 \
     --min-mirror-bytes 500000 \
-    httrack 'BASEURL/big/index.html' --retries=0 -c8 -%c100 -A100000000
+    httrack 'BASEURL/big/index.html' --retries=0 -c16 -%c100 -A100000000

tests/Makefile.am

@@ -44,6 +44,7 @@ TESTS = \
 	01_engine-header.test \
 	01_engine-idna.test \
 	01_engine-identurl.test \
+	01_engine-identabs.test \
 	01_engine-escape-room.test \
 	01_engine-inplace-escape.test \
 	01_engine-java.test \

tests/local-server.py

@@ -1645,7 +1645,12 @@ def main():
     def factory(*a, **kw):
         return Handler(*a, directory=root, **kw)
 
-    httpd = ThreadingHTTPServer((args.bind, 0), factory)
+    # macOS/BSD drop SYNs when the listen backlog overflows (Linux is lenient);
+    # raise it from Python's default 5 so a busy -c8 crawl can't lose fetches.
+    class BacklogHTTPServer(ThreadingHTTPServer):
+        request_queue_size = 128
+
+    httpd = BacklogHTTPServer((args.bind, 0), factory)
 
     if args.tls:
         import ssl

tests/webhttrack-smoke.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+# Smoke-test an installed webhttrack: launch it with a stub browser and assert
+# htsserver comes up and serves the web UI. Arg: the install prefix.
+set -euo pipefail
+
+prefix="${1:?usage: webhttrack-smoke.sh <install-prefix>}"
+wht="$prefix/bin/webhttrack"
+test -x "$wht" || {
+    echo "no webhttrack at $wht" >&2
+    exit 1
+}
+
+work="$(mktemp -d)"
+# webhttrack backgrounds htsserver, which outlives it; reap any stray one (scoped
+# to this prefix) so a lingering server can never hold the CI step open.
+trap 'pkill -f "$prefix/bin/htsserver" 2>/dev/null || true; rm -rf "$work"' EXIT
+export HOME="$work/home"
+mkdir -p "$HOME/websites"
+marker="$work/marker"
+
+stubdir="$work/bin"
+mkdir -p "$stubdir"
+
+# On Darwin webhttrack hardcodes "open -W", which launches a real GUI browser and
+# blocks headless. Shadow uname so it takes the generic path and picks the stub
+# browser below; htsserver and webhttrack's path resolution still run for real.
+cat >"$stubdir/uname" <<'EOF'
+#!/bin/bash
+[ "${1:-}" = "-s" ] && {
+    echo Linux
+    exit 0
+}
+exec /usr/bin/uname "$@"
+EOF
+chmod +x "$stubdir/uname"
+
+# Stub browser: webhttrack tries its browser-name list in order and runs the
+# first it finds, so shadow the first entry, "x-www-browser". It fetches the
+# server URL and records PASS only for the working UI: the brand string AND the
+# step-2 form action, which a truncated/degraded template page would lack (the
+# bare title alone is not enough). htsserver only lives until webhttrack exits,
+# so the check has to happen here.
+# -a: the UI is served ISO-8859-1, so grep must not treat it as binary.
+cat >"$stubdir/x-www-browser" <<EOF
+#!/bin/bash
+echo "stub browser invoked with: \$1" >&2
+if body="\$(curl -fsSL --max-time 20 "\$1")" && printf '%s' "\$body" | grep -qai httrack && printf '%s' "\$body" | grep -qaF step2.html; then
+    echo PASS >"$marker"
+else
+    echo "FAIL: unexpected response from \$1" >"$marker"
+fi
+EOF
+chmod +x "$stubdir/x-www-browser"
+export PATH="$stubdir:$prefix/bin:$PATH"
+
+echo "launching webhttrack"
+"$wht" </dev/null >"$work/webhttrack.log" 2>&1 &
+whpid=$!
+
+# Bounded poll for the marker (macOS has no timeout(1)); teardown below kills
+# webhttrack and reaps htsserver, so the run is bounded without a watchdog.
+for i in $(seq 1 45); do
+    test -f "$marker" && {
+        echo "marker written after ${i}s"
+        break
+    }
+    kill -0 "$whpid" 2>/dev/null || {
+        echo "webhttrack exited on its own after ${i}s"
+        break
+    }
+    sleep 1
+done
+
+# Reap webhttrack and the htsserver it spawned. Confirm death with a bounded poll
+# (not a blocking wait, which could hang on macOS); SIGKILL if it ignores TERM.
+echo "tearing down"
+kill "$whpid" 2>/dev/null || true
+if pkill -f "$prefix/bin/htsserver" 2>/dev/null; then
+    echo "reaped a lingering htsserver"
+else
+    echo "no lingering htsserver"
+fi
+for _ in $(seq 1 10); do
+    kill -0 "$whpid" 2>/dev/null || break
+    sleep 1
+done
+kill -9 "$whpid" 2>/dev/null || true
+
+echo "--- webhttrack.log ---"
+cat "$work/webhttrack.log" 2>/dev/null || true
+echo "--- end ---"
+echo "marker=[$(cat "$marker" 2>/dev/null || echo NONE)]"
+
+if test "$(cat "$marker" 2>/dev/null || true)" = PASS; then
+    echo "webhttrack smoke: PASS"
+else
+    echo "webhttrack smoke: FAIL" >&2
+    exit 1
+fi