Drop the leftover SIGSTKFLT comment

A comment documenting a signal we deliberately don't handle is noise.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -135,21 +135,52 @@ jobs:
       - name: Build
         run: make -j"$(sysctl -n hw.ncpu)"
 
+      - name: Add loopback aliases (macOS lacks 127.0.0.2/.3)
+        # 19_local-connect-fallback needs the dead 127.0.0.2/.3 to refuse
+        # instantly like Linux; alias them onto lo0 so they don't stall to timeout.
+        run: |
+          set -euo pipefail
+          sudo ifconfig lo0 alias 127.0.0.2 up
+          sudo ifconfig lo0 alias 127.0.0.3 up
+
       - name: Test
-        # bigcrawl's sustained -c8 crawl drops fetches on macOS's loopback when
-        # it competes with other crawls, flaking its exact file count (the #527
-        # macOS drop). Run everything else in parallel, then bigcrawl alone (its
-        # serial-safe condition). Linux tolerates the full parallel run.
         run: |
           jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          rest=$(cd tests && ls *.test | grep -v '^36_local-bigcrawl\.test$' | tr '\n' ' ')
-          make check -j"$jobs" TESTS="$rest"
-          make check TESTS=36_local-bigcrawl.test
+          make check -j"$jobs"
 
       - name: Print the test log on failure
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
+  # Runtime smoke of the WebHTTrack launcher on macOS: it carries a Darwin-only
+  # browser path (open -W) and nothing else exercises htsserver. Install into a
+  # temp prefix, then check webhttrack brings up htsserver and serves the UI.
+  webhttrack-macos:
+    name: webhttrack smoke (macOS arm64)
+    runs-on: macos-14
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: recursive
+
+      - name: Install build dependencies
+        run: |
+          set -euo pipefail
+          brew install autoconf automake libtool autoconf-archive
+
+      - name: Build and install into a temp prefix
+        run: |
+          set -euo pipefail
+          ssl="$(brew --prefix openssl@3)"
+          autoreconf -fi
+          ./configure CPPFLAGS="-I${ssl}/include" LDFLAGS="-L${ssl}/lib" \
+            --prefix="$RUNNER_TEMP/inst"
+          make -j"$(sysctl -n hw.ncpu)"
+          make install
+
+      - name: Smoke-test webhttrack
+        run: bash tests/webhttrack-smoke.sh "$RUNNER_TEMP/inst"
+
   # Portability/hardening: 32-bit (i386) build on the x86-64 runner via multilib
   # -- no extra hardware. Exercises the 32-bit size_t/pointer ABI, where size
   # and bounds math can truncate or wrap in ways 64-bit never reveals (the axis

AGENTS.md

@@ -14,10 +14,10 @@ the operational checklist: toolchain, invariants, and how to ship a change.
   automatically; only a test slower than the current longest raises the floor.
   On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
   spend much of their wall time asleep (server trickles, httrack self-pacing),
-  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)`. macOS runs
-  36_local-bigcrawl alone in a second pass: its sustained `-c8` crawl overloads
-  the macOS loopback when it competes with other crawls and flakes its exact
-  file count (Linux tolerates the full parallel run).
+  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
+  platform, macOS included: the test server raises its listen backlog
+  (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
+  `-c16` bigcrawl the way Python's default backlog of 5 did.
   Or run `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants

src/htsback.c

@@ -3908,10 +3908,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           if (fp) {
                             LLint alloc_mem = resume + 1;
 
-                            // Reject a hostile Content-Length that would
-                            // overflow the buffer size: drop the partial and
-                            // refetch.
-                            if (back[i].r.totalsize > INT64_MAX - alloc_mem) {
+                            // Bound the in-memory buffer to a 32-bit size (real
+                            // in-RAM resources are far smaller); a hostile
+                            // Content-Length that would overflow the add or the
+                            // (size_t) cast is dropped and refetched instead.
+                            if (back[i].r.totalsize > INT32_MAX - alloc_mem) {
                               url_savename_refname_remove(opt, back[i].url_adr,
                                                           back[i].url_fil);
                               UNLINK(back[i].url_sav);

src/htsfilters.c

@@ -121,12 +121,18 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
   return use1 ? jok1 : jok2;
 }
 
+/* Real filters/URLs fit HTS_URLMAXSIZE*2; past it a hostile pattern recurses
+   O(len) deep or runs O(n^2*stars). Cap length (depth) and steps (work). */
+#define STRJOKER_MAXLEN (HTS_URLMAXSIZE * 2)
+#define STRJOKER_MAXSTEPS 2000000u
+
 /* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
    pair keeps star-heavy patterns polynomial instead of exponential (#501). */
 typedef struct strjoker_memo {
   const char *chaine0, *joker0; /* offsets are relative to these bases */
   size_t stride;                /* strlen(joker0) + 1 */
   unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
+  size_t *nsteps; /* shared work counter; NULL: unbounded (oracle) */
 } strjoker_memo;
 
 static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
@@ -141,6 +147,8 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
   size_t bit = 0;
   const char *adr;
 
+  if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
+    return NULL; /* work budget spent: fail the match safely */
   if (memo->failed) {
     bit = (size_t) (chaine - memo->chaine0) * memo->stride +
           (size_t) (joker - memo->joker0);
@@ -153,22 +161,24 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
   return adr;
 }
 
-// wildcard comparator: match chaine against joker (pattern), case-insensitive
-// returns the address of the first matched letter past any leading joker
-// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
-                                LLint *size, int *size_flag) {
-  strjoker_memo memo = {chaine, joker, 0, NULL};
+/* Match chaine against joker with a shared work budget *nsteps (a strjokerfind
+   sweep passes one counter, bounding the whole scan). */
+static const char *strjoker_bounded(const char *chaine, const char *joker,
+                                    LLint *size, int *size_flag,
+                                    size_t *nsteps) {
+  strjoker_memo memo = {chaine, joker, 0, NULL, nsteps};
   unsigned char stackbits[2048];
   hts_boolean onheap = HTS_FALSE;
   const char *adr;
 
   if (chaine != NULL && joker != NULL) {
-    const size_t n1 = strlen(chaine) + 1;
+    const size_t l1 = strlen(chaine), l2 = strlen(joker);
 
-    memo.stride = strlen(joker) + 1;
-    if (n1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
-      const size_t bytes = (n1 * memo.stride + 7) / 8;
+    if (l1 > STRJOKER_MAXLEN || l2 > STRJOKER_MAXLEN)
+      return NULL; /* beyond any real filter/URL: bound depth+work */
+    memo.stride = l2 + 1;
+    if (l1 + 1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
+      const size_t bytes = ((l1 + 1) * memo.stride + 7) / 8;
 
       if (bytes <= sizeof(stackbits)) {
         memset(stackbits, 0, bytes);
@@ -185,6 +195,16 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
   return adr;
 }
 
+// wildcard comparator: match chaine against joker (pattern), case-insensitive
+// returns the address of the first matched letter past any leading joker
+// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
+                                LLint *size, int *size_flag) {
+  size_t nsteps = 0;
+
+  return strjoker_bounded(chaine, joker, size, size_flag, &nsteps);
+}
+
 /* Test-only oracle: the same matcher without the failure memo. */
 const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
                             int *size_flag) {
@@ -431,12 +451,18 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 // d'un strcpy sur une variable ayant un nom en lettres et copiant une chaine de chiffres
 // ATTENTION!! Eviter les jokers en début, où gare au temps machine!
 const char *strjokerfind(const char *chaine, const char *joker) {
+  size_t nsteps = 0; /* one budget for the whole scan */
   const char *adr;
 
+  if (chaine != NULL && strlen(chaine) > STRJOKER_MAXLEN)
+    return NULL;
   while(*chaine) {
-    if ((adr = strjoker(chaine, joker, NULL, NULL))) {  // ok trouvé
+    if ((adr = strjoker_bounded(chaine, joker, NULL, NULL,
+                                &nsteps))) { // ok trouvé
       return adr;
     }
+    if (nsteps > STRJOKER_MAXSTEPS) // scan budget spent: no match
+      return NULL;
     chaine++;
   }
   return NULL;

src/htslib.c

@@ -1059,10 +1059,10 @@ static int append_cookie_header(buff_struct *bstr, t_cookie *cookie,
   return cook;
 }
 
-/* Self-test entry for append_cookie_header(): build the request Cookie line
-   into dst (always NUL-terminated). Returns the number of cookies emitted. */
-int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
-                                const char *path, char *dst, size_t dst_size) {
+/* Build the request Cookie line for domain/path into dst (always
+   NUL-terminated). Returns the number of cookies emitted. */
+int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
+                       char *dst, size_t dst_size) {
   buff_struct bstr = {dst, dst_size, 0};
 
   assertf(dst != NULL && dst_size > 0);
@@ -2004,6 +2004,15 @@ LLint http_xfread1(htsblk * r, int bufl) {
 
   if (bufl > 0) {
     if (!r->is_write) {         // stocker en mémoire
+      // In-memory content must fit a 32-bit index (allocs below add 1, reads
+      // use int offsets): reject a hostile Content-Length or endless stream.
+      const LLint inmem_want =
+          (r->totalsize >= 0) ? r->totalsize : (r->size + bufl);
+      if (inmem_want >= INT32_MAX) {
+        r->statuscode = STATUSCODE_INVALID;
+        strcpybuff(r->msg, "In-memory content too large");
+        return READ_ERROR;
+      }
       if (r->totalsize >= 0) {  // totalsize déterminé ET ALLOUE
         if (r->adr == NULL) {
           r->adr = (char *) malloct((size_t) r->totalsize + 1);
@@ -2566,6 +2575,11 @@ int ident_url_absolute(const char *url, lien_adrfil *adrfil) {
     // recopier adrfil->adresse www..
     strncatbuff(adrfil->adr, p, ((int) (q - p)));
     // *( adrfil->adr+( ((int) q) - ((int) p) ) )=0;  // faut arrêter la fumette!
+
+    // fil[] holds the path plus a possible leading '/' the top strlen() misses.
+    if (strlen(q) >= sizeof(adrfil->fil) - (q[0] != '/' ? 1 : 0))
+      return -1;
+
     // recopier chemin /pub/..
     if (q[0] != '/')            // page par défaut (/)
       strcatbuff(adrfil->fil, "/");

src/htslib.h

@@ -184,10 +184,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
                   const char *referer_adr, const char *referer_fil,
                   htsblk * retour);
 /* Build the request "Cookie:" header line for stored cookies matching
-   domain/path into dst (NUL-terminated). Exposed for the -#Q self-test;
-   wraps the same logic http_sendhead() uses. Returns cookies emitted. */
-int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
-                                const char *path, char *dst, size_t dst_size);
+   domain/path into dst (NUL-terminated), wrapping the logic http_sendhead()
+   uses. Returns cookies emitted. */
+int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
+                       char *dst, size_t dst_size);
 
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
               int waitconnect);

src/htsparse.c

@@ -4311,6 +4311,8 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
       *forbidden_url == 0 && IS_DELAYED_EXT(afs->save) && !opt->state.stop) {
     int loops;
     int continue_loop;
+    char BIGSTK
+        cookie_before[16384]; /* #15: this URL's Cookie header, pre-request */
 
     hts_log_print(opt, LOG_DEBUG, "Waiting for type to be known: %s%s", afs->af.adr,
                   afs->af.fil);
@@ -4319,6 +4321,12 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
     for(loops = 0, continue_loop = 1;
         IS_DELAYED_EXT(afs->save) && continue_loop && loops < 7; loops++) {
       continue_loop = 0;
+      /* #15: snapshot the Cookie header THIS url would send, so only its own
+         self-redirect Set-Cookie trips the retry, not a concurrent slot's. */
+      cookie_before[0] = '\0';
+      if (opt->accept_cookie && opt->cookie != NULL)
+        http_cookie_header(opt->cookie, jump_identification_const(afs->af.adr),
+                           afs->af.fil, cookie_before, sizeof(cookie_before));
 
       /* Wait for an available slot */
       if (!hts_wait_available_socket(sback, opt, cache, ptr))
@@ -4516,6 +4524,17 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           /* Moved! */
           else if (HTTP_IS_REDIRECT(back[b].r.statuscode)) {
             char BIGSTK mov_url[HTS_URLMAXSIZE * 2];
+            char BIGSTK cookie_after[16384];
+            hts_boolean cookies_changed;
+
+            /* #15: cookie-wall signal: this URL's own Cookie header changed
+               across its self-redirect (a Set-Cookie it just set). */
+            cookie_after[0] = '\0';
+            if (opt->accept_cookie && opt->cookie != NULL)
+              http_cookie_header(
+                  opt->cookie, jump_identification_const(afs->af.adr),
+                  afs->af.fil, cookie_after, sizeof(cookie_after));
+            cookies_changed = strcmp(cookie_before, cookie_after) != 0;
 
             mov_url[0] = '\0';
             strcpybuff(mov_url, back[b].r.location);    // copier URL
@@ -4585,11 +4604,24 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
                   url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, 
                                opt, sback, cache, hash, ptr, numero_passe,
                                &delayed_back);
+                } else if (cookies_changed) {
+                  // #15: cookie-wall self-redirect; evict the cached
+                  // fast-header so the re-issue refetches with the cookie.
+                  if (cache->cached_tests != NULL)
+                    coucal_remove(cache->cached_tests,
+                                  concat(OPT_GET_BUFF(opt),
+                                         OPT_GET_BUFF_SIZE(opt), afs->af.adr,
+                                         afs->af.fil));
+                  afs->save[0] = '\0';
+                  url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, opt,
+                               sback, cache, hash, ptr, numero_passe,
+                               &delayed_back);
+                  continue_loop = 1;
                 } else {
                   hts_log_print(opt, LOG_WARNING,
                                 "Unable to test %s%s (loop to same filename)",
                                 afs->af.adr, afs->af.fil);
-                }               // loop to same location
+                } // loop to same location
               }                 // ident_url_relatif()
             }                   // location
           }                     // redirect

src/htsselftest.c

@@ -740,6 +740,40 @@ static int st_filterdual(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Length/work caps stop a hostile pattern stack-overflowing or hanging the
+   process (OSS-Fuzz 5060751291908096 / 5745936014573568). */
+static int st_filterbounds(httrackp *opt, int argc, char **argv) {
+  const size_t big = 100000; /* well past the length cap */
+  const size_t stars = 900;  /* star-heavy pattern, under the cap */
+  char *subj = malloct(big + 1);
+  char *pat = malloct(2 * stars + 2);
+  size_t i;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  memset(subj, 'a', big);
+  subj[big] = '\0';
+  /* '*' matches anything, but an over-length subject is refused by the cap */
+  assertf(strjoker(subj, "*", NULL, NULL) == NULL);
+  assertf(strjokerfind(subj, "*") == NULL);
+  /* within-cap star-heavy dead-end: the step budget keeps it bounded (a hang
+     would time the test out) */
+  for (i = 0; i < stars; i++) {
+    pat[2 * i] = '*';
+    pat[2 * i + 1] = 'a';
+  }
+  pat[2 * stars] = 'b'; /* never matches an all-'a' subject */
+  pat[2 * stars + 1] = '\0';
+  subj[stars] = '\0';
+  assertf(strjoker(subj, pat, NULL, NULL) == NULL);
+  assertf(strjokerfind(subj, pat) == NULL);
+  freet(pat);
+  freet(subj);
+  printf("filterbounds: OK\n");
+  return 0;
+}
+
 static int st_simplify(httrackp *opt, int argc, char **argv) {
   (void) opt;
   if (argc < 1) {
@@ -1266,6 +1300,30 @@ static int st_identurl(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Regression for the one-byte fil[] overflow: a 2047-byte hostless "?"-URL used
+   to abort in strncat_safe_ when the missing leading '/' pushed fil to 2048. */
+static int st_identabs(httrackp *opt, int argc, char **argv) {
+  lien_adrfil af;
+  const size_t len =
+      sizeof(af.fil) - 1; /* 2047: max URL the top guard admits */
+  char *url = malloct(len + 1);
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+  url[0] = '?';
+  memset(url + 1, 'a', len - 1);
+  url[len] = '\0';
+  assertf(ident_url_absolute(url, &af) == -1);
+  freet(url);
+  /* valid URLs still parse, so the guard is not over-rejecting */
+  assertf(ident_url_absolute("http://www.example.com/a/b/c.html?x=1", &af) ==
+          0);
+  assertf(ident_url_absolute("www.foo.com?bar=1", &af) == 0);
+  printf("identabs self-test OK\n");
+  return 0;
+}
+
 /* Extra args are key=value: adr= cdispo= statuscode= status= strip= urlhack=
    no-www= no-slash= no-query= n83= type=, plus repeatable prior=adr|fil|sav
    registering an already-crawled link (dedup/collision paths). */
@@ -1310,6 +1368,59 @@ static int st_headerlong(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* http_xfread1 must refuse an in-memory buffer whose size would exceed a 32-bit
+   index (hostile Content-Length or endless stream) rather than allocate it.
+   The guard returns before any socket read, so no real connection is needed. */
+static int st_xfread_limit(httrackp *opt, int argc, char **argv) {
+  htsblk r;
+
+  (void) opt;
+  (void) argc;
+  (void) argv;
+
+  // Content-Length just over 2 GiB.
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = (LLint) INT32_MAX + 1;
+  printf("bylen: refused=%d adr=%s msg=%s\n",
+         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
+         r.msg);
+  if (r.adr != NULL)
+    freet(r.adr);
+
+  // Unknown length, buffer already at the limit: the next read would exceed it.
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = -1;
+  r.size = (LLint) INT32_MAX;
+  printf("bygrow: refused=%d adr=%s msg=%s\n",
+         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
+         r.msg);
+  if (r.adr != NULL)
+    freet(r.adr);
+
+  // Exactly at the 2 GiB index (size + bufl == INT32_MAX): must also be
+  // refused, since the reallocs below add 1 (a `> INT32_MAX` check would let
+  // this through and overflow the int realloc size).
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = -1;
+  r.size = (LLint) INT32_MAX - 8192;
+  http_xfread1(&r, 8192);
+  printf("boundary: msg=%s\n", r.msg);
+
+  // A legitimate small size must NOT be refused by the guard (the read then
+  // fails on the invalid socket, but the size-too-large msg must not be set).
+  memset(&r, 0, sizeof(r));
+  r.soc = INVALID_SOCKET;
+  r.totalsize = 1000;
+  http_xfread1(&r, 8192);
+  printf("accept: msg=%s\n", r.msg);
+  if (r.adr != NULL)
+    freet(r.adr);
+  return 0;
+}
+
 /* Parse a Content-Range header and print the sanitized triple. A hostile value
    (negative or INT64 extreme) must clamp to 0 without signed-overflow UB. */
 static int st_crange(httrackp *opt, int argc, char **argv) {
@@ -1702,7 +1813,7 @@ static int st_cookies(httrackp *opt, int argc, char **argv) {
     return 1;
   }
 
-  http_cookie_header_selftest(&cookie, dom, "/", hdr, sizeof(hdr));
+  http_cookie_header(&cookie, dom, "/", hdr, sizeof(hdr));
   if (strcmp(hdr, expected) != 0)
     err = 1;
   if (strstr(hdr, "$Version") != NULL || strstr(hdr, "$Path") != NULL)
@@ -2484,6 +2595,8 @@ static const struct selftest_entry {
      "memoized vs unmemoized matcher differential", st_filtermemo},
     {"filterdual", "<string1> <string2> <filter>...",
      "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
+    {"filterbounds", "", "matcher length/work caps reject hostile patterns",
+     st_filterbounds},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
@@ -2514,6 +2627,8 @@ static const struct selftest_entry {
     {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
      st_resolve},
     {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
+    {"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
+     st_identabs},
     {"header", "<raw-header-line> ...", "response header-line parsing",
      st_header},
     {"headerlong", "[header-name:]",
@@ -2521,6 +2636,8 @@ static const struct selftest_entry {
      st_headerlong},
     {"crange", "<raw-content-range-line> ...",
      "Content-Range parse integer safety", st_crange},
+    {"xfread-limit", "", "in-memory receive buffer size bound",
+     st_xfread_limit},
     {"savename", "<fil> <content-type> [key=value ...]",
      "local save-name for a URL", st_savename},
     {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",

src/httrack.c

@@ -919,9 +919,6 @@ static void signal_handlers(void) {
 #ifdef SIGSEGV
   signal(SIGSEGV, sig_fatal);   // segmentation violation
 #endif
-#ifdef SIGSTKFLT
-  signal(SIGSTKFLT, sig_fatal); // stack fault
-#endif
 }
 
 // fin routines de détournement de SIGHUP & co

src/webhttrack

@@ -12,10 +12,9 @@ if test -n "${BROWSER}"; then
 fi
 # Patch for Darwin/Mac by Ross Williams
 if test "$(uname -s)" == "Darwin"; then
-    # Darwin/Mac OS X uses a system 'open' command to find
-    # the default browser. The -W flag causes it to wait for
-    # the browser to exit
-    BROWSEREXE="/usr/bin/open -W"
+    # Darwin's 'open -W' launches the default browser and waits for it to quit.
+    # Keep it only as a fallback so a configured $BROWSER still wins the search.
+    BROWSEREXEFALLBACK="/usr/bin/open -W"
 fi
 BINWD=$(dirname "$0")
 SRCHPATH=("$BINWD" /usr/local/bin /usr/share/bin /usr/bin /usr/lib/httrack /usr/local/lib/httrack /usr/local/share/httrack /opt/local/bin /sw/bin "${HOME}/usr/bin" "${HOME}/bin")
@@ -81,6 +80,7 @@ for i in "${SRCHBROWSEREXE[@]}"; do
     done
     test -n "$BROWSEREXE" && break
 done
+test -n "$BROWSEREXE" || BROWSEREXE="${BROWSEREXEFALLBACK}"
 test -n "$BROWSEREXE" || ! log "Could not find any suitable browser" || exit 1
 
 # "browse" command
@@ -132,12 +132,12 @@ function cleanup {
 }
 
 # Cleanup in case of emergency
-trap "cleanup now; exit" HUP INT QUIT ILL TRAP ABRT BUS FPE SEGV PIPE ALRM TERM STKFLT XCPU XFSZ
+trap "cleanup now; exit" HUP INT QUIT PIPE TERM
 
 # Got SRVURL, launch browser
 launch_browser "${BROWSEREXE}" "${SRVURL}"
 
 # That's all, folks!
-trap "" HUP INT QUIT ILL TRAP ABRT BUS FPE SEGV PIPE ALRM TERM STKFLT XCPU XFSZ
+trap "" HUP INT QUIT PIPE TERM
 cleanup
 exit 0

tests/01_engine-filter.test

@@ -192,3 +192,6 @@ test "$(with_timeout httrack -O /dev/null -#test=filter "$pat" "$subj")" == "$su
 # (guards against a single-backtrack-point matcher rewrite)
 match '*[aX]X*[a]Y' 'aXaXaY'
 nomatch '*[aX]X*[a]Y' 'aXbXaY'
+
+# hostile patterns must not stack-overflow or hang: length + work caps
+test "$(with_timeout httrack -O /dev/null -#test=filterbounds)" == "filterbounds: OK" || exit 1

tests/01_engine-identabs.test

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# A 2047-byte hostless "?"-URL used to overflow fil[] by one byte and abort in
+# strncat_safe_; the guard now returns -1 while valid URLs still parse.
+test "$(httrack -O /dev/null -#test=identabs)" == "identabs self-test OK" || exit 1

tests/01_engine-xfread.test

@@ -0,0 +1,29 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# http_xfread1 must refuse an in-memory receive buffer that would exceed a
+# 32-bit index (hostile Content-Length, or an endless stream) rather than
+# allocate it, while still accepting a normal small size. -#test=xfread-limit
+# drives both refusal paths and the accept path.
+
+out="$(httrack -O /dev/null -#test=xfread-limit)"
+for case in bylen bygrow; do
+    echo "$out" | grep -q "${case}: refused=1 adr=null msg=In-memory content too large" || {
+        echo "FAIL ${case}: $out"
+        exit 1
+    }
+done
+
+# Exactly INT32_MAX must be refused too (the reallocs add 1).
+echo "$out" | grep -q 'boundary: msg=In-memory content too large' || {
+    echo "FAIL boundary: $out"
+    exit 1
+}
+
+# The guard must NOT fire for a legitimate small size.
+if echo "$out" | grep -q 'accept: msg=In-memory content too large'; then
+    echo "FAIL accept (guard fired on a legit size): $out"
+    exit 1
+fi

tests/36_local-bigcrawl.test

@@ -53,4 +53,4 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --log-found ', no files updated' \
     --max-mirror-bytes 700000 \
     --min-mirror-bytes 500000 \
-    httrack 'BASEURL/big/index.html' --retries=0 -c8 -%c100 -A100000000
+    httrack 'BASEURL/big/index.html' --retries=0 -c16 -%c100 -A100000000

tests/49_local-cookiewall.test

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Cookie-wall self-redirect (#15): httrack replays a Set-Cookie to mirror the
+# real page, gives up when nothing changes, and stays bounded when it never does.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# Unknown-ext (wall.php) and known-ext (wall.html): the wall sets the cookie once,
+# then serves real content; httrack must replay it and mirror that content.
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
+    --found 'cookiewall/wall.html' \
+    --file-matches 'cookiewall/wall.html' 'REAL-CONTENT-BEHIND-COOKIE-WALL' \
+    httrack 'BASEURL/cookiewall/index.html'
+
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
+    --found 'cookiewall2/wall.html' \
+    --file-matches 'cookiewall2/wall.html' 'REAL-CONTENT-BEHIND-COOKIE-WALL' \
+    httrack 'BASEURL/cookiewall2/index.html'
+
+# No Set-Cookie: the jar never changes, so httrack must give up at once (retry
+# stays gated) and not mirror the wall. A dropped gate would skip this log line.
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
+    --not-found 'cookiewall3/wall.html' \
+    --log-found 'loop to same filename' \
+    httrack 'BASEURL/cookiewall3/index.html'
+
+# Ever-changing cookie never satisfies the wall: httrack must stop at the retry
+# cap (bounded, no early give-up) and not mirror it.
+bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
+    --not-found 'cookiewall4/wall.html' \
+    --log-not-found 'loop to same filename' \
+    httrack 'BASEURL/cookiewall4/index.html'

tests/Makefile.am

@@ -44,6 +44,7 @@ TESTS = \
 	01_engine-header.test \
 	01_engine-idna.test \
 	01_engine-identurl.test \
+	01_engine-identabs.test \
 	01_engine-escape-room.test \
 	01_engine-inplace-escape.test \
 	01_engine-java.test \
@@ -66,6 +67,7 @@ TESTS = \
 	01_engine-urlhack.test \
 	01_engine-unescape-bounds.test \
 	01_engine-useragent.test \
+	01_engine-xfread.test \
 	01_zlib-acceptencoding.test \
 	01_zlib-cache.test \
 	01_zlib-cache-corrupt.test \
@@ -118,6 +120,7 @@ TESTS = \
 	45_local-maxsize-recv.test \
 	46_local-update-304-resume.test \
 	47_local-crange-overflow.test \
-	48_local-crange-memresume.test
+	48_local-crange-memresume.test \
+	49_local-cookiewall.test
 
 CLEANFILES = check-network_sh.cache

tests/local-server.py

@@ -1143,6 +1143,56 @@ class Handler(SimpleHTTPRequestHandler):
     def route_delayed_empty(self):
         self.send_raw(b"", "text/html")  # 200 + Content-Length: 0
 
+    # --- /cookiewall/ (#15): a self-redirect that only sets a cookie is a
+    # consent wall; httrack must replay the cookie and fetch the real page.
+    WALL_MARK = b"REAL-CONTENT-BEHIND-COOKIE-WALL"
+
+    def route_cookiewall_index(self):
+        self.send_html('\t<a href="wall.php">wall</a>')
+
+    def route_cookiewall_wall(self):
+        self._cookiewall_reply("wall.php")
+
+    # Known-extension twin: .html so the type is not delayed-resolved.
+    def route_cookiewall2_index(self):
+        self.send_html('\t<a href="wall.html">wall</a>')
+
+    def route_cookiewall2_wall(self):
+        self._cookiewall_reply("wall.html")
+
+    def _cookiewall_reply(self, location):
+        if self.request_cookies().get("gate") == "1":
+            self.send_raw(
+                b"<html><body>" + self.WALL_MARK + b"</body></html>\n", "text/html"
+            )
+        else:
+            self._wall_redirect(location, "gate=1; Path=/")
+
+    # No-cookie self-redirect: the jar never changes, so httrack must give up at
+    # once rather than re-fetch (proves the cookie-wall retry stays gated on #15).
+    def route_cookiewall3_index(self):
+        self.send_html('\t<a href="wall.php">wall</a>')
+
+    def route_cookiewall3_wall(self):
+        self._wall_redirect("wall.php", None)
+
+    # Ever-changing cookie: every hit sets a fresh value, so the jar keeps
+    # changing; httrack must stop at the loops<7 cap, not spin forever.
+    def route_cookiewall4_index(self):
+        self.send_html('\t<a href="wall.php">wall</a>')
+
+    def route_cookiewall4_wall(self):
+        nonce = int(self.request_cookies().get("gate", "0")) + 1
+        self._wall_redirect("wall.php", f"gate={nonce}; Path=/")
+
+    def _wall_redirect(self, location, set_cookie):
+        self.send_response(302, "Found")
+        self.send_header("Location", location)
+        if set_cookie is not None:
+            self.send_header("Set-Cookie", set_cookie)
+        self.send_header("Content-Length", "0")
+        self.end_headers()
+
     # -E time-limit (#481): pages that trickle far longer than any -E budget,
     # so only an engine-side abort can end the crawl.
     TRICKLE_SECONDS = 60
@@ -1359,6 +1409,14 @@ class Handler(SimpleHTTPRequestHandler):
         "/delayed/chain7.php": route_delayed_chain,
         "/delayed/chain8.php": route_delayed_chain,
         "/delayed/chain9.php": route_delayed_chain,
+        "/cookiewall/index.html": route_cookiewall_index,
+        "/cookiewall/wall.php": route_cookiewall_wall,
+        "/cookiewall2/index.html": route_cookiewall2_index,
+        "/cookiewall2/wall.html": route_cookiewall2_wall,
+        "/cookiewall3/index.html": route_cookiewall3_index,
+        "/cookiewall3/wall.php": route_cookiewall3_wall,
+        "/cookiewall4/index.html": route_cookiewall4_index,
+        "/cookiewall4/wall.php": route_cookiewall4_wall,
         "/redir/index.html": route_redir_index,
         "/redir/go.php": route_redir_go,
         "/redir/target.html": route_redir_target,
@@ -1587,7 +1645,12 @@ def main():
     def factory(*a, **kw):
         return Handler(*a, directory=root, **kw)
 
-    httpd = ThreadingHTTPServer((args.bind, 0), factory)
+    # macOS/BSD drop SYNs when the listen backlog overflows (Linux is lenient);
+    # raise it from Python's default 5 so a busy -c8 crawl can't lose fetches.
+    class BacklogHTTPServer(ThreadingHTTPServer):
+        request_queue_size = 128
+
+    httpd = BacklogHTTPServer((args.bind, 0), factory)
 
     if args.tls:
         import ssl

tests/webhttrack-smoke.sh

@@ -0,0 +1,99 @@
+#!/bin/bash
+# Smoke-test an installed webhttrack: launch it with a stub browser and assert
+# htsserver comes up and serves the web UI. Arg: the install prefix.
+set -euo pipefail
+
+prefix="${1:?usage: webhttrack-smoke.sh <install-prefix>}"
+wht="$prefix/bin/webhttrack"
+test -x "$wht" || {
+    echo "no webhttrack at $wht" >&2
+    exit 1
+}
+
+work="$(mktemp -d)"
+# webhttrack backgrounds htsserver, which outlives it; reap any stray one (scoped
+# to this prefix) so a lingering server can never hold the CI step open.
+trap 'pkill -f "$prefix/bin/htsserver" 2>/dev/null || true; rm -rf "$work"' EXIT
+export HOME="$work/home"
+mkdir -p "$HOME/websites"
+marker="$work/marker"
+
+stubdir="$work/bin"
+mkdir -p "$stubdir"
+
+# On Darwin webhttrack hardcodes "open -W", which launches a real GUI browser and
+# blocks headless. Shadow uname so it takes the generic path and picks the stub
+# browser below; htsserver and webhttrack's path resolution still run for real.
+cat >"$stubdir/uname" <<'EOF'
+#!/bin/bash
+[ "${1:-}" = "-s" ] && {
+    echo Linux
+    exit 0
+}
+exec /usr/bin/uname "$@"
+EOF
+chmod +x "$stubdir/uname"
+
+# Stub browser: webhttrack tries its browser-name list in order and runs the
+# first it finds, so shadow the first entry, "x-www-browser". It fetches the
+# server URL and records PASS only for the working UI: the brand string AND the
+# step-2 form action, which a truncated/degraded template page would lack (the
+# bare title alone is not enough). htsserver only lives until webhttrack exits,
+# so the check has to happen here.
+# -a: the UI is served ISO-8859-1, so grep must not treat it as binary.
+cat >"$stubdir/x-www-browser" <<EOF
+#!/bin/bash
+echo "stub browser invoked with: \$1" >&2
+if body="\$(curl -fsSL --max-time 20 "\$1")" && printf '%s' "\$body" | grep -qai httrack && printf '%s' "\$body" | grep -qaF step2.html; then
+    echo PASS >"$marker"
+else
+    echo "FAIL: unexpected response from \$1" >"$marker"
+fi
+EOF
+chmod +x "$stubdir/x-www-browser"
+export PATH="$stubdir:$prefix/bin:$PATH"
+
+echo "launching webhttrack"
+"$wht" </dev/null >"$work/webhttrack.log" 2>&1 &
+whpid=$!
+
+# Bounded poll for the marker (macOS has no timeout(1)); teardown below kills
+# webhttrack and reaps htsserver, so the run is bounded without a watchdog.
+for i in $(seq 1 45); do
+    test -f "$marker" && {
+        echo "marker written after ${i}s"
+        break
+    }
+    kill -0 "$whpid" 2>/dev/null || {
+        echo "webhttrack exited on its own after ${i}s"
+        break
+    }
+    sleep 1
+done
+
+# Reap webhttrack and the htsserver it spawned. Confirm death with a bounded poll
+# (not a blocking wait, which could hang on macOS); SIGKILL if it ignores TERM.
+echo "tearing down"
+kill "$whpid" 2>/dev/null || true
+if pkill -f "$prefix/bin/htsserver" 2>/dev/null; then
+    echo "reaped a lingering htsserver"
+else
+    echo "no lingering htsserver"
+fi
+for _ in $(seq 1 10); do
+    kill -0 "$whpid" 2>/dev/null || break
+    sleep 1
+done
+kill -9 "$whpid" 2>/dev/null || true
+
+echo "--- webhttrack.log ---"
+cat "$work/webhttrack.log" 2>/dev/null || true
+echo "--- end ---"
+echo "marker=[$(cat "$marker" 2>/dev/null || echo NONE)]"
+
+if test "$(cat "$marker" 2>/dev/null || true)" = PASS; then
+    echo "webhttrack smoke: PASS"
+else
+    echo "webhttrack smoke: FAIL" >&2
+    exit 1
+fi