Isolate bigcrawl in a second pass on macOS to parallelize the rest

macOS was the CI bottleneck (~460s serial, the ~8min total): its loopback drops
fetches when bigcrawl's sustained -c8 crawl competes with other crawls, so a
parallel make check flaked bigcrawl's exact file count. Rather than keep the
whole macOS suite serial, run everything except bigcrawl in parallel (2x cores,
capped 16), then bigcrawl alone in a second pass, which is the serial-safe
condition green on master. Linux still runs the full suite in parallel.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -136,9 +136,15 @@ jobs:
         run: make -j"$(sysctl -n hw.ncpu)"
 
       - name: Test
+        # bigcrawl's sustained -c8 crawl drops fetches on macOS's loopback when
+        # it competes with other crawls, flaking its exact file count (the #527
+        # macOS drop). Run everything else in parallel, then bigcrawl alone (its
+        # serial-safe condition). Linux tolerates the full parallel run.
         run: |
           jobs=$(( $(sysctl -n hw.ncpu) * 2 )); [ "$jobs" -le 16 ] || jobs=16
-          make check -j"$jobs"
+          rest=$(cd tests && ls *.test | grep -v '^36_local-bigcrawl\.test$' | tr '\n' ' ')
+          make check -j"$jobs" TESTS="$rest"
+          make check TESTS=36_local-bigcrawl.test
 
       - name: Print the test log on failure
         if: failure()

AGENTS.md

@@ -14,10 +14,10 @@ the operational checklist: toolchain, invariants, and how to ship a change.
   automatically; only a test slower than the current longest raises the floor.
   On a few-core Linux box, `-j` at 2x the core count is faster still: the tests
   spend much of their wall time asleep (server trickles, httrack self-pacing),
-  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)` on every
-  platform, macOS included: the test server raises its listen backlog
-  (`request_queue_size`) so macOS/BSD don't drop connections under a parallel
-  `-c16` bigcrawl the way Python's default backlog of 5 did.
+  so an idle core covers a sleeping one. CI uses `min(2*cores, 16)`. macOS runs
+  36_local-bigcrawl alone in a second pass: its sustained `-c8` crawl overloads
+  the macOS loopback when it competes with other crawls and flakes its exact
+  file count (Linux tolerates the full parallel run).
   Or run `sh build.sh` to do bootstrap + configure + make in one shot.
 
 ## Hard invariants

src/htsback.c

@@ -3512,11 +3512,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     continue;
                   }
 
-                  // The server really sent 304 here; the *-hacks below force
-                  // NOT_MODIFIED only after confirming the file is complete.
-                  const hts_boolean server_sent_304 =
-                      (back[i].r.statuscode == HTTP_NOT_MODIFIED);
-
                   /*
                      Solve "false" 416 problems
                    */
@@ -3611,7 +3606,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       if (back[i].r.statuscode == HTTP_OK && !back[i].testmode) {       // 'OK'
                         if (!is_hypertext_mime(opt, back[i].r.contenttype, back[i].url_fil)) {  // not HTML
                           if (strnotempty(back[i].url_sav)) {   // target found
-                            off_t size = fsize_utf8(back[i].url_sav);
+                            int size = fsize_utf8(back[i].url_sav);     // target size
 
                             if (size >= 0) {
                               if (back[i].r.totalsize == size) {        // same size!
@@ -3706,21 +3701,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                     }
                   }
 
-                  // Out-of-protocol 304 to a Range resume: the file is still
-                  // partial, so drop it and refetch instead of trusting it.
-                  if (server_sent_304 && back[i].range_req_size > 0) {
-                    url_savename_refname_remove(opt, back[i].url_adr,
-                                                back[i].url_fil);
-                    UNLINK(back[i].url_sav);
-                    deletehttp(&back[i].r);
-                    back[i].r.soc = INVALID_SOCKET;
-                    back[i].r.statuscode = STATUSCODE_NON_FATAL;
-                    strcpybuff(back[i].r.msg,
-                               "Bogus 304 on resume, restarting");
-                    back[i].status = STATUS_READY;
-                    back_set_finished(sback, i);
-                  }
-
                   /* sinon, continuer */
                   /* if (back[i].r.soc!=INVALID_SOCKET) {   // ok récupérer body? */
                   // head: terminé
@@ -3839,7 +3819,7 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                       const hts_boolean range_ok =
                           back[i].r.crange > 0 && resume >= 0 &&
                           resume <= (LLint) sz &&
-                          back[i].r.crange_end == back[i].r.crange - 1 &&
+                          back[i].r.crange_end + 1 == back[i].r.crange &&
                           (back[i].r.totalsize < 0 ||
                            back[i].r.totalsize ==
                                back[i].r.crange_end - resume + 1);
@@ -3908,20 +3888,11 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                           if (fp) {
                             LLint alloc_mem = resume + 1;
 
-                            // Bound the in-memory buffer to a 32-bit size (real
-                            // in-RAM resources are far smaller); a hostile
-                            // Content-Length that would overflow the add or the
-                            // (size_t) cast is dropped and refetched instead.
-                            if (back[i].r.totalsize > INT32_MAX - alloc_mem) {
-                              url_savename_refname_remove(opt, back[i].url_adr,
-                                                          back[i].url_fil);
-                              UNLINK(back[i].url_sav);
-                              alloc_mem = -1;
-                            } else if (back[i].r.totalsize >= 0)
+                            if (back[i].r.totalsize >= 0)
                               alloc_mem += back[i].r.totalsize; // AJOUTER RESTANT!
-                            if (alloc_mem >= 0 && deleteaddr(&back[i].r) &&
-                                (back[i].r.adr =
-                                     (char *) malloct((size_t) alloc_mem))) {
+                            if (deleteaddr(&back[i].r)
+                                && (back[i].r.adr =
+                                    (char *) malloct((size_t) alloc_mem))) {
                               back[i].r.size = resume;
                               if (back[i].r.totalsize >= 0)
                                 back[i].r.totalsize += resume; // -> full size
@@ -4207,13 +4178,10 @@ static int back_maxtime_grace(const int maxtime) {
    No floor (unlike the time grace): a size overrun should abort promptly. */
 static LLint back_maxsize_grace(const LLint maxsite) { return maxsite / 10; }
 
-/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481).
-   -M measures received volume (HTS_TOTAL_RECV), not saved 200-only stat_bytes
-   which undercounts redirect/error-heavy crawls (#520). */
+/* Which cap has overrun its grace and must hard-stop the mirror (#77, #481). */
 static hts_mirror_limit back_mirror_limit(httrackp *opt) {
-  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
-      HTS_STAT.HTS_TOTAL_RECV - opt->maxsite >=
-          back_maxsize_grace(opt->maxsite))
+  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
+      HTS_STAT.stat_bytes - opt->maxsite >= back_maxsize_grace(opt->maxsite))
     return HTS_MIRROR_LIMIT_SIZE;
   if (opt->maxtime > 0) {
     const TStamp elapsed = time_local() - HTS_STAT.stat_timestart;
@@ -4227,7 +4195,7 @@ static hts_mirror_limit back_mirror_limit(httrackp *opt) {
 
 int back_checkmirror(httrackp *opt) {
   /* request a smooth stop the first time each cap is reached */
-  if (opt->maxsite > 0 && HTS_STAT.HTS_TOTAL_RECV >= opt->maxsite &&
+  if (opt->maxsite > 0 && HTS_STAT.stat_bytes >= opt->maxsite &&
       !opt->state.stop) {
     hts_log_print(opt, LOG_ERROR,
                   "More than " LLintP

src/htsfilters.c

@@ -121,18 +121,12 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
   return use1 ? jok1 : jok2;
 }
 
-/* Real filters/URLs fit HTS_URLMAXSIZE*2; past it a hostile pattern recurses
-   O(len) deep or runs O(n^2*stars). Cap length (depth) and steps (work). */
-#define STRJOKER_MAXLEN (HTS_URLMAXSIZE * 2)
-#define STRJOKER_MAXSTEPS 2000000u
-
 /* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
    pair keeps star-heavy patterns polynomial instead of exponential (#501). */
 typedef struct strjoker_memo {
   const char *chaine0, *joker0; /* offsets are relative to these bases */
   size_t stride;                /* strlen(joker0) + 1 */
   unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
-  size_t *nsteps; /* shared work counter; NULL: unbounded (oracle) */
 } strjoker_memo;
 
 static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
@@ -147,8 +141,6 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
   size_t bit = 0;
   const char *adr;
 
-  if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
-    return NULL; /* work budget spent: fail the match safely */
   if (memo->failed) {
     bit = (size_t) (chaine - memo->chaine0) * memo->stride +
           (size_t) (joker - memo->joker0);
@@ -161,24 +153,22 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
   return adr;
 }
 
-/* Match chaine against joker with a shared work budget *nsteps (a strjokerfind
-   sweep passes one counter, bounding the whole scan). */
-static const char *strjoker_bounded(const char *chaine, const char *joker,
-                                    LLint *size, int *size_flag,
-                                    size_t *nsteps) {
-  strjoker_memo memo = {chaine, joker, 0, NULL, nsteps};
+// wildcard comparator: match chaine against joker (pattern), case-insensitive
+// returns the address of the first matched letter past any leading joker
+// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
+HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
+                                LLint *size, int *size_flag) {
+  strjoker_memo memo = {chaine, joker, 0, NULL};
   unsigned char stackbits[2048];
   hts_boolean onheap = HTS_FALSE;
   const char *adr;
 
   if (chaine != NULL && joker != NULL) {
-    const size_t l1 = strlen(chaine), l2 = strlen(joker);
+    const size_t n1 = strlen(chaine) + 1;
 
-    if (l1 > STRJOKER_MAXLEN || l2 > STRJOKER_MAXLEN)
-      return NULL; /* beyond any real filter/URL: bound depth+work */
-    memo.stride = l2 + 1;
-    if (l1 + 1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
-      const size_t bytes = ((l1 + 1) * memo.stride + 7) / 8;
+    memo.stride = strlen(joker) + 1;
+    if (n1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
+      const size_t bytes = (n1 * memo.stride + 7) / 8;
 
       if (bytes <= sizeof(stackbits)) {
         memset(stackbits, 0, bytes);
@@ -195,16 +185,6 @@ static const char *strjoker_bounded(const char *chaine, const char *joker,
   return adr;
 }
 
-// wildcard comparator: match chaine against joker (pattern), case-insensitive
-// returns the address of the first matched letter past any leading joker
-// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
-HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
-                                LLint *size, int *size_flag) {
-  size_t nsteps = 0;
-
-  return strjoker_bounded(chaine, joker, size, size_flag, &nsteps);
-}
-
 /* Test-only oracle: the same matcher without the failure memo. */
 const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
                             int *size_flag) {
@@ -451,18 +431,12 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 // d'un strcpy sur une variable ayant un nom en lettres et copiant une chaine de chiffres
 // ATTENTION!! Eviter les jokers en début, où gare au temps machine!
 const char *strjokerfind(const char *chaine, const char *joker) {
-  size_t nsteps = 0; /* one budget for the whole scan */
   const char *adr;
 
-  if (chaine != NULL && strlen(chaine) > STRJOKER_MAXLEN)
-    return NULL;
   while(*chaine) {
-    if ((adr = strjoker_bounded(chaine, joker, NULL, NULL,
-                                &nsteps))) { // ok trouvé
+    if ((adr = strjoker(chaine, joker, NULL, NULL))) {  // ok trouvé
       return adr;
     }
-    if (nsteps > STRJOKER_MAXSTEPS) // scan budget spent: no match
-      return NULL;
     chaine++;
   }
   return NULL;

src/htslib.c

@@ -1059,10 +1059,10 @@ static int append_cookie_header(buff_struct *bstr, t_cookie *cookie,
   return cook;
 }
 
-/* Build the request Cookie line for domain/path into dst (always
-   NUL-terminated). Returns the number of cookies emitted. */
-int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
-                       char *dst, size_t dst_size) {
+/* Self-test entry for append_cookie_header(): build the request Cookie line
+   into dst (always NUL-terminated). Returns the number of cookies emitted. */
+int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
+                                const char *path, char *dst, size_t dst_size) {
   buff_struct bstr = {dst, dst_size, 0};
 
   assertf(dst != NULL && dst_size > 0);
@@ -1586,7 +1586,7 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
         a = strchr(rcvd + p, '/');
         if (a != NULL) {
           a++;
-          if (sscanf(a, LLintP, &retour->crange) == 1 && retour->crange >= 0) {
+          if (sscanf(a, LLintP, &retour->crange) == 1) {
             retour->crange_start = 0;
             retour->crange_end = retour->crange - 1;
           } else {
@@ -1594,12 +1594,6 @@ void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * ret
           }
         }
       }
-      // A valid Content-Range has no negative field; reject hostile values so
-      // the crange +/- 1 arithmetic downstream cannot sign-overflow (UB).
-      if (retour->crange_start < 0 || retour->crange_end < 0 ||
-          retour->crange < 0) {
-        retour->crange_start = retour->crange_end = retour->crange = 0;
-      }
     }
   } else if ((p = strfield(rcvd, "Connection:")) != 0) {
     char *a = rcvd + p;
@@ -2004,15 +1998,6 @@ LLint http_xfread1(htsblk * r, int bufl) {
 
   if (bufl > 0) {
     if (!r->is_write) {         // stocker en mémoire
-      // In-memory content must fit a 32-bit index (allocs below add 1, reads
-      // use int offsets): reject a hostile Content-Length or endless stream.
-      const LLint inmem_want =
-          (r->totalsize >= 0) ? r->totalsize : (r->size + bufl);
-      if (inmem_want >= INT32_MAX) {
-        r->statuscode = STATUSCODE_INVALID;
-        strcpybuff(r->msg, "In-memory content too large");
-        return READ_ERROR;
-      }
       if (r->totalsize >= 0) {  // totalsize déterminé ET ALLOUE
         if (r->adr == NULL) {
           r->adr = (char *) malloct((size_t) r->totalsize + 1);

src/htslib.h

@@ -184,10 +184,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode, const char *xsend
                   const char *referer_adr, const char *referer_fil,
                   htsblk * retour);
 /* Build the request "Cookie:" header line for stored cookies matching
-   domain/path into dst (NUL-terminated), wrapping the logic http_sendhead()
-   uses. Returns cookies emitted. */
-int http_cookie_header(t_cookie *cookie, const char *domain, const char *path,
-                       char *dst, size_t dst_size);
+   domain/path into dst (NUL-terminated). Exposed for the -#Q self-test;
+   wraps the same logic http_sendhead() uses. Returns cookies emitted. */
+int http_cookie_header_selftest(t_cookie *cookie, const char *domain,
+                                const char *path, char *dst, size_t dst_size);
 
 T_SOC newhttp(httrackp * opt, const char *iadr, htsblk * retour, int port,
               int waitconnect);

src/htsparse.c

@@ -4311,8 +4311,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
       *forbidden_url == 0 && IS_DELAYED_EXT(afs->save) && !opt->state.stop) {
     int loops;
     int continue_loop;
-    char BIGSTK
-        cookie_before[16384]; /* #15: this URL's Cookie header, pre-request */
 
     hts_log_print(opt, LOG_DEBUG, "Waiting for type to be known: %s%s", afs->af.adr,
                   afs->af.fil);
@@ -4321,12 +4319,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
     for(loops = 0, continue_loop = 1;
         IS_DELAYED_EXT(afs->save) && continue_loop && loops < 7; loops++) {
       continue_loop = 0;
-      /* #15: snapshot the Cookie header THIS url would send, so only its own
-         self-redirect Set-Cookie trips the retry, not a concurrent slot's. */
-      cookie_before[0] = '\0';
-      if (opt->accept_cookie && opt->cookie != NULL)
-        http_cookie_header(opt->cookie, jump_identification_const(afs->af.adr),
-                           afs->af.fil, cookie_before, sizeof(cookie_before));
 
       /* Wait for an available slot */
       if (!hts_wait_available_socket(sback, opt, cache, ptr))
@@ -4524,17 +4516,6 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
           /* Moved! */
           else if (HTTP_IS_REDIRECT(back[b].r.statuscode)) {
             char BIGSTK mov_url[HTS_URLMAXSIZE * 2];
-            char BIGSTK cookie_after[16384];
-            hts_boolean cookies_changed;
-
-            /* #15: cookie-wall signal: this URL's own Cookie header changed
-               across its self-redirect (a Set-Cookie it just set). */
-            cookie_after[0] = '\0';
-            if (opt->accept_cookie && opt->cookie != NULL)
-              http_cookie_header(
-                  opt->cookie, jump_identification_const(afs->af.adr),
-                  afs->af.fil, cookie_after, sizeof(cookie_after));
-            cookies_changed = strcmp(cookie_before, cookie_after) != 0;
 
             mov_url[0] = '\0';
             strcpybuff(mov_url, back[b].r.location);    // copier URL
@@ -4604,24 +4585,11 @@ int hts_wait_delayed(htsmoduleStruct * str, lien_adrfilsave *afs,
                   url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, 
                                opt, sback, cache, hash, ptr, numero_passe,
                                &delayed_back);
-                } else if (cookies_changed) {
-                  // #15: cookie-wall self-redirect; evict the cached
-                  // fast-header so the re-issue refetches with the cookie.
-                  if (cache->cached_tests != NULL)
-                    coucal_remove(cache->cached_tests,
-                                  concat(OPT_GET_BUFF(opt),
-                                         OPT_GET_BUFF_SIZE(opt), afs->af.adr,
-                                         afs->af.fil));
-                  afs->save[0] = '\0';
-                  url_savename(afs, former, heap(ptr)->adr, heap(ptr)->fil, opt,
-                               sback, cache, hash, ptr, numero_passe,
-                               &delayed_back);
-                  continue_loop = 1;
                 } else {
                   hts_log_print(opt, LOG_WARNING,
                                 "Unable to test %s%s (loop to same filename)",
                                 afs->af.adr, afs->af.fil);
-                } // loop to same location
+                }               // loop to same location
               }                 // ident_url_relatif()
             }                   // location
           }                     // redirect

src/htsselftest.c

@@ -740,40 +740,6 @@ static int st_filterdual(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* Length/work caps stop a hostile pattern stack-overflowing or hanging the
-   process (OSS-Fuzz 5060751291908096 / 5745936014573568). */
-static int st_filterbounds(httrackp *opt, int argc, char **argv) {
-  const size_t big = 100000; /* well past the length cap */
-  const size_t stars = 900;  /* star-heavy pattern, under the cap */
-  char *subj = malloct(big + 1);
-  char *pat = malloct(2 * stars + 2);
-  size_t i;
-
-  (void) opt;
-  (void) argc;
-  (void) argv;
-  memset(subj, 'a', big);
-  subj[big] = '\0';
-  /* '*' matches anything, but an over-length subject is refused by the cap */
-  assertf(strjoker(subj, "*", NULL, NULL) == NULL);
-  assertf(strjokerfind(subj, "*") == NULL);
-  /* within-cap star-heavy dead-end: the step budget keeps it bounded (a hang
-     would time the test out) */
-  for (i = 0; i < stars; i++) {
-    pat[2 * i] = '*';
-    pat[2 * i + 1] = 'a';
-  }
-  pat[2 * stars] = 'b'; /* never matches an all-'a' subject */
-  pat[2 * stars + 1] = '\0';
-  subj[stars] = '\0';
-  assertf(strjoker(subj, pat, NULL, NULL) == NULL);
-  assertf(strjokerfind(subj, pat) == NULL);
-  freet(pat);
-  freet(subj);
-  printf("filterbounds: OK\n");
-  return 0;
-}
-
 static int st_simplify(httrackp *opt, int argc, char **argv) {
   (void) opt;
   if (argc < 1) {
@@ -1344,82 +1310,6 @@ static int st_headerlong(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* http_xfread1 must refuse an in-memory buffer whose size would exceed a 32-bit
-   index (hostile Content-Length or endless stream) rather than allocate it.
-   The guard returns before any socket read, so no real connection is needed. */
-static int st_xfread_limit(httrackp *opt, int argc, char **argv) {
-  htsblk r;
-
-  (void) opt;
-  (void) argc;
-  (void) argv;
-
-  // Content-Length just over 2 GiB.
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = (LLint) INT32_MAX + 1;
-  printf("bylen: refused=%d adr=%s msg=%s\n",
-         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
-         r.msg);
-  if (r.adr != NULL)
-    freet(r.adr);
-
-  // Unknown length, buffer already at the limit: the next read would exceed it.
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = -1;
-  r.size = (LLint) INT32_MAX;
-  printf("bygrow: refused=%d adr=%s msg=%s\n",
-         http_xfread1(&r, 8192) == READ_ERROR, r.adr != NULL ? "alloc" : "null",
-         r.msg);
-  if (r.adr != NULL)
-    freet(r.adr);
-
-  // Exactly at the 2 GiB index (size + bufl == INT32_MAX): must also be
-  // refused, since the reallocs below add 1 (a `> INT32_MAX` check would let
-  // this through and overflow the int realloc size).
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = -1;
-  r.size = (LLint) INT32_MAX - 8192;
-  http_xfread1(&r, 8192);
-  printf("boundary: msg=%s\n", r.msg);
-
-  // A legitimate small size must NOT be refused by the guard (the read then
-  // fails on the invalid socket, but the size-too-large msg must not be set).
-  memset(&r, 0, sizeof(r));
-  r.soc = INVALID_SOCKET;
-  r.totalsize = 1000;
-  http_xfread1(&r, 8192);
-  printf("accept: msg=%s\n", r.msg);
-  if (r.adr != NULL)
-    freet(r.adr);
-  return 0;
-}
-
-/* Parse a Content-Range header and print the sanitized triple. A hostile value
-   (negative or INT64 extreme) must clamp to 0 without signed-overflow UB. */
-static int st_crange(httrackp *opt, int argc, char **argv) {
-  int i;
-
-  (void) opt;
-  if (argc < 1) {
-    fprintf(stderr, "crange: needs at least one raw Content-Range line\n");
-    return 1;
-  }
-  for (i = 0; i < argc; i++) {
-    htsblk r;
-    char BIGSTK line[HTS_URLMAXSIZE * 2];
-
-    memset(&r, 0, sizeof(r));
-    strcpybuff(line, argv[i]);
-    treathead(NULL, "www.example.com", "/", &r, line);
-    printf("crange_start=" LLintP " crange_end=" LLintP " crange=" LLintP "\n",
-           (LLint) r.crange_start, (LLint) r.crange_end, (LLint) r.crange);
-  }
-  return 0;
-}
-
 /* Decode a body argument ("hex:FFD8.." or literal text) into buf. */
 static size_t st_decode_body(const char *arg, char *buf, size_t size) {
   size_t n = 0;
@@ -1789,7 +1679,7 @@ static int st_cookies(httrackp *opt, int argc, char **argv) {
     return 1;
   }
 
-  http_cookie_header(&cookie, dom, "/", hdr, sizeof(hdr));
+  http_cookie_header_selftest(&cookie, dom, "/", hdr, sizeof(hdr));
   if (strcmp(hdr, expected) != 0)
     err = 1;
   if (strstr(hdr, "$Version") != NULL || strstr(hdr, "$Path") != NULL)
@@ -2571,8 +2461,6 @@ static const struct selftest_entry {
      "memoized vs unmemoized matcher differential", st_filtermemo},
     {"filterdual", "<string1> <string2> <filter>...",
      "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
-    {"filterbounds", "", "matcher length/work caps reject hostile patterns",
-     st_filterbounds},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
@@ -2608,10 +2496,6 @@ static const struct selftest_entry {
     {"headerlong", "[header-name:]",
      "over-long header value must not overflow the parse scratch",
      st_headerlong},
-    {"crange", "<raw-content-range-line> ...",
-     "Content-Range parse integer safety", st_crange},
-    {"xfread-limit", "", "in-memory receive buffer size bound",
-     st_xfread_limit},
     {"savename", "<fil> <content-type> [key=value ...]",
      "local save-name for a URL", st_savename},
     {"sniff", "<content-type> <hex:..|text>", "MIME magic consistency",

tests/01_engine-crange.test

@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# Content-Range parse (treathead via -#test=crange). Hostile values must clamp
-# to 0, and the crange +/- 1 arithmetic must not sign-overflow (UBSan aborts on
-# the pre-fix binary at the INT64_MIN cases).
-
-cr() {
-    local want="$1"
-    shift
-    out="$(httrack -O /dev/null -#test=crange "$@")"
-    test "$out" == "$want" || {
-        echo "FAIL: $* -> '$out' (want '$want')"
-        exit 1
-    }
-}
-
-# Normal range and the '*/N' fallback both parse through unchanged.
-cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes 0-70870/70871'
-cr 'crange_start=0 crange_end=70870 crange=70871' 'Content-Range: bytes */70871'
-
-# INT64_MIN total in the fallback would make crange-1 overflow: clamp to 0.
-cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes */-9223372036854775808'
-
-# Any negative field in the 3-field form clamps the whole triple to 0 (each
-# field independently: a leading negative start/end alone still zeroes all).
-cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -1--1/-1'
-cr 'crange_start=0 crange_end=0 crange=0' 'Content-Range: bytes -5-10/20'
-
-# INT64_MAX is non-negative: it passes through, the parse must not mangle it.
-cr 'crange_start=0 crange_end=9223372036854775807 crange=9223372036854775807' \
-    'Content-Range: bytes 0-9223372036854775807/9223372036854775807'

tests/01_engine-filter.test

@@ -192,6 +192,3 @@ test "$(with_timeout httrack -O /dev/null -#test=filter "$pat" "$subj")" == "$su
 # (guards against a single-backtrack-point matcher rewrite)
 match '*[aX]X*[a]Y' 'aXaXaY'
 nomatch '*[aX]X*[a]Y' 'aXbXaY'
-
-# hostile patterns must not stack-overflow or hang: length + work caps
-test "$(with_timeout httrack -O /dev/null -#test=filterbounds)" == "filterbounds: OK" || exit 1

tests/01_engine-xfread.test

@@ -1,29 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# http_xfread1 must refuse an in-memory receive buffer that would exceed a
-# 32-bit index (hostile Content-Length, or an endless stream) rather than
-# allocate it, while still accepting a normal small size. -#test=xfread-limit
-# drives both refusal paths and the accept path.
-
-out="$(httrack -O /dev/null -#test=xfread-limit)"
-for case in bylen bygrow; do
-    echo "$out" | grep -q "${case}: refused=1 adr=null msg=In-memory content too large" || {
-        echo "FAIL ${case}: $out"
-        exit 1
-    }
-done
-
-# Exactly INT32_MAX must be refused too (the reallocs add 1).
-echo "$out" | grep -q 'boundary: msg=In-memory content too large' || {
-    echo "FAIL boundary: $out"
-    exit 1
-}
-
-# The guard must NOT fire for a legitimate small size.
-if echo "$out" | grep -q 'accept: msg=In-memory content too large'; then
-    echo "FAIL accept (guard fired on a legit size): $out"
-    exit 1
-fi

tests/19_local-connect-fallback.test

@@ -66,12 +66,9 @@ test -n "$port" || {
 }
 
 out="$tmpdir/crawl"
-# macOS has only 127.0.0.1, so the dead 127.0.0.x connects block to the timeout
-# instead of refusing instantly (as on Linux); a small --timeout + --retries=0
-# bound the stall without changing what the fallback exercises.
 HTTRACK_DEBUG_RESOLVE="deadhost:127.0.0.2,127.0.0.1" \
     httrack "http://deadhost:$port/simple/basic.html" -O "$out" \
-    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log" 2>&1
+    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log" 2>&1
 
 log="$out/hts-log.txt"
 
@@ -95,12 +92,12 @@ test -f "$out/deadhost_$port/simple/basic.html" || {
     exit 1
 }
 
-# every address dead: the slot exhausts the list, then errors out (the harness
-# timeout would catch a hang/loop)
+# every address refused: the slot exhausts the list, then errors out (the
+# harness timeout would catch a hang/loop; refused connects are instant)
 out2="$tmpdir/crawl2"
 HTTRACK_DEBUG_RESOLVE="alldead:127.0.0.2,127.0.0.3" \
     httrack "http://alldead:$port/simple/basic.html" -O "$out2" \
-    -c1 --robots=0 --timeout=5 --retries=0 --quiet -Z >"$tmpdir/log2" 2>&1
+    -c1 --robots=0 --timeout=30 --quiet -Z >"$tmpdir/log2" 2>&1
 log2="$out2/hts-log.txt"
 
 grep -q "trying next address" "$log2" || {

tests/31_local-javaclass.test

@@ -17,10 +17,7 @@ printf 'GIF89a' >"$tmproot/javaclass/hello.gif"
 printf '\xCA\xFE\xBA\xBE\x00\x00\x00\x32\x00\x02\x01\x00\x09hello.gif\x00\x00\x00\x00' \
     >"$tmproot/javaclass/Foo.class"
 
-# --log-found first: a plugin-load failure (wrong soname, shadowing system
-# libhtsjava) trips here on the banner suffix, not as an opaque "not found".
 bash "$top_srcdir/tests/local-crawl.sh" --root "$tmproot" --errors 0 \
-    --log-found '\+libhtsjava' \
     --found 'javaclass/Foo.class' \
     --found 'javaclass/hello.gif' \
     httrack 'BASEURL/javaclass/index.html'

tests/36_local-bigcrawl.test

@@ -53,4 +53,4 @@ bash "$top_srcdir/tests/local-crawl.sh" --rerun \
     --log-found ', no files updated' \
     --max-mirror-bytes 700000 \
     --min-mirror-bytes 500000 \
-    httrack 'BASEURL/big/index.html' --retries=0 -c16 -%c100 -A100000000
+    httrack 'BASEURL/big/index.html' --retries=0 -c8 -%c100 -A100000000

tests/45_local-maxsize-recv.test

@@ -1,16 +0,0 @@
-#!/bin/bash
-#
-# -M caps received volume, not saved 200-only bytes (#520): links to large 404
-# bodies pump received >> saved; the cap must trip though little lands on disk.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-# The mirror bound is the load-bearing half: it fires while <100 KB is saved,
-# so the cap can only have tripped on received volume, not saved bytes (which
-# would exceed the cap and fetch all 16 links).
-bash "$top_srcdir/tests/local-crawl.sh" \
-    --log-found 'More than 500000 bytes have been transferred.. giving up' \
-    --max-mirror-bytes 100000 \
-    httrack 'BASEURL/maxrecv/index.html' -M500000 -c4

tests/46_local-update-304-resume.test

@@ -1,118 +0,0 @@
-#!/bin/bash
-# C7: a server 304 answering a resume's Range request is out of protocol; httrack
-# must drop the partial and refetch, not finalize it as complete at the partial
-# byte count. Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets a bogus
-# 304, and must recover the whole file.
-set -u
-
-: "${top_srcdir:=..}"
-testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
-
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
-
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_c7.XXXXXX") || exit 1
-serverpid=
-crawlpid=
-cleanup() {
-    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
-    rm -rf "$tmpdir"
-}
-trap cleanup EXIT HUP INT QUIT PIPE TERM
-
-serverlog="${tmpdir}/server.log"
-counter="${tmpdir}/reqcount"
-mark="${tmpdir}/got304"
-RESUME304_COUNTER="$counter" RESUME304_MARK="$mark" python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
-serverpid=$!
-port=
-for _ in $(seq 1 50); do
-    line=$(head -n1 "$serverlog" 2>/dev/null)
-    if test "${line%% *}" == "PORT"; then
-        port="${line#PORT }"
-        break
-    fi
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "server exited early: $(cat "$serverlog")"
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover server port"
-    exit 1
-}
-base="http://127.0.0.1:${port}"
-
-which httrack >/dev/null || {
-    echo "could not find httrack"
-    exit 1
-}
-out="${tmpdir}/crawl"
-mkdir "$out"
-common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
-refdir="${out}/hts-cache/ref"
-
-# --- pass 1: crawl, interrupt once the blob download is underway -------------
-printf '[pass 1: interrupt mid-download] ..\t'
-httrack "${common[@]}" "${base}/resume304/index.html" >"${tmpdir}/log1" 2>&1 &
-crawlpid=$!
-for _ in $(seq 1 300); do
-    test -s "$counter" && break
-    kill -0 "$crawlpid" 2>/dev/null || break
-    sleep 0.1
-done
-sleep 0.5
-kill -TERM "$crawlpid" 2>/dev/null
-wait "$crawlpid" 2>/dev/null
-crawlpid=
-test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
-    echo "FAIL: no temp-ref survived pass 1; cannot drive C7"
-    exit 1
-}
-echo "OK (temp-ref present)"
-before=$(wc -c <"$counter" 2>/dev/null || echo 0)
-
-# --- pass 2: --continue -> resume Range -> bogus 304 -> drop + refetch --------
-printf '[pass 2: resume gets 304, must refetch] ..\t'
-httrack "${common[@]}" --continue "${base}/resume304/index.html" >"${tmpdir}/log2" 2>&1
-echo "OK (terminated)"
-
-blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
-full=8200 # len(RESUME304_BODY) = len("C7DATA--") + 8192
-
-printf '[file recovered whole] ..\t'
-test -s "$blob" || {
-    echo "FAIL: blob.bin missing after the 304 resume (partial dropped, not refetched)"
-    exit 1
-}
-got=$(wc -c <"$blob")
-test "$got" -eq "$full" || {
-    echo "FAIL: blob.bin is ${got} bytes, expected ${full} (partial finalized as complete)"
-    exit 1
-}
-echo "OK (${got} bytes)"
-
-# Pass 2 must issue at least two requests: the resume (Range -> 304) and the
-# range-less refetch (-> 200). Exactly one means the partial was trusted.
-after=$(wc -c <"$counter" 2>/dev/null || echo 0)
-hits=$((after - before))
-printf '[resume then refetch] ..\t'
-test "$hits" -ge 2 || {
-    echo "FAIL: only ${hits} pass-2 request(s); the 304 was trusted, no refetch"
-    exit 1
-}
-echo "OK (${hits} requests)"
-
-# The recovery must go through the resume path: a Range request that drew the
-# bogus 304 (a fresh re-crawl sends no Range, so this marker stays empty).
-printf '[resume Range drew a 304] ..\t'
-test -s "$mark" || {
-    echo "FAIL: no Range request got a 304; the resume path was not exercised"
-    exit 1
-}
-echo "OK"

tests/47_local-crange-overflow.test

@@ -1,96 +0,0 @@
-#!/bin/bash
-# C2: a resume whose 206 carries a Content-Range end of INT64_MAX must not
-# sign-overflow the crange+1 range check (UBSan aborts on the pre-fix binary).
-# Pass 1 leaves a partial + temp-ref; pass 2 resumes, gets the hostile 206, and
-# must reject the range and refetch the whole file rather than overflow. Teeth
-# are UBSan-only: a normal build wraps to the same reject+restart.
-set -u
-
-: "${top_srcdir:=..}"
-testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
-
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
-
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crange.XXXXXX") || exit 1
-serverpid=
-crawlpid=
-cleanup() {
-    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
-    rm -rf "$tmpdir"
-}
-trap cleanup EXIT HUP INT QUIT PIPE TERM
-
-serverlog="${tmpdir}/server.log"
-python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
-serverpid=$!
-port=
-for _ in $(seq 1 50); do
-    line=$(head -n1 "$serverlog" 2>/dev/null)
-    if test "${line%% *}" == "PORT"; then
-        port="${line#PORT }"
-        break
-    fi
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "server exited early: $(cat "$serverlog")"
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover server port"
-    exit 1
-}
-base="http://127.0.0.1:${port}"
-
-which httrack >/dev/null || {
-    echo "could not find httrack"
-    exit 1
-}
-out="${tmpdir}/crawl"
-mkdir "$out"
-common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
-refdir="${out}/hts-cache/ref"
-
-# --- pass 1: crawl, interrupt once the blob download is underway -------------
-printf '[pass 1: interrupt mid-download] ..\t'
-httrack "${common[@]}" "${base}/crange206/index.html" >"${tmpdir}/log1" 2>&1 &
-crawlpid=$!
-for _ in $(seq 1 300); do
-    test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
-    kill -0 "$crawlpid" 2>/dev/null || break
-    sleep 0.1
-done
-sleep 0.3
-kill -TERM "$crawlpid" 2>/dev/null
-wait "$crawlpid" 2>/dev/null
-crawlpid=
-test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
-    echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
-    exit 1
-}
-echo "OK (temp-ref present)"
-
-# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
-printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
-httrack "${common[@]}" --continue "${base}/crange206/index.html" >"${tmpdir}/log2" 2>&1
-echo "OK (terminated)"
-
-blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
-full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
-
-printf '[file recovered whole] ..\t'
-test -s "$blob" || {
-    echo "FAIL: blob.bin missing after the hostile 206"
-    exit 1
-}
-got=$(wc -c <"$blob")
-test "$got" -eq "$full" || {
-    echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
-    exit 1
-}
-echo "OK (${got} bytes)"

tests/48_local-crange-memresume.test

@@ -1,96 +0,0 @@
-#!/bin/bash
-# C2 (memory branch): a resume whose 206 lies text/html with a matching
-# INT64_MAX Content-Length must not overflow the resume buffer-size add
-# (UBSan aborts on the pre-fix binary at alloc_mem += totalsize). Pass 1 leaves
-# a partial + temp-ref; pass 2 resumes and must reject and refetch the whole
-# file. Teeth are UBSan-only: a normal build wraps to the same reject+restart.
-set -u
-
-: "${top_srcdir:=..}"
-testdir=$(cd "$(dirname "$0")" && pwd)
-server="${testdir}/local-server.py"
-
-command -v python3 >/dev/null || ! echo "python3 not found; skipping" || exit 77
-
-tmpdir=$(mktemp -d "${TMPDIR:-/tmp}/httrack_crangemem.XXXXXX") || exit 1
-serverpid=
-crawlpid=
-cleanup() {
-    test -n "$crawlpid" && kill -9 "$crawlpid" 2>/dev/null
-    if test -n "$serverpid"; then
-        kill "$serverpid" 2>/dev/null
-        wait "$serverpid" 2>/dev/null
-    fi
-    rm -rf "$tmpdir"
-}
-trap cleanup EXIT HUP INT QUIT PIPE TERM
-
-serverlog="${tmpdir}/server.log"
-python3 "$server" --root "${testdir}/server-root" >"$serverlog" 2>&1 &
-serverpid=$!
-port=
-for _ in $(seq 1 50); do
-    line=$(head -n1 "$serverlog" 2>/dev/null)
-    if test "${line%% *}" == "PORT"; then
-        port="${line#PORT }"
-        break
-    fi
-    kill -0 "$serverpid" 2>/dev/null || {
-        echo "server exited early: $(cat "$serverlog")"
-        exit 1
-    }
-    sleep 0.1
-done
-test -n "$port" || {
-    echo "could not discover server port"
-    exit 1
-}
-base="http://127.0.0.1:${port}"
-
-which httrack >/dev/null || {
-    echo "could not find httrack"
-    exit 1
-}
-out="${tmpdir}/crawl"
-mkdir "$out"
-common=(-O "$out" --quiet --disable-security-limits --robots=0 --timeout=30 --retries=1 -c1)
-refdir="${out}/hts-cache/ref"
-
-# --- pass 1: crawl, interrupt once the blob download is underway -------------
-printf '[pass 1: interrupt mid-download] ..\t'
-httrack "${common[@]}" "${base}/crange206mem/index.html" >"${tmpdir}/log1" 2>&1 &
-crawlpid=$!
-for _ in $(seq 1 300); do
-    test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" && break
-    kill -0 "$crawlpid" 2>/dev/null || break
-    sleep 0.1
-done
-sleep 0.3
-kill -TERM "$crawlpid" 2>/dev/null
-wait "$crawlpid" 2>/dev/null
-crawlpid=
-test -n "$(find "$refdir" -name '*.ref' 2>/dev/null)" || {
-    echo "FAIL: no temp-ref survived pass 1; cannot drive the resume"
-    exit 1
-}
-echo "OK (temp-ref present)"
-
-# --- pass 2: --continue -> resume -> hostile INT64_MAX Content-Range ----------
-printf '[pass 2: hostile 206, no overflow, refetch] ..\t'
-httrack "${common[@]}" --continue "${base}/crange206mem/index.html" >"${tmpdir}/log2" 2>&1
-echo "OK (terminated)"
-
-blob=$(find "$out" -name blob.bin 2>/dev/null | head -1)
-full=6008 # len(CRANGE206_BODY) = len("CR206DAT") + 6000
-
-printf '[file recovered whole] ..\t'
-test -s "$blob" || {
-    echo "FAIL: blob.bin missing after the hostile 206"
-    exit 1
-}
-got=$(wc -c <"$blob")
-test "$got" -eq "$full" || {
-    echo "FAIL: blob.bin is ${got} bytes, expected ${full}"
-    exit 1
-}
-echo "OK (${got} bytes)"

tests/49_local-cookiewall.test

@@ -1,34 +0,0 @@
-#!/bin/bash
-#
-# Cookie-wall self-redirect (#15): httrack replays a Set-Cookie to mirror the
-# real page, gives up when nothing changes, and stays bounded when it never does.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-# Unknown-ext (wall.php) and known-ext (wall.html): the wall sets the cookie once,
-# then serves real content; httrack must replay it and mirror that content.
-bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
-    --found 'cookiewall/wall.html' \
-    --file-matches 'cookiewall/wall.html' 'REAL-CONTENT-BEHIND-COOKIE-WALL' \
-    httrack 'BASEURL/cookiewall/index.html'
-
-bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
-    --found 'cookiewall2/wall.html' \
-    --file-matches 'cookiewall2/wall.html' 'REAL-CONTENT-BEHIND-COOKIE-WALL' \
-    httrack 'BASEURL/cookiewall2/index.html'
-
-# No Set-Cookie: the jar never changes, so httrack must give up at once (retry
-# stays gated) and not mirror the wall. A dropped gate would skip this log line.
-bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
-    --not-found 'cookiewall3/wall.html' \
-    --log-found 'loop to same filename' \
-    httrack 'BASEURL/cookiewall3/index.html'
-
-# Ever-changing cookie never satisfies the wall: httrack must stop at the retry
-# cap (bounded, no early give-up) and not mirror it.
-bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
-    --not-found 'cookiewall4/wall.html' \
-    --log-not-found 'loop to same filename' \
-    httrack 'BASEURL/cookiewall4/index.html'

tests/Makefile.am

@@ -30,7 +30,6 @@ TESTS = \
 	01_engine-cmdline.test \
 	01_engine-cookies.test \
 	01_engine-copyopt.test \
-	01_engine-crange.test \
 	01_engine-dns.test \
 	01_engine-doitlog.test \
 	01_engine-entities.test \
@@ -66,7 +65,6 @@ TESTS = \
 	01_engine-urlhack.test \
 	01_engine-unescape-bounds.test \
 	01_engine-useragent.test \
-	01_engine-xfread.test \
 	01_zlib-acceptencoding.test \
 	01_zlib-cache.test \
 	01_zlib-cache-corrupt.test \
@@ -115,11 +113,6 @@ TESTS = \
 	41_local-utf8-link.test \
 	42_local-maxsize-slow.test \
 	43_local-update-truncate.test \
-	44_local-update-errormask.test \
-	45_local-maxsize-recv.test \
-	46_local-update-304-resume.test \
-	47_local-crange-overflow.test \
-	48_local-crange-memresume.test \
-	49_local-cookiewall.test
+	44_local-update-errormask.test
 
 CLEANFILES = check-network_sh.cache

tests/crawl-test.sh

@@ -105,7 +105,7 @@ function start-crawl {
     log="${tmp}/log"
     debug starting httrack -O "${tmp}" "${moreargs[@]}" "${@:pos}"
     info "running httrack ${*:pos}"
-    httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -mrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
+    httrack -O "${tmp}" --user-agent="httrack $ver ut ($(uname -omrs))" "${moreargs[@]}" "${@:pos}" >"${log}" 2>&1 &
     crawlpid="$!"
     debug "started cralwer on pid $crawlpid"
     wait "$crawlpid"

tests/local-crawl.sh

@@ -213,7 +213,7 @@ mkdir "$out" || die "could not create $out"
 declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
 log="${tmpdir}/log"
 info "running httrack ${hts[*]}"
-httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
+httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
 crawlpid=$!
 wait "$crawlpid"
 crawlres=$?
@@ -230,7 +230,7 @@ grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
 # --- optional second pass: re-mirror into the same dir (cache/update path) ----
 if test -n "$rerun"; then
     info "re-running httrack (update pass)"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -258,7 +258,7 @@ fi
 if test -n "$rerun_args"; then
     read -ra extra <<<"$rerun_args"
     info "re-running httrack with ${rerun_args}"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
         "${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -281,7 +281,7 @@ if test -n "$rerun_dead"; then
     wait "$serverpid" 2>/dev/null
     serverpid=
     info "re-running httrack against the stopped server"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -omrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
     crawlpid=$!
     wait "$crawlpid" || true

tests/local-server.py

@@ -790,56 +790,6 @@ class Handler(SimpleHTTPRequestHandler):
         self.send_header("Content-Length", "0")
         self.end_headers()
 
-    # C7: stall the first GET (partial + temp-ref), then answer the resume's
-    # Range with a bogus 304; httrack must drop the partial and refetch.
-    RESUME304_BODY = b"C7DATA--" + bytes((i * 7 + 3) % 256 for i in range(8192))
-    _resume304_started = False
-
-    def route_resume304_index(self):
-        self.send_html('\t<a href="blob.bin">blob</a>')
-
-    def route_resume304(self):
-        counter = os.environ.get("RESUME304_COUNTER")
-        if counter:
-            with open(counter, "a") as fp:
-                fp.write("x")
-        rng = self.headers.get("Range")
-        if not Handler._resume304_started:
-            Handler._resume304_started = True
-            self.send_response(200)
-            self.send_header("Content-Type", "application/octet-stream")
-            self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
-            self.send_header("Accept-Ranges", "bytes")
-            self.send_header("Last-Modified", BIG_LASTMOD)
-            self.send_header("ETag", '"c7"')
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.RESUME304_BODY[:4096])
-                self.wfile.flush()
-                try:
-                    while True:
-                        time.sleep(3600)
-                except OSError:
-                    pass
-            return
-        if rng is not None:  # resume request: bogus out-of-protocol 304
-            mark = os.environ.get("RESUME304_MARK")
-            if mark:
-                with open(mark, "a") as fp:
-                    fp.write("z")
-            self.send_response(304)
-            self.send_header("ETag", '"c7"')
-            self.send_header("Content-Length", "0")
-            self.end_headers()
-            return
-        # Range-less refetch after the partial is dropped: whole file.
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(len(self.RESUME304_BODY)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(self.RESUME304_BODY)
-
     # 206 resume must honor the server's Content-Range, not the offset we asked
     # for (#198): a server resuming a few bytes *before* the request must not
     # leave httrack duplicating the overlap onto the partial. flaky.bin
@@ -905,94 +855,6 @@ class Handler(SimpleHTTPRequestHandler):
         if self.command != "HEAD":
             self.wfile.write(part)
 
-    # C2: a resume answered with a 206 whose Content-Range end is INT64_MAX would
-    # sign-overflow the crange+1 range check (UBSan abort). Stall first (partial +
-    # ref), then answer the resume Range with that hostile 206; httrack must reject
-    # the range and refetch, never overflow.
-    CRANGE206_BODY = b"CR206DAT" + bytes((i * 5 + 1) % 256 for i in range(6000))
-    _crange206_started = False
-
-    def route_crange206_index(self):
-        self.send_html('\t<a href="blob.bin">blob</a>')
-
-    def route_crange206(self):
-        rng = self.headers.get("Range")
-        if not Handler._crange206_started:
-            Handler._crange206_started = True
-            self.send_response(200)
-            self.send_header("Content-Type", "application/octet-stream")
-            self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
-            self.send_header("Accept-Ranges", "bytes")
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.CRANGE206_BODY[:3000])
-                self.wfile.flush()
-                try:
-                    while True:
-                        time.sleep(3600)
-                except OSError:
-                    pass
-            return
-        if rng is not None:  # resume: hostile 206, Content-Range end = INT64_MAX
-            self.send_response(206, "Partial Content")
-            self.send_header("Content-Type", "application/octet-stream")
-            self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
-            self.send_header("Content-Range", "bytes 0-9223372036854775807/1")
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.CRANGE206_BODY)
-            return
-        # range-less refetch after the bad range is rejected: whole file.
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(self.CRANGE206_BODY)
-
-    # C2 memory branch: a resume answered with a 206 that lies text/html (so the
-    # resume buffers in memory) plus a matching INT64_MAX Content-Length would
-    # overflow the buffer-size add. Stall first, then send that hostile 206.
-    _crange206mem_started = False
-
-    def route_crange206mem_index(self):
-        self.send_html('\t<a href="blob.bin">blob</a>')
-
-    def route_crange206mem(self):
-        rng = self.headers.get("Range")
-        if not Handler._crange206mem_started:
-            Handler._crange206mem_started = True
-            self.send_response(200)
-            self.send_header("Content-Type", "application/octet-stream")
-            self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
-            self.send_header("Accept-Ranges", "bytes")
-            self.end_headers()
-            if self.command != "HEAD":
-                self.wfile.write(self.CRANGE206_BODY[:3000])
-                self.wfile.flush()
-                try:
-                    while True:
-                        time.sleep(3600)
-                except OSError:
-                    pass
-            return
-        if rng is not None:  # resume: text/html + matching INT64_MAX Content-Length
-            self.send_response(206, "Partial Content")
-            self.send_header("Content-Type", "text/html")
-            self.send_header("Content-Length", "9223372036854775807")
-            self.send_header(
-                "Content-Range", "bytes 0-9223372036854775806/9223372036854775807"
-            )
-            self.end_headers()
-            return  # the overflow is computed before any body read
-        # range-less refetch after the resume is rejected: whole file.
-        self.send_response(200)
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(len(self.CRANGE206_BODY)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(self.CRANGE206_BODY)
-
     # error pages / 0-byte files (#17): -o0 ("no error pages") must keep 4xx/5xx
     # bodies off disk; a genuine 0-byte 200 is a valid file and stays.
     def route_errpage_index(self):
@@ -1143,56 +1005,6 @@ class Handler(SimpleHTTPRequestHandler):
     def route_delayed_empty(self):
         self.send_raw(b"", "text/html")  # 200 + Content-Length: 0
 
-    # --- /cookiewall/ (#15): a self-redirect that only sets a cookie is a
-    # consent wall; httrack must replay the cookie and fetch the real page.
-    WALL_MARK = b"REAL-CONTENT-BEHIND-COOKIE-WALL"
-
-    def route_cookiewall_index(self):
-        self.send_html('\t<a href="wall.php">wall</a>')
-
-    def route_cookiewall_wall(self):
-        self._cookiewall_reply("wall.php")
-
-    # Known-extension twin: .html so the type is not delayed-resolved.
-    def route_cookiewall2_index(self):
-        self.send_html('\t<a href="wall.html">wall</a>')
-
-    def route_cookiewall2_wall(self):
-        self._cookiewall_reply("wall.html")
-
-    def _cookiewall_reply(self, location):
-        if self.request_cookies().get("gate") == "1":
-            self.send_raw(
-                b"<html><body>" + self.WALL_MARK + b"</body></html>\n", "text/html"
-            )
-        else:
-            self._wall_redirect(location, "gate=1; Path=/")
-
-    # No-cookie self-redirect: the jar never changes, so httrack must give up at
-    # once rather than re-fetch (proves the cookie-wall retry stays gated on #15).
-    def route_cookiewall3_index(self):
-        self.send_html('\t<a href="wall.php">wall</a>')
-
-    def route_cookiewall3_wall(self):
-        self._wall_redirect("wall.php", None)
-
-    # Ever-changing cookie: every hit sets a fresh value, so the jar keeps
-    # changing; httrack must stop at the loops<7 cap, not spin forever.
-    def route_cookiewall4_index(self):
-        self.send_html('\t<a href="wall.php">wall</a>')
-
-    def route_cookiewall4_wall(self):
-        nonce = int(self.request_cookies().get("gate", "0")) + 1
-        self._wall_redirect("wall.php", f"gate={nonce}; Path=/")
-
-    def _wall_redirect(self, location, set_cookie):
-        self.send_response(302, "Found")
-        self.send_header("Location", location)
-        if set_cookie is not None:
-            self.send_header("Set-Cookie", set_cookie)
-        self.send_header("Content-Length", "0")
-        self.end_headers()
-
     # -E time-limit (#481): pages that trickle far longer than any -E budget,
     # so only an engine-side abort can end the crawl.
     TRICKLE_SECONDS = 60
@@ -1292,22 +1104,6 @@ class Handler(SimpleHTTPRequestHandler):
         except OSError:
             pass
 
-    # -M received-volume cap (#520): links to large 404 bodies. httrack receives
-    # each (HTS_TOTAL_RECV climbs) but saves none, so saved stays far below -M.
-    def route_maxrecv_index(self):
-        self.send_html(
-            "".join('\t<a href="r%d.bin">r%d</a>\n' % (i, i) for i in range(16))
-        )
-
-    def route_maxrecv_404(self):
-        body = b"x" * self.BIGFILE_BYTES
-        self.send_response(404, "Not Found")
-        self.send_header("Content-Type", "application/octet-stream")
-        self.send_header("Content-Length", str(len(body)))
-        self.end_headers()
-        if self.command != "HEAD":
-            self.wfile.write(body)
-
     ROUTES = {
         "/cookies/entrance.php": route_entrance,
         "/cookies/second.php": route_second,
@@ -1337,15 +1133,9 @@ class Handler(SimpleHTTPRequestHandler):
         "/intl/" + INTL_NAME: route_intl_page,
         "/resume/index.html": route_resume_index,
         "/resume/blob.txt": route_resume,
-        "/resume304/index.html": route_resume304_index,
-        "/resume304/blob.bin": route_resume304,
         "/overlap/index.html": route_overlap_index,
         "/overlap/flaky.bin": route_overlap,
         "/overlap/full.bin": route_overlap_full,
-        "/crange206/index.html": route_crange206_index,
-        "/crange206/blob.bin": route_crange206,
-        "/crange206mem/index.html": route_crange206mem_index,
-        "/crange206mem/blob.bin": route_crange206mem,
         "/size/index.html": route_size_index,
         "/size/oversize.bin": route_size_oversize,
         "/errpage/index.html": route_errpage_index,
@@ -1409,14 +1199,6 @@ class Handler(SimpleHTTPRequestHandler):
         "/delayed/chain7.php": route_delayed_chain,
         "/delayed/chain8.php": route_delayed_chain,
         "/delayed/chain9.php": route_delayed_chain,
-        "/cookiewall/index.html": route_cookiewall_index,
-        "/cookiewall/wall.php": route_cookiewall_wall,
-        "/cookiewall2/index.html": route_cookiewall2_index,
-        "/cookiewall2/wall.html": route_cookiewall2_wall,
-        "/cookiewall3/index.html": route_cookiewall3_index,
-        "/cookiewall3/wall.php": route_cookiewall3_wall,
-        "/cookiewall4/index.html": route_cookiewall4_index,
-        "/cookiewall4/wall.php": route_cookiewall4_wall,
         "/redir/index.html": route_redir_index,
         "/redir/go.php": route_redir_go,
         "/redir/target.html": route_redir_target,
@@ -1424,23 +1206,6 @@ class Handler(SimpleHTTPRequestHandler):
         "/mini304/page.html": route_mini304_page,
         "/errmask/index.html": route_errmask_index,
         "/errmask/keep.dat": route_errmask_keep,
-        "/maxrecv/index.html": route_maxrecv_index,
-        "/maxrecv/r0.bin": route_maxrecv_404,
-        "/maxrecv/r1.bin": route_maxrecv_404,
-        "/maxrecv/r2.bin": route_maxrecv_404,
-        "/maxrecv/r3.bin": route_maxrecv_404,
-        "/maxrecv/r4.bin": route_maxrecv_404,
-        "/maxrecv/r5.bin": route_maxrecv_404,
-        "/maxrecv/r6.bin": route_maxrecv_404,
-        "/maxrecv/r7.bin": route_maxrecv_404,
-        "/maxrecv/r8.bin": route_maxrecv_404,
-        "/maxrecv/r9.bin": route_maxrecv_404,
-        "/maxrecv/r10.bin": route_maxrecv_404,
-        "/maxrecv/r11.bin": route_maxrecv_404,
-        "/maxrecv/r12.bin": route_maxrecv_404,
-        "/maxrecv/r13.bin": route_maxrecv_404,
-        "/maxrecv/r14.bin": route_maxrecv_404,
-        "/maxrecv/r15.bin": route_maxrecv_404,
     }
 
     # --- /big/ seeded pseudo-site ------------------------------------------
@@ -1645,12 +1410,7 @@ def main():
     def factory(*a, **kw):
         return Handler(*a, directory=root, **kw)
 
-    # macOS/BSD drop SYNs when the listen backlog overflows (Linux is lenient);
-    # raise it from Python's default 5 so a busy -c8 crawl can't lose fetches.
-    class BacklogHTTPServer(ThreadingHTTPServer):
-        request_queue_size = 128
-
-    httpd = BacklogHTTPServer((args.bind, 0), factory)
+    httpd = ThreadingHTTPServer((args.bind, 0), factory)
 
     if args.tls:
         import ssl