strjoker: catastrophic backtracking on *-heavy filter patterns (#513)

* Fix catastrophic backtracking in the strjoker filter matcher

The recursive wildcard matcher re-explored the same (subject, pattern)
positions exponentially on *-heavy patterns against a non-matching
subject: 9 stars against 50 characters already took 149 seconds, about
7x per added star. Subjects come from crawled URLs, so a hostile site
could stall matching against a user's filters (ReDoS).

Memoize failures instead: one bit per (subject, pattern) offset pair,
checked before and recorded after each recursion, which bounds the
work polynomially. A pair that failed once fails forever (*size is
only written on the success path), so the memo is sound and the
matcher's semantics are unchanged; a differential run of 9000 random
pattern/subject/size cases old vs new is byte-identical. The bitmap
lives on the stack for common sizes, on the heap above 2 KB.

The engine test adds the pathological pattern (instant now, hung
before) plus a case where a later star's dead-end must re-extend an
earlier star, which guards against a single-backtrack-point rewrite.
A corpus seed lets the fuzz replay job cover the same shape.

Closes #501

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

* Broaden matching-system test coverage

Adds near-miss negative cases for the arms the filter tests skipped
(word order, exact tails, range boundaries, class case sensitivity,
empty star runs, descending ranges, *[param], rule ordering, failed
size tests), first coverage of mime: filters via a new
-#test=filtermime self-test, and a -#test=filtermemo differential:
20000 seeded random pattern/subject/size cases where the memoized
matcher must agree with a new no-memo oracle (strjoker_nomemo) on
result, *size and *size_flag, asserting both polarities occur.
Corrupting the memo indexing by hand makes the differential fail on
case 8, so it genuinely guards the memo.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

---------

Signed-off-by: Xavier Roche <roche@httrack.com>
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>

fuzz/Makefile.am

@@ -36,6 +36,7 @@ EXTRA_DIST = README.md run-fuzzers.sh \
 	corpus/unescape/percent.txt \
 	corpus/filters/filter.bin corpus/filters/filter-size.bin \
 	corpus/filters/regress-empty-subject-unique.bin \
+	corpus/filters/redos-star-classes.bin \
 	corpus/url/http-url.txt corpus/url/relative-path.txt \
 	corpus/url/regress-file-empty-path.txt corpus/url/regress-long-path-abort.txt \
 	corpus/header/full-response.txt corpus/header/redirect.txt \

fuzz/run-fuzzers.sh

@@ -2,8 +2,8 @@
 # Drive every built harness against its seed corpus.
 #   run-fuzzers.sh <build-fuzz-dir> check       deterministic replay (CI smoke)
 #   run-fuzzers.sh <build-fuzz-dir> [seconds]   timed mutation run (discovery)
-# Replay is crash/leak-only and never mutates, so it can't hit strjoker's
-# catastrophic backtracking; the timed mode can, hence the per-unit -timeout.
+# Replay is crash/leak-only and never mutates; the per-unit -timeout guards
+# both modes against pathological slowdowns (e.g. pre-#501 strjoker).
 set -euo pipefail
 
 srcdir=$(cd "$(dirname "$0")" && pwd)

src/htsfilters.c

@@ -48,6 +48,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsbase.h"
 #include "htslib.h"
 #include <ctype.h>
+#include <stdint.h>
 /* END specific definitions */
 
 // à partir d'un tableau de {"+*.toto","-*.zip","+*.tata"} définit si nom est autorisé
@@ -120,16 +121,81 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
   return use1 ? jok1 : jok2;
 }
 
-// supercomparateur joker (tm)
-// compare a et b (b=avec joker dedans), case insensitive [voir CI]
-// renvoi l'adresse de la première lettre de la chaine
-// (càd *[..]toto.. renvoi adresse de toto dans la chaine)
-// accepte les délires du genre www.*.*/ * / * truc*.*
-// cet algo est 'un peu' récursif mais ne consomme pas trop de tm
-// * = toute lettre
-// --?-- : spécifique à HTTrack et aux ?
+/* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
+   pair keeps star-heavy patterns polynomial instead of exponential (#501). */
+typedef struct strjoker_memo {
+  const char *chaine0, *joker0; /* offsets are relative to these bases */
+  size_t stride;                /* strlen(joker0) + 1 */
+  unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
+} strjoker_memo;
+
+static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
+                                 const char *joker, LLint *size,
+                                 int *size_flag);
+
+/* A pair that failed once fails forever (*size is only written on the success
+   path), so record NULL results and cut the re-exploration. */
+static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
+                                const char *joker, LLint *size,
+                                int *size_flag) {
+  size_t bit = 0;
+  const char *adr;
+
+  if (memo->failed) {
+    bit = (size_t) (chaine - memo->chaine0) * memo->stride +
+          (size_t) (joker - memo->joker0);
+    if (memo->failed[bit >> 3] & (unsigned char) (1u << (bit & 7)))
+      return NULL;
+  }
+  adr = strjoker_impl(memo, chaine, joker, size, size_flag);
+  if (adr == NULL && memo->failed)
+    memo->failed[bit >> 3] |= (unsigned char) (1u << (bit & 7));
+  return adr;
+}
+
+// wildcard comparator: match chaine against joker (pattern), case-insensitive
+// returns the address of the first matched letter past any leading joker
+// (ie. *[..]toto.. returns the address of toto within chaine), NULL on mismatch
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
                                 LLint *size, int *size_flag) {
+  strjoker_memo memo = {chaine, joker, 0, NULL};
+  unsigned char stackbits[2048];
+  hts_boolean onheap = HTS_FALSE;
+  const char *adr;
+
+  if (chaine != NULL && joker != NULL) {
+    const size_t n1 = strlen(chaine) + 1;
+
+    memo.stride = strlen(joker) + 1;
+    if (n1 <= (SIZE_MAX - 7) / memo.stride) { // overflow-safe bitmap size
+      const size_t bytes = (n1 * memo.stride + 7) / 8;
+
+      if (bytes <= sizeof(stackbits)) {
+        memset(stackbits, 0, bytes);
+        memo.failed = stackbits;
+      } else {
+        memo.failed = (unsigned char *) calloct(bytes, 1);
+        onheap = memo.failed != NULL ? HTS_TRUE : HTS_FALSE;
+      }
+    }
+  }
+  adr = strjoker_rec(&memo, chaine, joker, size, size_flag);
+  if (onheap)
+    freet(memo.failed);
+  return adr;
+}
+
+/* Test-only oracle: the same matcher without the failure memo. */
+const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
+                            int *size_flag) {
+  const strjoker_memo memo = {chaine, joker, 0, NULL};
+
+  return strjoker_rec(&memo, chaine, joker, size, size_flag);
+}
+
+static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
+                                 const char *joker, LLint *size,
+                                 int *size_flag) {
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -303,7 +369,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
 
       // tester sans le joker (pas ()+ mais ()*)
       if (!unique) {
-        if ((adr = strjoker(chaine, joker + jmp, size, size_flag))) {
+        if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag))) {
           return adr;
         }
       }
@@ -315,7 +381,8 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
         max = strnotempty(chaine) ? 1 : 0; /* empty chaine: no char to eat */
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
-          if ((adr = strjoker(chaine + i + 1, joker + jmp, size, size_flag))) {
+          if ((adr = strjoker_rec(memo, chaine + i + 1, joker + jmp, size,
+                                  size_flag))) {
             return adr;
           }
           i++;
@@ -325,7 +392,8 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
 
       // tester chaîne vide
       if (i != max + 2)         // avant c'est ok
-        if ((adr = strjoker(chaine + max, joker + jmp, size, size_flag)))
+        if ((adr = strjoker_rec(memo, chaine + max, joker + jmp, size,
+                                size_flag)))
           return adr;
 
       return NULL;              // perdu
@@ -347,7 +415,7 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
       // comparaison ok?
       if (ok) {
         // continuer la comparaison.
-        if (strjoker(chaine + jmp, joker + jmp, size, size_flag))
+        if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag))
           return chaine;        // retourner 1e lettre
       }
 

src/htsfilters.h

@@ -48,6 +48,10 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
                      const char *nom2, LLint *size, int *size_flag, int *depth);
 HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * size,
                           int *size_flag);
+/* strjoker() without the failure memo (exponential worst case); test-only
+   oracle for the memoized matcher. */
+const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
+                            int *size_flag);
 const char *strjokerfind(const char *chaine, const char *joker);
 #endif
 

src/htsselftest.c

@@ -621,6 +621,105 @@ static int st_filtersize(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Mime-type filter verdict via fa_strjoker(type=1): only mime: rules apply. */
+static int st_filtermime(httrackp *opt, int argc, char **argv) {
+  int verdict;
+
+  (void) opt;
+  if (argc < 2) {
+    fprintf(stderr, "filtermime: needs <mime> <filter> [filter...]\n");
+    return 1;
+  }
+  verdict = fa_strjoker(1, &argv[1], argc - 1, argv[0], NULL, NULL, NULL);
+  printf("verdict=%s\n", verdict > 0   ? "allowed"
+                         : verdict < 0 ? "forbidden"
+                                       : "unknown");
+  return 0;
+}
+
+/* SplitMix64: deterministic, platform-independent case generator. */
+static uint64_t st_mix64(uint64_t *state) {
+  uint64_t z = (*state += UINT64_C(0x9E3779B97F4A7C15));
+
+  z = (z ^ (z >> 30)) * UINT64_C(0xBF58476D1CE4B5B9);
+  z = (z ^ (z >> 27)) * UINT64_C(0x94D049BB133111EB);
+  return z ^ (z >> 31);
+}
+
+/* Differential test: memoized strjoker vs the no-memo oracle on seeded random
+   pattern/subject/size cases must agree on result, *size and *size_flag. */
+static int st_filtermemo(httrackp *opt, int argc, char **argv) {
+  static const char *const pieces[] = {
+      "a",       "b",        "A",    "c",      ".",       "/",
+      "?",       "*",        "*[a]", "*[ab]",  "*[a-c]",  "*[A-Z]",
+      "*[<5]",   "*[>5]",    "*(a)", "*(a,b)", "*[file]", "*[path]",
+      "*[name]", "*[param]", "*[]",  "*[\\a]", "*[a-"};
+  static const char subject_chars[] = "abAc./?";
+  uint64_t rng = UINT64_C(0x501);
+  int iters = 20000, matched = 0, unmatched = 0;
+  int it;
+
+  (void) opt;
+  if (argc >= 1)
+    sscanf(argv[0], "%d", &iters);
+  for (it = 0; it < iters; it++) {
+    char pat[64], str[16];
+    size_t pl = 0;
+    const int npieces = (int) (st_mix64(&rng) % 5);
+    const int slen = (int) (st_mix64(&rng) % 13);
+    const int szsel = (int) (st_mix64(&rng) % 4); /* none/unknown/3/20 */
+    LLint sz1, sz2, off1, off2;
+    int i, flag1 = 0, flag2 = 0;
+    char *hpat, *hstr;
+    const char *adr;
+
+    for (i = 0; i < npieces; i++) {
+      const char *p =
+          pieces[st_mix64(&rng) % (sizeof(pieces) / sizeof(pieces[0]))];
+      const size_t len = strlen(p);
+
+      if (len >= sizeof(pat) - pl)
+        break;
+      memcpy(pat + pl, p, len);
+      pl += len;
+    }
+    pat[pl] = '\0';
+    for (i = 0; i < slen; i++)
+      str[i] = subject_chars[st_mix64(&rng) % (sizeof(subject_chars) - 1)];
+    str[slen] = '\0';
+    sz1 = sz2 = (szsel <= 1) ? -1 : (szsel == 2) ? 3 : 20;
+    /* exact-size heap copies per run so a sanitizer traps any over-read */
+    hpat = strdupt(pat);
+    hstr = strdupt(str);
+    adr = strjoker(hstr, hpat, szsel ? &sz1 : NULL, szsel ? &flag1 : NULL);
+    off1 = adr != NULL ? (LLint) (adr - hstr) : -1;
+    freet(hpat);
+    freet(hstr);
+    hpat = strdupt(pat);
+    hstr = strdupt(str);
+    adr =
+        strjoker_nomemo(hstr, hpat, szsel ? &sz2 : NULL, szsel ? &flag2 : NULL);
+    off2 = adr != NULL ? (LLint) (adr - hstr) : -1;
+    freet(hpat);
+    freet(hstr);
+    if (off1 != off2 || sz1 != sz2 || flag1 != flag2) {
+      printf("filtermemo MISMATCH pat=[%s] str=[%s] szsel=%d: "
+             "off %d/%d size %d/%d flag %d/%d\n",
+             pat, str, szsel, (int) off1, (int) off2, (int) sz1, (int) sz2,
+             flag1, flag2);
+      return 1;
+    }
+    if (off1 >= 0)
+      matched++;
+    else
+      unmatched++;
+  }
+  /* both polarities must actually occur or the test proves nothing */
+  assertf(matched > 0 && unmatched > 0);
+  printf("filtermemo: %d cases OK\n", iters);
+  return 0;
+}
+
 /* Merged two-form filter verdict via fa_strjoker_dual (see htsfilters.h). */
 static int st_filterdual(httrackp *opt, int argc, char **argv) {
   int depth = -1, verdict;
@@ -2279,6 +2378,10 @@ static const struct selftest_entry {
     {"filtersize", "<size> <string> <filter>...",
      "size-aware filter verdict (negative size = unknown/scan time)",
      st_filtersize},
+    {"filtermime", "<mime> <filter>...",
+     "mime-type filter verdict (fa_strjoker type=1)", st_filtermime},
+    {"filtermemo", "[iterations]",
+     "memoized vs unmemoized matcher differential", st_filtermemo},
     {"filterdual", "<string1> <string2> <filter>...",
      "merged two-form filter verdict (fa_strjoker_dual)", st_filterdual},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},

tests/01_engine-filter.test

@@ -131,3 +131,64 @@ match '*.aspx' 'page.aspx?y=2'
 # empty subject against a unique-match allow-all pattern (heap over-read regress)
 nomatch_empty '*(('
 nomatch_empty '**(('
+
+# near-miss negatives: word order, exact tails, range boundaries, class case
+nomatch 'foo*bar' 'foobaz'
+nomatch '*foo*bar' 'barfoo' # both words present, wrong order
+nomatch '*.gif' 'a.gifx'    # trailing junk after the extension
+nomatch '*.gif' 'a.gi'
+nomatch '*[b-y]' 'a'
+nomatch '*[b-y]' 'z'
+nomatch '*[a-z].txt' 'B.txt' # classes are case-sensitive...
+match 'FoO*' 'foobar'        # ...literals are not
+
+# a star may match an empty run, but never skip a required literal
+match '*[0-9].gif' '.gif'
+nomatch '*[0-9].gif' 'x.gif'
+
+# a descending range *[c-a] is an empty class, not c..a reversed
+match '*[c-a]x' 'x'
+nomatch '*[c-a]x' 'cx'
+
+# *[param] is all-or-nothing on '?': spans everything after one, else nothing
+match 'a*[param]' 'a?x=1'
+nomatch 'a*[param]' 'ab?x=1'
+
+# the latest matching rule wins, regardless of sign
+fsize 'verdict=forbidden size_flag=0' -1 a.gif '+*.gif' '-*.gif'
+fsize 'verdict=allowed size_flag=0' -1 a.gif '-*.gif' '+*.gif'
+
+# a failed size test sets size_flag but leaves the rule unmatched
+fsize 'verdict=unknown size_flag=1' 5 foo.jpg '-*.jpg*[>10]'
+
+# mime filters (-#test=filtermime = fa_strjoker type=1): only mime: rules
+# apply to a mime type, and mime: rules never apply to a URL
+fmime() {
+    local want="$1"
+    shift
+    test "$(httrack -O /dev/null -#test=filtermime "$@")" == "verdict=$want" || exit 1
+}
+fmime allowed 'image/png' '+mime:image/*'
+fmime unknown 'text/html' '+mime:image/*'
+fmime forbidden 'image/png' '+mime:image/*' '-mime:image/png' # last wins
+fmime allowed 'image/png' '-mime:image/png' '+mime:image/*'
+fmime unknown 'image/png' '+image/*'                               # URL rule: skipped
+fsize 'verdict=unknown size_flag=0' -1 'image/png' '+mime:image/*' # and vice versa
+
+# memoized vs no-memo differential over seeded random patterns and subjects,
+# matching and non-matching (the self-test asserts both polarities occurred)
+test "$(httrack -O /dev/null -#test=filtermemo)" == "filtermemo: 20000 cases OK" || exit 1
+
+# star-heavy non-match must stay polynomial (#501); 400 chars also exercises
+# the heap-allocated memo. timeout (where available) turns a hang into a FAIL.
+with_timeout() {
+    if command -v timeout >/dev/null; then timeout 60 "$@"; else "$@"; fi
+}
+pat=$(printf '*[a]%.0s' $(seq 1 12))b
+subj=$(printf 'a%.0s' $(seq 1 400))
+test "$(with_timeout httrack -O /dev/null -#test=filter "$pat" "$subj")" == "$subj does NOT match $pat" || exit 1
+
+# a later star's class dead-end must re-extend the earlier star, not give up
+# (guards against a single-backtrack-point matcher rewrite)
+match '*[aX]X*[a]Y' 'aXaXaY'
+nomatch '*[aX]X*[a]Y' 'aXbXaY'