Parse robots.txt with RFC 9309 Allow/Disallow precedence

The robots.txt handler only did substring Disallow matching against a flat
token blob: no Allow:, no path wildcards. Sites using "Disallow: /" plus
"Allow: /public/" were over-blocked, since Allow was never parsed.

Move the body parsing into robots_parse() (htsrobots.c) so both the crawler
and a self-test feed raw robots.txt. Rules are stored Allow/Disallow-tagged
and consulted with RFC 9309 precedence: the longest matching path pattern
wins, Allow breaking ties. Pattern matching supports '*' (any run) and a
trailing '$' (end-of-path anchor) via a linear two-pointer matcher with a
single resumable star position, so hostile patterns cannot trigger
exponential backtracking. Path matching is now case-sensitive per the RFC.

robots_wizard is internal (not in DevIncludes_DATA, no HTSEXT_API; htsopt.h
holds only an opaque pointer), so the in-memory format changed without an ABI
break. Sitemap:/Crawl-delay: lines are tolerated but ignored, as before.

New -#test=robots self-test plus tests/01_engine-robots.test cover the
Allow-over-Disallow longest match, the equal-length Allow tie, '*'/'$'
wildcards, and httrack-group selection.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsselftest.c

@@ -524,41 +524,6 @@ static int string_safety_selftests(void) {
       return 1;
   }
 
-  /* StringCatN/StringSetLength must eval SIZE once: (n_eval++, V) leaves
-     n_eval == 2 on a double-eval macro. */
-  {
-    String s = STRING_EMPTY;
-    int n_eval = 0;
-
-    StringCat(s, "hello");
-    StringCatN(s, "world", (n_eval++, 3)); /* strlen>SIZE so the clamp runs */
-    if (n_eval != 1 || strcmp(StringBuff(s), "hellowor") != 0) {
-      StringFree(s);
-      return 1;
-    }
-
-    n_eval = 0;
-    StringSetLength(s, (n_eval++, 5));
-    if (n_eval != 1 || StringLength(s) != 5) {
-      StringFree(s);
-      return 1;
-    }
-    StringFree(s);
-  }
-
-  /* StringSubRW still reads/writes after dropping its duplicate definition. */
-  {
-    String s = STRING_EMPTY;
-
-    StringCat(s, "abc");
-    StringSubRW(s, 1) = 'X';
-    if (StringSub(s, 1) != 'X' || strcmp(StringBuff(s), "aXc") != 0) {
-      StringFree(s);
-      return 1;
-    }
-    StringFree(s);
-  }
-
   return 0;
 }
 

src/htsstrings.h

@@ -121,6 +121,9 @@ struct String {
 /** Byte at POS (read/write). No bounds check; POS must be < StringLength. **/
 #define StringSubRW(BLK, POS) (StringBuffRW(BLK)[POS])
 
+/** Subcharacter (read/write) **/
+#define StringSubRW(BLK, POS) (StringBuffRW(BLK)[POS])
+
 /** Byte POS positions from the end (read). POS==1 is the last byte. **/
 #define StringRight(BLK, POS) (StringBuff(BLK)[StringLength(BLK) - POS])
 
@@ -188,9 +191,8 @@ HTS_STATIC char *StringBuffN_(String *blk, int size) {
     asserts SIZE fits the existing content; does not (re)allocate. **/
 #define StringSetLength(BLK, SIZE)                                             \
   do {                                                                         \
-    const int len__ = (SIZE); /* signed: negative means strlen(buffer_) */     \
-    if (len__ >= 0) {                                                          \
-      (BLK).length_ = len__;                                                   \
+    if (SIZE >= 0) {                                                           \
+      (BLK).length_ = SIZE;                                                    \
     } else {                                                                   \
       (BLK).length_ = strlen((BLK).buffer_);                                   \
     }                                                                          \
@@ -306,11 +308,10 @@ HTS_STATIC void StringAttach(String *blk, char **str) {
 #define StringCatN(BLK, STR, SIZE)                                             \
   do {                                                                         \
     const char *str__ = (STR);                                                 \
-    const size_t usize__ = (SIZE);                                             \
     if (str__ != NULL) {                                                       \
       size_t size__ = strlen(str__);                                           \
-      if (size__ > usize__) {                                                  \
-        size__ = usize__;                                                      \
+      if (size__ > (SIZE)) {                                                   \
+        size__ = (SIZE);                                                       \
       }                                                                        \
       StringMemcat(BLK, str__, size__);                                        \
     }                                                                          \

tests/19_local-connect-fallback.test

@@ -20,14 +20,6 @@ if ! command -v python3 >/dev/null 2>&1; then
     echo "python3 missing, skipping"
     exit 77
 fi
-# The fixture needs a second loopback IP (dead 127.0.0.2 + live 127.0.0.1) for
-# the fallback to have a target; GNU/Hurd has only 127.0.0.1, so skip there.
-case "$(uname -s)" in
-GNU | GNU/*)
-    echo "GNU/Hurd: single loopback IP, connect-fallback fixture unbuildable, skipping"
-    exit 77
-    ;;
-esac
 
 server="$top_srcdir/tests/local-server.py"
 root="$top_srcdir/tests/server-root"