htsparse: extract HT_ADD_END into hts_finish_html_file, drop dead macros

The final HTML-flush step lived in the ~60-line HT_ADD_END macro, captured by
lexical scope in htsparse(). Move it to a real function in htscore.c (next to
its sibling hts_finish_makeindex), passing the output buffer as a plain
pointer+length since the TypedArray is an anonymous struct type that can't cross
a function boundary; the caller keeps the size guard and the trailing
TypedArrayFree. Behavior-preserving: a two-crawl mirror produces byte-identical
output against master.

Also delete the _FILTERS/_FILTERS_PTR macros, dead in htsparse.c (htscore.c and
htswizard.c keep their own local copies); _ROBOTS stays.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsparse.c

@@ -108,97 +108,88 @@ Please visit our Website: http://www.httrack.com
 
 #define HT_ADD_FOP
 
-/* Mutable extended-context fields: one source of truth so the DEFINE/SET/SAVE
-   load and store lists can't drift apart. */
-/* clang-format off */
-#define ENGINE_MUTABLE_FIELDS(X) \
-  X(int, error, stre->error_) \
-  X(int, store_errpage, stre->store_errpage_) \
-  X(int, makeindex_done, stre->makeindex_done_) \
-  X(FILE *, makeindex_fp, stre->makeindex_fp_) \
-  X(int, makeindex_links, stre->makeindex_links_) \
-  X(LLint, stat_fragment, stre->stat_fragment_)
-
-#define ENGINE_FIELD_DECLARE(type, name, src) type name = *(src);
-#define ENGINE_FIELD_LOAD(type, name, src) name = *(src);
-#define ENGINE_FIELD_STORE(type, name, src) *(src) = name;
-
 #define ENGINE_DEFINE_CONTEXT() \
   ENGINE_DEFINE_CONTEXT_BASE(); \
+  /* */ \
   htsblk* const r HTS_UNUSED = stre->r_; \
   hash_struct* const hash HTS_UNUSED = stre->hash_; \
   char* const codebase HTS_UNUSED = stre->codebase; \
   char* const base HTS_UNUSED = stre->base; \
+  /* */ \
   const char * const template_header HTS_UNUSED = stre->template_header_; \
   const char * const template_body HTS_UNUSED = stre->template_body_; \
   const char * const template_footer HTS_UNUSED = stre->template_footer_; \
+  /* */ \
   HTS_UNUSED char* const makeindex_firstlink = stre->makeindex_firstlink_; \
-  ENGINE_MUTABLE_FIELDS(ENGINE_FIELD_DECLARE) \
-  /* load-once (kept out of SET/SAVE): re-reading would reset the throttle */ \
+  /* */ \
+  /* */ \
+  int error = * stre->error_; \
+  int store_errpage = * stre->store_errpage_; \
+  /* */ \
+  int makeindex_done = *stre->makeindex_done_; \
+  FILE* makeindex_fp = *stre->makeindex_fp_; \
+  int makeindex_links = *stre->makeindex_links_; \
+  /* */ \
+  LLint stat_fragment = *stre->stat_fragment_; \
   HTS_UNUSED TStamp makestat_time = stre->makestat_time; \
   HTS_UNUSED FILE* makestat_fp = stre->makestat_fp
 
+/* clang-format off: an edit realigns all backslashes, churning the macro. */
+/* clang-format off */
+/* Load-once: re-reading resets makestat_time (mutated locally, never SAVEd). */
 #define ENGINE_SET_CONTEXT() \
   ENGINE_SET_CONTEXT_BASE(); \
-  ENGINE_MUTABLE_FIELDS(ENGINE_FIELD_LOAD)
+  /* */ \
+  error = * stre->error_; \
+  store_errpage = * stre->store_errpage_; \
+  /* */ \
+  makeindex_done = *stre->makeindex_done_; \
+  makeindex_fp = *stre->makeindex_fp_; \
+  makeindex_links = *stre->makeindex_links_; \
+  /* */ \
+  stat_fragment = *stre->stat_fragment_
+/* clang-format on */
 
 #define ENGINE_LOAD_CONTEXT() \
   ENGINE_DEFINE_CONTEXT()
 
 #define ENGINE_SAVE_CONTEXT() \
   ENGINE_SAVE_CONTEXT_BASE(); \
-  ENGINE_MUTABLE_FIELDS(ENGINE_FIELD_STORE)
-/* clang-format on */
+  /* */ \
+  * stre->error_ = error; \
+  * stre->store_errpage_ = store_errpage; \
+  /* */ \
+  *stre->makeindex_done_ = makeindex_done; \
+  *stre->makeindex_fp_ = makeindex_fp; \
+  *stre->makeindex_links_ = makeindex_links; \
+  /* */ \
+  *stre->stat_fragment_ = stat_fragment
 
 #define _ROBOTS      ((robots_wizard*)opt->robotsptr)
 
-/* JS-detection automaton states; INSCRIPT_DEFAULT is the synthetic "any other
-   char" column of the transition table. */
-typedef enum {
-  INSCRIPT_START = 0,
-  INSCRIPT_ANTISLASH,
-  INSCRIPT_INQUOTE,
-  INSCRIPT_INQUOTE2,
-  INSCRIPT_SLASH,
-  INSCRIPT_SLASHSLASH,
-  INSCRIPT_COMMENT,
-  INSCRIPT_COMMENT2,
-  INSCRIPT_ANTISLASH_IN_QUOTE,
-  INSCRIPT_ANTISLASH_IN_QUOTE2,
-  INSCRIPT_DEFAULT = 256
-} INSCRIPT;
+/* Apply current *adr character for the script automate */
+#define AUTOMATE_LOOKUP_CURRENT_ADR() do { \
+  if (inscript) { \
+  int new_state_pos; \
+  new_state_pos=inscript_state[inscript_state_pos][(unsigned char)*html]; \
+  if (new_state_pos < 0) { \
+  new_state_pos=inscript_state[inscript_state_pos][INSCRIPT_DEFAULT]; \
+  } \
+  assertf(new_state_pos >= 0); \
+  assertf(new_state_pos*sizeof(inscript_state[0]) < sizeof(inscript_state)); \
+  inscript_state_pos=new_state_pos; \
+  } \
+} while(0)
 
-#define INSCRIPT_NSTATES 10 /* rows in the transition table */
-
-/* Live view of the parser's automaton locals, set up once so the helpers below
-   can drive it without capturing them by lexical scope. */
-typedef struct {
-  const int *inscript;             /* nonzero while inside a script body */
-  const signed char (*table)[257]; /* [INSCRIPT_NSTATES][257] transitions */
-  INSCRIPT *pos;                   /* current state */
-  const char **html;               /* parse cursor */
-} script_automate;
-
-/* Feed the current *html byte to the automaton. No-op outside a script body. */
-static void hts_automate_lookup(const script_automate *aut) {
-  if (*aut->inscript) {
-    int next = aut->table[*aut->pos][(unsigned char) **aut->html];
-    if (next < 0) {
-      next = aut->table[*aut->pos][INSCRIPT_DEFAULT];
-    }
-    assertf(next >= 0 && next < INSCRIPT_NSTATES);
-    *aut->pos = (INSCRIPT) next;
-  }
-}
-
-/* Advance the cursor by 'steps' bytes, feeding each to the automaton. */
-static void hts_automate_increment(const script_automate *aut, int steps) {
-  while (steps > 0) {
-    (*aut->html)++;
-    hts_automate_lookup(aut);
-    steps--;
-  }
-}
+/* Increment current pointer to 'steps' characters, modifying automate if necessary */
+#define INCREMENT_CURRENT_ADR(steps) do { \
+  int steps__ = (int) ( steps ); \
+  while(steps__ > 0) { \
+  html++; \
+  AUTOMATE_LOOKUP_CURRENT_ADR(); \
+  steps__ --; \
+  } \
+} while(0)
 
 /* Percent-encode the angle brackets of a string so it is safe to embed inside
    an HTML comment (the default footer) or any other HTML context. A URL holding
@@ -343,7 +334,20 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
       int incomment = 0;        // dans un <!--
       int inscript = 0;         // dans un scipt pour applets javascript)
       int inscript_locked = 0;  // in locked script (ie. js file)
-      signed char inscript_state[INSCRIPT_NSTATES][257];
+      signed char inscript_state[10][257];
+      typedef enum {
+        INSCRIPT_START = 0,
+        INSCRIPT_ANTISLASH,
+        INSCRIPT_INQUOTE,
+        INSCRIPT_INQUOTE2,
+        INSCRIPT_SLASH,
+        INSCRIPT_SLASHSLASH,
+        INSCRIPT_COMMENT,
+        INSCRIPT_COMMENT2,
+        INSCRIPT_ANTISLASH_IN_QUOTE,
+        INSCRIPT_ANTISLASH_IN_QUOTE2,
+        INSCRIPT_DEFAULT = 256
+      } INSCRIPT;
       INSCRIPT inscript_state_pos = INSCRIPT_START;
       const char *inscript_name = NULL;       // script tag name
       int inscript_tag = 0;     // on est dans un <body onLoad="... terminé par >
@@ -404,8 +408,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
       inscript_state[INSCRIPT_COMMENT2]['*'] = INSCRIPT_COMMENT2;
       inscript_state[INSCRIPT_ANTISLASH_IN_QUOTE][INSCRIPT_DEFAULT] = INSCRIPT_INQUOTE; /* #8: escape in '' */
       inscript_state[INSCRIPT_ANTISLASH_IN_QUOTE2][INSCRIPT_DEFAULT] = INSCRIPT_INQUOTE2;       /* #9: escape in "" */
-      const script_automate saut = {&inscript, inscript_state,
-                                    &inscript_state_pos, &html};
 
       /* Primary list or URLs */
       if (ptr == 0) {
@@ -844,7 +846,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             }
 
             /* automate */
-            hts_automate_lookup(&saut);
+            AUTOMATE_LOOKUP_CURRENT_ADR();
 
             // Note:
             // Certaines pages ne respectent pas le html
@@ -1760,7 +1762,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             }
             // sauter espaces
             // adr+=p;
-            hts_automate_increment(&saut, p);
+            INCREMENT_CURRENT_ADR(p);
             while((is_space(*html)
                    || (inscriptgen && html[0] == '\\' && is_space(html[1])
                    )
@@ -1775,7 +1777,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 }
               // puis quitter
               // html++;    // sauter les espaces, "" et cie
-              hts_automate_increment(&saut, 1);
+              INCREMENT_CURRENT_ADR(1);
             }
 
             /* Stop at \n (LF) if primary links or link lists */
@@ -1790,7 +1792,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               if (*html == '\\') {
                 if ((*(html + 1) == '\'') || (*(html + 1) == '"')) {      // \" ou \'
                   // html+=2;    // sauter
-                  hts_automate_increment(&saut, 2);
+                  INCREMENT_CURRENT_ADR(2);
                 }
               }
             }
@@ -1838,7 +1840,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             if (srcset_p) {
               while(html < r->adr + r->size
                     && (is_realspace(*html) || *html == ','))
-                hts_automate_increment(&saut, 1);
+                INCREMENT_CURRENT_ADR(1);
             }
             eadr = html;
 
@@ -3298,7 +3300,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
             assertf(eadr - html >= 0);   // Should not go back
             if (eadr > html) {
-              hts_automate_increment(&saut, (int) (eadr - 1 - html));
+              INCREMENT_CURRENT_ADR(eadr - 1 - html);
             }
             // adr=eadr-1;  // ** sauter
 
@@ -3317,8 +3319,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   q++;          // skip whitespace and empty candidates
                 if (q < endp && *q != '\0' && *q != ',' && *q != quote
                     && *q != '<' && *q != '>' && (unsigned char) *q >= 32) {
-                  hts_automate_increment(
-                      &saut, (int) (q - html)); // keep the automate in sync
+                  INCREMENT_CURRENT_ADR(q - html);   // keep the automate in sync
                   ok = 1;
                   goto srcset_next;
                 }

tests/30_local-fragment-link.test

@@ -1,13 +0,0 @@
-#!/bin/bash
-# Issue #279: an anchored link (target.html#sec, quoted or bare) fetches the
-# target with the fragment dropped (strict server 400s on a '#' in the request)
-# but keeps it in the rewritten local link so the anchor still works.
-set -e
-
-: "${top_srcdir:=..}"
-
-bash "$top_srcdir/tests/local-crawl.sh" --errors 0 \
-    --found 'fraglink/target.html' \
-    --file-matches 'fraglink/index.html' 'href=target\.html#sec' \
-    --file-matches 'fraglink/index.html' 'href="target\.html#sec2"' \
-    httrack 'BASEURL/fraglink/index.html'

tests/Makefile.am

@@ -6,7 +6,6 @@ EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
 	local-crawl.sh local-server.py server.crt server.key \
 	server-root/simple/basic.html server-root/simple/link.html \
 	server-root/stripquery/index.html server-root/stripquery/a.html \
-	server-root/fraglink/index.html server-root/fraglink/target.html \
 	fixtures/cache-golden/hts-cache/new.zip
 
 TESTS_ENVIRONMENT =
@@ -84,7 +83,6 @@ TESTS = \
 	26_local-strip-query.test \
 	27_local-cookies-file.test \
 	28_local-pause.test \
-	29_local-redirect-fragment.test \
-	30_local-fragment-link.test
+	29_local-redirect-fragment.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -15,11 +15,8 @@
 #   bash local-crawl.sh [--tls] [--root DIR] [--cookie NAME=VALUE ...] \
 #       --errors N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
-#       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
 #       httrack BASEURL/some/path [httrack-args...]
 # --log-found/--log-not-found grep (ERE) the crawl's hts-log.txt.
-# --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
-# host root), to assert rewritten link/content survived the crawl.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
 # which the ephemeral port forces into the cookie domain) and passes it to
 # httrack via --cookies-file, to exercise preloaded cookies.
@@ -124,10 +121,6 @@ while test "$pos" -lt "$nargs"; do
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
         ;;
-    --file-matches | --file-not-matches)
-        audit+=("${args[$pos]}" "${args[$((pos + 1))]}" "${args[$((pos + 2))]}")
-        pos=$((pos + 2))
-        ;;
     httrack)
         pos=$((pos + 1))
         break
@@ -301,24 +294,6 @@ while test "$i" -lt "${#audit[@]}"; do
             exit 1
         else result "OK"; fi
         ;;
-    --file-matches)
-        path="${audit[$((i + 1))]}"
-        i=$((i + 2))
-        info "checking ${path} matches ${audit[$i]}"
-        if grep -aqE "${audit[$i]}" "${hostroot}/${path}"; then result "OK"; else
-            result "no match"
-            exit 1
-        fi
-        ;;
-    --file-not-matches)
-        path="${audit[$((i + 1))]}"
-        i=$((i + 2))
-        info "checking ${path} lacks ${audit[$i]}"
-        if grep -aqE "${audit[$i]}" "${hostroot}/${path}"; then
-            result "matched"
-            exit 1
-        else result "OK"; fi
-        ;;
     esac
     i=$((i + 1))
 done

tests/server-root/fraglink/index.html

@@ -1,4 +0,0 @@
-<html><body>
-<a href=target.html#sec>unquoted fragment link</a>
-<a href="target.html#sec2">quoted fragment link</a>
-</body></html>

tests/server-root/fraglink/target.html

@@ -1 +0,0 @@
-<html><body><a name="sec"></a><a name="sec2"></a>target</body></html>