htsparse: keep makestat_time out of ENGINE_SET_CONTEXT

makestat_time throttles the makestat/maketrack stats to once per minute:
the wait loop compares time_local() against it and, when it fires, writes
it back to the local. But the field is by-value in the extended context,
so it can't round-trip through ENGINE_SAVE_CONTEXT, while ENGINE_SET_CONTEXT
re-read it from the load-once baseline on every loop iteration. That reset
the local before the next compare, so under -%v / maketrack the throttle
never held and the stats line plus the full back-stack dump were emitted
every iteration.

Drop makestat_time (and the never-changing makestat_fp) from SET_CONTEXT;
they belong to the load-once set. Wrapped the macro in clang-format off/on
for the same backslash-realignment reason as HT_ADD_END.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

debian/control

@@ -1,8 +1,9 @@
 Source: httrack
 Section: web
+Priority: optional
 Maintainer: Xavier Roche <roche@httrack.com>
 Standards-Version: 4.7.4
-Build-Depends: debhelper-compat (= 14), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
+Build-Depends: debhelper-compat (= 13), autoconf, autoconf-archive, automake, libtool, zlib1g-dev, libssl-dev
 Rules-Requires-Root: no
 Homepage: http://www.httrack.com
 Vcs-Git: https://github.com/xroche/httrack.git

debian/source/lintian-overrides

@@ -1,6 +1,4 @@
-# Maintainer uploads sign the changelog as xavier@debian.org while the control
-# Maintainer is roche@httrack.com; lintian reads the address mismatch as an NMU.
-httrack source: no-nmu-in-changelog
+httrack source: changelog-should-mention-nmu
 httrack source: source-nmu-has-incorrect-version-number
 
 # The bundled HTML pages are the genuine upstream documentation taken from

debian/upstream/metadata

@@ -1,6 +0,0 @@
----
-Repository: https://github.com/xroche/httrack.git
-Repository-Browse: https://github.com/xroche/httrack
-Bug-Database: https://github.com/xroche/httrack/issues
-Bug-Submit: https://github.com/xroche/httrack/issues/new
-Contact: Xavier Roche <roche@httrack.com>

src/htscore.c

@@ -441,72 +441,6 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
   *makeindex_done = 1;
 }
 
-/* Flush the parsed HTML output buffer to disk, skipping the rewrite when the
- * on-disk MD5 is unchanged. */
-void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
-                          FILE **fp, const char *ht_buff, size_t ht_len,
-                          const char *adr, const char *fil, const char *save) {
-  char digest[32 + 2];
-  off_t fsize_old =
-      fsize(fconv(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), save));
-  int ok = 0;
-
-  digest[0] = '\0';
-  domd5mem(ht_buff, ht_len, digest, 1);
-  if (fsize_old == (off_t) ht_len) {
-    int mlen = 0;
-    char *mbuff;
-
-    cache_readdata(cache, "//[HTML-MD5]//", save, &mbuff, &mlen);
-    if (mlen)
-      mbuff[mlen] = '\0';
-    if ((mlen == 32) && (strcmp(((mbuff != NULL) ? mbuff : ""), digest) == 0)) {
-      ok = 1;
-      hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s", save);
-    }
-    freet(mbuff);
-  }
-  if (!ok) {
-    file_notify(opt, adr, fil, save, 1, 1, r->notmodified);
-    *fp = filecreate(&opt->state.strc, save);
-    if (*fp) {
-      if (ht_len > 0 && fwrite(ht_buff, 1, ht_len, *fp) != ht_len) {
-        int fcheck = check_fatal_io_errno();
-
-        if (fcheck)
-          opt->state.exit_xh = -1;
-        if (opt->log) {
-          hts_log_print(opt, LOG_ERROR | LOG_ERRNO,
-                        "Unable to write HTML file %s", save);
-          if (fcheck)
-            hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
-        }
-      }
-      fclose(*fp);
-      *fp = NULL;
-      if (strnotempty(r->lastmodified))
-        set_filetime_rfc822(save, r->lastmodified);
-    } else {
-      int fcheck = check_fatal_io_errno();
-
-      if (fcheck) {
-        hts_log_print(opt, LOG_ERROR,
-                      "Mirror aborted: disk full or filesystem problems");
-        opt->state.exit_xh = -1;
-      }
-      hts_log_print(opt, LOG_ERROR | LOG_ERRNO, "Unable to save file %s", save);
-      if (fcheck)
-        hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");
-    }
-  } else {
-    file_notify(opt, adr, fil, save, 0, 0, r->notmodified);
-    filenote(&opt->state.strc, save, NULL);
-  }
-  if (cache->ndx)
-    cache_writedata(cache->ndx, cache->dat, "//[HTML-MD5]//", save, digest,
-                    (int) strlen(digest));
-}
-
 /* does it look like XML ? (SVG et al.) */
 static int look_like_xml(const char *s) {
   return strncmp(s, "<?xml", 5) == 0

src/htscore.h

@@ -370,12 +370,6 @@ void hts_finish_makeindex(httrackp *opt, int *makeindex_done,
                           const char *template_footer, const char *adr,
                           const char *fil);
 
-// Flush ht_buff[0..ht_len] to save on disk (skip if MD5 unchanged); *fp
-// closed+NULLed on write. Precondition: ht_len>0.
-void hts_finish_html_file(httrackp *opt, cache_back *cache, htsblk *r,
-                          FILE **fp, const char *ht_buff, size_t ht_len,
-                          const char *adr, const char *fil, const char *save);
-
 int filters_init(char ***ptrfilters, int maxfilter, int filterinc);
 
 int fspc(httrackp * opt, FILE * fp, const char *type);

src/htsparse.c

@@ -106,99 +106,159 @@ Please visit our Website: http://www.httrack.com
 // does nothing
 #define XH_uninit do {} while(0)
 
-#define HT_ADD_FOP
-
-/* Mutable extended-context fields: one source of truth so the DEFINE/SET/SAVE
-   load and store lists can't drift apart. */
+/* clang-format off: an edit realigns all backslashes, churning the macro. */
 /* clang-format off */
-#define ENGINE_MUTABLE_FIELDS(X) \
-  X(int, error, stre->error_) \
-  X(int, store_errpage, stre->store_errpage_) \
-  X(int, makeindex_done, stre->makeindex_done_) \
-  X(FILE *, makeindex_fp, stre->makeindex_fp_) \
-  X(int, makeindex_links, stre->makeindex_links_) \
-  X(LLint, stat_fragment, stre->stat_fragment_)
-
-#define ENGINE_FIELD_DECLARE(type, name, src) type name = *(src);
-#define ENGINE_FIELD_LOAD(type, name, src) name = *(src);
-#define ENGINE_FIELD_STORE(type, name, src) *(src) = name;
+#define HT_ADD_END { \
+  int ok=0;\
+  if (TypedArraySize(output_buffer) != 0) { \
+    const size_t ht_len = TypedArraySize(output_buffer); \
+    const char *const ht_buff = TypedArrayElts(output_buffer); \
+    char digest[32+2];\
+    off_t fsize_old = fsize(fconv(OPT_GET_BUFF(opt),OPT_GET_BUFF_SIZE(opt),savename()));\
+    digest[0] = '\0';\
+    domd5mem(TypedArrayElts(output_buffer), ht_len, digest, 1);\
+    if (fsize_old == (off_t) ht_len) { \
+      int mlen = 0;\
+      char* mbuff;\
+      cache_readdata(cache,"//[HTML-MD5]//",savename(),&mbuff,&mlen);\
+      if (mlen) \
+        mbuff[mlen]='\0';\
+      if ((mlen == 32) && (strcmp(((mbuff!=NULL)?mbuff:""),digest)==0)) {\
+        ok=1;\
+        hts_log_print(opt, LOG_DEBUG, "File not re-written (md5): %s",savename());\
+      } else {\
+        ok=0;\
+      } \
+      freet(mbuff);\
+    }\
+    if (!ok) { \
+      file_notify(opt,urladr(), urlfil(), savename(), 1, 1, r->notmodified); \
+      fp=filecreate(&opt->state.strc, savename()); \
+      if (fp) { \
+        if (ht_len>0) {\
+        if (fwrite(ht_buff,1,ht_len,fp) != ht_len) { \
+          int fcheck;\
+          if ((fcheck=check_fatal_io_errno())) {\
+            opt->state.exit_xh=-1;\
+          }\
+          if (opt->log) {   \
+            hts_log_print(opt, LOG_ERROR | LOG_ERRNO, "Unable to write HTML file %s", savename());\
+            if (fcheck) {\
+              hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");\
+            }\
+          }\
+        }\
+        }\
+        fclose(fp); fp=NULL; \
+        if (strnotempty(r->lastmodified)) \
+        set_filetime_rfc822(savename(),r->lastmodified); \
+      } else {\
+        int fcheck;\
+        if ((fcheck=check_fatal_io_errno())) {\
+  				hts_log_print(opt, LOG_ERROR, "Mirror aborted: disk full or filesystem problems"); \
+          opt->state.exit_xh=-1;\
+        }\
+        hts_log_print(opt, LOG_ERROR | LOG_ERRNO, "Unable to save file %s", savename());\
+        if (fcheck) {\
+          hts_log_print(opt, LOG_ERROR, "* * Fatal write error, giving up");\
+        }\
+      }\
+    } else {\
+      file_notify(opt,urladr(), urlfil(), savename(), 0, 0, r->notmodified); \
+      filenote(&opt->state.strc, savename(),NULL); \
+    }\
+    if (cache->ndx)\
+      cache_writedata(cache->ndx,cache->dat,"//[HTML-MD5]//",savename(),digest,(int)strlen(digest));\
+  } \
+  TypedArrayFree(output_buffer); \
+}
+/* clang-format on */
+#define HT_ADD_FOP
 
 #define ENGINE_DEFINE_CONTEXT() \
   ENGINE_DEFINE_CONTEXT_BASE(); \
+  /* */ \
   htsblk* const r HTS_UNUSED = stre->r_; \
   hash_struct* const hash HTS_UNUSED = stre->hash_; \
   char* const codebase HTS_UNUSED = stre->codebase; \
   char* const base HTS_UNUSED = stre->base; \
+  /* */ \
   const char * const template_header HTS_UNUSED = stre->template_header_; \
   const char * const template_body HTS_UNUSED = stre->template_body_; \
   const char * const template_footer HTS_UNUSED = stre->template_footer_; \
+  /* */ \
   HTS_UNUSED char* const makeindex_firstlink = stre->makeindex_firstlink_; \
-  ENGINE_MUTABLE_FIELDS(ENGINE_FIELD_DECLARE) \
-  /* load-once (kept out of SET/SAVE): re-reading would reset the throttle */ \
+  /* */ \
+  /* */ \
+  int error = * stre->error_; \
+  int store_errpage = * stre->store_errpage_; \
+  /* */ \
+  int makeindex_done = *stre->makeindex_done_; \
+  FILE* makeindex_fp = *stre->makeindex_fp_; \
+  int makeindex_links = *stre->makeindex_links_; \
+  /* */ \
+  LLint stat_fragment = *stre->stat_fragment_; \
   HTS_UNUSED TStamp makestat_time = stre->makestat_time; \
   HTS_UNUSED FILE* makestat_fp = stre->makestat_fp
 
+/* clang-format off: an edit realigns all backslashes, churning the macro. */
+/* clang-format off */
+/* Load-once: re-reading resets makestat_time (mutated locally, never SAVEd). */
 #define ENGINE_SET_CONTEXT() \
   ENGINE_SET_CONTEXT_BASE(); \
-  ENGINE_MUTABLE_FIELDS(ENGINE_FIELD_LOAD)
+  /* */ \
+  error = * stre->error_; \
+  store_errpage = * stre->store_errpage_; \
+  /* */ \
+  makeindex_done = *stre->makeindex_done_; \
+  makeindex_fp = *stre->makeindex_fp_; \
+  makeindex_links = *stre->makeindex_links_; \
+  /* */ \
+  stat_fragment = *stre->stat_fragment_
+/* clang-format on */
 
 #define ENGINE_LOAD_CONTEXT() \
   ENGINE_DEFINE_CONTEXT()
 
 #define ENGINE_SAVE_CONTEXT() \
   ENGINE_SAVE_CONTEXT_BASE(); \
-  ENGINE_MUTABLE_FIELDS(ENGINE_FIELD_STORE)
-/* clang-format on */
+  /* */ \
+  * stre->error_ = error; \
+  * stre->store_errpage_ = store_errpage; \
+  /* */ \
+  *stre->makeindex_done_ = makeindex_done; \
+  *stre->makeindex_fp_ = makeindex_fp; \
+  *stre->makeindex_links_ = makeindex_links; \
+  /* */ \
+  *stre->stat_fragment_ = stat_fragment
 
+#define _FILTERS     (*opt->filters.filters)
+#define _FILTERS_PTR (opt->filters.filptr)
 #define _ROBOTS      ((robots_wizard*)opt->robotsptr)
 
-/* JS-detection automaton states; INSCRIPT_DEFAULT is the synthetic "any other
-   char" column of the transition table. */
-typedef enum {
-  INSCRIPT_START = 0,
-  INSCRIPT_ANTISLASH,
-  INSCRIPT_INQUOTE,
-  INSCRIPT_INQUOTE2,
-  INSCRIPT_SLASH,
-  INSCRIPT_SLASHSLASH,
-  INSCRIPT_COMMENT,
-  INSCRIPT_COMMENT2,
-  INSCRIPT_ANTISLASH_IN_QUOTE,
-  INSCRIPT_ANTISLASH_IN_QUOTE2,
-  INSCRIPT_DEFAULT = 256
-} INSCRIPT;
+/* Apply current *adr character for the script automate */
+#define AUTOMATE_LOOKUP_CURRENT_ADR() do { \
+  if (inscript) { \
+  int new_state_pos; \
+  new_state_pos=inscript_state[inscript_state_pos][(unsigned char)*html]; \
+  if (new_state_pos < 0) { \
+  new_state_pos=inscript_state[inscript_state_pos][INSCRIPT_DEFAULT]; \
+  } \
+  assertf(new_state_pos >= 0); \
+  assertf(new_state_pos*sizeof(inscript_state[0]) < sizeof(inscript_state)); \
+  inscript_state_pos=new_state_pos; \
+  } \
+} while(0)
 
-#define INSCRIPT_NSTATES 10 /* rows in the transition table */
-
-/* Live view of the parser's automaton locals, set up once so the helpers below
-   can drive it without capturing them by lexical scope. */
-typedef struct {
-  const int *inscript;             /* nonzero while inside a script body */
-  const signed char (*table)[257]; /* [INSCRIPT_NSTATES][257] transitions */
-  INSCRIPT *pos;                   /* current state */
-  const char **html;               /* parse cursor */
-} script_automate;
-
-/* Feed the current *html byte to the automaton. No-op outside a script body. */
-static void hts_automate_lookup(const script_automate *aut) {
-  if (*aut->inscript) {
-    int next = aut->table[*aut->pos][(unsigned char) **aut->html];
-    if (next < 0) {
-      next = aut->table[*aut->pos][INSCRIPT_DEFAULT];
-    }
-    assertf(next >= 0 && next < INSCRIPT_NSTATES);
-    *aut->pos = (INSCRIPT) next;
-  }
-}
-
-/* Advance the cursor by 'steps' bytes, feeding each to the automaton. */
-static void hts_automate_increment(const script_automate *aut, int steps) {
-  while (steps > 0) {
-    (*aut->html)++;
-    hts_automate_lookup(aut);
-    steps--;
-  }
-}
+/* Increment current pointer to 'steps' characters, modifying automate if necessary */
+#define INCREMENT_CURRENT_ADR(steps) do { \
+  int steps__ = (int) ( steps ); \
+  while(steps__ > 0) { \
+  html++; \
+  AUTOMATE_LOOKUP_CURRENT_ADR(); \
+  steps__ --; \
+  } \
+} while(0)
 
 /* Percent-encode the angle brackets of a string so it is safe to embed inside
    an HTML comment (the default footer) or any other HTML context. A URL holding
@@ -343,7 +403,20 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
       int incomment = 0;        // dans un <!--
       int inscript = 0;         // dans un scipt pour applets javascript)
       int inscript_locked = 0;  // in locked script (ie. js file)
-      signed char inscript_state[INSCRIPT_NSTATES][257];
+      signed char inscript_state[10][257];
+      typedef enum {
+        INSCRIPT_START = 0,
+        INSCRIPT_ANTISLASH,
+        INSCRIPT_INQUOTE,
+        INSCRIPT_INQUOTE2,
+        INSCRIPT_SLASH,
+        INSCRIPT_SLASHSLASH,
+        INSCRIPT_COMMENT,
+        INSCRIPT_COMMENT2,
+        INSCRIPT_ANTISLASH_IN_QUOTE,
+        INSCRIPT_ANTISLASH_IN_QUOTE2,
+        INSCRIPT_DEFAULT = 256
+      } INSCRIPT;
       INSCRIPT inscript_state_pos = INSCRIPT_START;
       const char *inscript_name = NULL;       // script tag name
       int inscript_tag = 0;     // on est dans un <body onLoad="... terminé par >
@@ -404,8 +477,6 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
       inscript_state[INSCRIPT_COMMENT2]['*'] = INSCRIPT_COMMENT2;
       inscript_state[INSCRIPT_ANTISLASH_IN_QUOTE][INSCRIPT_DEFAULT] = INSCRIPT_INQUOTE; /* #8: escape in '' */
       inscript_state[INSCRIPT_ANTISLASH_IN_QUOTE2][INSCRIPT_DEFAULT] = INSCRIPT_INQUOTE2;       /* #9: escape in "" */
-      const script_automate saut = {&inscript, inscript_state,
-                                    &inscript_state_pos, &html};
 
       /* Primary list or URLs */
       if (ptr == 0) {
@@ -844,7 +915,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             }
 
             /* automate */
-            hts_automate_lookup(&saut);
+            AUTOMATE_LOOKUP_CURRENT_ADR();
 
             // Note:
             // Certaines pages ne respectent pas le html
@@ -1760,7 +1831,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             }
             // sauter espaces
             // adr+=p;
-            hts_automate_increment(&saut, p);
+            INCREMENT_CURRENT_ADR(p);
             while((is_space(*html)
                    || (inscriptgen && html[0] == '\\' && is_space(html[1])
                    )
@@ -1775,7 +1846,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                 }
               // puis quitter
               // html++;    // sauter les espaces, "" et cie
-              hts_automate_increment(&saut, 1);
+              INCREMENT_CURRENT_ADR(1);
             }
 
             /* Stop at \n (LF) if primary links or link lists */
@@ -1790,7 +1861,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
               if (*html == '\\') {
                 if ((*(html + 1) == '\'') || (*(html + 1) == '"')) {      // \" ou \'
                   // html+=2;    // sauter
-                  hts_automate_increment(&saut, 2);
+                  INCREMENT_CURRENT_ADR(2);
                 }
               }
             }
@@ -1838,7 +1909,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
             if (srcset_p) {
               while(html < r->adr + r->size
                     && (is_realspace(*html) || *html == ','))
-                hts_automate_increment(&saut, 1);
+                INCREMENT_CURRENT_ADR(1);
             }
             eadr = html;
 
@@ -3298,7 +3369,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
 
             assertf(eadr - html >= 0);   // Should not go back
             if (eadr > html) {
-              hts_automate_increment(&saut, (int) (eadr - 1 - html));
+              INCREMENT_CURRENT_ADR(eadr - 1 - html);
             }
             // adr=eadr-1;  // ** sauter
 
@@ -3317,8 +3388,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
                   q++;          // skip whitespace and empty candidates
                 if (q < endp && *q != '\0' && *q != ',' && *q != quote
                     && *q != '<' && *q != '>' && (unsigned char) *q >= 32) {
-                  hts_automate_increment(
-                      &saut, (int) (q - html)); // keep the automate in sync
+                  INCREMENT_CURRENT_ADR(q - html);   // keep the automate in sync
                   ok = 1;
                   goto srcset_next;
                 }
@@ -3458,12 +3528,7 @@ int htsparse(htsmoduleStruct * str, htsmoduleStructExtended * stre) {
         }
 
         /* Flush and save to disk */
-        if (TypedArraySize(output_buffer) != 0) {
-          hts_finish_html_file(
-              opt, cache, r, &fp, TypedArrayElts(output_buffer),
-              TypedArraySize(output_buffer), urladr(), urlfil(), savename());
-        }
-        TypedArrayFree(output_buffer);
+        HT_ADD_END;             // achever
       }
       //
       //