entities: harden the generator collision guard and widen test coverage

Review follow-up. The switch keys on the hash alone, so check hash-alone
uniqueness among emitted names (a same-hash/different-len pair would otherwise
slip the old (hash,len) check and surface only as a cryptic duplicate-case
compile error). Also hash the ~93 skipped multi-codepoint names and abort if any
aliases an emitted hash, so "skipped means verbatim" is enforced rather than
assumed on future regens.

Add a runtime sweep of common HTML4 names (copy/reg/trade/mdash/ndash/alpha/beta)
to 01_engine-entities.test: a regression guard against accidental drops and a
generator-vs-consumer hash cross-check on names beyond the handful already
probed.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/ci.yml

@@ -61,50 +61,6 @@ jobs:
         if: failure()
         run: cat tests/test-suite.log 2>/dev/null || true
 
-  # Reproduce the Debian buildds: they build in a minimal chroot with no
-  # python3, so the local-server tests must SKIP (exit 77), not fail. GitHub
-  # runners ship python3, so every other job hides this path; here we remove it
-  # before `make check`. This is the guard that would have caught the 3.49.10-1
-  # FTBFS (28_local-pause failed instead of skipping when python3 was absent).
-  buildd-no-python3:
-    name: build (no python3, Debian buildd)
-    runs-on: ubuntu-24.04
-    steps:
-      - uses: actions/checkout@v6
-        with:
-          submodules: recursive
-
-      - name: Install build dependencies
-        run: |
-          set -euo pipefail
-          sudo apt-get update
-          sudo apt-get install -y --no-install-recommends \
-            build-essential autoconf automake libtool autoconf-archive \
-            zlib1g-dev libssl-dev
-
-      - name: Configure
-        run: |
-          set -euo pipefail
-          autoreconf -fi
-          ./configure
-
-      - name: Build
-        run: make -j"$(nproc)"
-
-      - name: Test without python3
-        run: |
-          set -euo pipefail
-          # Hide every python3* so `command -v python3` fails like it does in the
-          # buildd chroot; masking with /bin/false would still resolve.
-          sudo find /usr/bin /usr/local/bin -maxdepth 1 -name 'python3*' \
-            -exec mv {} {}.hidden \;
-          ! command -v python3
-          make check
-
-      - name: Print the test log on failure
-        if: failure()
-        run: cat tests/test-suite.log 2>/dev/null || true
-
   # Portability: build and test on macOS (Darwin/clang) on a native runner --
   # no VM. The tree has no __APPLE__ branches, so Darwin exercises the
   # generic-Unix path on a second libc and kernel. brew's openssl@3 is keg-only,

src/htsglobal.h

@@ -229,10 +229,6 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DEFAULT_FOOTER                                                     \
   "<!-- Mirrored from %s%s by HTTrack Website Copier/" HTTRACK_AFF_VERSION     \
   " " HTTRACK_AFF_AUTHORS ", %s -->"
-/* Honest crawler User-Agent; no fake OS/browser to go stale. */
-#define HTS_DEFAULT_USER_AGENT                                                 \
-  "Mozilla/5.0 (compatible; HTTrack/" HTTRACK_AFF_VERSION                      \
-  "; +https://www.httrack.com/)"
 #define HTTRACK_WEB "http://www.httrack.com"
 #define HTS_UPDATE_WEBSITE                                                     \
   "http://www.httrack.com/"                                                    \

src/htslib.c

@@ -563,39 +563,6 @@ const char *hts_mime[][2] = {
   {"", ""}
 };
 
-/* Modern web formats (post-2010), kept in their own table: appending to the
-   legacy hts_mime[] above makes clang-format reflow its whole initializer.
-   Scanned after hts_mime[], so it never shadows a legacy mapping. */
-static const char *hts_mime_modern[][2] = {
-    {"image/webp", "webp"},
-    {"image/avif", "avif"},
-    {"image/heic", "heic"},
-    {"font/woff", "woff"},
-    {"font/woff2", "woff2"},
-    {"font/ttf", "ttf"},
-    {"font/otf", "otf"},
-    {"application/json", "json"},
-    {"application/ld+json", "jsonld"},
-    {"application/manifest+json", "webmanifest"},
-    {"application/wasm", "wasm"},
-    {"text/javascript", "js"},
-    {"text/javascript", "mjs"},
-    {"text/markdown", "md"},
-    {"video/mp4", "mp4"},
-    {"video/webm", "webm"},
-    {"video/ogg", "ogv"},
-    {"video/mp2t", "ts"},
-    {"audio/mp4", "m4a"},
-    {"audio/aac", "aac"},
-    {"audio/ogg", "oga"},
-    {"audio/opus", "opus"},
-    {"audio/flac", "flac"},
-    {"audio/webm", "weba"},
-    {"application/x-7z-compressed", "7z"},
-    {"application/x-rar-compressed", "rar"},
-    {"application/zstd", "zst"},
-    {"", ""}};
-
 // Reserved (RFC2396)
 #define CIS(c,ch) ( ((unsigned char)(c)) == (ch) )
 #define CHAR_RESERVED(c)  ( CIS(c,';') \
@@ -4341,20 +4308,6 @@ void guess_httptype(httrackp * opt, char *s, const char *fil) {
   (void) get_httptype_sized(opt, s, HTS_MIMETYPE_SIZE, fil, 1);
 }
 
-// first match in a NUL-terminated {mime,ext} table. key selects the lookup
-// column (0=mime, 1=ext); returns the other column, or NULL if no row matches
-// (a "*" partner means the row carries no value).
-static const char *hts_mime_lookup(const char *(*table)[2], int key,
-                                   const char *needle) {
-  int j;
-
-  for (j = 0; strnotempty(table[j][1]); j++) {
-    if (strfield2(table[j][key], needle) && table[j][!key][0] != '*')
-      return table[j][!key];
-  }
-  return NULL;
-}
-
 // write the mime type for fil into s (capacity ssize)
 // flag: 1 to always return a type (the "application/..." / octet-stream
 // fallback) returns 1 if a type was written to s, 0 otherwise
@@ -4378,15 +4331,17 @@ HTSEXT_API hts_boolean get_httptype_sized(httrackp *opt, char *s, size_t ssize,
     while ((a > fil) && (*a != '.') && (*a != '/'))
       a--;
     if (a >= fil && *a == '.' && strlen(a) < 32) {
-      const char *mime;
+      int j = 0;
 
       a++;
-      mime = hts_mime_lookup(hts_mime, 1, a);
-      if (mime == NULL)
-        mime = hts_mime_lookup(hts_mime_modern, 1, a);
-      if (mime != NULL) {
-        strlcpybuff(s, mime, ssize);
-        return 1;
+      while(strnotempty(hts_mime[j][1])) {
+        if (strfield2(hts_mime[j][1], a)) {
+          if (hts_mime[j][0][0] != '*') { // a match exists
+            strlcpybuff(s, hts_mime[j][0], ssize);
+            return 1;
+          }
+        }
+        j++;
       }
 
       if (flag) {
@@ -4521,16 +4476,18 @@ int get_userhttptype(httrackp * opt, char *s, const char *fil) {
 // returns 1 if an extension was found (and written to s), 0 otherwise
 int give_mimext(char *s, size_t ssize, const char *st) {
   int ok = 0;
-  const char *ext;
+  int j = 0;
 
   st = hts_effective_mime(st); /* no declared type: derive an html ext */
   s[0] = '\0';
-  ext = hts_mime_lookup(hts_mime, 0, st);
-  if (ext == NULL)
-    ext = hts_mime_lookup(hts_mime_modern, 0, st);
-  if (ext != NULL) {
-    strlcpybuff(s, ext, ssize);
-    ok = 1;
+  while((!ok) && (strnotempty(hts_mime[j][1]))) {
+    if (strfield2(hts_mime[j][0], st)) {
+      if (hts_mime[j][1][0] != '*') { // a match exists
+        strlcpybuff(s, hts_mime[j][1], ssize);
+        ok = 1;
+      }
+    }
+    j++;
   }
   // wrap "x" mimetypes, such as:
   // application/x-mp3
@@ -6048,7 +6005,8 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->shell = HTS_FALSE;
   opt->proxy.active = 0;        // pas de proxy
   opt->user_agent_send = HTS_TRUE;
-  StringCopy(opt->user_agent, HTS_DEFAULT_USER_AGENT);
+  StringCopy(opt->user_agent,
+             "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)");
   StringCopy(opt->referer, "");
   StringCopy(opt->from, "");
   opt->savename_83 = HTS_SAVENAME_83_LONG; // long names by default

src/htsselftest.c

@@ -239,14 +239,6 @@ static void basic_selftests(void) {
     assertf(strcmp(ext, "html") == 0);
     assertf(give_mimext(ext, sizeof(ext), "no/such-mime-type") == 0);
     assertf(ext[0] == '\0');
-    // modern web formats -> extension. Avoid MIME types the
-    // application/<=4-char-subtype fallback could fabricate without a row.
-    assertf(give_mimext(ext, sizeof(ext), "image/webp") == 1);
-    assertf(strcmp(ext, "webp") == 0);
-    assertf(give_mimext(ext, sizeof(ext), "application/manifest+json") == 1);
-    assertf(strcmp(ext, "webmanifest") == 0);
-    assertf(give_mimext(ext, sizeof(ext), "font/woff2") == 1);
-    assertf(strcmp(ext, "woff2") == 0);
   }
   // convtolower(): lower-cases into the caller buffer (bounded by its size).
   {
@@ -301,16 +293,6 @@ static void basic_selftests(void) {
     assertf(get_httptype_sized(opt, r.contenttype, sizeof(r.contenttype),
                                "x.gif", 0) == 1);
     assertf(strcmp(r.contenttype, "image/gif") == 0);
-    // modern extensions map back to their MIME type
-    assertf(get_httptype_sized(opt, r.contenttype, sizeof(r.contenttype),
-                               "x.webp", 0) == 1);
-    assertf(strcmp(r.contenttype, "image/webp") == 0);
-    assertf(get_httptype_sized(opt, r.contenttype, sizeof(r.contenttype),
-                               "app.wasm", 0) == 1);
-    assertf(strcmp(r.contenttype, "application/wasm") == 0);
-    assertf(get_httptype_sized(opt, r.contenttype, sizeof(r.contenttype),
-                               "mod.mjs", 0) == 1);
-    assertf(strcmp(r.contenttype, "text/javascript") == 0);
     // no extension and flag==0: nothing written, returns 0
     assertf(get_httptype_sized(opt, r.contenttype, sizeof(r.contenttype),
                                "noextfile", 0) == 0);
@@ -1302,22 +1284,6 @@ static int st_urlhack(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* Default User-Agent: honest HTTrack token, no resurrected Windows 98. */
-static int st_useragent(httrackp *opt, int argc, char **argv) {
-  const char *ua = StringBuff(opt->user_agent);
-  (void) argc;
-  (void) argv;
-  assertf(ua != NULL);
-  assertf(strcmp(ua, HTS_DEFAULT_USER_AGENT) == 0);
-  /* Teeth independent of the macro: honest token + self-identifier, and no
-     legacy Mozilla/4.x fake-browser string (rejects the whole relic family). */
-  assertf(strstr(ua, "HTTrack/") != NULL);
-  assertf(strstr(ua, "+https://www.httrack.com/") != NULL);
-  assertf(strstr(ua, "Mozilla/4.") == NULL);
-  printf("useragent self-test OK: %s\n", ua);
-  return 0;
-}
-
 /* ------------------------------------------------------------ */
 /* Registry: name -> handler, with a usage hint and a one-line description. */
 /* ------------------------------------------------------------ */
@@ -1364,7 +1330,6 @@ static const struct selftest_entry {
      st_cache_writefail},
     {"dns", "", "DNS resolver/cache self-test", st_dns},
     {"cookies", "", "cookie request-header self-test", st_cookies},
-    {"useragent", "", "default User-Agent self-test", st_useragent},
 };
 
 static void list_selftests(void) {

src/htsserver.c

@@ -358,12 +358,12 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
       {NULL, 0}
     };
     initStrElt initStr[] = {
-        {"user", HTS_DEFAULT_USER_AGENT},
-        {"footer", "<!-- Mirrored from %s%s by HTTrack Website Copier/3.x "
-                   "[XR&CO'2014], %s -->"},
-        {"url2",
-         "+*.png +*.gif +*.jpg +*.jpeg +*.css +*.js -ad.doubleclick.net/*"},
-        {NULL, NULL}};
+      {"user", "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"},
+      {"footer",
+       "<!-- Mirrored from %s%s by HTTrack Website Copier/3.x [XR&CO'2014], %s -->"},
+      {"url2", "+*.png +*.gif +*.jpg +*.jpeg +*.css +*.js -ad.doubleclick.net/*"},
+      {NULL, NULL}
+    };
     int i = 0;
 
     for(i = 0; initInt[i].name; i++) {

src/htswizard.c

@@ -80,10 +80,6 @@ htspair_t hts_detect_embed[] = {
   {NULL, NULL}
 };
 
-/* HTML5 media siblings of <img src>: same near-link treatment (#451) */
-static const htspair_t hts_detect_embed_html5[] = {
-    {"source", "src"}, {"source", "srcset"}, {"track", "src"}, {NULL, NULL}};
-
 /* Internal */
 static int hts_acceptlink_(httrackp * opt, int ptr, const char *adr,
                            const char *fil, const char *tag,
@@ -140,17 +136,6 @@ static int cmp_token(const char *tag, const char *cmp) {
           && !isalnum((unsigned char) tag[p]));
 }
 
-/* TRUE if (tag, attribute) matches an embedded-asset pair in the table */
-static hts_boolean is_embed_pair(const htspair_t *table, const char *tag,
-                                 const char *attribute) {
-  int i;
-  for (i = 0; table[i].tag != NULL; i++) {
-    if (cmp_token(tag, table[i].tag) && cmp_token(attribute, table[i].attr))
-      return HTS_TRUE;
-  }
-  return HTS_FALSE;
-}
-
 static int hts_acceptlink_(httrackp * opt, int ptr,
                            const char *adr, const char *fil, const char *tag,
                            const char *attribute, int *set_prio_to,
@@ -178,9 +163,15 @@ static int hts_acceptlink_(httrackp * opt, int ptr,
 
   /* Built-in known tags (<img src=..>, ..) */
   if (forbidden_url != 0 && opt->nearlink && tag != NULL && attribute != NULL) {
-    if (is_embed_pair(hts_detect_embed, tag, attribute) ||
-        is_embed_pair(hts_detect_embed_html5, tag, attribute)) {
-      embedded_triggered = 1;
+    int i;
+
+    for(i = 0; hts_detect_embed[i].tag != NULL; i++) {
+      if (cmp_token(tag, hts_detect_embed[i].tag)
+          && cmp_token(attribute, hts_detect_embed[i].attr)
+        ) {
+        embedded_triggered = 1;
+        break;
+      }
     }
   }
 

tests/01_engine-parse.test

@@ -323,33 +323,4 @@ grep -Fq 'href="ahref%20(4).gif"' "$saved9" ||
 ! grep -Eq '(src|href)="[^"]*%28' "$saved9" ||
     ! echo "FAIL #163: gate over-fired onto a non-url() attribute link" || exit 1
 
-# HTML5 <source>/<track> follow as embedded near-links past the -r2 depth boundary (#451).
-# img.gif positive control; plain.gif (bare <a href>) negative control proves the gate is selective.
-site10="$tmp/html5media"
-mkdir -p "$site10"
-for f in img ss plain; do gif "$site10/$f.gif"; done
-printf 'x' >"$site10/v.webm"
-printf 'x' >"$site10/subs.vtt"
-cat >"$site10/index.html" <<EOF
-<html><body><a href="leaf.html">leaf</a></body></html>
-EOF
-cat >"$site10/leaf.html" <<EOF
-<html><body>
-<img src="img.gif">
-<picture><source srcset="ss.gif 2x"></picture>
-<video><source src="v.webm"></video>
-<video><track src="subs.vtt"></video>
-<a href="plain.gif">plain link past the boundary</a>
-</body></html>
-EOF
-out10="$tmp/html5media-out"
-rm -rf "$out10"
-mkdir -p "$out10"
-httrack "file://$site10/index.html" -O "$out10" --quiet --near -r2 >"$out10/.log" 2>&1 || true
-found "img.gif" "$out10"
-found "ss.gif" "$out10"
-found "v.webm" "$out10"
-found "subs.vtt" "$out10"
-notfound "plain.gif" "$out10"
-
 exit 0

tests/01_engine-useragent.test

@@ -1,7 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# Default User-Agent (#449): honest HTTrack token, no Windows 98 relic.
-httrack -O /dev/null -#test=useragent run | grep -q "useragent self-test OK"

tests/28_local-pause.test

@@ -9,13 +9,6 @@ set -e
 
 : "${top_srcdir:=..}"
 
-# python3 runs the local server (mirror local-crawl.sh); skip when absent, else
-# run() swallows its exit-77 and the serverless 0s/0s crawl looks like a fail.
-command -v python3 >/dev/null || {
-    echo "python3 not found; skipping local crawl tests"
-    exit 77
-}
-
 run() { # echoes the wall-clock seconds of one crawl
     local t0 t1
     t0=$(date +%s)

tests/Makefile.am

@@ -50,7 +50,6 @@ TESTS = \
 	01_engine-stripquery.test \
 	01_engine-strsafe.test \
 	01_engine-urlhack.test \
-	01_engine-useragent.test \
 	02_manpage-regen.test \
 	02_update-cache.test \
 	10_crawl-simple.test \