Modernize the default User-Agent (#449 )

The default UA was "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)", a 25-year-old string naming a dead OS and trivially fingerprinted. Replace it with an honest crawler UA carrying the HTTrack token and a reference URL: "Mozilla/5.0 (compatible; HTTrack/3.x; +https://www.httrack.com/)". Both the engine default (hts_create_opt) and the webhttrack mini-server config default now share one HTS_DEFAULT_USER_AGENT macro, built from HTTRACK_AFF_VERSION so the version token tracks releases without going stale. Closes #449 Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com>
2026-06-30 14:05:47 +03:00 · 2026-06-28 23:58:59 +02:00
3 changed files with 10 additions and 49 deletions
--- a/src/htsselftest.c
+++ b/src/htsselftest.c
@@ -1309,8 +1309,7 @@ static int st_useragent(httrackp *opt, int argc, char **argv) {
  (void) argv;
  assertf(ua != NULL);
  assertf(strcmp(ua, HTS_DEFAULT_USER_AGENT) == 0);
-  /* Teeth independent of the macro: honest token + self-identifier, and no
-     legacy Mozilla/4.x fake-browser string (rejects the whole relic family). */
+  /* Macro-independent teeth: token, self-id URL, no Mozilla/4.x. */
  assertf(strstr(ua, "HTTrack/") != NULL);
  assertf(strstr(ua, "+https://www.httrack.com/") != NULL);
  assertf(strstr(ua, "Mozilla/4.") == NULL);
--- a/src/htswizard.c
+++ b/src/htswizard.c
@@ -80,10 +80,6 @@ htspair_t hts_detect_embed[] = {
  {NULL, NULL}
 };

-/* HTML5 media siblings of <img src>: same near-link treatment (#451) */
-static const htspair_t hts_detect_embed_html5[] = {
-    {"source", "src"}, {"source", "srcset"}, {"track", "src"}, {NULL, NULL}};
-
 /* Internal */
 static int hts_acceptlink_(httrackp * opt, int ptr, const char *adr,
                           const char *fil, const char *tag,
@@ -140,17 +136,6 @@ static int cmp_token(const char *tag, const char *cmp) {
          && !isalnum((unsigned char) tag[p]));
 }

-/* TRUE if (tag, attribute) matches an embedded-asset pair in the table */
-static hts_boolean is_embed_pair(const htspair_t *table, const char *tag,
-                                 const char *attribute) {
-  int i;
-  for (i = 0; table[i].tag != NULL; i++) {
-    if (cmp_token(tag, table[i].tag) && cmp_token(attribute, table[i].attr))
-      return HTS_TRUE;
-  }
-  return HTS_FALSE;
-}
-
 static int hts_acceptlink_(httrackp * opt, int ptr,
                           const char *adr, const char *fil, const char *tag,
                           const char *attribute, int *set_prio_to,
@@ -178,9 +163,15 @@ static int hts_acceptlink_(httrackp * opt, int ptr,

  /* Built-in known tags (<img src=..>, ..) */
  if (forbidden_url != 0 && opt->nearlink && tag != NULL && attribute != NULL) {
-    if (is_embed_pair(hts_detect_embed, tag, attribute) ||
-        is_embed_pair(hts_detect_embed_html5, tag, attribute)) {
-      embedded_triggered = 1;
+    int i;
+
+    for(i = 0; hts_detect_embed[i].tag != NULL; i++) {
+      if (cmp_token(tag, hts_detect_embed[i].tag)
+          && cmp_token(attribute, hts_detect_embed[i].attr)
+        ) {
+        embedded_triggered = 1;
+        break;
+      }
    }
  }

--- a/tests/01_engine-parse.test
+++ b/tests/01_engine-parse.test
@@ -323,33 +323,4 @@ grep -Fq 'href="ahref%20(4).gif"' "$saved9" ||
 ! grep -Eq '(src|href)="[^"]*%28' "$saved9" ||
    ! echo "FAIL #163: gate over-fired onto a non-url() attribute link" || exit 1

-# HTML5 <source>/<track> follow as embedded near-links past the -r2 depth boundary (#451).
-# img.gif positive control; plain.gif (bare <a href>) negative control proves the gate is selective.
-site10="$tmp/html5media"
-mkdir -p "$site10"
-for f in img ss plain; do gif "$site10/$f.gif"; done
-printf 'x' >"$site10/v.webm"
-printf 'x' >"$site10/subs.vtt"
-cat >"$site10/index.html" <<EOF
-<html><body><a href="leaf.html">leaf</a></body></html>
-EOF
-cat >"$site10/leaf.html" <<EOF
-<html><body>
-<img src="img.gif">
-<picture><source srcset="ss.gif 2x"></picture>
-<video><source src="v.webm"></video>
-<video><track src="subs.vtt"></video>
-<a href="plain.gif">plain link past the boundary</a>
-</body></html>
-EOF
-out10="$tmp/html5media-out"
-rm -rf "$out10"
-mkdir -p "$out10"
-httrack "file://$site10/index.html" -O "$out10" --quiet --near -r2 >"$out10/.log" 2>&1 || true
-found "img.gif" "$out10"
-found "ss.gif" "$out10"
-found "v.webm" "$out10"
-found "subs.vtt" "$out10"
-notfound "plain.gif" "$out10"
-
 exit 0