Modernize the default User-Agent (#449)

The default UA was "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)",
a 25-year-old string naming a dead OS and trivially fingerprinted. Replace
it with an honest crawler UA carrying the HTTrack token and a reference URL:
"Mozilla/5.0 (compatible; HTTrack/3.x; +https://www.httrack.com/)".

Both the engine default (hts_create_opt) and the webhttrack mini-server
config default now share one HTS_DEFAULT_USER_AGENT macro, built from
HTTRACK_AFF_VERSION so the version token tracks releases without going stale.

Closes #449

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htsglobal.h

@@ -229,6 +229,10 @@ Please visit our Website: http://www.httrack.com
 #define HTS_DEFAULT_FOOTER                                                     \
   "<!-- Mirrored from %s%s by HTTrack Website Copier/" HTTRACK_AFF_VERSION     \
   " " HTTRACK_AFF_AUTHORS ", %s -->"
+/* Honest crawler User-Agent; no fake OS/browser to go stale. */
+#define HTS_DEFAULT_USER_AGENT                                                 \
+  "Mozilla/5.0 (compatible; HTTrack/" HTTRACK_AFF_VERSION                      \
+  "; +https://www.httrack.com/)"
 #define HTTRACK_WEB "http://www.httrack.com"
 #define HTS_UPDATE_WEBSITE                                                     \
   "http://www.httrack.com/"                                                    \

src/htslib.c

@@ -6048,8 +6048,7 @@ HTSEXT_API httrackp *hts_create_opt(void) {
   opt->shell = HTS_FALSE;
   opt->proxy.active = 0;        // pas de proxy
   opt->user_agent_send = HTS_TRUE;
-  StringCopy(opt->user_agent,
-             "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)");
+  StringCopy(opt->user_agent, HTS_DEFAULT_USER_AGENT);
   StringCopy(opt->referer, "");
   StringCopy(opt->from, "");
   opt->savename_83 = HTS_SAVENAME_83_LONG; // long names by default

src/htsselftest.c

@@ -1302,6 +1302,21 @@ static int st_urlhack(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Default User-Agent: honest HTTrack token, no resurrected Windows 98. */
+static int st_useragent(httrackp *opt, int argc, char **argv) {
+  const char *ua = StringBuff(opt->user_agent);
+  (void) argc;
+  (void) argv;
+  assertf(ua != NULL);
+  assertf(strcmp(ua, HTS_DEFAULT_USER_AGENT) == 0);
+  /* Macro-independent teeth: token, self-id URL, no Mozilla/4.x. */
+  assertf(strstr(ua, "HTTrack/") != NULL);
+  assertf(strstr(ua, "+https://www.httrack.com/") != NULL);
+  assertf(strstr(ua, "Mozilla/4.") == NULL);
+  printf("useragent self-test OK: %s\n", ua);
+  return 0;
+}
+
 /* ------------------------------------------------------------ */
 /* Registry: name -> handler, with a usage hint and a one-line description. */
 /* ------------------------------------------------------------ */
@@ -1348,6 +1363,7 @@ static const struct selftest_entry {
      st_cache_writefail},
     {"dns", "", "DNS resolver/cache self-test", st_dns},
     {"cookies", "", "cookie request-header self-test", st_cookies},
+    {"useragent", "", "default User-Agent self-test", st_useragent},
 };
 
 static void list_selftests(void) {

src/htsserver.c

@@ -358,12 +358,12 @@ int smallserver(T_SOC soc, char *url, char *method, char *data, char *path) {
       {NULL, 0}
     };
     initStrElt initStr[] = {
-      {"user", "Mozilla/4.5 (compatible; HTTrack 3.0x; Windows 98)"},
-      {"footer",
-       "<!-- Mirrored from %s%s by HTTrack Website Copier/3.x [XR&CO'2014], %s -->"},
-      {"url2", "+*.png +*.gif +*.jpg +*.jpeg +*.css +*.js -ad.doubleclick.net/*"},
-      {NULL, NULL}
-    };
+        {"user", HTS_DEFAULT_USER_AGENT},
+        {"footer", "<!-- Mirrored from %s%s by HTTrack Website Copier/3.x "
+                   "[XR&CO'2014], %s -->"},
+        {"url2",
+         "+*.png +*.gif +*.jpg +*.jpeg +*.css +*.js -ad.doubleclick.net/*"},
+        {NULL, NULL}};
     int i = 0;
 
     for(i = 0; initInt[i].name; i++) {

tests/01_engine-useragent.test

@@ -0,0 +1,7 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# Default User-Agent (#449): honest HTTrack token, no Windows 98 relic.
+httrack -O /dev/null -#test=useragent run | grep -q "useragent self-test OK"

tests/Makefile.am

@@ -50,6 +50,7 @@ TESTS = \
 	01_engine-stripquery.test \
 	01_engine-strsafe.test \
 	01_engine-urlhack.test \
+	01_engine-useragent.test \
 	02_manpage-regen.test \
 	02_update-cache.test \
 	10_crawl-simple.test \