Attribute the proxy port range to TCP, not RFC 3986

RFC 3986 defines port as *DIGIT with no range; the 1..65535 bound comes from
TCP's 16-bit port field.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/windows-build.yml

@@ -158,21 +158,13 @@ jobs:
 
           pass=0 fail=0 skip=0 failed="" skipped=""
           for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
-                   *_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
-                   60_crawl-log-salvage.test; do
+                   *_local-*.test 13_crawl_proxy_https.test 58_watchdog.test; do
             rc=0
             run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
             case "$rc" in
               0) pass=$((pass + 1)); echo "PASS $t" ;;
               77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
-              124)
-                fail=$((fail + 1)) failed="$failed $t"
-                echo "FAIL $t (timed out, tree killed)"
-                # Re-running a wedge traced would just hang again: salvage the
-                # killed crawl's own logs into the artifact instead.
-                dump_crawl_logs >>"$t.log"
-                tail -n 25 "$t.log" | sed 's/^/      /'
-                ;;
+              124) fail=$((fail + 1)) failed="$failed $t"; echo "FAIL $t (timed out, tree killed)" ;;
               *)
                 fail=$((fail + 1)) failed="$failed $t"
                 echo "FAIL $t (exit $rc)"

lang.indexes

@@ -23,7 +23,7 @@ pt:7
 ro:25
 ru:8
 sk:20
-sl:24
+si:24
 sv:17
 tr:10
 uk:22

lang/Finnish.txt

@@ -930,6 +930,7 @@ Server terminated
 Palvelin lopetettu
 A fatal error has occurred during this mirror
 Tällä peilillä tapahtui vakava virhe
+
 Proxy type:
 Välityspalvelimen tyyppi:
 Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.

lang/Italiano.txt

@@ -374,7 +374,7 @@ Create error logging and report files
 Crea file di log per segnalare errori e informazioni
 Generate DOS 8-3 filenames ONLY
 Genera solo nomi di file in formato 8.3
-Generate ISO9660 filenames ONLY for CDROM medias
+Generate ISO 9660 filenames ONLY for CDROM medias
 Genera nomi di file in formato ISO9660 per i CDROM
 Do not create HTML error pages
 Non generare pagine d'errore HTML

lang/Portugues-Brasil.txt

@@ -918,7 +918,7 @@ normal\nextended\ndebug
 normal\nextendido\ncorrigir
 Download web site(s)\nDownload web site(s) + questions\nGet individual files\nDownload all sites in pages (multiple mirror)\nTest links in pages (bookmark test)\n* Continue interrupted download\n* Update existing download
 Copiar site(s) da Web\nCopiar site(s) interativos da Web (perguntas)\nReceber arquivos específicos\nCopiar todas as páginas do site (alternação múltipla)\nTestar links nas páginas (testar indicador)\n* Retomar download interrompido\n* Atualizar download existente
-Relative URI / Absolute URL (default)\nAbsolute URL / Absolute URL\nAbsolute URI / Absolute URL\nOriginal URL / Original URL
+Relative URL / Absolute URL (default)\nAbsolute URL / Absolute URL\nAbsolute URI / Absolute URL\nOriginal URL / Original URL
 URI Relativa / URL Absoluta (padrão)\nURL Absoluta / URL Absoluta\nURI Absoluta / URL Absoluta\nURL Original / URL Original
 Open Source offline browser
 Abrir origem offline no navegador

lang/Slovenian.txt

@@ -3,7 +3,7 @@ Slovenian
 LANGUAGE_FILE
 Slovenian
 LANGUAGE_ISO
-sl
+si
 LANGUAGE_AUTHOR
 Jadran Rudec,iur.\r\njrudec@email.si \r\n
 LANGUAGE_CHARSET

man/httrack.1

@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "16 July 2026" "httrack website copier"
+.TH httrack 1 "14 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -165,7 +165,7 @@ pause transfer if N bytes reached, and wait until lock file is deleted (\-\-max\
 .IP \-cN
 number of multiple connections (*c8) (\-\-sockets[=N])
 .IP \-TN
-timeout, number of seconds after a non\-responding link is shutdown; also bounds host name resolution (\-\-timeout[=N])
+timeout, number of seconds after a non\-responding link is shutdown (\-\-timeout[=N])
 .IP \-RN
 number of retries, in case of timeout or non\-fatal errors (*R1) (\-\-retries[=N])
 .IP \-JN

src/htsalias.c

@@ -567,29 +567,24 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
   return 0;
 }
 
-/* Get home directory, '.' if unset or empty */
+/* Get home directory, '.' if failed */
 /* example: /home/smith */
 const char *hts_gethome(void) {
   const char *home = getenv("HOME");
 
-  /* An empty $HOME would expand ~/foo into the absolute /foo */
-  return strnotempty(home) ? home : ".";
+  if (home)
+    return home;
+  else
+    return ".";
 }
 
-/* Convert ~/foo into /home/smith/foo (~user/ left alone: no getpwnam here) */
+/* Convert ~/foo into /home/smith/foo */
 void expand_home(String * str) {
-  if (StringNotEmpty(*str) && StringSub(*str, 0) == '~' &&
-      (StringLength(*str) == 1 || StringSub(*str, 1) == '/')) {
+  if (StringSub(*str, 1) == '~') {
     char BIGSTK tempo[HTS_URLMAXSIZE * 2];
-    const char *const home = hts_gethome();
-    const size_t homelen = strlen(home);
-    const size_t taillen = StringLength(*str) - 1;
 
-    /* Leave untouched rather than abort() in strcatbuff on a huge $HOME */
-    if (taillen < sizeof(tempo) && homelen < sizeof(tempo) - taillen) {
-      strcpybuff(tempo, home);
-      strcatbuff(tempo, StringBuff(*str) + 1);
-      StringCopy(*str, tempo);
-    }
+    strcpybuff(tempo, hts_gethome());
+    strcatbuff(tempo, StringBuff(*str) + 1);
+    StringCopy(*str, tempo);
   }
 }

src/htsback.c

@@ -2816,9 +2816,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 if (SSL_set_fd(back[i].r.ssl_con, (int) back[i].r.soc) == 1) {
                   SSL_set_connect_state(back[i].r.ssl_con);
                   back[i].status = STATUS_SSL_WAIT_HANDSHAKE;   /* handshake wait */
-                  // the handshake gets its own timeout window, as connect does
-                  if (back[i].timeout > 0)
-                    back[i].timeout_refresh = time_local();
                 } else
                   back[i].r.statuscode = STATUSCODE_SSL_HANDSHAKE;
               } else
@@ -2878,11 +2875,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
       }
 #if HTS_USEOPENSSL
       else if (back[i].status == STATUS_SSL_WAIT_HANDSHAKE) {       // wait for SSL handshake
-        // a peer that never speaks TLS must be reaped by --timeout too (#607)
-        if (!gestion_timeout)
-          if (back[i].timeout > 0)
-            gestion_timeout = 1;
-
         /* SSL mode */
         if (back[i].r.ssl) {
           int conn_code;
@@ -4273,8 +4265,6 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
                 strcpybuff(back[i].r.msg, "Connect Time Out");
               else if (back[i].status == STATUS_WAIT_DNS)
                 strcpybuff(back[i].r.msg, "DNS Time Out");
-              else if (back[i].status == STATUS_SSL_WAIT_HANDSHAKE)
-                strcpybuff(back[i].r.msg, "SSL/TLS Handshake Time Out");
               else
                 strcpybuff(back[i].r.msg, "Receive Time Out");
               back[i].status = STATUS_READY;    // terminé

src/htscoremain.c

@@ -113,7 +113,6 @@ HTSEXT_API int hts_main(int argc, char **argv) {
 }
 
 static int hts_main_internal(int argc, char **argv, httrackp * opt);
-static hts_boolean cmdl_shortopt_has(const char *s, char c);
 
 // Main, récupère les paramètres et appelle le robot
 HTSEXT_API int hts_main2(int argc, char **argv, httrackp * opt) {
@@ -305,12 +304,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                      hts_get_version_info(opt));
               return 0;
             } else {
-              if (strncmp(tmp_argv[0], "--", 2)) { /* not a long option */
-                if (cmdl_shortopt_has(tmp_argv[0], 'q'))
-                  opt->quiet = HTS_TRUE; // never ask questions (nohup)
-                if (cmdl_shortopt_has(tmp_argv[0], 'i')) { // doit.log!
-                  argv_url = -1;
-                  opt->quiet = HTS_TRUE;
+              if (strncmp(tmp_argv[0], "--", 2)) {      /* pas */
+                if ((strchr(tmp_argv[0], 'q') != NULL))
+                  opt->quiet = 1;       // ne pas poser de questions! (nohup par exemple)
+                if ((strchr(tmp_argv[0], 'i') != NULL)) {       // doit.log!
+                  argv_url = -1;        /* forcer */
+                  opt->quiet = 1;
                 }
               } else if (strcmp(tmp_argv[0] + 2, "quiet") == 0) {
                 opt->quiet = 1; // ne pas poser de questions! (nohup par exemple)
@@ -425,10 +424,25 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
 
       }                         // for
 
-      // path_html is already UTF-8 (argv is UTF-8 on Windows via
-      // hts_argv_utf8), so no re-encoding.
+      // Convert path to UTF-8
+#ifdef _WIN32
+      {
+        char *const path =
+          hts_convertStringSystemToUTF8(StringBuff(opt->path_html),
+                                        (int) StringLength(opt->path_html));
+        if (path != NULL) {
+          StringCopy(opt->path_html_utf8, path);
+          free(path);
+        } else {
+          StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html),
+                      StringLength(opt->path_html));
+        }
+      }
+#else
+      // Assume UTF-8 filesystem.
       StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html),
                   StringLength(opt->path_html));
+#endif
 
       /* if doit.log exists, or if new URL(s) defined, 
          then DO NOT load standard config files */
@@ -2471,12 +2485,9 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
          }
          } */
 
-      // vérifier existence de la structure (path_html/path_log are UTF-8, use
-      // the UTF-8 mkdir path)
-      structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                               StringBuff(opt->path_html), "/"));
-      structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                               StringBuff(opt->path_log), "/"));
+      // vérifier existence de la structure
+      structcheck(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "/"));
+      structcheck(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "/"));
 
       // reprise/update
       if (opt->cache) {
@@ -2796,32 +2807,6 @@ int check_path(String * s, char *defaultname) {
   return return_value;
 }
 
-/* Does the short-option cluster s carry c from the main option set (-i, -iC2,
-   -%Mi)? Walked as the parser does below: %, &, @ and # each take the letter
-   after them into another set, so the i of -%i is not the main-set -i. */
-static hts_boolean cmdl_shortopt_has(const char *s, char c) {
-  const char *com;
-
-  if (s[0] != '-' || s[1] == '-')
-    return HTS_FALSE;
-  for (com = s + 1; *com != '\0'; com++) {
-    switch (*com) {
-    case '%':
-    case '&':
-    case '@':
-    case '#':
-      if (*(com + 1) != '\0')
-        com++; /* skip the other set's letter */
-      break;
-    default:
-      if (*com == c)
-        return HTS_TRUE;
-      break;
-    }
-  }
-  return HTS_FALSE;
-}
-
 // détermine si l'argument est une option
 int cmdl_opt(char *s) {
   if (s[0] == '-') {            // c'est peut être une option

src/htsdns_selftest.c

@@ -63,14 +63,9 @@ typedef struct mock_host {
   int gai_err; /* non-zero: getaddrinfo returns this */
   int naddr;
   mock_addr addr[6];
-  int calls;   /* times the backend resolved this host */
-  int slow_ms; /* non-zero: block this long, as a black-hole resolver would */
+  int calls; /* times the backend resolved this host */
 } mock_host;
 
-/* Long enough to outlast the 1s --timeout the bounded resolve is checked
-   against, short enough to keep the self-test quick. */
-#define MOCK_SLOW_MS 3000
-
 static mock_host mock_hosts[] = {
     {"v4only.test", 0, 1, {{AF_INET, {1, 2, 3, 4}}}, 0},
     {"v6only.test", 0, 1, {{AF_INET6, {0x20, 0x01, 0x0d, 0xb8, [15] = 1}}}, 0},
@@ -100,15 +95,8 @@ static mock_host mock_hosts[] = {
       {AF_INET, {10, 0, 0, 6}}},
      0},
     {"nodns.test", EAI_NONAME, 0, {{0}}, 0},
-    /* resolves, but only well after --timeout: the #606 wedge */
-    {"slow.test", 0, 1, {{AF_INET, {127, 0, 0, 9}}}, 0, MOCK_SLOW_MS},
 };
 
-/* Serializes mock_host bookkeeping: a timed-out resolve is abandoned, so its
-   worker is still inside the backend while the test reads the counters. */
-static htsmutex mock_lock = HTSMUTEX_INIT;
-static int mock_finished = 0; /* backend calls that have returned */
-
 static mock_host *mock_find(const char *name) {
   for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++) {
     if (strcmp(mock_hosts[i].name, name) == 0)
@@ -118,34 +106,8 @@ static mock_host *mock_find(const char *name) {
 }
 
 static void mock_reset_calls(void) {
-  hts_mutexlock(&mock_lock);
   for (size_t i = 0; i < sizeof(mock_hosts) / sizeof(mock_hosts[0]); i++)
     mock_hosts[i].calls = 0;
-  mock_finished = 0;
-  hts_mutexrelease(&mock_lock);
-}
-
-static int mock_read_calls(const char *name) {
-  int calls;
-
-  hts_mutexlock(&mock_lock);
-  calls = mock_find(name)->calls;
-  hts_mutexrelease(&mock_lock);
-  return calls;
-}
-
-/* Wait for n backend calls to return, ordering their writes against ours. */
-static void mock_wait_finished(int n) {
-  for (;;) {
-    hts_boolean done;
-
-    hts_mutexlock(&mock_lock);
-    done = (mock_finished >= n) ? HTS_TRUE : HTS_FALSE;
-    hts_mutexrelease(&mock_lock);
-    if (done)
-      break;
-    Sleep(10);
-  }
 }
 
 /* Build one addrinfo node owning its sockaddr (freed by mock_freeaddrinfo). */
@@ -171,10 +133,10 @@ static struct addrinfo *mock_mkai(const mock_addr *a) {
   return ai;
 }
 
-static int HTS_RESOLVER_CALL mock_getaddrinfo_(const char *node,
-                                               const char *service,
-                                               const struct addrinfo *hints,
-                                               struct addrinfo **res) {
+static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
+                                              const char *service,
+                                              const struct addrinfo *hints,
+                                              struct addrinfo **res) {
   mock_host *const h = mock_find(node);
   const int want = (hints != NULL) ? hints->ai_family : PF_UNSPEC;
   struct addrinfo *head = NULL, *tail = NULL;
@@ -183,11 +145,7 @@ static int HTS_RESOLVER_CALL mock_getaddrinfo_(const char *node,
   *res = NULL;
   if (h == NULL)
     return EAI_NONAME;
-  hts_mutexlock(&mock_lock);
   h->calls++; /* a real backend hit; a cached host skips this */
-  hts_mutexrelease(&mock_lock);
-  if (h->slow_ms != 0)
-    Sleep(h->slow_ms);
   if (h->gai_err != 0)
     return h->gai_err;
   for (int i = 0; i < h->naddr; i++) {
@@ -207,18 +165,6 @@ static int HTS_RESOLVER_CALL mock_getaddrinfo_(const char *node,
   return 0;
 }
 
-static int HTS_RESOLVER_CALL mock_getaddrinfo(const char *node,
-                                              const char *service,
-                                              const struct addrinfo *hints,
-                                              struct addrinfo **res) {
-  const int ret = mock_getaddrinfo_(node, service, hints, res);
-
-  hts_mutexlock(&mock_lock);
-  mock_finished++;
-  hts_mutexrelease(&mock_lock);
-  return ret;
-}
-
 static void HTS_RESOLVER_CALL mock_freeaddrinfo(struct addrinfo *res) {
   while (res != NULL) {
     struct addrinfo *const next = res->ai_next;
@@ -402,50 +348,6 @@ int dns_selftests(httrackp *opt) {
       deletesoc(s);
   }
 
-  /* A URL port outside 1..65535 must refuse the link, not fold into range and
-     connect elsewhere (#614). *addr_count discriminates: 0 only if refused
-     before the resolve, still 2 for one merely truncated or defaulted. */
-  {
-    /* an empty "dual.test:" means the default port (WHATWG, curl): keep it */
-    static const char *const good[] = {"dual.test:1",    "dual.test:80",
-                                       "dual.test:8080", "dual.test:65535",
-                                       "dual.test:080",  "dual.test:"};
-    /* 65616 and 4294967376 are load-bearing: both wrap to a plausible 80 */
-    static const char *const bad[] = {
-        "dual.test:0",          "dual.test:65536",      "dual.test:65616",
-        "dual.test:99999",      "dual.test:2147483648", "dual.test:4294967296",
-        "dual.test:4294967376", "dual.test:-1",         "dual.test:-23437",
-        "dual.test:80x",        "dual.test:+80",        "dual.test: 80",
-        "dual.test:0x50"};
-    size_t k;
-
-    for (k = 0; k < sizeof(good) / sizeof(good[0]); k++) {
-      htsblk r;
-      int count = -1;
-      T_SOC s;
-
-      hts_init_htsblk(&r);
-      s = newhttp_addr(opt, good[k], &r, -1, 0, 0, &count);
-      CHECK(count == 2); /* accepted: reached the resolve */
-      if (s != INVALID_SOCKET)
-        deletesoc(s);
-    }
-
-    for (k = 0; k < sizeof(bad) / sizeof(bad[0]); k++) {
-      htsblk r;
-      int count = -1;
-      T_SOC s;
-
-      hts_init_htsblk(&r);
-      s = newhttp_addr(opt, bad[k], &r, -1, 0, 0, &count);
-      CHECK(s == INVALID_SOCKET);
-      CHECK(count == 0); /* refused before resolving, not a failed connect */
-      CHECK(strstr(r.msg, "Invalid port") != NULL);
-      if (s != INVALID_SOCKET)
-        deletesoc(s);
-    }
-  }
-
   /* Connect-fallback decision (consumer of the multi-address list): when a
      stuck connect should abandon the current address for the next one. */
   {
@@ -468,91 +370,6 @@ int dns_selftests(httrackp *opt) {
   return failures;
 }
 
-/* Probes how long acquiring opt->state.lock takes while a resolve is in
-   flight. hts_has_stopped() takes that same lock and mutates nothing, and the
-   API promises it stays callable from another thread during a mirror. */
-typedef struct lock_probe {
-  httrackp *opt;
-  htsmutex lock;
-  hts_boolean done;
-  TStamp blocked_ms;
-} lock_probe;
-
-static void lock_probe_thread(void *arg) {
-  lock_probe *const p = (lock_probe *) arg;
-  TStamp start;
-
-  Sleep(MOCK_SLOW_MS / 10); /* let the resolve get under way first */
-  start = mtime_local();
-  (void) hts_has_stopped(p->opt);
-  hts_mutexlock(&p->lock);
-  p->blocked_ms = mtime_local() - start;
-  p->done = HTS_TRUE;
-  hts_mutexrelease(&p->lock);
-}
-
-int dns_timeout_selftests(httrackp *opt) {
-  SOCaddr addrs[HTS_MAXADDRNUM];
-  const char *err = NULL;
-  lock_probe probe;
-  TStamp start, elapsed;
-  int count;
-
-  failures = 0;
-  hts_dns_set_resolver_backend(&mock_backend);
-  IPV6_resolver = 0;
-  mock_reset_calls();
-  opt->timeout = 1; /* the bound under test */
-
-  memset(&probe, 0, sizeof(probe));
-  probe.opt = opt;
-  probe.lock = HTSMUTEX_INIT;
-  hts_mutexinit(&probe.lock);
-  CHECK(hts_newthread(lock_probe_thread, &probe) == 0);
-
-  start = mtime_local();
-  count = hts_dns_resolve_all(opt, "slow.test", addrs, HTS_MAXADDRNUM, &err);
-  elapsed = mtime_local() - start;
-
-  /* the resolve returns on opt->timeout, not when the resolver deigns to
-     answer: this is what lets --max-time and --timeout fire (#606). The bound
-     is derived from opt->timeout, never from the mock's sleep, or a resolve
-     that ignored opt->timeout would still pass under the mock. */
-  CHECK(elapsed < (TStamp) opt->timeout * 1000 + 500);
-  CHECK(count == 0); /* a timeout is reported as "does not resolve" */
-
-  /* state.lock is not held across the resolve; a concurrent stop query, which
-     the mirror API promises stays live, is not blocked behind it */
-  for (;;) {
-    hts_boolean done;
-    TStamp blocked;
-
-    hts_mutexlock(&probe.lock);
-    done = probe.done;
-    blocked = probe.blocked_ms;
-    hts_mutexrelease(&probe.lock);
-    if (done) {
-      CHECK(blocked < 500);
-      break;
-    }
-    Sleep(20);
-  }
-  hts_mutexfree(&probe.lock);
-
-  /* a timeout is not an answer, so it must not be negative-cached: the host is
-     resolved again rather than written off for the rest of the crawl */
-  CHECK(mock_read_calls("slow.test") == 1);
-  count = hts_dns_resolve_all(opt, "slow.test", addrs, HTS_MAXADDRNUM, &err);
-  CHECK(count == 0);
-  CHECK(mock_read_calls("slow.test") == 2); /* re-resolved, not cached */
-
-  /* Both resolves were abandoned mid-backend; wait for their workers to leave
-     it before returning. The backend stays installed: an abandoned worker
-     still reads it (to free its addrinfo) after the last call returns. */
-  mock_wait_finished(2);
-  return failures;
-}
-
 #else
 
 int dns_selftests(httrackp *opt) {

src/htsdns_selftest.h

@@ -46,12 +46,6 @@ typedef struct httrackp httrackp;
    Returns the number of failed checks (0 == success). */
 int dns_selftests(httrackp *opt);
 
-/* Drive a deliberately slow (mock) resolver, asserting that a resolve is
-   bounded by opt->timeout, does not hold opt->state.lock while it runs, and
-   does not cache a timeout as an answer (#606). Takes a few seconds.
-   Returns the number of failed checks (0 == success). */
-int dns_timeout_selftests(httrackp *opt);
-
 #endif
 
 #endif

src/htshelp.c

@@ -496,8 +496,8 @@ void help(const char *app, int more) {
   infomsg("");
   infomsg("Flow control:");
   infomsg("  cN number of multiple connections (*c8)");
-  infomsg("  TN timeout, number of seconds after a non-responding link is"
-          " shutdown; also bounds host name resolution");
+  infomsg
+    ("  TN timeout, number of seconds after a non-responding link is shutdown");
   infomsg
     ("  RN number of retries, in case of timeout or non-fatal errors (*R1)");
   infomsg

src/htslib.c

@@ -2167,18 +2167,15 @@ T_SOC newhttp_addr(httrackp *opt, const char *_iadr, htsblk *retour, int port,
 #endif
 
       if (a != NULL) {
-        // folding a nonsense port into 0..65535 crawls one neither the link nor
-        // a port filter named; an empty "host:" just means the default (#614)
-        if (a[1] != '\0' && !hts_parse_url_port(a + 1, &port)) {
-          if (retour != NULL) {
-            snprintf(retour->msg, sizeof(retour->msg), "Invalid port: %s",
-                     a + 1);
-          }
-          return INVALID_SOCKET;
-        }
+        int i = -1;
 
         iadr2[0] = '\0';
-        // the address itself, without the ":port"
+        sscanf(a + 1, "%d", &i);
+        if (i != -1) {
+          port = (unsigned short int) i;
+        }
+
+        // adresse véritable (sans :xx)
         strncatbuff(iadr2, iadr, (int) (a - iadr));
         resolve_host = iadr2;
       }
@@ -3751,27 +3748,18 @@ static int proxy_default_port(const char *arg) {
   return hts_proxy_is_socks(arg) ? 1080 : 8080;
 }
 
-hts_boolean hts_parse_url_port(const char *a, int *port) {
+// port "a" of -P argument "arg": digits fitting TCP's 1..65535, else the scheme
+// default. Not sscanf("%d"): past INT_MAX it wraps to a garbage port (#602)
+static int parse_proxy_port(const char *a, const char *arg) {
   char *end;
   long p;
 
   if (!isdigit((unsigned char) *a)) // strtol would eat a sign or leading space
-    return HTS_FALSE;
+    return proxy_default_port(arg);
   p = strtol(a, &end, 10);
   if (*end != '\0' || p < 1 || p > 65535) // ERANGE lands out of range too
-    return HTS_FALSE;
-  *port = (int) p;
-  return HTS_TRUE;
-}
-
-// port "a" of -P argument "arg": digits fitting TCP's 1..65535, else the scheme
-// default. Not sscanf("%d"): past INT_MAX it wraps to a garbage port (#602)
-static int parse_proxy_port(const char *a, const char *arg) {
-  int port;
-
-  if (!hts_parse_url_port(a, &port))
     return proxy_default_port(arg);
-  return port;
+  return (int) p;
 }
 
 void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
@@ -5108,164 +5096,50 @@ HTSEXT_API int check_hostname_dns(const char *const hostname) {
   return hts_dns_resolve_nocache(hostname, &buffer) != NULL;
 }
 
-/* A resolve in flight. Refcounted: a timed-out resolve is abandoned, not
-   cancelled, so the last of caller/worker to leave frees the job. */
-typedef struct dns_resolve_job {
-  htsmutex lock;
-  int refcount;
-  hts_boolean done;
-  char *hostname;
-  SOCaddr addr[HTS_MAXADDRNUM];
+// Needs locking
+// Internal DNS cache. Fill out[0..count-1] with up to max addresses for _iadr,
+// resolving (and caching the full list) on a miss. Returns the count.
+static int hts_dns_resolve_list_(httrackp *opt, const char *_iadr,
+                                 SOCaddr *const out, const int max,
+                                 const char **error) {
+  char BIGSTK iadr[HTS_URLMAXSIZE * 2];
+  coucal cache = hts_cache(opt); // le cache dns
   int count;
-  const char *error;
-} dns_resolve_job;
-
-/* Copy the first min(count, max) addresses of src into dest. */
-static void dns_copy_addrs(SOCaddr *dest, SOCaddr *src, int count, int max) {
-  int i;
-
-  for (i = 0; i < count && i < max; i++)
-    SOCaddr_copy_SOCaddr(dest[i], src[i]);
-}
-
-static void dns_job_release(dns_resolve_job *job) {
-  hts_boolean last;
-
-  hts_mutexlock(&job->lock);
-  last = (--job->refcount == 0) ? HTS_TRUE : HTS_FALSE;
-  hts_mutexrelease(&job->lock);
-  if (last) {
-    hts_mutexfree(&job->lock);
-    freet(job->hostname);
-    freet(job);
-  }
-}
-
-/* Outlives a timed-out resolve, so it writes only the job: never opt (freed
-   before the thread wait at exit) nor the DNS cache. */
-static void dns_resolve_thread(void *arg) {
-  dns_resolve_job *const job = (dns_resolve_job *) arg;
-  SOCaddr resolved[HTS_MAXADDRNUM];
-  const char *error = NULL;
-  const int count = hts_dns_resolve_nocache_list(job->hostname, resolved,
-                                                 HTS_MAXADDRNUM, &error);
-
-  hts_mutexlock(&job->lock);
-  dns_copy_addrs(job->addr, resolved, count, HTS_MAXADDRNUM);
-  job->count = count;
-  job->error = error;
-  job->done = HTS_TRUE; /* published last: gates the caller's read of addr[] */
-  hts_mutexrelease(&job->lock);
-  dns_job_release(job);
-}
-
-/* Resolve hostname on a worker thread, giving up after timeout seconds.
-   Returns the address count, or -1 on timeout -- distinct from 0 ("does not
-   resolve"), which is a real answer and gets negative-cached. */
-static int hts_dns_resolve_nocache_list_bounded(const char *hostname,
-                                                SOCaddr *const out,
-                                                const int max,
-                                                const int timeout,
-                                                const char **error) {
-  dns_resolve_job *job;
-  TStamp deadline;
-  int count = -1;
-  int poll_ms = 1;
-
-  if (timeout <= 0) /* no bound asked for (--timeout 0) */
-    return hts_dns_resolve_nocache_list(hostname, out, max, error);
-
-  job = calloct(1, sizeof(*job));
-  assertf(job != NULL);
-  hts_mutexinit(&job->lock);
-  job->hostname = strdupt(hostname);
-  job->refcount = 2; /* this caller + the worker */
-  if (hts_newthread(dns_resolve_thread, job) != 0) {
-    job->refcount = 1; /* no worker: fall back to resolving inline */
-    dns_job_release(job);
-    return hts_dns_resolve_nocache_list(hostname, out, max, error);
-  }
-
-  deadline = mtime_local() + (TStamp) timeout * 1000;
-  for (;;) {
-    hts_boolean done;
-
-    hts_mutexlock(&job->lock);
-    done = job->done;
-    if (done) {
-      count = job->count;
-      dns_copy_addrs(out, job->addr, count, max);
-      if (error != NULL)
-        *error = job->error;
-    }
-    hts_mutexrelease(&job->lock);
-    if (done || mtime_local() >= deadline)
-      break;
-    Sleep(poll_ms);
-    if (poll_ms < 50) /* short first polls keep a fast resolve fast */
-      poll_ms *= 2;
-  }
-  dns_job_release(job);
-  return count;
-}
-
-int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
-                        const char **error) {
-  char BIGSTK host[HTS_URLMAXSIZE * 2];
-  SOCaddr resolved[HTS_MAXADDRNUM];
-  coucal cache;
-  int count, i;
 
   assertf(opt != NULL);
+  assertf(_iadr != NULL);
   assertf(out != NULL);
-  if (!strnotempty(iadr) || max <= 0) {
-    return 0;
-  }
 
-  /* cache key and resolver input: identification and any ":port" stripped */
-  strcpybuff(host, jump_identification_const(iadr));
+  strcpybuff(iadr, jump_identification_const(_iadr));
+  // couper éventuel :
   {
     char *a;
 
-    if ((a = jump_toport(host)))
+    if ((a = jump_toport(iadr)))
       *a = '\0';
   }
 
-  hts_mutexlock(&opt->state.lock);
-  cache = hts_cache(opt);
-#if HTS_INET6 != 0
-  hts_resolver_check_env(); /* settle the backend before a worker reads it */
-#endif
-  count = hts_ghbn_all(cache, host, out, max);
-  hts_mutexrelease(&opt->state.lock);
+  /* get IP from the dns cache */
+  count = hts_ghbn_all(cache, iadr, out, max);
   if (count >= 0) { // cache hit (0 == negative-cached)
     return count;
-  }
+  } else { // non présent dans le cache dns, tester
+    SOCaddr resolved[HTS_MAXADDRNUM];
+    t_dnscache *record;
+    int i;
 
 #if DEBUGDNS
-  printf("resolving (not cached) %s\n", host);
+    printf("resolving (not cached) %s\n", iadr);
 #endif
 
-  /* Resolve with no lock held: getaddrinfo can block for a long time, and
-     state.lock also gates the stop request (#606). */
-  count = hts_dns_resolve_nocache_list_bounded(host, resolved, HTS_MAXADDRNUM,
-                                               opt->timeout, error);
+    count = hts_dns_resolve_nocache_list(iadr, resolved, HTS_MAXADDRNUM, error);
 
 #if HTS_WIDE_DEBUG
-  DEBUG_W("gethostbyname done\n");
+    DEBUG_W("gethostbyname done\n");
 #endif
 
-  if (count < 0) { /* timed out: no answer to cache, and none to report */
-    if (error != NULL)
-      *error = "host name resolution timed out";
-    return 0;
-  }
-
-  hts_mutexlock(&opt->state.lock);
-  { /* store the full list (coucal owns the record and dups the host key; a
-       concurrent resolve of the same host replaces, and frees, this one) */
-    t_dnscache *const record = malloct(sizeof(t_dnscache));
-
+    /* attempt to store new entry (coucal owns it and dups the host key) */
+    record = malloct(sizeof(t_dnscache));
     if (record != NULL) {
       memset(record, 0, sizeof(*record));
       record->host_count = count;
@@ -5275,13 +5149,28 @@ int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
         memcpy(record->host_addr[i], &SOCaddr_sockaddr(resolved[i]),
                record->host_length[i]);
       }
-      coucal_add_pvoid(cache, host, record);
+      coucal_add_pvoid(cache, iadr, record);
     }
-  }
-  hts_mutexrelease(&opt->state.lock);
 
-  /* copy result to caller (cache store may have failed; result still valid) */
-  dns_copy_addrs(out, resolved, count, max);
+    /* copy result to caller (cache store may have failed; result still valid)
+     */
+    for (i = 0; i < count && i < max; i++) {
+      SOCaddr_copy_SOCaddr(out[i], resolved[i]);
+    }
+    return count;
+  } // retour hp du cache
+}
+
+int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
+                        const char **error) {
+  int count;
+
+  if (!strnotempty(iadr) || max <= 0) {
+    return 0;
+  }
+  hts_mutexlock(&opt->state.lock);
+  count = hts_dns_resolve_list_(opt, iadr, out, max, error);
+  hts_mutexrelease(&opt->state.lock);
   return count;
 }
 

src/htslib.h

@@ -222,8 +222,7 @@ LLint http_xfread1(htsblk * r, int bufl);
 /* Cached resolver: fill out[0..count-1] with up to max addresses for iadr (in
    resolver order), returning the count (0 = does not resolve, negative-cached).
    Resolves once per host; later calls read the DNS cache. Must hold no lock
-   (brackets opt->state.lock itself, never across the resolve). A miss resolves
-   on a worker thread bounded by opt->timeout; a timeout reports 0, uncached. */
+   (brackets opt->state.lock itself). */
 int hts_dns_resolve_all(httrackp *opt, const char *iadr, SOCaddr *out, int max,
                         const char **error);
 HTS_INLINE SOCaddr *hts_dns_resolve2(httrackp *opt, const char *iadr,
@@ -288,12 +287,6 @@ const char *strrchr_limit(const char *s, char c, const char *limit);
 char *jump_protocol(char *source);
 const char *jump_protocol_const(const char *source);
 
-/* Parse a URL's port text "a" (after the ':', up to the end of the string):
-   TRUE and *port set for a bare decimal in 1..65535, else FALSE and *port left
-   alone. Not sscanf("%d"), which range-checks nothing and wraps past INT_MAX.
- */
-hts_boolean hts_parse_url_port(const char *a, int *port);
-
 /* Split a -P proxy argument "[scheme://][user:pass@]host[:port]" into the proxy
    host string (scheme and any user:pass kept, for later stripping and auth),
    written NUL-terminated into name[name_size] (truncated to fit), and the port

src/htsproxy.c

@@ -491,11 +491,14 @@ static int socks5_handshake_stream(httrackp *opt, socks5_stream *st,
     if ((unsigned char) host[i] < ' ')
       return socks5_fail(msg, msgsize, "SOCKS5: invalid origin host");
   }
-  // the old range check ran after sscanf("%d") had wrapped a huge value into a
-  // plausible port (#614). An empty "host:" stays refused here, unlike the
-  // direct path, as it was before #614.
-  if (portsep != NULL && !hts_parse_url_port(portsep + 1, &port))
-    return socks5_fail(msg, msgsize, "SOCKS5: invalid origin port");
+  if (portsep != NULL) {
+    int p = -1;
+
+    sscanf(portsep + 1, "%d", &p);
+    if (p <= 0 || p > 65535)
+      return socks5_fail(msg, msgsize, "SOCKS5: invalid origin port");
+    port = p;
+  }
   if (link_has_authorization(proxy_name)) {
     if (!socks5_credentials(opt, proxy_name, user, sizeof(user), &userlen, pass,
                             sizeof(pass), &passlen, msg, msgsize))

src/htsselftest.c

@@ -45,7 +45,6 @@ Please visit our Website: http://www.httrack.com
 #include "htscore.h"
 #include "htsdefines.h"
 #include "htslib.h"
-#include "htsalias.h"
 #include "htsparse.h"
 #include "htscache.h"
 #include "htscache_selftest.h"
@@ -826,21 +825,6 @@ static int st_simplify(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-static int st_expandhome(httrackp *opt, int argc, char **argv) {
-  String path = STRING_EMPTY;
-
-  (void) opt;
-  if (argc < 1) {
-    fprintf(stderr, "expandhome: needs a path\n");
-    return 1;
-  }
-  StringCopy(path, argv[0]);
-  expand_home(&path);
-  printf("expanded=%s\n", StringBuff(path));
-  StringFree(path);
-  return 0;
-}
-
 static int st_mime(httrackp *opt, int argc, char **argv) {
   char mime[256];
 
@@ -1469,25 +1453,6 @@ static int st_socks5(httrackp *opt, int argc, char **argv) {
   assertf(socks5_handshake_scripted(opt, "origin.test:8443", proxy, &io) == 1);
   assertf(memcmp(io.sent + io.sent_len - 2, "\x20\xfb", 2) == 0);
 
-  /* a bad origin port is refused before any byte goes out (#614). 4294967376 is
-     the case the old range check could not see: it overflowed the sscanf("%d")
-     into a plausible 80 and passed. 65616 would not prove anything here, since
-     it fits an int and the old check already caught it. */
-  {
-    static const char *const bad[] = {"origin.test:4294967376",
-                                      "origin.test:80x", "origin.test:+80",
-                                      "origin.test: 80", "origin.test:8.0"};
-    size_t k;
-
-    for (k = 0; k < sizeof(bad) / sizeof(bad[0]); k++) {
-      len = socks5_reply(script, 0x01, v4, sizeof(v4));
-      io.reply = script;
-      io.reply_len = len;
-      assertf(socks5_handshake_scripted(opt, bad[k], proxy, &io) == 0);
-      assertf(io.sent_len == 0);
-    }
-  }
-
   /* credentials: split on the first colon of the escaped userinfo, so %3a stays
      inside the username and a colon in the password is not a delimiter */
   {
@@ -2144,15 +2109,6 @@ static int st_dns(httrackp *opt, int argc, char **argv) {
   return err;
 }
 
-static int st_dnstimeout(httrackp *opt, int argc, char **argv) {
-  const int err = dns_timeout_selftests(opt);
-
-  (void) argc;
-  (void) argv;
-  printf("dns-timeout-selftest: %s\n", err ? "FAIL" : "OK");
-  return err;
-}
-
 static int st_cookies(httrackp *opt, int argc, char **argv) {
   static t_cookie cookie;
   char hdr[1024];
@@ -2477,45 +2433,6 @@ static int st_makeindex(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-// hts_buildtopindex takes a system-charset path but verif_backblue() below it
-// expects utf-8, so on Windows a non-ASCII project dir gets the gifs written to
-// a mangled twin (issues #216/#217). argv[0] is a writable dir.
-static int st_topindex(httrackp *opt, int argc, char **argv) {
-  char projdir[HTS_URLMAXSIZE];
-  char path[HTS_URLMAXSIZE + 16]; /* projdir plus a basename */
-#ifdef _WIN32
-  /* the GUI hands hts_buildtopindex an ANSI path; mimic it. CP1252 'cafe' */
-  static const char *const projName = "caf\xE9";
-#else
-  /* POSIX system charset is UTF-8 */
-  static const char *const projName = "caf\xC3\xA9";
-#endif
-
-  assertf(argc >= 1);
-  snprintf(projdir, sizeof(projdir), "%s/%s", argv[0], projName);
-
-  /* structcheck(), not the utf-8 MKDIR family: same charset as buildtopindex */
-  snprintf(path, sizeof(path), "%s/", projdir);
-  assertf(structcheck(path) == 0);
-
-  /* returns 0 here: the dir holds no sub-project, only the gifs matter */
-  (void) hts_buildtopindex(opt, projdir, "");
-
-  /* the gifs must land in the project dir itself, not in a mangled sibling */
-  snprintf(path, sizeof(path), "%s/backblue.gif", projdir);
-  assertf(fexist(path));
-
-  /* raw unlink/rmdir: UNLINK is utf-8 on Windows, these paths aren't */
-  unlink(path);
-  snprintf(path, sizeof(path), "%s/fade.gif", projdir);
-  unlink(path);
-  snprintf(path, sizeof(path), "%s/index.html", projdir);
-  unlink(path);
-  rmdir(projdir);
-  printf("topindex self-test OK\n");
-  return 0;
-}
-
 /* Each inplace_escape_*() must equal escape_*() on a copy. */
 static int st_inplace_escape(httrackp *opt, int argc, char **argv) {
   /* >255 bytes forces the helper's malloct path, not the stack buffer */
@@ -3153,7 +3070,6 @@ static const struct selftest_entry {
     {"filterbounds", "", "matcher length/work caps reject hostile patterns",
      st_filterbounds},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
-    {"expandhome", "<path>", "expand a leading ~/ into $HOME", st_expandhome},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
     {"urlhack", "", "-%u url-hack sub-flag (www/slash/query) self-test",
@@ -3217,15 +3133,10 @@ static const struct selftest_entry {
     {"cache-corrupt", "<dir>", "cache read-side corruption self-test",
      st_cache_corrupt},
     {"dns", "", "DNS resolver/cache self-test", st_dns},
-    {"dnstimeout", "", "a slow DNS resolve is bounded and holds no lock",
-     st_dnstimeout},
     {"cookies", "", "cookie request-header self-test", st_cookies},
     {"useragent", "", "default User-Agent self-test", st_useragent},
     {"makeindex", "[dir]", "hts_finish_makeindex footer/refresh self-test",
      st_makeindex},
-    {"topindex", "[dir]",
-     "hts_buildtopindex charset handling of a non-ASCII project dir",
-     st_topindex},
     {"inplace-escape", "", "inplace_escape_* vs escape_* equivalence self-test",
      st_inplace_escape},
     {"escape-room", "", "HT_ADD_HTMLESCAPED* reservation-factor self-test",

src/htsserver.h

@@ -164,12 +164,13 @@ HTS_UNUSED static int linputsoc_t(T_SOC soc, char *s, int max, int timeout) {
   return -1;
 }
 
-/* Same contract as hts_gethome(), which is hidden and out of reach from here */
 static const char *gethomedir(void) {
   const char *home = getenv("HOME");
 
-  /* An empty $HOME would put the base path and httrack.ini at the root */
-  return strnotempty(home) ? home : ".";
+  if (home)
+    return home;
+  else
+    return ".";
 }
 static int linput_cpp(FILE * fp, char *s, int max) {
   int rlen = 0;

src/htstools.c

@@ -896,20 +896,7 @@ HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
     if (fpo) {
       find_handle h;
 
-      // générer gif. verif_backblue() is utf-8, but our path is the system
-      // charset, so on Windows convert it or the gifs land in a mangled twin
-      // dir (#217). Elsewhere the system charset is already utf-8.
-#ifdef _WIN32
-      {
-        const char *const base = concat(catbuff, sizeof(catbuff), rpath, "/");
-        char *const base_utf8 =
-            hts_convertStringSystemToUTF8(base, strlen(base));
-        verif_backblue(opt, base_utf8 != NULL ? base_utf8 : base);
-        free(base_utf8);
-      }
-#else
-      verif_backblue(opt, concat(catbuff, sizeof(catbuff), rpath, "/"));
-#endif
+      verif_backblue(opt, concat(catbuff, sizeof(catbuff), rpath, "/")); // générer gif
       // Header
       hts_template_format(fpo, toptemplate_header,
               "<!-- Mirror and index made by HTTrack Website Copier/"

src/proxy/proxytrack.h

@@ -107,12 +107,13 @@ HTS_UNUSED static void proxytrack_print_log(const char *severity, const char *fo
 	"<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \
 	"<!-- End Disable IE Friendly HTTP Error Messages -->\r\n"
 
-/* Same contract as hts_gethome(): proxytrack does not link libhttrack */
 HTS_UNUSED static const char *gethomedir(void) {
   const char *home = getenv("HOME");
 
-  /* An empty $HOME would resolve a relative path against the root */
-  return strnotempty(home) ? home : ".";
+  if (home)
+    return home;
+  else
+    return ".";
 }
 
 HTS_UNUSED static int linput(FILE * fp, char *s, int max) {

src/webhttrack

@@ -33,19 +33,6 @@ function log {
     return 0
 }
 
-# Map a POSIX locale ("zh_TW.UTF-8@euro") to its lang.indexes number, English (1) if unknown
-function lang_index {
-    local locale=$1 indexes=$2 tag n t
-    tag=$(echo "${locale}" | cut -f1 -d'.' | cut -f1 -d'@' | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z_')
-    # a few languages carry a region-specific entry (zh_tw, pt_br); else use the bare language
-    for t in "${tag}" "${tag%%_*}"; do
-        n=$(grep -E "^${t}:" "${indexes}" | cut -f2 -d':')
-        test -n "${n}" && break
-    done
-    test -n "${n}" || n=1
-    echo "${n}"
-}
-
 function launch_browser {
     log "Launching $1"
     browser=$1
@@ -72,11 +59,13 @@ test -f "${DISTPATH}/lang.indexes" || ! log "Could not find ${DISTPATH}/lang.ind
 test -d "${DISTPATH}/lang" || ! log "Could not find ${DISTPATH}/lang" || exit 1
 test -d "${DISTPATH}/html" || ! log "Could not find ${DISTPATH}/html" || exit 1
 
-# Locale: POSIX precedence, LC_ALL overrides LC_MESSAGES overrides LANG
-HTSLANG="${LC_ALL}"
-! test -n "${HTSLANG}" && HTSLANG="${LC_MESSAGES}"
+# Locale
+HTSLANG="${LC_MESSAGES}"
+! test -n "${HTSLANG}" && HTSLANG="${LC_ALL}"
 ! test -n "${HTSLANG}" && HTSLANG="${LANG}"
-LANGN=$(lang_index "${HTSLANG}" "${DISTPATH}/lang.indexes")
+HTSLANG="$(echo "$LANG" | cut -f1 -d'.' | cut -f1 -d'_')"
+LANGN=$(grep -E "^${HTSLANG}:" "${DISTPATH}/lang.indexes" | cut -f2 -d':')
+! test -n "${LANGN}" && LANGN=1
 
 # Find the browser
 # note: not all systems have sensible-browser or www-browser alternative

tests/01_engine-cmdline.test

@@ -54,13 +54,6 @@ refused() {
         ! echo "FAIL: $1 (exit $RC)" || exit 1
 }
 
-# assert continue mode was entered: it drops the URL list, so with no cache to
-# resume the run ends on the usage screen rather than on any other error
-continued() {
-    { test "$RC" -ne 0 && grep -q 'usage:' "$1/.log"; } ||
-        ! echo "FAIL: $2 (exit $RC)" || exit 1
-}
-
 # a value past the old 126/256 caps but within the cap is accepted, on both the
 # short and long form of each option
 long=$(nchars 900)
@@ -109,50 +102,4 @@ for bad in nan nan:5 5:nan inf 10:5 99999; do
     refused "#185: invalid --pause '$bad' not refused cleanly"
 done
 
-# An option is not -i (continue) merely because its name contains an 'i' (#615).
-# These used to wipe the URL given before them and exit on the usage screen.
-run "$tmp/ord-bti" --build-top-index
-accepted "$tmp/ord-bti" "#615: --build-top-index after the URL wiped the URL list"
-run "$tmp/ord-bti-s" "-%i"
-accepted "$tmp/ord-bti-s" "#615: -%i after the URL wiped the URL list"
-run "$tmp/ord-proto" --protocol 2
-accepted "$tmp/ord-proto" "#615: --protocol after the URL wiped the URL list"
-run "$tmp/ord-proto-s" "-@i2"
-accepted "$tmp/ord-proto-s" "#615: -@i2 after the URL wiped the URL list"
-
-# %, &, @ and # take the letter after them into another option set, wherever
-# they sit in the cluster, so the i of -q%i is not the main-set -i either.
-run "$tmp/ord-qpi" "-q%i"
-accepted "$tmp/ord-qpi" "#615: -q%i after the URL wiped the URL list"
-run "$tmp/ord-qai" "-q@i"
-accepted "$tmp/ord-qai" "#615: -q@i after the URL wiped the URL list"
-
-# -%q only forces quiet mode, which nothing here can see: a run whose stdout is
-# not a tty is quiet from the start (htscoremain.c:174). Just check it crawls.
-run "$tmp/ord-iqs" "-%q"
-accepted "$tmp/ord-iqs" "#615: -%q after the URL broke the crawl"
-
-# The real -i still forces continue mode and drops the URL list, in every form:
-# a fix that merely stopped looking for 'i' would pass the checks above.
-run "$tmp/cont-s" -i
-continued "$tmp/cont-s" "#615: -i after the URL no longer forces continue mode"
-run "$tmp/cont-c" -iC2
-continued "$tmp/cont-c" "#615: -iC2 after the URL no longer forces continue mode"
-run "$tmp/cont-l" --continue
-continued "$tmp/cont-l" "#615: --continue after the URL no longer forces continue mode"
-run "$tmp/cont-u" --update
-continued "$tmp/cont-u" "#615: --update after the URL no longer forces continue mode"
-
-# ...including where another set's letter precedes it in the cluster.
-run "$tmp/cont-pq" "-%qi"
-continued "$tmp/cont-pq" "#615: -%qi after the URL no longer forces continue mode"
-run "$tmp/cont-pm" "-%Mi"
-continued "$tmp/cont-pm" "#615: -%Mi after the URL no longer forces continue mode"
-
-# Placed before the URL these always worked; they must keep working.
-run_only "$tmp/pre-bti" "-%i" "file://$tmp/index.html"
-accepted "$tmp/pre-bti" "#615: -%i before the URL broke the crawl"
-run_only "$tmp/pre-proto" "-@i2" "file://$tmp/index.html"
-accepted "$tmp/pre-proto" "#615: -@i2 before the URL broke the crawl"
-
 exit 0

tests/01_engine-dnstimeout.test

@@ -1,15 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# A resolve against a deliberately slow mock getaddrinfo (no network) must
-# return on --timeout, must not hold opt->state.lock meanwhile, and must not
-# cache the timeout as an answer (#606).
-# 'run' is an ignored placeholder argument.
-out=$(httrack -#test=dnstimeout run)
-
-test "$out" = "dns-timeout-selftest: OK" || {
-    echo "expected 'dns-timeout-selftest: OK', got: $out" >&2
-    exit 1
-}

tests/01_engine-expandhome.test

@@ -1,53 +0,0 @@
-#!/bin/bash
-#
-
-# SC2088: the literal, unexpanded '~' is the input under test.
-# shellcheck disable=SC2088
-
-set -euo pipefail
-
-# ~ expansion for the -O base path (expand_home). $HOME is pinned so the cases
-# cannot depend on the caller's, but MSYS rewrites it into a Windows path
-# before the native httrack.exe reads it: ask the engine what it saw rather
-# than hardcoding the value.
-ask() {
-    HOME=HTSHOME httrack -O /dev/null -#test=expandhome "$1"
-}
-exp() {
-    test "$(ask "$1")" == "expanded=$2" || exit 1
-}
-
-home=$(ask '~')
-home=${home#expanded=}
-
-# or every case below is vacuous: '~' means expansion never fired, '.' means it
-# fell back to the default without ever reading $HOME
-test "$home" != '~' || exit 1
-test "$home" != '.' || exit 1
-
-exp '~' "$home"
-exp '~/foo' "$home/foo"
-exp '~/foo/bar/#' "$home/foo/bar/#"
-exp '~/' "$home/"
-exp '~/../x' "$home/../x"
-
-# ~user/ needs getpwnam: left alone rather than mangled into $HOME + "user/"
-exp '~smith/foo' '~smith/foo'
-exp '~root' '~root'
-
-# only a leading '~' expands; anything else is a literal path component
-exp 'a~/x' 'a~/x'
-exp './~/x' './~/x'
-exp 'foo~bar' 'foo~bar'
-
-# an unset or empty $HOME must not turn ~/foo into the absolute /foo
-test "$(env -u HOME httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=./foo" || exit 1
-test "$(HOME='' httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=./foo" || exit 1
-
-# A $HOME past the 2 * HTS_URLMAXSIZE buffer is left alone: strcatbuff aborts
-# rather than truncates. Only meaningful where the engine sees the value we set:
-# MSYS rewrites HOME into a path of its own choosing, long or not.
-if test "$home" = HTSHOME; then
-    long=$(printf '%04096d' 0 | tr '0' 'A')
-    test "$(HOME="$long" httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=~/foo" || exit 1
-fi

tests/01_engine-topindex.test

@@ -1,12 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# hts_buildtopindex takes a system-charset path but verif_backblue below it
-# expects utf-8, mangling a non-ASCII project dir on Windows (#216, #217).
-dir=$(mktemp -d)
-trap 'rm -rf "$dir"' EXIT
-
-httrack -O /dev/null -#test=topindex "$dir" run |
-    grep -q "topindex self-test OK"

tests/52_local-socks5.test

@@ -165,13 +165,6 @@ auth="$tmpdir/auth"
 start_server "$auth" ok
 run_crawl "$auth/out" "http://${host}:${http_port}/" \
     "socks5://user:secret@127.0.0.1:${socks_port}"
-# name a rejected version byte here: the crawl check below only knows the
-# handshake failed, not why
-grep -q "^AUTHVER-BAD" "$auth/socks.log" && {
-    echo "FAIL: sub-negotiation version was not RFC 1929's 0x01" >&2
-    cat "$auth/socks.log" >&2
-    exit 1
-}
 grep -rq "ORIGIN-PAGE-563" "$auth/out" || {
     echo "FAIL: origin not downloaded through the authenticated socks proxy" >&2
     cat "$auth/out.log" "$auth/socks.log" >&2

tests/59_local-tls-stall.test

@@ -1,95 +0,0 @@
-#!/bin/bash
-#
-# Issue #607: a peer that accepts the TCP connect but never speaks TLS leaves the
-# slot stuck in the handshake. The per-slot --timeout must reap it; before the
-# fix only --max-time did, so these crawls ran until the kill guard.
-
-set -euo pipefail
-
-: "${top_srcdir:=..}"
-
-# shellcheck source=tests/testlib.sh
-. "$top_srcdir/tests/testlib.sh"
-
-if test "${HTTPS_SUPPORT:-}" == "no"; then
-    echo "no https support compiled, skipping"
-    exit 77
-fi
-python=$(find_python) || {
-    echo "python3 missing, skipping"
-    exit 77
-}
-
-server=$(nativepath "$top_srcdir/tests/tls-stall-server.py")
-tmpdir=$(mktemp -d)
-serverpid=
-
-cleanup() {
-    stop_server "$serverpid"
-    rm -rf "$tmpdir"
-    return 0
-}
-trap cleanup EXIT
-
-# start_stall_server <tag> <mode-args...>: sets $port from the announced one.
-start_stall_server() {
-    local tag="$1"
-    shift
-    "$python" "$server" "$@" >"$tmpdir/$tag.out" 2>"$tmpdir/$tag.err" &
-    serverpid=$!
-    port=
-    for _ in $(seq 1 50); do
-        line=$(head -n1 "$tmpdir/$tag.out" 2>/dev/null || true)
-        if test "${line%% *}" == "PORT"; then
-            port="${line#PORT }"
-            return 0
-        fi
-        kill -0 "$serverpid" 2>/dev/null || {
-            echo "server exited early: $(cat "$tmpdir/$tag.err")"
-            exit 1
-        }
-        sleep 0.1
-    done
-    echo "could not discover server port"
-    exit 1
-}
-
-# no -E/--max-time on purpose: --timeout is the only thing that can end these.
-# The kill guard stands in for the hang, so a wall time near it means no reap.
-crawl_wall() {
-    local out="$tmpdir/$1"
-    shift
-    local start
-    start=$(date +%s)
-    run_with_timeout 60 httrack -O "$out" -c1 --robots=0 --retries=0 --quiet -Z \
-        "$@" >>"$tmpdir/log" 2>&1 || true
-    echo $(($(date +%s) - start))
-}
-
-# 1. handshake stalled from the first byte: reaped at --timeout, not before.
-start_stall_server direct direct
-wall=$(crawl_wall crawl1 "https://127.0.0.1:$port/" --timeout=5)
-if test "$wall" -ge 30 || test "$wall" -lt 3; then
-    echo "FAIL: stalled handshake reaped after ${wall}s, expected about 5s" >&2
-    cat "$tmpdir/log" >&2
-    exit 1
-fi
-grep -q 'Handshake Time Out' "$tmpdir/crawl1/hts-log.txt" || {
-    echo "FAIL: crawl ended in ${wall}s but not on a handshake timeout" >&2
-    cat "$tmpdir/crawl1/hts-log.txt" >&2
-    exit 1
-}
-echo "OK: stalled TLS handshake reaped by --timeout after ${wall}s"
-
-# 2. the handshake window is its own, not what the connect left over: a proxy
-# that takes 4s to answer CONNECT must still leave the full --timeout=5 for the
-# handshake (~9s total). Sharing the connect's clock would reap at ~5s.
-stop_server "$serverpid"
-start_stall_server proxy proxy 4
-wall=$(crawl_wall crawl2 "https://127.0.0.1:443/" -P "127.0.0.1:$port" --timeout=5)
-if test "$wall" -ge 20 || test "$wall" -lt 7; then
-    echo "FAIL: handshake after a slow connect reaped at ${wall}s, expected about 9s" >&2
-    cat "$tmpdir/log" >&2
-    exit 1
-fi
-echo "OK: handshake keeps its own timeout window after a slow connect (${wall}s)"

tests/60_crawl-log-salvage.test

@@ -1,46 +0,0 @@
-#!/bin/bash
-#
-# Unit-tests dump_crawl_logs (testlib.sh): the killed test's hts-log.txt must
-# reach the uploaded logs, or a Windows wedge past --max-time stays
-# undiagnosable (#605).
-
-set -euo pipefail
-
-testdir=$(cd "$(dirname "$0")" && pwd)
-# shellcheck source=tests/testlib.sh
-. "${testdir}/testlib.sh"
-
-TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/httrack_salv.XXXXXX")
-export TMPDIR
-trap 'rm -rf "$TMPDIR"' EXIT
-fail() {
-    echo "FAIL: $*" >&2
-    exit 1
-}
-
-verdict="More than 120 seconds passed.. giving up"
-
-# Vacuity control: with nothing left behind the assertions below cannot pass on
-# ambient output.
-test -z "$(dump_crawl_logs)" || fail "dumped something with no crawl tmpdir"
-
-# A killed local-crawl.sh leaves exactly this shape.
-d="${TMPDIR}/httrack_local.probe"
-mkdir -p "$d/crawl"
-echo "$verdict" >"$d/crawl/hts-log.txt"
-echo "httrack stdout marker" >"$d/log"
-
-# Append to an unterminated line, as a killed test's own log always ends.
-art="${TMPDIR}/artifact.log"
-printf '[running httrack ...] ..\t' >"$art"
-dump_crawl_logs >>"$art"
-
-grep -qF "$verdict" "$art" || fail "hts-log.txt verdict not salvaged"
-grep -qF "httrack stdout marker" "$art" || fail "httrack stdout not salvaged"
-grep -q '^--- .*hts-log.txt' "$art" || fail "dump glued its header onto the unterminated line"
-
-# Cleared, so the next timing-out test cannot re-report this crawl as its own.
-test ! -d "$d" || fail "salvaged tmpdir left behind"
-test -z "$(dump_crawl_logs)" || fail "salvaged crawl reported twice"
-
-echo "crawl log salvage OK"

tests/61_webhttrack-locale.test

@@ -1,91 +0,0 @@
-#!/bin/bash
-#
-# webhttrack must map the caller's locale to the right lang.indexes number.
-# Guards: LC_ALL/LC_MESSAGES were ignored, and zh_tw/pt_br were unreachable.
-
-set -euo pipefail
-
-testdir=$(cd "$(dirname "$0")" && pwd)
-distdir=$(cd "${testdir}/.." && pwd)
-script="${distdir}/src/webhttrack"
-
-fail() {
-    echo "FAIL: $*" >&2
-    exit 1
-}
-
-test -r "${script}" || fail "no ${script}"
-test -r "${distdir}/lang.indexes" || fail "no ${distdir}/lang.indexes"
-
-# Lift the locale code out of the launcher, so this tests the shipped logic itself.
-funcs=$(sed -n '/^function lang_index/,/^}/p' "${script}")
-# End on the next section, not a blank line: a blank line grown inside the block
-# would truncate the lift and silently stop testing everything below it.
-grep -q '^# Find the browser' "${script}" || fail "locale block terminator moved in ${script}"
-block=$(sed -n '/^# Locale/,/^# Find the browser/p' "${script}" | sed '$d')
-echo "${block}" | grep -q 'LANGN=.*lang_index' || fail "could not lift the whole locale block from ${script}"
-
-# Run the lifted block against whatever locale vars the caller exported.
-run_block() {
-    # shellcheck disable=SC2034 # read by the eval'd locale block
-    DISTPATH="${distdir}"
-    # the launcher runs under neither -e ("! test -n x && y" returns 1) nor -u
-    set +eu
-    eval "${funcs}"
-    eval "${block}"
-    # shellcheck disable=SC2153 # LANGN is set by the eval'd block, not a LANG typo
-    echo "${LANGN}"
-}
-
-# Resolve LANG/LC_ALL/LC_MESSAGES (empty = unset) to the language number.
-lang_of() {
-    (
-        unset LANG LC_ALL LC_MESSAGES
-        # only the value matters here; hide bash's setlocale gripe when the locale is not installed
-        # shellcheck disable=SC2030 # confining the locale to this subshell is the point
-        {
-            if test -n "$1"; then export LANG="$1"; fi
-            if test -n "$2"; then export LC_ALL="$2"; fi
-            if test -n "$3"; then export LC_MESSAGES="$3"; fi
-        } 2>/dev/null
-        run_block
-    )
-}
-
-check() {
-    local want=$1 got desc
-    shift
-    desc="LANG='$1' LC_ALL='$2' LC_MESSAGES='$3'"
-    got=$(lang_of "$1" "$2" "$3")
-    test "${got}" = "${want}" || fail "${desc}: want ${want}, got ${got}"
-    echo "ok: ${desc} -> ${got}"
-}
-
-#      want  LANG            LC_ALL          LC_MESSAGES
-check 2 "fr_FR.UTF-8" "" ""
-check 14 "zh_TW.UTF-8" "" ""                      # region-qualified: Chinese-BIG5, not Simplified (13)
-check 12 "pt_BR.UTF-8" "" ""                      # region-qualified: Portugues-Brasil, not Portugues (7)
-check 13 "zh_CN.UTF-8" "" ""                      # unlisted region falls back to the bare language
-check 24 "sl_SI.UTF-8" "" ""                      # Slovenian is ISO 639-1 "sl"
-check 4 "" "de_DE.UTF-8" ""                       # LC_ALL alone
-check 4 "" "" "de_DE.UTF-8"                       # LC_MESSAGES alone
-check 4 "fr_FR.UTF-8" "de_DE.UTF-8" ""            # LC_ALL wins over LANG
-check 4 "fr_FR.UTF-8" "" "de_DE.UTF-8"            # LC_MESSAGES wins over LANG
-check 2 "de_DE.UTF-8" "fr_FR.UTF-8" "de_DE.UTF-8" # LC_ALL wins over LC_MESSAGES
-check 2 "fr_FR@euro" "" ""                        # @modifier without a charset
-check 1 "C" "" ""                                 # C, unknown and unset all mean English
-check 1 "xx_YY.UTF-8" "" ""
-check 1 "" "" ""
-
-# Exported but empty is unset (POSIX), so it must not mask LANG. lang_of cannot
-# express this: it exports nothing for an empty argument.
-got=$(
-    unset LANG LC_ALL LC_MESSAGES
-    # shellcheck disable=SC2031 # confining the locale to this subshell is the point
-    { export LANG="fr_FR.UTF-8" LC_ALL="" LC_MESSAGES=""; } 2>/dev/null
-    run_block
-)
-test "${got}" = 2 || fail "empty LC_ALL/LC_MESSAGES must not mask LANG: want 2, got ${got}"
-echo "ok: LANG='fr_FR.UTF-8' with LC_ALL='' exported -> ${got}"
-
-echo "PASS"

tests/62_lang-integrity.test

@@ -1,90 +0,0 @@
-#!/bin/bash
-#
-# lang/*.txt are line pairs: an English msgid, then its translation. A stray
-# blank or a drifted msgid silently unhooks every translation that follows,
-# rather than failing, so assert the pairing and the join against English.txt.
-# Pairs are physical lines here; a msgid ending in \ continues onto the next one
-# for the engine (linput_cpp) but not for us. None do today.
-
-set -euo pipefail
-
-langdir="${top_srcdir:-..}/lang"
-def="${top_srcdir:-..}/lang.def"
-eng="$langdir/English.txt"
-for f in "$eng" "$def"; do
-    [ -f "$f" ] || {
-        echo "cannot find $f"
-        exit 1
-    }
-done
-
-tmp=$(mktemp -d)
-trap 'rm -rf "$tmp"' EXIT
-keys="$tmp/english.keys"
-
-# Byte-wise: each file is in its own declared legacy charset, lang.def is CRLF.
-LC_ALL=C awk 'NR%2==1 { sub(/\r$/, ""); print }' "$eng" >"$keys"
-
-# An empty parse would make every check below pass vacuously.
-nkeys=$(wc -l <"$keys")
-if [ "$nkeys" -lt 400 ]; then
-    echo "only $nkeys msgids parsed from $eng; the parse is broken"
-    exit 1
-fi
-
-nfiles=0
-fail=0
-for f in "$langdir"/*.txt; do
-    nfiles=$((nfiles + 1))
-    LC_ALL=C awk -v keys="$keys" -v name="${f##*/}" '
-        BEGIN { while ((getline k < keys) > 0) known[k] = 1 }
-        { sub(/\r$/, "") }
-        NR % 2 == 1 {
-            if ($0 == "")
-                printf "%s:%d: blank line where an English msgid belongs\n", name, NR
-            else if (!($0 in known))
-                printf "%s:%d: msgid absent from English.txt: %s\n", name, NR, $0
-            else
-                next
-            bad++
-        }
-        END {
-            if (NR % 2) {
-                printf "%s: odd line count (%d): msgid/translation pairing is broken\n", name, NR
-                bad++
-            }
-            exit(bad > 0)
-        }
-    ' "$f" || fail=1
-done
-
-if [ "$nfiles" -lt 25 ]; then
-    echo "only $nfiles language files found in $langdir"
-    exit 1
-fi
-
-# lang.def maps each LANG_* macro to the English string the GUI compiles in,
-# which is the lookup key into every lang/*.txt: no msgid, no translations.
-LC_ALL=C awk -v keys="$keys" '
-    BEGIN { while ((getline k < keys) > 0) known[k] = 1 }
-    { sub(/\r$/, "") }
-    NR % 2 == 1 { macro = $0; next }
-    macro ~ /^(LANG_|LISTDEF_)/ {
-        n++
-        if (!($0 in known)) {
-            printf "lang.def:%d: %s maps to \"%s\", which is not a msgid in English.txt\n", NR, macro, $0
-            bad++
-        }
-    }
-    END {
-        if (n < 400) {
-            printf "lang.def: only %d LANG_ entries parsed; the parse is broken\n", n
-            bad++
-        }
-        exit(bad > 0)
-    }
-' "$def" || fail=1
-
-[ "$fail" -eq 0 ] || exit 1
-
-echo "$nfiles language files consistent with $nkeys English msgids"

tests/63_webhttrack-home.test

@@ -1,88 +0,0 @@
-#!/bin/bash
-#
-# htsserver's $HOME lookup, through the web UI it actually renders. An exported
-# but empty $HOME must not resolve the default base path against the root.
-
-set -euo pipefail
-
-testdir=$(cd "$(dirname "$0")" && pwd)
-distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
-distdir=$(cd "${distdir}" && pwd)
-
-fail() {
-    echo "FAIL: $*" >&2
-    exit 1
-}
-
-command -v htsserver >/dev/null || fail "no htsserver in PATH"
-# Mirror local-crawl.sh's skip-with-77: the Debian buildd chroot has no python3.
-command -v python3 >/dev/null || {
-    echo "python3 not found; skipping" >&2
-    exit 77
-}
-
-srv=
-cleanup() {
-    test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
-}
-trap cleanup EXIT HUP INT QUIT PIPE TERM
-
-log=$(mktemp)
-trap 'cleanup; rm -f "${log}"' EXIT
-
-# Ask htsserver for its rendered default base path, under the given $HOME
-# ("-" = leave $HOME unset).
-pathfor() {
-    local homeval=$1 port url got
-    port=$(python3 -c 'import socket
-s = socket.socket()
-s.bind(("127.0.0.1", 0))
-print(s.getsockname()[1])
-s.close()')
-
-    : >"${log}"
-    (
-        # htsserver ignores SIGTERM/SIGTTOU handling from the harness shell
-        trap '' TERM TTOU
-        if test "${homeval}" = -; then unset HOME; else export HOME="${homeval}"; fi
-        exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
-    ) &
-    srv=$!
-
-    for _ in $(seq 1 40); do
-        url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
-        kill -0 "${srv}" 2>/dev/null || break
-        sleep 0.25
-    done
-    test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
-
-    # The UI is served ISO-8859-1, so keep python out of text mode.
-    got=$(python3 -c 'import re, sys, urllib.request
-body = urllib.request.urlopen(sys.argv[1] + "server/step2.html", timeout=20).read()
-m = re.search(br"name=\"path\" value=\"([^\"]*)\"", body)
-print(m.group(1).decode("latin-1") if m else "")' "${url}")
-
-    kill -9 "${srv}" 2>/dev/null || true
-    wait "${srv}" 2>/dev/null || true
-    srv=
-    test -n "${got}" || fail "no path field in the rendered step2.html"
-    echo "${got}"
-}
-
-check() {
-    local desc=$1 homeval=$2 want=$3 got
-    got=$(pathfor "${homeval}")
-    test "${got}" = "${want}" || fail "${desc}: want ${want}, got ${got}"
-    echo "ok: ${desc} -> ${got}"
-}
-
-# A real HOME must reach the page, or the two cases below are vacuous: they
-# also pass when gethomedir() ignores the environment and always answers ".".
-check 'a set HOME is used' /nonexistent/hts-home /nonexistent/hts-home/websites
-
-# Exported but empty is the regression: getenv() returns "", not NULL, so a
-# bare NULL check let the base path collapse onto /websites.
-check 'an empty HOME' '' ./websites
-check 'an unset HOME' - ./websites
-
-echo "PASS"

tests/64_local-intl-outdir.test

@@ -1,13 +0,0 @@
-#!/bin/bash
-#
-# A non-ASCII -O output dir must receive the mirror, not a double-encoded twin
-# (#621). argv is UTF-8 on Windows, so path_html was re-encoded as if it were
-# the ANSI codepage, prefixing every saved file with a mangled directory. Only
-# path_html is the non-ASCII "café"; path_log stays ASCII so the harness reads
-# hts-log.txt/hts-cache normally and this isolates the -O encoding path.
-
-: "${top_srcdir:=..}"
-
-bash "$top_srcdir/tests/local-crawl.sh" --html-subdir 'café' --errors 0 --files 5 \
-    --found 'simple/basic.html' \
-    httrack 'BASEURL/simple/basic.html'

tests/Makefile.am

@@ -3,7 +3,7 @@
 # silently drop it from the dist tarball and break "make distcheck".
 EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
 	proxy-https-server.py socks5-server.py proxy-connect-server.py \
-	proxytestlib.py tls-stall-server.py \
+	proxytestlib.py \
 	local-crawl.sh local-server.py testlib.sh server.crt server.key \
 	server-root/simple/basic.html server-root/simple/link.html \
 	server-root/stripquery/index.html server-root/stripquery/a.html \
@@ -35,7 +35,6 @@ TESTS = \
 	01_engine-copyopt.test \
 	01_engine-crange.test \
 	01_engine-dns.test \
-	01_engine-dnstimeout.test \
 	01_engine-doitlog.test \
 	01_engine-entities.test \
 	01_engine-filelist.test \
@@ -59,7 +58,6 @@ TESTS = \
 	01_engine-pause.test \
 	01_engine-rcfile.test \
 	01_engine-reconcile.test \
-	01_engine-expandhome.test \
 	01_engine-fsize.test \
 	01_engine-redirect.test \
 	01_engine-relative.test \
@@ -71,7 +69,6 @@ TESTS = \
 	01_engine-status.test \
 	01_engine-stripquery.test \
 	01_engine-strsafe.test \
-	01_engine-topindex.test \
 	01_engine-urlhack.test \
 	01_engine-unescape-bounds.test \
 	01_engine-useragent.test \
@@ -139,12 +136,6 @@ TESTS = \
 	55_local-chunked.test \
 	56_local-proxy-noleak.test \
 	57_local-proxy-connect.test \
-	58_watchdog.test \
-	59_local-tls-stall.test \
-	60_crawl-log-salvage.test \
-	61_webhttrack-locale.test \
-	62_lang-integrity.test \
-	63_webhttrack-home.test \
-	64_local-intl-outdir.test
+	58_watchdog.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -47,7 +47,6 @@ key="${testdir}/server.key"
 
 tls=
 verbose=
-html_subdir=
 rerun=
 rerun_args=
 rerun_dead=
@@ -133,13 +132,6 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         rerun_args="${args[$pos]}"
         ;;
-    --html-subdir)
-        # Mirror into "$out/NAME" (path_html) but keep logs/cache in "$out"
-        # (path_log): a non-ASCII NAME exercises the -O path encoding (#621)
-        # without routing the harness-read hts-log.txt through a non-ASCII path.
-        pos=$((pos + 1))
-        html_subdir="${args[$pos]}"
-        ;;
     --errors | --errors-content | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
@@ -215,21 +207,12 @@ test -n "$ver" || die "could not run httrack"
 
 out="${tmpdir}/crawl"
 mkdir "$out" || die "could not create $out"
-# path_html holds the mirror + index; path_log holds hts-cache/hts-log.txt.
-# Default: both are "$out". With --html-subdir, path_html becomes "$out/NAME"
-# (the mirror root the audits inspect) while path_log stays "$out".
-mirrorroot="$out"
-odir="$out"
-if test -n "$html_subdir"; then
-    mirrorroot="${out}/${html_subdir}"
-    odir="${mirrorroot},${out}"
-fi
 # Localhost is fast; disable the rate/bandwidth safety limits but keep a
 # max-time backstop so a hang cannot wedge the suite.
 declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
 log="${tmpdir}/log"
 info "running httrack ${hts[*]}"
-httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
+httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
 crawlpid=$!
 wait "$crawlpid"
 crawlres=$?
@@ -246,7 +229,7 @@ grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
 # --- optional second pass: re-mirror into the same dir (cache/update path) ----
 if test -n "$rerun"; then
     info "re-running httrack (update pass)"
-    httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -274,7 +257,7 @@ fi
 if test -n "$rerun_args"; then
     read -ra extra <<<"$rerun_args"
     info "re-running httrack with ${rerun_args}"
-    httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -296,7 +279,7 @@ if test -n "$rerun_dead"; then
     stop_server "$serverpid"
     serverpid=
     info "re-running httrack against the stopped server"
-    httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
     crawlpid=$!
     wait "$crawlpid" || true
@@ -325,7 +308,7 @@ fi
 
 # --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
 hostroot=
-for cand in "${mirrorroot}/127.0.0.1_${port}" "${mirrorroot}/127.0.0.1"; do
+for cand in "${out}/127.0.0.1_${port}" "${out}/127.0.0.1"; do
     if test -d "$cand"; then
         hostroot="$cand"
         break

tests/proxytestlib.py

@@ -18,15 +18,6 @@ PROXY_LOG = "proxy.log"
 ORIGIN_LOG = "origin-headers.log"
 
 
-def bind_ephemeral():
-    """Listening socket on a free loopback port, and that port."""
-    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
-    srv.bind(("127.0.0.1", 0))
-    srv.listen(16)
-    return srv, srv.getsockname()[1]
-
-
 def pipe(src, dst):
     """Relay bytes one way until EOF, then tear both ends down."""
     try:
@@ -122,7 +113,11 @@ def handle_client(conn, logdir, mode, default_port):
 
 
 def start_proxy(logdir, mode, default_port):
-    srv, port = bind_ephemeral()
+    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    srv.bind(("127.0.0.1", 0))
+    srv.listen(16)
+    port = srv.getsockname()[1]
 
     def accept_loop():
         while True:

tests/socks5-server.py

@@ -19,12 +19,11 @@ import threading
 # python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
 sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
 
-from proxytestlib import bind_ephemeral, pipe  # noqa: E402
+from proxytestlib import pipe  # noqa: E402
 
 # The one name the proxy answers for; a .invalid TLD never resolves (RFC 6761),
 # so a locally-resolving client could not reach us -- success proves remote DNS.
 REMOTE_HOST = b"socks-origin.invalid"
-AUTH_VERSION = 0x01  # RFC 1929 sub-negotiation version
 # index links the subpages so an -r3 crawl reuses one keep-alive socket
 LINKS = "".join('<a href="/p%d.html">%d</a>' % (i, i) for i in range(1, 6))
 ORIGIN_BODY = ("<html><body>ORIGIN-PAGE-563 " + LINKS + "</body></html>").encode()
@@ -105,12 +104,6 @@ def negotiate_auth(conn, logdir):
     if 0x02 in methods:  # prefer user/pass so the auth test exercises RFC 1929
         conn.sendall(b"\x05\x02")
         (subver,) = recvn(conn, 1)
-        # RFC 1929's version byte is 0x01, not SOCKS5's 0x05: a real proxy
-        # rejects a mismatch instead of tunnelling anyway
-        if subver != AUTH_VERSION:
-            log(logdir, "AUTHVER-BAD %d" % subver)
-            conn.sendall(bytes([AUTH_VERSION, 0x01]))  # sub-negotiation failure
-            return False
         (ulen,) = recvn(conn, 1)
         uname = recvn(conn, ulen)
         (plen,) = recvn(conn, 1)
@@ -187,7 +180,11 @@ def handle_socks(conn, logdir, mode):
 
 
 def start_socks(logdir, mode):
-    srv, port = bind_ephemeral()
+    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    srv.bind(("127.0.0.1", 0))
+    srv.listen(16)
+    port = srv.getsockname()[1]
 
     def serve():
         while True:

tests/testlib.sh

@@ -44,25 +44,6 @@ stop_server() {
     return 0
 }
 
-# Dump and clear the crawl logs a hard-killed test leaves in TMPDIR (its cleanup
-# trap never ran): hts-log.txt alone records "More than N seconds passed.. giving
-# up", so a wedge past --max-time is undiagnosable without it (#605).
-dump_crawl_logs() {
-    local d f
-    for d in "${TMPDIR:-/tmp}"/httrack_local.*; do
-        test -d "$d" || continue
-        for f in "$d/crawl/hts-log.txt" "$d/log" "$d/log.2"; do
-            test -f "$f" || continue
-            # Leading newline: the killed test's last line has no terminator.
-            printf '\n--- %s (last 200 lines)\n' "$f"
-            tail -n 200 "$f"
-        done
-        # so a later test's dump cannot re-report this one; never fatal, the
-        # caller is already handling a failure and Windows may still hold a file
-        rm -rf "$d" || true
-    done
-}
-
 # Kill a backgrounded job and its whole descendant tree. POSIX: the caller must
 # have put the job in its own process group (run_with_timeout does) so we signal
 # the group; a bare kill would orphan the grandchildren. Windows: the tree is

tests/tls-stall-server.py

@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-"""Peers that accept a connection and never speak TLS (#607).
-
-Modes: "direct" stalls the handshake straight away; "proxy <secs>" answers a
-CONNECT after <secs> before stalling, so the handshake starts on a clock the
-connect has already eaten into. Prints "PORT <n>" once listening.
-Usage: tls-stall-server.py [direct | proxy <secs>]
-"""
-import os
-import sys
-import threading
-import time
-
-# python3 -P (PYTHONSAFEPATH) drops the script's own directory from sys.path
-sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
-
-from proxytestlib import bind_ephemeral  # noqa: E402
-
-held = []  # keep every socket open: a close would fail the handshake outright
-
-
-def stall(conn, delay):
-    if delay is not None:
-        rfile = conn.makefile("rb")
-        while rfile.readline() not in (b"\r\n", b"\n", b""):
-            pass
-        time.sleep(delay)
-        conn.sendall(b"HTTP/1.0 200 Connection established\r\n\r\n")
-    held.append(conn)
-
-
-def main():
-    mode = sys.argv[1] if len(sys.argv) > 1 else "direct"
-    delay = float(sys.argv[2]) if mode == "proxy" else None
-    srv, port = bind_ephemeral()
-    sys.stdout.reconfigure(newline="\n")  # Windows would emit \r\n
-    print("PORT %d" % port, flush=True)
-    while True:
-        conn, _ = srv.accept()
-        threading.Thread(target=stall, args=(conn, delay), daemon=True).start()
-
-
-main()