Keep the vcxproj source list alphabetical

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com>
Extract the proxy tunneling code into htsproxy.c
2026-07-14 21:00:57 +03:00 · 2026-07-14 16:11:38 +02:00 · 2026-07-14 15:47:24 +02:00 · 2026-07-14 15:38:47 +02:00 · 2026-07-14 14:29:12 +02:00 · 2026-07-14 14:11:57 +02:00
23 changed files with 1600 additions and 232 deletions
--- a/man/httrack.1
+++ b/man/httrack.1
@@ -3,7 +3,7 @@
 .\"
 .\" This file is generated by man/makeman.sh; do not edit by hand.
 .\" SPDX-License-Identifier: GPL-3.0-or-later
-.TH httrack 1 "12 July 2026" "httrack website copier"
+.TH httrack 1 "14 July 2026" "httrack website copier"
 .SH NAME
 httrack \- offline browser : copy websites to a local directory
 .SH SYNOPSIS
@@ -135,7 +135,7 @@ continue an interrupted mirror using the cache (\-\-continue)
 mirror ALL links located in the first level pages (mirror links) (\-\-mirrorlinks)
 .SS Proxy options:
 .IP \-P
-proxy use (\-P proxy:port or \-P user:pass@proxy:port) (\-\-proxy <param>)
+proxy use (\-P [socks5://][user:pass@]proxy:port) (\-\-proxy <param>)
 .IP \-%f
 *use proxy for ftp (f0 don't use) (\-\-httpproxy\-ftp[=N])
 .IP \-%b
--- a/src/Makefile.am
+++ b/src/Makefile.am
@@ -61,7 +61,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htshelp.c htslib.c htscoremain.c \
 	htsname.c htsrobots.c htstools.c htswizard.c \
 	htsalias.c htsthread.c htsindex.c htsbauth.c \
-	htsmd5.c htscodec.c htszlib.c htswrap.c htsconcat.c \
+	htsmd5.c htscodec.c htsproxy.c htszlib.c htswrap.c htsconcat.c \
 	htsmodules.c htscharset.c punycode.c htsencoding.c htssniff.c \
 	md5.c \
 	minizip/ioapi.c minizip/mztools.c minizip/unzip.c minizip/zip.c \
@@ -72,7 +72,7 @@ libhttrack_la_SOURCES =  htscore.c htsparse.c htsback.c htscache.c \
 	htshelp.h htsindex.h htslib.h htsmd5.h \
 	htsmodules.h htsname.h htsnet.h htssniff.h \
 	htsopt.h htsrobots.h htsthread.h  \
-	htstools.h htswizard.h htswrap.h htscodec.h htszlib.h  \
+	htstools.h htswizard.h htswrap.h htscodec.h htsproxy.h htszlib.h  \
 	htsstrings.h htsarrays.h httrack-library.h \
 	htscharset.h punycode.h htsencoding.h \
 	htsentities.h htsentities.sh htsbasiccharsets.sh htscodepages.h \
--- a/src/htsback.c
+++ b/src/htsback.c
@@ -45,6 +45,7 @@ Please visit our Website: http://www.httrack.com

 #include "htsftp.h"
 #include "htscodec.h"
+#include "htsproxy.h"
 #if HTS_USEZLIB
 #include "htszlib.h"
 #else
@@ -557,6 +558,14 @@ static int create_back_tmpfile(httrackp *opt, lien_back *const back,
  return 0;
 }

+/* Move src onto dst; RENAME does not clobber an existing target on Windows. */
+static hts_boolean replace_file(const char *src, const char *dst) {
+  if (RENAME(src, dst) == 0)
+    return HTS_TRUE;
+  (void) UNLINK(dst);
+  return RENAME(src, dst) == 0 ? HTS_TRUE : HTS_FALSE;
+}
+
 /* Commit or restore a re-fetch backup (#77 follow-up): a re-fetch over an
   existing file moved the good copy to back->tmpfile before truncating url_sav.
   commit keeps the new file and drops the backup; else restore it so an aborted
@@ -572,10 +581,9 @@ static void back_finalize_backup(httrackp *opt, lien_back *const back,
      fclose(back->r.out);
      back->r.out = NULL;
    }
-    (void) UNLINK(back->url_sav); /* drop the failed partial */
-    /* On rename failure keep the backup: it still holds the good copy, so
-       losing it would be worse than an orphaned temp. */
-    if (RENAME(back->tmpfile, back->url_sav) != 0)
+    /* On failure keep the backup: an orphaned temp beats losing the good copy.
+     */
+    if (!replace_file(back->tmpfile, back->url_sav))
      hts_log_print(opt, LOG_WARNING | LOG_ERRNO,
                    "could not restore %s; previous copy kept as %s",
                    back->url_sav, back->tmpfile);
@@ -676,24 +684,45 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
              if (back[p].url_sav[0]) {
                const hts_codec codec =
                    hts_codec_parse(back[p].r.contentencoding);
+                /* Never decode over url_sav: a failed decode would destroy the
+                   copy an --update re-fetch is supposed to refresh (#557). */
+                char BIGSTK unpacked[HTS_URLMAXSIZE * 2 + 4]; // room for ".u"
                LLint size;

-                file_notify(opt, back[p].url_adr, back[p].url_fil,
-                            back[p].url_sav, 1, 1, back[p].r.notmodified);
-                filecreateempty(&opt->state.strc, back[p].url_sav);     // filenote & co
+                snprintf(unpacked, sizeof(unpacked), "%s.u", back[p].url_sav);
                if ((size = hts_codec_unpack(codec, back[p].tmpfile,
-                                             back[p].url_sav)) >= 0) {
+                                             unpacked)) >= 0) {
                  back[p].r.size = back[p].r.totalsize = size;
-                  // fichier -> mémoire
                  if (!back[p].r.is_write) {
+                    // fichier -> mémoire ; le fichier est écrit plus tard
                    deleteaddr(&back[p].r);
-                    back[p].r.adr = readfile_utf8(back[p].url_sav);
+                    back[p].r.adr = readfile_utf8(unpacked);
                    if (!back[p].r.adr) {
                      back[p].r.statuscode = STATUSCODE_INVALID;
                      strcpybuff(back[p].r.msg,
                                 "Read error when decompressing");
                    }
-                    UNLINK(back[p].url_sav);
+                    UNLINK(unpacked);
+                  } else if (replace_file(unpacked, back[p].url_sav)) {
+                    /* The temp bypassed filecreate(), which is what chmods. */
+#ifndef _WIN32
+                    chmod(back[p].url_sav, HTS_ACCESS_FILE);
+#endif
+                    file_notify(opt, back[p].url_adr, back[p].url_fil,
+                                back[p].url_sav, 1, 1, back[p].r.notmodified);
+                    filenote(&opt->state.strc, back[p].url_sav, NULL);
+                  } else {
+                    back[p].r.statuscode = STATUSCODE_INVALID;
+                    strcpybuff(back[p].r.msg,
+                               "Write error when decompressing (can not rename "
+                               "the temporary file)");
+                    /* Keep the decoded body: the failed replace may have
+                       removed the previous copy, leaving this as the only one.
+                     */
+                    hts_log_print(
+                        opt, LOG_WARNING | LOG_ERRNO,
+                        "could not replace %s; decoded copy kept as %s",
+                        back[p].url_sav, unpacked);
                  }
                } else {
                  back[p].r.statuscode = STATUSCODE_INVALID;
@@ -702,12 +731,18 @@ int back_finalize(httrackp * opt, cache_back * cache, struct_back * sback,
                               ? "Unsupported Content-Encoding (%s)"
                               : "Error when decompressing (%s)",
                           back[p].r.contentencoding);
-                  /* Drop the undecoded body and its placeholder so the writer
-                     can't commit the coded bytes as the page. */
+                  /* Drop the undecoded body so the writer can't commit the
+                     coded bytes as the page; url_sav is left untouched. */
                  if (!back[p].r.is_write)
                    deleteaddr(&back[p].r);
-                  UNLINK(back[p].url_sav);
+                  UNLINK(unpacked);
                }
+                /* A failed decode keeps the previously-mirrored copy: note it,
+                   or the update purge (in old.lst, absent from new.lst) would
+                   delete what we just took care not to overwrite. */
+                if (back[p].r.statuscode == STATUSCODE_INVALID &&
+                    fexist_utf8(back[p].url_sav))
+                  filenote(&opt->state.strc, back[p].url_sav, NULL);
              }
              /* ensure that no remaining temporary file exists */
              unlink(back[p].tmpfile);
@@ -2077,6 +2112,17 @@ int back_add(struct_back * sback, httrackp * opt, cache_back * cache, const char
                        "error: forbidden test with ftp link for back_add");
          return -1;            // erreur pas de test permis
        }
+        // the ftp client dials the origin itself: over socks that would bypass
+        // the proxy, so fail the link rather than leak the connection (#563)
+        if (back[p].r.req.proxy.active &&
+            hts_proxy_is_socks(back[p].r.req.proxy.name)) {
+          back[p].r.statuscode = STATUSCODE_NON_FATAL;
+          strcpybuff(back[p].r.msg,
+                     "ftp:// is not supported over a SOCKS proxy");
+          back[p].status = STATUS_READY;
+          back_set_finished(sback, p);
+          return 0;
+        }
        if (!(back[p].r.req.proxy.active && opt->ftp_proxy)) {  // connexion directe, gérée en thread
          FTPDownloadStruct *str =
            (FTPDownloadStruct *) malloc(sizeof(FTPDownloadStruct));
@@ -2679,13 +2725,40 @@ void back_wait(struct_back * sback, httrackp * opt, cache_back * cache,
          }
          busy_state = 1;

+          // socks5: tunnel to the origin before anything is written, for http
+          // as well as https. Skip on a reused keep-alive socket (already
+          // tunneled) and on the post-TLS re-entry (ssl_con set) (#563).
+          if (back[i].r.req.proxy.active &&
+              hts_proxy_is_socks(back[i].r.req.proxy.name) &&
+              !back[i].r.keep_alive
+#if HTS_USEOPENSSL
+              && back[i].r.ssl_con == NULL
+#endif
+          ) {
+            const int timeout = back[i].timeout > 0 ? back[i].timeout : 30;
+
+            if (!socks5_handshake(opt, &back[i].r, back[i].url_adr, timeout)) {
+              if (!strnotempty(back[i].r.msg))
+                strcpybuff(back[i].r.msg, "SOCKS5 handshake failed");
+              deletehttp(&back[i].r);
+              back[i].r.soc = INVALID_SOCKET;
+              back[i].r.statuscode = STATUSCODE_NON_FATAL;
+              back[i].status = STATUS_READY;
+              back_set_finished(sback, i);
+              continue;
+            }
+          }
+
 #if HTS_USEOPENSSL
          /* SSL mode */
          if (back[i].r.ssl) {
            int tunnel_ok = 1;

-            // https via proxy: CONNECT-tunnel before TLS (#85)
-            if (back[i].r.req.proxy.active && back[i].r.ssl_con == NULL) {
+            // https via an http proxy: CONNECT-tunnel before TLS (#85); socks
+            // already carries the origin connection
+            if (back[i].r.req.proxy.active &&
+                !hts_proxy_is_socks(back[i].r.req.proxy.name) &&
+                back[i].r.ssl_con == NULL) {
              const int timeout = back[i].timeout > 0 ? back[i].timeout : 30;

              tunnel_ok =
--- a/src/htscache_selftest.c
+++ b/src/htscache_selftest.c
@@ -232,11 +232,10 @@ static int disk_fallback_selftest(httrackp *opt) {
  static const char body[] = "BINARY-on-disk-body-0123456789-no-trailing-nul";
  const size_t body_len = sizeof(body) - 1;

-  /* X-Save must start with path_html_utf8 so the reader resolves it verbatim
-     (otherwise it re-roots it as a pre-3.40 relative path); then the body we
-     create at fconv(save) is exactly where cache_readex looks for it. */
-  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
-          "example.com/blob.bin");
+  /* a DOS-ified X-Save loses the path_html_utf8 prefix the reader matches on,
+     and gets re-rooted as a pre-3.40 relative path; fconv() only on access */
+  concat(save, sizeof(save), StringBuff(opt->path_html_utf8),
+         "example.com/blob.bin");

  /* write only the header (X-In-Cache: 0); the body stays on disk */
  selftest_open_for_write(&cache, opt);
@@ -1253,8 +1252,9 @@ static void corrupt_build_disk(httrackp *opt) {
  memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
  remove(reconcile_st_path(opt, "hts-cache/new.zip"));
  remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
-          CORRUPT_ADR "/victim.bin");
+  /* X-Save stored verbatim: see disk_fallback_selftest */
+  concat(save, sizeof(save), StringBuff(opt->path_html_utf8),
+         CORRUPT_ADR "/victim.bin");
  selftest_open_for_write(&cache, opt);
  store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
              "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,
--- a/src/htscoremain.c
+++ b/src/htscoremain.c
@@ -2253,21 +2253,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
              htsmain_free();
              return -1;
            } else {
-              char *a;
+              char BIGSTK pname[HTS_URLMAXSIZE * 2];

              na++;
              opt->proxy.active = 1;
-              // Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
-              a = argv[na] + strlen(argv[na]) - 1;
-              while((a > argv[na]) && (*a != ':') && (*a != '@'))
-                a--;
-              if (*a == ':') {  // un port est présent, <proxy>:port
-                sscanf(a + 1, "%d", &opt->proxy.port);
-                StringCopyN(opt->proxy.name, argv[na], (int) (a - argv[na]));
-              } else {          // <proxy>
-                opt->proxy.port = 8080;
-                StringCopy(opt->proxy.name, argv[na]);
-              }
+              hts_parse_proxy(argv[na], pname, sizeof(pname), &opt->proxy.port);
+              StringCopy(opt->proxy.name, pname);
            }
            break;
          case 'F':            // user-agent field
--- a/src/htshelp.c
+++ b/src/htshelp.c
@@ -477,7 +477,7 @@ void help(const char *app, int more) {
    ("  Y   mirror ALL links located in the first level pages (mirror links)");
  infomsg("");
  infomsg("Proxy options:");
-  infomsg("  P  proxy use (-P proxy:port or -P user:pass@proxy:port)");
+  infomsg("  P  proxy use (-P [socks5://][user:pass@]proxy:port)");
  infomsg(" %f *use proxy for ftp (f0 don't use)");
  infomsg(" %b  use this local hostname to make/send requests (-%b hostname)");
  infomsg("");
--- a/src/htslib.c
+++ b/src/htslib.c
@@ -649,165 +649,6 @@ T_SOC http_fopen(httrackp * opt, const char *adr, const char *fil, htsblk * reto
  return http_xfopen(opt, 0, 1, 1, NULL, adr, fil, retour);
 }

-// Read a CRLF line from a non-blocking socket (waits up to timeout per recv).
-// Returns the line length (0 = empty), or -1 on timeout/EOF/error.
-static int proxy_getline(T_SOC soc, char *s, int max, int timeout) {
-  int j = 0;
-
-  for (;;) {
-    unsigned char ch;
-    int n;
-
-    if (!check_readinput_t(soc, timeout))
-      return -1; // timed out waiting for data
-    n = (int) recv(soc, &ch, 1, 0);
-    if (n == 1) {
-      if (ch == 13) // CR
-        continue;
-      if (ch == 10) // LF: end of line
-        break;
-      if (j >= max - 1)
-        return -1; // line too long: bound the read against a hostile proxy
-      s[j++] = (char) ch;
-    } else if (n == 0) {
-      return -1; // connection closed
-    } else {
-#ifdef _WIN32
-      if (WSAGetLastError() == WSAEWOULDBLOCK)
-        continue;
-#else
-      if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
-        continue;
-#endif
-      return -1;
-    }
-  }
-  s[j] = '\0';
-  return j;
-}
-
-int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
-                      int timeout) {
-  const T_SOC soc = retour->soc;
-  const char *const host = jump_identification_const(adr); // host[:port]
-  const char *const portsep = jump_toport_const(adr);      // ":port" or NULL
-  char BIGSTK authority[HTS_URLMAXSIZE * 2];
-  char BIGSTK req[HTS_URLMAXSIZE * 4 + 1100];
-  char line[1024];
-  int code;
-
-  if (soc == INVALID_SOCKET)
-    return 0;
-
-  // CONNECT needs an explicit host:port; default the https port
-  authority[0] = '\0';
-  if (portsep != NULL)
-    strlcatbuff(authority, host, sizeof(authority)); // already host:port
-  else
-    snprintf(authority, sizeof(authority), "%s:%d", host, 443);
-
-  // backstop: never let a stray CR/LF in the host smuggle a second line into
-  // the CONNECT request (the host is already sanitized upstream)
-  {
-    const char *c;
-
-    for (c = authority; *c != '\0'; c++) {
-      if ((unsigned char) *c < ' ') {
-        strcpybuff(retour->msg, "proxy CONNECT: invalid host");
-        return 0;
-      }
-    }
-  }
-
-  snprintf(req, sizeof(req), "CONNECT %s HTTP/1.0" H_CRLF "Host: %s" H_CRLF,
-           authority, authority);
-
-  // creds go on the CONNECT, not the tunneled origin request
-  if (link_has_authorization(retour->req.proxy.name)) {
-    const char *a = jump_identification_const(retour->req.proxy.name);
-    const char *astart = jump_protocol_const(retour->req.proxy.name);
-    char autorisation[1100];
-    char user_pass[256];
-
-    autorisation[0] = user_pass[0] = '\0';
-    strncatbuff(user_pass, astart, (int) (a - astart) - 1);
-    strcpybuff(user_pass, unescape_http(OPT_GET_BUFF(opt),
-                                        OPT_GET_BUFF_SIZE(opt), user_pass));
-    code64((unsigned char *) user_pass, (int) strlen(user_pass),
-           (unsigned char *) autorisation, 0);
-    strlcatbuff(req, "Proxy-Authorization: Basic ", sizeof(req));
-    strlcatbuff(req, autorisation, sizeof(req));
-    strlcatbuff(req, H_CRLF, sizeof(req));
-  }
-  strlcatbuff(req, H_CRLF, sizeof(req)); // end of request headers
-
-  // raw send: ssl is set, so sendc() would route to TLS
-  {
-    const char *p = req;
-    size_t remain = strlen(req);
-    int stalls = 0;
-
-    while (remain > 0) {
-      const int n = (int) send(soc, p, (int) remain, 0);
-
-      if (n > 0) {
-        p += n;
-        remain -= (size_t) n;
-        stalls = 0;
-      } else {
-#ifdef _WIN32
-        const int wouldblock = (WSAGetLastError() == WSAEWOULDBLOCK);
-#else
-        const int wouldblock =
-            (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR);
-#endif
-        // don't spin forever on a fatal error or an unwritable socket
-        if (!wouldblock || !check_writeinput_t(soc, timeout) ||
-            ++stalls > 100) {
-          strcpybuff(retour->msg, "proxy CONNECT: write error");
-          return 0;
-        }
-      }
-    }
-  }
-
-  // proxy status line: "HTTP/1.x <code> ..."
-  if (proxy_getline(soc, line, sizeof(line), timeout) < 0) {
-    strcpybuff(retour->msg, "proxy CONNECT: no response");
-    return 0;
-  }
-  if (sscanf(line, "HTTP/%*d.%*d %d", &code) < 1)
-    code = 0;
-  if (code < 200 || code >= 300) {
-    snprintf(retour->msg, sizeof(retour->msg), "proxy CONNECT refused: %s",
-             strnotempty(line) ? line : "(no status)");
-    return 0;
-  }
-
-  // drain headers to the blank line; cap the count so a flooding proxy can't
-  // stall the crawl
-  {
-    int headers = 0;
-
-    for (;;) {
-      const int n = proxy_getline(soc, line, sizeof(line), timeout);
-
-      if (n < 0) {
-        strcpybuff(retour->msg, "proxy CONNECT: truncated response");
-        return 0;
-      }
-      if (n == 0)
-        break; // blank line: tunnel ready
-      if (++headers > 64) {
-        strcpybuff(retour->msg, "proxy CONNECT: too many response headers");
-        return 0;
-      }
-    }
-  }
-
-  return 1;
-}
-
 // ouverture d'une liaison http, envoi d'une requète
 // mode: 0 GET  1 HEAD  [2 POST]
 // treat: traiter header?
@@ -1106,8 +947,9 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
          /* widths bound method[256], url[HTS_URLMAXSIZE*2], protocol[256] */
          if (sscanf(line, "%255s %2047s %255s", method, url, protocol) == 3) {
            size_t ret;
-            // selon que l'on a ou pas un proxy
-            if (retour->req.proxy.active) {
+            // absolute-URI for an http proxy; a socks tunnel takes origin-form
+            if (retour->req.proxy.active &&
+                !hts_proxy_is_socks(retour->req.proxy.name)) {
              print_buffer(&bstr,
                      "%s http://%s%s %s\r\n", method, adr, url,
                      protocol);
@@ -1143,8 +985,10 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
        print_buffer(&bstr, "HEAD ");
    }

-    // si on gère un proxy, il faut une Absolute URI: on ajoute avant http://www.adr.dom
-    if (retour->req.proxy.active && (strncmp(adr, "https://", 8) != 0)) {
+    // an http proxy needs an absolute URI; a socks tunnel does not
+    if (retour->req.proxy.active &&
+        !hts_proxy_is_socks(retour->req.proxy.name) &&
+        (strncmp(adr, "https://", 8) != 0)) {
      if (!link_has_authority(adr)) {   // default http
 #if HDEBUG
        printf("Proxy Use: for %s%s proxy %d port %d\n", adr, fil,
@@ -1188,8 +1032,11 @@ int http_sendhead(httrackp * opt, t_cookie * cookie, int mode,
    if (xsend)
      print_buffer(&bstr, "%s", xsend);  // éventuelles autres lignes

-    // for https, auth rides the CONNECT (the tunneled GET would leak it)
-    if (retour->req.proxy.active && strncmp(adr, "https://", 8) != 0) {
+    // for https, auth rides the CONNECT (the tunneled GET would leak it); for
+    // socks, the handshake
+    if (retour->req.proxy.active &&
+        !hts_proxy_is_socks(retour->req.proxy.name) &&
+        strncmp(adr, "https://", 8) != 0) {
      if (link_has_authorization(retour->req.proxy.name)) {     // et hop, authentification proxy!
        const char *a = jump_identification_const(retour->req.proxy.name);
        const char *astart = jump_protocol_const(retour->req.proxy.name);
@@ -3866,6 +3713,10 @@ const char *jump_protocol_const(const char *source) {
    source += p;
  else if ((p = strfield(source, "file:")))
    source += p;
+  else if ((p = strfield(source, "socks5h:")))
+    source += p;
+  else if ((p = strfield(source, "socks5:")))
+    source += p;
  // net_path
  if (strncmp(source, "//", 2) == 0)
    source += 2;
@@ -3874,6 +3725,48 @@ const char *jump_protocol_const(const char *source) {

 DECLARE_NON_CONST_VERSION(jump_protocol)

+hts_boolean hts_proxy_is_socks(const char *name) {
+  if (name == NULL)
+    return HTS_FALSE;
+  return (strfield(name, "socks5h:") || strfield(name, "socks5:")) ? HTS_TRUE
+                                                                   : HTS_FALSE;
+}
+
+// default proxy port for a -P argument, keyed on the scheme
+static int proxy_default_port(const char *arg) {
+  return hts_proxy_is_socks(arg) ? 1080 : 8080;
+}
+
+void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
+  const char *authority = strstr(arg, "://");
+  const char *a;
+  size_t namelen;
+
+  if (name_size == 0)
+    return;
+  // scan back to the port ':' (or userinfo '@'), but never past the authority,
+  // so a scheme's own colon is not read as a port separator; inspect a[-1] from
+  // one-past-end so no pointer below the string is ever formed
+  authority = (authority != NULL) ? authority + 3 : arg;
+  a = arg + strlen(arg);
+  while (a > authority && a[-1] != ':' && a[-1] != '@')
+    a--;
+  if (a > authority && a[-1] == ':') {
+    int p = -1;
+
+    sscanf(a, "%d", &p);
+    *port = (p > 0) ? p : proxy_default_port(arg);
+    namelen = (size_t) (a - 1 - arg);
+  } else {
+    *port = proxy_default_port(arg);
+    namelen = strlen(arg);
+  }
+  if (namelen >= name_size) // arg is user-controlled: truncate, don't overflow
+    namelen = name_size - 1;
+  memcpy(name, arg, namelen);
+  name[namelen] = '\0';
+}
+
 // codage base 64 a vers b
 void code64(unsigned char *a, int size_a, unsigned char *b, int crlf) {
  int i1 = 0, i2 = 0, i3 = 0, i4 = 0;
--- a/src/htslib.h
+++ b/src/htslib.h
@@ -207,15 +207,9 @@ int check_readinput(htsblk * r);
 int check_readinput_t(T_SOC soc, int timeout);
 int check_writeinput_t(T_SOC soc, int timeout);

-/* Open an HTTP CONNECT tunnel through the active proxy for an https request:
-   `retour->soc` must already be TCP-connected to the proxy, and `adr` is the
-   origin authority (url_adr, e.g. "https://host:port"). Sends the CONNECT
-   request (with Proxy-Authorization when the proxy carries credentials) and
-   reads the proxy's status line, so the caller's TLS handshake then runs
-   end-to-end with the origin. Blocks up to `timeout` seconds. Returns 1 on a
-   2xx tunnel, 0 on failure (retour->msg/statuscode set). */
-int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
-                      int timeout);
+/* TRUE if this -P proxy name (which keeps its scheme) is a SOCKS5 proxy. */
+hts_boolean hts_proxy_is_socks(const char *name);
+
 void treathead(t_cookie * cookie, const char *adr, const char *fil, htsblk * retour,
               char *rcvd);
 void treatfirstline(htsblk * retour, const char *rcvd);
@@ -289,6 +283,12 @@ int may_unknown2(httrackp * opt, const char *mime, const char *filename);
 const char *strrchr_limit(const char *s, char c, const char *limit);
 char *jump_protocol(char *source);
 const char *jump_protocol_const(const char *source);
+
+/* Split a -P proxy argument "[scheme://][user:pass@]host[:port]" into the proxy
+   host string (scheme and any user:pass kept, for later stripping and auth),
+   written NUL-terminated into name[name_size] (truncated to fit), and the port
+   in *port. The port defaults by scheme: 1080 for socks5/socks5h, else 8080. */
+void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port);
 void code64(unsigned char *a, int size_a, unsigned char *b, int crlf);

 #define copychar(catbuff,a) concat(catbuff,(a),NULL)
--- a/src/htsproxy.c
+++ b/src/htsproxy.c
@@ -0,0 +1,539 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: Proxy tunneling (HTTP CONNECT, SOCKS5)                 */
+/* Author: Xavier Roche                                         */
+/* ------------------------------------------------------------ */
+
+/* Internal engine bytecode */
+#define HTS_INTERNAL_BYTECODE
+
+#include "htscore.h"
+
+#include "htslib.h"
+#include "htsproxy.h"
+
+#include <string.h>
+
+// Read a CRLF line from a non-blocking socket (waits up to timeout per recv).
+// Returns the line length (0 = empty), or -1 on timeout/EOF/error.
+static int proxy_getline(T_SOC soc, char *s, int max, int timeout) {
+  int j = 0;
+
+  for (;;) {
+    unsigned char ch;
+    int n;
+
+    if (!check_readinput_t(soc, timeout))
+      return -1; // timed out waiting for data
+    n = (int) recv(soc, &ch, 1, 0);
+    if (n == 1) {
+      if (ch == 13) // CR
+        continue;
+      if (ch == 10) // LF: end of line
+        break;
+      if (j >= max - 1)
+        return -1; // line too long: bound the read against a hostile proxy
+      s[j++] = (char) ch;
+    } else if (n == 0) {
+      return -1; // connection closed
+    } else {
+#ifdef _WIN32
+      if (WSAGetLastError() == WSAEWOULDBLOCK)
+        continue;
+#else
+      if (errno == EINTR || errno == EAGAIN || errno == EWOULDBLOCK)
+        continue;
+#endif
+      return -1;
+    }
+  }
+  s[j] = '\0';
+  return j;
+}
+
+int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
+                      int timeout) {
+  const T_SOC soc = retour->soc;
+  const char *const host = jump_identification_const(adr); // host[:port]
+  const char *const portsep = jump_toport_const(adr);      // ":port" or NULL
+  char BIGSTK authority[HTS_URLMAXSIZE * 2];
+  char BIGSTK req[HTS_URLMAXSIZE * 4 + 1100];
+  char line[1024];
+  int code;
+
+  if (soc == INVALID_SOCKET)
+    return 0;
+
+  // CONNECT needs an explicit host:port; default the https port
+  authority[0] = '\0';
+  if (portsep != NULL)
+    strlcatbuff(authority, host, sizeof(authority)); // already host:port
+  else
+    snprintf(authority, sizeof(authority), "%s:%d", host, 443);
+
+  // backstop: never let a stray CR/LF in the host smuggle a second line into
+  // the CONNECT request (the host is already sanitized upstream)
+  {
+    const char *c;
+
+    for (c = authority; *c != '\0'; c++) {
+      if ((unsigned char) *c < ' ') {
+        strcpybuff(retour->msg, "proxy CONNECT: invalid host");
+        return 0;
+      }
+    }
+  }
+
+  snprintf(req, sizeof(req), "CONNECT %s HTTP/1.0" H_CRLF "Host: %s" H_CRLF,
+           authority, authority);
+
+  // creds go on the CONNECT, not the tunneled origin request
+  if (link_has_authorization(retour->req.proxy.name)) {
+    const char *a = jump_identification_const(retour->req.proxy.name);
+    const char *astart = jump_protocol_const(retour->req.proxy.name);
+    char autorisation[1100];
+    char user_pass[256];
+
+    autorisation[0] = user_pass[0] = '\0';
+    strncatbuff(user_pass, astart, (int) (a - astart) - 1);
+    strcpybuff(user_pass, unescape_http(OPT_GET_BUFF(opt),
+                                        OPT_GET_BUFF_SIZE(opt), user_pass));
+    code64((unsigned char *) user_pass, (int) strlen(user_pass),
+           (unsigned char *) autorisation, 0);
+    strlcatbuff(req, "Proxy-Authorization: Basic ", sizeof(req));
+    strlcatbuff(req, autorisation, sizeof(req));
+    strlcatbuff(req, H_CRLF, sizeof(req));
+  }
+  strlcatbuff(req, H_CRLF, sizeof(req)); // end of request headers
+
+  // raw send: ssl is set, so sendc() would route to TLS
+  {
+    const char *p = req;
+    size_t remain = strlen(req);
+    int stalls = 0;
+
+    while (remain > 0) {
+      const int n = (int) send(soc, p, (int) remain, 0);
+
+      if (n > 0) {
+        p += n;
+        remain -= (size_t) n;
+        stalls = 0;
+      } else {
+#ifdef _WIN32
+        const int wouldblock = (WSAGetLastError() == WSAEWOULDBLOCK);
+#else
+        const int wouldblock =
+            (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR);
+#endif
+        // don't spin forever on a fatal error or an unwritable socket
+        if (!wouldblock || !check_writeinput_t(soc, timeout) ||
+            ++stalls > 100) {
+          strcpybuff(retour->msg, "proxy CONNECT: write error");
+          return 0;
+        }
+      }
+    }
+  }
+
+  // proxy status line: "HTTP/1.x <code> ..."
+  if (proxy_getline(soc, line, sizeof(line), timeout) < 0) {
+    strcpybuff(retour->msg, "proxy CONNECT: no response");
+    return 0;
+  }
+  if (sscanf(line, "HTTP/%*d.%*d %d", &code) < 1)
+    code = 0;
+  if (code < 200 || code >= 300) {
+    snprintf(retour->msg, sizeof(retour->msg), "proxy CONNECT refused: %s",
+             strnotempty(line) ? line : "(no status)");
+    return 0;
+  }
+
+  // drain headers to the blank line; cap the count so a flooding proxy can't
+  // stall the crawl
+  {
+    int headers = 0;
+
+    for (;;) {
+      const int n = proxy_getline(soc, line, sizeof(line), timeout);
+
+      if (n < 0) {
+        strcpybuff(retour->msg, "proxy CONNECT: truncated response");
+        return 0;
+      }
+      if (n == 0)
+        break; // blank line: tunnel ready
+      if (++headers > 64) {
+        strcpybuff(retour->msg, "proxy CONNECT: too many response headers");
+        return 0;
+      }
+    }
+  }
+
+  return 1;
+}
+
+/* SOCKS5 client (RFC 1928, RFC 1929 auth), hostname mode only: the proxy
+   resolves the origin name (remote DNS, curl's socks5h). The stream is the
+   proxy socket, or a scripted buffer under -#test=socks5. */
+
+#define SOCKS5_VERSION 0x05
+#define SOCKS5_AUTH_VERSION 0x01
+#define SOCKS5_METHOD_NONE 0x00
+#define SOCKS5_METHOD_USERPASS 0x02
+#define SOCKS5_METHOD_NOACCEPTABLE 0xFF
+#define SOCKS5_CMD_CONNECT 0x01
+#define SOCKS5_ATYP_IPV4 0x01
+#define SOCKS5_ATYP_DOMAIN 0x03
+#define SOCKS5_ATYP_IPV6 0x04
+#define SOCKS5_MAXFIELD 255 /* one length byte: host, user and password */
+
+typedef struct socks5_stream {
+  T_SOC soc; /* INVALID_SOCKET when scripted (self-test) */
+  int timeout;
+  socks5_test_io *io;
+} socks5_stream;
+
+static int socks5_fail(char *msg, size_t msgsize, const char *text) {
+  if (msgsize != 0)
+    strlcpybuff(msg, text, msgsize);
+  return 0;
+}
+
+/* Read exactly n bytes, or fail: SOCKS frames are not self-delimiting, so a
+   short read would desync the stream shared with the origin traffic. */
+static int socks5_read_n(socks5_stream *st, unsigned char *buf, size_t n) {
+  size_t got = 0;
+  int stalls = 0;
+
+  if (st->soc == INVALID_SOCKET) { /* scripted */
+    socks5_test_io *const io = st->io;
+
+    if (n > io->reply_len - io->consumed)
+      return 0;
+    memcpy(buf, io->reply + io->consumed, n);
+    io->consumed += n;
+    return 1;
+  }
+  while (got < n) {
+    const int r = (int) recv(st->soc, (char *) buf + got, (int) (n - got), 0);
+
+    if (r > 0) {
+      got += (size_t) r;
+      stalls = 0;
+    } else if (r == 0) {
+      return 0; // proxy closed mid-frame
+    } else {
+#ifdef _WIN32
+      const int wouldblock = (WSAGetLastError() == WSAEWOULDBLOCK);
+#else
+      const int wouldblock =
+          (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR);
+#endif
+      if (!wouldblock || !check_readinput_t(st->soc, st->timeout) ||
+          ++stalls > 100)
+        return 0;
+    }
+  }
+  return 1;
+}
+
+static int socks5_write_all(socks5_stream *st, const unsigned char *buf,
+                            size_t len) {
+  size_t remain = len;
+  int stalls = 0;
+
+  if (st->soc == INVALID_SOCKET) { /* scripted */
+    socks5_test_io *const io = st->io;
+
+    if (len > sizeof(io->sent) - io->sent_len)
+      return 0;
+    memcpy(io->sent + io->sent_len, buf, len);
+    io->sent_len += len;
+    return 1;
+  }
+  while (remain > 0) {
+    // raw send: the socket is still plain here, sendc() would route to TLS
+    const int n = (int) send(st->soc, (const char *) buf, (int) remain, 0);
+
+    if (n > 0) {
+      buf += n;
+      remain -= (size_t) n;
+      stalls = 0;
+    } else {
+#ifdef _WIN32
+      const int wouldblock = (WSAGetLastError() == WSAEWOULDBLOCK);
+#else
+      const int wouldblock =
+          (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR);
+#endif
+      if (!wouldblock || !check_writeinput_t(st->soc, st->timeout) ||
+          ++stalls > 100)
+        return 0;
+    }
+  }
+  return 1;
+}
+
+static const char *socks5_rep_message(unsigned char rep) {
+  switch (rep) {
+  case 0x01:
+    return "general failure";
+  case 0x02:
+    return "connection not allowed by ruleset";
+  case 0x03:
+    return "network unreachable";
+  case 0x04:
+    return "host unreachable";
+  case 0x05:
+    return "connection refused";
+  case 0x06:
+    return "TTL expired";
+  case 0x07:
+    return "command not supported";
+  case 0x08:
+    return "address type not supported";
+  default:
+    return "unknown error";
+  }
+}
+
+/* The reply's BND.ADDR is variable-length: consume exactly what ATYP says, or
+   the leftover bytes prepend to the first origin (or TLS) read. */
+static int socks5_read_reply(socks5_stream *st, char *msg, size_t msgsize) {
+  unsigned char head[4];
+  unsigned char addr[SOCKS5_MAXFIELD + 2];
+  size_t addrlen;
+
+  if (!socks5_read_n(st, head, sizeof(head)))
+    return socks5_fail(msg, msgsize, "SOCKS5: no reply from proxy");
+  if (head[0] != SOCKS5_VERSION)
+    return socks5_fail(msg, msgsize, "SOCKS5: bad version in reply");
+  if (head[1] != 0x00) {
+    snprintf(msg, msgsize, "SOCKS5 connect failed: %s",
+             socks5_rep_message(head[1]));
+    return 0;
+  }
+  switch (head[3]) {
+  case SOCKS5_ATYP_IPV4:
+    addrlen = 4;
+    break;
+  case SOCKS5_ATYP_IPV6:
+    addrlen = 16;
+    break;
+  case SOCKS5_ATYP_DOMAIN: {
+    unsigned char len;
+
+    if (!socks5_read_n(st, &len, 1))
+      return socks5_fail(msg, msgsize, "SOCKS5: truncated reply");
+    addrlen = len; // <= 255, always fits addr[]
+    break;
+  }
+  default: // unknown length: the stream cannot be resynchronized
+    return socks5_fail(msg, msgsize, "SOCKS5: unknown address type in reply");
+  }
+  if (addrlen != 0 && !socks5_read_n(st, addr, addrlen))
+    return socks5_fail(msg, msgsize, "SOCKS5: truncated reply address");
+  if (!socks5_read_n(st, addr, 2)) // BND.PORT, unused
+    return socks5_fail(msg, msgsize, "SOCKS5: truncated reply port");
+  return 1;
+}
+
+static int socks5_negotiate(socks5_stream *st, const char *host, size_t hostlen,
+                            int port, const char *user, size_t userlen,
+                            const char *pass, size_t passlen, int want_auth,
+                            char *msg, size_t msgsize) {
+  unsigned char frame[3 + SOCKS5_MAXFIELD + SOCKS5_MAXFIELD];
+  unsigned char rep[2];
+
+  /* greeting */
+  frame[0] = SOCKS5_VERSION;
+  frame[1] = (unsigned char) (want_auth ? 2 : 1);
+  frame[2] = SOCKS5_METHOD_NONE;
+  frame[3] = SOCKS5_METHOD_USERPASS;
+  if (!socks5_write_all(st, frame, (size_t) 2 + frame[1]))
+    return socks5_fail(msg, msgsize, "SOCKS5: write error");
+  if (!socks5_read_n(st, rep, sizeof(rep)))
+    return socks5_fail(msg, msgsize, "SOCKS5: no method reply from proxy");
+  if (rep[0] != SOCKS5_VERSION)
+    return socks5_fail(msg, msgsize, "SOCKS5: bad version in method reply");
+  switch (rep[1]) {
+  case SOCKS5_METHOD_NONE:
+    break;
+  case SOCKS5_METHOD_USERPASS:
+    if (!want_auth)
+      return socks5_fail(msg, msgsize,
+                         "SOCKS5: proxy requires authentication, none given");
+    /* RFC 1929 sub-negotiation; its version byte is 0x01, not 0x05 */
+    frame[0] = SOCKS5_AUTH_VERSION;
+    frame[1] = (unsigned char) userlen;
+    memcpy(frame + 2, user, userlen);
+    frame[2 + userlen] = (unsigned char) passlen;
+    memcpy(frame + 3 + userlen, pass, passlen);
+    if (!socks5_write_all(st, frame, 3 + userlen + passlen))
+      return socks5_fail(msg, msgsize, "SOCKS5: write error");
+    if (!socks5_read_n(st, rep, sizeof(rep)))
+      return socks5_fail(msg, msgsize, "SOCKS5: no authentication reply");
+    if (rep[1] != 0x00)
+      return socks5_fail(msg, msgsize, "SOCKS5: authentication failed");
+    break;
+  case SOCKS5_METHOD_NOACCEPTABLE:
+    return socks5_fail(
+        msg, msgsize,
+        "SOCKS5: proxy accepts no authentication method we offer");
+  default:
+    return socks5_fail(msg, msgsize,
+                       "SOCKS5: proxy selected an unknown method");
+  }
+
+  /* CONNECT to the origin, by name */
+  frame[0] = SOCKS5_VERSION;
+  frame[1] = SOCKS5_CMD_CONNECT;
+  frame[2] = 0x00; // RSV
+  frame[3] = SOCKS5_ATYP_DOMAIN;
+  frame[4] = (unsigned char) hostlen;
+  memcpy(frame + 5, host, hostlen);
+  frame[5 + hostlen] = (unsigned char) (port >> 8);
+  frame[6 + hostlen] = (unsigned char) (port & 0xFF);
+  if (!socks5_write_all(st, frame, 7 + hostlen))
+    return socks5_fail(msg, msgsize, "SOCKS5: write error");
+
+  return socks5_read_reply(st, msg, msgsize);
+}
+
+/* Decode the proxy userinfo into the two RFC 1929 fields. Split on the first
+   colon of the still-escaped string, so a %3A stays inside the username. */
+static int socks5_credentials(httrackp *opt, const char *proxy_name, char *user,
+                              size_t user_size, size_t *userlen, char *pass,
+                              size_t pass_size, size_t *passlen, char *msg,
+                              size_t msgsize) {
+  const char *const a = jump_identification_const(proxy_name); // past the '@'
+  const char *const astart = jump_protocol_const(proxy_name);
+  char userinfo[1024];
+  char *colon;
+
+  if (a <= astart || (size_t) (a - astart) - 1 >= sizeof(userinfo))
+    return socks5_fail(msg, msgsize, "SOCKS5: credentials too long");
+  userinfo[0] = '\0';
+  strncatbuff(userinfo, astart, (int) (a - astart) - 1);
+  colon = strchr(userinfo, ':');
+  if (colon != NULL)
+    *colon++ = '\0';
+  strlcpybuff(
+      user, unescape_http(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), userinfo),
+      user_size);
+  strlcpybuff(pass,
+              colon != NULL ? unescape_http(OPT_GET_BUFF(opt),
+                                            OPT_GET_BUFF_SIZE(opt), colon)
+                            : "",
+              pass_size);
+  *userlen = strlen(user);
+  *passlen = strlen(pass);
+  // reject, never truncate: a clipped secret would authenticate as another one
+  if (*userlen > SOCKS5_MAXFIELD || *passlen > SOCKS5_MAXFIELD)
+    return socks5_fail(msg, msgsize, "SOCKS5: credentials too long");
+  if (*userlen == 0)
+    return socks5_fail(msg, msgsize, "SOCKS5: empty proxy username");
+  return 1;
+}
+
+static int socks5_handshake_stream(httrackp *opt, socks5_stream *st,
+                                   const char *adr, const char *proxy_name,
+                                   int ssl, char *msg, size_t msgsize) {
+  const char *const host = jump_identification_const(adr);
+  const char *const portsep = jump_toport_const(adr);
+  const size_t hostlen =
+      portsep != NULL ? (size_t) (portsep - host) : strlen(host);
+  // sized for the whole userinfo: decoding shrinks, so an over-long credential
+  // lands intact and is rejected below rather than silently clipped
+  char user[1024];
+  char pass[1024];
+  size_t userlen = 0, passlen = 0;
+  int want_auth = 0;
+  int port = ssl ? 443 : 80;
+  size_t i;
+
+  if (hostlen == 0 || hostlen > SOCKS5_MAXFIELD)
+    return socks5_fail(msg, msgsize, "SOCKS5: invalid origin host");
+  if (host[0] == '[') // ATYP=domain cannot carry an IPv6 literal
+    return socks5_fail(msg, msgsize,
+                       "SOCKS5: IPv6 literal origin is not supported");
+  for (i = 0; i < hostlen; i++) {
+    if ((unsigned char) host[i] < ' ')
+      return socks5_fail(msg, msgsize, "SOCKS5: invalid origin host");
+  }
+  if (portsep != NULL) {
+    int p = -1;
+
+    sscanf(portsep + 1, "%d", &p);
+    if (p <= 0 || p > 65535)
+      return socks5_fail(msg, msgsize, "SOCKS5: invalid origin port");
+    port = p;
+  }
+  if (link_has_authorization(proxy_name)) {
+    if (!socks5_credentials(opt, proxy_name, user, sizeof(user), &userlen, pass,
+                            sizeof(pass), &passlen, msg, msgsize))
+      return 0;
+    want_auth = 1;
+  }
+  return socks5_negotiate(st, host, hostlen, port, user, userlen, pass, passlen,
+                          want_auth, msg, msgsize);
+}
+
+int socks5_handshake(httrackp *opt, htsblk *retour, const char *adr,
+                     int timeout) {
+  socks5_stream st;
+#if HTS_USEOPENSSL
+  const int ssl = retour->ssl;
+#else
+  const int ssl = 0;
+#endif
+
+  if (retour->soc == INVALID_SOCKET)
+    return 0;
+  st.soc = retour->soc;
+  st.timeout = timeout;
+  st.io = NULL;
+  return socks5_handshake_stream(opt, &st, adr, retour->req.proxy.name, ssl,
+                                 retour->msg, sizeof(retour->msg));
+}
+
+int socks5_handshake_scripted(httrackp *opt, const char *adr,
+                              const char *proxy_name, socks5_test_io *io) {
+  socks5_stream st;
+
+  st.soc = INVALID_SOCKET;
+  st.timeout = 0;
+  st.io = io;
+  io->consumed = 0;
+  io->sent_len = 0;
+  io->msg[0] = '\0';
+  return socks5_handshake_stream(opt, &st, adr, proxy_name, 0, io->msg,
+                                 sizeof(io->msg));
+}
--- a/src/htsproxy.h
+++ b/src/htsproxy.h
@@ -0,0 +1,82 @@
+/* ------------------------------------------------------------ */
+/*
+HTTrack Website Copier, Offline Browser for Windows and Unix
+Copyright (C) 2026 Xavier Roche and other contributors
+
+SPDX-License-Identifier: GPL-3.0-or-later
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+Ethical use: we kindly ask that you NOT use this software to harvest email
+addresses or to collect any other private information about people. Doing so
+would dishonor our work and waste the many hours we have spent on it.
+
+Please visit our Website: http://www.httrack.com
+*/
+
+/* ------------------------------------------------------------ */
+/* File: Proxy tunneling (HTTP CONNECT, SOCKS5) .h              */
+/* Author: Xavier Roche                                         */
+/* ------------------------------------------------------------ */
+
+#ifndef HTS_DEFPROXY
+#define HTS_DEFPROXY
+
+#include "htsglobal.h"
+
+/* Forward definitions */
+#ifndef HTS_DEF_FWSTRUCT_httrackp
+#define HTS_DEF_FWSTRUCT_httrackp
+typedef struct httrackp httrackp;
+#endif
+#ifndef HTS_DEF_FWSTRUCT_htsblk
+#define HTS_DEF_FWSTRUCT_htsblk
+typedef struct htsblk htsblk;
+#endif
+
+/* Open an HTTP CONNECT tunnel through the active proxy for an https request:
+   `retour->soc` must already be TCP-connected to the proxy, and `adr` is the
+   origin authority (url_adr, e.g. "https://host:port"). Sends the CONNECT
+   request (with Proxy-Authorization when the proxy carries credentials) and
+   reads the proxy's status line, so the caller's TLS handshake then runs
+   end-to-end with the origin. Blocks up to `timeout` seconds. Returns 1 on a
+   2xx tunnel, 0 on failure (retour->msg/statuscode set). */
+int http_proxy_tunnel(httrackp *opt, htsblk *retour, const char *adr,
+                      int timeout);
+
+/* SOCKS5 (RFC 1928, RFC 1929 auth) handshake on the raw proxy socket, before
+   any TLS: `retour->soc` must be TCP-connected to the proxy and `adr` is the
+   origin (url_adr). The origin name is sent verbatim (ATYP=domain), so the
+   proxy resolves it; a bracketed IPv6 literal origin is rejected. Blocks up to
+   `timeout` seconds. Returns 1 with the stream positioned on the first origin
+   byte, 0 on failure (retour->msg set). */
+int socks5_handshake(httrackp *opt, htsblk *retour, const char *adr,
+                     int timeout);
+
+/* Self-test hook (-#test=socks5): run the handshake against `reply`, a scripted
+   server byte stream, instead of a socket. `sent` captures the client frames,
+   `consumed` the reply bytes eaten (the frame-desync guard). */
+typedef struct socks5_test_io {
+  const unsigned char *reply;
+  size_t reply_len;
+  size_t consumed;
+  unsigned char sent[1024];
+  size_t sent_len;
+  char msg[256];
+} socks5_test_io;
+
+int socks5_handshake_scripted(httrackp *opt, const char *adr,
+                              const char *proxy_name, socks5_test_io *io);
+
+#endif
--- a/src/htsselftest.c
+++ b/src/htsselftest.c
@@ -55,6 +55,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsmd5.h"
 #include "htssniff.h"
 #include "htscodec.h"
+#include "htsproxy.h"
 #if HTS_USEZLIB
 #include "htszlib.h"
 #endif
@@ -1308,6 +1309,162 @@ static int st_identurl(httrackp *opt, int argc, char **argv) {
  return 0;
 }

+static int st_proxyurl(httrackp *opt, int argc, char **argv) {
+  char BIGSTK name[HTS_URLMAXSIZE * 2];
+  int port = -1;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "proxyurl: needs a proxy argument\n");
+    return 1;
+  }
+  hts_parse_proxy(argv[0], name, sizeof(name), &port);
+  // host= is what the connect actually resolves (scheme + user:pass stripped)
+  printf("name=%s port=%d host=%s\n", name, port,
+         jump_identification_const(name));
+  return 0;
+}
+
+/* Scripted SOCKS5 server: no-auth method reply, then a success CONNECT reply
+   carrying the given ATYP address, then a sentinel origin byte the handshake
+   must leave unread (an off-by-one in the reply framing eats it). */
+static size_t socks5_reply(unsigned char *buf, unsigned char atyp,
+                           const unsigned char *addr, size_t addrlen) {
+  size_t n = 0;
+
+  buf[n++] = 0x05;
+  buf[n++] = 0x00; /* method: no authentication */
+  buf[n++] = 0x05;
+  buf[n++] = 0x00; /* REP: succeeded */
+  buf[n++] = 0x00;
+  buf[n++] = atyp;
+  memcpy(buf + n, addr, addrlen);
+  n += addrlen;
+  buf[n++] = 0x1f;
+  buf[n++] = 0x90; /* BND.PORT */
+  buf[n++] = 0xAA; /* sentinel */
+  return n;
+}
+
+#define SOCKS5_SENTINEL_LEFT(io, len) ((io).consumed == (len) - 1)
+
+static int st_socks5(httrackp *opt, int argc, char **argv) {
+  static const unsigned char v4[4] = {127, 0, 0, 1};
+  static const unsigned char v6[16] = {0};
+  static const unsigned char domain[6] = {5, 'p', 'r', 'o', 'x', 'y'};
+  const char *const proxy = "socks5://127.0.0.1";
+  unsigned char script[64];
+  socks5_test_io io;
+  size_t len;
+
+  (void) argc;
+  (void) argv;
+
+  /* each reply address type is drained exactly, sentinel untouched */
+  len = socks5_reply(script, 0x01, v4, sizeof(v4));
+  io.reply = script;
+  io.reply_len = len;
+  assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 1);
+  assertf(SOCKS5_SENTINEL_LEFT(io, len));
+  /* the greeting offers no-auth only, and the origin goes out by name, port 80
+   */
+  assertf(io.sent_len == 3 + 7 + 11);
+  assertf(memcmp(io.sent, "\x05\x01\x00", 3) == 0);
+  assertf(memcmp(io.sent + 3, "\x05\x01\x00\x03\x0borigin.test\x00\x50", 18) ==
+          0);
+
+  len = socks5_reply(script, 0x04, v6, sizeof(v6));
+  io.reply = script;
+  io.reply_len = len;
+  assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 1);
+  assertf(SOCKS5_SENTINEL_LEFT(io, len));
+
+  len = socks5_reply(script, 0x03, domain, sizeof(domain));
+  io.reply = script;
+  io.reply_len = len;
+  assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 1);
+  assertf(SOCKS5_SENTINEL_LEFT(io, len));
+
+  /* an unknown address type has no known length: fail, never guess */
+  len = socks5_reply(script, 0x02, v4, sizeof(v4));
+  io.reply = script;
+  io.reply_len = len;
+  assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 0);
+
+  /* truncated frames (here: no BND.PORT) fail instead of over-reading */
+  len = socks5_reply(script, 0x01, v4, sizeof(v4)) - 3;
+  io.reply = script;
+  io.reply_len = len;
+  assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 0);
+
+  /* explicit origin port is encoded big-endian (8443 = 0x20fb) */
+  len = socks5_reply(script, 0x01, v4, sizeof(v4));
+  io.reply = script;
+  io.reply_len = len;
+  assertf(socks5_handshake_scripted(opt, "origin.test:8443", proxy, &io) == 1);
+  assertf(memcmp(io.sent + io.sent_len - 2, "\x20\xfb", 2) == 0);
+
+  /* credentials: split on the first colon of the escaped userinfo, so %3a stays
+     inside the username and a colon in the password is not a delimiter */
+  {
+    static const unsigned char auth_script[] = {
+        0x05, 0x02,             /* method: user/pass */
+        0x01, 0x00,             /* auth: success */
+        0x05, 0x00, 0x00, 0x01, /* REP: succeeded, ATYP ipv4 */
+        127,  0,    0,    1,    0x1f, 0x90, 0xAA};
+
+    io.reply = auth_script;
+    io.reply_len = sizeof(auth_script);
+    assertf(socks5_handshake_scripted(opt, "origin.test",
+                                      "socks5://us%3aer:p:ass@127.0.0.1",
+                                      &io) == 1);
+    assertf(SOCKS5_SENTINEL_LEFT(io, sizeof(auth_script)));
+    assertf(memcmp(io.sent, "\x05\x02\x00\x02", 4) == 0);
+    assertf(memcmp(io.sent + 4, "\x01\x05us:er\x05p:ass", 13) == 0);
+  }
+
+  /* a proxy demanding auth we cannot provide, and one refusing every method */
+  io.reply = (const unsigned char *) "\x05\x02";
+  io.reply_len = 2;
+  assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 0);
+  io.reply = (const unsigned char *) "\x05\xff";
+  io.reply_len = 2;
+  assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 0);
+
+  /* a refused CONNECT is an error, not a tunnel */
+  io.reply = (const unsigned char
+                  *) "\x05\x00\x05\x05\x00\x01\x7f\x00\x00\x01\x1f\x90";
+  io.reply_len = 12;
+  assertf(socks5_handshake_scripted(opt, "origin.test", proxy, &io) == 0);
+
+  /* over-long host or credentials are rejected before anything is sent */
+  {
+    char host[512];
+    char name[1024];
+    size_t i;
+
+    host[0] = '\0';
+    for (i = 0; i < 256; i++)
+      strcatbuff(host, "a");
+    io.reply = script;
+    io.reply_len = len;
+    assertf(socks5_handshake_scripted(opt, host, proxy, &io) == 0);
+    assertf(io.sent_len == 0);
+
+    strcpybuff(name, "socks5://user:");
+    for (i = 0; i < 256; i++)
+      strcatbuff(name, "p");
+    strcatbuff(name, "@127.0.0.1");
+    io.reply = script;
+    io.reply_len = len;
+    assertf(socks5_handshake_scripted(opt, "origin.test", name, &io) == 0);
+    assertf(io.sent_len == 0);
+  }
+
+  printf("socks5 self-test OK\n");
+  return 0;
+}
+
 /* Regression for the one-byte fil[] overflow: a 2047-byte hostless "?"-URL used
   to abort in strncat_safe_ when the missing leading '/' pushed fil to 2048. */
 static int st_identabs(httrackp *opt, int argc, char **argv) {
@@ -2756,6 +2913,10 @@ static const struct selftest_entry {
    {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
     st_resolve},
    {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
+    {"proxyurl", "<proxy-arg>", "parse a -P proxy URL into host/port",
+     st_proxyurl},
+    {"socks5", "", "SOCKS5 handshake framing and credential self-test",
+     st_socks5},
    {"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
     st_identabs},
    {"header", "<raw-header-line> ...", "response header-line parsing",
--- a/src/httrack.rc
+++ b/src/httrack.rc
@@ -1,5 +1,5 @@
-// Version resource for httrack.exe, the command line program. See version.rc.
-#define VER_FILE_DESCRIPTION "HTTrack Website Copier (command line)"
-#define VER_ORIGINAL_FILENAME "httrack.exe"
-#define VER_FILETYPE VFT_APP
-#include "version.rc"
+// Version resource for httrack.exe, the command line program. See version.rc.
+#define VER_FILE_DESCRIPTION "HTTrack Website Copier (command line)"
+#define VER_ORIGINAL_FILENAME "httrack.exe"
+#define VER_FILETYPE VFT_APP
+#include "version.rc"
--- a/src/libhttrack.rc
+++ b/src/libhttrack.rc
@@ -1,5 +1,5 @@
-// Version resource for libhttrack.dll. See version.rc.
-#define VER_FILE_DESCRIPTION "HTTrack Website Copier engine"
-#define VER_ORIGINAL_FILENAME "libhttrack.dll"
-#define VER_FILETYPE VFT_DLL
-#include "version.rc"
+// Version resource for libhttrack.dll. See version.rc.
+#define VER_FILE_DESCRIPTION "HTTrack Website Copier engine"
+#define VER_ORIGINAL_FILENAME "libhttrack.dll"
+#define VER_FILETYPE VFT_DLL
+#include "version.rc"
--- a/src/libhttrack.vcxproj
+++ b/src/libhttrack.vcxproj
@@ -125,6 +125,7 @@
    <ClCompile Include="htsmodules.c" />
    <ClCompile Include="htsname.c" />
    <ClCompile Include="htsparse.c" />
+    <ClCompile Include="htsproxy.c" />
    <ClCompile Include="htsrobots.c" />
    <ClCompile Include="htsselftest.c" />
    <ClCompile Include="htssniff.c" />
--- a/tests/01_engine-proxyurl.test
+++ b/tests/01_engine-proxyurl.test
@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# hts_parse_proxy splits a -P argument "[scheme://][user:pass@]host[:port]" into
+# the proxy host string and port. -#test=proxyurl <arg> prints
+# "name=.. port=.. host=..", where host= is what the connect resolves.
+
+p() {
+    test "$(httrack -O /dev/null -#test=proxyurl "$1")" == "$2" || exit 1
+}
+
+# bare host, with and without an explicit port
+p 'proxy.example.com:8080' 'name=proxy.example.com port=8080 host=proxy.example.com'
+p 'proxy.example.com' 'name=proxy.example.com port=8080 host=proxy.example.com'
+
+# user:pass@ is kept in name (for proxy auth) and stripped from host
+p 'user:pass@proxy:3128' 'name=user:pass@proxy port=3128 host=proxy'
+p 'user:pass@proxy' 'name=user:pass@proxy port=8080 host=proxy'
+
+# a scheme colon must not be read as a port: http://host with no port used to
+# parse to name=http with a garbage port
+p 'http://proxy:8080' 'name=http://proxy port=8080 host=proxy'
+p 'http://proxy' 'name=http://proxy port=8080 host=proxy'
+
+# socks5/socks5h default to 1080 and resolve to the bare host
+p 'socks5://host:1080' 'name=socks5://host port=1080 host=host'
+p 'socks5://host' 'name=socks5://host port=1080 host=host'
+p 'socks5h://host' 'name=socks5h://host port=1080 host=host'
+# explicit port wins over the 1080 default, with and without userinfo
+p 'socks5://host:9050' 'name=socks5://host port=9050 host=host'
+p 'socks5://user:pass@host:9050' 'name=socks5://user:pass@host port=9050 host=host'
+
+# the split is byte-transparent: percent-escapes stay encoded (decoded later at
+# auth time), only structural ASCII ':'/'@' delimit, and the last '@' ends the
+# userinfo. So %40/%3A and UTF-8 bytes never mis-split host or port.
+p 'socks5://user:p%40ss@host:1080' 'name=socks5://user:p%40ss@host port=1080 host=host'
+p 'socks5://us%3Aer:pass@host:1080' 'name=socks5://us%3Aer:pass@host port=1080 host=host'
+p 'socks5://a@b:c@host:1080' 'name=socks5://a@b:c@host port=1080 host=host'
+p 'socks5://naïve:pass@héllo:1080' 'name=socks5://naïve:pass@héllo port=1080 host=héllo'
+
+# a lone ':' must not underflow the backward scan
+p ':' 'name= port=8080 host='
--- a/tests/01_engine-sniff.test
+++ b/tests/01_engine-sniff.test
@@ -35,7 +35,8 @@ chk image/avif hex:0000001C6674797061766966 "$yes"
 chk image/avif hex:0000001C6674797068656963 "$no" # heic brand is not avif
 chk image/heic hex:0000001C6674797068656963 "$yes"
 chk image/svg+xml '<svg xmlns="x">' "$yes"
-chk image/svg+xml $'\xef\xbb\xbf  <?xml version="1.0"?>' "$yes" # BOM+ws skip
+# BOM+ws skip; hex, as Windows argv cannot carry the raw BOM through the ANSI codepage
+chk image/svg+xml hex:EFBBBF20203C3F786D6C2076657273696F6E3D22312E30223F3E "$yes"

 # audio / video
 chk audio/mpeg 'ID3xxx' "$yes"
--- a/tests/01_engine-socks5.test
+++ b/tests/01_engine-socks5.test
@@ -0,0 +1,9 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# The SOCKS5 handshake framing and credential split, driven against scripted
+# server replies (frame draining, oversize rejects, RFC 1929 fields).
+
+httrack -O /dev/null '-#test=socks5' | grep -q "socks5 self-test OK"
--- a/tests/51_local-update-codec.test
+++ b/tests/51_local-update-codec.test
@@ -0,0 +1,38 @@
+#!/bin/bash
+#
+# A coded re-fetch that fails to decode must not destroy the mirrored file (#557).
+# Pass 1 mirrors from valid gzip; pass 2 (--update) serves an undecodable body for
+# mem.html (in-memory), disk.bin (direct-to-disk) and unsup.html (a coding we have
+# no decoder for). All three must keep their pass-1 content. fresh.html and
+# freshdisk.bin decode on both passes: they are the controls that a good update
+# still lands, in memory and direct-to-disk (the latter renaming the decoded temp
+# over an existing file). Unfixed, the decode target is the mirror itself, so the
+# first three are lost.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+# A restrictive umask makes the decoded direct-to-disk file's mode observable:
+# it is committed by renaming a temp, so it needs the chmod filecreate() does.
+umask 077
+
+bash "$top_srcdir/tests/local-crawl.sh" \
+    --rerun-args '--update' \
+    --errors 3 \
+    --log-found 'decompressing.*upcodec/mem\.html' \
+    --log-found 'decompressing.*upcodec/disk\.bin' \
+    --log-found 'Unsupported Content-Encoding.*upcodec/unsup\.html' \
+    --file-matches 'upcodec/mem.html' 'MIRRORED-MEM-V1' \
+    --file-matches 'upcodec/disk.bin' 'MIRRORED-DISK-V1' \
+    --file-min-bytes 'upcodec/disk.bin' 32768 \
+    --file-mode 'upcodec/disk.bin' 644 \
+    --file-matches 'upcodec/unsup.html' 'MIRRORED-UNSUP-V1' \
+    --file-matches 'upcodec/fresh.html' 'FRESH-V2' \
+    --file-not-matches 'upcodec/fresh.html' 'FRESH-V1' \
+    --file-mode 'upcodec/fresh.html' 644 \
+    --file-matches 'upcodec/freshdisk.bin' 'FRESHDISK-V2' \
+    --file-not-matches 'upcodec/freshdisk.bin' 'FRESHDISK-V1' \
+    --file-min-bytes 'upcodec/freshdisk.bin' 32768 \
+    --file-mode 'upcodec/freshdisk.bin' 644 \
+    httrack 'BASEURL/upcodec/index.html'
--- a/tests/52_local-socks5.test
+++ b/tests/52_local-socks5.test
@@ -0,0 +1,211 @@
+#!/bin/bash
+#
+# Issue #563: a SOCKS5 proxy must carry both http and https, resolve the origin
+# name REMOTELY (ATYP=domain, never a local lookup), and authenticate per RFC
+# 1929 without leaking the credentials to the origin.
+
+set -euo pipefail
+
+: "${top_srcdir:=..}"
+
+if test "${HTTPS_SUPPORT:-}" == "no"; then
+    echo "no https support compiled, skipping"
+    exit 77
+fi
+if ! command -v python3 >/dev/null 2>&1 || ! command -v openssl >/dev/null 2>&1; then
+    echo "python3/openssl missing, skipping"
+    exit 77
+fi
+
+# a .invalid name never resolves (RFC 6761): reaching the page at all proves the
+# proxy did the DNS
+host="socks-origin.invalid"
+server="$top_srcdir/tests/socks5-server.py"
+tmpdir=$(mktemp -d)
+pids=
+
+cleanup() {
+    for pid in $pids; do
+        kill "$pid" 2>/dev/null || true
+    done
+    rm -rf "$tmpdir"
+}
+trap cleanup EXIT
+
+openssl req -x509 -newkey rsa:2048 -keyout "$tmpdir/key.pem" \
+    -out "$tmpdir/cert.pem" -days 2 -nodes -subj "/CN=127.0.0.1" \
+    >/dev/null 2>&1
+cat "$tmpdir/key.pem" "$tmpdir/cert.pem" >"$tmpdir/both.pem"
+
+# start_server <logdir> <mode>: sets $tls_port/$http_port/$socks_port
+start_server() {
+    local dir="$1" mode="$2" ports
+    mkdir -p "$dir"
+    ports="$dir/ports.txt"
+    python3 "$server" "$tmpdir/both.pem" "$dir" "$mode" \
+        >"$ports" 2>"$dir/server.err" &
+    pids="$pids $!"
+    for _ in $(seq 1 100); do
+        grep -q "^ready" "$ports" 2>/dev/null && break
+        sleep 0.1
+    done
+    grep -q "^ready" "$ports" 2>/dev/null || {
+        echo "server ($mode) did not start" >&2
+        cat "$dir/server.err" >&2
+        exit 1
+    }
+    tls_port=$(awk '/^TLS/{print $2}' "$ports")
+    http_port=$(awk '/^HTTP/{print $2}' "$ports")
+    socks_port=$(awk '/^SOCKS/{print $2}' "$ports")
+}
+
+# kill a hung httrack so a missing read bound surfaces as $HANG_RC instead of
+# stalling the job (macOS has no `timeout`)
+HANG_RC=137 # 128 + SIGKILL
+run_crawl() {
+    local out="$1" url="$2" proxy="$3"
+    shift 3
+    local extra="$*"
+    test -n "$extra" || extra="-r1 -s0"
+    rm -rf "$out"
+    # shellcheck disable=SC2086
+    httrack "$url" --proxy "$proxy" \
+        -O "$out" $extra --timeout=10 >"$out.log" 2>&1 &
+    local pid=$!
+    (sleep 60 && kill -9 "$pid" 2>/dev/null) &
+    local guard=$!
+    local rc=0
+    wait "$pid" 2>/dev/null || rc=$?
+    kill "$guard" 2>/dev/null || true
+    wait "$guard" 2>/dev/null || true
+    return "$rc"
+}
+
+# --- https over socks, remote DNS -------------------------------------------
+ok="$tmpdir/ok"
+start_server "$ok" ok
+
+run_crawl "$ok/out" "https://${host}:${tls_port}/" "socks5://127.0.0.1:${socks_port}"
+grep -rq "ORIGIN-PAGE-563" "$ok/out" || {
+    echo "FAIL: origin page not downloaded through the socks proxy" >&2
+    cat "$ok/out.log" "$ok/server.err" >&2
+    exit 1
+}
+grep -q "^ATYP domain ${host}\$" "$ok/socks.log" || {
+    echo "FAIL: proxy did not receive ATYP=domain ${host} (no remote DNS)" >&2
+    cat "$ok/socks.log" >&2
+    exit 1
+}
+# a client that resolved locally would have sent an address, not a name
+grep -q "^ATYP ipv4" "$ok/socks.log" && {
+    echo "FAIL: httrack resolved locally and sent an IPv4 to the proxy" >&2
+    cat "$ok/socks.log" >&2
+    exit 1
+}
+echo "OK: https tunneled through socks5 with remote DNS"
+
+# --- plain http over socks --------------------------------------------------
+: >"$ok/origin-headers.log" # the https crawl above also logged a request line
+run_crawl "$ok/outh" "http://${host}:${http_port}/" "socks5://127.0.0.1:${socks_port}"
+grep -rq "ORIGIN-PAGE-563" "$ok/outh" || {
+    echo "FAIL: plain http not tunneled through the socks proxy" >&2
+    cat "$ok/outh.log" "$ok/socks.log" >&2
+    exit 1
+}
+# the tunneled request is origin-form, as on a direct connection: the http-proxy
+# absolute URI would reach the origin as "GET http://host/"
+grep -q "^REQUEST GET / " "$ok/origin-headers.log" || {
+    echo "FAIL: tunneled request is not origin-form" >&2
+    cat "$ok/origin-headers.log" >&2
+    exit 1
+}
+grep -q "^REQUEST GET http" "$ok/origin-headers.log" && {
+    echo "FAIL: absolute-URI request sent through the socks tunnel" >&2
+    cat "$ok/origin-headers.log" >&2
+    exit 1
+}
+echo "OK: plain http tunneled through socks5"
+
+# --- keep-alive socket reuse must NOT re-run the handshake ------------------
+# A reused keep-alive socket is already tunneled; re-injecting the SOCKS
+# greeting corrupts the origin stream. -c1 + the origin's max>1 advert makes
+# httrack serve the whole plain-http crawl over one tunnel: exactly one
+# handshake, every subpage intact.
+ka="$tmpdir/ka"
+start_server "$ka" ok
+run_crawl "$ka/out" "http://${host}:${http_port}/" "socks5://127.0.0.1:${socks_port}" -r3 -c1
+missing=0
+for i in 1 2 3 4 5; do
+    grep -rq "SUBPAGE-563 /p${i}.html" "$ka/out" || missing=1
+done
+test "$missing" -eq 0 || {
+    echo "FAIL: a keep-alive subpage was lost (re-handshake corrupted the reused socket)" >&2
+    cat "$ka/out.log" "$ka/socks.log" >&2
+    exit 1
+}
+handshakes=$(grep -c "^CMD " "$ka/socks.log" || true)
+test "$handshakes" -eq 1 || {
+    echo "FAIL: expected 1 SOCKS handshake for the reused socket, got $handshakes" >&2
+    cat "$ka/socks.log" >&2
+    exit 1
+}
+echo "OK: keep-alive socket reused over one socks tunnel, no re-handshake"
+
+# --- authenticated socks (RFC 1929) -----------------------------------------
+# over plain http: that is the request an http proxy would decorate with
+# Proxy-Authorization, so the no-leak check below has teeth
+auth="$tmpdir/auth"
+start_server "$auth" ok
+run_crawl "$auth/out" "http://${host}:${http_port}/" \
+    "socks5://user:secret@127.0.0.1:${socks_port}"
+grep -rq "ORIGIN-PAGE-563" "$auth/out" || {
+    echo "FAIL: origin not downloaded through the authenticated socks proxy" >&2
+    cat "$auth/out.log" "$auth/socks.log" >&2
+    exit 1
+}
+grep -q "^METHOD userpass\$" "$auth/socks.log" || {
+    echo "FAIL: user/pass method not negotiated (auth ignored)" >&2
+    cat "$auth/socks.log" >&2
+    exit 1
+}
+grep -q "^AUTH user secret\$" "$auth/socks.log" || {
+    echo "FAIL: wrong socks credentials sent" >&2
+    cat "$auth/socks.log" >&2
+    exit 1
+}
+grep -qi "secret\|proxy-authorization" "$auth/origin-headers.log" && {
+    echo "FAIL: socks credentials leaked into the origin request" >&2
+    cat "$auth/origin-headers.log" >&2
+    exit 1
+}
+echo "OK: socks5 user/pass negotiated, creds not leaked to origin"
+
+# --- hostile: refusing proxy ------------------------------------------------
+ref="$tmpdir/refuse"
+start_server "$ref" refuse
+rc=0
+run_crawl "$ref/out" "https://${host}:${tls_port}/" "socks5://127.0.0.1:${socks_port}" || rc=$?
+test "$rc" -ne "$HANG_RC" || {
+    echo "FAIL: crawl hung on a refusing socks proxy" >&2
+    exit 1
+}
+grep -rq "ORIGIN-PAGE-563" "$ref/out" 2>/dev/null && {
+    echo "FAIL: refusing proxy unexpectedly served the page" >&2
+    exit 1
+}
+echo "OK: refusing socks proxy fails cleanly, no hang"
+
+# --- hostile: truncated handshake reply -------------------------------------
+trunc="$tmpdir/trunc"
+start_server "$trunc" truncate
+rc=0
+run_crawl "$trunc/out" "https://${host}:${tls_port}/" "socks5://127.0.0.1:${socks_port}" || rc=$?
+test "$rc" -ne "$HANG_RC" || {
+    echo "FAIL: crawl hung on a truncated socks reply (bounded read missing)" >&2
+    exit 1
+}
+grep -rq "ORIGIN-PAGE-563" "$trunc/out" 2>/dev/null && {
+    echo "FAIL: truncated-reply proxy unexpectedly served the page" >&2
+    exit 1
+}
+echo "OK: bounded socks handshake, no hang on a truncated reply"
--- a/tests/Makefile.am
+++ b/tests/Makefile.am
@@ -2,7 +2,7 @@
 # explicitly: automake does not expand wildcards in EXTRA_DIST, so a glob would
 # silently drop it from the dist tarball and break "make distcheck".
 EXTRA_DIST = $(TESTS) crawl-test.sh run-all-tests.sh check-network.sh \
-	proxy-https-server.py \
+	proxy-https-server.py socks5-server.py \
 	local-crawl.sh local-server.py server.crt server.key \
 	server-root/simple/basic.html server-root/simple/link.html \
 	server-root/stripquery/index.html server-root/stripquery/a.html \
@@ -46,6 +46,8 @@ TESTS = \
 	01_engine-header.test \
 	01_engine-idna.test \
 	01_engine-identurl.test \
+	01_engine-proxyurl.test \
+	01_engine-socks5.test \
 	01_engine-identabs.test \
 	01_engine-escape-room.test \
 	01_engine-inplace-escape.test \
@@ -124,6 +126,8 @@ TESTS = \
 	47_local-crange-overflow.test \
 	48_local-crange-memresume.test \
 	49_local-cookiewall.test \
-	50_local-contentcodings.test
+	50_local-contentcodings.test \
+	51_local-update-codec.test \
+	52_local-socks5.test

 CLEANFILES = check-network_sh.cache
--- a/tests/local-crawl.sh
+++ b/tests/local-crawl.sh
@@ -17,7 +17,7 @@
 #       --errors N --errors-content N --files N --found PATH ... --directory PATH ... \
 #       --log-found REGEX ... --log-not-found REGEX ... \
 #       --file-matches PATH REGEX ... --file-not-matches PATH REGEX ... \
-#       --file-min-bytes PATH N --max-mirror-bytes N \
+#       --file-min-bytes PATH N --file-mode PATH OCTAL --max-mirror-bytes N \
 #       httrack BASEURL/some/path [httrack-args...]
 # --errors counts every "Error:" log line; --errors-content drops transient
 # network failures (codes -2..-6) that flake on busy loopback under -c8.
@@ -26,6 +26,7 @@
 # --file-matches/--file-not-matches grep (ERE) a mirrored file (PATH under the
 # host root), to assert rewritten link/content survived the crawl.
 # --file-min-bytes asserts a mirrored file (PATH) is at least N bytes.
+# --file-mode asserts its octal permissions (e.g. 644); POSIX hosts only.
 # --rerun-args runs a second pass (same server and mirror dir) with the given
 # extra httrack args appended, e.g. an --update run under a cap.
 # --cookie writes a Netscape cookies.txt (scoped to the discovered host:port,
@@ -141,7 +142,7 @@ while test "$pos" -lt "$nargs"; do
        audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
        pos=$((pos + 1))
        ;;
-    --file-matches | --file-not-matches | --file-min-bytes)
+    --file-matches | --file-not-matches | --file-min-bytes | --file-mode)
        audit+=("${args[$pos]}" "${args[$((pos + 1))]}" "${args[$((pos + 2))]}")
        pos=$((pos + 2))
        ;;
@@ -319,9 +320,10 @@ done
 test -n "$hostroot" || die "could not find host root under $out"
 debug "host root: $hostroot"

-# No crawl, even a cancelled one, may leave .delayed temporaries (#107, #483).
-info "checking for leftover .delayed files"
-leftovers=$(find "$out" -name '*.delayed' 2>/dev/null | head -5)
+# No crawl, even a cancelled one, may leave engine temporaries: .delayed (#107,
+# #483), or the .z/.u content-coding temps (#557).
+info "checking for leftover engine temporaries"
+leftovers=$(find "$out" \( -name '*.delayed' -o -name '*.z' -o -name '*.u' \) 2>/dev/null | head -5)
 if test -z "$leftovers"; then result "OK"; else
    result "leftover: $leftovers"
    exit 1
@@ -436,6 +438,17 @@ while test "$i" -lt "${#audit[@]}"; do
            exit 1
        fi
        ;;
+    --file-mode)
+        path="${audit[$((i + 1))]}"
+        i=$((i + 2))
+        mode=$(stat -c '%a' "${hostroot}/${path}" 2>/dev/null ||
+            stat -f '%Lp' "${hostroot}/${path}" 2>/dev/null)
+        info "checking ${path} mode ${mode:-none} is ${audit[$i]}"
+        if test "$mode" = "${audit[$i]}"; then result "OK"; else
+            result "wrong mode"
+            exit 1
+        fi
+        ;;
    esac
    i=$((i + 1))
 done
--- a/tests/local-server.py
+++ b/tests/local-server.py
@@ -695,6 +695,80 @@ class Handler(SimpleHTTPRequestHandler):
            extra_headers=[("Content-Encoding", "compress")],
        )

+    # --- coded re-fetch that fails to decode (#557) -------------------------
+    # Pass 1 mirrors each file from a valid gzip body; pass 2 (--update) serves
+    # a body that cannot be decoded. The previously-mirrored copy must survive.
+    # fresh.html is the control: its pass-2 body decodes, so it must be updated.
+    UPCODEC_SEEN = {}
+
+    def upcodec_pass(self):
+        """1 on the first body fetch of this path, 2 on the next ones. HEADs
+        don't count, so a stray one can't shift which pass gets the bad body."""
+        if self.command == "HEAD":
+            return 1
+        seen = Handler.UPCODEC_SEEN.get(self.path, 0) + 1
+        Handler.UPCODEC_SEEN[self.path] = seen
+        return seen
+
+    @staticmethod
+    def gzipped(body):
+        return gzip.compress(body)
+
+    @staticmethod
+    def bad_gzip(body):
+        """A gzip stream whose deflate payload is mangled: inflate fails partway
+        through, after some plausible output has already been produced."""
+        raw = bytearray(gzip.compress(body))
+        raw[20:40] = b"\xff" * 20
+        return bytes(raw[:-4])
+
+    def send_coded(self, body, content_type, coding="gzip"):
+        self.send_raw(body, content_type, extra_headers=[("Content-Encoding", coding)])
+
+    def route_upcodec_index(self):
+        self.send_html(
+            '\t<a href="mem.html">mem</a>\n'
+            '\t<a href="disk.bin">disk</a>\n'
+            '\t<a href="unsup.html">unsup</a>\n'
+            '\t<a href="fresh.html">fresh</a>\n'
+            '\t<a href="freshdisk.bin">freshdisk</a>\n'
+        )
+
+    MEM_V1 = b"<html><body><p>MIRRORED-MEM-V1</p></body></html>"
+    DISK_V1 = b"MIRRORED-DISK-V1\n" + b"\x00\x01\x02\xff" * 8192
+    UNSUP_V1 = b"<html><body><p>MIRRORED-UNSUP-V1</p></body></html>"
+
+    def route_upcodec_mem(self):
+        if self.upcodec_pass() == 1:
+            self.send_coded(self.gzipped(self.MEM_V1), "text/html")
+        else:
+            self.send_coded(self.bad_gzip(self.MEM_V1), "text/html")
+
+    def route_upcodec_disk(self):
+        if self.upcodec_pass() == 1:
+            self.send_coded(self.gzipped(self.DISK_V1), "application/octet-stream")
+        else:
+            self.send_coded(self.bad_gzip(self.DISK_V1), "application/octet-stream")
+
+    # Pass 2 switches to a coding we have no decoder for.
+    def route_upcodec_unsup(self):
+        if self.upcodec_pass() == 1:
+            self.send_coded(self.gzipped(self.UNSUP_V1), "text/html")
+        else:
+            self.send_coded(self.UNSUP_V1, "text/html", coding="compress")
+
+    def route_upcodec_fresh(self):
+        pass1 = self.upcodec_pass() == 1
+        body = b"<html><body><p>FRESH-V%d</p></body></html>" % (1 if pass1 else 2)
+        self.send_coded(self.gzipped(body), "text/html")
+
+    # Same, direct-to-disk: the update pass decodes, so the temp is renamed over
+    # an existing mirror file.
+    def route_upcodec_freshdisk(self):
+        pass1 = self.upcodec_pass() == 1
+        body = b"FRESHDISK-V%d\n" % (1 if pass1 else 2) + b"\x03\x02\x01\xfe" * 8192
+        self.send_coded(self.gzipped(body), "application/octet-stream")
+
    # Echo what httrack advertised, so a crawl can assert the header.
    def route_codec_ae(self):
        self.send_raw(
@@ -1392,6 +1466,12 @@ class Handler(SimpleHTTPRequestHandler):
        "/codec/bad.html": route_codec_bad,
        "/codec/bin.dat": route_codec_bin,
        "/codec/ae.html": route_codec_ae,
+        "/upcodec/index.html": route_upcodec_index,
+        "/upcodec/mem.html": route_upcodec_mem,
+        "/upcodec/disk.bin": route_upcodec_disk,
+        "/upcodec/unsup.html": route_upcodec_unsup,
+        "/upcodec/fresh.html": route_upcodec_fresh,
+        "/upcodec/freshdisk.bin": route_upcodec_freshdisk,
        "/types/index.html": route_types_index,
        "/types/control.php": route_types,
        "/types/photo.png": route_types,
--- a/tests/socks5-server.py
+++ b/tests/socks5-server.py
@@ -0,0 +1,228 @@
+#!/usr/bin/env python3
+"""Local SOCKS5 proxy (RFC 1928/1929) + TLS and plain HTTP origins for #563.
+
+Logs the auth method, any user/pass, and the CONNECT ATYP + literal domain, so
+the test proves httrack sent a hostname (remote DNS) and negotiated auth.
+Modes (argv[3]): ok | refuse (REP=5) | truncate (partial reply, must not hang).
+Usage: socks5-server.py <cert.pem> <logdir> [mode]
+Prints "TLS <port>", "HTTP <port>", "SOCKS <port>", "ready" on stdout.
+"""
+import http.server
+import os
+import socket
+import socketserver
+import ssl
+import struct
+import sys
+import threading
+
+# The one name the proxy answers for; a .invalid TLD never resolves (RFC 6761),
+# so a locally-resolving client could not reach us -- success proves remote DNS.
+REMOTE_HOST = b"socks-origin.invalid"
+# index links the subpages so an -r3 crawl reuses one keep-alive socket
+LINKS = "".join('<a href="/p%d.html">%d</a>' % (i, i) for i in range(1, 6))
+ORIGIN_BODY = ("<html><body>ORIGIN-PAGE-563 " + LINKS + "</body></html>").encode()
+SOCKS_LOG = "socks.log"
+ORIGIN_LOG = "origin-headers.log"
+
+
+def make_origin(logdir):
+    class Origin(http.server.BaseHTTPRequestHandler):
+        # HTTP/1.1 + a max>1 advertisement lets httrack keep the socket alive
+        protocol_version = "HTTP/1.1"
+
+        def do_GET(self):
+            with open(os.path.join(logdir, ORIGIN_LOG), "a") as handle:
+                handle.write(
+                    "REQUEST %s %s %s\n"
+                    % (self.command, self.path, self.request_version)
+                )
+                for key in self.headers.keys():
+                    handle.write(key + ": " + self.headers[key] + "\n")
+            if "p" in self.path and ".html" in self.path:
+                body = (
+                    b"<html><body>SUBPAGE-563 " + self.path.encode() + b"</body></html>"
+                )
+            else:
+                body = ORIGIN_BODY
+            self.send_response(200)
+            self.send_header("Content-Type", "text/html")
+            self.send_header("Content-Length", str(len(body)))
+            self.send_header("Keep-Alive", "timeout=15, max=100")
+            self.send_header("Connection", "keep-alive")
+            self.end_headers()
+            self.wfile.write(body)
+
+        def log_message(self, *args):
+            pass
+
+    return Origin
+
+
+class ThreadingTCPServer(socketserver.ThreadingTCPServer):
+    allow_reuse_address = True
+    daemon_threads = True
+
+
+def start_origin(logdir, certfile=None):
+    httpd = ThreadingTCPServer(("127.0.0.1", 0), make_origin(logdir))
+    if certfile is not None:
+        ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+        ctx.load_cert_chain(certfile)
+        httpd.socket = ctx.wrap_socket(httpd.socket, server_side=True)
+    port = httpd.socket.getsockname()[1]
+    threading.Thread(target=httpd.serve_forever, daemon=True).start()
+    return port
+
+
+def recvn(conn, n):
+    buf = b""
+    while len(buf) < n:
+        chunk = conn.recv(n - len(buf))
+        if not chunk:
+            raise OSError("short read")
+        buf += chunk
+    return buf
+
+
+def pipe(src, dst):
+    try:
+        while True:
+            data = src.recv(65536)
+            if not data:
+                break
+            dst.sendall(data)
+    except OSError:
+        pass
+    finally:
+        for sock in (src, dst):
+            try:
+                sock.shutdown(socket.SHUT_RDWR)
+            except OSError:
+                pass
+
+
+def log(logdir, line):
+    with open(os.path.join(logdir, SOCKS_LOG), "a") as handle:
+        handle.write(line + "\n")
+
+
+def negotiate_auth(conn, logdir):
+    """RFC 1928 greeting + RFC 1929 sub-negotiation. Returns True to proceed."""
+    ver, nmethods = recvn(conn, 2)
+    if ver != 0x05:
+        return False
+    methods = recvn(conn, nmethods)
+    if 0x02 in methods:  # prefer user/pass so the auth test exercises RFC 1929
+        conn.sendall(b"\x05\x02")
+        (subver,) = recvn(conn, 1)
+        (ulen,) = recvn(conn, 1)
+        uname = recvn(conn, ulen)
+        (plen,) = recvn(conn, 1)
+        passwd = recvn(conn, plen)
+        log(logdir, "METHOD userpass")
+        log(logdir, "AUTH %s %s" % (uname.decode(), passwd.decode()))
+        conn.sendall(b"\x01\x00")  # RFC 1929 success
+    elif 0x00 in methods:
+        conn.sendall(b"\x05\x00")
+        log(logdir, "METHOD noauth")
+    else:
+        conn.sendall(b"\x05\xff")  # no acceptable method
+        return False
+    return True
+
+
+def read_request(conn, logdir):
+    """RFC 1928 CONNECT. Logs CMD, ATYP, domain/addr, port. Returns dest port."""
+    ver, cmd, _rsv, atyp = recvn(conn, 4)
+    log(logdir, "CMD %d" % cmd)
+    if atyp == 0x01:
+        addr = socket.inet_ntoa(recvn(conn, 4))
+        log(logdir, "ATYP ipv4 %s" % addr)
+    elif atyp == 0x03:
+        (dlen,) = recvn(conn, 1)
+        domain = recvn(conn, dlen)
+        log(logdir, "ATYP domain %s" % domain.decode())
+    elif atyp == 0x04:
+        recvn(conn, 16)
+        log(logdir, "ATYP ipv6")
+        domain = b""
+    else:
+        return None, None
+    (port,) = struct.unpack(">H", recvn(conn, 2))
+    log(logdir, "PORT %d" % port)
+    name = domain if atyp == 0x03 else b""
+    return name, port
+
+
+def reply(conn, rep):
+    conn.sendall(bytes([0x05, rep, 0x00, 0x01]) + b"\x00\x00\x00\x00" + b"\x00\x00")
+
+
+def handle_socks(conn, logdir, mode):
+    try:
+        if not negotiate_auth(conn, logdir):
+            conn.close()
+            return
+        if mode == "truncate":
+            # one byte of the 10-byte reply, then hold the socket: the client
+            # must bound this handshake read and give up, not hang.
+            conn.sendall(b"\x05")
+            threading.Event().wait()
+            return
+        name, port = read_request(conn, logdir)
+        if mode == "refuse" or name != REMOTE_HOST or port is None:
+            reply(conn, 0x05)  # connection refused
+            conn.close()
+            return
+        try:
+            upstream = socket.create_connection(("127.0.0.1", port))
+        except OSError:
+            reply(conn, 0x05)
+            conn.close()
+            return
+        reply(conn, 0x00)  # succeeded
+        threading.Thread(target=pipe, args=(conn, upstream), daemon=True).start()
+        pipe(upstream, conn)
+    except OSError:
+        try:
+            conn.close()
+        except OSError:
+            pass
+
+
+def start_socks(logdir, mode):
+    srv = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    srv.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+    srv.bind(("127.0.0.1", 0))
+    srv.listen(16)
+    port = srv.getsockname()[1]
+
+    def serve():
+        while True:
+            conn, _ = srv.accept()
+            threading.Thread(
+                target=handle_socks, args=(conn, logdir, mode), daemon=True
+            ).start()
+
+    threading.Thread(target=serve, daemon=True).start()
+    return port
+
+
+def main():
+    certfile, logdir = sys.argv[1], sys.argv[2]
+    mode = sys.argv[3] if len(sys.argv) > 3 else "ok"
+    for name in (SOCKS_LOG, ORIGIN_LOG):
+        open(os.path.join(logdir, name), "w").close()
+    tls_port = start_origin(logdir, certfile)
+    http_port = start_origin(logdir, None)
+    socks_port = start_socks(logdir, mode)
+    print("TLS %d" % tls_port, flush=True)
+    print("HTTP %d" % http_port, flush=True)
+    print("SOCKS %d" % socks_port, flush=True)
+    print("ready", flush=True)
+    threading.Event().wait()
+
+
+if __name__ == "__main__":
+    main()
Author	SHA1	Message	Date
Xavier Roche	3ab3726508	Keep the vcxproj source list alphabetical Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com>	2026-07-14 16:11:38 +02:00
Xavier Roche	573e62e08c	Extract the proxy tunneling code into htsproxy.c htslib.c is the engine's catch-all "subroutines" file; the two proxy tunnels living in it (HTTP CONNECT and the SOCKS5 handshake) are one coherent thing: blocking raw-socket negotiation with the proxy, run from the same back_wait callsite before TLS. Move both, plus proxy_getline and the socks5 statics, into htsproxy.c/.h, which also gives the socks5_test_io self-test seam a home away from the public-ish htslib.h. Pure code motion: the moved bytes are identical, and hts_proxy_is_socks stays in htslib.c next to hts_parse_proxy and jump_protocol_const, being a URL-scheme predicate rather than handshake logic. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com>	2026-07-14 15:47:24 +02:00
Xavier Roche	1fa0184468	SOCKS5 proxy support (#570 ) * Tunnel crawls through a SOCKS5 proxy (RFC 1928/1929) -P socks5://[user:pass@]host[:port] now negotiates a SOCKS5 tunnel to the origin instead of speaking HTTP to the proxy. The handshake runs on the raw socket right after connect, so it covers plain http as well as https (TLS then runs end-to-end over the tunnel, no CONNECT); the origin request stays origin-form, and the proxy credentials ride the RFC 1929 sub-negotiation rather than a Proxy-Authorization header. The origin hostname is sent as-is (ATYP=domain): the proxy resolves it, like curl's socks5h, which is what makes .onion and split-horizon names work and keeps the crawl from leaking DNS. ftp:// links fail under a socks proxy rather than silently dial the origin direct. Reply frames are drained exactly per ATYP, since the socket is shared with the origin stream: one byte too few and the leftovers corrupt the first TLS record. -#test=socks5 asserts that byte count against scripted replies, plus the truncated/hostile cases; 52_local-socks5.test crawls http and https through a local SOCKS5 server that only answers for a .invalid name, so a locally-resolved origin could not reach it. Closes #563 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * Don't re-run the SOCKS5 handshake on a reused keep-alive socket back_wait re-enters the connect-completed block for a keep-alive socket revived by back_trylive (plain http; https is covered by the ssl_con guard). The socket is already tunneled, so re-sending the SOCKS greeting injected \x05\x01\x00... into the established origin stream and corrupted every request after the first. Gate the handshake on !keep_alive, which back_connxfr carries onto the revived slot and which is zero on a fresh connect. 52_local-socks5.test now crawls a keep-alive origin (advertising max>1) with -c1: the whole plain-http crawl rides one tunnel, exactly one handshake, every subpage intact. Without the gate the reused stream desyncs and the crawl hangs. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * Use scheme-less origins in the SOCKS5 self-test (CodeQL cpp/non-https-url) The scripted handshake strips the scheme and takes the TLS flag as a parameter, so the "http://origin.test" fixtures were inert cleartext-URL literals that tripped CodeQL. Pass the bare authority instead; the parsing is identical and the real scheme handling is covered end-to-end by 52_local-socks5.test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> --------- Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>	2026-07-14 15:38:47 +02:00
Xavier Roche	79e46eeceb	Store the cache self-tests' X-Save verbatim, and feed the sniff BOM as hex (#567 ) The cache self-tests built the X-Save path with fconcat(), which rewrites every '/' to '\' on Windows -- including the path_html_utf8 prefix the reader matches on. cache_readex then took the pre-3.40 compat branch and re-rooted the name into a path that does not exist, so -#test=cache's disk-fallback read and -#test=cache-corrupt's headers-only and in-memory cases all failed on MSVC builds. The engine keeps save names in '/' form and DOS-ifies only at the syscall boundary; the self-tests now do the same. The sniff test passed a UTF-8 BOM as raw bytes through argv, where Windows routes it via the ANSI codepage and U+FEFF has no mapping, so it arrived as '?'. Use the hex: body form the self-test already accepts. Both are self-test bugs: neither path is reachable from the crawler, which never sniffs argv and never DOS-ifies a stored save name. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>	2026-07-14 14:29:12 +02:00
Xavier Roche	436ac53793	Parse -P proxy URLs scheme-aware, recognize socks5 schemes (#566 ) * Parse -P proxy URLs scheme-aware, and recognize socks5 schemes The -P argument parser scanned backward for the port ':' from the end of the string with no notion of a scheme, so a scheme-prefixed proxy with no explicit port ("-P http://proxy") mistook the scheme's own colon for a port separator: the name came out as "http" with a garbage port. A port or a "user:pass@" prefix happened to shield the scan, so this only bit the no-port case. Extract the parse into hts_parse_proxy(), which skips any "scheme://" before hunting the port, so the scheme colon is never read as a port. Teach jump_protocol_const the socks5/socks5h schemes (so the proxy host resolves with the scheme stripped) and default the port by scheme (1080 for socks, 8080 otherwise). This is the parsing groundwork for SOCKS5 support (#563); the handshake lands separately. Add a -#test=proxyurl self-test and tests/01_engine-proxyurl.test covering bare hosts, user:pass, the scheme-colon regression, and the socks defaults. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * hts_parse_proxy: scan from one-past-end, avoid forming arg-1 The backward port scan started at arg+strlen(arg)-1 and decremented while a >= authority, so on any argument with no ':' or '@' in the authority (every bare-host proxy, and -P "") it stepped a to arg-1: a pointer formed before the string, which is undefined behavior. It is not a memory-safety bug (never dereferenced, no wrap, so UBSan stays green), but it is latent UB reachable from -P. Scan from one-past-end and test a[-1] with a > authority instead, so no pointer below the string is ever formed. Guard name_size == 0 against a future small-buffer caller (the name_size - 1 truncation would otherwise underflow). Add a userinfo-free socks explicit-port case and a lone ':' boundary case to the self-test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * proxyurl test: percent-encoded specials and UTF-8 Pin that the host/port split is byte-transparent: a percent-encoded '@' (%40) or ':' (%3A) in the userinfo stays encoded and never mis-splits host or port, a second raw '@' is handled by the last-'@' host rule, and UTF-8 bytes in the username and host label pass through intact. Decoding happens later at auth time, not in the parser. Raw UTF-8 CLI args match the existing 01_engine-idna.test. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> --------- Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>	2026-07-14 14:11:57 +02:00
Xavier Roche	bc4b2c7b8e	Keep the mirrored file when a coded re-fetch fails to decode (#560 ) * Keep the mirrored file when a coded re-fetch fails to decode An --update re-fetch of a content-coded URL decoded straight into url_sav: the live mirror was truncated (filecreateempty) before the decode was even attempted, so a truncated gzip, a corrupt br/zstd stream, a decode-size overrun or a coding we have no decoder for destroyed the good copy we were merely refreshing. The uncompressed path has been rollback-safe since #522, but its backup is explicitly skipped for coded bodies, so nothing covered them. Decode into a temporary file instead and only commit it over the mirror once the decode succeeded; on failure url_sav is never touched. The failure path also has to filenote() the surviving copy, or the end-of-update purge (in old.lst, absent from new.lst) would delete the very file the decode took care not to overwrite. Closes #557 Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * Never drop the decoded body when the mirror replace fails, and probe the good path replace_file() unlinks the destination and retries when the first rename is refused (Windows never clobbers). If that retry also fails, the previous copy is gone, so deleting the decoded temp as well destroyed the last copy of the body: keep it and log where it went, the way the backup restore does. That restore is the same remove-then-rename, so it now shares the helper. The test grew the missing positive control: freshdisk.bin decodes on both passes on the direct-to-disk path, so the update pass renames a decoded temp over an existing mirror file, which no case exercised. The leftover-temporaries sweep in local-crawl.sh now covers the .z/.u decode temps for every crawl test, not just the three names this one asserted. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * Restore the 0644 mode on a decoded direct-to-disk file Committing the decode by renaming the temp bypassed filecreate(), which is what chmods the mirrored file to HTS_ACCESS_FILE. Under a restrictive umask a coded binary landed 0600 while every other mirrored file stayed 0644, so a mirror served from a web root or shared with a group silently lost read access. chmod it at the rename, as the plain direct-to-disk path already does. The test now crawls under umask 077 and asserts the mode, via a --file-mode assertion in local-crawl.sh; without the chmod it fails with 0600. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> * Pin the codec-update assertions to the URLs and to fresh content The exact error count did not say which links failed: an engine that logged one URL twice while another failed silently still counted three and passed. Match the three decode errors per URL instead. The content checks only looked for the new marker, so a fix that appended the decoded body to the old file rather than replacing it would pass too; assert the pass-1 marker is gone. Also assert the mode on a kept file, not just on the two rewritten ones. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com> Signed-off-by: Xavier Roche <roche@httrack.com> --------- Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>	2026-07-14 13:10:28 +02:00
Xavier Roche	4fafe283b4	Store the .rc blobs LF-normalized, as .gitattributes declares (#561 ) .gitattributes marks *.rc as "text eol=crlf": the blob is LF, the checkout is CRLF. The two resource scripts added in #558 went in with CRLF blobs, so git renormalizes them on every add and every checkout reports them as modified. Renormalized with git add --renormalize; version.rc was already correct. The working-tree files stay CRLF, which is what the Windows toolchain reads. Signed-off-by: Xavier Roche <roche@httrack.com> Co-authored-by: Claude Opus 4.8 (1M context) <noreply@anthropic.com>	2026-07-14 12:42:19 +02:00