Validate the RFC 1929 sub-negotiation version in the SOCKS5 test server

The server read the sub-negotiation version byte and dropped it on the floor,
so a client sending SOCKS5's 0x05 where RFC 1929 wants 0x01 kept the test
green while every real proxy rejected the handshake. Reject the mismatch the
way a real proxy does, and assert the version reached the proxy.

Verified by mutation: with htsproxy.c patched to send 0x05, the test on master
passes and the fixed test fails.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

lang/Finnish.txt

@@ -930,7 +930,6 @@ Server terminated
 Palvelin lopetettu
 A fatal error has occurred during this mirror
 Tällä peilillä tapahtui vakava virhe
-
 Proxy type:
 Välityspalvelimen tyyppi:
 Proxy protocol. HTTP: standard proxy. HTTP (CONNECT tunnel): sends every request through a CONNECT tunnel, for CONNECT-only proxies like Tor's HTTPTunnelPort. SOCKS5: default port 1080.

lang/Italiano.txt

@@ -374,7 +374,7 @@ Create error logging and report files
 Crea file di log per segnalare errori e informazioni
 Generate DOS 8-3 filenames ONLY
 Genera solo nomi di file in formato 8.3
-Generate ISO 9660 filenames ONLY for CDROM medias
+Generate ISO9660 filenames ONLY for CDROM medias
 Genera nomi di file in formato ISO9660 per i CDROM
 Do not create HTML error pages
 Non generare pagine d'errore HTML

lang/Portugues-Brasil.txt

@@ -918,7 +918,7 @@ normal\nextended\ndebug
 normal\nextendido\ncorrigir
 Download web site(s)\nDownload web site(s) + questions\nGet individual files\nDownload all sites in pages (multiple mirror)\nTest links in pages (bookmark test)\n* Continue interrupted download\n* Update existing download
 Copiar site(s) da Web\nCopiar site(s) interativos da Web (perguntas)\nReceber arquivos específicos\nCopiar todas as páginas do site (alternação múltipla)\nTestar links nas páginas (testar indicador)\n* Retomar download interrompido\n* Atualizar download existente
-Relative URL / Absolute URL (default)\nAbsolute URL / Absolute URL\nAbsolute URI / Absolute URL\nOriginal URL / Original URL
+Relative URI / Absolute URL (default)\nAbsolute URL / Absolute URL\nAbsolute URI / Absolute URL\nOriginal URL / Original URL
 URI Relativa / URL Absoluta (padrão)\nURL Absoluta / URL Absoluta\nURI Absoluta / URL Absoluta\nURL Original / URL Original
 Open Source offline browser
 Abrir origem offline no navegador

src/htsalias.c

@@ -567,24 +567,29 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
   return 0;
 }
 
-/* Get home directory, '.' if failed */
+/* Get home directory, '.' if unset or empty */
 /* example: /home/smith */
 const char *hts_gethome(void) {
   const char *home = getenv("HOME");
 
-  if (home)
-    return home;
-  else
-    return ".";
+  /* An empty $HOME would expand ~/foo into the absolute /foo */
+  return strnotempty(home) ? home : ".";
 }
 
-/* Convert ~/foo into /home/smith/foo */
+/* Convert ~/foo into /home/smith/foo (~user/ left alone: no getpwnam here) */
 void expand_home(String * str) {
-  if (StringSub(*str, 1) == '~') {
+  if (StringNotEmpty(*str) && StringSub(*str, 0) == '~' &&
+      (StringLength(*str) == 1 || StringSub(*str, 1) == '/')) {
     char BIGSTK tempo[HTS_URLMAXSIZE * 2];
+    const char *const home = hts_gethome();
+    const size_t homelen = strlen(home);
+    const size_t taillen = StringLength(*str) - 1;
 
-    strcpybuff(tempo, hts_gethome());
-    strcatbuff(tempo, StringBuff(*str) + 1);
-    StringCopy(*str, tempo);
+    /* Leave untouched rather than abort() in strcatbuff on a huge $HOME */
+    if (taillen < sizeof(tempo) && homelen < sizeof(tempo) - taillen) {
+      strcpybuff(tempo, home);
+      strcatbuff(tempo, StringBuff(*str) + 1);
+      StringCopy(*str, tempo);
+    }
   }
 }

src/htsselftest.c

@@ -45,6 +45,7 @@ Please visit our Website: http://www.httrack.com
 #include "htscore.h"
 #include "htsdefines.h"
 #include "htslib.h"
+#include "htsalias.h"
 #include "htsparse.h"
 #include "htscache.h"
 #include "htscache_selftest.h"
@@ -825,6 +826,21 @@ static int st_simplify(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+static int st_expandhome(httrackp *opt, int argc, char **argv) {
+  String path = STRING_EMPTY;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "expandhome: needs a path\n");
+    return 1;
+  }
+  StringCopy(path, argv[0]);
+  expand_home(&path);
+  printf("expanded=%s\n", StringBuff(path));
+  StringFree(path);
+  return 0;
+}
+
 static int st_mime(httrackp *opt, int argc, char **argv) {
   char mime[256];
 
@@ -3079,6 +3095,7 @@ static const struct selftest_entry {
     {"filterbounds", "", "matcher length/work caps reject hostile patterns",
      st_filterbounds},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
+    {"expandhome", "<path>", "expand a leading ~/ into $HOME", st_expandhome},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
     {"urlhack", "", "-%u url-hack sub-flag (www/slash/query) self-test",

tests/01_engine-expandhome.test

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+
+# SC2088: the literal, unexpanded '~' is the input under test.
+# shellcheck disable=SC2088
+
+set -euo pipefail
+
+# ~ expansion for the -O base path (expand_home). $HOME is pinned so the cases
+# cannot depend on the caller's, but MSYS rewrites it into a Windows path
+# before the native httrack.exe reads it: ask the engine what it saw rather
+# than hardcoding the value.
+ask() {
+    HOME=HTSHOME httrack -O /dev/null -#test=expandhome "$1"
+}
+exp() {
+    test "$(ask "$1")" == "expanded=$2" || exit 1
+}
+
+home=$(ask '~')
+home=${home#expanded=}
+
+# or every case below is vacuous: '~' means expansion never fired, '.' means it
+# fell back to the default without ever reading $HOME
+test "$home" != '~' || exit 1
+test "$home" != '.' || exit 1
+
+exp '~' "$home"
+exp '~/foo' "$home/foo"
+exp '~/foo/bar/#' "$home/foo/bar/#"
+exp '~/' "$home/"
+exp '~/../x' "$home/../x"
+
+# ~user/ needs getpwnam: left alone rather than mangled into $HOME + "user/"
+exp '~smith/foo' '~smith/foo'
+exp '~root' '~root'
+
+# only a leading '~' expands; anything else is a literal path component
+exp 'a~/x' 'a~/x'
+exp './~/x' './~/x'
+exp 'foo~bar' 'foo~bar'
+
+# an unset or empty $HOME must not turn ~/foo into the absolute /foo
+test "$(env -u HOME httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=./foo" || exit 1
+test "$(HOME='' httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=./foo" || exit 1
+
+# A $HOME past the 2 * HTS_URLMAXSIZE buffer is left alone: strcatbuff aborts
+# rather than truncates. Only meaningful where the engine sees the value we set:
+# MSYS rewrites HOME into a path of its own choosing, long or not.
+if test "$home" = HTSHOME; then
+    long=$(printf '%04096d' 0 | tr '0' 'A')
+    test "$(HOME="$long" httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=~/foo" || exit 1
+fi

tests/52_local-socks5.test

@@ -180,6 +180,12 @@ grep -q "^AUTH user secret\$" "$auth/socks.log" || {
     cat "$auth/socks.log" >&2
     exit 1
 }
+# the sub-negotiation carries RFC 1929's 0x01, not the 0x05 of the SOCKS5 greeting
+grep -q "^AUTHVER 1\$" "$auth/socks.log" || {
+    echo "FAIL: wrong RFC 1929 sub-negotiation version" >&2
+    cat "$auth/socks.log" >&2
+    exit 1
+}
 grep -qi "secret\|proxy-authorization" "$auth/origin-headers.log" && {
     echo "FAIL: socks credentials leaked into the origin request" >&2
     cat "$auth/origin-headers.log" >&2

tests/62_lang-integrity.test

@@ -0,0 +1,90 @@
+#!/bin/bash
+#
+# lang/*.txt are line pairs: an English msgid, then its translation. A stray
+# blank or a drifted msgid silently unhooks every translation that follows,
+# rather than failing, so assert the pairing and the join against English.txt.
+# Pairs are physical lines here; a msgid ending in \ continues onto the next one
+# for the engine (linput_cpp) but not for us. None do today.
+
+set -euo pipefail
+
+langdir="${top_srcdir:-..}/lang"
+def="${top_srcdir:-..}/lang.def"
+eng="$langdir/English.txt"
+for f in "$eng" "$def"; do
+    [ -f "$f" ] || {
+        echo "cannot find $f"
+        exit 1
+    }
+done
+
+tmp=$(mktemp -d)
+trap 'rm -rf "$tmp"' EXIT
+keys="$tmp/english.keys"
+
+# Byte-wise: each file is in its own declared legacy charset, lang.def is CRLF.
+LC_ALL=C awk 'NR%2==1 { sub(/\r$/, ""); print }' "$eng" >"$keys"
+
+# An empty parse would make every check below pass vacuously.
+nkeys=$(wc -l <"$keys")
+if [ "$nkeys" -lt 400 ]; then
+    echo "only $nkeys msgids parsed from $eng; the parse is broken"
+    exit 1
+fi
+
+nfiles=0
+fail=0
+for f in "$langdir"/*.txt; do
+    nfiles=$((nfiles + 1))
+    LC_ALL=C awk -v keys="$keys" -v name="${f##*/}" '
+        BEGIN { while ((getline k < keys) > 0) known[k] = 1 }
+        { sub(/\r$/, "") }
+        NR % 2 == 1 {
+            if ($0 == "")
+                printf "%s:%d: blank line where an English msgid belongs\n", name, NR
+            else if (!($0 in known))
+                printf "%s:%d: msgid absent from English.txt: %s\n", name, NR, $0
+            else
+                next
+            bad++
+        }
+        END {
+            if (NR % 2) {
+                printf "%s: odd line count (%d): msgid/translation pairing is broken\n", name, NR
+                bad++
+            }
+            exit(bad > 0)
+        }
+    ' "$f" || fail=1
+done
+
+if [ "$nfiles" -lt 25 ]; then
+    echo "only $nfiles language files found in $langdir"
+    exit 1
+fi
+
+# lang.def maps each LANG_* macro to the English string the GUI compiles in,
+# which is the lookup key into every lang/*.txt: no msgid, no translations.
+LC_ALL=C awk -v keys="$keys" '
+    BEGIN { while ((getline k < keys) > 0) known[k] = 1 }
+    { sub(/\r$/, "") }
+    NR % 2 == 1 { macro = $0; next }
+    macro ~ /^(LANG_|LISTDEF_)/ {
+        n++
+        if (!($0 in known)) {
+            printf "lang.def:%d: %s maps to \"%s\", which is not a msgid in English.txt\n", NR, macro, $0
+            bad++
+        }
+    }
+    END {
+        if (n < 400) {
+            printf "lang.def: only %d LANG_ entries parsed; the parse is broken\n", n
+            bad++
+        }
+        exit(bad > 0)
+    }
+' "$def" || fail=1
+
+[ "$fail" -eq 0 ] || exit 1
+
+echo "$nfiles language files consistent with $nkeys English msgids"

tests/Makefile.am

@@ -59,6 +59,7 @@ TESTS = \
 	01_engine-pause.test \
 	01_engine-rcfile.test \
 	01_engine-reconcile.test \
+	01_engine-expandhome.test \
 	01_engine-fsize.test \
 	01_engine-redirect.test \
 	01_engine-relative.test \
@@ -140,6 +141,7 @@ TESTS = \
 	58_watchdog.test \
 	59_local-tls-stall.test \
 	60_crawl-log-salvage.test \
-	61_webhttrack-locale.test
+	61_webhttrack-locale.test \
+	62_lang-integrity.test
 
 CLEANFILES = check-network_sh.cache

tests/socks5-server.py

@@ -24,6 +24,7 @@ from proxytestlib import bind_ephemeral, pipe  # noqa: E402
 # The one name the proxy answers for; a .invalid TLD never resolves (RFC 6761),
 # so a locally-resolving client could not reach us -- success proves remote DNS.
 REMOTE_HOST = b"socks-origin.invalid"
+AUTH_VERSION = 0x01  # RFC 1929 sub-negotiation version
 # index links the subpages so an -r3 crawl reuses one keep-alive socket
 LINKS = "".join('<a href="/p%d.html">%d</a>' % (i, i) for i in range(1, 6))
 ORIGIN_BODY = ("<html><body>ORIGIN-PAGE-563 " + LINKS + "</body></html>").encode()
@@ -104,6 +105,13 @@ def negotiate_auth(conn, logdir):
     if 0x02 in methods:  # prefer user/pass so the auth test exercises RFC 1929
         conn.sendall(b"\x05\x02")
         (subver,) = recvn(conn, 1)
+        # RFC 1929's version byte is 0x01, not SOCKS5's 0x05: a real proxy
+        # rejects a mismatch instead of tunnelling anyway
+        if subver != AUTH_VERSION:
+            log(logdir, "AUTHVER-BAD %d" % subver)
+            conn.sendall(bytes([AUTH_VERSION, 0x01]))  # sub-negotiation failure
+            return False
+        log(logdir, "AUTHVER %d" % subver)
         (ulen,) = recvn(conn, 1)
         uname = recvn(conn, ulen)
         (plen,) = recvn(conn, 1)