Run the oversized-$HOME case only where HOME survives the shell

MSYS rewrites HOME into a path of its own, so the engine never saw the long
value: it expanded ~/foo against the build directory instead of bailing. Gate
the case on the engine reporting the HOME we set; it still runs, and still
fails without the guard, on every platform that passes HOME through.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.github/workflows/windows-build.yml

@@ -158,21 +158,13 @@ jobs:
 
           pass=0 fail=0 skip=0 failed="" skipped=""
           for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test \
-                   *_local-*.test 13_crawl_proxy_https.test 58_watchdog.test \
-                   60_crawl-log-salvage.test; do
+                   *_local-*.test 13_crawl_proxy_https.test 58_watchdog.test; do
             rc=0
             run_with_timeout 600 bash "$t" >"$t.log" 2>&1 || rc=$?
             case "$rc" in
               0) pass=$((pass + 1)); echo "PASS $t" ;;
               77) skip=$((skip + 1)) skipped="$skipped $t"; echo "SKIP $t" ;;
-              124)
-                fail=$((fail + 1)) failed="$failed $t"
-                echo "FAIL $t (timed out, tree killed)"
-                # Re-running a wedge traced would just hang again: salvage the
-                # killed crawl's own logs into the artifact instead.
-                dump_crawl_logs >>"$t.log"
-                tail -n 25 "$t.log" | sed 's/^/      /'
-                ;;
+              124) fail=$((fail + 1)) failed="$failed $t"; echo "FAIL $t (timed out, tree killed)" ;;
               *)
                 fail=$((fail + 1)) failed="$failed $t"
                 echo "FAIL $t (exit $rc)"

lang.indexes

@@ -23,7 +23,7 @@ pt:7
 ro:25
 ru:8
 sk:20
-sl:24
+si:24
 sv:17
 tr:10
 uk:22

lang/Slovenian.txt

@@ -3,7 +3,7 @@ Slovenian
 LANGUAGE_FILE
 Slovenian
 LANGUAGE_ISO
-sl
+si
 LANGUAGE_AUTHOR
 Jadran Rudec,iur.\r\njrudec@email.si \r\n
 LANGUAGE_CHARSET

src/htsalias.c

@@ -567,24 +567,29 @@ int optinclude_file(const char *name, int *argc, char **argv, char *x_argvblk,
   return 0;
 }
 
-/* Get home directory, '.' if failed */
+/* Get home directory, '.' if unset or empty */
 /* example: /home/smith */
 const char *hts_gethome(void) {
   const char *home = getenv("HOME");
 
-  if (home)
-    return home;
-  else
-    return ".";
+  /* An empty $HOME would expand ~/foo into the absolute /foo */
+  return strnotempty(home) ? home : ".";
 }
 
-/* Convert ~/foo into /home/smith/foo */
+/* Convert ~/foo into /home/smith/foo (~user/ left alone: no getpwnam here) */
 void expand_home(String * str) {
-  if (StringSub(*str, 1) == '~') {
+  if (StringNotEmpty(*str) && StringSub(*str, 0) == '~' &&
+      (StringLength(*str) == 1 || StringSub(*str, 1) == '/')) {
     char BIGSTK tempo[HTS_URLMAXSIZE * 2];
+    const char *const home = hts_gethome();
+    const size_t homelen = strlen(home);
+    const size_t taillen = StringLength(*str) - 1;
 
-    strcpybuff(tempo, hts_gethome());
-    strcatbuff(tempo, StringBuff(*str) + 1);
-    StringCopy(*str, tempo);
+    /* Leave untouched rather than abort() in strcatbuff on a huge $HOME */
+    if (taillen < sizeof(tempo) && homelen < sizeof(tempo) - taillen) {
+      strcpybuff(tempo, home);
+      strcatbuff(tempo, StringBuff(*str) + 1);
+      StringCopy(*str, tempo);
+    }
   }
 }

src/htscoremain.c

@@ -113,7 +113,6 @@ HTSEXT_API int hts_main(int argc, char **argv) {
 }
 
 static int hts_main_internal(int argc, char **argv, httrackp * opt);
-static hts_boolean cmdl_shortopt_has(const char *s, char c);
 
 // Main, récupère les paramètres et appelle le robot
 HTSEXT_API int hts_main2(int argc, char **argv, httrackp * opt) {
@@ -305,12 +304,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                      hts_get_version_info(opt));
               return 0;
             } else {
-              if (strncmp(tmp_argv[0], "--", 2)) { /* not a long option */
-                if (cmdl_shortopt_has(tmp_argv[0], 'q'))
-                  opt->quiet = HTS_TRUE; // never ask questions (nohup)
-                if (cmdl_shortopt_has(tmp_argv[0], 'i')) { // doit.log!
-                  argv_url = -1;
-                  opt->quiet = HTS_TRUE;
+              if (strncmp(tmp_argv[0], "--", 2)) {      /* pas */
+                if ((strchr(tmp_argv[0], 'q') != NULL))
+                  opt->quiet = 1;       // ne pas poser de questions! (nohup par exemple)
+                if ((strchr(tmp_argv[0], 'i') != NULL)) {       // doit.log!
+                  argv_url = -1;        /* forcer */
+                  opt->quiet = 1;
                 }
               } else if (strcmp(tmp_argv[0] + 2, "quiet") == 0) {
                 opt->quiet = 1; // ne pas poser de questions! (nohup par exemple)
@@ -2808,32 +2807,6 @@ int check_path(String * s, char *defaultname) {
   return return_value;
 }
 
-/* Does the short-option cluster s carry c from the main option set (-i, -iC2,
-   -%Mi)? Walked as the parser does below: %, &, @ and # each take the letter
-   after them into another set, so the i of -%i is not the main-set -i. */
-static hts_boolean cmdl_shortopt_has(const char *s, char c) {
-  const char *com;
-
-  if (s[0] != '-' || s[1] == '-')
-    return HTS_FALSE;
-  for (com = s + 1; *com != '\0'; com++) {
-    switch (*com) {
-    case '%':
-    case '&':
-    case '@':
-    case '#':
-      if (*(com + 1) != '\0')
-        com++; /* skip the other set's letter */
-      break;
-    default:
-      if (*com == c)
-        return HTS_TRUE;
-      break;
-    }
-  }
-  return HTS_FALSE;
-}
-
 // détermine si l'argument est une option
 int cmdl_opt(char *s) {
   if (s[0] == '-') {            // c'est peut être une option

src/htsselftest.c

@@ -45,6 +45,7 @@ Please visit our Website: http://www.httrack.com
 #include "htscore.h"
 #include "htsdefines.h"
 #include "htslib.h"
+#include "htsalias.h"
 #include "htsparse.h"
 #include "htscache.h"
 #include "htscache_selftest.h"
@@ -825,6 +826,21 @@ static int st_simplify(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+static int st_expandhome(httrackp *opt, int argc, char **argv) {
+  String path = STRING_EMPTY;
+
+  (void) opt;
+  if (argc < 1) {
+    fprintf(stderr, "expandhome: needs a path\n");
+    return 1;
+  }
+  StringCopy(path, argv[0]);
+  expand_home(&path);
+  printf("expanded=%s\n", StringBuff(path));
+  StringFree(path);
+  return 0;
+}
+
 static int st_mime(httrackp *opt, int argc, char **argv) {
   char mime[256];
 
@@ -3079,6 +3095,7 @@ static const struct selftest_entry {
     {"filterbounds", "", "matcher length/work caps reject hostile patterns",
      st_filterbounds},
     {"simplify", "<path>", "collapse ./ and ../ in a path", st_simplify},
+    {"expandhome", "<path>", "expand a leading ~/ into $HOME", st_expandhome},
     {"stripquery", "", "--strip-query pattern/key stripping self-test",
      st_stripquery},
     {"urlhack", "", "-%u url-hack sub-flag (www/slash/query) self-test",

src/webhttrack

@@ -33,19 +33,6 @@ function log {
     return 0
 }
 
-# Map a POSIX locale ("zh_TW.UTF-8@euro") to its lang.indexes number, English (1) if unknown
-function lang_index {
-    local locale=$1 indexes=$2 tag n t
-    tag=$(echo "${locale}" | cut -f1 -d'.' | cut -f1 -d'@' | tr '[:upper:]' '[:lower:]' | tr -cd 'a-z_')
-    # a few languages carry a region-specific entry (zh_tw, pt_br); else use the bare language
-    for t in "${tag}" "${tag%%_*}"; do
-        n=$(grep -E "^${t}:" "${indexes}" | cut -f2 -d':')
-        test -n "${n}" && break
-    done
-    test -n "${n}" || n=1
-    echo "${n}"
-}
-
 function launch_browser {
     log "Launching $1"
     browser=$1
@@ -72,11 +59,13 @@ test -f "${DISTPATH}/lang.indexes" || ! log "Could not find ${DISTPATH}/lang.ind
 test -d "${DISTPATH}/lang" || ! log "Could not find ${DISTPATH}/lang" || exit 1
 test -d "${DISTPATH}/html" || ! log "Could not find ${DISTPATH}/html" || exit 1
 
-# Locale: POSIX precedence, LC_ALL overrides LC_MESSAGES overrides LANG
-HTSLANG="${LC_ALL}"
-! test -n "${HTSLANG}" && HTSLANG="${LC_MESSAGES}"
+# Locale
+HTSLANG="${LC_MESSAGES}"
+! test -n "${HTSLANG}" && HTSLANG="${LC_ALL}"
 ! test -n "${HTSLANG}" && HTSLANG="${LANG}"
-LANGN=$(lang_index "${HTSLANG}" "${DISTPATH}/lang.indexes")
+HTSLANG="$(echo "$LANG" | cut -f1 -d'.' | cut -f1 -d'_')"
+LANGN=$(grep -E "^${HTSLANG}:" "${DISTPATH}/lang.indexes" | cut -f2 -d':')
+! test -n "${LANGN}" && LANGN=1
 
 # Find the browser
 # note: not all systems have sensible-browser or www-browser alternative

tests/01_engine-cmdline.test

@@ -54,13 +54,6 @@ refused() {
         ! echo "FAIL: $1 (exit $RC)" || exit 1
 }
 
-# assert continue mode was entered: it drops the URL list, so with no cache to
-# resume the run ends on the usage screen rather than on any other error
-continued() {
-    { test "$RC" -ne 0 && grep -q 'usage:' "$1/.log"; } ||
-        ! echo "FAIL: $2 (exit $RC)" || exit 1
-}
-
 # a value past the old 126/256 caps but within the cap is accepted, on both the
 # short and long form of each option
 long=$(nchars 900)
@@ -109,50 +102,4 @@ for bad in nan nan:5 5:nan inf 10:5 99999; do
     refused "#185: invalid --pause '$bad' not refused cleanly"
 done
 
-# An option is not -i (continue) merely because its name contains an 'i' (#615).
-# These used to wipe the URL given before them and exit on the usage screen.
-run "$tmp/ord-bti" --build-top-index
-accepted "$tmp/ord-bti" "#615: --build-top-index after the URL wiped the URL list"
-run "$tmp/ord-bti-s" "-%i"
-accepted "$tmp/ord-bti-s" "#615: -%i after the URL wiped the URL list"
-run "$tmp/ord-proto" --protocol 2
-accepted "$tmp/ord-proto" "#615: --protocol after the URL wiped the URL list"
-run "$tmp/ord-proto-s" "-@i2"
-accepted "$tmp/ord-proto-s" "#615: -@i2 after the URL wiped the URL list"
-
-# %, &, @ and # take the letter after them into another option set, wherever
-# they sit in the cluster, so the i of -q%i is not the main-set -i either.
-run "$tmp/ord-qpi" "-q%i"
-accepted "$tmp/ord-qpi" "#615: -q%i after the URL wiped the URL list"
-run "$tmp/ord-qai" "-q@i"
-accepted "$tmp/ord-qai" "#615: -q@i after the URL wiped the URL list"
-
-# -%q only forces quiet mode, which nothing here can see: a run whose stdout is
-# not a tty is quiet from the start (htscoremain.c:174). Just check it crawls.
-run "$tmp/ord-iqs" "-%q"
-accepted "$tmp/ord-iqs" "#615: -%q after the URL broke the crawl"
-
-# The real -i still forces continue mode and drops the URL list, in every form:
-# a fix that merely stopped looking for 'i' would pass the checks above.
-run "$tmp/cont-s" -i
-continued "$tmp/cont-s" "#615: -i after the URL no longer forces continue mode"
-run "$tmp/cont-c" -iC2
-continued "$tmp/cont-c" "#615: -iC2 after the URL no longer forces continue mode"
-run "$tmp/cont-l" --continue
-continued "$tmp/cont-l" "#615: --continue after the URL no longer forces continue mode"
-run "$tmp/cont-u" --update
-continued "$tmp/cont-u" "#615: --update after the URL no longer forces continue mode"
-
-# ...including where another set's letter precedes it in the cluster.
-run "$tmp/cont-pq" "-%qi"
-continued "$tmp/cont-pq" "#615: -%qi after the URL no longer forces continue mode"
-run "$tmp/cont-pm" "-%Mi"
-continued "$tmp/cont-pm" "#615: -%Mi after the URL no longer forces continue mode"
-
-# Placed before the URL these always worked; they must keep working.
-run_only "$tmp/pre-bti" "-%i" "file://$tmp/index.html"
-accepted "$tmp/pre-bti" "#615: -%i before the URL broke the crawl"
-run_only "$tmp/pre-proto" "-@i2" "file://$tmp/index.html"
-accepted "$tmp/pre-proto" "#615: -@i2 before the URL broke the crawl"
-
 exit 0

tests/01_engine-expandhome.test

@@ -0,0 +1,53 @@
+#!/bin/bash
+#
+
+# SC2088: the literal, unexpanded '~' is the input under test.
+# shellcheck disable=SC2088
+
+set -euo pipefail
+
+# ~ expansion for the -O base path (expand_home). $HOME is pinned so the cases
+# cannot depend on the caller's, but MSYS rewrites it into a Windows path
+# before the native httrack.exe reads it: ask the engine what it saw rather
+# than hardcoding the value.
+ask() {
+    HOME=HTSHOME httrack -O /dev/null -#test=expandhome "$1"
+}
+exp() {
+    test "$(ask "$1")" == "expanded=$2" || exit 1
+}
+
+home=$(ask '~')
+home=${home#expanded=}
+
+# or every case below is vacuous: '~' means expansion never fired, '.' means it
+# fell back to the default without ever reading $HOME
+test "$home" != '~' || exit 1
+test "$home" != '.' || exit 1
+
+exp '~' "$home"
+exp '~/foo' "$home/foo"
+exp '~/foo/bar/#' "$home/foo/bar/#"
+exp '~/' "$home/"
+exp '~/../x' "$home/../x"
+
+# ~user/ needs getpwnam: left alone rather than mangled into $HOME + "user/"
+exp '~smith/foo' '~smith/foo'
+exp '~root' '~root'
+
+# only a leading '~' expands; anything else is a literal path component
+exp 'a~/x' 'a~/x'
+exp './~/x' './~/x'
+exp 'foo~bar' 'foo~bar'
+
+# an unset or empty $HOME must not turn ~/foo into the absolute /foo
+test "$(env -u HOME httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=./foo" || exit 1
+test "$(HOME='' httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=./foo" || exit 1
+
+# A $HOME past the 2 * HTS_URLMAXSIZE buffer is left alone: strcatbuff aborts
+# rather than truncates. Only meaningful where the engine sees the value we set:
+# MSYS rewrites HOME into a path of its own choosing, long or not.
+if test "$home" = HTSHOME; then
+    long=$(printf '%04096d' 0 | tr '0' 'A')
+    test "$(HOME="$long" httrack -O /dev/null -#test=expandhome '~/foo')" == "expanded=~/foo" || exit 1
+fi

tests/60_crawl-log-salvage.test

@@ -1,46 +0,0 @@
-#!/bin/bash
-#
-# Unit-tests dump_crawl_logs (testlib.sh): the killed test's hts-log.txt must
-# reach the uploaded logs, or a Windows wedge past --max-time stays
-# undiagnosable (#605).
-
-set -euo pipefail
-
-testdir=$(cd "$(dirname "$0")" && pwd)
-# shellcheck source=tests/testlib.sh
-. "${testdir}/testlib.sh"
-
-TMPDIR=$(mktemp -d "${TMPDIR:-/tmp}/httrack_salv.XXXXXX")
-export TMPDIR
-trap 'rm -rf "$TMPDIR"' EXIT
-fail() {
-    echo "FAIL: $*" >&2
-    exit 1
-}
-
-verdict="More than 120 seconds passed.. giving up"
-
-# Vacuity control: with nothing left behind the assertions below cannot pass on
-# ambient output.
-test -z "$(dump_crawl_logs)" || fail "dumped something with no crawl tmpdir"
-
-# A killed local-crawl.sh leaves exactly this shape.
-d="${TMPDIR}/httrack_local.probe"
-mkdir -p "$d/crawl"
-echo "$verdict" >"$d/crawl/hts-log.txt"
-echo "httrack stdout marker" >"$d/log"
-
-# Append to an unterminated line, as a killed test's own log always ends.
-art="${TMPDIR}/artifact.log"
-printf '[running httrack ...] ..\t' >"$art"
-dump_crawl_logs >>"$art"
-
-grep -qF "$verdict" "$art" || fail "hts-log.txt verdict not salvaged"
-grep -qF "httrack stdout marker" "$art" || fail "httrack stdout not salvaged"
-grep -q '^--- .*hts-log.txt' "$art" || fail "dump glued its header onto the unterminated line"
-
-# Cleared, so the next timing-out test cannot re-report this crawl as its own.
-test ! -d "$d" || fail "salvaged tmpdir left behind"
-test -z "$(dump_crawl_logs)" || fail "salvaged crawl reported twice"
-
-echo "crawl log salvage OK"

tests/61_webhttrack-locale.test

@@ -1,91 +0,0 @@
-#!/bin/bash
-#
-# webhttrack must map the caller's locale to the right lang.indexes number.
-# Guards: LC_ALL/LC_MESSAGES were ignored, and zh_tw/pt_br were unreachable.
-
-set -euo pipefail
-
-testdir=$(cd "$(dirname "$0")" && pwd)
-distdir=$(cd "${testdir}/.." && pwd)
-script="${distdir}/src/webhttrack"
-
-fail() {
-    echo "FAIL: $*" >&2
-    exit 1
-}
-
-test -r "${script}" || fail "no ${script}"
-test -r "${distdir}/lang.indexes" || fail "no ${distdir}/lang.indexes"
-
-# Lift the locale code out of the launcher, so this tests the shipped logic itself.
-funcs=$(sed -n '/^function lang_index/,/^}/p' "${script}")
-# End on the next section, not a blank line: a blank line grown inside the block
-# would truncate the lift and silently stop testing everything below it.
-grep -q '^# Find the browser' "${script}" || fail "locale block terminator moved in ${script}"
-block=$(sed -n '/^# Locale/,/^# Find the browser/p' "${script}" | sed '$d')
-echo "${block}" | grep -q 'LANGN=.*lang_index' || fail "could not lift the whole locale block from ${script}"
-
-# Run the lifted block against whatever locale vars the caller exported.
-run_block() {
-    # shellcheck disable=SC2034 # read by the eval'd locale block
-    DISTPATH="${distdir}"
-    # the launcher runs under neither -e ("! test -n x && y" returns 1) nor -u
-    set +eu
-    eval "${funcs}"
-    eval "${block}"
-    # shellcheck disable=SC2153 # LANGN is set by the eval'd block, not a LANG typo
-    echo "${LANGN}"
-}
-
-# Resolve LANG/LC_ALL/LC_MESSAGES (empty = unset) to the language number.
-lang_of() {
-    (
-        unset LANG LC_ALL LC_MESSAGES
-        # only the value matters here; hide bash's setlocale gripe when the locale is not installed
-        # shellcheck disable=SC2030 # confining the locale to this subshell is the point
-        {
-            if test -n "$1"; then export LANG="$1"; fi
-            if test -n "$2"; then export LC_ALL="$2"; fi
-            if test -n "$3"; then export LC_MESSAGES="$3"; fi
-        } 2>/dev/null
-        run_block
-    )
-}
-
-check() {
-    local want=$1 got desc
-    shift
-    desc="LANG='$1' LC_ALL='$2' LC_MESSAGES='$3'"
-    got=$(lang_of "$1" "$2" "$3")
-    test "${got}" = "${want}" || fail "${desc}: want ${want}, got ${got}"
-    echo "ok: ${desc} -> ${got}"
-}
-
-#      want  LANG            LC_ALL          LC_MESSAGES
-check 2 "fr_FR.UTF-8" "" ""
-check 14 "zh_TW.UTF-8" "" ""                      # region-qualified: Chinese-BIG5, not Simplified (13)
-check 12 "pt_BR.UTF-8" "" ""                      # region-qualified: Portugues-Brasil, not Portugues (7)
-check 13 "zh_CN.UTF-8" "" ""                      # unlisted region falls back to the bare language
-check 24 "sl_SI.UTF-8" "" ""                      # Slovenian is ISO 639-1 "sl"
-check 4 "" "de_DE.UTF-8" ""                       # LC_ALL alone
-check 4 "" "" "de_DE.UTF-8"                       # LC_MESSAGES alone
-check 4 "fr_FR.UTF-8" "de_DE.UTF-8" ""            # LC_ALL wins over LANG
-check 4 "fr_FR.UTF-8" "" "de_DE.UTF-8"            # LC_MESSAGES wins over LANG
-check 2 "de_DE.UTF-8" "fr_FR.UTF-8" "de_DE.UTF-8" # LC_ALL wins over LC_MESSAGES
-check 2 "fr_FR@euro" "" ""                        # @modifier without a charset
-check 1 "C" "" ""                                 # C, unknown and unset all mean English
-check 1 "xx_YY.UTF-8" "" ""
-check 1 "" "" ""
-
-# Exported but empty is unset (POSIX), so it must not mask LANG. lang_of cannot
-# express this: it exports nothing for an empty argument.
-got=$(
-    unset LANG LC_ALL LC_MESSAGES
-    # shellcheck disable=SC2031 # confining the locale to this subshell is the point
-    { export LANG="fr_FR.UTF-8" LC_ALL="" LC_MESSAGES=""; } 2>/dev/null
-    run_block
-)
-test "${got}" = 2 || fail "empty LC_ALL/LC_MESSAGES must not mask LANG: want 2, got ${got}"
-echo "ok: LANG='fr_FR.UTF-8' with LC_ALL='' exported -> ${got}"
-
-echo "PASS"

tests/Makefile.am

@@ -59,6 +59,7 @@ TESTS = \
 	01_engine-pause.test \
 	01_engine-rcfile.test \
 	01_engine-reconcile.test \
+	01_engine-expandhome.test \
 	01_engine-fsize.test \
 	01_engine-redirect.test \
 	01_engine-relative.test \
@@ -138,8 +139,6 @@ TESTS = \
 	56_local-proxy-noleak.test \
 	57_local-proxy-connect.test \
 	58_watchdog.test \
-	59_local-tls-stall.test \
-	60_crawl-log-salvage.test \
-	61_webhttrack-locale.test
+	59_local-tls-stall.test
 
 CLEANFILES = check-network_sh.cache

tests/testlib.sh

@@ -44,25 +44,6 @@ stop_server() {
     return 0
 }
 
-# Dump and clear the crawl logs a hard-killed test leaves in TMPDIR (its cleanup
-# trap never ran): hts-log.txt alone records "More than N seconds passed.. giving
-# up", so a wedge past --max-time is undiagnosable without it (#605).
-dump_crawl_logs() {
-    local d f
-    for d in "${TMPDIR:-/tmp}"/httrack_local.*; do
-        test -d "$d" || continue
-        for f in "$d/crawl/hts-log.txt" "$d/log" "$d/log.2"; do
-            test -f "$f" || continue
-            # Leading newline: the killed test's last line has no terminator.
-            printf '\n--- %s (last 200 lines)\n' "$f"
-            tail -n 200 "$f"
-        done
-        # so a later test's dump cannot re-report this one; never fatal, the
-        # caller is already handling a failure and Windows may still hold a file
-        rm -rf "$d" || true
-    done
-}
-
 # Kill a backgrounded job and its whole descendant tree. POSIX: the caller must
 # have put the job in its own process group (run_with_timeout does) so we signal
 # the group; a bare kill would orphan the grandchildren. Windows: the tree is