Non-ASCII -O path is double-encoded on Windows now that argv is UTF-8 (#628)

Since argv became UTF-8 through hts_argv_utf8(), opt->path_html is already
UTF-8, but hts_main_internal() still re-encoded it with
hts_convertStringSystemToUTF8(), reinterpreting the bytes as the ANSI codepage.
path_html_utf8 prefixes every saved file, so a non-ASCII -O path sent the whole
mirror into a double-encoded directory. Collapse the Windows branch to the
plain UTF-8 copy the POSIX side already used, and create the mirror root with
structcheck_utf8() so the pre-created directory matches where files are written.
On a UTF-8 filesystem both resolve to the same mkdir, so only Windows changes.
This fixes the CLI and the WinHTTrack GUI, which both hand the engine UTF-8 argv.

A local-crawl test mirrors into a non-ASCII -O directory and asserts the files
land there; it runs on the Windows CI leg where the bug is reachable.

Two adjacent gaps stay open with their own issues: a single-directory -O still
leaves a mangled path_log twin holding logs and cache (#630), and webhttrack
never converts its argv to UTF-8 (#629).

Closes #621

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

src/htscoremain.c

@@ -425,25 +425,10 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
 
       }                         // for
 
-      // Convert path to UTF-8
-#ifdef _WIN32
-      {
-        char *const path =
-          hts_convertStringSystemToUTF8(StringBuff(opt->path_html),
-                                        (int) StringLength(opt->path_html));
-        if (path != NULL) {
-          StringCopy(opt->path_html_utf8, path);
-          free(path);
-        } else {
-          StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html),
-                      StringLength(opt->path_html));
-        }
-      }
-#else
-      // Assume UTF-8 filesystem.
+      // path_html is already UTF-8 (argv is UTF-8 on Windows via
+      // hts_argv_utf8), so no re-encoding.
       StringCopyN(opt->path_html_utf8, StringBuff(opt->path_html),
                   StringLength(opt->path_html));
-#endif
 
       /* if doit.log exists, or if new URL(s) defined, 
          then DO NOT load standard config files */
@@ -2486,9 +2471,12 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
          }
          } */
 
-      // vérifier existence de la structure
-      structcheck(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_html), "/"));
-      structcheck(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "/"));
+      // vérifier existence de la structure (path_html/path_log are UTF-8, use
+      // the UTF-8 mkdir path)
+      structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                               StringBuff(opt->path_html), "/"));
+      structcheck_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
+                               StringBuff(opt->path_log), "/"));
 
       // reprise/update
       if (opt->cache) {

src/htsselftest.c

@@ -2477,6 +2477,45 @@ static int st_makeindex(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+// hts_buildtopindex takes a system-charset path but verif_backblue() below it
+// expects utf-8, so on Windows a non-ASCII project dir gets the gifs written to
+// a mangled twin (issues #216/#217). argv[0] is a writable dir.
+static int st_topindex(httrackp *opt, int argc, char **argv) {
+  char projdir[HTS_URLMAXSIZE];
+  char path[HTS_URLMAXSIZE + 16]; /* projdir plus a basename */
+#ifdef _WIN32
+  /* the GUI hands hts_buildtopindex an ANSI path; mimic it. CP1252 'cafe' */
+  static const char *const projName = "caf\xE9";
+#else
+  /* POSIX system charset is UTF-8 */
+  static const char *const projName = "caf\xC3\xA9";
+#endif
+
+  assertf(argc >= 1);
+  snprintf(projdir, sizeof(projdir), "%s/%s", argv[0], projName);
+
+  /* structcheck(), not the utf-8 MKDIR family: same charset as buildtopindex */
+  snprintf(path, sizeof(path), "%s/", projdir);
+  assertf(structcheck(path) == 0);
+
+  /* returns 0 here: the dir holds no sub-project, only the gifs matter */
+  (void) hts_buildtopindex(opt, projdir, "");
+
+  /* the gifs must land in the project dir itself, not in a mangled sibling */
+  snprintf(path, sizeof(path), "%s/backblue.gif", projdir);
+  assertf(fexist(path));
+
+  /* raw unlink/rmdir: UNLINK is utf-8 on Windows, these paths aren't */
+  unlink(path);
+  snprintf(path, sizeof(path), "%s/fade.gif", projdir);
+  unlink(path);
+  snprintf(path, sizeof(path), "%s/index.html", projdir);
+  unlink(path);
+  rmdir(projdir);
+  printf("topindex self-test OK\n");
+  return 0;
+}
+
 /* Each inplace_escape_*() must equal escape_*() on a copy. */
 static int st_inplace_escape(httrackp *opt, int argc, char **argv) {
   /* >255 bytes forces the helper's malloct path, not the stack buffer */
@@ -3184,6 +3223,9 @@ static const struct selftest_entry {
     {"useragent", "", "default User-Agent self-test", st_useragent},
     {"makeindex", "[dir]", "hts_finish_makeindex footer/refresh self-test",
      st_makeindex},
+    {"topindex", "[dir]",
+     "hts_buildtopindex charset handling of a non-ASCII project dir",
+     st_topindex},
     {"inplace-escape", "", "inplace_escape_* vs escape_* equivalence self-test",
      st_inplace_escape},
     {"escape-room", "", "HT_ADD_HTMLESCAPED* reservation-factor self-test",

src/htsserver.h

@@ -164,13 +164,12 @@ HTS_UNUSED static int linputsoc_t(T_SOC soc, char *s, int max, int timeout) {
   return -1;
 }
 
+/* Same contract as hts_gethome(), which is hidden and out of reach from here */
 static const char *gethomedir(void) {
   const char *home = getenv("HOME");
 
-  if (home)
-    return home;
-  else
-    return ".";
+  /* An empty $HOME would put the base path and httrack.ini at the root */
+  return strnotempty(home) ? home : ".";
 }
 static int linput_cpp(FILE * fp, char *s, int max) {
   int rlen = 0;

src/htstools.c

@@ -896,7 +896,20 @@ HTSEXT_API int hts_buildtopindex(httrackp * opt, const char *path,
     if (fpo) {
       find_handle h;
 
-      verif_backblue(opt, concat(catbuff, sizeof(catbuff), rpath, "/")); // générer gif
+      // générer gif. verif_backblue() is utf-8, but our path is the system
+      // charset, so on Windows convert it or the gifs land in a mangled twin
+      // dir (#217). Elsewhere the system charset is already utf-8.
+#ifdef _WIN32
+      {
+        const char *const base = concat(catbuff, sizeof(catbuff), rpath, "/");
+        char *const base_utf8 =
+            hts_convertStringSystemToUTF8(base, strlen(base));
+        verif_backblue(opt, base_utf8 != NULL ? base_utf8 : base);
+        free(base_utf8);
+      }
+#else
+      verif_backblue(opt, concat(catbuff, sizeof(catbuff), rpath, "/"));
+#endif
       // Header
       hts_template_format(fpo, toptemplate_header,
               "<!-- Mirror and index made by HTTrack Website Copier/"

src/proxy/proxytrack.h

@@ -107,13 +107,12 @@ HTS_UNUSED static void proxytrack_print_log(const char *severity, const char *fo
 	"<!-- _-._.--._._-._.--._._-._.--._._-._.--._._-._.--._. -->\r\n" \
 	"<!-- End Disable IE Friendly HTTP Error Messages -->\r\n"
 
+/* Same contract as hts_gethome(): proxytrack does not link libhttrack */
 HTS_UNUSED static const char *gethomedir(void) {
   const char *home = getenv("HOME");
 
-  if (home)
-    return home;
-  else
-    return ".";
+  /* An empty $HOME would resolve a relative path against the root */
+  return strnotempty(home) ? home : ".";
 }
 
 HTS_UNUSED static int linput(FILE * fp, char *s, int max) {

tests/01_engine-topindex.test

@@ -0,0 +1,12 @@
+#!/bin/bash
+#
+
+set -euo pipefail
+
+# hts_buildtopindex takes a system-charset path but verif_backblue below it
+# expects utf-8, mangling a non-ASCII project dir on Windows (#216, #217).
+dir=$(mktemp -d)
+trap 'rm -rf "$dir"' EXIT
+
+httrack -O /dev/null -#test=topindex "$dir" run |
+    grep -q "topindex self-test OK"

tests/63_webhttrack-home.test

@@ -0,0 +1,88 @@
+#!/bin/bash
+#
+# htsserver's $HOME lookup, through the web UI it actually renders. An exported
+# but empty $HOME must not resolve the default base path against the root.
+
+set -euo pipefail
+
+testdir=$(cd "$(dirname "$0")" && pwd)
+distdir=${top_srcdir:-$(cd "${testdir}/.." && pwd)}
+distdir=$(cd "${distdir}" && pwd)
+
+fail() {
+    echo "FAIL: $*" >&2
+    exit 1
+}
+
+command -v htsserver >/dev/null || fail "no htsserver in PATH"
+# Mirror local-crawl.sh's skip-with-77: the Debian buildd chroot has no python3.
+command -v python3 >/dev/null || {
+    echo "python3 not found; skipping" >&2
+    exit 77
+}
+
+srv=
+cleanup() {
+    test -z "${srv}" || kill -9 "${srv}" 2>/dev/null || true
+}
+trap cleanup EXIT HUP INT QUIT PIPE TERM
+
+log=$(mktemp)
+trap 'cleanup; rm -f "${log}"' EXIT
+
+# Ask htsserver for its rendered default base path, under the given $HOME
+# ("-" = leave $HOME unset).
+pathfor() {
+    local homeval=$1 port url got
+    port=$(python3 -c 'import socket
+s = socket.socket()
+s.bind(("127.0.0.1", 0))
+print(s.getsockname()[1])
+s.close()')
+
+    : >"${log}"
+    (
+        # htsserver ignores SIGTERM/SIGTTOU handling from the harness shell
+        trap '' TERM TTOU
+        if test "${homeval}" = -; then unset HOME; else export HOME="${homeval}"; fi
+        exec htsserver "${distdir}/" --port "${port}" >"${log}" 2>&1
+    ) &
+    srv=$!
+
+    for _ in $(seq 1 40); do
+        url=$(sed -n 's/^URL=//p' "${log}" 2>/dev/null) && test -n "${url}" && break
+        kill -0 "${srv}" 2>/dev/null || break
+        sleep 0.25
+    done
+    test -n "${url:-}" || fail "htsserver did not come up: $(cat "${log}")"
+
+    # The UI is served ISO-8859-1, so keep python out of text mode.
+    got=$(python3 -c 'import re, sys, urllib.request
+body = urllib.request.urlopen(sys.argv[1] + "server/step2.html", timeout=20).read()
+m = re.search(br"name=\"path\" value=\"([^\"]*)\"", body)
+print(m.group(1).decode("latin-1") if m else "")' "${url}")
+
+    kill -9 "${srv}" 2>/dev/null || true
+    wait "${srv}" 2>/dev/null || true
+    srv=
+    test -n "${got}" || fail "no path field in the rendered step2.html"
+    echo "${got}"
+}
+
+check() {
+    local desc=$1 homeval=$2 want=$3 got
+    got=$(pathfor "${homeval}")
+    test "${got}" = "${want}" || fail "${desc}: want ${want}, got ${got}"
+    echo "ok: ${desc} -> ${got}"
+}
+
+# A real HOME must reach the page, or the two cases below are vacuous: they
+# also pass when gethomedir() ignores the environment and always answers ".".
+check 'a set HOME is used' /nonexistent/hts-home /nonexistent/hts-home/websites
+
+# Exported but empty is the regression: getenv() returns "", not NULL, so a
+# bare NULL check let the base path collapse onto /websites.
+check 'an empty HOME' '' ./websites
+check 'an unset HOME' - ./websites
+
+echo "PASS"

tests/64_local-intl-outdir.test

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# A non-ASCII -O output dir must receive the mirror, not a double-encoded twin
+# (#621). argv is UTF-8 on Windows, so path_html was re-encoded as if it were
+# the ANSI codepage, prefixing every saved file with a mangled directory. Only
+# path_html is the non-ASCII "café"; path_log stays ASCII so the harness reads
+# hts-log.txt/hts-cache normally and this isolates the -O encoding path.
+
+: "${top_srcdir:=..}"
+
+bash "$top_srcdir/tests/local-crawl.sh" --html-subdir 'café' --errors 0 --files 5 \
+    --found 'simple/basic.html' \
+    httrack 'BASEURL/simple/basic.html'

tests/Makefile.am

@@ -71,6 +71,7 @@ TESTS = \
 	01_engine-status.test \
 	01_engine-stripquery.test \
 	01_engine-strsafe.test \
+	01_engine-topindex.test \
 	01_engine-urlhack.test \
 	01_engine-unescape-bounds.test \
 	01_engine-useragent.test \
@@ -142,6 +143,8 @@ TESTS = \
 	59_local-tls-stall.test \
 	60_crawl-log-salvage.test \
 	61_webhttrack-locale.test \
-	62_lang-integrity.test
+	62_lang-integrity.test \
+	63_webhttrack-home.test \
+	64_local-intl-outdir.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -47,6 +47,7 @@ key="${testdir}/server.key"
 
 tls=
 verbose=
+html_subdir=
 rerun=
 rerun_args=
 rerun_dead=
@@ -132,6 +133,13 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         rerun_args="${args[$pos]}"
         ;;
+    --html-subdir)
+        # Mirror into "$out/NAME" (path_html) but keep logs/cache in "$out"
+        # (path_log): a non-ASCII NAME exercises the -O path encoding (#621)
+        # without routing the harness-read hts-log.txt through a non-ASCII path.
+        pos=$((pos + 1))
+        html_subdir="${args[$pos]}"
+        ;;
     --errors | --errors-content | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
@@ -207,12 +215,21 @@ test -n "$ver" || die "could not run httrack"
 
 out="${tmpdir}/crawl"
 mkdir "$out" || die "could not create $out"
+# path_html holds the mirror + index; path_log holds hts-cache/hts-log.txt.
+# Default: both are "$out". With --html-subdir, path_html becomes "$out/NAME"
+# (the mirror root the audits inspect) while path_log stays "$out".
+mirrorroot="$out"
+odir="$out"
+if test -n "$html_subdir"; then
+    mirrorroot="${out}/${html_subdir}"
+    odir="${mirrorroot},${out}"
+fi
 # Localhost is fast; disable the rate/bandwidth safety limits but keep a
 # max-time backstop so a hang cannot wedge the suite.
 declare -a moreargs=(--quiet --max-time=120 --timeout=30 --disable-security-limits --robots=0)
 log="${tmpdir}/log"
 info "running httrack ${hts[*]}"
-httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
+httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" "${moreargs[@]}" "${hts[@]}" >"$log" 2>&1 &
 crawlpid=$!
 wait "$crawlpid"
 crawlres=$?
@@ -229,7 +246,7 @@ grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
 # --- optional second pass: re-mirror into the same dir (cache/update path) ----
 if test -n "$rerun"; then
     info "re-running httrack (update pass)"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -257,7 +274,7 @@ fi
 if test -n "$rerun_args"; then
     read -ra extra <<<"$rerun_args"
     info "re-running httrack with ${rerun_args}"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" "${extra[@]}" >"${log}.2" 2>&1 &
     crawlpid=$!
     wait "$crawlpid"
@@ -279,7 +296,7 @@ if test -n "$rerun_dead"; then
     stop_server "$serverpid"
     serverpid=
     info "re-running httrack against the stopped server"
-    httrack -O "$out" --user-agent="httrack $ver local ($(uname -mrs))" \
+    httrack -O "$odir" --user-agent="httrack $ver local ($(uname -mrs))" \
         "${moreargs[@]}" "${hts[@]}" >"${log}.dead" 2>&1 &
     crawlpid=$!
     wait "$crawlpid" || true
@@ -308,7 +325,7 @@ fi
 
 # --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
 hostroot=
-for cand in "${out}/127.0.0.1_${port}" "${out}/127.0.0.1"; do
+for cand in "${mirrorroot}/127.0.0.1_${port}" "${mirrorroot}/127.0.0.1"; do
     if test -d "$cand"; then
         hostroot="$cand"
         break