Merge origin/master into fix-623-delayed-marker

Resolve tests/Makefile.am: keep both 66_engine-port80-strip (from #632) and
67_engine-delayed-truncate.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

src/htscoremain.c

@@ -2368,28 +2368,30 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
       opt->errlog = stderr;
     } else if (httrack_logmode >= 2) {
       // deux fichiers log
-      // path_log holds UTF-8 bytes (argv is UTF-8): the ANSI file calls would
-      // read them as the codepage and drop the logs into a mangled twin (#630).
-      structcheck_utf8(StringBuff(opt->path_log));
-      if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                              StringBuff(opt->path_log), "hts-log.txt")))
-        UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                       StringBuff(opt->path_log), "hts-log.txt"));
-      if (fexist_utf8(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                              StringBuff(opt->path_log), "hts-err.txt")))
-        UNLINK(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                       StringBuff(opt->path_log), "hts-err.txt"));
+      structcheck(StringBuff(opt->path_log));
+      if (fexist
+          (fconcat
+           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt")))
+        remove(fconcat
+               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"));
+      if (fexist
+          (fconcat
+           (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt")))
+        remove(fconcat
+               (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"));
 
       /* Check FS directory structure created */
-      structcheck_utf8(StringBuff(opt->path_log));
+      structcheck(StringBuff(opt->path_log));
 
-      opt->log = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                               StringBuff(opt->path_log), "hts-log.txt"),
-                       "w");
+      opt->log =
+        fopen(fconcat
+              (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-log.txt"),
+              "w");
       if (httrack_logmode == 2)
-        opt->errlog = FOPEN(fconcat(OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt),
-                                    StringBuff(opt->path_log), "hts-err.txt"),
-                            "w");
+        opt->errlog =
+          fopen(fconcat
+                (OPT_GET_BUFF(opt), OPT_GET_BUFF_SIZE(opt), StringBuff(opt->path_log), "hts-err.txt"),
+                "w");
       else
         opt->errlog = opt->log;
       if (opt->log == NULL) {

tests/01_engine-xfread.test

@@ -9,23 +9,21 @@ set -euo pipefail
 # drives both refusal paths and the accept path.
 
 out="$(httrack -O /dev/null -#test=xfread-limit)"
-# Match with here-strings, not `echo | grep -q`: under pipefail the early grep
-# exit SIGPIPEs echo and fails the pipeline even when the pattern matched.
 for case in bylen bygrow; do
-    grep -q "${case}: refused=1 adr=null msg=In-memory content too large" <<<"$out" || {
+    echo "$out" | grep -q "${case}: refused=1 adr=null msg=In-memory content too large" || {
         echo "FAIL ${case}: $out"
         exit 1
     }
 done
 
 # Exactly INT32_MAX must be refused too (the reallocs add 1).
-grep -q 'boundary: msg=In-memory content too large' <<<"$out" || {
+echo "$out" | grep -q 'boundary: msg=In-memory content too large' || {
     echo "FAIL boundary: $out"
     exit 1
 }
 
 # The guard must NOT fire for a legitimate small size.
-if grep -q 'accept: msg=In-memory content too large' <<<"$out"; then
+if echo "$out" | grep -q 'accept: msg=In-memory content too large'; then
     echo "FAIL accept (guard fired on a legit size): $out"
     exit 1
 fi

tests/69_local-intl-logdir.test

@@ -1,15 +0,0 @@
-#!/bin/bash
-#
-# A single non-ASCII -O sets both path_html and path_log to "café" (#630). The
-# logs (hts-log.txt/hts-err.txt) must land there, not in an ANSI-mangled twin:
-# on Windows path_log holds UTF-8 bytes the raw file calls read as the codepage.
-# The --errors/--files/--log-found audits all grep logroot=café/hts-log.txt, so
-# a log written to the twin fails them. POSIX has no twin, so this bites on the
-# Windows CI leg (like test 64). The cache twin is a separate, larger fix.
-
-: "${top_srcdir:=..}"
-
-bash "$top_srcdir/tests/local-crawl.sh" --outdir-intl 'café' --errors 0 --files 5 \
-    --found 'simple/basic.html' \
-    --log-found 'mirror complete in' \
-    httrack 'BASEURL/simple/basic.html'

tests/Makefile.am

@@ -148,7 +148,6 @@ TESTS = \
 	64_local-intl-outdir.test \
 	65_port-siblings.test \
 	66_engine-port80-strip.test \
-	67_engine-delayed-truncate.test \
-	69_local-intl-logdir.test
+	67_engine-delayed-truncate.test
 
 CLEANFILES = check-network_sh.cache

tests/local-crawl.sh

@@ -48,7 +48,6 @@ key="${testdir}/server.key"
 tls=
 verbose=
 html_subdir=
-outdir_intl=
 rerun=
 rerun_args=
 rerun_dead=
@@ -141,13 +140,6 @@ while test "$pos" -lt "$nargs"; do
         pos=$((pos + 1))
         html_subdir="${args[$pos]}"
         ;;
-    --outdir-intl)
-        # Single non-ASCII -O "$out/NAME": path_html AND path_log are NAME, so
-        # the logs (and the harness reads of them) go through the non-ASCII path
-        # (#630). Distinct from --html-subdir, which keeps path_log ASCII.
-        pos=$((pos + 1))
-        outdir_intl="${args[$pos]}"
-        ;;
     --errors | --errors-content | --files)
         audit+=("${args[$pos]}" "${args[$((pos + 1))]}")
         pos=$((pos + 1))
@@ -224,18 +216,13 @@ test -n "$ver" || die "could not run httrack"
 out="${tmpdir}/crawl"
 mkdir "$out" || die "could not create $out"
 # path_html holds the mirror + index; path_log holds hts-cache/hts-log.txt.
-# Default: both are "$out". --html-subdir moves path_html to "$out/NAME" while
-# path_log (logroot) stays "$out"; --outdir-intl moves both to "$out/NAME".
+# Default: both are "$out". With --html-subdir, path_html becomes "$out/NAME"
+# (the mirror root the audits inspect) while path_log stays "$out".
 mirrorroot="$out"
-logroot="$out"
 odir="$out"
 if test -n "$html_subdir"; then
     mirrorroot="${out}/${html_subdir}"
     odir="${mirrorroot},${out}"
-elif test -n "$outdir_intl"; then
-    mirrorroot="${out}/${outdir_intl}"
-    logroot="$mirrorroot"
-    odir="$mirrorroot"
 fi
 # Localhost is fast; disable the rate/bandwidth safety limits but keep a
 # max-time backstop so a hang cannot wedge the suite.
@@ -254,7 +241,7 @@ test "$crawlres" -eq 0 || ! result "httrack exited $crawlres" || {
     exit 1
 }
 result "OK"
-grep -iE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt" >&2
+grep -iE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt" >&2
 
 # --- optional second pass: re-mirror into the same dir (cache/update path) ----
 if test -n "$rerun"; then
@@ -273,7 +260,7 @@ if test -n "$rerun"; then
     # The update summary reports "files updated"; a fresh crawl never does. Assert
     # it so a regression that bypasses the cache (re-crawls fresh) can't pass.
     info "checking update used the cache"
-    if grep -aqE "mirror complete in .*files updated" "${logroot}/hts-log.txt"; then
+    if grep -aqE "mirror complete in .*files updated" "${out}/hts-log.txt"; then
         result "OK"
     else
         result "update pass did not report cache activity"
@@ -305,7 +292,7 @@ if test -n "$rerun_dead"; then
     zip="${out}/hts-cache/new.zip"
     test -s "$zip" || die "no cache was written by the first pass"
     cp "$zip" "${tmpdir}/cache-before.zip"
-    cp "${logroot}/hts-log.txt" "${tmpdir}/log-before.txt"
+    cp "${out}/hts-log.txt" "${tmpdir}/log-before.txt"
     stop_server "$serverpid"
     serverpid=
     info "re-running httrack against the stopped server"
@@ -318,7 +305,7 @@ if test -n "$rerun_dead"; then
     # The dead pass must have gone through the no-data rollback, not bailed out
     # before the mirror loop (which would leave the cache trivially untouched).
     info "checking the dead pass hit the rollback"
-    if grep -aq "No data seems to have been transferred" "${logroot}/hts-log.txt"; then
+    if grep -aq "No data seems to have been transferred" "${out}/hts-log.txt"; then
         result "OK"
     else
         result "rollback notice not found in hts-log.txt"
@@ -333,7 +320,7 @@ if test -n "$rerun_dead"; then
         exit 1
     fi
     # Audits below describe the healthy crawl, not the dead pass.
-    cp "${tmpdir}/log-before.txt" "${logroot}/hts-log.txt"
+    cp "${tmpdir}/log-before.txt" "${out}/hts-log.txt"
 fi
 
 # --- discover the single host root (127.0.0.1_<port> or 127.0.0.1) -----------
@@ -363,19 +350,19 @@ while test "$i" -lt "${#audit[@]}"; do
     --errors)
         i=$((i + 1))
         assert_equals "checking errors" "${audit[$i]}" \
-            "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")"
+            "$(grep -iEc "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")"
         ;;
     --errors-content)
         i=$((i + 1))
-        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${logroot}/hts-log.txt")
+        total=$(grep -icE "^[0-9:]*[[:space:]]Error:" "${out}/hts-log.txt")
         # transient network failures (statuscode -2..-6) flake on busy loopback;
         # the code parens are followed by " at link" or " after N retries at link"
-        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${logroot}/hts-log.txt" || true)
+        transient=$(grep -cE '\(-[2-6]\) (at link|after )' "${out}/hts-log.txt" || true)
         assert_equals "checking content errors" "${audit[$i]}" "$((total - transient))"
         ;;
     --files)
         i=$((i + 1))
-        nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${logroot}/hts-log.txt" |
+        nFiles=$(grep -E "^HTTrack Website Copier/[^ ]* mirror complete in " "${out}/hts-log.txt" |
             sed -e 's/.*[[:space:]]\([^ ]*\)[[:space:]]files written.*/\1/g')
         assert_equals "checking files" "${audit[$i]}" "$nFiles"
         ;;
@@ -406,7 +393,7 @@ while test "$i" -lt "${#audit[@]}"; do
     --log-found)
         i=$((i + 1))
         info "checking log matches ${audit[$i]}"
-        if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then result "OK"; else
+        if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then result "OK"; else
             result "not in log"
             exit 1
         fi
@@ -414,7 +401,7 @@ while test "$i" -lt "${#audit[@]}"; do
     --log-not-found)
         i=$((i + 1))
         info "checking log lacks ${audit[$i]}"
-        if grep -aqE "${audit[$i]}" "${logroot}/hts-log.txt"; then
+        if grep -aqE "${audit[$i]}" "${out}/hts-log.txt"; then
             result "present in log"
             exit 1
         else result "OK"; fi