Fix two more tests the Win32 CI leg exposed

Both are test bugs surfaced by running on 32-bit Windows for the first
time, not engine bugs:

- strsafe drove its overflow through a char[4]. On a 32-bit build that
  equals sizeof(char*), so htssafe's array-vs-pointer heuristic
  (sizeof(A) != sizeof(char*), the MSVC path) reads the array as a
  pointer and skips the bound: the copy then genuinely overflows and
  crashes instead of aborting cleanly. Size the buffer off the pointer
  width so the checked path runs everywhere.

- idna's malformed-UTF-8 rejection case passed the bad bytes through
  argv. They cannot survive a Windows command line: the UTF-16 round-trip
  turns them into valid U+FFFD, which the encoder then accepts. Feed them
  as hex so the exact bytes reach the bundled punycode encoder, which
  rejects them identically on every platform.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
Signed-off-by: Xavier Roche <roche@httrack.com>

.gitattributes

@@ -1,3 +1,8 @@
 # Resource scripts are Windows tooling input and must stay CRLF: the engine has no
 # encoding guard, so an autocrlf checkout could otherwise flatten them silently.
 *.rc text eol=crlf
+
+# The test scripts run under Git Bash on the Windows CI; a CRLF checkout makes
+# bash die on $'\r' on every line of them.
+*.test text eol=lf
+*.sh text eol=lf

.github/workflows/windows-build.yml

@@ -119,38 +119,61 @@ jobs:
             throw "CRT mismatch across the boundary: libhttrack.dll=$a httrack.exe=$b"
           }
 
-      # The engine self-tests exercise the codecs the DLL was linked with: a
-      # missing brotli/zstd decoder shows up here, not just at link time.
-      - name: Run the content-coding self-tests
-        shell: pwsh
+      # The engine ships ~90 tests and none had ever run on Windows: "make check"
+      # is Linux/macOS only. These are the offline ones, driven from Git Bash
+      # against the native httrack.exe. They subsume the self-tests this step
+      # used to run inline (codecs, cache, fsize).
+      - name: Run the engine test suite (offline tests)
+        shell: bash
+        working-directory: tests
         run: |
-          $ErrorActionPreference = "Stop"
-          $exe = "src\${{ matrix.platform }}\${{ matrix.configuration }}\httrack.exe"
-          $tmp = New-Item -ItemType Directory -Path (Join-Path $env:RUNNER_TEMP "st")
-          foreach ($t in @("acceptencoding", "contentcodings")) {
-            $out = & $exe -O NUL "-#test=$t" $tmp.FullName run
-            if ($LASTEXITCODE -ne 0) { throw "$t failed" }
-            if (-not ($out | Select-String -SimpleMatch "$t self-test OK")) {
-              throw "$t did not report OK: $out"
-            }
-            Write-Host ($out -join "`n")
-          }
+          set -u
+          bin="$(cygpath -u "$GITHUB_WORKSPACE")/src/${{ matrix.platform }}/${{ matrix.configuration }}"
+          export PATH="$bin:$PATH"
+          command -v httrack >/dev/null || { echo "::error::no httrack.exe in $bin"; exit 1; }
 
-      # These fail only on Windows: the cache ones on a DOS-ified save name (fconv
-      # is a no-op on POSIX, so make check cannot see it), fsize on a 32-bit
-      # st_size/off_t. They report failure by exit code, not an "OK" line.
-      - name: Run the Windows-sensitive self-tests
-        shell: pwsh
-        run: |
-          $ErrorActionPreference = "Stop"
-          $exe = "src\${{ matrix.platform }}\${{ matrix.configuration }}\httrack.exe"
-          foreach ($t in @("cache", "cache-corrupt", "fsize")) {
-            $tmp = New-Item -ItemType Directory -Force `
-              -Path (Join-Path $env:RUNNER_TEMP "st-$t")
-            $out = & $exe -O NUL "-#test=$t" $tmp.FullName run 2>&1
-            Write-Host ($out -join "`n")
-            if ($LASTEXITCODE -ne 0) { throw "$t failed (exit $LASTEXITCODE)" }
-          }
+          # httrack.exe is native, so MSYS rewrites any argument shaped like a
+          # POSIX path, and a URL path is shaped exactly like one: "/a/b.html"
+          # reached the engine as "C:/Program Files/Git/a/b.html". Switch that
+          # off, and hand the tests a TMPDIR that is already a Windows path.
+          export MSYS_NO_PATHCONV=1
+          export MSYS2_ARG_CONV_EXCL='*'
+          TMPDIR="$(cygpath -m "$RUNNER_TEMP")"
+          export TMPDIR
+
+          pass=0 fail=0 skip=0 failed=""
+          for t in 00_runnable.test 01_engine-*.test 01_zlib-*.test; do
+            rc=0
+            bash "$t" >"$t.log" 2>&1 || rc=$?
+            case "$rc" in
+              0) pass=$((pass + 1)); echo "PASS $t" ;;
+              77) skip=$((skip + 1)); echo "SKIP $t" ;;
+              *)
+                fail=$((fail + 1)) failed="$failed $t"
+                echo "FAIL $t (exit $rc)"
+                # These assert with `test "$(...)" == "..." || exit 1`, which
+                # says nothing at all on failure. Re-run traced.
+                bash -x "$t" >>"$t.log" 2>&1 || true
+                tail -n 25 "$t.log" | sed 's/^/      /'
+                ;;
+            esac
+          done
+          echo "ran=$((pass + fail + skip)) pass=$pass fail=$fail skip=$skip" |
+            tee -a "$GITHUB_STEP_SUMMARY"
+
+          # Every gate in these scripts exits 77, so a suite that degraded to
+          # all-skipped would report green having tested nothing: assert a floor
+          # on what actually ran, not just the absence of failures.
+          [ "$pass" -ge 45 ] || { echo "::error::only $pass tests passed ($skip skipped)"; exit 1; }
+          [ "$fail" -eq 0 ] || { echo "::error::failing:$failed"; exit 1; }
+
+      - name: Upload the test logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: engine-tests-${{ matrix.platform }}-${{ matrix.configuration }}
+          path: tests/*.log
+          if-no-files-found: warn
 
       - name: Upload MSBuild logs
         if: always()

src/htscharset.c

@@ -31,6 +31,9 @@ Please visit our Website: http://www.httrack.com
 /* ------------------------------------------------------------ */
 
 #include "htscharset.h"
+#ifdef _WIN32
+#include <shellapi.h>
+#endif
 #include "htsbase.h"
 #include "punycode.h"
 #include "htssafe.h"
@@ -386,6 +389,40 @@ char *hts_convertStringSystemToUTF8(const char *s, size_t size) {
   return hts_convertStringCPToUTF8(s, size, GetACP());
 }
 
+HTSEXT_API void hts_argv_utf8(int *pargc, char ***pargv) {
+  int wargc = 0;
+  LPWSTR *const wargv = CommandLineToArgvW(GetCommandLineW(), &wargc);
+  char **argv;
+  int i;
+
+  // On any failure keep the CRT's ANSI argv: lossy, but never half-converted.
+  if (wargv == NULL)
+    return;
+  if (wargc <= 0 ||
+      (argv = calloct((size_t) wargc + 1, sizeof(char *))) == NULL) {
+    LocalFree(wargv);
+    return;
+  }
+  for (i = 0; i < wargc; i++) {
+    const int wsize = (int) wcslen(wargv[i]);
+
+    // hts_convertUCS2StringToUTF8() returns NULL on an empty string
+    argv[i] =
+        wsize != 0 ? hts_convertUCS2StringToUTF8(wargv[i], wsize) : strdupt("");
+    if (argv[i] == NULL) {
+      while (i-- != 0)
+        freet(argv[i]);
+      freet(argv);
+      LocalFree(wargv);
+      return;
+    }
+  }
+  argv[wargc] = NULL; // callers may rely on argv[argc] == NULL
+  LocalFree(wargv);
+  *pargc = wargc;
+  *pargv = argv; // never freed: argv lives for the process
+}
+
 #else
 
 #include <errno.h>

src/htscharset.h

@@ -34,6 +34,7 @@ Please visit our Website: http://www.httrack.com
 #define HTS_CHARSET_DEFH
 
 /** Standard includes. **/
+#include "htsglobal.h"
 #include <stdlib.h>
 #include <string.h>
 #ifdef _WIN32
@@ -175,6 +176,15 @@ extern char *hts_convertUCS2StringToUTF8(LPWSTR woutput, int wsize);
  * This function is WIN32 specific.
  **/
 extern char *hts_convertStringSystemToUTF8(const char *s, size_t size);
+
+/**
+ * Replace the CRT's ANSI argv by a UTF-8 one decoded from the real UTF-16
+ * command line: every char* is UTF-8 on Windows (FOPEN, STAT, ... convert at
+ * the syscall boundary). Keeps the CRT's argv on failure; the new array is
+ * writable, NULL-terminated, and lives for the process.
+ * This function is WIN32 specific.
+ **/
+HTSEXT_API void hts_argv_utf8(int *pargc, char ***pargv);
 #endif
 
 #endif

src/htscoremain.c

@@ -2584,21 +2584,11 @@ static int hts_main_internal(int argc, char **argv, httrackp * opt) {
                 hts_get_version_info(opt), t, url);
         fprintf(opt->log, "(");
         for(i = 0; i < argc; i++) {
-#ifdef _WIN32
-          char *carg =
-            hts_convertStringSystemToUTF8(argv[i], (int) strlen(argv[i]));
-          char *arg = carg != NULL ? carg : argv[i];
-#else
-          const char *arg = argv[i];
-#endif
+          const char *arg = argv[i]; // already UTF-8 on every platform
           if (strchr(arg, ' ') == NULL || strchr(arg, '\"') != NULL)
             fprintf(opt->log, "%s ", arg);
           else                  // entre "" (si espace(s) et pas déja de ")
             fprintf(opt->log, "\"%s\" ", arg);
-#ifdef _WIN32
-          if (carg != NULL)
-            free(carg);
-#endif
         }
         fprintf(opt->log, ")" LF);
         fprintf(opt->log, LF);

src/htsfilters.c

@@ -122,8 +122,11 @@ int fa_strjoker_dual(int type, char **filters, int nfil, const char *nom1,
 }
 
 /* Real filters/URLs fit HTS_URLMAXSIZE*2; past it a hostile pattern recurses
-   O(len) deep or runs O(n^2*stars). Cap length (depth) and steps (work). */
+   O(len) deep or runs O(n^2*stars). Cap length, recursion depth and steps
+   (work): the length cap alone still allows ~2000 frames, ~900KB of stack,
+   which overflows the 1MB a Windows thread gets (#574). */
 #define STRJOKER_MAXLEN (HTS_URLMAXSIZE * 2)
+#define STRJOKER_MAXDEPTH 256u
 #define STRJOKER_MAXSTEPS 2000000u
 
 /* Failure memo for the recursive matcher: one bit per (chaine, joker) offset
@@ -133,20 +136,28 @@ typedef struct strjoker_memo {
   size_t stride;                /* strlen(joker0) + 1 */
   unsigned char *failed;        /* failed-pair bitmap; NULL: no memo */
   size_t *nsteps; /* shared work counter; NULL: unbounded (oracle) */
+  size_t maxdepth; /* deepest recursion reached */
+  hts_boolean cut; /* a branch hit the depth cap: its failures are not final */
 } strjoker_memo;
 
-static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
-                                 const char *joker, LLint *size,
-                                 int *size_flag);
+static const char *strjoker_impl(strjoker_memo *memo, const char *chaine,
+                                 const char *joker, LLint *size, int *size_flag,
+                                 size_t depth);
 
 /* A pair that failed once fails forever (*size is only written on the success
    path), so record NULL results and cut the re-exploration. */
-static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
-                                const char *joker, LLint *size,
-                                int *size_flag) {
+static const char *strjoker_rec(strjoker_memo *memo, const char *chaine,
+                                const char *joker, LLint *size, int *size_flag,
+                                size_t depth) {
   size_t bit = 0;
   const char *adr;
 
+  if (depth > memo->maxdepth)
+    memo->maxdepth = depth;
+  if (depth >= STRJOKER_MAXDEPTH) {
+    memo->cut = HTS_TRUE;
+    return NULL; /* nesting beyond any real filter: fail the branch safely */
+  }
   if (memo->nsteps != NULL && ++*memo->nsteps > STRJOKER_MAXSTEPS)
     return NULL; /* work budget spent: fail the match safely */
   if (memo->failed) {
@@ -155,8 +166,9 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
     if (memo->failed[bit >> 3] & (unsigned char) (1u << (bit & 7)))
       return NULL;
   }
-  adr = strjoker_impl(memo, chaine, joker, size, size_flag);
-  if (adr == NULL && memo->failed)
+  adr = strjoker_impl(memo, chaine, joker, size, size_flag, depth + 1);
+  /* a cut branch may fail here yet match when reached at a shallower depth */
+  if (adr == NULL && memo->failed && !memo->cut)
     memo->failed[bit >> 3] |= (unsigned char) (1u << (bit & 7));
   return adr;
 }
@@ -164,13 +176,15 @@ static const char *strjoker_rec(const strjoker_memo *memo, const char *chaine,
 /* Match chaine against joker with a shared work budget *nsteps (a strjokerfind
    sweep passes one counter, bounding the whole scan). */
 static const char *strjoker_bounded(const char *chaine, const char *joker,
-                                    LLint *size, int *size_flag,
-                                    size_t *nsteps) {
-  strjoker_memo memo = {chaine, joker, 0, NULL, nsteps};
+                                    LLint *size, int *size_flag, size_t *nsteps,
+                                    size_t *maxdepth) {
+  strjoker_memo memo = {chaine, joker, 0, NULL, nsteps, 0, HTS_FALSE};
   unsigned char stackbits[2048];
   hts_boolean onheap = HTS_FALSE;
   const char *adr;
 
+  if (maxdepth != NULL)
+    *maxdepth = 0;
   if (chaine != NULL && joker != NULL) {
     const size_t l1 = strlen(chaine), l2 = strlen(joker);
 
@@ -189,9 +203,11 @@ static const char *strjoker_bounded(const char *chaine, const char *joker,
       }
     }
   }
-  adr = strjoker_rec(&memo, chaine, joker, size, size_flag);
+  adr = strjoker_rec(&memo, chaine, joker, size, size_flag, 0);
   if (onheap)
     freet(memo.failed);
+  if (maxdepth != NULL)
+    *maxdepth = memo.maxdepth;
   return adr;
 }
 
@@ -202,34 +218,39 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker,
                                 LLint *size, int *size_flag) {
   size_t nsteps = 0;
 
-  return strjoker_bounded(chaine, joker, size, size_flag, &nsteps);
+  return strjoker_bounded(chaine, joker, size, size_flag, &nsteps, NULL);
 }
 
 /* Test-only oracle: the same matcher without the failure memo. */
 const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
                             int *size_flag) {
-  const strjoker_memo memo = {chaine, joker, 0, NULL};
+  strjoker_memo memo = {chaine, joker, 0, NULL, NULL, 0, HTS_FALSE};
 
-  return strjoker_rec(&memo, chaine, joker, size, size_flag);
+  return strjoker_rec(&memo, chaine, joker, size, size_flag, 0);
 }
 
-/* Test-only: strjoker() reporting the work-budget steps spent and the cap, so a
-   self-test can prove the budget bounds a hostile pattern's work. */
-const char *strjoker_steps(const char *chaine, const char *joker,
-                           size_t *nsteps_out, size_t *maxsteps_out) {
-  size_t nsteps = 0;
-  const char *r = strjoker_bounded(chaine, joker, NULL, NULL, &nsteps);
+/* Test-only: strjoker() reporting the work and depth spent and their caps, so a
+   self-test can prove both bound a hostile pattern. */
+const char *strjoker_bounds(const char *chaine, const char *joker,
+                            size_t *nsteps_out, size_t *maxsteps_out,
+                            size_t *depth_out, size_t *maxdepth_out) {
+  size_t nsteps = 0, depth = 0;
+  const char *r = strjoker_bounded(chaine, joker, NULL, NULL, &nsteps, &depth);
 
   if (nsteps_out != NULL)
     *nsteps_out = nsteps;
   if (maxsteps_out != NULL)
     *maxsteps_out = STRJOKER_MAXSTEPS;
+  if (depth_out != NULL)
+    *depth_out = depth;
+  if (maxdepth_out != NULL)
+    *maxdepth_out = STRJOKER_MAXDEPTH;
   return r;
 }
 
-static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
-                                 const char *joker, LLint *size,
-                                 int *size_flag) {
+static const char *strjoker_impl(strjoker_memo *memo, const char *chaine,
+                                 const char *joker, LLint *size, int *size_flag,
+                                 size_t depth) {
   if (strnotempty(joker) == 0) {        // fin de chaine joker
     if (strnotempty(chaine) == 0)       // fin aussi pour la chaine: ok
       return chaine;
@@ -403,7 +424,8 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
 
       // tester sans le joker (pas ()+ mais ()*)
       if (!unique) {
-        if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag))) {
+        if ((adr = strjoker_rec(memo, chaine, joker + jmp, size, size_flag,
+                                depth))) {
           return adr;
         }
       }
@@ -416,7 +438,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       while(i < (int) max) {
         if (pass[(int) (unsigned char) chaine[i]]) {    // caractère autorisé
           if ((adr = strjoker_rec(memo, chaine + i + 1, joker + jmp, size,
-                                  size_flag))) {
+                                  size_flag, depth))) {
             return adr;
           }
           i++;
@@ -427,7 +449,7 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       // tester chaîne vide
       if (i != max + 2)         // avant c'est ok
         if ((adr = strjoker_rec(memo, chaine + max, joker + jmp, size,
-                                size_flag)))
+                                size_flag, depth)))
           return adr;
 
       return NULL;              // perdu
@@ -449,7 +471,8 @@ static const char *strjoker_impl(const strjoker_memo *memo, const char *chaine,
       // comparaison ok?
       if (ok) {
         // continuer la comparaison.
-        if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag))
+        if (strjoker_rec(memo, chaine + jmp, joker + jmp, size, size_flag,
+                         depth))
           return chaine;        // retourner 1e lettre
       }
 
@@ -471,8 +494,8 @@ const char *strjokerfind(const char *chaine, const char *joker) {
   if (chaine != NULL && strlen(chaine) > STRJOKER_MAXLEN)
     return NULL;
   while(*chaine) {
-    if ((adr = strjoker_bounded(chaine, joker, NULL, NULL,
-                                &nsteps))) { // ok trouvé
+    if ((adr = strjoker_bounded(chaine, joker, NULL, NULL, &nsteps,
+                                NULL))) { // ok trouvé
       return adr;
     }
     if (nsteps > STRJOKER_MAXSTEPS) // scan budget spent: no match

src/htsfilters.h

@@ -52,10 +52,12 @@ HTS_INLINE const char *strjoker(const char *chaine, const char *joker, LLint * s
    oracle for the memoized matcher. */
 const char *strjoker_nomemo(const char *chaine, const char *joker, LLint *size,
                             int *size_flag);
-/* strjoker() reporting the work-budget steps it spent and the cap; test-only,
-   lets a self-test assert the budget bounds a hostile pattern's work. */
-const char *strjoker_steps(const char *chaine, const char *joker,
-                           size_t *nsteps_out, size_t *maxsteps_out);
+/* strjoker() reporting the work-budget steps and the recursion depth it spent,
+   with their caps; test-only, lets a self-test assert both bound a hostile
+   pattern (depth bounds the stack: an uncapped one overflows Windows' 1MB). */
+const char *strjoker_bounds(const char *chaine, const char *joker,
+                            size_t *nsteps_out, size_t *maxsteps_out,
+                            size_t *depth_out, size_t *maxdepth_out);
 const char *strjokerfind(const char *chaine, const char *joker);
 #endif
 

src/htsselftest.c

@@ -757,7 +757,7 @@ static int st_filterbounds(httrackp *opt, int argc, char **argv) {
   const size_t subjlen = 2048;
   char *subj = malloct(big + 1);
   char *pat = malloct(2 * stars + 2);
-  size_t steps = 0, maxsteps = 0, i;
+  size_t steps = 0, maxsteps = 0, depth = 0, maxdepth = 0, i;
 
   (void) opt;
   (void) argc;
@@ -778,8 +778,11 @@ static int st_filterbounds(httrackp *opt, int argc, char **argv) {
   subj[subjlen] = '\0';
   /* Budget must fire and hold: steps > cap (deleting the budget zeroes the
      counter that is the enforcement), steps < 10*cap (unbudgeted ~1.26e9). */
-  assertf(strjoker_steps(subj, pat, &steps, &maxsteps) == NULL);
+  assertf(strjoker_bounds(subj, pat, &steps, &maxsteps, &depth, &maxdepth) ==
+          NULL);
   assertf(steps > maxsteps && steps < 10 * maxsteps);
+  /* Depth caps the stack: uncapped this recurses 2046 frames, ~900KB (#574). */
+  assertf(depth == maxdepth);
   assertf(strjokerfind(subj, pat) == NULL);
   freet(pat);
   freet(subj);
@@ -817,7 +820,11 @@ static int st_mime(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+static size_t st_decode_body(const char *arg, char *buf, size_t size);
+
 static int st_charset(httrackp *opt, int argc, char **argv) {
+  char buf[512];
+  size_t len;
   char *s;
 
   (void) opt;
@@ -825,7 +832,8 @@ static int st_charset(httrackp *opt, int argc, char **argv) {
     fprintf(stderr, "charset: needs a charset and a string\n");
     return 1;
   }
-  s = hts_convertStringToUTF8(argv[1], strlen(argv[1]), argv[0]);
+  len = st_decode_body(argv[1], buf, sizeof(buf));
+  s = hts_convertStringToUTF8(buf, len, argv[0]);
   if (s != NULL) {
     printf("%s\n", s);
     freet(s);
@@ -850,16 +858,22 @@ static int st_metacharset(httrackp *opt, int argc, char **argv) {
 }
 
 static int st_isutf8(httrackp *opt, int argc, char **argv) {
+  char buf[512];
+  size_t len;
+
   (void) opt;
   if (argc < 1) {
     fprintf(stderr, "isutf8: needs a string\n");
     return 1;
   }
-  printf("%d\n", hts_isStringUTF8(argv[0], strlen(argv[0])) ? 1 : 0);
+  len = st_decode_body(argv[0], buf, sizeof(buf));
+  printf("%d\n", hts_isStringUTF8(buf, len) ? 1 : 0);
   return 0;
 }
 
 static int st_idna_encode(httrackp *opt, int argc, char **argv) {
+  char buf[512];
+  size_t len;
   char *s;
 
   (void) opt;
@@ -867,7 +881,8 @@ static int st_idna_encode(httrackp *opt, int argc, char **argv) {
     fprintf(stderr, "idna-encode: needs a hostname\n");
     return 1;
   }
-  s = hts_convertStringUTF8ToIDNA(argv[0], strlen(argv[0]));
+  len = st_decode_body(argv[0], buf, sizeof(buf));
+  s = hts_convertStringUTF8ToIDNA(buf, len);
   if (s != NULL) {
     printf("%s\n", s);
     freet(s);
@@ -1129,7 +1144,10 @@ static int st_strsafe(httrackp *opt, int argc, char **argv) {
        comes from argv so its length is opaque to the compiler (no static
        -Wstringop-overflow, genuine runtime check). "overflow-buff" exercises
        htsbuff. */
-    char small[4];
+    /* Not sizeof(char*): on ILP32 a char[4] equals the pointer size, and the
+       MSVC array-vs-pointer heuristic (sizeof(A) != sizeof(char*)) then reads
+       it as a pointer and silently skips the bound. */
+    char small[6];
     const char *const src = (argc >= 2) ? argv[1] : "overflowing";
 
     if (strcmp(argv[0], "overflow-buff") == 0) {
@@ -1613,7 +1631,8 @@ static int st_crange(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
-/* Decode a body argument ("hex:FFD8.." or literal text) into buf. */
+/* Decode an argument ("hex:FFD8.." or literal text) into buf. Raw non-UTF-8
+   bytes cannot survive a Windows command line, so hostile inputs go as hex. */
 static size_t st_decode_body(const char *arg, char *buf, size_t size) {
   size_t n = 0;
 
@@ -2887,11 +2906,43 @@ static int st_robots(httrackp *opt, int argc, char **argv) {
   return 0;
 }
 
+/* Connected stream pair over loopback; Windows has no socketpair(). */
+static int st_socketpair(T_SOC sv[2]) {
+  struct sockaddr_in sa;
+  socklen_t len = sizeof(sa);
+  T_SOC srv, cli, acc;
+
+  memset(&sa, 0, sizeof(sa));
+  sa.sin_family = AF_INET;
+  sa.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+  if ((srv = (T_SOC) socket(AF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET)
+    return -1;
+  if (bind(srv, (struct sockaddr *) &sa, sizeof(sa)) != 0 ||
+      listen(srv, 1) != 0 ||
+      getsockname(srv, (struct sockaddr *) &sa, &len) != 0) {
+    deletesoc(srv);
+    return -1;
+  }
+  if ((cli = (T_SOC) socket(AF_INET, SOCK_STREAM, 0)) == INVALID_SOCKET) {
+    deletesoc(srv);
+    return -1;
+  }
+  if (connect(cli, (struct sockaddr *) &sa, sizeof(sa)) != 0 ||
+      (acc = (T_SOC) accept(srv, NULL, NULL)) == INVALID_SOCKET) {
+    deletesoc(cli);
+    deletesoc(srv);
+    return -1;
+  }
+  deletesoc(srv);
+  sv[0] = acc;
+  sv[1] = cli;
+  return 0;
+}
+
 /* get_ftp_line must bound a hostile, CRLF-less reply into its internal
    1024-byte buffer; ASan turns the pre-fix overflow into an abort here. */
-#ifndef _WIN32
 static int st_ftpline(httrackp *opt, int argc, char **argv) {
-  int sv[2];
+  T_SOC sv[2];
   char line[2048];
   char flood[4096];
 
@@ -2899,19 +2950,19 @@ static int st_ftpline(httrackp *opt, int argc, char **argv) {
   (void) argc;
   (void) argv;
   memset(flood, 'x', sizeof(flood));
-  assertf(socketpair(AF_UNIX, SOCK_STREAM, 0, sv) == 0);
-  assertf(write(sv[1], "220 ", 4) == 4); // valid 3-digit code
-  assertf(write(sv[1], flood, sizeof(flood)) == (ssize_t) sizeof(flood));
-  assertf(write(sv[1], "\r\n", 2) == 2); // end the line so we return
-  close(sv[1]);
+  assertf(st_socketpair(sv) == 0);
+  // the 4102-byte reply fits the loopback send buffer, so no reader is needed
+  assertf(send(sv[1], "220 ", 4, 0) == 4); // valid 3-digit code
+  assertf(send(sv[1], flood, (int) sizeof(flood), 0) == (int) sizeof(flood));
+  assertf(send(sv[1], "\r\n", 2, 0) == 2); // end the line so we return
+  deletesoc(sv[1]);
   line[0] = '\0';
   get_ftp_line(sv[0], line, sizeof(line), 5);
-  close(sv[0]);
+  deletesoc(sv[0]);
   printf("ftp-line self-test OK (bounded %d-byte reply)\n",
          (int) sizeof(flood));
   return 0;
 }
-#endif
 
 /* ftp_split_userpass: well-formed split, plus a hostile over-long userinfo
    that pre-fix overran user[256]/pass[256]. */
@@ -2983,11 +3034,11 @@ static const struct selftest_entry {
     {"redirect-samefile", "", "same-file redirect detection self-test (#159)",
      st_redirect_samefile},
     {"mime", "<filename>", "MIME type for a filename", st_mime},
-    {"charset", "<charset> <string>",
+    {"charset", "<charset> <hex:..|string>",
      "convert a string to UTF-8 from a charset", st_charset},
     {"metacharset", "<html>", "extract the <meta> charset from an HTML page",
      st_metacharset},
-    {"isutf8", "<string>", "is the string valid UTF-8 (1/0)", st_isutf8},
+    {"isutf8", "<hex:..|string>", "is the string valid UTF-8 (1/0)", st_isutf8},
     {"idna-encode", "<host>", "encode a hostname to IDNA/punycode",
      st_idna_encode},
     {"idna-decode", "<host>", "decode an IDNA/punycode hostname",
@@ -3055,10 +3106,8 @@ static const struct selftest_entry {
      st_contentcodings},
     {"robots", "", "robots.txt RFC 9309 Allow/Disallow precedence self-test",
      st_robots},
-#ifndef _WIN32
     {"ftp-line", "", "get_ftp_line bounds a hostile FTP reply line",
      st_ftpline},
-#endif
     {"ftp-userpass", "", "ftp_split_userpass bounds URL userinfo", st_ftpuser},
 };
 

src/httrack.c

@@ -45,6 +45,7 @@ Please visit our Website: http://www.httrack.com
 #include "htsdefines.h"
 #include "httrack.h"
 #include "htslib.h"
+#include "htscharset.h" // after htslib.h: winsock2.h must precede windows.h
 
 /* Static definitions */
 static int fexist(const char *s);
@@ -206,6 +207,7 @@ int main(int argc, char **argv) {
   httrackp *opt;
 
 #ifdef _WIN32
+  hts_argv_utf8(&argc, &argv);
   {
     WORD wVersionRequested;     // requested version WinSock API
     WSADATA wsadata;            // Windows Sockets API data

src/libhttrack.vcxproj

@@ -70,7 +70,7 @@
     <Link>
       <!-- vcpkg auto-links libssl/libcrypto/zlib/brotli/zstd; only the OS import
            lib is explicit. -->
-      <AdditionalDependencies>Ws2_32.lib;%(AdditionalDependencies)</AdditionalDependencies>
+      <AdditionalDependencies>Ws2_32.lib;Shell32.lib;%(AdditionalDependencies)</AdditionalDependencies>
       <GenerateDebugInformation>true</GenerateDebugInformation>
     </Link>
   </ItemDefinitionGroup>

tests/01_engine-charset.test

@@ -23,7 +23,7 @@ conv 'iso-8859-1' 'café' 'café'
 
 # windows-1252 is NOT latin-1: 0x80 is the euro sign, not U+0080. This is the
 # case that actually exercises the cp1252 table (the 0x80-0x9F range).
-conv 'windows-1252' $'\x80' '€'
+conv 'windows-1252' 'hex:80' '€'
 
 # pure ASCII is charset-invariant
 conv 'us-ascii' 'hello' 'hello'
@@ -34,8 +34,8 @@ conv 'no-such-charset-xyz' 'abc' 'abc'
 test "$(httrack -O /dev/null -#test=charset 'no-such-charset-xyz' 'café' 2>/dev/null)" == "" || exit 1
 
 # malformed UTF-8 (lone continuation byte, truncated lead byte) must not crash
-runs 'utf-8' $'\x80'
-runs 'utf-8' $'\xc3'
+runs 'utf-8' 'hex:80'
+runs 'utf-8' 'hex:c3'
 
 # hts_getCharsetFromMeta: <meta> charset extraction, HTML5 and legacy forms.
 # -#test=metacharset <html> prints the charset, or "(none)".
@@ -89,11 +89,11 @@ utf8() {
 utf8 'plain' '1'
 utf8 'café' '1'
 utf8 '统计' '1'
-utf8 "$(printf 'caf\xe9')" '0'              # latin-1 accent
-utf8 "$(printf '\xc3')" '0'                 # truncated lead byte
-utf8 "$(printf '\xa9')" '0'                 # lone continuation byte
-utf8 "$(printf '\xf0\x9f\x98\x80')" '1'     # 4-byte emoji
-utf8 "$(printf '\xc0\xaf')" '0'             # overlong (latin-1 "À¯" must stay convertible)
-utf8 "$(printf '\xed\xa0\x80')" '0'         # UTF-16 surrogate
-utf8 "$(printf '\xf4\x90\x80\x80')" '0'     # above U+10FFFF
-utf8 "$(printf '\xf8\x88\x80\x80\x80')" '0' # legacy 5-byte form
+utf8 'hex:636166e9' '0'   # latin-1 accent
+utf8 'hex:c3' '0'         # truncated lead byte
+utf8 'hex:a9' '0'         # lone continuation byte
+utf8 'hex:f09f9880' '1'   # 4-byte emoji
+utf8 'hex:c0af' '0'       # overlong (latin-1 "À¯" must stay convertible)
+utf8 'hex:eda080' '0'     # UTF-16 surrogate
+utf8 'hex:f4908080' '0'   # above U+10FFFF
+utf8 'hex:f888808080' '0' # legacy 5-byte form

tests/01_engine-filelist.test

@@ -52,14 +52,15 @@ run "$tmp/dir" "$tmp/adir"
 loghas 'Could not include URL list "[^"]+": not a regular file'
 
 # unreadable regular file: the fopen() errno arm fires, distinct from the
-# directory branch. Root bypasses mode 000, so skip it there.
-if test "$(id -u)" -ne 0; then
-    : >"$tmp/noperm.txt"
-    chmod 000 "$tmp/noperm.txt"
+# directory branch. Needs an OS that enforces mode 000: root bypasses it, and
+# Windows ignores chmod entirely, so probe rather than assume.
+: >"$tmp/noperm.txt"
+chmod 000 "$tmp/noperm.txt"
+if ! cat "$tmp/noperm.txt" >/dev/null 2>&1; then
     run "$tmp/perm" "$tmp/noperm.txt"
-    chmod 644 "$tmp/noperm.txt"
     loghas 'Could not include URL list "[^"]+": .+'
     lognot 'not a regular file'
 fi
+chmod 644 "$tmp/noperm.txt"
 
 exit 0

tests/01_engine-filter.test

@@ -193,5 +193,13 @@ test "$(with_timeout httrack -O /dev/null -#test=filter "$pat" "$subj")" == "$su
 match '*[aX]X*[a]Y' 'aXaXaY'
 nomatch '*[aX]X*[a]Y' 'aXbXaY'
 
-# hostile patterns must not stack-overflow or hang: length + work caps
+# hostile patterns must not stack-overflow or hang: length + depth + work caps
 test "$(with_timeout httrack -O /dev/null -#test=filterbounds)" == "filterbounds: OK" || exit 1
+
+# and the depth cap must hold on the 1MB stack a Windows thread gets (#574):
+# uncapped, the same pattern recursed ~900KB deep. Skipped where 512K is too
+# small to even start the binary.
+if (ulimit -s 512 && httrack --version >/dev/null 2>&1); then
+    out=$(ulimit -s 512 && with_timeout httrack -O /dev/null -#test=filterbounds)
+    test "$out" == "filterbounds: OK" || exit 1
+fi

tests/01_engine-idna.test

@@ -38,4 +38,7 @@ dec 'xn--!!!' '㚯'
 dec 'xn--already-encoded.com' 'ǮalǭǮreaǭǫdy.com'
 
 # second label has a truncated UTF-8 sequence: used to leak the segment buffer.
-enc_bad "$(printf 'b\xc3\xbcchev.\xe4\xbe\x8b\xe5\xad\x62\xc3\xbccheple')"
+# Fed as hex: the raw bytes must reach the encoder intact, but invalid UTF-8
+# cannot survive a Windows command line (it round-trips through UTF-16 to valid
+# U+FFFD, which then encodes fine instead of being rejected).
+enc_bad 'hex:62c3bc636865762ee4be8be5ad62c3bc636865706c65'

tests/01_engine-rcfile.test

@@ -32,6 +32,13 @@ esac
 tmp=$(mktemp -d "${TMPDIR:-/tmp}/httrack_rcfile.XXXXXX") || exit 1
 trap 'rm -rf "$tmp"' EXIT HUP INT QUIT PIPE TERM
 
+# HTS_HTTRACKRC is ".httrackrc" on POSIX but "httrackrc" on Windows: write both,
+# each platform reads the one it knows.
+write_rc() {
+    cat >"$1/.httrackrc"
+    cp "$1/.httrackrc" "$1/httrackrc"
+}
+
 # --- 1. alias token survives the bound intact -------------------------------
 d1="$tmp/intact"
 mkdir -p "$d1"
@@ -40,7 +47,7 @@ echo '<html><body>hello</body></html>' >"$d1/index.html"
 # optinclude_file() lowercases each config line, so the marker is lowercase to
 # survive the comparison verbatim.
 marker='zzz_rcfile_marker_0123456789_abcdefghijklmnopqrstuvwxyz_intact'
-printf 'user-agent=%s\n' "$marker" >"$d1/.httrackrc"
+printf 'user-agent=%s\n' "$marker" | write_rc "$d1"
 
 # Run with no -O so the working-directory .httrackrc is loaded (an -O path makes
 # the engine skip the rc files). Output lands in the temp dir. Guard the run so a
@@ -73,7 +80,7 @@ echo '<html><body>hi</body></html>' >"$d2/index.html"
 val=$(printf 'a%.0s' $(seq 1 200))
 for _ in $(seq 1 400); do
     printf 'user-agent=%s\n' "$val"
-done >"$d2/.httrackrc"
+done | write_rc "$d2"
 
 # The process aborts (httrack turns the fatal signal into exit 134 either way),
 # so the exit code does not distinguish the bounded abort from a heap overflow;

tests/01_engine-strsafe.test

@@ -32,10 +32,10 @@ case "$err" in
 esac
 
 # Same guarantee for the htsbuff builder. The source is exactly the buffer
-# capacity (4 bytes into a 4-byte buffer), so this also pins the boundary: a
+# capacity (6 bytes into a 6-byte buffer), so this also pins the boundary: a
 # '<=' off-by-one in the capacity check would let it through (and print "NOT
 # aborted"). Match the specific htsbuff abort message, not just any assert.
-err=$(httrack -#test=strsafe overflow-buff "abcd" 2>&1) || true
+err=$(httrack -#test=strsafe overflow-buff "abcdef" 2>&1) || true
 case "$err" in
 *"strsafe: NOT aborted"*)
     echo "htsbuff over-capacity write was NOT caught" >&2